Backups/multi-user
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'file_replication' into 'develop'

Browse Source 
Basic file replication interface

See merge request slumber/multi-user!48
This commit is contained in:
Swann Martinez
2020-09-21 16:17:58 +00:00
parent 8d2b9e5580 167b39f15e
commit 0ae34d5702
13 changed files with 385 additions and 144 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
.gitlab/ci/test.gitlab-ci.yml
View File

@ -3,8 +3,3 @@ test:
image: slumber/blender-addon-testing:latest
script:
- python3 scripts/test_addon.py
only:
refs:
- master
- develop

BIN
docs/getting_started/img/quickstart_advanced_cache.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 7.6 KiB

20
docs/getting_started/quickstart.rst
View File

@ -355,6 +355,26 @@ Replication
- **Refresh**: pushed data update rate (in second)
- **Apply**: pulled data update rate (in second)
-----
Cache
-----
The multi-user allows to replicate external blend dependencies such as images, movies sounds.
On each client, those files are stored into the cache folder.
.. figure:: img/quickstart_advanced_cache.png
:align: center
Advanced cache settings
**cache_directory** allows to choose where cached files (images, sound, movies) will be saved.
**Clear memory filecache** will save memory space at runtime by removing the file content from memory as soon as it have been written to the disk.
**Clear cache** will remove all file from the cache folder.
.. warning:: Clear cash could break your scene image/movie/sound if they are used into the blend !
---
Log
---

3
multi_user/bl_types/__init__.py
View File

@ -36,7 +36,8 @@ __all__ = [
'bl_lightprobe',
'bl_speaker',
'bl_font',
'bl_sound'
'bl_sound',
'bl_file'
] # Order here defines execution order
from . import *

132
multi_user/bl_types/bl_file.py Normal file
View File

@ -0,0 +1,132 @@
# ##### BEGIN GPL LICENSE BLOCK #####
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
#
# ##### END GPL LICENSE BLOCK #####
import logging
import os
import sys
from pathlib import Path
import bpy
import mathutils
from replication.constants import DIFF_BINARY, UP
from replication.data import ReplicatedDatablock
from .. import utils
from .dump_anything import Dumper, Loader
def get_filepath(filename):
"""
Construct the local filepath
"""
return str(Path(
utils.get_preferences().cache_directory,
filename
))
def ensure_unpacked(datablock):
if datablock.packed_file:
logging.info(f"Unpacking {datablock.name}")
filename = Path(bpy.path.abspath(datablock.filepath)).name
datablock.filepath = get_filepath(filename)
datablock.unpack(method="WRITE_ORIGINAL")
class BlFile(ReplicatedDatablock):
bl_id = 'file'
bl_name = "file"
bl_class = Path
bl_delay_refresh = 0
bl_delay_apply = 1
bl_automatic_push = True
bl_check_common = False
bl_icon = 'FILE'
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.instance = kwargs.get('instance', None)
self.preferences = utils.get_preferences()
self.diff_method = DIFF_BINARY
def resolve(self):
if self.data:
self.instance = Path(get_filepath(self.data['name']))
def push(self, socket, identity=None):
super().push(socket, identity=None)
if self.preferences.clear_memory_filecache:
del self.data['file']
def _dump(self, instance=None):
"""
Read the file and return a dict as:
{
name : filename
extension :
file: file content
}
"""
logging.info(f"Extracting file metadata")
data = {
'name': self.instance.name,
}
logging.info(
f"Reading {self.instance.name} content: {self.instance.stat().st_size} bytes")
try:
file = open(self.instance, "rb")
data['file'] = file.read()
file.close()
except IOError:
logging.warning(f"{self.instance} doesn't exist, skipping")
else:
file.close()
return data
def _load(self, data, target):
"""
Writing the file
"""
# TODO: check for empty data
if target.exists() and not self.diff():
logging.info(f"{data['name']} already on the disk, skipping.")
return
try:
file = open(target, "wb")
file.write(data['file'])
if self.preferences.clear_memory_filecache:
del self.data['file']
except IOError:
logging.warning(f"{target} doesn't exist, skipping")
else:
file.close()
def diff(self):
memory_size = sys.getsizeof(self.data['file'])-33
disk_size = self.instance.stat().st_size
return memory_size == disk_size

53
multi_user/bl_types/bl_font.py
View File

@ -16,14 +16,16 @@
# ##### END GPL LICENSE BLOCK #####
import bpy
import mathutils
import os
import logging
import pathlib
from .. import utils
from .dump_anything import Loader, Dumper
import os
from pathlib import Path
import bpy
from .bl_datablock import BlDatablock
from .bl_file import get_filepath, ensure_unpacked
from .dump_anything import Dumper, Loader
class BlFont(BlDatablock):
bl_id = "fonts"
@ -37,33 +39,28 @@ class BlFont(BlDatablock):
def _construct(self, data):
if data['filepath'] == '<builtin>':
return bpy.data.fonts.load(data['filepath'])
elif 'font_file' in data.keys():
prefs = utils.get_preferences()
ext = pathlib.Path(data['filepath']).suffix
font_name = f"{self.uuid}{ext}"
font_path = os.path.join(prefs.cache_directory, font_name)
os.makedirs(prefs.cache_directory, exist_ok=True)
file = open(font_path, 'wb')
file.write(data["font_file"])
file.close()
logging.info(f'loading {font_path}')
return bpy.data.fonts.load(font_path)
else:
filename = Path(data['filepath']).name
return bpy.data.fonts.load(get_filepath(filename))
def _load(self, data, target):
pass
loader = Loader()
loader.load(target, data)
def _dump(self, instance=None):
data = {
'filepath':instance.filepath,
'name':instance.name
return {
'filepath': instance.filepath,
'name': instance.name
}
if instance.filepath != '<builtin>' and not instance.is_embedded_data:
file = open(instance.filepath, "rb")
data['font_file'] = file.read()
file.close()
return data
def diff(self):
return False
def _resolve_deps_implementation(self):
deps = []
if self.instance.filepath and self.instance.filepath != '<builtin>':
ensure_unpacked(self.instance)
deps.append(Path(self.instance.filepath))
return deps

87
multi_user/bl_types/bl_image.py
View File

@ -16,13 +16,17 @@
# ##### END GPL LICENSE BLOCK #####
import logging
import os
from pathlib import Path
import bpy
import mathutils
import os
import logging
from .. import utils
from .dump_anything import Loader, Dumper
from .bl_datablock import BlDatablock
from .dump_anything import Dumper, Loader
from .bl_file import get_filepath, ensure_unpacked
format_to_ext = {
'BMP': 'bmp',
@ -53,29 +57,6 @@ class BlImage(BlDatablock):
bl_check_common = False
bl_icon = 'IMAGE_DATA'
def dump_image(self, image):
pixels = None
if image.source == "GENERATED" or image.packed_file is not None:
prefs = utils.get_preferences()
img_name = f"{self.uuid}.{format_to_ext[image.file_format]}"
# Cache the image on the disk
image.filepath_raw = os.path.join(prefs.cache_directory, img_name)
os.makedirs(prefs.cache_directory, exist_ok=True)
image.save()
if image.source == "FILE":
image_path = bpy.path.abspath(image.filepath_raw)
image_directory = os.path.dirname(image_path)
os.makedirs(image_directory, exist_ok=True)
image.save()
file = open(image_path, "rb")
pixels = file.read()
file.close()
else:
raise ValueError()
return pixels
def _construct(self, data):
return bpy.data.images.new(
name=data['name'],
@ -84,28 +65,23 @@ class BlImage(BlDatablock):
)
def _load(self, data, target):
image = target
prefs = utils.get_preferences()
img_format = data['file_format']
img_name = f"{self.uuid}.{format_to_ext[img_format]}"
img_path = os.path.join(prefs.cache_directory, img_name)
os.makedirs(prefs.cache_directory, exist_ok=True)
file = open(img_path, 'wb')
file.write(data["pixels"])
file.close()
image.source = 'FILE'
image.filepath = img_path
image.colorspace_settings.name = data["colorspace_settings"]["name"]
loader = Loader()
loader.load(data, target)
target.source = 'FILE'
target.filepath_raw = get_filepath(data['filename'])
target.colorspace_settings.name = data["colorspace_settings"]["name"]
def _dump(self, instance=None):
assert(instance)
data = {}
data['pixels'] = self.dump_image(instance)
filename = Path(instance.filepath).name
data = {
"filename": filename
}
dumper = Dumper()
dumper.depth = 2
dumper.include_filter = [
@ -114,13 +90,9 @@ class BlImage(BlDatablock):
'height',
'alpha',
'float_buffer',
'file_format',
'alpha_mode',
'filepath',
'source',
'colorspace_settings']
data.update(dumper.dump(instance))
return data
def diff(self):
@ -128,3 +100,24 @@ class BlImage(BlDatablock):
return True
else:
return False
def _resolve_deps_implementation(self):
deps = []
if self.instance.filepath:
if self.instance.packed_file:
filename = Path(bpy.path.abspath(self.instance.filepath)).name
self.instance.filepath = get_filepath(filename)
self.instance.save()
# An image can't be unpacked to the modified path
# TODO: make a bug report
self.instance.unpack(method="REMOVE")
elif self.instance.source == "GENERATED":
filename = f"{self.instance.name}.png"
self.instance.filepath = get_filepath(filename)
self.instance.save()
deps.append(Path(self.instance.filepath))
return deps

61
multi_user/bl_types/bl_sound.py
View File

@ -16,14 +16,16 @@
# ##### END GPL LICENSE BLOCK #####
import bpy
import mathutils
import os
import logging
import pathlib
from .. import utils
from .dump_anything import Loader, Dumper
import os
from pathlib import Path
import bpy
from .bl_file import get_filepath, ensure_unpacked
from .bl_datablock import BlDatablock
from .dump_anything import Dumper, Loader
class BlSound(BlDatablock):
bl_id = "sounds"
@ -35,40 +37,27 @@ class BlSound(BlDatablock):
bl_icon = 'SOUND'
def _construct(self, data):
if 'file' in data.keys():
prefs = utils.get_preferences()
ext = data['filepath'].split(".")[-1]
sound_name = f"{self.uuid}.{ext}"
sound_path = os.path.join(prefs.cache_directory, sound_name)
os.makedirs(prefs.cache_directory, exist_ok=True)
file = open(sound_path, 'wb')
file.write(data["file"])
file.close()
logging.info(f'loading {sound_path}')
return bpy.data.sounds.load(sound_path)
filename = Path(data['filepath']).name
return bpy.data.sounds.load(get_filepath(filename))
def _load(self, data, target):
loader = Loader()
loader.load(target, data)
def _dump(self, instance=None):
if not instance.packed_file:
# prefs = utils.get_preferences()
# ext = pathlib.Path(instance.filepath).suffix
# sound_name = f"{self.uuid}{ext}"
# sound_path = os.path.join(prefs.cache_directory, sound_name)
# instance.filepath = sound_path
instance.pack()
#TODO:use file locally with unpack(method='USE_ORIGINAL') ?
return {
'filepath':instance.filepath,
'name':instance.name,
'file': instance.packed_file.data
}
def diff(self):
return False
def _dump(self, instance=None):
return {
'filepath': instance.filepath,
'name': instance.name
}
def _resolve_deps_implementation(self):
deps = []
if self.instance.filepath and self.instance.filepath != '<builtin>':
ensure_unpacked(self.instance)
deps.append(Path(self.instance.filepath))
return deps

32
multi_user/operators.py
View File

@ -27,6 +27,8 @@ from operator import itemgetter
from pathlib import Path
from subprocess import PIPE, Popen, TimeoutExpired
import zmq
import shutil
from pathlib import Path
import bpy
import mathutils
@ -608,6 +610,34 @@ class ApplyArmatureOperator(bpy.types.Operator):
stop_modal_executor = False
class ClearCache(bpy.types.Operator):
"Clear local session cache"
bl_idname = "session.clear_cache"
bl_label = "Modal Executor Operator"
@classmethod
def poll(cls, context):
return True
def execute(self, context):
cache_dir = utils.get_preferences().cache_directory
try:
for root, dirs, files in os.walk(cache_dir):
for name in files:
Path(root, name).unlink()
except Exception as e:
self.report({'ERROR'}, repr(e))
return {"FINISHED"}
def invoke(self, context, event):
return context.window_manager.invoke_props_dialog(self)
def draw(self, context):
row = self.layout
row.label(text=f" Do you really want to remove local cache ? ")
classes = (
SessionStartOperator,
SessionStopOperator,
@ -620,7 +650,7 @@ classes = (
ApplyArmatureOperator,
SessionKickOperator,
SessionInitOperator,
ClearCache,
)

28
multi_user/preferences.py
View File

@ -20,6 +20,9 @@ import logging
import bpy
import string
import re
import os
from pathlib import Path
from . import bl_types, environment, addon_updater_ops, presence, ui
from .utils import get_preferences, get_expanded_icon
@ -68,6 +71,16 @@ def update_port(self, context):
self['ipc_port'] = random.randrange(self.port+4, 10000)
def update_directory(self, context):
new_dir = Path(self.cache_directory)
if new_dir.exists() and any(Path(self.cache_directory).iterdir()):
logging.error("The folder is not empty, choose another one.")
self['cache_directory'] = environment.DEFAULT_CACHE_DIR
elif not new_dir.exists():
logging.info("Target cache folder doesn't exist, creating it.")
os.makedirs(self.cache_directory, exist_ok=True)
def set_log_level(self, value):
logging.getLogger().setLevel(value)
@ -136,7 +149,8 @@ class SessionPrefs(bpy.types.AddonPreferences):
cache_directory: bpy.props.StringProperty(
name="cache directory",
subtype="DIR_PATH",
default=environment.DEFAULT_CACHE_DIR)
default=environment.DEFAULT_CACHE_DIR,
update=update_directory)
connection_timeout: bpy.props.IntProperty(
name='connection timeout',
description='connection timeout before disconnection',
@ -162,6 +176,11 @@ class SessionPrefs(bpy.types.AddonPreferences):
description="Enable objects update in edit mode (! Impact performances !)",
default=False
)
clear_memory_filecache: bpy.props.BoolProperty(
name="Clear memory filecache",
description="Remove filecache from memory",
default=False
)
# for UI
category: bpy.props.EnumProperty(
name="Category",
@ -230,6 +249,12 @@ class SessionPrefs(bpy.types.AddonPreferences):
description="sidebar_advanced_net_expanded",
default=False
)
sidebar_advanced_cache_expanded: bpy.props.BoolProperty(
name="sidebar_advanced_cache_expanded",
description="sidebar_advanced_cache_expanded",
default=False
)
auto_check_update: bpy.props.BoolProperty(
name="Auto-check for Update",
description="If enabled, auto-check for updates using an interval",
@ -343,6 +368,7 @@ class SessionPrefs(bpy.types.AddonPreferences):
emboss=False)
if self.conf_session_cache_expanded:
box.row().prop(self, "cache_directory", text="Cache directory")
box.row().prop(self, "clear_memory_filecache", text="Clear memory filecache")
# INTERFACE SETTINGS
box = grid.box()

19
multi_user/ui.py
View File

@ -19,7 +19,7 @@
import bpy
from . import operators
from .utils import get_preferences, get_expanded_icon
from .utils import get_preferences, get_expanded_icon, get_folder_size
from replication.constants import (ADDED, ERROR, FETCHED,
MODIFIED, RP_COMMON, UP,
STATE_ACTIVE, STATE_AUTH,
@ -356,6 +356,23 @@ class SESSION_PT_advanced_settings(bpy.types.Panel):
replication_timers.label(text="Update rate (ms):")
replication_timers.prop(settings, "depsgraph_update_rate", text="")
cache_section = layout.row().box()
cache_section.prop(
settings,
"sidebar_advanced_cache_expanded",
text="Cache",
icon=get_expanded_icon(settings.sidebar_advanced_cache_expanded),
emboss=False)
if settings.sidebar_advanced_cache_expanded:
cache_section_row = cache_section.row()
cache_section_row.label(text="Cache directory:")
cache_section_row = cache_section.row()
cache_section_row.prop(settings, "cache_directory", text="")
cache_section_row = cache_section.row()
cache_section_row.label(text="Clear memory filecache:")
cache_section_row.prop(settings, "clear_memory_filecache", text="")
cache_section_row = cache_section.row()
cache_section_row.operator('session.clear_cache', text=f"Clear cache ({get_folder_size(settings.cache_directory)})")
log_section = layout.row().box()
log_section.prop(
settings,

68
multi_user/utils.py
View File

@ -21,8 +21,10 @@ import logging
import os
import sys
import time
from uuid import uuid4
from collections.abc import Iterable
from pathlib import Path
from uuid import uuid4
import math
import bpy
import mathutils
@ -47,7 +49,7 @@ def get_datablock_users(datablock):
if hasattr(datablock, 'users_group') and datablock.users_scene:
users.extend(list(datablock.users_scene))
for datatype in supported_types:
if datatype.bl_name != 'users':
if datatype.bl_name != 'users' and hasattr(bpy.data, datatype.bl_name):
root = getattr(bpy.data, datatype.bl_name)
for item in root:
if hasattr(item, 'data') and datablock == item.data or \
@ -78,6 +80,7 @@ def resolve_from_id(id, optionnal_type=None):
return root[id]
return None
def get_datablock_from_uuid(uuid, default, ignore=[]):
if not uuid:
return default
@ -87,9 +90,10 @@ def get_datablock_from_uuid(uuid, default, ignore=[]):
if isinstance(root, Iterable) and category not in ignore:
for item in root:
if getattr(item, 'uuid', None) == uuid:
return item
return item
return default
def get_preferences():
return bpy.context.preferences.addons[__package__].preferences
@ -103,3 +107,61 @@ def get_expanded_icon(prop: bpy.types.BoolProperty) -> str:
return 'DISCLOSURE_TRI_DOWN'
else:
return 'DISCLOSURE_TRI_RIGHT'
# Taken from here: https://stackoverflow.com/a/55659577
def get_folder_size(folder):
return ByteSize(sum(file.stat().st_size for file in Path(folder).rglob('*')))
class ByteSize(int):
_kB = 1024
_suffixes = 'B', 'kB', 'MB', 'GB', 'PB'
def __new__(cls, *args, **kwargs):
return super().__new__(cls, *args, **kwargs)
def __init__(self, *args, **kwargs):
self.bytes = self.B = int(self)
self.kilobytes = self.kB = self / self._kB**1
self.megabytes = self.MB = self / self._kB**2
self.gigabytes = self.GB = self / self._kB**3
self.petabytes = self.PB = self / self._kB**4
*suffixes, last = self._suffixes
suffix = next((
suffix
for suffix in suffixes
if 1 < getattr(self, suffix) < self._kB
), last)
self.readable = suffix, getattr(self, suffix)
super().__init__()
def __str__(self):
return self.__format__('.2f')
def __repr__(self):
return '{}({})'.format(self.__class__.__name__, super().__repr__())
def __format__(self, format_spec):
suffix, val = self.readable
return '{val:{fmt}} {suf}'.format(val=math.ceil(val), fmt=format_spec, suf=suffix)
def __sub__(self, other):
return self.__class__(super().__sub__(other))
def __add__(self, other):
return self.__class__(super().__add__(other))
def __mul__(self, other):
return self.__class__(super().__mul__(other))
def __rsub__(self, other):
return self.__class__(super().__sub__(other))
def __radd__(self, other):
return self.__class__(super().__add__(other))
def __rmul__(self, other):
return self.__class__(super().__rmul__(other))

21
tests/test_bl_types/test_image.py
View File

@ -1,21 +0,0 @@
import os
import pytest
from deepdiff import DeepDiff
import bpy
import random
from multi_user.bl_types.bl_image import BlImage
def test_image(clear_blend):
datablock = bpy.data.images.new('asd',2000,2000)
implementation = BlImage()
expected = implementation._dump(datablock)
bpy.data.images.remove(datablock)
test = implementation._construct(expected)
implementation._load(expected, test)
result = implementation._dump(test)
assert not DeepDiff(expected, result)