diff --git a/.gitlab/ci/test.gitlab-ci.yml b/.gitlab/ci/test.gitlab-ci.yml index 8ee1972..ccacbb3 100644 --- a/.gitlab/ci/test.gitlab-ci.yml +++ b/.gitlab/ci/test.gitlab-ci.yml @@ -3,8 +3,3 @@ test: image: slumber/blender-addon-testing:latest script: - python3 scripts/test_addon.py - - only: - refs: - - master - - develop diff --git a/docs/getting_started/img/quickstart_advanced_cache.png b/docs/getting_started/img/quickstart_advanced_cache.png new file mode 100644 index 0000000..fd56043 Binary files /dev/null and b/docs/getting_started/img/quickstart_advanced_cache.png differ diff --git a/docs/getting_started/quickstart.rst b/docs/getting_started/quickstart.rst index d0e26eb..cc4751f 100644 --- a/docs/getting_started/quickstart.rst +++ b/docs/getting_started/quickstart.rst @@ -355,6 +355,26 @@ Replication - **Refresh**: pushed data update rate (in second) - **Apply**: pulled data update rate (in second) +----- +Cache +----- + +The multi-user allows to replicate external blend dependencies such as images, movies sounds. +On each client, those files are stored into the cache folder. + +.. figure:: img/quickstart_advanced_cache.png + :align: center + + Advanced cache settings + +**cache_directory** allows to choose where cached files (images, sound, movies) will be saved. + +**Clear memory filecache** will save memory space at runtime by removing the file content from memory as soon as it have been written to the disk. + +**Clear cache** will remove all file from the cache folder. + +.. warning:: Clear cash could break your scene image/movie/sound if they are used into the blend ! + --- Log --- diff --git a/multi_user/bl_types/__init__.py b/multi_user/bl_types/__init__.py index ad35fe8..add7058 100644 --- a/multi_user/bl_types/__init__.py +++ b/multi_user/bl_types/__init__.py @@ -36,7 +36,8 @@ __all__ = [ 'bl_lightprobe', 'bl_speaker', 'bl_font', - 'bl_sound' + 'bl_sound', + 'bl_file' ] # Order here defines execution order from . import * diff --git a/multi_user/bl_types/bl_file.py b/multi_user/bl_types/bl_file.py new file mode 100644 index 0000000..6b07c43 --- /dev/null +++ b/multi_user/bl_types/bl_file.py @@ -0,0 +1,132 @@ +# ##### BEGIN GPL LICENSE BLOCK ##### +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# ##### END GPL LICENSE BLOCK ##### + + +import logging +import os +import sys +from pathlib import Path + +import bpy +import mathutils +from replication.constants import DIFF_BINARY, UP +from replication.data import ReplicatedDatablock + +from .. import utils +from .dump_anything import Dumper, Loader + + +def get_filepath(filename): + """ + Construct the local filepath + """ + return str(Path( + utils.get_preferences().cache_directory, + filename + )) + + +def ensure_unpacked(datablock): + if datablock.packed_file: + logging.info(f"Unpacking {datablock.name}") + + filename = Path(bpy.path.abspath(datablock.filepath)).name + datablock.filepath = get_filepath(filename) + + datablock.unpack(method="WRITE_ORIGINAL") + + +class BlFile(ReplicatedDatablock): + bl_id = 'file' + bl_name = "file" + bl_class = Path + bl_delay_refresh = 0 + bl_delay_apply = 1 + bl_automatic_push = True + bl_check_common = False + bl_icon = 'FILE' + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.instance = kwargs.get('instance', None) + self.preferences = utils.get_preferences() + self.diff_method = DIFF_BINARY + + def resolve(self): + if self.data: + self.instance = Path(get_filepath(self.data['name'])) + + def push(self, socket, identity=None): + super().push(socket, identity=None) + + if self.preferences.clear_memory_filecache: + del self.data['file'] + + def _dump(self, instance=None): + """ + Read the file and return a dict as: + { + name : filename + extension : + file: file content + } + """ + logging.info(f"Extracting file metadata") + + data = { + 'name': self.instance.name, + } + + logging.info( + f"Reading {self.instance.name} content: {self.instance.stat().st_size} bytes") + + try: + file = open(self.instance, "rb") + data['file'] = file.read() + + file.close() + except IOError: + logging.warning(f"{self.instance} doesn't exist, skipping") + else: + file.close() + + return data + + def _load(self, data, target): + """ + Writing the file + """ + # TODO: check for empty data + + if target.exists() and not self.diff(): + logging.info(f"{data['name']} already on the disk, skipping.") + return + try: + file = open(target, "wb") + file.write(data['file']) + + if self.preferences.clear_memory_filecache: + del self.data['file'] + except IOError: + logging.warning(f"{target} doesn't exist, skipping") + else: + file.close() + + def diff(self): + memory_size = sys.getsizeof(self.data['file'])-33 + disk_size = self.instance.stat().st_size + return memory_size == disk_size diff --git a/multi_user/bl_types/bl_font.py b/multi_user/bl_types/bl_font.py index 599f3ad..157f9f2 100644 --- a/multi_user/bl_types/bl_font.py +++ b/multi_user/bl_types/bl_font.py @@ -16,14 +16,16 @@ # ##### END GPL LICENSE BLOCK ##### -import bpy -import mathutils -import os import logging -import pathlib -from .. import utils -from .dump_anything import Loader, Dumper +import os +from pathlib import Path + +import bpy + from .bl_datablock import BlDatablock +from .bl_file import get_filepath, ensure_unpacked +from .dump_anything import Dumper, Loader + class BlFont(BlDatablock): bl_id = "fonts" @@ -37,33 +39,28 @@ class BlFont(BlDatablock): def _construct(self, data): if data['filepath'] == '': return bpy.data.fonts.load(data['filepath']) - elif 'font_file' in data.keys(): - prefs = utils.get_preferences() - ext = pathlib.Path(data['filepath']).suffix - font_name = f"{self.uuid}{ext}" - font_path = os.path.join(prefs.cache_directory, font_name) - - os.makedirs(prefs.cache_directory, exist_ok=True) - file = open(font_path, 'wb') - file.write(data["font_file"]) - file.close() - - logging.info(f'loading {font_path}') - return bpy.data.fonts.load(font_path) + else: + filename = Path(data['filepath']).name + return bpy.data.fonts.load(get_filepath(filename)) def _load(self, data, target): - pass + loader = Loader() + loader.load(target, data) def _dump(self, instance=None): - data = { - 'filepath':instance.filepath, - 'name':instance.name + return { + 'filepath': instance.filepath, + 'name': instance.name } - if instance.filepath != '' and not instance.is_embedded_data: - file = open(instance.filepath, "rb") - data['font_file'] = file.read() - file.close() - return data def diff(self): return False + + def _resolve_deps_implementation(self): + deps = [] + if self.instance.filepath and self.instance.filepath != '': + ensure_unpacked(self.instance) + + deps.append(Path(self.instance.filepath)) + + return deps diff --git a/multi_user/bl_types/bl_image.py b/multi_user/bl_types/bl_image.py index 669d9a9..6026ca2 100644 --- a/multi_user/bl_types/bl_image.py +++ b/multi_user/bl_types/bl_image.py @@ -16,13 +16,17 @@ # ##### END GPL LICENSE BLOCK ##### +import logging +import os +from pathlib import Path + import bpy import mathutils -import os -import logging + from .. import utils -from .dump_anything import Loader, Dumper from .bl_datablock import BlDatablock +from .dump_anything import Dumper, Loader +from .bl_file import get_filepath, ensure_unpacked format_to_ext = { 'BMP': 'bmp', @@ -53,29 +57,6 @@ class BlImage(BlDatablock): bl_check_common = False bl_icon = 'IMAGE_DATA' - def dump_image(self, image): - pixels = None - if image.source == "GENERATED" or image.packed_file is not None: - prefs = utils.get_preferences() - img_name = f"{self.uuid}.{format_to_ext[image.file_format]}" - - # Cache the image on the disk - image.filepath_raw = os.path.join(prefs.cache_directory, img_name) - os.makedirs(prefs.cache_directory, exist_ok=True) - image.save() - - if image.source == "FILE": - image_path = bpy.path.abspath(image.filepath_raw) - image_directory = os.path.dirname(image_path) - os.makedirs(image_directory, exist_ok=True) - image.save() - file = open(image_path, "rb") - pixels = file.read() - file.close() - else: - raise ValueError() - return pixels - def _construct(self, data): return bpy.data.images.new( name=data['name'], @@ -84,28 +65,23 @@ class BlImage(BlDatablock): ) def _load(self, data, target): - image = target - prefs = utils.get_preferences() - img_format = data['file_format'] - img_name = f"{self.uuid}.{format_to_ext[img_format]}" - - img_path = os.path.join(prefs.cache_directory, img_name) - os.makedirs(prefs.cache_directory, exist_ok=True) - file = open(img_path, 'wb') - file.write(data["pixels"]) - file.close() - - image.source = 'FILE' - image.filepath = img_path - image.colorspace_settings.name = data["colorspace_settings"]["name"] - loader = Loader() loader.load(data, target) + target.source = 'FILE' + target.filepath_raw = get_filepath(data['filename']) + target.colorspace_settings.name = data["colorspace_settings"]["name"] + + def _dump(self, instance=None): assert(instance) - data = {} - data['pixels'] = self.dump_image(instance) + + filename = Path(instance.filepath).name + + data = { + "filename": filename + } + dumper = Dumper() dumper.depth = 2 dumper.include_filter = [ @@ -114,13 +90,9 @@ class BlImage(BlDatablock): 'height', 'alpha', 'float_buffer', - 'file_format', 'alpha_mode', - 'filepath', - 'source', 'colorspace_settings'] data.update(dumper.dump(instance)) - return data def diff(self): @@ -128,3 +100,24 @@ class BlImage(BlDatablock): return True else: return False + + def _resolve_deps_implementation(self): + deps = [] + if self.instance.filepath: + + if self.instance.packed_file: + filename = Path(bpy.path.abspath(self.instance.filepath)).name + self.instance.filepath = get_filepath(filename) + self.instance.save() + # An image can't be unpacked to the modified path + # TODO: make a bug report + self.instance.unpack(method="REMOVE") + + elif self.instance.source == "GENERATED": + filename = f"{self.instance.name}.png" + self.instance.filepath = get_filepath(filename) + self.instance.save() + + deps.append(Path(self.instance.filepath)) + + return deps diff --git a/multi_user/bl_types/bl_sound.py b/multi_user/bl_types/bl_sound.py index bd9bafb..93f3990 100644 --- a/multi_user/bl_types/bl_sound.py +++ b/multi_user/bl_types/bl_sound.py @@ -16,14 +16,16 @@ # ##### END GPL LICENSE BLOCK ##### -import bpy -import mathutils -import os import logging -import pathlib -from .. import utils -from .dump_anything import Loader, Dumper +import os +from pathlib import Path + +import bpy + +from .bl_file import get_filepath, ensure_unpacked from .bl_datablock import BlDatablock +from .dump_anything import Dumper, Loader + class BlSound(BlDatablock): bl_id = "sounds" @@ -35,40 +37,27 @@ class BlSound(BlDatablock): bl_icon = 'SOUND' def _construct(self, data): - if 'file' in data.keys(): - prefs = utils.get_preferences() - ext = data['filepath'].split(".")[-1] - sound_name = f"{self.uuid}.{ext}" - sound_path = os.path.join(prefs.cache_directory, sound_name) - - os.makedirs(prefs.cache_directory, exist_ok=True) - file = open(sound_path, 'wb') - file.write(data["file"]) - file.close() - - logging.info(f'loading {sound_path}') - return bpy.data.sounds.load(sound_path) + filename = Path(data['filepath']).name + return bpy.data.sounds.load(get_filepath(filename)) def _load(self, data, target): loader = Loader() loader.load(target, data) - def _dump(self, instance=None): - if not instance.packed_file: - # prefs = utils.get_preferences() - # ext = pathlib.Path(instance.filepath).suffix - # sound_name = f"{self.uuid}{ext}" - # sound_path = os.path.join(prefs.cache_directory, sound_name) - # instance.filepath = sound_path - instance.pack() - #TODO:use file locally with unpack(method='USE_ORIGINAL') ? - - return { - 'filepath':instance.filepath, - 'name':instance.name, - 'file': instance.packed_file.data - } - - def diff(self): return False + + def _dump(self, instance=None): + return { + 'filepath': instance.filepath, + 'name': instance.name + } + + def _resolve_deps_implementation(self): + deps = [] + if self.instance.filepath and self.instance.filepath != '': + ensure_unpacked(self.instance) + + deps.append(Path(self.instance.filepath)) + + return deps diff --git a/multi_user/operators.py b/multi_user/operators.py index 381d824..7353823 100644 --- a/multi_user/operators.py +++ b/multi_user/operators.py @@ -27,6 +27,8 @@ from operator import itemgetter from pathlib import Path from subprocess import PIPE, Popen, TimeoutExpired import zmq +import shutil +from pathlib import Path import bpy import mathutils @@ -608,6 +610,34 @@ class ApplyArmatureOperator(bpy.types.Operator): stop_modal_executor = False +class ClearCache(bpy.types.Operator): + "Clear local session cache" + bl_idname = "session.clear_cache" + bl_label = "Modal Executor Operator" + + @classmethod + def poll(cls, context): + return True + + def execute(self, context): + cache_dir = utils.get_preferences().cache_directory + try: + for root, dirs, files in os.walk(cache_dir): + for name in files: + Path(root, name).unlink() + + except Exception as e: + self.report({'ERROR'}, repr(e)) + + return {"FINISHED"} + + def invoke(self, context, event): + return context.window_manager.invoke_props_dialog(self) + + def draw(self, context): + row = self.layout + row.label(text=f" Do you really want to remove local cache ? ") + classes = ( SessionStartOperator, SessionStopOperator, @@ -620,7 +650,7 @@ classes = ( ApplyArmatureOperator, SessionKickOperator, SessionInitOperator, - + ClearCache, ) diff --git a/multi_user/preferences.py b/multi_user/preferences.py index 90ec490..905833f 100644 --- a/multi_user/preferences.py +++ b/multi_user/preferences.py @@ -20,6 +20,9 @@ import logging import bpy import string import re +import os + +from pathlib import Path from . import bl_types, environment, addon_updater_ops, presence, ui from .utils import get_preferences, get_expanded_icon @@ -68,6 +71,16 @@ def update_port(self, context): self['ipc_port'] = random.randrange(self.port+4, 10000) +def update_directory(self, context): + new_dir = Path(self.cache_directory) + if new_dir.exists() and any(Path(self.cache_directory).iterdir()): + logging.error("The folder is not empty, choose another one.") + self['cache_directory'] = environment.DEFAULT_CACHE_DIR + elif not new_dir.exists(): + logging.info("Target cache folder doesn't exist, creating it.") + os.makedirs(self.cache_directory, exist_ok=True) + + def set_log_level(self, value): logging.getLogger().setLevel(value) @@ -136,7 +149,8 @@ class SessionPrefs(bpy.types.AddonPreferences): cache_directory: bpy.props.StringProperty( name="cache directory", subtype="DIR_PATH", - default=environment.DEFAULT_CACHE_DIR) + default=environment.DEFAULT_CACHE_DIR, + update=update_directory) connection_timeout: bpy.props.IntProperty( name='connection timeout', description='connection timeout before disconnection', @@ -162,6 +176,11 @@ class SessionPrefs(bpy.types.AddonPreferences): description="Enable objects update in edit mode (! Impact performances !)", default=False ) + clear_memory_filecache: bpy.props.BoolProperty( + name="Clear memory filecache", + description="Remove filecache from memory", + default=False + ) # for UI category: bpy.props.EnumProperty( name="Category", @@ -230,6 +249,12 @@ class SessionPrefs(bpy.types.AddonPreferences): description="sidebar_advanced_net_expanded", default=False ) + sidebar_advanced_cache_expanded: bpy.props.BoolProperty( + name="sidebar_advanced_cache_expanded", + description="sidebar_advanced_cache_expanded", + default=False + ) + auto_check_update: bpy.props.BoolProperty( name="Auto-check for Update", description="If enabled, auto-check for updates using an interval", @@ -343,6 +368,7 @@ class SessionPrefs(bpy.types.AddonPreferences): emboss=False) if self.conf_session_cache_expanded: box.row().prop(self, "cache_directory", text="Cache directory") + box.row().prop(self, "clear_memory_filecache", text="Clear memory filecache") # INTERFACE SETTINGS box = grid.box() diff --git a/multi_user/ui.py b/multi_user/ui.py index ef79e9c..954a1f0 100644 --- a/multi_user/ui.py +++ b/multi_user/ui.py @@ -19,7 +19,7 @@ import bpy from . import operators -from .utils import get_preferences, get_expanded_icon +from .utils import get_preferences, get_expanded_icon, get_folder_size from replication.constants import (ADDED, ERROR, FETCHED, MODIFIED, RP_COMMON, UP, STATE_ACTIVE, STATE_AUTH, @@ -356,6 +356,23 @@ class SESSION_PT_advanced_settings(bpy.types.Panel): replication_timers.label(text="Update rate (ms):") replication_timers.prop(settings, "depsgraph_update_rate", text="") + cache_section = layout.row().box() + cache_section.prop( + settings, + "sidebar_advanced_cache_expanded", + text="Cache", + icon=get_expanded_icon(settings.sidebar_advanced_cache_expanded), + emboss=False) + if settings.sidebar_advanced_cache_expanded: + cache_section_row = cache_section.row() + cache_section_row.label(text="Cache directory:") + cache_section_row = cache_section.row() + cache_section_row.prop(settings, "cache_directory", text="") + cache_section_row = cache_section.row() + cache_section_row.label(text="Clear memory filecache:") + cache_section_row.prop(settings, "clear_memory_filecache", text="") + cache_section_row = cache_section.row() + cache_section_row.operator('session.clear_cache', text=f"Clear cache ({get_folder_size(settings.cache_directory)})") log_section = layout.row().box() log_section.prop( settings, diff --git a/multi_user/utils.py b/multi_user/utils.py index f66ab90..75c1391 100644 --- a/multi_user/utils.py +++ b/multi_user/utils.py @@ -21,8 +21,10 @@ import logging import os import sys import time -from uuid import uuid4 from collections.abc import Iterable +from pathlib import Path +from uuid import uuid4 +import math import bpy import mathutils @@ -47,7 +49,7 @@ def get_datablock_users(datablock): if hasattr(datablock, 'users_group') and datablock.users_scene: users.extend(list(datablock.users_scene)) for datatype in supported_types: - if datatype.bl_name != 'users': + if datatype.bl_name != 'users' and hasattr(bpy.data, datatype.bl_name): root = getattr(bpy.data, datatype.bl_name) for item in root: if hasattr(item, 'data') and datablock == item.data or \ @@ -78,6 +80,7 @@ def resolve_from_id(id, optionnal_type=None): return root[id] return None + def get_datablock_from_uuid(uuid, default, ignore=[]): if not uuid: return default @@ -87,9 +90,10 @@ def get_datablock_from_uuid(uuid, default, ignore=[]): if isinstance(root, Iterable) and category not in ignore: for item in root: if getattr(item, 'uuid', None) == uuid: - return item + return item return default + def get_preferences(): return bpy.context.preferences.addons[__package__].preferences @@ -103,3 +107,61 @@ def get_expanded_icon(prop: bpy.types.BoolProperty) -> str: return 'DISCLOSURE_TRI_DOWN' else: return 'DISCLOSURE_TRI_RIGHT' + + +# Taken from here: https://stackoverflow.com/a/55659577 +def get_folder_size(folder): + return ByteSize(sum(file.stat().st_size for file in Path(folder).rglob('*'))) + + +class ByteSize(int): + + _kB = 1024 + _suffixes = 'B', 'kB', 'MB', 'GB', 'PB' + + def __new__(cls, *args, **kwargs): + return super().__new__(cls, *args, **kwargs) + + def __init__(self, *args, **kwargs): + self.bytes = self.B = int(self) + self.kilobytes = self.kB = self / self._kB**1 + self.megabytes = self.MB = self / self._kB**2 + self.gigabytes = self.GB = self / self._kB**3 + self.petabytes = self.PB = self / self._kB**4 + *suffixes, last = self._suffixes + suffix = next(( + suffix + for suffix in suffixes + if 1 < getattr(self, suffix) < self._kB + ), last) + self.readable = suffix, getattr(self, suffix) + + super().__init__() + + def __str__(self): + return self.__format__('.2f') + + def __repr__(self): + return '{}({})'.format(self.__class__.__name__, super().__repr__()) + + def __format__(self, format_spec): + suffix, val = self.readable + return '{val:{fmt}} {suf}'.format(val=math.ceil(val), fmt=format_spec, suf=suffix) + + def __sub__(self, other): + return self.__class__(super().__sub__(other)) + + def __add__(self, other): + return self.__class__(super().__add__(other)) + + def __mul__(self, other): + return self.__class__(super().__mul__(other)) + + def __rsub__(self, other): + return self.__class__(super().__sub__(other)) + + def __radd__(self, other): + return self.__class__(super().__add__(other)) + + def __rmul__(self, other): + return self.__class__(super().__rmul__(other)) diff --git a/tests/test_bl_types/test_image.py b/tests/test_bl_types/test_image.py deleted file mode 100644 index e597526..0000000 --- a/tests/test_bl_types/test_image.py +++ /dev/null @@ -1,21 +0,0 @@ -import os - -import pytest -from deepdiff import DeepDiff - -import bpy -import random -from multi_user.bl_types.bl_image import BlImage - -def test_image(clear_blend): - datablock = bpy.data.images.new('asd',2000,2000) - - implementation = BlImage() - expected = implementation._dump(datablock) - bpy.data.images.remove(datablock) - - test = implementation._construct(expected) - implementation._load(expected, test) - result = implementation._dump(test) - - assert not DeepDiff(expected, result)