Merge branch 'file_replication' into 'develop'

Basic file replication interface

See merge request slumber/multi-user!48

.gitlab/ci/test.gitlab-ci.yml

@@ -3,8 +3,3 @@ test:
   image: slumber/blender-addon-testing:latest
   script:
     - python3 scripts/test_addon.py
-
-  only:
-    refs:
-    - master
-    - develop

docs/getting_started/quickstart.rst

@@ -355,6 +355,26 @@ Replication
 - **Refresh**: pushed data update rate (in second)
 - **Apply**: pulled data update rate (in second)
 
+-----
+Cache
+-----
+
+The multi-user allows to replicate external blend dependencies such as images, movies sounds. 
+On each client, those files are stored into the cache folder.
+
+.. figure:: img/quickstart_advanced_cache.png
+   :align: center
+
+   Advanced cache settings
+
+**cache_directory** allows to choose where cached files (images, sound, movies) will be saved.
+
+**Clear memory filecache** will save memory space at runtime by removing the file content from memory as soon as it have been written to the disk.
+
+**Clear cache** will remove all file from the cache folder. 
+
+.. warning:: Clear cash could break your scene image/movie/sound if they are used into the blend ! 
+
 ---
 Log
 ---

multi_user/bl_types/__init__.py

@@ -36,7 +36,8 @@ __all__ = [
     'bl_lightprobe',
     'bl_speaker',
     'bl_font',
-    'bl_sound'
+    'bl_sound',
+    'bl_file'
 ]  # Order here defines execution order
 
 from . import *

multi_user/bl_types/bl_file.py

@@ -0,0 +1,132 @@
+# ##### BEGIN GPL LICENSE BLOCK #####
+#
+#   This program is free software: you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation, either version 3 of the License, or
+#   (at your option) any later version.
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License
+#   along with this program.  If not, see <https://www.gnu.org/licenses/>.
+#
+# ##### END GPL LICENSE BLOCK #####
+
+
+import logging
+import os
+import sys
+from pathlib import Path
+
+import bpy
+import mathutils
+from replication.constants import DIFF_BINARY, UP
+from replication.data import ReplicatedDatablock
+
+from .. import utils
+from .dump_anything import Dumper, Loader
+
+
+def get_filepath(filename):
+    """
+    Construct the local filepath 
+    """
+    return str(Path(
+        utils.get_preferences().cache_directory,
+        filename
+    ))
+
+
+def ensure_unpacked(datablock):
+    if datablock.packed_file:
+        logging.info(f"Unpacking {datablock.name}")
+
+        filename = Path(bpy.path.abspath(datablock.filepath)).name
+        datablock.filepath = get_filepath(filename)
+
+        datablock.unpack(method="WRITE_ORIGINAL")
+
+
+class BlFile(ReplicatedDatablock):
+    bl_id = 'file'
+    bl_name = "file"
+    bl_class = Path
+    bl_delay_refresh = 0
+    bl_delay_apply = 1
+    bl_automatic_push = True
+    bl_check_common = False
+    bl_icon = 'FILE'
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.instance = kwargs.get('instance', None)
+        self.preferences = utils.get_preferences()
+        self.diff_method = DIFF_BINARY
+
+    def resolve(self):
+        if self.data:
+            self.instance = Path(get_filepath(self.data['name']))
+
+    def push(self, socket, identity=None):
+        super().push(socket, identity=None)
+        
+        if self.preferences.clear_memory_filecache:
+                del self.data['file']
+
+    def _dump(self, instance=None):
+        """
+        Read the file and return a dict as:
+        {
+            name : filename
+            extension :
+            file: file content
+        }
+        """
+        logging.info(f"Extracting file metadata")
+
+        data = {
+            'name': self.instance.name,
+        }
+
+        logging.info(
+            f"Reading {self.instance.name} content: {self.instance.stat().st_size} bytes")
+
+        try:
+            file = open(self.instance, "rb")
+            data['file'] = file.read()
+
+            file.close()
+        except IOError:
+            logging.warning(f"{self.instance} doesn't exist, skipping")
+        else:
+            file.close()
+
+        return data
+
+    def _load(self, data, target):
+        """
+        Writing the file
+        """
+        # TODO: check for empty data
+
+        if target.exists() and not self.diff():
+            logging.info(f"{data['name']} already on the disk, skipping.")
+            return
+        try:
+            file = open(target, "wb")
+            file.write(data['file'])
+            
+            if self.preferences.clear_memory_filecache:
+                del self.data['file']
+        except IOError:
+            logging.warning(f"{target} doesn't exist, skipping")
+        else:
+            file.close()
+
+    def diff(self):
+        memory_size = sys.getsizeof(self.data['file'])-33
+        disk_size = self.instance.stat().st_size
+        return memory_size == disk_size

multi_user/bl_types/bl_font.py

@@ -16,14 +16,16 @@
 # ##### END GPL LICENSE BLOCK #####
 
 
-import bpy
-import mathutils
-import os
 import logging
-import pathlib
+import os
-from .. import utils
+from pathlib import Path
-from .dump_anything import Loader, Dumper
+
+import bpy
+
 from .bl_datablock import BlDatablock
+from .bl_file import get_filepath, ensure_unpacked
+from .dump_anything import Dumper, Loader
+
 
 class BlFont(BlDatablock):
     bl_id = "fonts"
@@ -37,33 +39,28 @@ class BlFont(BlDatablock):
     def _construct(self, data):
         if data['filepath'] == '<builtin>':
             return bpy.data.fonts.load(data['filepath'])
-        elif 'font_file' in data.keys():
+        else:
-            prefs = utils.get_preferences()
+            filename = Path(data['filepath']).name
-            ext = pathlib.Path(data['filepath']).suffix
+            return bpy.data.fonts.load(get_filepath(filename))
-            font_name = f"{self.uuid}{ext}"
-            font_path = os.path.join(prefs.cache_directory, font_name)
-            
-            os.makedirs(prefs.cache_directory, exist_ok=True)
-            file = open(font_path, 'wb')
-            file.write(data["font_file"])
-            file.close()
-
-            logging.info(f'loading {font_path}')
-            return bpy.data.fonts.load(font_path)
 
     def _load(self, data, target):
-        pass
+        loader = Loader()
+        loader.load(target, data)
 
     def _dump(self, instance=None):
-        data = {
+        return {
             'filepath': instance.filepath,
             'name': instance.name
         }
-        if instance.filepath != '<builtin>' and not instance.is_embedded_data:            
-            file = open(instance.filepath, "rb")
-            data['font_file'] = file.read()
-            file.close()
-        return data
 
     def diff(self):
         return False
+
+    def _resolve_deps_implementation(self):
+        deps = []
+        if self.instance.filepath and self.instance.filepath != '<builtin>':
+            ensure_unpacked(self.instance)
+            
+            deps.append(Path(self.instance.filepath))
+
+        return deps

multi_user/bl_types/bl_image.py

@@ -16,13 +16,17 @@
 # ##### END GPL LICENSE BLOCK #####
 
 
+import logging
+import os
+from pathlib import Path
+
 import bpy
 import mathutils
-import os
+
-import logging
 from .. import utils
-from .dump_anything import Loader, Dumper
 from .bl_datablock import BlDatablock
+from .dump_anything import Dumper, Loader
+from .bl_file import get_filepath, ensure_unpacked
 
 format_to_ext = {
     'BMP': 'bmp',
@@ -53,29 +57,6 @@ class BlImage(BlDatablock):
     bl_check_common = False
     bl_icon = 'IMAGE_DATA'
 
-    def dump_image(self, image):
-        pixels = None
-        if image.source == "GENERATED" or image.packed_file is not None:
-            prefs = utils.get_preferences()
-            img_name = f"{self.uuid}.{format_to_ext[image.file_format]}"
-
-            # Cache the image on the disk
-            image.filepath_raw = os.path.join(prefs.cache_directory, img_name)
-            os.makedirs(prefs.cache_directory, exist_ok=True)
-            image.save()
-
-        if image.source == "FILE":
-            image_path = bpy.path.abspath(image.filepath_raw)
-            image_directory = os.path.dirname(image_path)
-            os.makedirs(image_directory, exist_ok=True)
-            image.save()
-            file = open(image_path, "rb")
-            pixels = file.read()
-            file.close()
-        else:
-            raise ValueError()
-        return pixels
-
     def _construct(self, data):
         return bpy.data.images.new(
             name=data['name'],
@@ -84,28 +65,23 @@ class BlImage(BlDatablock):
         )
 
     def _load(self, data, target):
-        image = target
-        prefs = utils.get_preferences()
-        img_format = data['file_format']
-        img_name = f"{self.uuid}.{format_to_ext[img_format]}"
-
-        img_path = os.path.join(prefs.cache_directory, img_name)
-        os.makedirs(prefs.cache_directory, exist_ok=True)
-        file = open(img_path, 'wb')
-        file.write(data["pixels"])
-        file.close()
-
-        image.source = 'FILE'
-        image.filepath = img_path
-        image.colorspace_settings.name = data["colorspace_settings"]["name"]
-
         loader = Loader()
         loader.load(data, target)
 
+        target.source = 'FILE'
+        target.filepath_raw = get_filepath(data['filename'])
+        target.colorspace_settings.name = data["colorspace_settings"]["name"]
+       
+
     def _dump(self, instance=None):
         assert(instance)
-        data = {}
+
-        data['pixels'] = self.dump_image(instance)
+        filename = Path(instance.filepath).name
+
+        data = {
+            "filename": filename
+        }
+
         dumper = Dumper()
         dumper.depth = 2
         dumper.include_filter = [
@@ -114,13 +90,9 @@ class BlImage(BlDatablock):
             'height',
             'alpha',
             'float_buffer',
-            'file_format',
             'alpha_mode',
-            'filepath',
-            'source',
             'colorspace_settings']
         data.update(dumper.dump(instance))
-
         return data
 
     def diff(self):
@@ -128,3 +100,24 @@ class BlImage(BlDatablock):
             return True
         else:
             return False
+
+    def _resolve_deps_implementation(self):
+        deps = []
+        if self.instance.filepath:
+
+            if self.instance.packed_file:
+                filename = Path(bpy.path.abspath(self.instance.filepath)).name
+                self.instance.filepath = get_filepath(filename)
+                self.instance.save()
+                # An image can't be unpacked to the modified path
+                # TODO: make a bug report
+                self.instance.unpack(method="REMOVE")
+
+            elif self.instance.source == "GENERATED":
+                filename = f"{self.instance.name}.png"
+                self.instance.filepath = get_filepath(filename)
+                self.instance.save()
+
+            deps.append(Path(self.instance.filepath))
+
+        return deps

multi_user/bl_types/bl_sound.py

@@ -16,14 +16,16 @@
 # ##### END GPL LICENSE BLOCK #####
 
 
-import bpy
-import mathutils
-import os
 import logging
-import pathlib
+import os
-from .. import utils
+from pathlib import Path
-from .dump_anything import Loader, Dumper
+
+import bpy
+
+from .bl_file import get_filepath, ensure_unpacked
 from .bl_datablock import BlDatablock
+from .dump_anything import Dumper, Loader
+
 
 class BlSound(BlDatablock):
     bl_id = "sounds"
@@ -35,40 +37,27 @@ class BlSound(BlDatablock):
     bl_icon = 'SOUND'
 
     def _construct(self, data):
-        if 'file' in data.keys():
+        filename = Path(data['filepath']).name
-            prefs = utils.get_preferences()
+        return bpy.data.sounds.load(get_filepath(filename))
-            ext = data['filepath'].split(".")[-1]
-            sound_name = f"{self.uuid}.{ext}"
-            sound_path = os.path.join(prefs.cache_directory, sound_name)
-            
-            os.makedirs(prefs.cache_directory, exist_ok=True)
-            file = open(sound_path, 'wb')
-            file.write(data["file"])
-            file.close()
-
-            logging.info(f'loading {sound_path}')
-            return bpy.data.sounds.load(sound_path)
 
     def _load(self, data, target):
         loader = Loader()
         loader.load(target, data)
 
-    def _dump(self, instance=None):
-        if not instance.packed_file:
-            # prefs = utils.get_preferences()
-            # ext = pathlib.Path(instance.filepath).suffix
-            # sound_name = f"{self.uuid}{ext}"
-            # sound_path = os.path.join(prefs.cache_directory, sound_name)
-            # instance.filepath = sound_path
-            instance.pack()
-            #TODO:use file locally with unpack(method='USE_ORIGINAL') ?
-
-        return {
-            'filepath':instance.filepath,
-            'name':instance.name,
-            'file': instance.packed_file.data
-        }
-
-
     def diff(self):
         return False
+
+    def _dump(self, instance=None):
+        return {
+            'filepath': instance.filepath,
+            'name': instance.name
+        }
+
+    def _resolve_deps_implementation(self):
+        deps = []
+        if self.instance.filepath and self.instance.filepath != '<builtin>':
+            ensure_unpacked(self.instance)
+            
+            deps.append(Path(self.instance.filepath))
+
+        return deps

multi_user/operators.py

@@ -27,6 +27,8 @@ from operator import itemgetter
 from pathlib import Path
 from subprocess import PIPE, Popen, TimeoutExpired
 import zmq
+import shutil
+from pathlib import Path
 
 import bpy
 import mathutils
@@ -608,6 +610,34 @@ class ApplyArmatureOperator(bpy.types.Operator):
         stop_modal_executor = False
 
 
+class ClearCache(bpy.types.Operator):
+    "Clear local session cache"
+    bl_idname = "session.clear_cache"
+    bl_label = "Modal Executor Operator"
+
+    @classmethod
+    def poll(cls, context):
+        return True
+
+    def execute(self, context):
+        cache_dir =  utils.get_preferences().cache_directory
+        try:
+            for root, dirs, files in os.walk(cache_dir):
+                for name in files:
+                    Path(root, name).unlink()
+
+        except Exception as e:
+            self.report({'ERROR'}, repr(e))
+
+        return {"FINISHED"}
+
+    def invoke(self, context, event):
+        return context.window_manager.invoke_props_dialog(self)
+
+    def draw(self, context):
+        row = self.layout
+        row.label(text=f" Do you really want to remove local cache ? ")
+
 classes = (
     SessionStartOperator,
     SessionStopOperator,
@@ -620,7 +650,7 @@ classes = (
     ApplyArmatureOperator,
     SessionKickOperator,
     SessionInitOperator,
-
+    ClearCache,
 )
 
 

multi_user/preferences.py

@@ -20,6 +20,9 @@ import logging
 import bpy
 import string
 import re
+import os
+
+from pathlib import Path
 
 from . import bl_types, environment, addon_updater_ops, presence, ui
 from .utils import get_preferences, get_expanded_icon
@@ -68,6 +71,16 @@ def update_port(self, context):
         self['ipc_port'] = random.randrange(self.port+4, 10000)
 
 
+def update_directory(self, context):
+    new_dir = Path(self.cache_directory)
+    if new_dir.exists() and any(Path(self.cache_directory).iterdir()):
+        logging.error("The folder is not empty, choose another one.")
+        self['cache_directory'] = environment.DEFAULT_CACHE_DIR
+    elif not new_dir.exists():
+        logging.info("Target cache folder doesn't exist, creating it.")
+        os.makedirs(self.cache_directory, exist_ok=True)
+
+
 def set_log_level(self, value):
     logging.getLogger().setLevel(value)
 
@@ -136,7 +149,8 @@ class SessionPrefs(bpy.types.AddonPreferences):
     cache_directory: bpy.props.StringProperty(
         name="cache directory",
         subtype="DIR_PATH",
-        default=environment.DEFAULT_CACHE_DIR)
+        default=environment.DEFAULT_CACHE_DIR,
+        update=update_directory)
     connection_timeout: bpy.props.IntProperty(
         name='connection timeout',
         description='connection timeout before disconnection',
@@ -162,6 +176,11 @@ class SessionPrefs(bpy.types.AddonPreferences):
         description="Enable objects update in edit mode (! Impact performances !)",
         default=False
     )
+    clear_memory_filecache: bpy.props.BoolProperty(
+        name="Clear memory filecache",
+        description="Remove filecache from memory",
+        default=False
+    )
     # for UI
     category: bpy.props.EnumProperty(
         name="Category",
@@ -230,6 +249,12 @@ class SessionPrefs(bpy.types.AddonPreferences):
         description="sidebar_advanced_net_expanded",
         default=False
     )
+    sidebar_advanced_cache_expanded: bpy.props.BoolProperty(
+        name="sidebar_advanced_cache_expanded",
+        description="sidebar_advanced_cache_expanded",
+        default=False
+    )
+
     auto_check_update: bpy.props.BoolProperty(
         name="Auto-check for Update",
         description="If enabled, auto-check for updates using an interval",
@@ -343,6 +368,7 @@ class SessionPrefs(bpy.types.AddonPreferences):
                 emboss=False)
             if self.conf_session_cache_expanded:
                 box.row().prop(self, "cache_directory", text="Cache directory")
+                box.row().prop(self, "clear_memory_filecache", text="Clear memory filecache")
 
             # INTERFACE SETTINGS
             box = grid.box()

multi_user/ui.py

@@ -19,7 +19,7 @@
 import bpy
 
 from . import operators
-from .utils import get_preferences, get_expanded_icon
+from .utils import get_preferences, get_expanded_icon, get_folder_size
 from replication.constants import (ADDED, ERROR, FETCHED,
                                                      MODIFIED, RP_COMMON, UP,
                                                      STATE_ACTIVE, STATE_AUTH,
@@ -356,6 +356,23 @@ class SESSION_PT_advanced_settings(bpy.types.Panel):
                 replication_timers.label(text="Update rate (ms):")
                 replication_timers.prop(settings, "depsgraph_update_rate", text="")
         
+        cache_section = layout.row().box()
+        cache_section.prop(
+            settings,
+            "sidebar_advanced_cache_expanded",
+            text="Cache",
+            icon=get_expanded_icon(settings.sidebar_advanced_cache_expanded), 
+            emboss=False)
+        if settings.sidebar_advanced_cache_expanded:
+            cache_section_row = cache_section.row()
+            cache_section_row.label(text="Cache directory:")
+            cache_section_row = cache_section.row()
+            cache_section_row.prop(settings, "cache_directory", text="")
+            cache_section_row = cache_section.row()
+            cache_section_row.label(text="Clear memory filecache:")
+            cache_section_row.prop(settings, "clear_memory_filecache", text="")
+            cache_section_row = cache_section.row()
+            cache_section_row.operator('session.clear_cache', text=f"Clear cache ({get_folder_size(settings.cache_directory)})")
         log_section = layout.row().box()
         log_section.prop(
             settings,

multi_user/utils.py

@@ -21,8 +21,10 @@ import logging
 import os
 import sys
 import time
-from uuid import uuid4
 from collections.abc import Iterable
+from pathlib import Path
+from uuid import uuid4
+import math
 
 import bpy
 import mathutils
@@ -47,7 +49,7 @@ def get_datablock_users(datablock):
     if hasattr(datablock, 'users_group') and datablock.users_scene:
         users.extend(list(datablock.users_scene))
     for datatype in supported_types:
-        if datatype.bl_name != 'users':
+        if datatype.bl_name != 'users' and hasattr(bpy.data, datatype.bl_name):
             root = getattr(bpy.data, datatype.bl_name)
             for item in root:
                 if hasattr(item, 'data') and datablock == item.data or \
@@ -78,6 +80,7 @@ def resolve_from_id(id, optionnal_type=None):
                 return root[id]
     return None
 
+
 def get_datablock_from_uuid(uuid, default, ignore=[]):
     if not uuid:
         return default
@@ -90,6 +93,7 @@ def get_datablock_from_uuid(uuid, default, ignore=[]):
                     return item
     return default
 
+
 def get_preferences():
     return bpy.context.preferences.addons[__package__].preferences
 
@@ -103,3 +107,61 @@ def get_expanded_icon(prop: bpy.types.BoolProperty) -> str:
         return 'DISCLOSURE_TRI_DOWN'
     else:
         return 'DISCLOSURE_TRI_RIGHT'
+
+
+# Taken from here: https://stackoverflow.com/a/55659577
+def get_folder_size(folder):
+    return ByteSize(sum(file.stat().st_size for file in Path(folder).rglob('*')))
+
+
+class ByteSize(int):
+
+    _kB = 1024
+    _suffixes = 'B', 'kB', 'MB', 'GB', 'PB'
+
+    def __new__(cls, *args, **kwargs):
+        return super().__new__(cls, *args, **kwargs)
+
+    def __init__(self, *args, **kwargs):
+        self.bytes = self.B = int(self)
+        self.kilobytes = self.kB = self / self._kB**1
+        self.megabytes = self.MB = self / self._kB**2
+        self.gigabytes = self.GB = self / self._kB**3
+        self.petabytes = self.PB = self / self._kB**4
+        *suffixes, last = self._suffixes
+        suffix = next((
+            suffix
+            for suffix in suffixes
+            if 1 < getattr(self, suffix) < self._kB
+        ), last)
+        self.readable = suffix, getattr(self, suffix)
+
+        super().__init__()
+
+    def __str__(self):
+        return self.__format__('.2f')
+
+    def __repr__(self):
+        return '{}({})'.format(self.__class__.__name__, super().__repr__())
+
+    def __format__(self, format_spec):
+        suffix, val = self.readable
+        return '{val:{fmt}} {suf}'.format(val=math.ceil(val), fmt=format_spec, suf=suffix)
+
+    def __sub__(self, other):
+        return self.__class__(super().__sub__(other))
+
+    def __add__(self, other):
+        return self.__class__(super().__add__(other))
+
+    def __mul__(self, other):
+        return self.__class__(super().__mul__(other))
+
+    def __rsub__(self, other):
+        return self.__class__(super().__sub__(other))
+
+    def __radd__(self, other):
+        return self.__class__(super().__add__(other))
+
+    def __rmul__(self, other):
+        return self.__class__(super().__rmul__(other))

tests/test_bl_types/test_image.py

@@ -1,21 +0,0 @@
-import os
-
-import pytest
-from deepdiff import DeepDiff
-
-import bpy
-import random
-from multi_user.bl_types.bl_image import BlImage
-
-def test_image(clear_blend):
-    datablock = bpy.data.images.new('asd',2000,2000)
-    
-    implementation = BlImage()
-    expected = implementation._dump(datablock)
-    bpy.data.images.remove(datablock)
-
-    test = implementation._construct(expected)
-    implementation._load(expected, test)
-    result = implementation._dump(test)
-
-    assert not DeepDiff(expected, result)