diff --git a/.gitlab/ci/test.gitlab-ci.yml b/.gitlab/ci/test.gitlab-ci.yml
index 8ee1972..ccacbb3 100644
--- a/.gitlab/ci/test.gitlab-ci.yml
+++ b/.gitlab/ci/test.gitlab-ci.yml
@@ -3,8 +3,3 @@ test:
image: slumber/blender-addon-testing:latest
script:
- python3 scripts/test_addon.py
-
- only:
- refs:
- - master
- - develop
diff --git a/docs/getting_started/img/quickstart_advanced_cache.png b/docs/getting_started/img/quickstart_advanced_cache.png
new file mode 100644
index 0000000..fd56043
Binary files /dev/null and b/docs/getting_started/img/quickstart_advanced_cache.png differ
diff --git a/docs/getting_started/quickstart.rst b/docs/getting_started/quickstart.rst
index d0e26eb..cc4751f 100644
--- a/docs/getting_started/quickstart.rst
+++ b/docs/getting_started/quickstart.rst
@@ -355,6 +355,26 @@ Replication
- **Refresh**: pushed data update rate (in second)
- **Apply**: pulled data update rate (in second)
+-----
+Cache
+-----
+
+The multi-user allows to replicate external blend dependencies such as images, movies sounds.
+On each client, those files are stored into the cache folder.
+
+.. figure:: img/quickstart_advanced_cache.png
+ :align: center
+
+ Advanced cache settings
+
+**cache_directory** allows to choose where cached files (images, sound, movies) will be saved.
+
+**Clear memory filecache** will save memory space at runtime by removing the file content from memory as soon as it have been written to the disk.
+
+**Clear cache** will remove all file from the cache folder.
+
+.. warning:: Clear cash could break your scene image/movie/sound if they are used into the blend !
+
---
Log
---
diff --git a/multi_user/bl_types/__init__.py b/multi_user/bl_types/__init__.py
index ad35fe8..add7058 100644
--- a/multi_user/bl_types/__init__.py
+++ b/multi_user/bl_types/__init__.py
@@ -36,7 +36,8 @@ __all__ = [
'bl_lightprobe',
'bl_speaker',
'bl_font',
- 'bl_sound'
+ 'bl_sound',
+ 'bl_file'
] # Order here defines execution order
from . import *
diff --git a/multi_user/bl_types/bl_file.py b/multi_user/bl_types/bl_file.py
new file mode 100644
index 0000000..6b07c43
--- /dev/null
+++ b/multi_user/bl_types/bl_file.py
@@ -0,0 +1,132 @@
+# ##### BEGIN GPL LICENSE BLOCK #####
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+#
+# ##### END GPL LICENSE BLOCK #####
+
+
+import logging
+import os
+import sys
+from pathlib import Path
+
+import bpy
+import mathutils
+from replication.constants import DIFF_BINARY, UP
+from replication.data import ReplicatedDatablock
+
+from .. import utils
+from .dump_anything import Dumper, Loader
+
+
+def get_filepath(filename):
+ """
+ Construct the local filepath
+ """
+ return str(Path(
+ utils.get_preferences().cache_directory,
+ filename
+ ))
+
+
+def ensure_unpacked(datablock):
+ if datablock.packed_file:
+ logging.info(f"Unpacking {datablock.name}")
+
+ filename = Path(bpy.path.abspath(datablock.filepath)).name
+ datablock.filepath = get_filepath(filename)
+
+ datablock.unpack(method="WRITE_ORIGINAL")
+
+
+class BlFile(ReplicatedDatablock):
+ bl_id = 'file'
+ bl_name = "file"
+ bl_class = Path
+ bl_delay_refresh = 0
+ bl_delay_apply = 1
+ bl_automatic_push = True
+ bl_check_common = False
+ bl_icon = 'FILE'
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.instance = kwargs.get('instance', None)
+ self.preferences = utils.get_preferences()
+ self.diff_method = DIFF_BINARY
+
+ def resolve(self):
+ if self.data:
+ self.instance = Path(get_filepath(self.data['name']))
+
+ def push(self, socket, identity=None):
+ super().push(socket, identity=None)
+
+ if self.preferences.clear_memory_filecache:
+ del self.data['file']
+
+ def _dump(self, instance=None):
+ """
+ Read the file and return a dict as:
+ {
+ name : filename
+ extension :
+ file: file content
+ }
+ """
+ logging.info(f"Extracting file metadata")
+
+ data = {
+ 'name': self.instance.name,
+ }
+
+ logging.info(
+ f"Reading {self.instance.name} content: {self.instance.stat().st_size} bytes")
+
+ try:
+ file = open(self.instance, "rb")
+ data['file'] = file.read()
+
+ file.close()
+ except IOError:
+ logging.warning(f"{self.instance} doesn't exist, skipping")
+ else:
+ file.close()
+
+ return data
+
+ def _load(self, data, target):
+ """
+ Writing the file
+ """
+ # TODO: check for empty data
+
+ if target.exists() and not self.diff():
+ logging.info(f"{data['name']} already on the disk, skipping.")
+ return
+ try:
+ file = open(target, "wb")
+ file.write(data['file'])
+
+ if self.preferences.clear_memory_filecache:
+ del self.data['file']
+ except IOError:
+ logging.warning(f"{target} doesn't exist, skipping")
+ else:
+ file.close()
+
+ def diff(self):
+ memory_size = sys.getsizeof(self.data['file'])-33
+ disk_size = self.instance.stat().st_size
+ return memory_size == disk_size
diff --git a/multi_user/bl_types/bl_font.py b/multi_user/bl_types/bl_font.py
index 599f3ad..157f9f2 100644
--- a/multi_user/bl_types/bl_font.py
+++ b/multi_user/bl_types/bl_font.py
@@ -16,14 +16,16 @@
# ##### END GPL LICENSE BLOCK #####
-import bpy
-import mathutils
-import os
import logging
-import pathlib
-from .. import utils
-from .dump_anything import Loader, Dumper
+import os
+from pathlib import Path
+
+import bpy
+
from .bl_datablock import BlDatablock
+from .bl_file import get_filepath, ensure_unpacked
+from .dump_anything import Dumper, Loader
+
class BlFont(BlDatablock):
bl_id = "fonts"
@@ -37,33 +39,28 @@ class BlFont(BlDatablock):
def _construct(self, data):
if data['filepath'] == '':
return bpy.data.fonts.load(data['filepath'])
- elif 'font_file' in data.keys():
- prefs = utils.get_preferences()
- ext = pathlib.Path(data['filepath']).suffix
- font_name = f"{self.uuid}{ext}"
- font_path = os.path.join(prefs.cache_directory, font_name)
-
- os.makedirs(prefs.cache_directory, exist_ok=True)
- file = open(font_path, 'wb')
- file.write(data["font_file"])
- file.close()
-
- logging.info(f'loading {font_path}')
- return bpy.data.fonts.load(font_path)
+ else:
+ filename = Path(data['filepath']).name
+ return bpy.data.fonts.load(get_filepath(filename))
def _load(self, data, target):
- pass
+ loader = Loader()
+ loader.load(target, data)
def _dump(self, instance=None):
- data = {
- 'filepath':instance.filepath,
- 'name':instance.name
+ return {
+ 'filepath': instance.filepath,
+ 'name': instance.name
}
- if instance.filepath != '' and not instance.is_embedded_data:
- file = open(instance.filepath, "rb")
- data['font_file'] = file.read()
- file.close()
- return data
def diff(self):
return False
+
+ def _resolve_deps_implementation(self):
+ deps = []
+ if self.instance.filepath and self.instance.filepath != '':
+ ensure_unpacked(self.instance)
+
+ deps.append(Path(self.instance.filepath))
+
+ return deps
diff --git a/multi_user/bl_types/bl_image.py b/multi_user/bl_types/bl_image.py
index 669d9a9..6026ca2 100644
--- a/multi_user/bl_types/bl_image.py
+++ b/multi_user/bl_types/bl_image.py
@@ -16,13 +16,17 @@
# ##### END GPL LICENSE BLOCK #####
+import logging
+import os
+from pathlib import Path
+
import bpy
import mathutils
-import os
-import logging
+
from .. import utils
-from .dump_anything import Loader, Dumper
from .bl_datablock import BlDatablock
+from .dump_anything import Dumper, Loader
+from .bl_file import get_filepath, ensure_unpacked
format_to_ext = {
'BMP': 'bmp',
@@ -53,29 +57,6 @@ class BlImage(BlDatablock):
bl_check_common = False
bl_icon = 'IMAGE_DATA'
- def dump_image(self, image):
- pixels = None
- if image.source == "GENERATED" or image.packed_file is not None:
- prefs = utils.get_preferences()
- img_name = f"{self.uuid}.{format_to_ext[image.file_format]}"
-
- # Cache the image on the disk
- image.filepath_raw = os.path.join(prefs.cache_directory, img_name)
- os.makedirs(prefs.cache_directory, exist_ok=True)
- image.save()
-
- if image.source == "FILE":
- image_path = bpy.path.abspath(image.filepath_raw)
- image_directory = os.path.dirname(image_path)
- os.makedirs(image_directory, exist_ok=True)
- image.save()
- file = open(image_path, "rb")
- pixels = file.read()
- file.close()
- else:
- raise ValueError()
- return pixels
-
def _construct(self, data):
return bpy.data.images.new(
name=data['name'],
@@ -84,28 +65,23 @@ class BlImage(BlDatablock):
)
def _load(self, data, target):
- image = target
- prefs = utils.get_preferences()
- img_format = data['file_format']
- img_name = f"{self.uuid}.{format_to_ext[img_format]}"
-
- img_path = os.path.join(prefs.cache_directory, img_name)
- os.makedirs(prefs.cache_directory, exist_ok=True)
- file = open(img_path, 'wb')
- file.write(data["pixels"])
- file.close()
-
- image.source = 'FILE'
- image.filepath = img_path
- image.colorspace_settings.name = data["colorspace_settings"]["name"]
-
loader = Loader()
loader.load(data, target)
+ target.source = 'FILE'
+ target.filepath_raw = get_filepath(data['filename'])
+ target.colorspace_settings.name = data["colorspace_settings"]["name"]
+
+
def _dump(self, instance=None):
assert(instance)
- data = {}
- data['pixels'] = self.dump_image(instance)
+
+ filename = Path(instance.filepath).name
+
+ data = {
+ "filename": filename
+ }
+
dumper = Dumper()
dumper.depth = 2
dumper.include_filter = [
@@ -114,13 +90,9 @@ class BlImage(BlDatablock):
'height',
'alpha',
'float_buffer',
- 'file_format',
'alpha_mode',
- 'filepath',
- 'source',
'colorspace_settings']
data.update(dumper.dump(instance))
-
return data
def diff(self):
@@ -128,3 +100,24 @@ class BlImage(BlDatablock):
return True
else:
return False
+
+ def _resolve_deps_implementation(self):
+ deps = []
+ if self.instance.filepath:
+
+ if self.instance.packed_file:
+ filename = Path(bpy.path.abspath(self.instance.filepath)).name
+ self.instance.filepath = get_filepath(filename)
+ self.instance.save()
+ # An image can't be unpacked to the modified path
+ # TODO: make a bug report
+ self.instance.unpack(method="REMOVE")
+
+ elif self.instance.source == "GENERATED":
+ filename = f"{self.instance.name}.png"
+ self.instance.filepath = get_filepath(filename)
+ self.instance.save()
+
+ deps.append(Path(self.instance.filepath))
+
+ return deps
diff --git a/multi_user/bl_types/bl_sound.py b/multi_user/bl_types/bl_sound.py
index bd9bafb..93f3990 100644
--- a/multi_user/bl_types/bl_sound.py
+++ b/multi_user/bl_types/bl_sound.py
@@ -16,14 +16,16 @@
# ##### END GPL LICENSE BLOCK #####
-import bpy
-import mathutils
-import os
import logging
-import pathlib
-from .. import utils
-from .dump_anything import Loader, Dumper
+import os
+from pathlib import Path
+
+import bpy
+
+from .bl_file import get_filepath, ensure_unpacked
from .bl_datablock import BlDatablock
+from .dump_anything import Dumper, Loader
+
class BlSound(BlDatablock):
bl_id = "sounds"
@@ -35,40 +37,27 @@ class BlSound(BlDatablock):
bl_icon = 'SOUND'
def _construct(self, data):
- if 'file' in data.keys():
- prefs = utils.get_preferences()
- ext = data['filepath'].split(".")[-1]
- sound_name = f"{self.uuid}.{ext}"
- sound_path = os.path.join(prefs.cache_directory, sound_name)
-
- os.makedirs(prefs.cache_directory, exist_ok=True)
- file = open(sound_path, 'wb')
- file.write(data["file"])
- file.close()
-
- logging.info(f'loading {sound_path}')
- return bpy.data.sounds.load(sound_path)
+ filename = Path(data['filepath']).name
+ return bpy.data.sounds.load(get_filepath(filename))
def _load(self, data, target):
loader = Loader()
loader.load(target, data)
- def _dump(self, instance=None):
- if not instance.packed_file:
- # prefs = utils.get_preferences()
- # ext = pathlib.Path(instance.filepath).suffix
- # sound_name = f"{self.uuid}{ext}"
- # sound_path = os.path.join(prefs.cache_directory, sound_name)
- # instance.filepath = sound_path
- instance.pack()
- #TODO:use file locally with unpack(method='USE_ORIGINAL') ?
-
- return {
- 'filepath':instance.filepath,
- 'name':instance.name,
- 'file': instance.packed_file.data
- }
-
-
def diff(self):
return False
+
+ def _dump(self, instance=None):
+ return {
+ 'filepath': instance.filepath,
+ 'name': instance.name
+ }
+
+ def _resolve_deps_implementation(self):
+ deps = []
+ if self.instance.filepath and self.instance.filepath != '':
+ ensure_unpacked(self.instance)
+
+ deps.append(Path(self.instance.filepath))
+
+ return deps
diff --git a/multi_user/operators.py b/multi_user/operators.py
index 381d824..7353823 100644
--- a/multi_user/operators.py
+++ b/multi_user/operators.py
@@ -27,6 +27,8 @@ from operator import itemgetter
from pathlib import Path
from subprocess import PIPE, Popen, TimeoutExpired
import zmq
+import shutil
+from pathlib import Path
import bpy
import mathutils
@@ -608,6 +610,34 @@ class ApplyArmatureOperator(bpy.types.Operator):
stop_modal_executor = False
+class ClearCache(bpy.types.Operator):
+ "Clear local session cache"
+ bl_idname = "session.clear_cache"
+ bl_label = "Modal Executor Operator"
+
+ @classmethod
+ def poll(cls, context):
+ return True
+
+ def execute(self, context):
+ cache_dir = utils.get_preferences().cache_directory
+ try:
+ for root, dirs, files in os.walk(cache_dir):
+ for name in files:
+ Path(root, name).unlink()
+
+ except Exception as e:
+ self.report({'ERROR'}, repr(e))
+
+ return {"FINISHED"}
+
+ def invoke(self, context, event):
+ return context.window_manager.invoke_props_dialog(self)
+
+ def draw(self, context):
+ row = self.layout
+ row.label(text=f" Do you really want to remove local cache ? ")
+
classes = (
SessionStartOperator,
SessionStopOperator,
@@ -620,7 +650,7 @@ classes = (
ApplyArmatureOperator,
SessionKickOperator,
SessionInitOperator,
-
+ ClearCache,
)
diff --git a/multi_user/preferences.py b/multi_user/preferences.py
index 90ec490..905833f 100644
--- a/multi_user/preferences.py
+++ b/multi_user/preferences.py
@@ -20,6 +20,9 @@ import logging
import bpy
import string
import re
+import os
+
+from pathlib import Path
from . import bl_types, environment, addon_updater_ops, presence, ui
from .utils import get_preferences, get_expanded_icon
@@ -68,6 +71,16 @@ def update_port(self, context):
self['ipc_port'] = random.randrange(self.port+4, 10000)
+def update_directory(self, context):
+ new_dir = Path(self.cache_directory)
+ if new_dir.exists() and any(Path(self.cache_directory).iterdir()):
+ logging.error("The folder is not empty, choose another one.")
+ self['cache_directory'] = environment.DEFAULT_CACHE_DIR
+ elif not new_dir.exists():
+ logging.info("Target cache folder doesn't exist, creating it.")
+ os.makedirs(self.cache_directory, exist_ok=True)
+
+
def set_log_level(self, value):
logging.getLogger().setLevel(value)
@@ -136,7 +149,8 @@ class SessionPrefs(bpy.types.AddonPreferences):
cache_directory: bpy.props.StringProperty(
name="cache directory",
subtype="DIR_PATH",
- default=environment.DEFAULT_CACHE_DIR)
+ default=environment.DEFAULT_CACHE_DIR,
+ update=update_directory)
connection_timeout: bpy.props.IntProperty(
name='connection timeout',
description='connection timeout before disconnection',
@@ -162,6 +176,11 @@ class SessionPrefs(bpy.types.AddonPreferences):
description="Enable objects update in edit mode (! Impact performances !)",
default=False
)
+ clear_memory_filecache: bpy.props.BoolProperty(
+ name="Clear memory filecache",
+ description="Remove filecache from memory",
+ default=False
+ )
# for UI
category: bpy.props.EnumProperty(
name="Category",
@@ -230,6 +249,12 @@ class SessionPrefs(bpy.types.AddonPreferences):
description="sidebar_advanced_net_expanded",
default=False
)
+ sidebar_advanced_cache_expanded: bpy.props.BoolProperty(
+ name="sidebar_advanced_cache_expanded",
+ description="sidebar_advanced_cache_expanded",
+ default=False
+ )
+
auto_check_update: bpy.props.BoolProperty(
name="Auto-check for Update",
description="If enabled, auto-check for updates using an interval",
@@ -343,6 +368,7 @@ class SessionPrefs(bpy.types.AddonPreferences):
emboss=False)
if self.conf_session_cache_expanded:
box.row().prop(self, "cache_directory", text="Cache directory")
+ box.row().prop(self, "clear_memory_filecache", text="Clear memory filecache")
# INTERFACE SETTINGS
box = grid.box()
diff --git a/multi_user/ui.py b/multi_user/ui.py
index ef79e9c..954a1f0 100644
--- a/multi_user/ui.py
+++ b/multi_user/ui.py
@@ -19,7 +19,7 @@
import bpy
from . import operators
-from .utils import get_preferences, get_expanded_icon
+from .utils import get_preferences, get_expanded_icon, get_folder_size
from replication.constants import (ADDED, ERROR, FETCHED,
MODIFIED, RP_COMMON, UP,
STATE_ACTIVE, STATE_AUTH,
@@ -356,6 +356,23 @@ class SESSION_PT_advanced_settings(bpy.types.Panel):
replication_timers.label(text="Update rate (ms):")
replication_timers.prop(settings, "depsgraph_update_rate", text="")
+ cache_section = layout.row().box()
+ cache_section.prop(
+ settings,
+ "sidebar_advanced_cache_expanded",
+ text="Cache",
+ icon=get_expanded_icon(settings.sidebar_advanced_cache_expanded),
+ emboss=False)
+ if settings.sidebar_advanced_cache_expanded:
+ cache_section_row = cache_section.row()
+ cache_section_row.label(text="Cache directory:")
+ cache_section_row = cache_section.row()
+ cache_section_row.prop(settings, "cache_directory", text="")
+ cache_section_row = cache_section.row()
+ cache_section_row.label(text="Clear memory filecache:")
+ cache_section_row.prop(settings, "clear_memory_filecache", text="")
+ cache_section_row = cache_section.row()
+ cache_section_row.operator('session.clear_cache', text=f"Clear cache ({get_folder_size(settings.cache_directory)})")
log_section = layout.row().box()
log_section.prop(
settings,
diff --git a/multi_user/utils.py b/multi_user/utils.py
index f66ab90..75c1391 100644
--- a/multi_user/utils.py
+++ b/multi_user/utils.py
@@ -21,8 +21,10 @@ import logging
import os
import sys
import time
-from uuid import uuid4
from collections.abc import Iterable
+from pathlib import Path
+from uuid import uuid4
+import math
import bpy
import mathutils
@@ -47,7 +49,7 @@ def get_datablock_users(datablock):
if hasattr(datablock, 'users_group') and datablock.users_scene:
users.extend(list(datablock.users_scene))
for datatype in supported_types:
- if datatype.bl_name != 'users':
+ if datatype.bl_name != 'users' and hasattr(bpy.data, datatype.bl_name):
root = getattr(bpy.data, datatype.bl_name)
for item in root:
if hasattr(item, 'data') and datablock == item.data or \
@@ -78,6 +80,7 @@ def resolve_from_id(id, optionnal_type=None):
return root[id]
return None
+
def get_datablock_from_uuid(uuid, default, ignore=[]):
if not uuid:
return default
@@ -87,9 +90,10 @@ def get_datablock_from_uuid(uuid, default, ignore=[]):
if isinstance(root, Iterable) and category not in ignore:
for item in root:
if getattr(item, 'uuid', None) == uuid:
- return item
+ return item
return default
+
def get_preferences():
return bpy.context.preferences.addons[__package__].preferences
@@ -103,3 +107,61 @@ def get_expanded_icon(prop: bpy.types.BoolProperty) -> str:
return 'DISCLOSURE_TRI_DOWN'
else:
return 'DISCLOSURE_TRI_RIGHT'
+
+
+# Taken from here: https://stackoverflow.com/a/55659577
+def get_folder_size(folder):
+ return ByteSize(sum(file.stat().st_size for file in Path(folder).rglob('*')))
+
+
+class ByteSize(int):
+
+ _kB = 1024
+ _suffixes = 'B', 'kB', 'MB', 'GB', 'PB'
+
+ def __new__(cls, *args, **kwargs):
+ return super().__new__(cls, *args, **kwargs)
+
+ def __init__(self, *args, **kwargs):
+ self.bytes = self.B = int(self)
+ self.kilobytes = self.kB = self / self._kB**1
+ self.megabytes = self.MB = self / self._kB**2
+ self.gigabytes = self.GB = self / self._kB**3
+ self.petabytes = self.PB = self / self._kB**4
+ *suffixes, last = self._suffixes
+ suffix = next((
+ suffix
+ for suffix in suffixes
+ if 1 < getattr(self, suffix) < self._kB
+ ), last)
+ self.readable = suffix, getattr(self, suffix)
+
+ super().__init__()
+
+ def __str__(self):
+ return self.__format__('.2f')
+
+ def __repr__(self):
+ return '{}({})'.format(self.__class__.__name__, super().__repr__())
+
+ def __format__(self, format_spec):
+ suffix, val = self.readable
+ return '{val:{fmt}} {suf}'.format(val=math.ceil(val), fmt=format_spec, suf=suffix)
+
+ def __sub__(self, other):
+ return self.__class__(super().__sub__(other))
+
+ def __add__(self, other):
+ return self.__class__(super().__add__(other))
+
+ def __mul__(self, other):
+ return self.__class__(super().__mul__(other))
+
+ def __rsub__(self, other):
+ return self.__class__(super().__sub__(other))
+
+ def __radd__(self, other):
+ return self.__class__(super().__add__(other))
+
+ def __rmul__(self, other):
+ return self.__class__(super().__rmul__(other))
diff --git a/tests/test_bl_types/test_image.py b/tests/test_bl_types/test_image.py
deleted file mode 100644
index e597526..0000000
--- a/tests/test_bl_types/test_image.py
+++ /dev/null
@@ -1,21 +0,0 @@
-import os
-
-import pytest
-from deepdiff import DeepDiff
-
-import bpy
-import random
-from multi_user.bl_types.bl_image import BlImage
-
-def test_image(clear_blend):
- datablock = bpy.data.images.new('asd',2000,2000)
-
- implementation = BlImage()
- expected = implementation._dump(datablock)
- bpy.data.images.remove(datablock)
-
- test = implementation._construct(expected)
- implementation._load(expected, test)
- result = implementation._dump(test)
-
- assert not DeepDiff(expected, result)