work: save startup because I'm sick of the arnoldesque materials

scripts/addons/flamenco/__init__.py

@@ -5,7 +5,7 @@
 bl_info = {
     "name": "Flamenco",
     "author": "Sybren A. Stüvel",
-    "version": (3, 8, 2),
+    "version": (3, 8, 5),
     "blender": (3, 1, 0),
     "description": "Flamenco client for Blender.",
     "location": "Output Properties > Flamenco",

scripts/addons/flamenco/manager/__init__.py

@@ -10,7 +10,7 @@
 """
 
 
-__version__ = "3.8.2"
+__version__ = "3.8.5"
 
 # import ApiClient
 from flamenco.manager.api_client import ApiClient

scripts/addons/flamenco/manager/api_client.py

@@ -76,7 +76,7 @@ class ApiClient(object):
             self.default_headers[header_name] = header_value
         self.cookie = cookie
         # Set default User-Agent.
-        self.user_agent = 'Flamenco/3.8.2 (Blender add-on)'
+        self.user_agent = 'Flamenco/3.8.5 (Blender add-on)'
 
     def __enter__(self):
         return self

scripts/addons/flamenco/manager/configuration.py

@@ -404,7 +404,7 @@ conf = flamenco.manager.Configuration(
                "OS: {env}\n"\
                "Python Version: {pyversion}\n"\
                "Version of the API: 1.0.0\n"\
-               "SDK Package Version: 3.8.2".\
+               "SDK Package Version: 3.8.5".\
                format(env=sys.platform, pyversion=sys.version)
 
     def get_host_settings(self):

scripts/addons/flamenco/manager_README.md

@@ -4,7 +4,7 @@ Render Farm manager API
 The `flamenco.manager` package is automatically generated by the [OpenAPI Generator](https://openapi-generator.tech) project:
 
 - API version: 1.0.0
-- Package version: 3.8.2
+- Package version: 3.8.5
 - Build package: org.openapitools.codegen.languages.PythonClientCodegen
 For more information, please visit [https://flamenco.blender.org/](https://flamenco.blender.org/)
 

scripts/addons/flamenco/operators.py

@@ -3,6 +3,7 @@
 
 import datetime
 import logging
+import os
 import time
 from pathlib import Path, PurePosixPath
 from typing import Optional, TYPE_CHECKING
@@ -363,11 +364,132 @@ class FLAMENCO_OT_submit_job(FlamencoOpMixin, bpy.types.Operator):
 
         return filepath
 
+    def _bat_project_path(
+        self, context: bpy.types.Context, blendfile: Path
+    ) -> Path:
+        """BAT 'project' directory: preference root, or wider if links lie outside it.
+
+        Linked blends outside the configured project force BAT to use KEEP_PATH;
+        Shaman packing can then fail path rewriting. Using a common ancestor of
+        the submission blend and all linked library paths keeps assets inside
+        the project tree for BAT.
+        """
+        prefs = preferences.get(context)
+        configured = bpathlib.make_absolute(
+            Path(bpy.path.abspath(str(prefs.project_root())))
+        )
+        blend_abs = bpathlib.make_absolute(blendfile)
+
+        lib_paths: list[Path] = []
+        for lib in bpy.data.libraries:
+            if not lib.filepath:
+                continue
+            lib_paths.append(
+                bpathlib.make_absolute(Path(bpy.path.abspath(lib.filepath)))
+            )
+
+        def is_under(root: Path, path: Path) -> bool:
+            try:
+                path.resolve().relative_to(root.resolve())
+                return True
+            except ValueError:
+                return False
+
+        need_widen = (not is_under(configured, blend_abs)) or any(
+            not is_under(configured, lp) for lp in lib_paths
+        )
+        if not need_widen:
+            return configured
+
+        all_paths = [blend_abs] + lib_paths
+        try:
+            common = Path(os.path.commonpath([str(p) for p in all_paths])).resolve()
+        except ValueError:
+            self.log.warning(
+                "Could not compute common path for BAT project root, using preferences"
+            )
+            return configured
+
+        self.log.info(
+            "BAT project root widened from %s to %s (assets outside preference project)",
+            configured,
+            common,
+        )
+        return common
+
+    def _convert_relpaths_to_absolute(self, context: bpy.types.Context) -> None:
+        """Convert all relative paths in the blend file to absolute paths.
+
+        Covers libraries, images, clips, sounds, fonts, volumes, and cache_files
+        (point caches / GN caches). Allows the blend to be sent as-is without BAT.
+        """
+        # Convert library paths to absolute
+        for library in bpy.data.libraries:
+            if library.filepath:
+                old_path = library.filepath
+                abs_path = bpy.path.abspath(library.filepath)
+                library.filepath = abs_path
+                self.log.debug("Converted library path: %s -> %s", old_path, abs_path)
+        
+        # Convert image paths to absolute
+        for image in bpy.data.images:
+            if image.filepath and not image.packed_file:
+                old_path = image.filepath
+                abs_path = bpy.path.abspath(image.filepath)
+                image.filepath = abs_path
+                self.log.debug("Converted image path: %s -> %s", old_path, abs_path)
+        
+        # Convert movie paths to absolute
+        for movie in bpy.data.movieclips:
+            if movie.filepath:
+                old_path = movie.filepath
+                abs_path = bpy.path.abspath(movie.filepath)
+                movie.filepath = abs_path
+                self.log.debug("Converted movie path: %s -> %s", old_path, abs_path)
+        
+        # Convert sound paths to absolute
+        for sound in bpy.data.sounds:
+            if sound.filepath:
+                old_path = sound.filepath
+                abs_path = bpy.path.abspath(sound.filepath)
+                sound.filepath = abs_path
+                self.log.debug("Converted sound path: %s -> %s", old_path, abs_path)
+        
+        # Convert font paths to absolute (skip VectorFont - its filepath is read-only)
+        for font in bpy.data.fonts:
+            if font.filepath:
+                try:
+                    old_path = font.filepath
+                    abs_path = bpy.path.abspath(font.filepath)
+                    font.filepath = abs_path
+                    self.log.debug("Converted font path: %s -> %s", old_path, abs_path)
+                except (TypeError, AttributeError):
+                    self.log.debug("Skipping font %s (filepath is read-only)", font.name)
+        
+        # Convert volume paths to absolute
+        for volume in bpy.data.volumes:
+            if volume.filepath:
+                old_path = volume.filepath
+                abs_path = bpy.path.abspath(volume.filepath)
+                volume.filepath = abs_path
+                self.log.debug("Converted volume path: %s -> %s", old_path, abs_path)
+
+        # Point / mesh cache files (e.g. .pc2), geometry-nodes caches, etc.
+        for cache_file in bpy.data.cache_files:
+            if cache_file.filepath:
+                old_path = cache_file.filepath
+                abs_path = bpy.path.abspath(cache_file.filepath)
+                cache_file.filepath = abs_path
+                self.log.debug("Converted cache_file path: %s -> %s", old_path, abs_path)
+
     def _submit_files(self, context: bpy.types.Context, blendfile: Path) -> bool:
         """Ensure that the files are somewhere in the shared storage.
 
         Returns True if a packing thread has been started, and False otherwise.
         """
+        prefs = preferences.get(context)
+        if prefs.bat_bypass:
+            return self._submit_files_bat_bypass(context, blendfile)
 
         from .bat import interface as bat_interface
 
@@ -405,6 +527,179 @@ class FLAMENCO_OT_submit_job(FlamencoOpMixin, bpy.types.Operator):
 
         return True
 
+    def _submit_files_bat_bypass(self, context: bpy.types.Context, blendfile: Path) -> bool:
+        """Bypass BAT: absolute paths, then upload or copy the blend only."""
+
+        manager = self._manager_info(context)
+        if not manager:
+            return False
+
+        self.log.info("Converting all relative paths to absolute")
+        self._convert_relpaths_to_absolute(context)
+
+        self.log.info("Saving blend file with absolute paths")
+        blendfile = self._save_blendfile(context)
+        blendfile = bpathlib.make_absolute(blendfile)
+
+        if manager.shared_storage.shaman_enabled:
+            self.log.info("Uploading blend file directly to Shaman (bypassing BAT)")
+            self._upload_blendfile_to_shaman(context, blendfile)
+            self._quit(context)
+            return False
+        if job_submission.is_file_inside_job_storage(context, blendfile):
+            self.log.info(
+                "File is already in job storage location, submitting it as-is"
+            )
+            self._use_blendfile_directly(context, blendfile)
+            return False
+        self.log.info(
+            "File is not already in job storage location, copying it there"
+        )
+        try:
+            self._copy_blendfile_to_storage(context, blendfile)
+        except FileNotFoundError:
+            self._quit(context)
+            return False
+        return False
+
+    def _upload_blendfile_to_shaman(
+        self, context: bpy.types.Context, blendfile: Path
+    ) -> None:
+        """Upload blend file directly to Shaman without BAT.
+        
+        Creates a Shaman checkout with just the blend file, maintaining its
+        relative path from the project root.
+        """
+        from .bat import cache
+        from .manager.apis import ShamanApi
+        from .manager.models import (
+            ShamanFileSpec,
+            ShamanCheckout,
+        )
+        from .manager.exceptions import ApiException
+        from . import preferences
+        
+        api_client = self.get_api_client(context)
+        shaman_api = ShamanApi(api_client)
+        
+        # Get project root to calculate relative path
+        prefs = preferences.get(context)
+        project_path: Path = prefs.project_root()
+        project_path = bpathlib.make_absolute(Path(bpy.path.abspath(str(project_path))))
+        
+        # Calculate relative path from project root
+        try:
+            blendfile_rel_path = blendfile.relative_to(project_path)
+            # Convert to POSIX path for Shaman
+            blendfile_path_in_checkout = PurePosixPath(blendfile_rel_path.as_posix())
+        except ValueError:
+            # Blend file is not under project root, use just the filename
+            self.log.warning(
+                "Blend file %s is not under project root %s, using filename only",
+                blendfile,
+                project_path,
+            )
+            blendfile_path_in_checkout = PurePosixPath(blendfile.name)
+        
+        # Compute checksum and file size
+        self.log.info("Computing checksum for %s", blendfile.name)
+        checksum = cache.compute_cached_checksum(blendfile)
+        filesize = blendfile.stat().st_size
+        
+        # Upload the blend file to Shaman
+        self.log.info("Uploading blend file to Shaman: %s", blendfile.name)
+        try:
+            with blendfile.open("rb") as file_reader:
+                shaman_api.shaman_file_store(
+                    checksum=checksum,
+                    filesize=filesize,
+                    body=file_reader,
+                    x_shaman_can_defer_upload=True,
+                    x_shaman_original_filename=blendfile.name,
+                )
+        except ApiException as ex:
+            if ex.status == 208:
+                # File already known to Shaman
+                self.log.info("Blend file already known to Shaman")
+            elif ex.status == 425:
+                # Defer upload - someone else is uploading
+                self.log.info("Blend file is being uploaded by another client, deferring")
+                # Retry after a short delay
+                import time
+                time.sleep(1)
+                with blendfile.open("rb") as file_reader:
+                    shaman_api.shaman_file_store(
+                        checksum=checksum,
+                        filesize=filesize,
+                        body=file_reader,
+                        x_shaman_can_defer_upload=False,
+                        x_shaman_original_filename=blendfile.name,
+                    )
+            else:
+                self.log.error("Error uploading to Shaman: %s", ex)
+                self.report({"ERROR"}, f"Error uploading to Shaman: {ex}")
+                return
+        
+        # Create checkout definition with just the blend file
+        checkout_path = self._shaman_checkout_path()
+        filespec = ShamanFileSpec(
+            sha=checksum,
+            size=filesize,
+            path=str(blendfile_path_in_checkout),  # Relative path from project root
+        )
+        
+        # Create the checkout
+        self.log.info("Creating Shaman checkout: %s", checkout_path)
+        self.log.info("Blend file path in checkout: %s", blendfile_path_in_checkout)
+        checkout = ShamanCheckout(
+            files=[filespec],
+            checkout_path=str(checkout_path),
+        )
+        
+        try:
+            result = shaman_api.shaman_checkout(checkout)
+            self.actual_shaman_checkout_path = PurePosixPath(result.checkout_path)
+            # The checkout itself is created in a unique subdirectory. The job's
+            # blendfile must include that checkout path.
+            self.blendfile_on_farm = (
+                PurePosixPath("{jobs}")
+                / self.actual_shaman_checkout_path
+                / blendfile_path_in_checkout
+            )
+            self.log.info("Shaman checkout created: %s", self.actual_shaman_checkout_path)
+            self._submit_job(context)
+        except ApiException as ex:
+            self.log.error("Error creating Shaman checkout: %s", ex)
+            self.report({"ERROR"}, f"Error creating Shaman checkout: {ex}")
+            return
+
+    def _copy_blendfile_to_storage(
+        self, context: bpy.types.Context, blendfile: Path
+    ) -> None:
+        """Copy blend file to job storage without BAT."""
+        import shutil
+        
+        manager = self._manager_info(context)
+        if not manager:
+            raise FileNotFoundError("Manager info not known")
+        
+        unique_dir = "%s-%s" % (
+            datetime.datetime.now().isoformat("-").replace(":", ""),
+            self.job_name,
+        )
+        pack_target_dir = Path(manager.shared_storage.location) / unique_dir
+        pack_target_dir.mkdir(parents=True, exist_ok=True)
+        
+        pack_target_file = pack_target_dir / blendfile.name
+        self.log.info("Copying blend file to %s", pack_target_file)
+        
+        shutil.copy2(blendfile, pack_target_file)
+        
+        self.blendfile_on_farm = PurePosixPath(pack_target_file.as_posix())
+        self.actual_shaman_checkout_path = None
+        
+        self._submit_job(context)
+
     def _bat_pack_filesystem(
         self, context: bpy.types.Context, blendfile: Path
     ) -> PurePosixPath:
@@ -414,9 +709,7 @@ class FLAMENCO_OT_submit_job(FlamencoOpMixin, bpy.types.Operator):
         """
         from .bat import interface as bat_interface
 
-        # Get project path from addon preferences.
-        prefs = preferences.get(context)
-        project_path: Path = prefs.project_root()
+        project_path = self._bat_project_path(context, blendfile)
         project_path = bpathlib.make_absolute(Path(bpy.path.abspath(str(project_path))))
 
         if not project_path.exists():
@@ -442,7 +735,9 @@ class FLAMENCO_OT_submit_job(FlamencoOpMixin, bpy.types.Operator):
             project=project_path,
             target=str(pack_target_dir),
             exclusion_filter="",  # TODO: get from GUI.
-            relative_only=True,  # TODO: get from GUI.
+            # False: relative_only=True can leave linked blends on KEEP_PATH and hit BAT
+            # _rewrite_paths assertions with Shaman (same as stock 3.8.x + BAT 1.x).
+            relative_only=False,
         )
 
         return PurePosixPath(pack_target_file.as_posix())
@@ -473,15 +768,15 @@ class FLAMENCO_OT_submit_job(FlamencoOpMixin, bpy.types.Operator):
         assert self.job is not None
         self.log.info("Sending BAT pack to Shaman")
 
-        prefs = preferences.get(context)
-        project_path: Path = prefs.project_root()
+        project_path = self._bat_project_path(context, blendfile)
 
         self.packthread = bat_interface.copy(
             base_blendfile=blendfile,
             project=project_path,
             target="/",  # Target directory irrelevant for Shaman transfers.
             exclusion_filter="",  # TODO: get from GUI.
-            relative_only=True,  # TODO: get from GUI.
+            # See _bat_pack_filesystem: avoid BAT+Shaman KEEP_PATH / _rewrite_paths failure.
+            relative_only=False,
             packer_class=bat_shaman.Packer,
             packer_kwargs=dict(
                 api_client=self.get_api_client(context),

scripts/addons/flamenco/preferences.py

@@ -65,6 +65,16 @@ class FlamencoPreferences(bpy.types.AddonPreferences):
         items=_project_finder_enum_items,
     )
 
+    bat_bypass: bpy.props.BoolProperty(  # type: ignore
+        name="Bypass BAT",
+        description=(
+            "When enabled, submission skips Blender Asset Tracer: paths in the blend "
+            "are written as absolute, then the blend is uploaded or copied without a BAT pack. "
+            "When disabled, Flamenco uses the normal BAT pack (Shaman or job storage)"
+        ),
+        default=True,
+    )
+
     # Property that gets its value from the above _job_storage, and cannot be
     # set. This makes it read-only in the GUI.
     job_storage_for_gui: bpy.props.StringProperty(  # type: ignore
@@ -118,6 +128,10 @@ class FlamencoPreferences(bpy.types.AddonPreferences):
         else:
             text_row(col, str(project_root))
 
+        col = layout.column(align=True)
+        col.label(text="Submission")
+        col.prop(self, "bat_bypass")
+
     def project_root(self) -> Path:
         """Use the configured project finder to find the project root directory."""