WIP: bundle

contrib/bash-completion/bob

@@ -121,7 +121,7 @@ __bob_clean()
 
 __bob_cook()
 {
-   if [[ "$prev" = "--destination" ]] ; then
+   if [[ "$prev" = "--destination" || "$prev" == "--bundle" || "$prev" == "--unbundle" ]] ; then
       __bob_complete_dir "$cur"
    elif [[ "$prev" = "--download" ]] ; then
          __bob_complete_words "yes no deps forced forced-deps forced-fallback"
@@ -130,7 +130,7 @@ __bob_cook()
    elif [[ "$prev" = "--always-checkout" ]] ; then
       COMPREPLY=( )
    else
-      __bob_complete_path "--destination -j --jobs -k --keep-going -f --force -n --no-deps -p --with-provided --without-provided -A --no-audit --audit -b --build-only -B --checkout-only --normal --clean --incremental --always-checkout --resume -q --quiet -v --verbose --no-logfiles -D -c -e -E -M --upload --link-deps --no-link-deps --download --download-layer --shared --no-shared --install --no-install --sandbox --no-sandbox --clean-checkout --attic --no-attic"
+      __bob_complete_path "--destination -j --jobs -k --keep-going -f --force -n --no-deps -p --with-provided --without-provided -A --no-audit --audit -b --build-only -B --checkout-only --normal --clean --incremental --always-checkout --resume -q --quiet -v --verbose --no-logfiles -D -c -e -E -M --upload --link-deps --no-link-deps --download --download-layer --shared --no-shared --install --no-install --sandbox --no-sandbox --clean-checkout --attic --no-attic --bundle --bundle-exclude --unbundle"
    fi
 }
 

doc/manpages/bob-build-dev.rst

@@ -35,6 +35,16 @@ Options
 
     This is the default unless the user changed it in ``default.yaml``.
 
+``--bundle BUNDLE``
+    Bundle all the sources needed to build the package. The bunlde is a tar-file
+    containing the sources and a overrides file. To use the bundle call bob
+    dev/build with ``-c`` pointing to the scmOverrides-file. In addition to this
+    the ``LOCAL_BUNDLE_BASE`` environment variable needs to be set to point to
+    the base-directoy where the bundle has been extracted.
+
+``--bundle-exclude RE``
+    Do not add packages matching  RE to the bundle.
+
 ``--clean``
     Do clean builds by clearing the build directory before executing the build
     commands. It will *not* clean all build results (e.g. like ``make clean``)

doc/manpages/bob-build.rst

@@ -24,6 +24,7 @@ Synopsis
               [--shared | --no-shared] [--install | --no-install]
               [--sandbox | --no-sandbox] [--clean-checkout]
               [--attic | --no-attic]
+              [--bundle BUNDLE] [--bundle-exclude BUNDLE_EXCLUDE]
               PACKAGE [PACKAGE ...]
 
 

doc/manpages/bob-dev.rst

@@ -24,6 +24,7 @@ Synopsis
             [--shared | --no-shared] [--install | --no-install]
             [--sandbox | --no-sandbox] [--clean-checkout]
             [--attic | --no-attic]
+            [--bundle BUNDLE] [--bundle-exclude BUNDLE_EXCLUDE]
             PACKAGE [PACKAGE ...]
 
 

pym/bob/builder.py

@@ -6,6 +6,7 @@
 from . import BOB_VERSION
 from .archive import DummyArchive
 from .audit import Audit
+from .bundle import Bundler
 from .errors import BobError, BuildError, MultiBobError
 from .input import RecipeSet
 from .invoker import Invoker, InvocationMode
@@ -407,6 +408,7 @@
         self.__installSharedPackages = False
         self.__executor = None
         self.__attic = True
+        self.__bundler = None
 
     def setExecutor(self, executor):
         self.__executor = executor
@@ -505,6 +507,10 @@
     def setAtticEnable(self, enable):
         self.__attic = enable
 
+    def setBundle(self, dest, excludes):
+        if dest is not None:
+            self.__bundler = Bundler(dest, excludes)
+
     def setShareHandler(self, handler):
         self.__share = handler
 
@@ -618,6 +624,10 @@
             self.__workspaceLocks[path] = ret = asyncio.Lock()
         return ret
 
+    def bundle(self):
+        if self.__bundler:
+            self.__bundler.finalize()
+
     async def _generateAudit(self, step, depth, resultHash, buildId, executed=True):
         auditPath = os.path.join(os.path.dirname(step.getWorkspacePath()), "audit.json.gz")
         if os.path.lexists(auditPath): removePath(auditPath)
@@ -1237,7 +1247,10 @@
                     oldCheckoutHash = datetime.datetime.now()
                     BobState().setResultHash(prettySrcPath, oldCheckoutHash)
 
-                with stepExec(checkoutStep, "CHECKOUT",
+                action = "CHECKOUT"
+                if checkoutStep.getBundle() is not None:
+                    action = "UNBUNDLE"
+                with stepExec(checkoutStep, action,
                               "{} ({}) {}".format(prettySrcPath, checkoutReason, overridesString)) as a:
                     await self._runShell(checkoutStep, "checkout", a)
                 self.__statistic.checkouts += 1
@@ -1284,6 +1297,9 @@
             assert predicted, "Non-predicted incorrect Build-Id found!"
             self.__handleChangedBuildId(checkoutStep, checkoutHash)
 
+        if self.__bundler:
+            await self.__bundler.bundle(checkoutStep, self.__executor)
+
     async def _cookBuildStep(self, buildStep, depth, buildBuildId):
         # Add the execution path of the build step to the buildDigest to
         # detect changes between sandbox and non-sandbox builds. This is

pym/bob/bundle.py

@@ -0,0 +1,126 @@
+# Bob build tool
+# Copyright (C) 2024 Secunet Security Networks AG
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from .errors import BuildError
+from .tty import stepExec, EXECUTED
+from .utils import hashFile
+
+import asyncio
+import concurrent.futures
+import fnmatch
+import gzip
+import hashlib
+import os
+import schema
+import signal
+import tarfile
+import tempfile
+import yaml
+
+class Bundler:
+    def __init__(self, name, excludes):
+        self.__name = name
+        self.__bundleFile = os.path.join(os.getcwd(), self.__name) + ".tar"
+        self.__excludes = excludes
+        self.__tempDir = tempfile.TemporaryDirectory()
+        self.__tempDirPath = os.path.join(self.__tempDir.name, self.__name)
+        self.__bundled = {}
+
+        if os.path.exists(self.__bundleFile):
+            raise BuildError(f"Bundle {self.__bundleFile} already exists!")
+        os.mkdir(self.__tempDirPath)
+
+    def _bundle(self, workspace, bundleFile):
+        def reset(tarinfo):
+            tarinfo.uid = tarinfo.gid = 0
+            tarinfo.uname = tarinfo.gname = "root"
+            tarinfo.mtime = 0
+            return tarinfo
+
+        # Set default signal handler so that KeyboardInterrupt is raised.
+        # Needed to gracefully handle ctrl+c.
+        signal.signal(signal.SIGINT, signal.default_int_handler)
+
+        try:
+            files = []
+            for root, dirs, filenames in os.walk(workspace):
+                for f in filenames:
+                    files.append(os.path.join(root, f))
+            files.sort()
+            with open(bundleFile, 'wb') as outfile:
+                with gzip.GzipFile(fileobj=outfile, mode='wb', mtime=0) as zipfile:
+                    with tarfile.open(fileobj=zipfile, mode="w:") as bundle:
+                        for f in files:
+                            bundle.add(f, arcname=os.path.relpath(f, workspace),
+                                       recursive=False, filter=reset)
+            digest = hashFile(bundleFile, hashlib.sha256).hex()
+
+        except (tarfile.TarError, OSError) as e:
+            raise BuildError("Cannot bundle workspace: " + str(e))
+        finally:
+            # Restore signals to default so that Ctrl+C kills process. Needed
+            # to prevent ugly backtraces when user presses ctrl+c.
+            signal.signal(signal.SIGINT, signal.SIG_DFL)
+
+        return ("ok", EXECUTED, digest)
+
+    async def bundle(self, step, executor):
+        for e in self.__excludes:
+            if fnmatch.fnmatch(step.getPackage().getName(), e): return
+
+        checkoutVariantId = step.getPackage().getCheckoutStep().getVariantId().hex()
+        dest = os.path.join(self.__tempDirPath, step.getPackage().getRecipe().getName(),
+                            checkoutVariantId)
+        os.makedirs(dest)
+        bundleFile = os.path.join(dest, "bundle.tgz")
+
+        loop = asyncio.get_event_loop()
+        with stepExec(step, "BUNDLE", "{}".format(step.getWorkspacePath())) as a:
+            try:
+                msg, kind, digest = await loop.run_in_executor(executor, Bundler._bundle,
+                    self, step.getWorkspacePath(), bundleFile)
+                a.setResult(msg, kind)
+            except (concurrent.futures.CancelledError, concurrent.futures.process.BrokenProcessPool):
+                raise BuildError("Upload of bundling interrupted.")
+
+        self.__bundled[checkoutVariantId] = (step.getPackage().getRecipe().getName(), digest, bundleFile)
+
+    def finalize(self):
+        bundle = []
+        with tarfile.open(self.__bundleFile, "w") as bundle_tar:
+
+            for vid, (package, digest, bundleFile) in sorted(self.__bundled.items()):
+                bundle.append({vid : {"digestSHA256" : digest,
+                                "name" : package}})
+                print(f"add to bundle: {bundleFile}")
+                bundle_tar.add(bundleFile,
+                              arcname=os.path.relpath(bundleFile, self.__tempDir.name))
+
+            bundleConfig = self.__name + ".yaml"
+            bundleConfigPath = os.path.join(self.__tempDirPath, bundleConfig)
+            with open(bundleConfigPath, "w") as f:
+                yaml.dump(bundle, f, default_flow_style=False)
+            bundle_tar.add(bundleConfigPath, arcname=os.path.join(self.__name, bundleConfig))
+
+class Unbundler:
+    BUNDLE_SCHEMA = schema.Schema([{
+      str : schema.Schema({
+                "name" : str,
+                "digestSHA256" : str
+            })
+    }])
+
+    def __init__(self, bundles):
+        self.__bundles = bundles
+
+    def getFromBundle(self, variantId):
+        for bundleFile, items in self.__bundles.items():
+            for b in items:
+                if variantId.hex() in b:
+                    data = b.get(variantId.hex())
+                    return (bundleFile, os.path.join(os.path.dirname(bundleFile), data['name'], variantId.hex(),
+                                         "bundle.tgz"), data['digestSHA256'])
+        return None
+

pym/bob/cmds/build/build.py

@@ -214,6 +214,12 @@
         help="Move scm to attic if inline switch is not possible (default).")
     group.add_argument('--no-attic', action='store_false', default=None, dest='attic',
         help="Do not move to attic, instead fail the build.")
+    parser.add_argument('--bundle', metavar='BUNDLE', default=None,
+        help="Bundle all matching packages to BUNDLE")
+    parser.add_argument('--bundle-exclude', action='append', default=[],
+        help="Do not add matching packages to bundle.")
+    parser.add_argument('--unbundle', default=[], action='append',
+        help="Use sources from bundle")
     args = parser.parse_args(argv)
 
     defines = processDefines(args.defines)
@@ -230,6 +236,7 @@
         if args.build_mode != 'build-only':
             setVerbosity(args.verbose)
             updateLayers(recipes, loop, defines, args.verbose, args.attic, args.layerConfig)
+        recipes.setBundleFiles(args.unbundle)
         recipes.parse(defines)
 
         # if arguments are not passed on cmdline use them from default.yaml or set to default yalue
@@ -302,6 +309,9 @@
         packages = recipes.generatePackages(nameFormatter, args.sandbox)
         if develop: developPersister.prime(packages)
 
+        if args.bundle and args.build_mode == 'build-only':
+            parser.error("--bundle can't be used with --build-only")
+
         verbosity = cfg.get('verbosity', 0) + args.verbose - args.quiet
         setVerbosity(verbosity)
         builder = LocalBuilder(verbosity, args.force,
@@ -325,6 +335,7 @@
         builder.setShareHandler(getShare(recipes.getShareConfig()))
         builder.setShareMode(args.shared, args.install)
         builder.setAtticEnable(args.attic)
+        builder.setBundle(args.bundle, args.bundle_exclude)
         if args.resume: builder.loadBuildState()
 
         backlog = []
@@ -386,6 +397,8 @@
                     + " package" + ("s" if (stats.packagesBuilt != 1) else "") + " built, "
                 + str(stats.packagesDownloaded) + " downloaded.")
 
+        builder.bundle()
+
         # Copy build result if requested. It's ok to overwrite files that are
         # already at the destination. Warn if built packages overwrite
         # themselves, though.