diff --git a/contrib/bash-completion/bob b/contrib/bash-completion/bob index 3a2309a0..79b79991 100644 --- a/contrib/bash-completion/bob +++ b/contrib/bash-completion/bob @@ -121,7 +121,7 @@ __bob_clean() __bob_cook() { - if [[ "$prev" = "--destination" ]] ; then + if [[ "$prev" = "--destination" || "$prev" == "--bundle" || "$prev" == "--unbundle" ]] ; then __bob_complete_dir "$cur" elif [[ "$prev" = "--download" ]] ; then __bob_complete_words "yes no deps forced forced-deps forced-fallback" @@ -130,7 +130,7 @@ __bob_cook() elif [[ "$prev" = "--always-checkout" ]] ; then COMPREPLY=( ) else - __bob_complete_path "--destination -j --jobs -k --keep-going -f --force -n --no-deps -p --with-provided --without-provided -A --no-audit --audit -b --build-only -B --checkout-only --normal --clean --incremental --always-checkout --resume -q --quiet -v --verbose --no-logfiles -D -c -e -E -M --upload --link-deps --no-link-deps --download --download-layer --shared --no-shared --install --no-install --sandbox --no-sandbox --clean-checkout --attic --no-attic" + __bob_complete_path "--destination -j --jobs -k --keep-going -f --force -n --no-deps -p --with-provided --without-provided -A --no-audit --audit -b --build-only -B --checkout-only --normal --clean --incremental --always-checkout --resume -q --quiet -v --verbose --no-logfiles -D -c -e -E -M --upload --link-deps --no-link-deps --download --download-layer --shared --no-shared --install --no-install --sandbox --no-sandbox --clean-checkout --attic --no-attic --bundle --bundle-exclude --unbundle" fi } diff --git a/doc/manpages/bob-build-dev.rst b/doc/manpages/bob-build-dev.rst index 1ecdf4da..7ade5909 100644 --- a/doc/manpages/bob-build-dev.rst +++ b/doc/manpages/bob-build-dev.rst @@ -35,6 +35,16 @@ Options This is the default unless the user changed it in ``default.yaml``. +``--bundle BUNDLE`` + Bundle all the sources needed to build the package. The bunlde is a tar-file + containing the sources and a overrides file. To use the bundle call bob + dev/build with ``-c`` pointing to the scmOverrides-file. In addition to this + the ``LOCAL_BUNDLE_BASE`` environment variable needs to be set to point to + the base-directoy where the bundle has been extracted. + +``--bundle-exclude RE`` + Do not add packages matching RE to the bundle. + ``--clean`` Do clean builds by clearing the build directory before executing the build commands. It will *not* clean all build results (e.g. like ``make clean``) diff --git a/doc/manpages/bob-build.rst b/doc/manpages/bob-build.rst index 10cb76a4..5d590cff 100644 --- a/doc/manpages/bob-build.rst +++ b/doc/manpages/bob-build.rst @@ -24,6 +24,7 @@ Synopsis [--shared | --no-shared] [--install | --no-install] [--sandbox | --no-sandbox] [--clean-checkout] [--attic | --no-attic] + [--bundle BUNDLE] [--bundle-exclude BUNDLE_EXCLUDE] PACKAGE [PACKAGE ...] diff --git a/doc/manpages/bob-dev.rst b/doc/manpages/bob-dev.rst index 24fb2e79..6ef96e9c 100644 --- a/doc/manpages/bob-dev.rst +++ b/doc/manpages/bob-dev.rst @@ -24,6 +24,7 @@ Synopsis [--shared | --no-shared] [--install | --no-install] [--sandbox | --no-sandbox] [--clean-checkout] [--attic | --no-attic] + [--bundle BUNDLE] [--bundle-exclude BUNDLE_EXCLUDE] PACKAGE [PACKAGE ...] diff --git a/pym/bob/builder.py b/pym/bob/builder.py index e7bd85b6..dfe6b236 100644 --- a/pym/bob/builder.py +++ b/pym/bob/builder.py @@ -6,6 +6,7 @@ from . import BOB_VERSION from .archive import DummyArchive from .audit import Audit +from .bundle import Bundler from .errors import BobError, BuildError, MultiBobError from .input import RecipeSet from .invoker import Invoker, InvocationMode @@ -407,6 +408,7 @@ def __init__(self, verbose, force, skipDeps, buildOnly, preserveEnv, self.__installSharedPackages = False self.__executor = None self.__attic = True + self.__bundler = None def setExecutor(self, executor): self.__executor = executor @@ -505,6 +507,10 @@ def setAuditMeta(self, keys): def setAtticEnable(self, enable): self.__attic = enable + def setBundle(self, dest, excludes): + if dest is not None: + self.__bundler = Bundler(dest, excludes) + def setShareHandler(self, handler): self.__share = handler @@ -618,6 +624,10 @@ def __workspaceLock(self, step): self.__workspaceLocks[path] = ret = asyncio.Lock() return ret + def bundle(self): + if self.__bundler: + self.__bundler.finalize() + async def _generateAudit(self, step, depth, resultHash, buildId, executed=True): auditPath = os.path.join(os.path.dirname(step.getWorkspacePath()), "audit.json.gz") if os.path.lexists(auditPath): removePath(auditPath) @@ -1237,7 +1247,10 @@ async def _cookCheckoutStep(self, checkoutStep, depth): oldCheckoutHash = datetime.datetime.now() BobState().setResultHash(prettySrcPath, oldCheckoutHash) - with stepExec(checkoutStep, "CHECKOUT", + action = "CHECKOUT" + if checkoutStep.getBundle() is not None: + action = "UNBUNDLE" + with stepExec(checkoutStep, action, "{} ({}) {}".format(prettySrcPath, checkoutReason, overridesString)) as a: await self._runShell(checkoutStep, "checkout", a) self.__statistic.checkouts += 1 @@ -1284,6 +1297,9 @@ async def _cookCheckoutStep(self, checkoutStep, depth): assert predicted, "Non-predicted incorrect Build-Id found!" self.__handleChangedBuildId(checkoutStep, checkoutHash) + if self.__bundler: + await self.__bundler.bundle(checkoutStep, self.__executor) + async def _cookBuildStep(self, buildStep, depth, buildBuildId): # Add the execution path of the build step to the buildDigest to # detect changes between sandbox and non-sandbox builds. This is diff --git a/pym/bob/bundle.py b/pym/bob/bundle.py new file mode 100644 index 00000000..557d4bc5 --- /dev/null +++ b/pym/bob/bundle.py @@ -0,0 +1,126 @@ +# Bob build tool +# Copyright (C) 2024 Secunet Security Networks AG +# +# SPDX-License-Identifier: GPL-3.0-or-later + +from .errors import BuildError +from .tty import stepExec, EXECUTED +from .utils import hashFile + +import asyncio +import concurrent.futures +import fnmatch +import gzip +import hashlib +import os +import schema +import signal +import tarfile +import tempfile +import yaml + +class Bundler: + def __init__(self, name, excludes): + self.__name = name + self.__bundleFile = os.path.join(os.getcwd(), self.__name) + ".tar" + self.__excludes = excludes + self.__tempDir = tempfile.TemporaryDirectory() + self.__tempDirPath = os.path.join(self.__tempDir.name, self.__name) + self.__bundled = {} + + if os.path.exists(self.__bundleFile): + raise BuildError(f"Bundle {self.__bundleFile} already exists!") + os.mkdir(self.__tempDirPath) + + def _bundle(self, workspace, bundleFile): + def reset(tarinfo): + tarinfo.uid = tarinfo.gid = 0 + tarinfo.uname = tarinfo.gname = "root" + tarinfo.mtime = 0 + return tarinfo + + # Set default signal handler so that KeyboardInterrupt is raised. + # Needed to gracefully handle ctrl+c. + signal.signal(signal.SIGINT, signal.default_int_handler) + + try: + files = [] + for root, dirs, filenames in os.walk(workspace): + for f in filenames: + files.append(os.path.join(root, f)) + files.sort() + with open(bundleFile, 'wb') as outfile: + with gzip.GzipFile(fileobj=outfile, mode='wb', mtime=0) as zipfile: + with tarfile.open(fileobj=zipfile, mode="w:") as bundle: + for f in files: + bundle.add(f, arcname=os.path.relpath(f, workspace), + recursive=False, filter=reset) + digest = hashFile(bundleFile, hashlib.sha256).hex() + + except (tarfile.TarError, OSError) as e: + raise BuildError("Cannot bundle workspace: " + str(e)) + finally: + # Restore signals to default so that Ctrl+C kills process. Needed + # to prevent ugly backtraces when user presses ctrl+c. + signal.signal(signal.SIGINT, signal.SIG_DFL) + + return ("ok", EXECUTED, digest) + + async def bundle(self, step, executor): + for e in self.__excludes: + if fnmatch.fnmatch(step.getPackage().getName(), e): return + + checkoutVariantId = step.getPackage().getCheckoutStep().getVariantId().hex() + dest = os.path.join(self.__tempDirPath, step.getPackage().getRecipe().getName(), + checkoutVariantId) + os.makedirs(dest) + bundleFile = os.path.join(dest, "bundle.tgz") + + loop = asyncio.get_event_loop() + with stepExec(step, "BUNDLE", "{}".format(step.getWorkspacePath())) as a: + try: + msg, kind, digest = await loop.run_in_executor(executor, Bundler._bundle, + self, step.getWorkspacePath(), bundleFile) + a.setResult(msg, kind) + except (concurrent.futures.CancelledError, concurrent.futures.process.BrokenProcessPool): + raise BuildError("Upload of bundling interrupted.") + + self.__bundled[checkoutVariantId] = (step.getPackage().getRecipe().getName(), digest, bundleFile) + + def finalize(self): + bundle = [] + with tarfile.open(self.__bundleFile, "w") as bundle_tar: + + for vid, (package, digest, bundleFile) in sorted(self.__bundled.items()): + bundle.append({vid : {"digestSHA256" : digest, + "name" : package}}) + print(f"add to bundle: {bundleFile}") + bundle_tar.add(bundleFile, + arcname=os.path.relpath(bundleFile, self.__tempDir.name)) + + bundleConfig = self.__name + ".yaml" + bundleConfigPath = os.path.join(self.__tempDirPath, bundleConfig) + with open(bundleConfigPath, "w") as f: + yaml.dump(bundle, f, default_flow_style=False) + bundle_tar.add(bundleConfigPath, arcname=os.path.join(self.__name, bundleConfig)) + +class Unbundler: + BUNDLE_SCHEMA = schema.Schema([{ + str : schema.Schema({ + "name" : str, + "digestSHA256" : str + }) + }]) + + def __init__(self, bundles): + self.__bundles = bundles + + def getFromBundle(self, variantId): + for bundleFile, items in self.__bundles.items(): + for b in items: + if variantId.hex() in b: + data = b.get(variantId.hex()) + return (bundleFile, os.path.join(os.path.dirname(bundleFile), data['name'], variantId.hex(), + "bundle.tgz"), data['digestSHA256']) + return None + diff --git a/pym/bob/cmds/build/build.py b/pym/bob/cmds/build/build.py index d590f723..33a5ba1d 100644 --- a/pym/bob/cmds/build/build.py +++ b/pym/bob/cmds/build/build.py @@ -214,6 +214,12 @@ def _downloadLayerArgument(arg): help="Move scm to attic if inline switch is not possible (default).") group.add_argument('--no-attic', action='store_false', default=None, dest='attic', help="Do not move to attic, instead fail the build.") + parser.add_argument('--bundle', metavar='BUNDLE', default=None, + help="Bundle all matching packages to BUNDLE") + parser.add_argument('--bundle-exclude', action='append', default=[], + help="Do not add matching packages to bundle.") + parser.add_argument('--unbundle', default=[], action='append', + help="Use sources from bundle") args = parser.parse_args(argv) defines = processDefines(args.defines) @@ -230,6 +236,7 @@ def _downloadLayerArgument(arg): if args.build_mode != 'build-only': setVerbosity(args.verbose) updateLayers(recipes, loop, defines, args.verbose, args.attic, args.layerConfig) + recipes.setBundleFiles(args.unbundle) recipes.parse(defines) # if arguments are not passed on cmdline use them from default.yaml or set to default yalue @@ -302,6 +309,9 @@ def _downloadLayerArgument(arg): packages = recipes.generatePackages(nameFormatter, args.sandbox) if develop: developPersister.prime(packages) + if args.bundle and args.build_mode == 'build-only': + parser.error("--bundle can't be used with --build-only") + verbosity = cfg.get('verbosity', 0) + args.verbose - args.quiet setVerbosity(verbosity) builder = LocalBuilder(verbosity, args.force, @@ -325,6 +335,7 @@ def _downloadLayerArgument(arg): builder.setShareHandler(getShare(recipes.getShareConfig())) builder.setShareMode(args.shared, args.install) builder.setAtticEnable(args.attic) + builder.setBundle(args.bundle, args.bundle_exclude) if args.resume: builder.loadBuildState() backlog = [] @@ -386,6 +397,8 @@ def _downloadLayerArgument(arg): + " package" + ("s" if (stats.packagesBuilt != 1) else "") + " built, " + str(stats.packagesDownloaded) + " downloaded.") + builder.bundle() + # Copy build result if requested. It's ok to overwrite files that are # already at the destination. Warn if built packages overwrite # themselves, though. diff --git a/pym/bob/input.py b/pym/bob/input.py index 440012fe..81690d24 100644 --- a/pym/bob/input.py +++ b/pym/bob/input.py @@ -4,6 +4,7 @@ # SPDX-License-Identifier: GPL-3.0-or-later from . import BOB_VERSION, BOB_INPUT_HASH, DEBUG +from .bundle import Unbundler from .errors import ParseError, BobError from .languages import getLanguage, ScriptLanguage, BashLanguage, PwshLanguage from .pathspec import PackageSet @@ -1232,12 +1233,12 @@ def toolDepWeak(self): class CoreCheckoutStep(CoreStep): - __slots__ = ( "scmList", "__checkoutUpdateIf", "__checkoutUpdateDeterministic", "__checkoutAsserts" ) + __slots__ = ( "scmList", "__checkoutUpdateIf", "__checkoutUpdateDeterministic", "__checkoutAsserts", "__bundle" ) def __init__(self, corePackage, checkout=None, checkoutSCMs=[], fullEnv=Env(), digestEnv=Env(), env=Env(), args=[], checkoutUpdateIf=[], checkoutUpdateDeterministic=True, - toolDep=set(), toolDepWeak=set()): + toolDep=set(), toolDepWeak=set(), bundle=False): if checkout: recipeSet = corePackage.recipe.getRecipeSet() overrides = recipeSet.scmOverrides() @@ -1277,6 +1278,7 @@ def __init__(self, corePackage, checkout=None, checkoutSCMs=[], self.__checkoutAsserts = [ CheckoutAssert (a, fullEnv) for a in corePackage.recipe.checkoutAsserts ] self.__checkoutUpdateIf = checkoutUpdateIf self.__checkoutUpdateDeterministic = checkoutUpdateDeterministic + self.__bundle = None deterministic = corePackage.recipe.checkoutDeterministic super().__init__(corePackage, isValid, deterministic, digestEnv, env, args, toolDep, toolDepWeak) @@ -1286,6 +1288,12 @@ def refDeref(self, stack, inputTools, inputSandbox, pathFormatter, cache=None): package._setCheckoutStep(ret) return ret + def setBundle(self, bundle): + self.__bundle = bundle + + def getBundle(self): + return self.__bundle + def getLabel(self): return "src" @@ -1308,10 +1316,10 @@ def getJenkinsPreRunCmds(self): return [ s.getProperties(True) for s in self.scmList if not s.hasJenkinsPlugin() ] def getSetupScript(self): - return self.corePackage.recipe.checkoutSetupScript + return self.corePackage.recipe.checkoutSetupScript if self.__bundle is None else "" def getMainScript(self): - return self.corePackage.recipe.checkoutMainScript + return self.corePackage.recipe.checkoutMainScript if self.__bundle is None else "" def getPostRunCmds(self): return [s.getProperties() for s in self.__checkoutAsserts] @@ -1358,6 +1366,8 @@ def hasLiveBuildId(self): def hasNetAccess(self): return True + def getBundle(self): + return self._coreStep.getBundle() class CoreBuildStep(CoreStep): __slots__ = ["fingerprintMask"] @@ -1512,6 +1522,9 @@ def getMetaEnv(self): def jobServer(self): return self.recipe.jobServer() + def setCoreCheckoutBundle(self, bundleScm): + self.checkoutStep.setBundle(bundleScm) + class Package(object): """Representation of a package that was created from a recipe. @@ -2590,6 +2603,16 @@ def prepare(self, inputEnv, sandboxEnabled, inputStates, inputSandbox=None, self.__checkoutSCMs, env, checkoutDigestEnv, checkoutEnv, checkoutDeps, checkoutUpdateIf, checkoutUpdateDeterministic, toolDepCheckout, toolDepCheckoutWeak) + + fromBundle = self.__recipeSet.getFromBundle(srcCoreStep.variantId) + if fromBundle is not None: + print(" -> from Bundle!") + bundleScm = [{'scm': 'url', + 'url': fromBundle[1], + 'recipe': fromBundle[0], + 'digestSHA256' : fromBundle[2] + }] + p.setCoreCheckoutBundle(bundleScm) else: srcCoreStep = p.createInvalidCoreCheckoutStep() @@ -3061,6 +3084,7 @@ def __init__(self): self.__hooks = {} self.__projectGenerators = {} self.__configFiles = [] + self.__bundleFiles = [] self.__properties = {} self.__states = {} self.__cache = YamlCache() @@ -3116,6 +3140,10 @@ def removeWhiteList(x): "archive" : BuiltinSetting(archiveValidator, updateArchive, True), "archiveAppend" : BuiltinSetting(archiveValidator, appendArchive, True, 100), "archivePrepend" : BuiltinSetting(archiveValidator, prependArchive, True, 100), + "bundles" : BuiltinSetting( + schema.Schema([str]), + lambda x: self.__bundleFiles.extend(x) + ), "command" : BuiltinSetting( schema.Schema({ schema.Optional('dev') : self.BUILD_DEV_SCHEMA, @@ -3206,6 +3234,7 @@ def removeWhiteList(x): priority=100 ), } + self.__unbundler = None def __addRecipe(self, recipe): name = recipe.getPackageName() @@ -3349,6 +3378,14 @@ def defineHook(self, name, value): def setConfigFiles(self, configFiles): self.__configFiles = configFiles + def getFromBundle(self, variantId): + if self.__unbundler is not None: + return self.__unbundler.getFromBundle(variantId) + return None + + def setBundleFiles(self, bundleFiles): + self.__bundleFiles = bundleFiles + def getCommandConfig(self): return self.__commandConfig @@ -3498,6 +3535,12 @@ def __parse(self, envOverrides, platform, recipesRoot="", noLayers=False): raise ParseError("Config file {} does not exist!".format(c)) self.__parseUserConfig(c) + if len(self.__bundleFiles) > 0: + bundledSources = {} + for b in self.__bundleFiles: + bundledSources[b] = self.loadYaml(b, (Unbundler.BUNDLE_SCHEMA, b'')) + self.__unbundler = Unbundler(bundledSources) + # calculate start environment osEnv = Env(os.environ) osEnv.setFuns(self.__stringFunctions) diff --git a/pym/bob/intermediate.py b/pym/bob/intermediate.py index 0b0d3236..206e0da7 100644 --- a/pym/bob/intermediate.py +++ b/pym/bob/intermediate.py @@ -88,6 +88,7 @@ def fromStep(cls, step, graph, partial=False): self.__data['hasNetAccess'] = step.hasNetAccess() if self.__data['isCheckoutStep']: self.__data['hasLiveBuildId'] = step.hasLiveBuildId() + self.__data['bundle'] = step.getBundle() self.__data['scmList'] = [ (s.getProperties(self.JENKINS), [ o.__getstate__() for o in s.getActiveOverrides()]) for s in step.getScmList() @@ -264,6 +265,9 @@ def getLabel(self): def isDeterministic(self): return self.__data['isDeterministic'] + def getBundle(self): + return self.__data['bundle'] + def isUpdateDeterministic(self): return self.__data['isUpdateDeterministic'] diff --git a/test/unit/test_input_recipe.py b/test/unit/test_input_recipe.py index b66d9b4e..5c631fb3 100644 --- a/test/unit/test_input_recipe.py +++ b/test/unit/test_input_recipe.py @@ -201,6 +201,7 @@ def parseAndPrepare(self, recipe, classes={}, name="foo", env={}): recipeSet.loadBinary = MagicMock() recipeSet.scriptLanguage = self.SCRIPT_LANGUAGE recipeSet.getPolicy = lambda x: None + recipeSet.getFromBundle = lambda x: None cc = { n : Recipe(recipeSet, self.applyRecipeDefaults(r), "", n+".yaml", cwd, n, n, {}, False)