From df375502125fbcf9f81448d8e81799621bff18be Mon Sep 17 00:00:00 2001 From: Ralf Hubert Date: Mon, 23 Sep 2024 08:59:44 +0200 Subject: [PATCH] urlscm: separate download directory --- doc/manual/policies.rst | 16 ++++++++++++++++ pym/bob/builder.py | 2 ++ pym/bob/input.py | 6 ++++++ pym/bob/intermediate.py | 1 + pym/bob/scm/__init__.py | 3 ++- pym/bob/scm/url.py | 24 +++++++++++++++++------- 6 files changed, 44 insertions(+), 8 deletions(-) diff --git a/doc/manual/policies.rst b/doc/manual/policies.rst index 69748578..44e2e474 100644 --- a/doc/manual/policies.rst +++ b/doc/manual/policies.rst @@ -626,3 +626,19 @@ New behavior passed to the ``fingerprintScript``. Other environment variables are unset but whitelisted variables (see :ref:`configuration-config-whitelist`) are still available. + +scmOnlyExtracted +~~~~~~~~~~~~~~~~ + +Introduced in: 0.25 + +This policy controls if the downloaded file of a UrlScm is stored in the source +workspace when extractors are available for it. + +Old behavior + Downloaded original and `.extracted` canary file are part of the source + workspace. + +New behavior + If a extractor is available the downloaded file and the canary are stored in + workspace/../_downloaded. diff --git a/pym/bob/builder.py b/pym/bob/builder.py index f7e10b9a..6c4caea5 100644 --- a/pym/bob/builder.py +++ b/pym/bob/builder.py @@ -1216,6 +1216,8 @@ async def _cookCheckoutStep(self, checkoutStep, depth): os.rename(scmPath, atticPath) BobState().setAtticDirectoryState(atticPath, scmSpec) atticPaths.add(scmPath, atticPath) + downloadPath = os.path.normpath(os.path.join(prettySrcPath, "..", "_download", scmDir)) + shutil.rmtree(downloadPath, ignore_errors=True) del oldCheckoutState[scmDir] BobState().setDirectoryState(prettySrcPath, oldCheckoutState) diff --git a/pym/bob/input.py b/pym/bob/input.py index 10b09aad..0b7473d1 100644 --- a/pym/bob/input.py +++ b/pym/bob/input.py @@ -2959,6 +2959,7 @@ class RecipeSet: schema.Optional('gitCommitOnBranch') : bool, schema.Optional('fixImportScmVariant') : bool, schema.Optional('defaultFileMode') : bool, + schema.Optional('scmOnlyExtracted') : bool, }, error="Invalid policy specified! Are you using an appropriate version of Bob?" ), @@ -3046,6 +3047,11 @@ def __init__(self): InfoOnce("defaultFileMode policy not set. File mode of URL SCMs not set for locally copied files.", help="See http://bob-build-tool.readthedocs.io/en/latest/manual/policies.html#defaultfilemode for more information.") ), + "scmOnlyExtracted": ( + "0.24.1.dev105", + InfoOnce("scmOnlyExtracted policy is not set. Downloaded original is part of the source workspace", + help="See http://bob-build-tool.readthedocs.io/en/latest/manual/policies.html#scmOnlyExtracted for more information.") + ), } self.__buildHooks = {} self.__sandboxOpts = {} diff --git a/pym/bob/intermediate.py b/pym/bob/intermediate.py index 206e0da7..230226aa 100644 --- a/pym/bob/intermediate.py +++ b/pym/bob/intermediate.py @@ -549,6 +549,7 @@ def fromRecipeSet(cls, recipeSet): 'gitCommitOnBranch' : recipeSet.getPolicy('gitCommitOnBranch'), 'fixImportScmVariant' : recipeSet.getPolicy('fixImportScmVariant'), 'defaultFileMode' : recipeSet.getPolicy('defaultFileMode'), + 'scmOnlyExtracted' : recipeSet.getPolicy('scmOnlyExtracted'), } self.__data['archiveSpec'] = recipeSet.archiveSpec() self.__data['envWhiteList'] = sorted(recipeSet.envWhiteList()) diff --git a/pym/bob/scm/__init__.py b/pym/bob/scm/__init__.py index 0788a6f3..1266afb4 100644 --- a/pym/bob/scm/__init__.py +++ b/pym/bob/scm/__init__.py @@ -62,6 +62,7 @@ def getScm(spec, overrides=[], recipeSet=None): recipeSet and recipeSet.getPolicy('scmIgnoreUser'), recipeSet.getPreMirrors() if recipeSet else [], recipeSet.getFallbackMirrors() if recipeSet else [], - recipeSet and recipeSet.getPolicy('defaultFileMode')) + recipeSet and recipeSet.getPolicy('defaultFileMode'), + recipeSet and recipeSet.getPolicy('scmOnlyExtracted')) else: raise ParseError("Unknown SCM '{}'".format(scm)) diff --git a/pym/bob/scm/url.py b/pym/bob/scm/url.py index e155242a..14c2067b 100644 --- a/pym/bob/scm/url.py +++ b/pym/bob/scm/url.py @@ -223,7 +223,8 @@ class UrlScm(Scm): } def __init__(self, spec, overrides=[], stripUser=None, - preMirrors=[], fallbackMirrors=[], defaultFileMode=None): + preMirrors=[], fallbackMirrors=[], defaultFileMode=None, + onlyExtracted=True): super().__init__(spec, overrides) self.__url = spec["url"] self.__digestSha1 = spec.get("digestSHA1") @@ -262,6 +263,7 @@ def __init__(self, spec, overrides=[], stripUser=None, self.__fallbackMirrorsUrls = spec.get("fallbackMirrors") self.__fallbackMirrorsUpload = spec.get("__fallbackMirrorsUpload") self.__fileMode = spec.get("fileMode", 0o600 if defaultFileMode else None) + self.__onlyExtracted = spec.get("onlyExtracted", onlyExtracted) def getProperties(self, isJenkins, pretty=False): ret = super().getProperties(isJenkins) @@ -282,6 +284,7 @@ def getProperties(self, isJenkins, pretty=False): 'fallbackMirrors' : self.__getFallbackMirrorsUrls(), '__fallbackMirrorsUpload' : self.__getFallbackMirrorsUpload(), 'fileMode' : dumpMode(self.__fileMode) if pretty else self.__fileMode, + 'onlyExtracted' : self.__onlyExtracted, }) return ret @@ -536,9 +539,14 @@ async def switch(self, invoker, oldScm): return True async def invoke(self, invoker): - os.makedirs(invoker.joinPath(self.__dir), exist_ok=True) - workspaceFile = os.path.join(self.__dir, self.__fn) - destination = invoker.joinPath(self.__dir, self.__fn) + extractors = self.__getExtractors() + downloadDestination = "" + if extractors and self.__onlyExtracted: + downloadDestination = os.path.join("..", "_download", self.__dir) + + os.makedirs(invoker.joinPath(self.__dir, downloadDestination), exist_ok=True) + destination = invoker.joinPath(self.__dir, downloadDestination, self.__fn) + workspaceFile = os.path.join(self.__dir, downloadDestination, self.__fn) # Download only if necessary if not self.isDeterministic() or not os.path.isfile(destination): @@ -587,8 +595,7 @@ async def invoke(self, invoker): await self._put(invoker, workspaceFile, destination, url) # Run optional extractors - extractors = self.__getExtractors() - canary = invoker.joinPath(self.__dir, "." + self.__fn + ".extracted") + canary = invoker.joinPath(self.__dir, downloadDestination, "." + self.__fn + ".extracted") if extractors and isYounger(destination, canary): for cmd in extractors: if shutil.which(cmd[0]) is None: continue @@ -672,7 +679,10 @@ def __getExtractors(self): strip = [extractor[3].format(self.__strip)] else: strip = [] - ret.append([extractor[1]] + [a.format(self.__fn) for a in extractor[2]] + strip) + fileToExtract = self.__fn + if self.__onlyExtracted: + fileToExtract = os.path.join("..", "_download", self.__dir, self.__fn) + ret.append([extractor[1]] + [a.format(fileToExtract) for a in extractor[2]] + strip) if not ret: raise BuildError("Extractor does not support 'stripComponents'!")