diff --git a/atomic_reactor/constants.py b/atomic_reactor/constants.py index 8f3510579..40897173d 100644 --- a/atomic_reactor/constants.py +++ b/atomic_reactor/constants.py @@ -193,6 +193,8 @@ # Name of downloaded remote sources tarball REMOTE_SOURCE_TARBALL_FILENAME = 'remote-source.tar.gz' REMOTE_SOURCE_JSON_FILENAME = 'remote-source.json' +REMOTE_SOURCE_JSON_CONFIG_FILENAME = 'remote-source.config.json' +REMOTE_SOURCE_JSON_ENV_FILENAME = 'remote-source.env.json' ICM_JSON_FILENAME = 'icm-{}.json' # koji osbs_build metadata diff --git a/atomic_reactor/plugins/koji_import.py b/atomic_reactor/plugins/koji_import.py index 512cb1fd8..9dd190d95 100644 --- a/atomic_reactor/plugins/koji_import.py +++ b/atomic_reactor/plugins/koji_import.py @@ -322,6 +322,8 @@ def set_remote_sources_metadata(self, extra): "archives": [ remote_source["remote_source_json"]["filename"], remote_source["remote_source_tarball"]["filename"], + remote_source["remote_source_json_env"]["filename"], + remote_source["remote_source_json_config"]["filename"], ], } for remote_source in remote_source_result @@ -622,15 +624,18 @@ def _collect_remote_sources(self) -> Iterable[ArtifactOutputInfo]: dest_filename = remote_source_tarball['filename'] yield local_filename, dest_filename, KOJI_BTYPE_REMOTE_SOURCES, None - remote_source_json = remote_source['remote_source_json'] - remote_source_json_filename = remote_source_json['filename'] - file_path = os.path.join(tmpdir, remote_source_json_filename) - with open(file_path, 'w') as f: - json.dump(remote_source_json['json'], f, indent=4, sort_keys=True) - yield (file_path, - remote_source_json_filename, - KOJI_BTYPE_REMOTE_SOURCES, - None) + for source_key in ( + "remote_source_json", "remote_source_json_env", "remote_source_json_config", + ): + data_json = remote_source[source_key] + data_json_filename = data_json['filename'] + file_path = os.path.join(tmpdir, data_json_filename) + with open(file_path, 'w') as f: + json.dump(data_json['json'], f, indent=4, sort_keys=True) + yield (file_path, + data_json_filename, + KOJI_BTYPE_REMOTE_SOURCES, + None) def _collect_exported_operator_manifests(self) -> Iterable[ArtifactOutputInfo]: wf_data = self.workflow.data diff --git a/atomic_reactor/plugins/resolve_remote_source.py b/atomic_reactor/plugins/resolve_remote_source.py index dad5b62d3..a7df9de7a 100644 --- a/atomic_reactor/plugins/resolve_remote_source.py +++ b/atomic_reactor/plugins/resolve_remote_source.py @@ -23,6 +23,8 @@ REMOTE_SOURCE_DIR, REMOTE_SOURCE_JSON_FILENAME, REMOTE_SOURCE_TARBALL_FILENAME, + REMOTE_SOURCE_JSON_CONFIG_FILENAME, + REMOTE_SOURCE_JSON_ENV_FILENAME, ) from atomic_reactor.dirs import BuildDir from atomic_reactor.plugin import Plugin @@ -48,7 +50,8 @@ class RemoteSource: id: int name: Optional[str] json_data: dict - build_args: Dict[str, str] + json_env_data: Dict[str, Dict[str, str]] + json_config_data: List[Dict[str, str]] tarball_path: Path @classmethod @@ -65,6 +68,38 @@ def json_filename(cls, name: Optional[str]): else: return REMOTE_SOURCE_JSON_FILENAME + @classmethod + def json_config_filename(cls, name: Optional[str]): + if name: + return f"remote-source-{name}.config.json" + else: + return REMOTE_SOURCE_JSON_CONFIG_FILENAME + + @classmethod + def json_env_filename(cls, name: Optional[str]): + if name: + return f"remote-source-{name}.env.json" + else: + return REMOTE_SOURCE_JSON_ENV_FILENAME + + @property + def build_args(self) -> Dict[str, str]: + build_args = {} + + for env_var, value_info in self.json_env_data.items(): + build_arg_value = value_info['value'] + kind = value_info['kind'] + if kind == 'path': + name = self.name or '' + build_arg_value = os.path.join(REMOTE_SOURCE_DIR, name, value_info['value']) + build_args[env_var] = build_arg_value + elif kind == 'literal': + build_args[env_var] = build_arg_value + else: + raise RuntimeError(f'Unknown kind {kind} got from Cachito.') + + return build_args + class ResolveRemoteSourcePlugin(Plugin): """Initiate a new Cachito request for sources @@ -219,42 +254,13 @@ def inject_into_build_dir( with tarfile.open(remote_source.tarball_path) as tar: safe_extractall(tar, str(dest_dir)) - config_files = self.cachito_session.get_request_config_files(remote_source.id) - self.generate_cachito_config_files(dest_dir, config_files) + self.generate_cachito_config_files(dest_dir, remote_source.json_config_data) # Create cachito.env file with environment variables received from cachito request self.generate_cachito_env_file(dest_dir, remote_source.build_args) return created_dirs - def get_buildargs(self, request_id: int, remote_source_name: Optional[str]) -> Dict[str, str]: - build_args = {} - env_vars = self.cachito_session.get_request_env_vars(request_id) - - for env_var, value_info in env_vars.items(): - build_arg_value = value_info['value'] - kind = value_info['kind'] - if kind == 'path': - name = remote_source_name or '' - build_arg_value = os.path.join(REMOTE_SOURCE_DIR, name, value_info['value']) - self.log.debug( - 'Setting the Cachito environment variable "%s" to the absolute path "%s"', - env_var, - build_arg_value, - ) - build_args[env_var] = build_arg_value - elif kind == 'literal': - self.log.debug( - 'Setting the Cachito environment variable "%s" to a literal value "%s"', - env_var, - build_arg_value, - ) - build_args[env_var] = build_arg_value - else: - raise RuntimeError(f'Unknown kind {kind} got from Cachito.') - - return build_args - def source_request_to_json(self, source_request): """Create a relevant representation of the source request""" required = ('packages', 'ref', 'repo') @@ -309,13 +315,14 @@ def process_request(self, source_request: dict, name: Optional[str]) -> RemoteSo dest_filename=tarball_filename, ) - build_args = self.get_buildargs(source_request["id"], name) + env_vars = self.cachito_session.get_request_env_vars(source_request["id"]) remote_source = RemoteSource( id=source_request["id"], name=name, json_data=self.source_request_to_json(source_request), - build_args=build_args, + json_env_data=env_vars, + json_config_data=self.cachito_session.get_request_config_files(source_request["id"]), tarball_path=Path(tarball_dest_path), ) return remote_source @@ -323,7 +330,6 @@ def process_request(self, source_request: dict, name: Optional[str]) -> RemoteSo def remote_source_to_output(self, remote_source: RemoteSource) -> Dict[str, Any]: """Convert a processed remote source to a dict to be used as output of this plugin.""" download_url = self.cachito_session.assemble_download_url(remote_source.id) - json_filename = RemoteSource.json_filename(remote_source.name) return { "id": remote_source.id, @@ -331,7 +337,15 @@ def remote_source_to_output(self, remote_source: RemoteSource) -> Dict[str, Any] "url": download_url, "remote_source_json": { "json": remote_source.json_data, - "filename": json_filename, + "filename": RemoteSource.json_filename(remote_source.name), + }, + "remote_source_json_env": { + "json": remote_source.json_env_data, + "filename": RemoteSource.json_env_filename(remote_source.name), + }, + "remote_source_json_config": { + "json": remote_source.json_config_data, + "filename": RemoteSource.json_config_filename(remote_source.name), }, "remote_source_tarball": { "filename": remote_source.tarball_path.name, diff --git a/tests/plugins/test_koji_import.py b/tests/plugins/test_koji_import.py index b9f571e17..c5cb5f00c 100644 --- a/tests/plugins/test_koji_import.py +++ b/tests/plugins/test_koji_import.py @@ -49,6 +49,8 @@ PARENT_IMAGES_KEY, OPERATOR_MANIFESTS_ARCHIVE, REMOTE_SOURCE_TARBALL_FILENAME, REMOTE_SOURCE_JSON_FILENAME, + REMOTE_SOURCE_JSON_CONFIG_FILENAME, + REMOTE_SOURCE_JSON_ENV_FILENAME, MEDIA_TYPE_DOCKER_V2_SCHEMA2, MEDIA_TYPE_DOCKER_V2_MANIFEST_LIST, KOJI_BTYPE_REMOTE_SOURCE_FILE, @@ -533,6 +535,14 @@ def custom_get(method, url, headers, **kwargs): "filename": REMOTE_SOURCE_JSON_FILENAME, "json": {"stub": "data"}, }, + "remote_source_json_config": { + "filename": REMOTE_SOURCE_JSON_CONFIG_FILENAME, + "json": [{"stub": "data"}], + }, + "remote_source_json_env": { + "filename": REMOTE_SOURCE_JSON_ENV_FILENAME, + "json": {"var": {"stub": "data"}}, + }, "remote_source_tarball": { "filename": REMOTE_SOURCE_TARBALL_FILENAME, "path": str(source_path), @@ -2003,7 +2013,10 @@ def test_remote_sources(self, workflow, source_dir, { 'name': None, 'url': 'https://cachito.com/api/v1/requests/21048', - 'archives': ['remote-source.json', 'remote-source.tar.gz'], + 'archives': [ + 'remote-source.json', 'remote-source.tar.gz', + 'remote-source.env.json', 'remote-source.config.json' + ], } ] assert REMOTE_SOURCE_TARBALL_FILENAME in session.uploaded_files.keys() diff --git a/tests/plugins/test_resolve_remote_source.py b/tests/plugins/test_resolve_remote_source.py index 9ece1d725..5dc5746b7 100644 --- a/tests/plugins/test_resolve_remote_source.py +++ b/tests/plugins/test_resolve_remote_source.py @@ -29,6 +29,8 @@ REMOTE_SOURCE_DIR, REMOTE_SOURCE_TARBALL_FILENAME, REMOTE_SOURCE_JSON_FILENAME, + REMOTE_SOURCE_JSON_CONFIG_FILENAME, + REMOTE_SOURCE_JSON_ENV_FILENAME, ) from atomic_reactor.plugin import PluginFailedException from atomic_reactor.plugins.resolve_remote_source import ( @@ -391,6 +393,14 @@ def mock_cachito_api_multiple_remote_sources(workflow, user=KOJI_TASK_OWNER): .ordered() ) + ( + flexmock(CachitoAPI) + .should_receive("get_request_config_files") + .with_args(CACHITO_SOURCE_REQUEST["id"]) + .and_return(CACHITO_CONFIG_FILES) + .ordered() + ) + ( flexmock(CachitoAPI) .should_receive("download_sources") @@ -411,14 +421,6 @@ def mock_cachito_api_multiple_remote_sources(workflow, user=KOJI_TASK_OWNER): .ordered() ) - ( - flexmock(CachitoAPI) - .should_receive("get_request_config_files") - .with_args(CACHITO_SOURCE_REQUEST["id"]) - .and_return(CACHITO_CONFIG_FILES) - .ordered() - ) - ( flexmock(CachitoAPI) .should_receive("get_request_config_files") @@ -564,6 +566,14 @@ def test_resolve_remote_source(workflow, scratch, dr_strs, dependency_replacemen "json": REMOTE_SOURCE_JSON, "filename": REMOTE_SOURCE_JSON_FILENAME, }, + "remote_source_json_config": { + "json": CACHITO_CONFIG_FILES, + "filename": REMOTE_SOURCE_JSON_CONFIG_FILENAME, + }, + "remote_source_json_env": { + "json": CACHITO_ENV_VARS_JSON, + "filename": REMOTE_SOURCE_JSON_ENV_FILENAME, + }, "remote_source_tarball": { "filename": REMOTE_SOURCE_TARBALL_FILENAME, "path": expected_dowload_path(workflow), @@ -653,6 +663,14 @@ def test_no_koji_user(workflow, caplog): "json": REMOTE_SOURCE_JSON, "filename": REMOTE_SOURCE_JSON_FILENAME, }, + "remote_source_json_config": { + "json": CACHITO_CONFIG_FILES, + "filename": REMOTE_SOURCE_JSON_CONFIG_FILENAME, + }, + "remote_source_json_env": { + "json": CACHITO_ENV_VARS_JSON, + "filename": REMOTE_SOURCE_JSON_ENV_FILENAME, + }, "remote_source_tarball": { "filename": REMOTE_SOURCE_TARBALL_FILENAME, "path": expected_dowload_path(workflow), @@ -732,6 +750,14 @@ def test_bad_build_metadata(workflow, task_id, log_entry, caplog): "json": REMOTE_SOURCE_JSON, "filename": REMOTE_SOURCE_JSON_FILENAME, }, + "remote_source_json_config": { + "json": CACHITO_CONFIG_FILES, + "filename": REMOTE_SOURCE_JSON_CONFIG_FILENAME, + }, + "remote_source_json_env": { + "json": CACHITO_ENV_VARS_JSON, + "filename": REMOTE_SOURCE_JSON_ENV_FILENAME, + }, "remote_source_tarball": { "filename": REMOTE_SOURCE_TARBALL_FILENAME, "path": expected_dowload_path(workflow), @@ -749,9 +775,13 @@ def test_allow_multiple_remote_sources(workflow, allow_multiple_remote_sources): first_remote_source_name = 'gomod' first_remote_tarball_filename = 'remote-source-gomod.tar.gz' first_remote_json_filename = 'remote-source-gomod.json' + first_remote_json_config_filename = 'remote-source-gomod.config.json' + first_remote_json_env_filename = 'remote-source-gomod.env.json' second_remote_source_name = 'pip' second_remote_tarball_filename = 'remote-source-pip.tar.gz' second_remote_json_filename = 'remote-source-pip.json' + second_remote_json_config_filename = 'remote-source-pip.config.json' + second_remote_json_env_filename = 'remote-source-pip.env.json' container_yaml_config = dedent( """\ @@ -806,6 +836,14 @@ def test_allow_multiple_remote_sources(workflow, allow_multiple_remote_sources): "json": REMOTE_SOURCE_JSON, "filename": first_remote_json_filename, }, + "remote_source_json_config": { + "json": CACHITO_CONFIG_FILES, + "filename": first_remote_json_config_filename, + }, + "remote_source_json_env": { + "json": CACHITO_ENV_VARS_JSON, + "filename": first_remote_json_env_filename, + }, "remote_source_tarball": { "filename": first_remote_tarball_filename, "path": expected_dowload_path(workflow, "gomod"), @@ -819,6 +857,14 @@ def test_allow_multiple_remote_sources(workflow, allow_multiple_remote_sources): "json": SECOND_REMOTE_SOURCE_JSON, "filename": second_remote_json_filename, }, + "remote_source_json_config": { + "json": SECOND_CACHITO_CONFIG_FILES, + "filename": second_remote_json_config_filename, + }, + "remote_source_json_env": { + "json": SECOND_CACHITO_ENV_VARS_JSON, + "filename": second_remote_json_env_filename, + }, "remote_source_tarball": { "filename": second_remote_tarball_filename, "path": expected_dowload_path(workflow, "pip"), @@ -932,7 +978,8 @@ def test_inject_remote_sources_dest_already_exists(workflow): id=CACHITO_REQUEST_ID, name=None, json_data={}, - build_args={}, + json_env_data={}, + json_config_data={}, tarball_path=Path("/does/not/matter"), ), ]