Skip to content

Commit

Permalink
feat(remote_source): upload all metadata
Browse files Browse the repository at this point in the history 
Upload all metadata from cachito, se we can avoid having extra queries
for cachito endpoints and keep all data in koji

STONEBLD-2655

Signed-off-by: Martin Basti <[email protected]>
  • Loading branch information
@MartinBasti
MartinBasti committed Aug 9, 2024
1 parent bd931b0 commit 9d2ebfe
Show file tree
Hide file tree
Showing 5 changed files with 119 additions and 25 deletions.
2 changes: 2 additions & 0 deletions atomic_reactor/constants.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,8 @@
# Name of downloaded remote sources tarball
REMOTE_SOURCE_TARBALL_FILENAME = 'remote-source.tar.gz'
REMOTE_SOURCE_JSON_FILENAME = 'remote-source.json'
REMOTE_SOURCE_JSON_CONFIG_FILENAME = 'remote-source.json.config'
REMOTE_SOURCE_JSON_ENV_FILENAME = 'remote-source.json.env'
ICM_JSON_FILENAME = 'icm-{}.json'

# koji osbs_build metadata
Expand Down
23 changes: 14 additions & 9 deletions atomic_reactor/plugins/koji_import.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -322,6 +322,8 @@ def set_remote_sources_metadata(self, extra):
"archives": [
remote_source["remote_source_json"]["filename"],
remote_source["remote_source_tarball"]["filename"],
remote_source["remote_source_json_env"]["filename"],
remote_source["remote_source_json_config"]["filename"],
],
}
for remote_source in remote_source_result
Expand Down Expand Up @@ -622,15 +624,18 @@ def _collect_remote_sources(self) -> Iterable[ArtifactOutputInfo]:
dest_filename = remote_source_tarball['filename']
yield local_filename, dest_filename, KOJI_BTYPE_REMOTE_SOURCES, None

remote_source_json = remote_source['remote_source_json']
remote_source_json_filename = remote_source_json['filename']
file_path = os.path.join(tmpdir, remote_source_json_filename)
with open(file_path, 'w') as f:
json.dump(remote_source_json['json'], f, indent=4, sort_keys=True)
yield (file_path,
remote_source_json_filename,
KOJI_BTYPE_REMOTE_SOURCES,
None)
for source_key in (
"remote_source_json", "remote_source_json_env", "remote_source_json_config",
):
data_json = remote_source[source_key]
data_json_filename = data_json['filename']
file_path = os.path.join(tmpdir, data_json_filename)
with open(file_path, 'w') as f:
json.dump(data_json['json'], f, indent=4, sort_keys=True)
yield (file_path,
data_json_filename,
KOJI_BTYPE_REMOTE_SOURCES,
None)

def _collect_exported_operator_manifests(self) -> Iterable[ArtifactOutputInfo]:
wf_data = self.workflow.data
Expand Down
40 changes: 33 additions & 7 deletions atomic_reactor/plugins/resolve_remote_source.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@
REMOTE_SOURCE_DIR,
REMOTE_SOURCE_JSON_FILENAME,
REMOTE_SOURCE_TARBALL_FILENAME,
REMOTE_SOURCE_JSON_CONFIG_FILENAME,
REMOTE_SOURCE_JSON_ENV_FILENAME,
)
from atomic_reactor.dirs import BuildDir
from atomic_reactor.plugin import Plugin
Expand All @@ -48,6 +50,8 @@ class RemoteSource:
id: int
name: Optional[str]
json_data: dict
json_env_data: Dict[str, Dict[str, str]]
json_config_data: List[Dict[str, str]]
build_args: Dict[str, str]
tarball_path: Path

Expand All @@ -65,6 +69,20 @@ def json_filename(cls, name: Optional[str]):
else:
return REMOTE_SOURCE_JSON_FILENAME

@classmethod
def json_config_filename(cls, name: Optional[str]):
if name:
return f"remote-source-{name}.json.config"
else:
return REMOTE_SOURCE_JSON_CONFIG_FILENAME

@classmethod
def json_env_filename(cls, name: Optional[str]):
if name:
return f"remote-source-{name}.json.env"
else:
return REMOTE_SOURCE_JSON_ENV_FILENAME


class ResolveRemoteSourcePlugin(Plugin):
"""Initiate a new Cachito request for sources
Expand Down Expand Up @@ -219,17 +237,15 @@ def inject_into_build_dir(
with tarfile.open(remote_source.tarball_path) as tar:
safe_extractall(tar, str(dest_dir))

config_files = self.cachito_session.get_request_config_files(remote_source.id)
self.generate_cachito_config_files(dest_dir, config_files)
self.generate_cachito_config_files(dest_dir, remote_source.json_config_data)

# Create cachito.env file with environment variables received from cachito request
self.generate_cachito_env_file(dest_dir, remote_source.build_args)

return created_dirs

def get_buildargs(self, request_id: int, remote_source_name: Optional[str]) -> Dict[str, str]:
def get_buildargs(self, env_vars: Dict[str, Dict[str, str]], remote_source_name: Optional[str]) -> Dict[str, str]:
build_args = {}
env_vars = self.cachito_session.get_request_env_vars(request_id)

for env_var, value_info in env_vars.items():
build_arg_value = value_info['value']
Expand Down Expand Up @@ -309,12 +325,15 @@ def process_request(self, source_request: dict, name: Optional[str]) -> RemoteSo
dest_filename=tarball_filename,
)

build_args = self.get_buildargs(source_request["id"], name)
env_vars = self.cachito_session.get_request_env_vars(source_request["id"])
build_args = self.get_buildargs(env_vars, name)

remote_source = RemoteSource(
id=source_request["id"],
name=name,
json_data=self.source_request_to_json(source_request),
json_env_data=env_vars,
json_config_data=self.cachito_session.get_request_config_files(source_request["id"]),
build_args=build_args,
tarball_path=Path(tarball_dest_path),
)
Expand All @@ -323,15 +342,22 @@ def process_request(self, source_request: dict, name: Optional[str]) -> RemoteSo
def remote_source_to_output(self, remote_source: RemoteSource) -> Dict[str, Any]:
"""Convert a processed remote source to a dict to be used as output of this plugin."""
download_url = self.cachito_session.assemble_download_url(remote_source.id)
json_filename = RemoteSource.json_filename(remote_source.name)

return {
"id": remote_source.id,
"name": remote_source.name,
"url": download_url,
"remote_source_json": {
"json": remote_source.json_data,
"filename": json_filename,
"filename": RemoteSource.json_filename(remote_source.name),
},
"remote_source_json_env": {
"json": remote_source.json_env_data,
"filename": RemoteSource.json_env_filename(remote_source.name),
},
"remote_source_json_config": {
"json": remote_source.json_config_data,
"filename": RemoteSource.json_config_filename(remote_source.name),
},
"remote_source_tarball": {
"filename": remote_source.tarball_path.name,
Expand Down
15 changes: 14 additions & 1 deletion tests/plugins/test_koji_import.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,8 @@
PARENT_IMAGES_KEY, OPERATOR_MANIFESTS_ARCHIVE,
REMOTE_SOURCE_TARBALL_FILENAME,
REMOTE_SOURCE_JSON_FILENAME,
REMOTE_SOURCE_JSON_CONFIG_FILENAME,
REMOTE_SOURCE_JSON_ENV_FILENAME,
MEDIA_TYPE_DOCKER_V2_SCHEMA2,
MEDIA_TYPE_DOCKER_V2_MANIFEST_LIST,
KOJI_BTYPE_REMOTE_SOURCE_FILE,
Expand Down Expand Up @@ -533,6 +535,14 @@ def custom_get(method, url, headers, **kwargs):
"filename": REMOTE_SOURCE_JSON_FILENAME,
"json": {"stub": "data"},
},
"remote_source_json_config": {
"filename": REMOTE_SOURCE_JSON_CONFIG_FILENAME,
"json": [{"stub": "data"}],
},
"remote_source_json_env": {
"filename": REMOTE_SOURCE_JSON_ENV_FILENAME,
"json": {"var": {"stub": "data"}},
},
"remote_source_tarball": {
"filename": REMOTE_SOURCE_TARBALL_FILENAME,
"path": str(source_path),
Expand Down Expand Up @@ -2003,7 +2013,10 @@ def test_remote_sources(self, workflow, source_dir,
{
'name': None,
'url': 'https://cachito.com/api/v1/requests/21048',
'archives': ['remote-source.json', 'remote-source.tar.gz'],
'archives': [
'remote-source.json', 'remote-source.tar.gz',
'remote-source.json.env', 'remote-source.json.config'
],
}
]
assert REMOTE_SOURCE_TARBALL_FILENAME in session.uploaded_files.keys()
Expand Down
64 changes: 56 additions & 8 deletions tests/plugins/test_resolve_remote_source.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@
REMOTE_SOURCE_DIR,
REMOTE_SOURCE_TARBALL_FILENAME,
REMOTE_SOURCE_JSON_FILENAME,
REMOTE_SOURCE_JSON_CONFIG_FILENAME,
REMOTE_SOURCE_JSON_ENV_FILENAME,
)
from atomic_reactor.plugin import PluginFailedException
from atomic_reactor.plugins.resolve_remote_source import (
Expand Down Expand Up @@ -391,6 +393,14 @@ def mock_cachito_api_multiple_remote_sources(workflow, user=KOJI_TASK_OWNER):
.ordered()
)

(
flexmock(CachitoAPI)
.should_receive("get_request_config_files")
.with_args(CACHITO_SOURCE_REQUEST["id"])
.and_return(CACHITO_CONFIG_FILES)
.ordered()
)

(
flexmock(CachitoAPI)
.should_receive("download_sources")
Expand All @@ -411,14 +421,6 @@ def mock_cachito_api_multiple_remote_sources(workflow, user=KOJI_TASK_OWNER):
.ordered()
)

(
flexmock(CachitoAPI)
.should_receive("get_request_config_files")
.with_args(CACHITO_SOURCE_REQUEST["id"])
.and_return(CACHITO_CONFIG_FILES)
.ordered()
)

(
flexmock(CachitoAPI)
.should_receive("get_request_config_files")
Expand Down Expand Up @@ -564,6 +566,14 @@ def test_resolve_remote_source(workflow, scratch, dr_strs, dependency_replacemen
"json": REMOTE_SOURCE_JSON,
"filename": REMOTE_SOURCE_JSON_FILENAME,
},
"remote_source_json_config": {
"json": CACHITO_CONFIG_FILES,
"filename": REMOTE_SOURCE_JSON_CONFIG_FILENAME,
},
"remote_source_json_env": {
"json": CACHITO_ENV_VARS_JSON,
"filename": REMOTE_SOURCE_JSON_ENV_FILENAME,
},
"remote_source_tarball": {
"filename": REMOTE_SOURCE_TARBALL_FILENAME,
"path": expected_dowload_path(workflow),
Expand Down Expand Up @@ -653,6 +663,14 @@ def test_no_koji_user(workflow, caplog):
"json": REMOTE_SOURCE_JSON,
"filename": REMOTE_SOURCE_JSON_FILENAME,
},
"remote_source_json_config": {
"json": CACHITO_CONFIG_FILES,
"filename": REMOTE_SOURCE_JSON_CONFIG_FILENAME,
},
"remote_source_json_env": {
"json": CACHITO_ENV_VARS_JSON,
"filename": REMOTE_SOURCE_JSON_ENV_FILENAME,
},
"remote_source_tarball": {
"filename": REMOTE_SOURCE_TARBALL_FILENAME,
"path": expected_dowload_path(workflow),
Expand Down Expand Up @@ -732,6 +750,14 @@ def test_bad_build_metadata(workflow, task_id, log_entry, caplog):
"json": REMOTE_SOURCE_JSON,
"filename": REMOTE_SOURCE_JSON_FILENAME,
},
"remote_source_json_config": {
"json": CACHITO_CONFIG_FILES,
"filename": REMOTE_SOURCE_JSON_CONFIG_FILENAME,
},
"remote_source_json_env": {
"json": CACHITO_ENV_VARS_JSON,
"filename": REMOTE_SOURCE_JSON_ENV_FILENAME,
},
"remote_source_tarball": {
"filename": REMOTE_SOURCE_TARBALL_FILENAME,
"path": expected_dowload_path(workflow),
Expand All @@ -749,9 +775,13 @@ def test_allow_multiple_remote_sources(workflow, allow_multiple_remote_sources):
first_remote_source_name = 'gomod'
first_remote_tarball_filename = 'remote-source-gomod.tar.gz'
first_remote_json_filename = 'remote-source-gomod.json'
first_remote_json_config_filename = 'remote-source-gomod.json.config'
first_remote_json_env_filename = 'remote-source-gomod.json.env'
second_remote_source_name = 'pip'
second_remote_tarball_filename = 'remote-source-pip.tar.gz'
second_remote_json_filename = 'remote-source-pip.json'
second_remote_json_config_filename = 'remote-source-pip.json.config'
second_remote_json_env_filename = 'remote-source-pip.json.env'

container_yaml_config = dedent(
"""\
Expand Down Expand Up @@ -806,6 +836,14 @@ def test_allow_multiple_remote_sources(workflow, allow_multiple_remote_sources):
"json": REMOTE_SOURCE_JSON,
"filename": first_remote_json_filename,
},
"remote_source_json_config": {
"json": CACHITO_CONFIG_FILES,
"filename": first_remote_json_config_filename,
},
"remote_source_json_env": {
"json": CACHITO_ENV_VARS_JSON,
"filename": first_remote_json_env_filename,
},
"remote_source_tarball": {
"filename": first_remote_tarball_filename,
"path": expected_dowload_path(workflow, "gomod"),
Expand All @@ -819,6 +857,14 @@ def test_allow_multiple_remote_sources(workflow, allow_multiple_remote_sources):
"json": SECOND_REMOTE_SOURCE_JSON,
"filename": second_remote_json_filename,
},
"remote_source_json_config": {
"json": SECOND_CACHITO_CONFIG_FILES,
"filename": second_remote_json_config_filename,
},
"remote_source_json_env": {
"json": SECOND_CACHITO_ENV_VARS_JSON,
"filename": second_remote_json_env_filename,
},
"remote_source_tarball": {
"filename": second_remote_tarball_filename,
"path": expected_dowload_path(workflow, "pip"),
Expand Down Expand Up @@ -932,6 +978,8 @@ def test_inject_remote_sources_dest_already_exists(workflow):
id=CACHITO_REQUEST_ID,
name=None,
json_data={},
json_env_data={},
json_config_data={},
build_args={},
tarball_path=Path("/does/not/matter"),
),
Expand Down

0 comments on commit 9d2ebfe

Please sign in to comment.