Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(remote_source): upload all metadata #2106

Merged
merged 2 commits into from
Aug 14, 2024
Merged

feat(remote_source): upload all metadata #2106

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
908fd3c
feat(remote_source): upload all metadata
MartinBasti Aug 9, 2024
68216d8
refactor(remote_sources): make build_args property
MartinBasti Aug 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions atomic_reactor/constants.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,8 @@
# Name of downloaded remote sources tarball
REMOTE_SOURCE_TARBALL_FILENAME = 'remote-source.tar.gz'
REMOTE_SOURCE_JSON_FILENAME = 'remote-source.json'
REMOTE_SOURCE_JSON_CONFIG_FILENAME = 'remote-source.config.json'
REMOTE_SOURCE_JSON_ENV_FILENAME = 'remote-source.env.json'
ICM_JSON_FILENAME = 'icm-{}.json'

# koji osbs_build metadata
Expand Down
23 changes: 14 additions & 9 deletions atomic_reactor/plugins/koji_import.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -322,6 +322,8 @@ def set_remote_sources_metadata(self, extra):
"archives": [
remote_source["remote_source_json"]["filename"],
remote_source["remote_source_tarball"]["filename"],
remote_source["remote_source_json_env"]["filename"],
remote_source["remote_source_json_config"]["filename"],
],
}
for remote_source in remote_source_result
Expand Down Expand Up @@ -622,15 +624,18 @@ def _collect_remote_sources(self) -> Iterable[ArtifactOutputInfo]:
dest_filename = remote_source_tarball['filename']
yield local_filename, dest_filename, KOJI_BTYPE_REMOTE_SOURCES, None

remote_source_json = remote_source['remote_source_json']
remote_source_json_filename = remote_source_json['filename']
file_path = os.path.join(tmpdir, remote_source_json_filename)
with open(file_path, 'w') as f:
json.dump(remote_source_json['json'], f, indent=4, sort_keys=True)
yield (file_path,
remote_source_json_filename,
KOJI_BTYPE_REMOTE_SOURCES,
None)
for source_key in (
mkosiarc marked this conversation as resolved.
Show resolved Hide resolved
"remote_source_json", "remote_source_json_env", "remote_source_json_config",
):
data_json = remote_source[source_key]
data_json_filename = data_json['filename']
file_path = os.path.join(tmpdir, data_json_filename)
with open(file_path, 'w') as f:
json.dump(data_json['json'], f, indent=4, sort_keys=True)
yield (file_path,
data_json_filename,
KOJI_BTYPE_REMOTE_SOURCES,
None)

def _collect_exported_operator_manifests(self) -> Iterable[ArtifactOutputInfo]:
wf_data = self.workflow.data
Expand Down
43 changes: 36 additions & 7 deletions atomic_reactor/plugins/resolve_remote_source.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@
REMOTE_SOURCE_DIR,
REMOTE_SOURCE_JSON_FILENAME,
REMOTE_SOURCE_TARBALL_FILENAME,
REMOTE_SOURCE_JSON_CONFIG_FILENAME,
REMOTE_SOURCE_JSON_ENV_FILENAME,
)
from atomic_reactor.dirs import BuildDir
from atomic_reactor.plugin import Plugin
Expand All @@ -48,6 +50,8 @@ class RemoteSource:
id: int
name: Optional[str]
json_data: dict
json_env_data: Dict[str, Dict[str, str]]
json_config_data: List[Dict[str, str]]
build_args: Dict[str, str]
tarball_path: Path

Expand All @@ -65,6 +69,20 @@ def json_filename(cls, name: Optional[str]):
else:
return REMOTE_SOURCE_JSON_FILENAME

@classmethod
def json_config_filename(cls, name: Optional[str]):
if name:
return f"remote-source-{name}.config.json"
else:
return REMOTE_SOURCE_JSON_CONFIG_FILENAME

@classmethod
def json_env_filename(cls, name: Optional[str]):
if name:
return f"remote-source-{name}.env.json"
else:
return REMOTE_SOURCE_JSON_ENV_FILENAME


class ResolveRemoteSourcePlugin(Plugin):
"""Initiate a new Cachito request for sources
Expand Down Expand Up @@ -219,17 +237,18 @@ def inject_into_build_dir(
with tarfile.open(remote_source.tarball_path) as tar:
safe_extractall(tar, str(dest_dir))

config_files = self.cachito_session.get_request_config_files(remote_source.id)
self.generate_cachito_config_files(dest_dir, config_files)
self.generate_cachito_config_files(dest_dir, remote_source.json_config_data)

# Create cachito.env file with environment variables received from cachito request
self.generate_cachito_env_file(dest_dir, remote_source.build_args)

return created_dirs

def get_buildargs(self, request_id: int, remote_source_name: Optional[str]) -> Dict[str, str]:
def get_buildargs(
self, env_vars: Dict[str, Dict[str, str]],
remote_source_name: Optional[str]
) -> Dict[str, str]:
chmeliik marked this conversation as resolved.
Show resolved Hide resolved
build_args = {}
env_vars = self.cachito_session.get_request_env_vars(request_id)

for env_var, value_info in env_vars.items():
build_arg_value = value_info['value']
Expand Down Expand Up @@ -309,12 +328,15 @@ def process_request(self, source_request: dict, name: Optional[str]) -> RemoteSo
dest_filename=tarball_filename,
)

build_args = self.get_buildargs(source_request["id"], name)
env_vars = self.cachito_session.get_request_env_vars(source_request["id"])
build_args = self.get_buildargs(env_vars, name)

remote_source = RemoteSource(
id=source_request["id"],
name=name,
json_data=self.source_request_to_json(source_request),
json_env_data=env_vars,
json_config_data=self.cachito_session.get_request_config_files(source_request["id"]),
build_args=build_args,
tarball_path=Path(tarball_dest_path),
)
Expand All @@ -323,15 +345,22 @@ def process_request(self, source_request: dict, name: Optional[str]) -> RemoteSo
def remote_source_to_output(self, remote_source: RemoteSource) -> Dict[str, Any]:
"""Convert a processed remote source to a dict to be used as output of this plugin."""
download_url = self.cachito_session.assemble_download_url(remote_source.id)
json_filename = RemoteSource.json_filename(remote_source.name)

return {
"id": remote_source.id,
"name": remote_source.name,
"url": download_url,
"remote_source_json": {
"json": remote_source.json_data,
"filename": json_filename,
"filename": RemoteSource.json_filename(remote_source.name),
},
"remote_source_json_env": {
"json": remote_source.json_env_data,
"filename": RemoteSource.json_env_filename(remote_source.name),
},
"remote_source_json_config": {
"json": remote_source.json_config_data,
"filename": RemoteSource.json_config_filename(remote_source.name),
},
"remote_source_tarball": {
"filename": remote_source.tarball_path.name,
Expand Down
15 changes: 14 additions & 1 deletion tests/plugins/test_koji_import.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,8 @@
PARENT_IMAGES_KEY, OPERATOR_MANIFESTS_ARCHIVE,
REMOTE_SOURCE_TARBALL_FILENAME,
REMOTE_SOURCE_JSON_FILENAME,
REMOTE_SOURCE_JSON_CONFIG_FILENAME,
REMOTE_SOURCE_JSON_ENV_FILENAME,
MEDIA_TYPE_DOCKER_V2_SCHEMA2,
MEDIA_TYPE_DOCKER_V2_MANIFEST_LIST,
KOJI_BTYPE_REMOTE_SOURCE_FILE,
Expand Down Expand Up @@ -533,6 +535,14 @@ def custom_get(method, url, headers, **kwargs):
"filename": REMOTE_SOURCE_JSON_FILENAME,
"json": {"stub": "data"},
},
"remote_source_json_config": {
"filename": REMOTE_SOURCE_JSON_CONFIG_FILENAME,
"json": [{"stub": "data"}],
},
"remote_source_json_env": {
"filename": REMOTE_SOURCE_JSON_ENV_FILENAME,
"json": {"var": {"stub": "data"}},
},
"remote_source_tarball": {
"filename": REMOTE_SOURCE_TARBALL_FILENAME,
"path": str(source_path),
Expand Down Expand Up @@ -2003,7 +2013,10 @@ def test_remote_sources(self, workflow, source_dir,
{
'name': None,
'url': 'https://cachito.com/api/v1/requests/21048',
'archives': ['remote-source.json', 'remote-source.tar.gz'],
'archives': [
'remote-source.json', 'remote-source.tar.gz',
'remote-source.env.json', 'remote-source.config.json'
],
}
]
assert REMOTE_SOURCE_TARBALL_FILENAME in session.uploaded_files.keys()
Expand Down
64 changes: 56 additions & 8 deletions tests/plugins/test_resolve_remote_source.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@
REMOTE_SOURCE_DIR,
REMOTE_SOURCE_TARBALL_FILENAME,
REMOTE_SOURCE_JSON_FILENAME,
REMOTE_SOURCE_JSON_CONFIG_FILENAME,
REMOTE_SOURCE_JSON_ENV_FILENAME,
)
from atomic_reactor.plugin import PluginFailedException
from atomic_reactor.plugins.resolve_remote_source import (
Expand Down Expand Up @@ -391,6 +393,14 @@ def mock_cachito_api_multiple_remote_sources(workflow, user=KOJI_TASK_OWNER):
.ordered()
)

(
flexmock(CachitoAPI)
.should_receive("get_request_config_files")
.with_args(CACHITO_SOURCE_REQUEST["id"])
.and_return(CACHITO_CONFIG_FILES)
.ordered()
)

(
flexmock(CachitoAPI)
.should_receive("download_sources")
Expand All @@ -411,14 +421,6 @@ def mock_cachito_api_multiple_remote_sources(workflow, user=KOJI_TASK_OWNER):
.ordered()
)

(
flexmock(CachitoAPI)
.should_receive("get_request_config_files")
.with_args(CACHITO_SOURCE_REQUEST["id"])
.and_return(CACHITO_CONFIG_FILES)
.ordered()
)

(
flexmock(CachitoAPI)
.should_receive("get_request_config_files")
Expand Down Expand Up @@ -564,6 +566,14 @@ def test_resolve_remote_source(workflow, scratch, dr_strs, dependency_replacemen
"json": REMOTE_SOURCE_JSON,
"filename": REMOTE_SOURCE_JSON_FILENAME,
},
"remote_source_json_config": {
"json": CACHITO_CONFIG_FILES,
"filename": REMOTE_SOURCE_JSON_CONFIG_FILENAME,
},
"remote_source_json_env": {
"json": CACHITO_ENV_VARS_JSON,
"filename": REMOTE_SOURCE_JSON_ENV_FILENAME,
},
"remote_source_tarball": {
"filename": REMOTE_SOURCE_TARBALL_FILENAME,
"path": expected_dowload_path(workflow),
Expand Down Expand Up @@ -653,6 +663,14 @@ def test_no_koji_user(workflow, caplog):
"json": REMOTE_SOURCE_JSON,
"filename": REMOTE_SOURCE_JSON_FILENAME,
},
"remote_source_json_config": {
"json": CACHITO_CONFIG_FILES,
"filename": REMOTE_SOURCE_JSON_CONFIG_FILENAME,
},
"remote_source_json_env": {
"json": CACHITO_ENV_VARS_JSON,
"filename": REMOTE_SOURCE_JSON_ENV_FILENAME,
},
"remote_source_tarball": {
"filename": REMOTE_SOURCE_TARBALL_FILENAME,
"path": expected_dowload_path(workflow),
Expand Down Expand Up @@ -732,6 +750,14 @@ def test_bad_build_metadata(workflow, task_id, log_entry, caplog):
"json": REMOTE_SOURCE_JSON,
"filename": REMOTE_SOURCE_JSON_FILENAME,
},
"remote_source_json_config": {
"json": CACHITO_CONFIG_FILES,
"filename": REMOTE_SOURCE_JSON_CONFIG_FILENAME,
},
"remote_source_json_env": {
"json": CACHITO_ENV_VARS_JSON,
"filename": REMOTE_SOURCE_JSON_ENV_FILENAME,
},
"remote_source_tarball": {
"filename": REMOTE_SOURCE_TARBALL_FILENAME,
"path": expected_dowload_path(workflow),
Expand All @@ -749,9 +775,13 @@ def test_allow_multiple_remote_sources(workflow, allow_multiple_remote_sources):
first_remote_source_name = 'gomod'
first_remote_tarball_filename = 'remote-source-gomod.tar.gz'
first_remote_json_filename = 'remote-source-gomod.json'
first_remote_json_config_filename = 'remote-source-gomod.config.json'
first_remote_json_env_filename = 'remote-source-gomod.env.json'
second_remote_source_name = 'pip'
second_remote_tarball_filename = 'remote-source-pip.tar.gz'
second_remote_json_filename = 'remote-source-pip.json'
second_remote_json_config_filename = 'remote-source-pip.config.json'
second_remote_json_env_filename = 'remote-source-pip.env.json'

container_yaml_config = dedent(
"""\
Expand Down Expand Up @@ -806,6 +836,14 @@ def test_allow_multiple_remote_sources(workflow, allow_multiple_remote_sources):
"json": REMOTE_SOURCE_JSON,
"filename": first_remote_json_filename,
},
"remote_source_json_config": {
"json": CACHITO_CONFIG_FILES,
"filename": first_remote_json_config_filename,
},
"remote_source_json_env": {
"json": CACHITO_ENV_VARS_JSON,
"filename": first_remote_json_env_filename,
},
"remote_source_tarball": {
"filename": first_remote_tarball_filename,
"path": expected_dowload_path(workflow, "gomod"),
Expand All @@ -819,6 +857,14 @@ def test_allow_multiple_remote_sources(workflow, allow_multiple_remote_sources):
"json": SECOND_REMOTE_SOURCE_JSON,
"filename": second_remote_json_filename,
},
"remote_source_json_config": {
"json": SECOND_CACHITO_CONFIG_FILES,
"filename": second_remote_json_config_filename,
},
"remote_source_json_env": {
"json": SECOND_CACHITO_ENV_VARS_JSON,
"filename": second_remote_json_env_filename,
},
"remote_source_tarball": {
"filename": second_remote_tarball_filename,
"path": expected_dowload_path(workflow, "pip"),
Expand Down Expand Up @@ -932,6 +978,8 @@ def test_inject_remote_sources_dest_already_exists(workflow):
id=CACHITO_REQUEST_ID,
name=None,
json_data={},
json_env_data={},
json_config_data={},
build_args={},
tarball_path=Path("/does/not/matter"),
),
Expand Down