From 85ab30850f4f94c55624e444d2675d6abd8a6ef5 Mon Sep 17 00:00:00 2001 From: Petr Stodulka Date: Fri, 8 Nov 2024 17:40:01 +0100 Subject: [PATCH 1/8] Packaging: Require leapp-framework 6.x + update leapp deps The leapp actors configuration feature is present since leapp-framework 6.0. Update the dependencies to ensure the correct version of the framework is installed on the system. Also, leapp requirements have been updated - requiring python3-PyYAML as it requires YAML parser, bumping leapp-framework-dependencies to 6. Address the change in leapp-deps metapackage to satisfy leapp dependencies during the upgrade process. --- packaging/leapp-repository.spec | 2 +- packaging/other_specs/leapp-el7toel8-deps.spec | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/packaging/leapp-repository.spec b/packaging/leapp-repository.spec index 0d63ba02a5..570d0df2fd 100644 --- a/packaging/leapp-repository.spec +++ b/packaging/leapp-repository.spec @@ -120,7 +120,7 @@ Requires: leapp-repository-dependencies = %{leapp_repo_deps} # IMPORTANT: this is capability provided by the leapp framework rpm. # Check that 'version' instead of the real framework rpm version. -Requires: leapp-framework >= 5.0, leapp-framework < 6 +Requires: leapp-framework >= 6.0, leapp-framework < 7 # Since we provide sub-commands for the leapp utility, we expect the leapp # tool to be installed as well. diff --git a/packaging/other_specs/leapp-el7toel8-deps.spec b/packaging/other_specs/leapp-el7toel8-deps.spec index d9e94faa55..2c662a37f1 100644 --- a/packaging/other_specs/leapp-el7toel8-deps.spec +++ b/packaging/other_specs/leapp-el7toel8-deps.spec @@ -14,7 +14,7 @@ %define leapp_repo_deps 10 -%define leapp_framework_deps 5 +%define leapp_framework_deps 6 # NOTE: the Version contains the %{rhel} macro just for the convenience to # have always upgrade path between newer and older deps packages. So for @@ -112,6 +112,7 @@ Requires: python3 Requires: python3-six Requires: python3-setuptools Requires: python3-requests +Requires: python3-PyYAML %description -n %{ldname} From 1d85a6bb84b9f6c033d1a644200fb9abd9dce430 Mon Sep 17 00:00:00 2001 From: Michal Hecko Date: Sun, 6 Oct 2024 21:01:13 +0200 Subject: [PATCH 2/8] spec: create /etc/leapp/actor_conf.d Add additional build steps to the specfile that create the actor configuration directory. The directory is owned by the package, so it gets removed when the user uninstalls leapp. Also prepared some comment lines for future when we will want to include some configuration files as part of the rpm. --- etc/leapp/actor_conf.d/.gitkeep | 0 packaging/leapp-repository.spec | 7 +++++++ 2 files changed, 7 insertions(+) create mode 100644 etc/leapp/actor_conf.d/.gitkeep diff --git a/etc/leapp/actor_conf.d/.gitkeep b/etc/leapp/actor_conf.d/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packaging/leapp-repository.spec b/packaging/leapp-repository.spec index 570d0df2fd..828355bf19 100644 --- a/packaging/leapp-repository.spec +++ b/packaging/leapp-repository.spec @@ -250,6 +250,11 @@ install -m 0755 -d %{buildroot}%{_sysconfdir}/leapp/files/ install -m 0644 etc/leapp/transaction/* %{buildroot}%{_sysconfdir}/leapp/transaction install -m 0644 etc/leapp/files/* %{buildroot}%{_sysconfdir}/leapp/files +# Actor configuration dir +install -m 0755 -d %{buildroot}%{_sysconfdir}/leapp/actor_conf.d/ +# uncomment to install existing configs +#install -m 0644 etc/leapp/actor_conf.d/* %%{buildroot}%%{_sysconfdir}/leapp/actor_conf.d + # install CLI commands for the leapp utility on the expected path install -m 0755 -d %{buildroot}%{leapp_python_sitelib}/leapp/cli/ cp -r commands %{buildroot}%{leapp_python_sitelib}/leapp/cli/ @@ -295,6 +300,8 @@ done; %dir %{custom_repositorydir} %dir %{leapp_python_sitelib}/leapp/cli/commands %config %{_sysconfdir}/leapp/files/* +# uncomment to package installed configs +#%%config %%{_sysconfdir}/leapp/actor_conf.d/* %{_sysconfdir}/leapp/repos.d/* %{_sysconfdir}/leapp/transaction/* %{repositorydir}/* From 30ff05074f7ec53f12fae6f0bb905c04eff4e78a Mon Sep 17 00:00:00 2001 From: Petr Stodulka Date: Wed, 13 Nov 2024 15:05:50 +0100 Subject: [PATCH 3/8] spec: drop .gitkeep files from the RPM We have several .gitkeep files in the repo as we want to have some directories present in git however these directories are empty otherwise. This is common hack to achieve this, but we do not want to have these files really in the resulting RPMs. So we just remove them. --- packaging/leapp-repository.spec | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packaging/leapp-repository.spec b/packaging/leapp-repository.spec index 828355bf19..2bb52505b7 100644 --- a/packaging/leapp-repository.spec +++ b/packaging/leapp-repository.spec @@ -272,6 +272,9 @@ rm -rf %{buildroot}%{repositorydir}/common/actors/testactor find %{buildroot}%{repositorydir}/common -name "test.py" -delete rm -rf `find %{buildroot}%{repositorydir} -name "tests" -type d` find %{buildroot}%{repositorydir} -name "Makefile" -delete +# .gitkeep file is used to have a directory in the repo. but we do not want these +# files in the resulting RPM +find %{buildroot} -name .gitkeep -delete for DIRECTORY in $(find %{buildroot}%{repositorydir}/ -mindepth 1 -maxdepth 1 -type d); do From 1b7a85830ce419f25dfc641f42c4189885951618 Mon Sep 17 00:00:00 2001 From: Michal Hecko Date: Sun, 10 Nov 2024 13:54:20 +0100 Subject: [PATCH 4/8] cli: load actor configuration Load actor configuration when running `leapp upgrade` or `leapp preupgrade`. The configuration is loaded, saved to leapp's DB, and remains available to all actors via framework's global variable. --- commands/command_utils.py | 32 +++++++++++++++++++++++++++++++- commands/preupgrade/__init__.py | 3 +++ commands/upgrade/__init__.py | 3 +++ 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/commands/command_utils.py b/commands/command_utils.py index 2810a542cb..190f5f0339 100644 --- a/commands/command_utils.py +++ b/commands/command_utils.py @@ -1,10 +1,12 @@ +import hashlib import json import os import re import resource +from leapp.actors import config as actor_config from leapp.exceptions import CommandError -from leapp.utils import path +from leapp.utils import audit, path HANA_BASE_PATH = '/hana/shared' HANA_SAPCONTROL_PATH_X86_64 = 'exe/linuxx86_64/hdb/sapcontrol' @@ -178,3 +180,31 @@ def set_resource_limit(resource_type, soft, hard): if soft_fsize != fsize_limit: set_resource_limit(resource.RLIMIT_FSIZE, fsize_limit, fsize_limit) + + +def load_actor_configs_and_store_it_in_db(context, repositories, framework_cfg): + """ + Load actor configuration so that actor's can access it and store it into leapp db. + + :param context: Current execution context + :param repositories: Discovered repositories + :param framework_cfg: Leapp's configuration + """ + # Read the Actor Config and validate it against the schemas saved in the + # configuration. + + actor_config_schemas = tuple(actor.config_schemas for actor in repositories.actors) + actor_config_schemas = actor_config.normalize_schemas(actor_config_schemas) + actor_config_path = framework_cfg.get('actor_config', 'path') + + # Note: actor_config.load() stores the loaded actor config into a global + # variable which can then be accessed by functions in that file. Is this + # the right way to store that information? + actor_cfg = actor_config.load(actor_config_path, actor_config_schemas) + + # Dump the collected configuration, checksum it and store it inside the DB + config_text = json.dumps(actor_cfg) + config_text_hash = hashlib.sha256(config_text.encode('utf-8')).hexdigest() + config_data = audit.ActorConfigData(config=config_text, hash_id=config_text_hash) + db_config = audit.ActorConfig(config=config_data, context=context) + db_config.store() diff --git a/commands/preupgrade/__init__.py b/commands/preupgrade/__init__.py index a9fa40e077..631eca6bd1 100644 --- a/commands/preupgrade/__init__.py +++ b/commands/preupgrade/__init__.py @@ -62,6 +62,9 @@ def preupgrade(args, breadcrumbs): command_utils.set_resource_limits() workflow = repositories.lookup_workflow('IPUWorkflow')() + + command_utils.load_actor_configs_and_store_it_in_db(context, repositories, cfg) + util.warn_if_unsupported(configuration) util.process_whitelist_experimental(repositories, workflow, configuration, logger) with beautify_actor_exception(): diff --git a/commands/upgrade/__init__.py b/commands/upgrade/__init__.py index c7487fded8..3dedd43802 100644 --- a/commands/upgrade/__init__.py +++ b/commands/upgrade/__init__.py @@ -93,6 +93,9 @@ def upgrade(args, breadcrumbs): command_utils.set_resource_limits() workflow = repositories.lookup_workflow('IPUWorkflow')(auto_reboot=args.reboot) + + command_utils.load_actor_configs_and_store_it_in_db(context, repositories, cfg) + util.process_whitelist_experimental(repositories, workflow, configuration, logger) util.warn_if_unsupported(configuration) with beautify_actor_exception(): From 04183abc3c74e990a9ecaf6f32702108345e73d1 Mon Sep 17 00:00:00 2001 From: Michal Hecko Date: Sun, 10 Nov 2024 14:35:26 +0100 Subject: [PATCH 5/8] configs(common): introduce RHUI configuration Introduce a common configuration definition for RHUI related decisions. The configuration has an atomic nature - if the user wants to overwrite leapp's decisions, he/she must overwrite all of them. Essentially, all fields of the RHUI_SETUPS cloud map entry can be configured. Almost no non-empty defaults are provided, as no reasonable defaults can be given. This is due to all setup parameters are different from provider to provider. Therefore, default values are empty values, so that it can later be detected by an actor whether all fields of the RHUI config has been filled. Jira ref: RHEL-56251 --- repos/system_upgrade/common/configs/rhui.py | 127 ++++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 repos/system_upgrade/common/configs/rhui.py diff --git a/repos/system_upgrade/common/configs/rhui.py b/repos/system_upgrade/common/configs/rhui.py new file mode 100644 index 0000000000..ade9bab9cf --- /dev/null +++ b/repos/system_upgrade/common/configs/rhui.py @@ -0,0 +1,127 @@ +""" +Configuration keys for RHUI. + +In case of RHUI in private regions it usual that publicly known RHUI data +is not valid. In such cases it's possible to provide the correct expected +RHUI data to correct the in-place upgrade process. +""" + +from leapp.actors.config import Config +from leapp.models import fields + +RHUI_CONFIG_SECTION = 'rhui' + + +# @Note(mhecko): We use to distinguish config instantiated from default values that we should ignore +# # Maybe we could make all config values None and detect it that way, but then we cannot +# # give the user an example how the config should look like. +class RhuiUseConfig(Config): + section = RHUI_CONFIG_SECTION + name = "use_config" + type_ = fields.Boolean() + default = False + description = """ + Use values provided in the configuration file to override leapp's decisions. + """ + + +class RhuiSourcePkgs(Config): + section = RHUI_CONFIG_SECTION + name = "source_clients" + type_ = fields.List(fields.String()) + default = [] + description = """ + The name of the source RHUI client RPMs (to be removed from the system). + """ + + +class RhuiTargetPkgs(Config): + section = RHUI_CONFIG_SECTION + name = "target_clients" + type_ = fields.List(fields.String()) + default = [] + description = """ + The name of the target RHUI client RPM (to be installed on the system). + """ + + +class RhuiCloudProvider(Config): + section = RHUI_CONFIG_SECTION + name = "cloud_provider" + type_ = fields.String() + default = "" + description = """ + Cloud provider name that should be used internally by leapp. + + Leapp recognizes the following cloud providers: + - azure + - aws + - google + + Cloud provider information is used for triggering some provider-specific modifications. The value also + influences how leapp determines target repositories to enable. + """ + + +# @Note(mhecko): We likely don't need this. We need the variant primarily to grab files from a correct directory +# in leapp-rhui- folders. +class RhuiCloudVariant(Config): + section = RHUI_CONFIG_SECTION + name = "image_variant" + type_ = fields.String() + default = "ordinary" + description = """ + RHEL variant of the source system - is the source system SAP-specific image? + + Leapp recognizes the following cloud providers: + - ordinary # The source system has not been deployed from a RHEL with SAP image + - sap # RHEL SAP images + - sap-apps # RHEL SAP Apps images (Azure only) + - sap-ha # RHEL HA Apps images (HA only) + + Cloud provider information is used for triggering some provider-specific modifications. The value also + influences how leapp determines target repositories to enable. + + Default: + "ordinary" + """ + + +class RhuiUpgradeFiles(Config): + section = RHUI_CONFIG_SECTION + name = "upgrade_files" + type_ = fields.StringMap(fields.String()) + default = dict() + description = """ + A mapping from source file paths to the destination where should they be + placed in the upgrade container. + + Typically, these files should be provided by leapp-rhui- packages. + + These files are needed to facilitate access to target repositories. Typical examples are: repofile(s), + certificates and keys. + """ + + +class RhuiTargetRepositoriesToUse(Config): + section = RHUI_CONFIG_SECTION + name = "rhui_target_repositories_to_use" + type_ = fields.List(fields.String()) + description = """ + List of target repositories enabled during the upgrade. Similar to executing leapp with --enablerepo. + + The repositories to be enabled need to be either in the repofiles listed in the `upgrade_files` field, + or in repofiles present on the source system. + """ + default = list() + + +all_rhui_cfg = ( + RhuiTargetPkgs, + RhuiUpgradeFiles, + RhuiTargetRepositoriesToUse, + RhuiCloudProvider, + RhuiCloudVariant, + RhuiSourcePkgs, + RhuiUseConfig +) From f025759b52b4cc60871b1e186abda0955fb60378 Mon Sep 17 00:00:00 2001 From: Michal Hecko Date: Sun, 10 Nov 2024 14:36:07 +0100 Subject: [PATCH 6/8] check_rhui: read RHUI configuration Extend the check_rhui actor to read user-provided RHUI configuration. If the provided configuration values say that the user wants to overrwrite leapp's decisions, then the patch checks whether all values are provided. If so, corresponding RHUIInfo message is produced. The only implemented safe-guards are those that prevent the user from accidentaly specifying a non-existing file to be copied into the scrach container during us preparing to download target userspace content. If the user provides only some of the configuration values the upgrade is terminated early with an error, providing quick feedback about misconfiguration. The patch has been designed to allow development of upgrades on previously unknown clouds (clouds without an entry in RHUI_SETUPS). Jira ref: RHEL-56251 --- .../common/actors/cloud/checkrhui/actor.py | 4 + .../cloud/checkrhui/libraries/checkrhui.py | 102 +++++++++- .../tests/component_test_checkrhui.py | 178 ++++++++++++++++-- 3 files changed, 265 insertions(+), 19 deletions(-) diff --git a/repos/system_upgrade/common/actors/cloud/checkrhui/actor.py b/repos/system_upgrade/common/actors/cloud/checkrhui/actor.py index 593e73e51f..933ffcb36e 100644 --- a/repos/system_upgrade/common/actors/cloud/checkrhui/actor.py +++ b/repos/system_upgrade/common/actors/cloud/checkrhui/actor.py @@ -1,4 +1,5 @@ from leapp.actors import Actor +from leapp.configs.common.rhui import all_rhui_cfg from leapp.libraries.actor import checkrhui as checkrhui_lib from leapp.models import ( CopyFile, @@ -8,6 +9,7 @@ RequiredTargetUserspacePackages, RHUIInfo, RpmTransactionTasks, + TargetRepositories, TargetUserSpacePreupgradeTasks ) from leapp.reporting import Report @@ -21,6 +23,7 @@ class CheckRHUI(Actor): """ name = 'checkrhui' + config_schemas = all_rhui_cfg consumes = (InstalledRPM,) produces = ( KernelCmdlineArg, @@ -28,6 +31,7 @@ class CheckRHUI(Actor): RequiredTargetUserspacePackages, Report, DNFPluginTask, RpmTransactionTasks, + TargetRepositories, TargetUserSpacePreupgradeTasks, CopyFile, ) diff --git a/repos/system_upgrade/common/actors/cloud/checkrhui/libraries/checkrhui.py b/repos/system_upgrade/common/actors/cloud/checkrhui/libraries/checkrhui.py index 3b2179175e..64e36e083c 100644 --- a/repos/system_upgrade/common/actors/cloud/checkrhui/libraries/checkrhui.py +++ b/repos/system_upgrade/common/actors/cloud/checkrhui/libraries/checkrhui.py @@ -2,17 +2,29 @@ import os from collections import namedtuple +import leapp.configs.common.rhui as rhui_config_lib from leapp import reporting +from leapp.configs.common.rhui import ( # Import all config fields so we are not using their name attributes directly + RhuiCloudProvider, + RhuiCloudVariant, + RhuiSourcePkgs, + RhuiTargetPkgs, + RhuiTargetRepositoriesToUse, + RhuiUpgradeFiles, + RhuiUseConfig +) from leapp.exceptions import StopActorExecutionError from leapp.libraries.common import rhsm, rhui from leapp.libraries.common.config import version from leapp.libraries.stdlib import api from leapp.models import ( CopyFile, + CustomTargetRepository, DNFPluginTask, InstalledRPM, RHUIInfo, RpmTransactionTasks, + TargetRepositories, TargetRHUIPostInstallTasks, TargetRHUIPreInstallTasks, TargetRHUISetupInfo, @@ -291,11 +303,11 @@ def produce_rhui_info_to_setup_target(rhui_family, source_setup_desc, target_set api.produce(rhui_info) -def produce_rpms_to_install_into_target(source_setup, target_setup): - to_install = sorted(target_setup.clients - source_setup.clients) - to_remove = sorted(source_setup.clients - target_setup.clients) +def produce_rpms_to_install_into_target(source_clients, target_clients): + to_install = sorted(target_clients - source_clients) + to_remove = sorted(source_clients - target_clients) - api.produce(TargetUserSpacePreupgradeTasks(install_rpms=sorted(target_setup.clients))) + api.produce(TargetUserSpacePreupgradeTasks(install_rpms=sorted(target_clients))) if to_install or to_remove: api.produce(RpmTransactionTasks(to_install=to_install, to_remove=to_remove)) @@ -316,7 +328,85 @@ def inform_about_upgrade_with_rhui_without_no_rhsm(): return False +def emit_rhui_setup_tasks_based_on_config(rhui_config_dict): + config_upgrade_files = rhui_config_dict[RhuiUpgradeFiles.name] + + nonexisting_files_to_copy = [] + for source_path in config_upgrade_files: + if not os.path.exists(source_path): + nonexisting_files_to_copy.append(source_path) + + if nonexisting_files_to_copy: + details_lines = ['The following files were not found:'] + # Use .format and put backticks around paths so that weird unicode spaces will be easily seen + details_lines.extend(' - `{0}`'.format(path) for path in nonexisting_files_to_copy) + details = '\n'.join(details_lines) + + reason = 'RHUI config lists nonexisting files in its `{0}` field.'.format(RhuiUpgradeFiles.name) + raise StopActorExecutionError(reason, details={'details': details}) + + files_to_copy_into_overlay = [CopyFile(src=key, dst=value) for key, value in config_upgrade_files.items()] + preinstall_tasks = TargetRHUIPreInstallTasks(files_to_copy_into_overlay=files_to_copy_into_overlay) + + target_client_setup_info = TargetRHUISetupInfo( + preinstall_tasks=preinstall_tasks, + postinstall_tasks=TargetRHUIPostInstallTasks(), + bootstrap_target_client=False, # We don't need to install the client into overlay - user provided all files + ) + + rhui_info = RHUIInfo( + provider=rhui_config_dict[RhuiCloudProvider.name], + variant=rhui_config_dict[RhuiCloudVariant.name], + src_client_pkg_names=rhui_config_dict[RhuiSourcePkgs.name], + target_client_pkg_names=rhui_config_dict[RhuiTargetPkgs.name], + target_client_setup_info=target_client_setup_info + ) + api.produce(rhui_info) + + +def request_configured_repos_to_be_enabled(rhui_config): + config_repos_to_enable = rhui_config[RhuiTargetRepositoriesToUse.name] + custom_repos = [CustomTargetRepository(repoid=repoid) for repoid in config_repos_to_enable] + if custom_repos: + target_repos = TargetRepositories(custom_repos=custom_repos, rhel_repos=[]) + api.produce(target_repos) + + +def stop_with_err_if_config_missing_fields(config): + required_fields = [ + RhuiTargetRepositoriesToUse, + RhuiCloudProvider, + # RhuiCloudVariant, <- this is not required + RhuiSourcePkgs, + RhuiTargetPkgs, + RhuiUpgradeFiles, + ] + + missing_fields = tuple(field for field in required_fields if not config[field.name]) + if missing_fields: + field_names = (field.name for field in missing_fields) + missing_fields_str = ', '.join(field_names) + details = 'The following required RHUI config fields are missing or they are set to an empty value: {}' + details = details.format(missing_fields_str) + raise StopActorExecutionError('Provided RHUI config is missing values for required fields.', + details={'details': details}) + + def process(): + rhui_config = api.current_actor().config[rhui_config_lib.RHUI_CONFIG_SECTION] + + if rhui_config[RhuiUseConfig.name]: + api.current_logger().info('Skipping RHUI upgrade auto-configuration - using provided config instead.') + stop_with_err_if_config_missing_fields(rhui_config) + emit_rhui_setup_tasks_based_on_config(rhui_config) + + src_clients = set(rhui_config[RhuiSourcePkgs.name]) + target_clients = set(rhui_config[RhuiTargetPkgs.name]) + produce_rpms_to_install_into_target(src_clients, target_clients) + + request_configured_repos_to_be_enabled(rhui_config) + return + installed_rpm = itertools.chain(*[installed_rpm_msg.items for installed_rpm_msg in api.consume(InstalledRPM)]) installed_pkgs = {rpm.name for rpm in installed_rpm} @@ -342,7 +432,9 @@ def process(): # Instruction on how to access the target content produce_rhui_info_to_setup_target(src_rhui_setup.family, src_rhui_setup.description, target_setup_desc) - produce_rpms_to_install_into_target(src_rhui_setup.description, target_setup_desc) + source_clients = src_rhui_setup.description.clients + target_clients = target_setup_desc.clients + produce_rpms_to_install_into_target(source_clients, target_clients) if src_rhui_setup.family.provider == rhui.RHUIProvider.AWS: # We have to disable Amazon-id plugin in the initramdisk phase as there is no network diff --git a/repos/system_upgrade/common/actors/cloud/checkrhui/tests/component_test_checkrhui.py b/repos/system_upgrade/common/actors/cloud/checkrhui/tests/component_test_checkrhui.py index 27e70eea7f..3ac9c1b883 100644 --- a/repos/system_upgrade/common/actors/cloud/checkrhui/tests/component_test_checkrhui.py +++ b/repos/system_upgrade/common/actors/cloud/checkrhui/tests/component_test_checkrhui.py @@ -1,30 +1,43 @@ -from collections import namedtuple +import itertools +import os +from collections import defaultdict from enum import Enum import pytest from leapp import reporting +from leapp.configs.common.rhui import ( + all_rhui_cfg, + RhuiCloudProvider, + RhuiCloudVariant, + RhuiSourcePkgs, + RhuiTargetPkgs, + RhuiTargetRepositoriesToUse, + RhuiUpgradeFiles, + RhuiUseConfig +) from leapp.exceptions import StopActorExecutionError from leapp.libraries.actor import checkrhui as checkrhui_lib from leapp.libraries.common import rhsm, rhui -from leapp.libraries.common.config import mock_configs, version from leapp.libraries.common.rhui import mk_rhui_setup, RHUIFamily -from leapp.libraries.common.testutils import create_report_mocked, CurrentActorMocked, produce_mocked +from leapp.libraries.common.testutils import ( + _make_default_config, + create_report_mocked, + CurrentActorMocked, + produce_mocked +) from leapp.libraries.stdlib import api from leapp.models import ( - CopyFile, InstalledRPM, - RequiredTargetUserspacePackages, RHUIInfo, RPM, RpmTransactionTasks, + TargetRepositories, TargetRHUIPostInstallTasks, TargetRHUIPreInstallTasks, TargetRHUISetupInfo, TargetUserSpacePreupgradeTasks ) -from leapp.reporting import Report -from leapp.snactor.fixture import current_actor_context RH_PACKAGER = 'Red Hat, Inc. ' @@ -95,7 +108,8 @@ def mk_cloud_map(variants): ] ) def test_determine_rhui_src_variant(monkeypatch, extra_pkgs, rhui_setups, expected_result): - monkeypatch.setattr(api, 'current_actor', CurrentActorMocked(src_ver='7.9')) + actor = CurrentActorMocked(src_ver='7.9', config=_make_default_config(all_rhui_cfg)) + monkeypatch.setattr(api, 'current_actor', actor) installed_pkgs = {'zip', 'zsh', 'bash', 'grubby'}.union(set(extra_pkgs)) if expected_result and not isinstance(expected_result, RHUIFamily): # An exception @@ -167,7 +181,8 @@ def test_google_specific_customization(provider, should_mutate): ) def test_aws_specific_customization(monkeypatch, rhui_family, target_major, should_mutate): dst_ver = '{major}.0'.format(major=target_major) - monkeypatch.setattr(api, 'current_actor', CurrentActorMocked(dst_ver=dst_ver)) + actor = CurrentActorMocked(dst_ver=dst_ver, config=_make_default_config(all_rhui_cfg)) + monkeypatch.setattr(api, 'current_actor', actor) setup_info = mk_setup_info() checkrhui_lib.customize_rhui_setup_for_aws(rhui_family, setup_info) @@ -215,12 +230,12 @@ def produce_rhui_info_to_setup_target(monkeypatch): def test_produce_rpms_to_install_into_target(monkeypatch): - source_rhui_setup = mk_rhui_setup(clients={'src_pkg'}, leapp_pkg='leapp_pkg') - target_rhui_setup = mk_rhui_setup(clients={'target_pkg'}, leapp_pkg='leapp_pkg') + source_clients = {'src_pkg'} + target_clients = {'target_pkg'} monkeypatch.setattr(api, 'produce', produce_mocked()) - checkrhui_lib.produce_rpms_to_install_into_target(source_rhui_setup, target_rhui_setup) + checkrhui_lib.produce_rpms_to_install_into_target(source_clients, target_clients) assert len(api.produce.model_instances) == 2 userspace_tasks, target_rpm_tasks = api.produce.model_instances[0], api.produce.model_instances[1] @@ -276,7 +291,8 @@ def test_process(monkeypatch, extra_installed_pkgs, skip_rhsm, expected_action): installed_rpms = InstalledRPM(items=installed_pkgs) monkeypatch.setattr(api, 'produce', produce_mocked()) - monkeypatch.setattr(api, 'current_actor', CurrentActorMocked(src_ver='7.9', msgs=[installed_rpms])) + actor = CurrentActorMocked(src_ver='7.9', msgs=[installed_rpms], config=_make_default_config(all_rhui_cfg)) + monkeypatch.setattr(api, 'current_actor', actor) monkeypatch.setattr(reporting, 'create_report', create_report_mocked()) monkeypatch.setattr(rhsm, 'skip_rhsm', lambda: skip_rhsm) monkeypatch.setattr(rhui, 'RHUI_SETUPS', known_setups) @@ -315,7 +331,8 @@ def test_unknown_target_rhui_setup(monkeypatch, is_target_setup_known): installed_rpms = InstalledRPM(items=installed_pkgs) monkeypatch.setattr(api, 'produce', produce_mocked()) - monkeypatch.setattr(api, 'current_actor', CurrentActorMocked(src_ver='7.9', msgs=[installed_rpms])) + actor = CurrentActorMocked(src_ver='7.9', msgs=[installed_rpms], config=_make_default_config(all_rhui_cfg)) + monkeypatch.setattr(api, 'current_actor', actor) monkeypatch.setattr(reporting, 'create_report', create_report_mocked()) monkeypatch.setattr(rhsm, 'skip_rhsm', lambda: True) monkeypatch.setattr(rhui, 'RHUI_SETUPS', known_setups) @@ -374,3 +391,136 @@ def test_select_chronologically_closest(monkeypatch, setups, desired_minor, expe setup = setups[0] assert setup == expected_setup + + +def test_config_overwrites_everything(monkeypatch): + rhui_config = { + RhuiUseConfig.name: True, + RhuiSourcePkgs.name: ['client_source'], + RhuiTargetPkgs.name: ['client_target'], + RhuiCloudProvider.name: 'aws', + RhuiUpgradeFiles.name: { + '/root/file.repo': '/etc/yum.repos.d/' + }, + RhuiTargetRepositoriesToUse.name: [ + 'repoid_to_use' + ] + } + all_config = {'rhui': rhui_config} + + actor = CurrentActorMocked(config=all_config) + monkeypatch.setattr(api, 'current_actor', actor) + + function_calls = defaultdict(int) + + def mk_function_probe(fn_name): + def probe(*args, **kwargs): + function_calls[fn_name] += 1 + return probe + + monkeypatch.setattr(checkrhui_lib, + 'emit_rhui_setup_tasks_based_on_config', + mk_function_probe('emit_rhui_setup_tasks_based_on_config')) + monkeypatch.setattr(checkrhui_lib, + 'stop_with_err_if_config_missing_fields', + mk_function_probe('stop_with_err_if_config_missing_fields')) + monkeypatch.setattr(checkrhui_lib, + 'produce_rpms_to_install_into_target', + mk_function_probe('produce_rpms_to_install_into_target')) + monkeypatch.setattr(checkrhui_lib, + 'request_configured_repos_to_be_enabled', + mk_function_probe('request_configured_repos_to_be_enabled')) + + checkrhui_lib.process() + + expected_function_calls = { + 'emit_rhui_setup_tasks_based_on_config': 1, + 'stop_with_err_if_config_missing_fields': 1, + 'produce_rpms_to_install_into_target': 1, + 'request_configured_repos_to_be_enabled': 1, + } + + assert function_calls == expected_function_calls + + +def test_request_configured_repos_to_be_enabled(monkeypatch): + monkeypatch.setattr(api, 'produce', produce_mocked()) + + rhui_config = { + RhuiUseConfig.name: True, + RhuiSourcePkgs.name: ['client_source'], + RhuiTargetPkgs.name: ['client_target'], + RhuiCloudProvider.name: 'aws', + RhuiUpgradeFiles.name: { + '/root/file.repo': '/etc/yum.repos.d/' + }, + RhuiTargetRepositoriesToUse.name: [ + 'repoid1', + 'repoid2', + 'repoid3', + ] + } + + checkrhui_lib.request_configured_repos_to_be_enabled(rhui_config) + + assert api.produce.called + assert len(api.produce.model_instances) == 1 + + target_repos = api.produce.model_instances[0] + assert isinstance(target_repos, TargetRepositories) + assert not target_repos.rhel_repos + + custom_repoids = sorted(custom_repo_model.repoid for custom_repo_model in target_repos.custom_repos) + assert custom_repoids == ['repoid1', 'repoid2', 'repoid3'] + + +@pytest.mark.parametrize( + ('upgrade_files', 'existing_files'), + ( + (['/root/a', '/root/b'], ['/root/a', '/root/b']), + (['/root/a', '/root/b'], ['/root/b']), + (['/root/a', '/root/b'], []), + ) +) +def test_missing_files_in_config(monkeypatch, upgrade_files, existing_files): + upgrade_files_map = dict((source_path, '/tmp/dummy') for source_path in upgrade_files) + + rhui_config = { + RhuiUseConfig.name: True, + RhuiSourcePkgs.name: ['client_source'], + RhuiTargetPkgs.name: ['client_target'], + RhuiCloudProvider.name: 'aws', + RhuiCloudVariant.name: 'ordinary', + RhuiUpgradeFiles.name: upgrade_files_map, + RhuiTargetRepositoriesToUse.name: [ + 'repoid_to_use' + ] + } + + monkeypatch.setattr(os.path, 'exists', lambda path: path in existing_files) + monkeypatch.setattr(api, 'produce', produce_mocked()) + + should_error = (len(upgrade_files) != len(existing_files)) + if should_error: + with pytest.raises(StopActorExecutionError): + checkrhui_lib.emit_rhui_setup_tasks_based_on_config(rhui_config) + else: + checkrhui_lib.emit_rhui_setup_tasks_based_on_config(rhui_config) + assert api.produce.called + assert len(api.produce.model_instances) == 1 + + rhui_info = api.produce.model_instances[0] + assert isinstance(rhui_info, RHUIInfo) + assert rhui_info.provider == 'aws' + assert rhui_info.variant == 'ordinary' + assert rhui_info.src_client_pkg_names == ['client_source'] + assert rhui_info.target_client_pkg_names == ['client_target'] + + setup_info = rhui_info.target_client_setup_info + assert not setup_info.bootstrap_target_client + + _copies_to_perform = setup_info.preinstall_tasks.files_to_copy_into_overlay + copies_to_perform = sorted((copy.src, copy.dst) for copy in _copies_to_perform) + expected_copies = sorted(zip(upgrade_files, itertools.repeat('/tmp/dummy'))) + + assert copies_to_perform == expected_copies From 2b818a72d12e8916a1f7557e6462742156c3d126 Mon Sep 17 00:00:00 2001 From: Michal Hecko Date: Wed, 16 Oct 2024 17:38:36 +0200 Subject: [PATCH 7/8] testutils: add support for configs Extend the CurrentActorMocked class to accept a `config` value, allowing developers to mock actors that rely on configuration. A library function `_make_default_config` is also introduced, allowing to instantiate default configs from config schemas. --- repos/system_upgrade/common/libraries/testutils.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/repos/system_upgrade/common/libraries/testutils.py b/repos/system_upgrade/common/libraries/testutils.py index c538af1a13..afeb360a64 100644 --- a/repos/system_upgrade/common/libraries/testutils.py +++ b/repos/system_upgrade/common/libraries/testutils.py @@ -4,6 +4,7 @@ from collections import namedtuple from leapp import reporting +from leapp.actors.config import _normalize_config, normalize_schemas from leapp.libraries.common.config import architecture from leapp.models import EnvVar from leapp.utils.deprecation import deprecated @@ -67,9 +68,15 @@ def __call__(self): return self +def _make_default_config(actor_config_schema): + """ Make a config dict populated with default values. """ + merged_schema = normalize_schemas((actor_config_schema, )) + return _normalize_config({}, merged_schema) # Will fill default values during normalization + + class CurrentActorMocked(object): # pylint:disable=R0904 def __init__(self, arch=architecture.ARCH_X86_64, envars=None, kernel='3.10.0-957.43.1.el7.x86_64', - release_id='rhel', src_ver='7.8', dst_ver='8.1', msgs=None, flavour='default'): + release_id='rhel', src_ver='7.8', dst_ver='8.1', msgs=None, flavour='default', config=None): envarsList = [EnvVar(name=k, value=v) for k, v in envars.items()] if envars else [] version = namedtuple('Version', ['source', 'target'])(src_ver, dst_ver) release = namedtuple('OS_release', ['release_id', 'version_id'])(release_id, src_ver) @@ -82,6 +89,7 @@ def __init__(self, arch=architecture.ARCH_X86_64, envars=None, kernel='3.10.0-95 'configuration', ['architecture', 'kernel', 'leapp_env_vars', 'os_release', 'version', 'flavour'] )(arch, kernel, envarsList, release, version, flavour) self._msgs = msgs or [] + self.config = {} if config is None else config def __call__(self): return self From 37e9f77f85548adf4d30e4bfb35c6a3952f1e6cd Mon Sep 17 00:00:00 2001 From: Michal Hecko Date: Sun, 20 Oct 2024 16:08:49 +0200 Subject: [PATCH 8/8] userspacegen(rhui): remove repofiles only if now owned by an RPM We copy files into the target userspace when setting up target repository content. If this file is named equally as some of the files installed by the target RHUI client installed during early phases of target userspace setup process, we would delete it in cleanup. Therefore, if we copy a repofile named /etc/yum.repos.d/X.repo and the target client also owns a file /etc/yum.repos.d/X.repo, we would remove it, making the container loose access to target content. This patch prevents us from blindly deleting files, keeping files that are owned by some RPM (usually that would be the target RHUI client). --- .../libraries/userspacegen.py | 30 ++++++++++++++----- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/repos/system_upgrade/common/actors/targetuserspacecreator/libraries/userspacegen.py b/repos/system_upgrade/common/actors/targetuserspacecreator/libraries/userspacegen.py index d7698056f5..12736ab738 100644 --- a/repos/system_upgrade/common/actors/targetuserspacecreator/libraries/userspacegen.py +++ b/repos/system_upgrade/common/actors/targetuserspacecreator/libraries/userspacegen.py @@ -1120,6 +1120,27 @@ def _get_target_userspace(): return constants.TARGET_USERSPACE.format(get_target_major_version()) +def _remove_injected_repofiles_from_our_rhui_packages(target_userspace_ctx, rhui_setup_info): + target_userspace_path = _get_target_userspace() + for copy in rhui_setup_info.preinstall_tasks.files_to_copy_into_overlay: + dst_in_container = get_copy_location_from_copy_in_task(target_userspace_path, copy) + dst_in_container = dst_in_container.strip('/') + dst_in_host = os.path.join(target_userspace_path, dst_in_container) + + if os.path.isfile(dst_in_host) and dst_in_host.endswith('.repo'): + # The repofile might have been replaced by a new one provided by the RHUI client if names collide + # Performance: Do the query here and not earlier, because we would be running rpm needlessly + try: + path_with_root = '/' + dst_in_container + target_userspace_ctx.call(['rpm', '-q', '--whatprovides', path_with_root]) + api.current_logger().debug('Repofile {0} kept as it is owned by some RPM.'.format(dst_in_host)) + except CalledProcessError: + # rpm exists with 1 if the file is not owned by any RPM. We might be catching all kinds of other + # problems here, but still better than always removing repofiles. + api.current_logger().debug('Removing repofile - not owned by any RPM: {0}'.format(dst_in_host)) + os.remove(dst_in_host) + + def _create_target_userspace(context, indata, packages, files, target_repoids): """Create the target userspace.""" target_path = _get_target_userspace() @@ -1139,14 +1160,7 @@ def _create_target_userspace(context, indata, packages, files, target_repoids): ) setup_info = indata.rhui_info.target_client_setup_info if not setup_info.bootstrap_target_client: - target_userspace_path = _get_target_userspace() - for copy in setup_info.preinstall_tasks.files_to_copy_into_overlay: - dst_in_container = get_copy_location_from_copy_in_task(target_userspace_path, copy) - dst_in_container = dst_in_container.strip('/') - dst_in_host = os.path.join(target_userspace_path, dst_in_container) - if os.path.isfile(dst_in_host) and dst_in_host.endswith('.repo'): - api.current_logger().debug('Removing repofile: {0}'.format(dst_in_host)) - os.remove(dst_in_host) + _remove_injected_repofiles_from_our_rhui_packages(context, setup_info) # and do not forget to set the rhsm into the container mode again with mounting.NspawnActions(_get_target_userspace()) as target_context: