From 7d4dd357346b751f13fc561b8df541e9d808ed9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcus=20Sch=C3=A4fer?= Date: Sun, 22 Sep 2024 11:02:39 +0200 Subject: [PATCH] Fix/Refactor DMS packaging Add tox target to test implementation against py 3.6 and 3.11 stack Actually also wanted to add py3.4 but none of the github runners offers such an old python stack Add mypy tests. This is enabled to make sure we get notified if static type checking is enabled in the code. In this case we will be able to check if the affected code part is allowed to use type checking because the pre-checks python code must not use this feature of python as it is expected to run on SLE12 with py3.4 Fixup the overall python packaging. The former version of the package stack duplicates the module code in conflicting packages and also packaged python byte code binaries. This commit refactors the packaging such that the python module code is packages as a proper %{pythons}-migration package which can build for different python versions. The spec file change is done in a similar way to allow SLE12 and SLE15 builds as we do it for other PCT python modules. The respective DMS packages suse-migration-services, suse-migration-pre-checks can now simply require the DMS python stack and package only the main entry-point commands and services. This also allows to add further implementations to the DMS without ending in a package nightmare Fixup unit tests to finally get back to 100% test coverage as it was before I left the project. I'm not sure who tought it would be a good idea to weaken --cov-fail-under --- .github/workflows/ci-testing.yml | 38 ++--- .virtualenv.dev-requirements.txt | 1 + Makefile | 2 +- package/suse-migration-services-spec-template | 133 ++++++++---------- pip.sh | 3 - setup.cfg | 3 + test/data/migration-config.yml | 6 +- test/unit/logger_test.py | 4 +- tox.ini | 48 +++++-- 9 files changed, 122 insertions(+), 116 deletions(-) delete mode 100755 pip.sh diff --git a/.github/workflows/ci-testing.yml b/.github/workflows/ci-testing.yml index c3e5c7d9..2d66d6a3 100644 --- a/.github/workflows/ci-testing.yml +++ b/.github/workflows/ci-testing.yml @@ -3,24 +3,28 @@ name: CILint on: [push] jobs: - build: - - #runs-on: ubuntu-latest - runs-on: ubuntu-20.04 + unit_tests: + name: Unit tests for DMS + runs-on: ${{ matrix.os }} strategy: matrix: - python-version: [3.6, 3.8] + python-version: ["3.11"] + os: [ubuntu-latest] + include: + # Use ubuntu 20.04 for python 3.6 (dropped from later versions) + - python-version: 3.6 + os: ubuntu-20.04 steps: - - uses: actions/checkout@v2 - - name: Python${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Install - run: | - python -m pip install --upgrade pip - python -m pip install tox==2.9.1 - - name: Tox - run: | - tox + - uses: actions/checkout@v3 + - name: Python${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install + run: | + python -m pip install --upgrade pip + python -m pip install tox + - name: Tox + run: | + tox diff --git a/.virtualenv.dev-requirements.txt b/.virtualenv.dev-requirements.txt index d5fd27b1..5ac532b4 100644 --- a/.virtualenv.dev-requirements.txt +++ b/.virtualenv.dev-requirements.txt @@ -10,6 +10,7 @@ mock # Python style guide checker flake8 +mypy # Version-bump your software with a single command! bumpversion diff --git a/Makefile b/Makefile index 01fc08a3..b4eedbf1 100644 --- a/Makefile +++ b/Makefile @@ -64,7 +64,7 @@ sle15_activation: check .PHONY: test test: - tox -e unit_py3 + tox check: tox -e check diff --git a/package/suse-migration-services-spec-template b/package/suse-migration-services-spec-template index 1ddf9cd4..a124dfce 100644 --- a/package/suse-migration-services-spec-template +++ b/package/suse-migration-services-spec-template @@ -1,7 +1,7 @@ # # spec file for package suse-migration-services # -# Copyright (c) 2018 SUSE LINUX GmbH, Nuernberg, Germany. +# Copyright (c) 2024 SUSE LLC # # All modifications and additions to the file contributed by third parties # remain the property of their copyright owners, unless otherwise agreed @@ -11,10 +11,15 @@ # case the license is the MIT License). An "Open Source License" is a # license that conforms to the Open Source Definition (Version 1.9) # published by the Open Source Initiative. - -# Please submit bugfixes or comments via http://bugs.opensuse.org/ # - +# Please submit bugfixes or comments via https://bugs.opensuse.org/ +# +%if 0%{?suse_version} >= 1600 +%define pythons %{primary_python} +%else +%define pythons python3 +%endif +%global _sitelibdir %{%{pythons}_sitelib} Name: suse-migration-services Version: %%VERSION @@ -25,20 +30,27 @@ License: GPL-3.0+ Group: System/Management Source: suse-migration-services.tar.gz BuildRoot: %{_tmppath}/%{name}-%{version}-build -Conflicts: suse-migration-sle15-activation -BuildRequires: python3-devel -BuildRequires: python3-setuptools +BuildRequires: fdupes +BuildRequires: python-rpm-macros +BuildRequires: %{pythons}-setuptools +BuildRequires: %{pythons}-Cerberus +BuildRequires: %{pythons}-PyYAML +%if 0%{?suse_version} >= 1600 +BuildRequires: %{pythons}-pip +BuildRequires: %{pythons}-wheel +%endif BuildRequires: systemd-rpm-macros -Requires: suse-migration-pre-checks -Requires: python3-Cerberus -Requires: python3-PyYAML -Requires: python3-setuptools +Requires: %{pythons}-Cerberus +Requires: %{pythons}-PyYAML +Requires: %{pythons}-setuptools +Requires: %{pythons}-migration Requires: util-linux Requires: kexec-tools Requires: ca-certificates Requires: dialog Requires: rsync Requires: suseconnect-ng +Requires: suse-migration-services = %{version}-%{release} Requires(preun): systemd Requires(postun): systemd BuildArch: noarch @@ -46,11 +58,17 @@ BuildArch: noarch %description Systemd services to prepare and run a distribution migration process. +%package -n python3-migration +Summary: The systemd services for python3-migration +Group: System/Management + +%description -n python3-migration +Python based implementation for suse major code stream upgrade system + %package -n suse-migration-pre-checks -Summary: The pre-checks code used with suse-migration-services +Summary: The pre-checks code used with python3-migration Group: System/Management -Requires: python3-Cerberus -Requires: python3-PyYAML +Requires: %{pythons}-migration Conflicts: suse-migration-sle15-activation < 2.0.33 %description -n suse-migration-pre-checks @@ -60,10 +78,18 @@ Checks to be performed before starting a distribution migration process %setup -q -n suse_migration_services-%{version} %build -python3 setup.py build +%if 0%{?suse_version} >= 1600 +%pyproject_wheel +%else +%{pythons} setup.py build +%endif %install -python3 setup.py install --prefix=%{_prefix} --root=%{buildroot} +%if 0%{?suse_version} >= 1600 +%pyproject_install +%else +%{pythons} setup.py install --prefix=%{_prefix} --root=%{buildroot} +%endif install -D -m 755 tools/run_migration \ %{buildroot}%{_sbindir}/run_migration @@ -118,83 +144,38 @@ install -D -m 644 systemd/suse-migration-console-log.service \ # need for use of the "standard" systemd service handling macros. %files -%defattr(-,root,root,-) -%{python3_sitelib}/suse_migration_services/suse_connect.py -%{python3_sitelib}/suse_migration_services/units -%{python3_sitelib}/suse_migration_services/__pycache__/migration_config.cpython-*.pyc -%{python3_sitelib}/suse_migration_services/__pycache__/path.cpython-*.pyc -%{python3_sitelib}/suse_migration_services/__pycache__/schema.cpython-*.pyc -%{python3_sitelib}/suse_migration_services/__pycache__/suse_connect.cpython-*.pyc -%{python3_sitelib}/suse_migration_services/__pycache__/suse_product.cpython-*.pyc - %{_sbindir}/run_migration - %{_bindir}/suse-migration-ssh-keys -%{_unitdir}/suse-migration-ssh-keys.service - %{_bindir}/suse-migration-mount-system -%{_unitdir}/suse-migration-mount-system.service - -# The %{_bindir}/suse-migration-pre-checks is -# installed from the suse-migration-pre-checks package -%{_unitdir}/suse-migration-pre-checks.service - %{_bindir}/suse-migration-post-mount-system -%{_unitdir}/suse-migration-post-mount-system.service - %{_bindir}/suse-migration-setup-host-network -%{_unitdir}/suse-migration-setup-host-network.service - %{_bindir}/suse-migration-prepare -%{_unitdir}/suse-migration-prepare.service - %{_bindir}/suse-migration +%{_bindir}/suse-migration-grub-setup +%{_bindir}/suse-migration-update-bootloader +%{_bindir}/suse-migration-product-setup +%{_bindir}/suse-migration-regenerate-initrd +%{_bindir}/suse-migration-kernel-load +%{_bindir}/suse-migration-reboot +%{_unitdir}/suse-migration-ssh-keys.service +%{_unitdir}/suse-migration-mount-system.service +%{_unitdir}/suse-migration-post-mount-system.service +%{_unitdir}/suse-migration-setup-host-network.service +%{_unitdir}/suse-migration-prepare.service %{_unitdir}/suse-migration.service %{_unitdir}/suse-migration-console-log.service - -%{_bindir}/suse-migration-grub-setup %{_unitdir}/suse-migration-grub-setup.service - -%{_bindir}/suse-migration-update-bootloader %{_unitdir}/suse-migration-update-bootloader.service - -%{_bindir}/suse-migration-product-setup %{_unitdir}/suse-migration-product-setup.service - -%{_bindir}/suse-migration-regenerate-initrd %{_unitdir}/suse-migration-regenerate-initrd.service - -%{_bindir}/suse-migration-kernel-load %{_unitdir}/suse-migration-kernel-load.service - -%{_bindir}/suse-migration-reboot %{_unitdir}/suse-migration-reboot.service +%files -n python3-migration +%{python_sitelib}/* + %files -n suse-migration-pre-checks -%defattr(-,root,root,-) -%{python3_sitelib}/suse_migration_services/command.py -%{python3_sitelib}/suse_migration_services/defaults.py -%{python3_sitelib}/suse_migration_services/exceptions.py -%{python3_sitelib}/suse_migration_services/fstab.py -%{python3_sitelib}/suse_migration_services/logger.py -%{python3_sitelib}/suse_migration_services/migration_config.py -%{python3_sitelib}/suse_migration_services/path.py -%{python3_sitelib}/suse_migration_services/schema.py -%{python3_sitelib}/suse_migration_services/suse_product.py -%{python3_sitelib}/suse_migration_services/version.py -%{python3_sitelib}/suse_migration_services/__init__.py -%{python3_sitelib}/suse_migration_services/prechecks %{_bindir}/suse-migration-pre-checks -%{python3_sitelib}/suse_migration_services-* -%{python3_sitelib}/suse_migration_services/__pycache__/__init__.cpython-*.pyc -%{python3_sitelib}/suse_migration_services/__pycache__/command.cpython-*.pyc -%{python3_sitelib}/suse_migration_services/__pycache__/defaults.cpython-*.pyc -%{python3_sitelib}/suse_migration_services/__pycache__/exceptions.cpython-*.pyc -%{python3_sitelib}/suse_migration_services/__pycache__/fstab.cpython-*.pyc -%{python3_sitelib}/suse_migration_services/__pycache__/logger.cpython-*.pyc -%{python3_sitelib}/suse_migration_services/__pycache__/version.cpython-*.pyc -%{python3_sitelib}/suse_migration_services-*/ -%dir %{python3_sitelib}/suse_migration_services -%dir %{python3_sitelib}/suse_migration_services/__pycache__ +%{_unitdir}/suse-migration-pre-checks.service %changelog diff --git a/pip.sh b/pip.sh deleted file mode 100755 index bb9f89fe..00000000 --- a/pip.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -pip install --upgrade pip -pip install "$@" diff --git a/setup.cfg b/setup.cfg index 4a829d13..1556af8f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -10,3 +10,6 @@ testpaths = test/unit/ exclude = .tox*/* # we allow long lines ignore = E501 + +[mypy] +ignore_missing_imports = True diff --git a/test/data/migration-config.yml b/test/data/migration-config.yml index f2410d16..e2f3f2f6 100644 --- a/test/data/migration-config.yml +++ b/test/data/migration-config.yml @@ -1,6 +1,6 @@ preserve: rules: - - /etc/udev/rules.d/a.rules - - /etc/udev/rules.d/b.rules + - /etc/udev/rules.d/a.rules + - /etc/udev/rules.d/b.rules static: - - /etc/sysconfig/proxy + - /etc/sysconfig/proxy diff --git a/test/unit/logger_test.py b/test/unit/logger_test.py index 1c697747..2b3c90d8 100644 --- a/test/unit/logger_test.py +++ b/test/unit/logger_test.py @@ -17,7 +17,7 @@ def test_setup(self, mock_Path_create): '/system-root/var/log' ) mock_open.assert_called_once_with( - '/system-root/var/log/distro_migration.log', 'a', encoding=None + '/system-root/var/log/distro_migration.log', 'a', encoding='locale', errors=None ) def test_setup_no_system_root(self, mock_Path_create): @@ -29,5 +29,5 @@ def test_setup_no_system_root(self, mock_Path_create): '/var/log' ) mock_open.assert_called_once_with( - '/var/log/distro_migration.log', 'a', encoding=None + '/var/log/distro_migration.log', 'a', encoding='locale', errors=None ) diff --git a/tox.ini b/tox.ini index 963e5f4c..89adb5e8 100644 --- a/tox.ini +++ b/tox.ini @@ -1,19 +1,30 @@ [tox] +minversion = 3.3.0 +isolated_build = True +skip_missing_interpreters = True skipsdist = True envlist = - unit_py3, check + check, + unit_py3_11 [testenv] -install_command = {toxinidir}/pip.sh {opts} {packages} -whitelist_externals = +description = + {unit_py3_11}: Unit Test run with basepython set to {basepython} +allowlist_externals = bash + travis-sphinx + cp + make + rm + mv + flake8 + python + pytest basepython = - {check}: python3 - unit_py3: python3 -envdir = - {check}: {toxworkdir}/3 - unit_py3: {toxworkdir}/3 + {check,devel}: python3 + unit_py3_11: python3.11 + release: python3.11 passenv = * usedevelop = True @@ -21,12 +32,20 @@ deps = -r.virtualenv.dev-requirements.txt -# Unit Test run with basepython set to 3.x -[testenv:unit_py3] +# Unit Test run with basepython set to 3.11 +[testenv:unit_py3_11] +setenv = + PYTHONPATH={toxinidir}/test +changedir=test/unit +commands = + {[testenv:unit]commands} + + +[testenv:unit] +description = Unit Test Base skip_install = True usedevelop = True setenv = - PYTHONPATH={toxinidir}/test PYTHONUNBUFFERED=yes WITH_COVERAGE=yes passenv = @@ -35,9 +54,10 @@ deps = {[testenv]deps} changedir=test/unit commands = bash -c 'cd ../../ && ./setup.py develop' - py.test --no-cov-on-fail --cov=suse_migration_services \ - --cov-report=term-missing --cov-fail-under=95 --cov-config .coveragerc - + bash -c 'cd ../../ && mypy --install-types --non-interactive suse_migration_services' + pytest --doctest-modules --no-cov-on-fail --cov=suse_migration_services \ + --cov-report=term-missing --cov-fail-under=100 \ + --cov-config .coveragerc {posargs} # Source code quality/integrity check