Merge branch 'develop' into fs_mount_opts

.github/workflows/main.yml

@@ -44,9 +44,9 @@ jobs:
       uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
+    - name: Install dependencies and docs
       run: |
-        ./bootstrap.sh
+        ./bootstrap.sh +docs
     - name: Generic Unittests
       run: |
         ./test_reframe.py
@@ -140,12 +140,15 @@ jobs:
 
   docvalidation:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
     steps:
       - uses: actions/checkout@v3
-      - name: Setup up Python 3.8
+      - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v4
         with:
-          python-version: 3.8
+          python-version: ${{ matrix.python-version }}
       - name: Install Doc Requirements
         run: |
           python -m pip install -r docs/requirements.txt

.github/workflows/test-flux.yaml

@@ -10,7 +10,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        container: ['fluxrm/flux-sched:focal']
+        container: ['fluxrm/flux-sched:focal-v0.28.0']
 
     container:
       image: ${{ matrix.container }}

docs/config_reference.rst

@@ -260,8 +260,9 @@ System Partition Configuration
    The job scheduler that will be used to launch jobs on this partition.
    Supported schedulers are the following:
 
-   - ``local``: Jobs will be launched locally without using any job scheduler.
    - ``flux``: Jobs will be launched using the `Flux Framework <https://flux-framework.org/>`_ scheduler.
+   - ``local``: Jobs will be launched locally without using any job scheduler.
+   - ``lsf``: Jobs will be launched using the `LSF <https://www.ibm.com/docs/en/spectrum-lsf/10.1.0?topic=lsf-session-scheduler>`__ scheduler.
    - ``oar``: Jobs will be launched using the `OAR <https://oar.imag.fr/>`__ scheduler.
    - ``pbs``: Jobs will be launched using the `PBS Pro <https://en.wikipedia.org/wiki/Portable_Batch_System>`__ scheduler.
    - ``sge``: Jobs will be launched using the `Sun Grid Engine <https://arc.liv.ac.uk/SGE/htmlman/manuals.html>`__ scheduler.
@@ -270,8 +271,24 @@ System Partition Configuration
      If not, you should consider using the ``squeue`` backend below.
    - ``squeue``: Jobs will be launched using the `Slurm <https://www.schedmd.com/>`__ scheduler.
      This backend does not rely on job accounting to retrieve job statuses, but ReFrame does its best to query the job state as reliably as possible.
+   - ``ssh``: Jobs will be launched on a remote host using SSH.
+
+     The remote host will be selected from the list of hosts specified in :attr:`~systems.partitions.sched_options.ssh_hosts`.
+     The scheduler keeps track of the hosts that it has submitted jobs to, and it will select the next available one in a round-robin fashion.
+     For connecting to a remote host, the options specified in :attr:`~systems.partitions.access` will be used.
+
+     When a job is submitted with this scheduler, its stage directory will be copied over to a unique temporary directory on the remote host, then the job will be executed and, finally, any produced artifacts will be copied back.
+
+     The contents of the stage directory are copied to the remote host either using ``rsync``, if available, or ``scp`` as a second choice.
+     The same :attr:`~systems.partitions.access` options will be used in those operations as well.
+     Please note, that the connection options of ``ssh`` and ``scp`` differ and ReFrame will not attempt to translate any options between the two utilities in case ``scp`` is selected for copying to the remote host.
+     In this case, it is preferable to set up the host connection options in ``~/.ssh/config`` and leave :attr:`~systems.partition.access` blank.
+
+     Job-scheduler command line options can be used to interact with the ``ssh`` backend.
+     More specifically, if the :option:`--distribute` option is used, a test will be generated for each host listed in :attr:`~systems.partitions.sched_options.ssh_hosts`.
+     You can also pin a test to a specific host if you pass the ``#host`` directive to the :option:`-J` option, e.g., ``-J '#host=myhost'``.
+
    - ``torque``: Jobs will be launched using the `Torque <https://en.wikipedia.org/wiki/TORQUE>`__ scheduler.
-   - ``lsf``: Jobs will be launched using the `LSF <https://www.ibm.com/docs/en/spectrum-lsf/10.1.0?topic=lsf-session-scheduler>`__ scheduler.
 
    .. versionadded:: 3.7.2
       Support for the SGE scheduler is added.
@@ -282,6 +299,9 @@ System Partition Configuration
    .. versionadded:: 3.11.0
       Support for the LSF scheduler is added.
 
+   .. versionadded:: 4.4
+      The ``ssh`` scheduler is added.
+
    .. note::
 
       The way that multiple node jobs are submitted using the SGE scheduler can be very site-specific.
@@ -337,6 +357,14 @@ System Partition Configuration
    .. warning::
       This option is broken in 4.0.
 
+.. py:attribute:: systems.partitions.sched_options.ssh_hosts
+
+   :required: No
+   :default: ``[]``
+
+   List of hosts in a partition that uses the ``ssh`` scheduler.
+
+
 .. py:attribute:: systems.partitions.sched_options.ignore_reqnodenotavail
 
    :required: No
@@ -1095,6 +1123,9 @@ All logging handlers share the following set of common attributes:
 .. py:attribute:: logging.handlers.format_perfvars
 .. py:attribute:: logging.handlers_perflog.format_perfvars
 
+   :required: No
+   :default: ``""``
+
    Format specifier for logging the performance variables.
 
    This defines how the ``%(check_perfvalues)s`` will be formatted.
@@ -1216,9 +1247,9 @@ The additional properties for the ``filelog`` handler are the following:
      {basedir}/
         system1/
             partition1/
-                test_short_name.log
+                <test_class_name>.log
             partition2/
-                test_short_name.log
+                <test_class_name>.log
             ...
         system2/
         ...
@@ -1238,6 +1269,12 @@ The additional properties for the ``filelog`` handler are the following:
    Examples of changes in the logged information are when the log record format changes or a new performance metric is added, deleted or has its name changed.
    This behavior guarantees that each log file is consistent and it will not break existing parsers.
 
+.. versionchanged:: 4.3
+
+   In the generated log file, the name of the test class name is used instead of the test's short name (which included the test's hash).
+   This allows the results of different variants of a parameterized test to be stored in the same log file facilitating post-processing.
+
+
 The ``graylog`` log handler
 ---------------------------
 

docs/manpage.rst

@@ -68,6 +68,21 @@ This happens recursively so that if test ``T1`` depends on ``T2`` and ``T2`` dep
    The value of this attribute is not required to be non-zero for GPU tests.
    Tests may or may not make use of it.
 
+   .. deprecated:: 4.4
+
+      Please use ``-E 'not num_gpus_per_node'`` instead.
+
+.. option:: -E, --filter-expr=EXPR
+
+   Select only tests that satisfy the given expression.
+
+   The expression ``EXPR`` can be any valid Python expression on the test variables or parameters.
+   For example, ``-E num_tasks > 10`` will select all tests, whose :attr:`~reframe.core.pipeline.RegressionTest.num_tasks` exceeds ``10``.
+   You may use any test variable in expression, even user-defined.
+   Multiple variables can also be included such as ``-E num_tasks >= my_param``, where ``my_param`` is user-defined parameter.
+
+   .. versionadded:: 4.4
+
 .. option:: --failed
 
    Select only the failed test cases for a previous run.
@@ -77,13 +92,18 @@ This happens recursively so that if test ``T1`` depends on ``T2`` and ``T2`` dep
 
    .. versionadded:: 3.4
 
+
 .. option:: --gpu-only
 
    Select tests that can run on GPUs.
 
    These are all tests with :attr:`num_gpus_per_node` greater than zero.
    This option and :option:`--cpu-only` are mutually exclusive.
 
+   .. deprecated:: 4.4
+
+      Please use ``-E num_gpus_per_node`` instead.
+
 .. option:: --maintainer=MAINTAINER
 
    Filter tests by maintainer.
@@ -101,6 +121,7 @@ This happens recursively so that if test ``T1`` depends on ``T2`` and ``T2`` dep
       The ``MAINTAINER`` pattern is matched anywhere in the maintainer's name and not at its beginning.
       If you want to match at the beginning of the name, you should prepend ``^``.
 
+
 .. option:: -n, --name=NAME
 
    Filter tests by name.
@@ -650,6 +671,9 @@ Options controlling ReFrame execution
    - Sequence types: ``-S seqvar=1,2,3,4``
    - Mapping types: ``-S mapvar=a:1,b:2,c:3``
 
+   Nested mapping types can also be converted using JSON syntax.
+   For example, the :attr:`~reframe.core.pipeline.RegressionTest.extra_resources` complex dictionary could be set with ``-S extra_resources='{"gpu": {"num_gpus_per_node":8}}'``.
+
    Conversions to arbitrary objects are also supported.
    See :class:`~reframe.utility.typecheck.ConvertibleType` for more details.
 
@@ -705,6 +729,9 @@ Options controlling ReFrame execution
 
       Allow setting variables in fixtures.
 
+   .. versionchanged:: 4.4
+
+      Allow setting nested mapping types using JSON syntax.
 
 .. option:: --skip-performance-check
 

docs/regression_test_api.rst

@@ -67,9 +67,9 @@ The use of this module is required only when creating new tests programmatically
 
 .. autodecorator:: reframe.core.builtins.require_deps
 
-.. autodecorator:: reframe.core.builtins.run_after(stage)
+.. autodecorator:: reframe.core.builtins.run_after(stage, *, always_last=False)
 
-.. autodecorator:: reframe.core.builtins.run_before(stage)
+.. autodecorator:: reframe.core.builtins.run_before(stage, *, always_last=False)
 
 .. autodecorator:: reframe.core.builtins.sanity_function
 

docs/requirements.txt

@@ -1,7 +1,9 @@
 archspec==0.2.1
-docutils==0.17.1 # https://github.com/sphinx-doc/sphinx/issues/9001
+docutils==0.18.1
 jsonschema==3.2.0
 semver==2.13.0; python_version == '3.6'
-semver==3.0.0; python_version >= '3.7'
-Sphinx==5.3.0
-sphinx-rtd-theme==1.2.1
+semver==3.0.1; python_version >= '3.7'
+Sphinx==5.3.0; python_version < '3.8'
+Sphinx==7.1.2; python_version == '3.8'
+Sphinx==7.2.6; python_version >= '3.9'
+sphinx-rtd-theme==1.3.0

hpctestlib/system/ssh/host_keys.py

@@ -0,0 +1,61 @@
+# Copyright 2016-2023 Swiss National Supercomputing Centre (CSCS/ETH Zurich)
+# ReFrame Project Developers. See the top-level LICENSE file for details.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+import time
+
+import reframe as rfm
+import reframe.utility.sanity as sn
+import reframe.utility.typecheck as typ
+
+
+@rfm.simple_test
+class ssh_host_keys_check(rfm.RunOnlyRegressionTest):
+    '''SSH host keys age check
+
+    The test checks if the list of SSH keys has been updated recently.
+    In this case, we are checking against the max_key_age variable
+    '''
+
+    #: Parameter list with all host keys to check
+    #:
+    #: The test is skipped if a key is not found
+    #:
+    #: :type: :class:`str`
+    #: :values: ``['/etc/ssh/ssh_host_rsa_key',
+    #:              '/etc/ssh/ssh_host_ecdsa_key',
+    #:              '/etc/ssh/ssh_host_ed25519_key']``
+    ssh_host_keys = parameter([
+        '/etc/ssh/ssh_host_rsa_key',
+        '/etc/ssh/ssh_host_ecdsa_key',
+        '/etc/ssh/ssh_host_ed25519_key',
+    ], fmt=lambda x: x.split('_')[2], loggable=True)
+
+    #: The max age of the keys in ReFrame duration format
+    #:
+    #: :type: :class:`str`
+    #: :default: ``'365d'``
+    max_key_age = variable(str, value='365d', loggable=True)
+
+    executable = 'stat'
+    executable_opts = ['-c', '%Y']
+    tags = {'system', 'ssh'}
+
+    @run_after('init')
+    def set_hosts_keys(self):
+        self.executable_opts += [self.ssh_host_keys]
+
+    @sanity_function
+    def assert_file_age(self):
+        current_time = time.time()
+
+        skip_me = sn.extractall('No such file or directory', self.stderr)
+        self.skip_if(skip_me, msg=f'Skipping test because {self.ssh_host_keys}'
+                                  f' was not found')
+
+        return sn.assert_lt(current_time -
+                         sn.extractsingle(r'\d+', self.stdout, 0, int),
+                         typ.Duration(self.max_key_age),
+                         msg=f'File {self.ssh_host_keys} is older than '
+                             f'{self.max_key_age}')

reframe/__init__.py

@@ -6,7 +6,7 @@
 import os
 import sys
 
-VERSION = '4.3.2'
+VERSION = '4.5.0-dev.0'
 INSTALL_PREFIX = os.path.normpath(
     os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
 )

reframe/core/backends.py

@@ -23,7 +23,8 @@
     'reframe.core.schedulers.pbs',
     'reframe.core.schedulers.oar',
     'reframe.core.schedulers.sge',
-    'reframe.core.schedulers.slurm'
+    'reframe.core.schedulers.slurm',
+    'reframe.core.schedulers.ssh'
 ]
 _schedulers = {}
 

reframe/core/builtins.py

@@ -37,7 +37,7 @@ def final(fn):
 
 # Hook-related builtins
 
-def run_before(stage):
+def run_before(stage, *, always_last=False):
     '''Attach the decorated function before a certain pipeline stage.
 
     The function will run just before the specified pipeline stage and it
@@ -47,14 +47,25 @@ def run_before(stage):
 
     :param stage: The pipeline stage where this function will be attached to.
         See :ref:`pipeline-hooks` for the list of valid stage values.
+
+    :param always_last: Run this hook always as the last one of the stage. In
+        a whole test hierarchy, only a single hook can be explicitly pinned at
+        the end of the same-stage sequence of hooks. If another hook is
+        declared as ``always_last`` in the same stage, an error will be
+        issued.
+
+    .. versionchanged:: 4.4
+       The ``always_last`` argument was added.
+
     '''
-    return hooks.attach_to('pre_' + stage)
+
+    return hooks.attach_to('pre_' + stage, always_last)
 
 
-def run_after(stage):
+def run_after(stage, *, always_last=False):
     '''Attach the decorated function after a certain pipeline stage.
 
-    This is analogous to :func:`~RegressionMixin.run_before`, except that the
+    This is analogous to :func:`run_before`, except that the
     hook will execute right after the stage it was attached to. This decorator
     also supports ``'init'`` as a valid ``stage`` argument, where in this
     case, the hook will execute right after the test is initialized (i.e.
@@ -81,7 +92,7 @@ def __init__(self):
        Add support for post-init hooks.
 
     '''
-    return hooks.attach_to('post_' + stage)
+    return hooks.attach_to('post_' + stage, always_last)
 
 
 require_deps = hooks.require_deps