Merge pull request #182 from feelpp/117-discoverer

resolves: 117 discoverer

.github/workflows/benchmark.yml

@@ -9,12 +9,19 @@ on:
             machines_config:
                 description: 'Machine related configurations'
                 required: True
+                default: 67504e9a4c9ccbdde21a46fe
             benchmark_config:
                 description: 'Applcation related configuration'
                 required: True
+                default: 67504e9a4c9ccbdde21a4701
+            plots_config:
+                description: 'Plots related configuration'
+                required: True
+                default: 675053424c9ccbdde21a470a
             girder_folder_id:
                 description: 'ID of the folder to upload to'
                 required: True
+                default: 67504ecd4c9ccbdde21a4704
 
 jobs:
 
@@ -68,25 +75,28 @@ jobs:
                 girder-download -gid $machine_cfg_id -o ./tmp/ -fn "machines_config.json"
             env:
                 GIRDER_API_KEY: ${{secrets.GIRDER}}
-        -   id: hpc-systems
-            name: Set HPC systems matrix
-            run: |
-                source .venv/bin/activate
-                matrix=$(hpc-dispatch -mp ./tmp/machines_config.json -o ./tmp/machines/)
-                echo $matrix
-                echo "matrix={ include : $matrix }" >> $GITHUB_OUTPUT
         -   name: Donwload benchmark configuration
             run: |
                 source .venv/bin/activate
 
                 if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
                     bench_cfg_id=${{ github.event.inputs.benchmark_config }};
+                    plots_cfg_id=${{ github.event.inputs.plots_config }};
                 elif [[ "${{ github.event_name}}" == "repository_dispatch" ]]; then
                     bench_cfg_id=${{ github.event.client_payload.benchmark_config }};
+                    plots_cfg_id=${{ github.event.client_payload.plots_config }};
                 fi
                 girder-download -gid $bench_cfg_id -o ./tmp/ -fn "benchmark_config.json"
+                girder-download -gid $plots_cfg_id -o ./tmp/ -fn "plots.json"
             env:
                 GIRDER_API_KEY: ${{secrets.GIRDER}}
+        -   id: hpc-systems
+            name: Set HPC systems matrix
+            run: |
+                source .venv/bin/activate
+                matrix=$(hpc-dispatch -mcp ./tmp/machines_config.json -mod ./tmp/machines/ -bcp ./tmp/benchmark_config.json -pcp ./tmp/plots.json)
+                echo $matrix
+                echo "matrix={ include : $matrix }" >> $GITHUB_OUTPUT
         -   name: pull_images
             run: |
                 source .venv/bin/activate
@@ -102,6 +112,7 @@ jobs:
                 name: config-artifacts
                 path: |
                     ./tmp/benchmark_config.json
+                    ./tmp/plots.json
                     ./tmp/machines/
 
     benchmark:
@@ -113,6 +124,7 @@ jobs:
         timeout-minutes: 7200
         name: ${{matrix.machine}}
         steps:
+        -   uses: actions/checkout@v4
         -   name: Download wheel
             uses: actions/download-artifact@v4
             with:
@@ -123,20 +135,13 @@ jobs:
             with:
                 name: config-artifacts
                 path: ./tmp/
-        -   name: Create Virtual Environment
-            run: |
-                python3 -m venv .venv
-                source .venv/bin/activate
-                pip3 install -r requirements.txt
         -   name: Execute benchmarks
-            run: |
-                source .venv/bin/activate
-                execute-benchmark -ec ./${{matrix.machine_cfg}} --config ./tmp/benchmark_config.json --move-results ./tmp/results/ -v
+            run: ${{matrix.submit_command}}
         -   name: Upload reframe report
             uses: actions/upload-artifact@v4
             with:
-                name: benchmark-results
-                path: ./tmp/results/
+                name: benchmark-results-${{matrix.machine}}
+                path: ${{matrix.reports_path}}
 
     results:
         runs-on: self-ubuntu-22.04
@@ -148,8 +153,9 @@ jobs:
         -   name: Download results
             uses: actions/download-artifact@v4
             with:
-                name: benchmark-results
+                pattern: benchmark-results-*
                 path: ./tmp/results/
+                merge-multiple: false
         -   name: Create Virtual Environment
             run: |
                 python3 -m venv .venv
@@ -168,28 +174,4 @@ jobs:
                 girder-upload --directory $new_foldername --girder_id $girder_upload_id
                 rm -r $new_foldername
             env:
-                GIRDER_API_KEY: ${{ secrets.GIRDER }}
-        -   name:  Reset reports
-            run: |
-                rm -r ./docs/modules/ROOT/pages/applications/
-                rm -r ./docs/modules/ROOT/pages/machines/
-                rm -r ./docs/modules/ROOT/pages/reports/
-                rm -r ./docs/modules/ROOT/pages/use_cases/
-                rm -r ./reports/
-        -   name: Render reports
-            run: |
-                source .venv/bin/activate
-                render-benchmarks
-            env:
-                GIRDER_API_KEY: ${{ secrets.GIRDER }}
-
-        -   name: Create Pull Request
-            uses: peter-evans/create-pull-request@v7
-            with:
-                title: "Add benchmark for ${{ needs.factory.outputs.executable_name }} - ${{ needs.factory.outputs.use_case }}"
-                body: |
-                    Auto-generated by [create-pull-request][1]
-                    [1]: https://github.com/peter-evans/create-pull-request
-                reviewers: JavierCladellas
-            env:
-                GITHUB_TOKEN: ${{ secrets.CR_PAT }}
+                GIRDER_API_KEY: ${{ secrets.GIRDER }}

config/toolbox_heat/thermal_bridges_case_3.json

@@ -1,17 +1,18 @@
 {
     "executable": "feelpp_toolbox_heat",
-    "output_directory": "{{machine.output_app_dir}}/toolboxes/heat/thermal_bridges_case_3",
-    "use_case_name": "thermal_bridges_case_3",
-    "timeout":"0-01:00:00",
+    "output_directory": "{{machine.output_app_dir}}/toolboxes/heat/ThermalBridgesENISO10211/Case3",
+    "use_case_name": "ThermalBridgesENISO10211",
+    "timeout":"0-00:10:00",
     "platforms": {
         "apptainer":{
             "image": {
-                "name":"{{machine.containers.apptainer.image_base_dir}}/feelpp.sif"
+                "name":"{{machine.containers.apptainer.image_base_dir}}/feelpp-noble.sif"
             },
             "input_dir":"/input_data/",
             "options": [
                 "--home {{machine.output_app_dir}}",
-                "--bind {{machine.input_dataset_base_dir}}/{{use_case_name}}/:{{platforms.apptainer.input_dir}}"
+                "--bind {{machine.input_dataset_base_dir}}/{{use_case_name}}/:{{platforms.apptainer.input_dir}}",
+                "--env OMP_NUM_THREADS=1"
             ],
             "append_app_option":[]
         },
@@ -21,15 +22,17 @@
         }
     },
     "options": [
-        "--config-files {{platforms.{{machine.platform}}.input_dir}}/case3.cfg",
+        "--config-files /usr/share/feelpp/data/testcases/toolboxes/heat/cases/Building/ThermalBridgesENISO10211/case3.cfg {{platforms.{{machine.platform}}.input_dir}}/{{parameters.solver.value}}.cfg",
         "--directory {{output_directory}}/{{instance}}",
         "--repository.case {{use_case_name}}",
-        "--fail-on-unknown-option 1",
         "--heat.scalability-save=1",
         "--repository.append.np 0",
         "--case.discretization {{parameters.discretization.value}}",
-        "--heat.json.patch='{\"op\": \"replace\",\"path\": \"/Meshes/heat/Import/filename\",\"value\": \"$cfgdir/{{parameters.meshes.value}}/case3_p{{parameters.nb_tasks.tasks.value}}.json\" }'"
+        "--heat.json.patch='{\"op\": \"replace\",\"path\": \"/Meshes/heat/Import/filename\",\"value\": \"{{platforms.{{machine.platform}}.input_dir}}/partitioning/case3/{{parameters.meshes.value}}/case3_p{{parameters.nb_tasks.tasks.value}}.json\" }'"
     ],
+    "env_variables":{
+        "OMP_NUM_THREADS":1
+    },
     "outputs": [
         {
             "filepath": "{{output_directory}}/{{instance}}/{{use_case_name}}/heat.measures/values.csv",
@@ -67,7 +70,7 @@
         {
             "name": "nb_tasks",
             "sequence": [
-                {"tasks":128,"nodes":1,"exclusive_access":true}
+                {"tasks":128,"tasks_per_node":128,"exclusive_access":true}
             ]
         },
         {
@@ -77,6 +80,10 @@
         {
             "name": "discretization",
             "sequence": ["P1"]
+        },
+        {
+            "name": "solver",
+            "sequence": ["gamg"]
         }
     ]
 }

pyproject.toml

@@ -45,7 +45,7 @@ render-benchmarks = "feelpp.benchmarking.report.__main__:main_cli"
 execute-benchmark = "feelpp.benchmarking.reframe.__main__:main_cli"
 girder-download = "feelpp.benchmarking.scripts.girder:download_cli"
 girder-upload = "feelpp.benchmarking.scripts.girder:upload_cli"
-hpc-dispatch = "feelpp.benchmarking.scripts.hpcSystems:parseHpcSystems_cli"
+hpc-dispatch = "feelpp.benchmarking.scripts.hpcSystems:hpcSystemDispatcher_cli"
 
 [tool.pytest.ini_options]
 minversion = "6.0"

requirements.txt

@@ -13,4 +13,5 @@ pandas
 nbmake
 traitlets
 tabulate
+typing-extensions>=4.12.2
 .

src/feelpp/benchmarking/reframe/__main__.py

@@ -33,6 +33,15 @@ def createReportFolder(self,executable,use_case):
 
         return str(self.report_folder_path)
 
+    def buildExecutionMode(self):
+        """Write the ReFrame execution flag depending on the parser arguments.
+            Examples are --dry-run or -r
+        """
+        if self.parser.args.dry_run:
+            return "--dry-run"
+        else:
+            return "-r"
+
     def buildCommand(self,timeout):
         assert self.report_folder_path is not None, "Report folder path not set"
         cmd = [
@@ -47,7 +56,7 @@ def buildCommand(self,timeout):
             f"-J '#SBATCH --time={timeout}'",
             f'--perflogdir={os.path.join(self.machine_config.reframe_base_dir,"logs")}',
             f'{"-"+"v"*self.parser.args.verbose  if self.parser.args.verbose else ""}',
-            '-r',
+            f'{self.buildExecutionMode()}'
         ]
         return ' '.join(cmd)
 
@@ -56,7 +65,7 @@ def main_cli():
     parser = Parser()
     parser.printArgs()
 
-    machine_reader = ConfigReader(parser.args.machine_config,MachineConfig)
+    machine_reader = ConfigReader(parser.args.machine_config,MachineConfig,dry_run=parser.args.dry_run)
     machine_reader.updateConfig()
 
     #Sets the cachedir and tmpdir directories for containers
@@ -82,7 +91,7 @@ def main_cli():
         configs = [config_filepath]
         if parser.args.plots_config:
             configs += [parser.args.plots_config]
-        app_reader = ConfigReader(configs,ConfigFile)
+        app_reader = ConfigReader(configs,ConfigFile,dry_run=parser.args.dry_run)
         executable_name = os.path.basename(app_reader.config.executable).split(".")[0]
         report_folder_path = cmd_builder.createReportFolder(executable_name,app_reader.config.use_case_name)
         app_reader.updateConfig(machine_reader.processor.flattenDict(machine_reader.config,"machine"))

src/feelpp/benchmarking/reframe/config/configMachines.py

@@ -10,10 +10,13 @@ class Container(BaseModel):
 
     @field_validator("cachedir","tmpdir","image_base_dir",mode="before")
     @classmethod
-    def checkDirectories(cls,v):
+    def checkDirectories(cls,v, info):
         """Checks that the directories exists"""
         if v and not os.path.exists(v):
-            raise FileNotFoundError(f"Cannot find {v}")
+            if info.context.get("dry_run", False):
+                print(f"Dry Run: Skipping directory check for {v}")
+            else:
+                raise FileNotFoundError(f"Cannot find {v}")
 
         return v
 
@@ -36,7 +39,6 @@ class MachineConfig(BaseModel):
     #TODO: maybe skipJsonSchema or something like that.
     environment_map: Optional[Dict[str,List[str]]] = {}
 
-
     @model_validator(mode="after")
     def parseTargets(self):
         if not self.targets:

src/feelpp/benchmarking/reframe/config/configReader.py

@@ -70,12 +70,15 @@ def decode(self, s: str):
 
 class ConfigReader:
     """ Class to load config files"""
-    def __init__(self, config_paths, schema):
+    def __init__(self, config_paths, schema, dry_run=False):
         """
         Args:
             config_paths (str | list[str]) : Path to the config JSON file. If a list is provided, files will be merged.
         """
         self.schema = schema
+        self.context = {
+            "dry_run":dry_run
+        }
         self.config = self.load(
             config_paths if type(config_paths) == list else [config_paths],
             schema
@@ -97,7 +100,7 @@ def load(self,config_paths, schema):
             with open(config, "r") as cfg:
                 self.config.update(json.load(cfg, cls=JSONWithCommentsDecoder))
 
-        self.config = schema(**self.config)
+        self.config = schema.model_validate(self.config, context=self.context)
 
         return self.config
 
@@ -109,7 +112,7 @@ def updateConfig(self, flattened_replace = None):
         """
         if not flattened_replace:
             flattened_replace = self.processor.flattenDict(self.config.model_dump())
-        self.config = self.schema(**self.processor.recursiveReplace(self.config.model_dump(),flattened_replace))
+        self.config = self.schema.model_validate(self.processor.recursiveReplace(self.config.model_dump(),flattened_replace), context=self.context)
 
     def __repr__(self):
         return json.dumps(self.config.dict(), indent=4)

src/feelpp/benchmarking/reframe/config/configSchemas.py

@@ -44,17 +44,29 @@ class Image(BaseModel):
     protocol:Optional[Literal["oras","docker","library","local"]] = None
     name:str
 
-    @model_validator(mode="after")
-    def extractProtocol(self):
+    @field_validator("protocol",mode="before")
+    @classmethod
+    def extractProtocol(cls, v, info):
         """ Extracts the image protocol (oras, docker, etc..) or if a local image is provided.
         If local, checks if the image exists """
 
-        if "://" in self.name:
-            self.protocol = self.name.split("://")[0]
+        name = info.data.get("name","")
+        if "://" in name:
+            return name.split("://")[0]
         else:
-            self.protocol = "local"
+            return "local"
 
-        return self
+    @field_validator("name", mode="before")
+    @classmethod
+    def checkImage(cls,v,info):
+        if info.data["protocol"] == "local":
+            if not os.path.exists(v):
+                if info.context.get("dry_run", False):
+                   print(f"Dry Run: Skipping image check for {v}")
+                else:
+                    raise FileExistsError(f"Cannot find image {v}")
+
+        return v
 
 
 class Platform(BaseModel):
@@ -74,6 +86,7 @@ class ConfigFile(BaseModel):
     output_directory:Optional[str] = ""
     use_case_name: str
     options: List[str]
+    env_variables:Optional[Dict] = {}
     outputs: List[AppOutput]
     scalability: Scalability
     sanity: Sanity