[Postmortem 4.1] Make mlperf.conf static in loadgen, enable automatic…

… Pypi release (#1882) * Extend the generate_final_reports script to output a json file of results * Fix an error in merging perf and power results * Export weight_data_types in the submission checker csv * Fix positional argument * Quote model precision info in submission checker csv * Quote model precision info in submission checker csv * Fix submission generation for v4.0 * Autoload mlperf.conf in loadgen Added VERSION file * Fix loadgen version in R50 test * Fix format for setup.py * Added soft link to mlperf.conf in root * Fix format with clang * Fixed softlink for mlperf.conf * Fixed Python API binding for default argument in FromConfig * Fixed Python API binding for default argument in FromConfig * Fix a bug in FromConfig * Fix deprecation of pkg_resources * Fix deprecation of pkg_resources * Fix deprecation of pkg_resources * Fix package info * Fix package info * Added __init__.py * Fix package info * Restrict the use of only one conf file in loadgen * Remove mlperf.conf arg from the reference implementations * Use custom loadgen version for the github actions (retinanet+bert) * Added version patch * Remove mlperf.conf for vision benchmarks * Fix compilation warnings * formatted by clang * Modify the build_wheels action to use VERSION * Fix build_wheels gh action * Do version increment in github action only for linux * VERSION -> VERSION.txt * Fix the build_wheel github action on windows * VERSION -> VERSION.txt * Support python 3.13 loadgen build * VERSION -> VERSION.txt * VERSION update * Build wheels on macos and windows but upload from only ubuntu * Build wheels on macos and windows but upload from only ubuntu * Build wheels on macos and windows but upload from only ubuntu * Fixes for github action * Fixes for github action * Fixes for github action * Fixes for github action * Make version number consistent for all OS * Make version number consistent for all OS * Do not release pypi wheels from loadgen-dev branch * Split the keys specific to mlperf.conf * Split the keys specific to mlperf.conf * Split the keys specific to mlperf.conf
mlcommons · Oct 22, 2024 · ecb8801 · ecb8801
1 parent f5c8f17
commit ecb8801
Show file tree

Hide file tree

Showing 26 changed files with 1,040 additions and 470 deletions.
diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml
@@ -6,21 +6,69 @@ on:
   push:
     branches:
       - master
+      - loadgen-release
     paths:
-      - loadgen/setup.py
+      - loadgen/**
 
 jobs:
+  update_version:
+    name: Update version only on ubuntu but used by windows and macos
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      # Step 3: Check if VERSION.txt file has changed in this push
+      - name: Check if VERSION.txt file has changed
+        id: version_changed
+        run: |
+          if git diff --name-only HEAD~1 | grep -q "VERSION.txt"; then
+            echo "VERSION.txt file has been modified"
+            echo "::set-output name=version_changed::true"
+            new_version=$(cat VERSION.txt)
+          else
+            echo "VERSION file has NOT been modified"
+            echo "::set-output name=version_changed::false"
+          fi
+          echo "::set-output name=new_version::$new_version"
+
+      # Step 4: Increment version if VERSION was not changed
+      - name: Increment version if necessary
+        id: do_version_increment
+        if: steps.version_changed.outputs.version_changed == 'false'
+        run: |
+          cd loadgen
+          # Check if VERSION file exists, else initialize it
+          if [ ! -f VERSION.txt ]; then
+            echo "0.0.0" > VERSION.txt
+          fi
+
+          version=$(cat VERSION.txt)
+          IFS='.' read -r major minor patch <<< "$version"
+          patch=$((patch + 1))
+          new_version="$major.$minor.$patch"
+          echo $new_version > VERSION.txt
+          echo "New version: $new_version"
+          echo "::set-output name=new_version::$new_version"
+
+      # Step 5: Commit the updated version to the repository
+      - name: Commit updated version
+        if: steps.version_changed.outputs.version_changed == 'false'
+        run: |
+          cd loadgen
+          git config --global user.name "${{ github.actor }}"
+          git config --global user.email "${{ github.actor }}@users.noreply.github.com"
+          git add VERSION.txt
+          git commit -m "Increment version to ${{ steps.do_version_increment.outputs.new_version }}"
+          git push
+  
   build_wheels:
     name: Build wheels on ${{ matrix.os }}
+    needs: update_version
     runs-on: ${{ matrix.os }}
-    environment: release
-    permissions:
-      # IMPORTANT: this permission is mandatory for trusted publishing
-      id-token: write
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, windows-latest, macOS-latest]
+        os: [ubuntu-latest, windows-latest, macos-latest]
 
     steps:
       - uses: actions/checkout@v3
@@ -33,6 +81,41 @@ jobs:
       - name: Build wheels
         run: python -m cibuildwheel loadgen/ --output-dir wheels
 
+       # Save wheels as artifacts
+      - name: Upload built wheels
+        uses: actions/upload-artifact@v3
+        with:
+          name: wheels-${{ matrix.os }}
+          path: wheels
+
+  publish_wheels:
+    needs: build_wheels  # Wait for the build_wheels job to complete
+    runs-on: ubuntu-latest  # Only run this job on Linux
+    environment: release
+    permissions:
+      # IMPORTANT: this permission is mandatory for trusted publishing
+      id-token: write
+    steps:
+      - uses: actions/checkout@v3
+
+      # Download the built wheels from ubuntu
+      - name: Download Ubuntu wheels
+        uses: actions/download-artifact@v3
+        with:
+          name: wheels-ubuntu-latest
+          path: wheels
+      # Download the built wheels from macOS
+      - name: Download macOS wheels
+        uses: actions/download-artifact@v3
+        with:
+          name: wheels-macos-latest
+          path: wheels
+      # Download the built wheels from Windows
+      - name: Download Windows wheels
+        uses: actions/download-artifact@v3
+        with:
+          name: wheels-windows-latest
+          path: wheels
       - name: Publish
         uses: pypa/gh-action-pypi-publish@release/v1
         with:

diff --git a/.github/workflows/test-bert.yml b/.github/workflows/test-bert.yml
@@ -33,4 +33,4 @@ jobs:
         python3 -m pip install cm4mlops
     - name: Test BERT and end to end submission generation
       run: |
-        cm run script --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --quiet --submitter="MLCommons" --hw_name=default --model=bert-99 --implementation=reference --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --adr.compiler.tags=gcc  --adr.inference-src.version=custom --adr.inference-src.tags=_repo.${{ github.event.pull_request.head.repo.html_url }},_branch.${{ github.event.pull_request.head.ref }}
+        cm run script --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --quiet --submitter="MLCommons" --hw_name=default --model=bert-99 --implementation=reference --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --adr.compiler.tags=gcc  --adr.inference-src.version=custom --adr.inference-src.tags=_repo.${{ github.event.pull_request.head.repo.html_url }},_branch.${{ github.event.pull_request.head.ref }} --adr.loadgen.version=custom
diff --git a/.github/workflows/test-loadgen.yml b/.github/workflows/test-loadgen.yml
@@ -18,7 +18,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10"]
+        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
 
     steps:
     - uses: actions/checkout@v3
@@ -31,4 +31,4 @@ jobs:
         python3 -m pip install cm4mlops
     - name: Test Loadgen
       run: |
-        cm run script --tags=get,mlperf,inference,loadgen --quiet --version=custom --adr.inference-src.tags=_repo.${{ github.event.pull_request.head.repo.html_url }},_branch.${{ github.event.pull_request.head.ref }}
+        cm run script --tags=get,mlperf,inference,loadgen --quiet --version=custom --adr.inference-src.tags=_repo.${{ github.event.pull_request.head.repo.html_url }},_branch.${{ github.event.pull_request.head.ref }} --adr.loadgen.tags=_no-compilation-warnings
diff --git a/.github/workflows/test-resnet50.yml b/.github/workflows/test-resnet50.yml
@@ -8,6 +8,7 @@ on:
     branches: [ "master", "dev" ]
     paths:
       - vision/classification_and_detection/**
+      - loadgen/**
       - tools/submission/**
       - .github/workflows/test-resnet50.yml
       - '!**.md'
@@ -33,4 +34,4 @@ jobs:
         python3 -m pip install cm4mlops
     - name: Test Resnet50 and end to end submission generation
       run: |
-        cm run script --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --quiet --submitter="MLCommons" --hw_name=default --model=resnet50 --implementation=reference --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --adr.compiler.tags=gcc --adr.inference-src.tags=_branch.${{ github.event.pull_request.head.ref }},_repo.${{ github.event.pull_request.head.repo.html_url }} --adr.inference-src.version=custom
+        cm run script --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --quiet --submitter="MLCommons" --hw_name=default --model=resnet50 --implementation=reference --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --adr.compiler.tags=gcc --adr.inference-src.tags=_branch.${{ github.event.pull_request.head.ref }},_repo.${{ github.event.pull_request.head.repo.html_url }} --adr.inference-src.version=custom --adr.loadgen.version=custom
diff --git a/.github/workflows/test-retinanet.yml b/.github/workflows/test-retinanet.yml
@@ -33,4 +33,4 @@ jobs:
         python3 -m pip install cm4mlops
     - name: Test Retinanet and end to end submission generation
       run: |
-        cm run script --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --quiet --submitter="MLCommons" --hw_name=default --model=retinanet --implementation=reference --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=10 --adr.compiler.tags=gcc  --adr.inference-src.version=custom --adr.inference-src.tags=_repo.${{ github.event.pull_request.head.repo.html_url }},_branch.${{ github.event.pull_request.head.ref }}
+        cm run script --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --quiet --submitter="MLCommons" --hw_name=default --model=retinanet --implementation=reference --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=10 --adr.compiler.tags=gcc  --adr.inference-src.version=custom --adr.inference-src.tags=_repo.${{ github.event.pull_request.head.repo.html_url }},_branch.${{ github.event.pull_request.head.ref }} --adr.loadgen.version=custom
diff --git a/language/bert/run.py b/language/bert/run.py
@@ -15,20 +15,21 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from absl import flags
+from absl import app
 import subprocess
 import mlperf_loadgen as lg
 import argparse
 import os
 import sys
 sys.path.insert(0, os.getcwd())
 sys.path.insert(0, os.path.join(os.getcwd(), "..", "..", "lon"))
-from absl import app
-from absl import flags
+
 
 def get_args():
     parser = argparse.ArgumentParser()
     parser.add_argument(
-            "--backend", choices=["tf", "pytorch", "onnxruntime", "tf_estimator", "ray"], default="tf", help="Backend")
+        "--backend", choices=["tf", "pytorch", "onnxruntime", "tf_estimator", "ray"], default="tf", help="Backend")
     parser.add_argument("--scenario", choices=["SingleStream", "Offline",
                                                "Server", "MultiStream"], default="Offline", help="Scenario")
     parser.add_argument("--accuracy", action="store_true",
@@ -37,30 +38,36 @@ def get_args():
                         help="use quantized model (only valid for onnxruntime backend)")
     parser.add_argument("--profile", action="store_true",
                         help="enable profiling (only valid for onnxruntime backend)")
-    parser.add_argument(
-            "--mlperf_conf", default="build/mlperf.conf", help="mlperf rules config")
     parser.add_argument("--user_conf", default="user.conf",
                         help="user config for user LoadGen settings such as target QPS")
     parser.add_argument("--audit_conf", default="audit.conf",
                         help="audit config for LoadGen settings during compliance runs")
     parser.add_argument("--max_examples", type=int,
                         help="Maximum number of examples to consider (not limited by default)")
-    parser.add_argument("--network", choices=["sut","lon",None], default=None, help="Loadgen network mode")
+    parser.add_argument(
+        "--network",
+        choices=[
+            "sut",
+            "lon",
+            None],
+        default=None,
+        help="Loadgen network mode")
     parser.add_argument('--node', type=str, default="")
     parser.add_argument('--port', type=int, default=8000)
-    parser.add_argument('--sut_server', nargs="*", default= ['http://localhost:8000'],
-                    help='Address of the server(s) under test.')
+    parser.add_argument('--sut_server', nargs="*", default=['http://localhost:8000'],
+                        help='Address of the server(s) under test.')
 
     args = parser.parse_args()
     return args
 
 
 scenario_map = {
-        "SingleStream": lg.TestScenario.SingleStream,
-        "Offline": lg.TestScenario.Offline,
-        "Server": lg.TestScenario.Server,
-        "MultiStream": lg.TestScenario.MultiStream
-        }
+    "SingleStream": lg.TestScenario.SingleStream,
+    "Offline": lg.TestScenario.Offline,
+    "Server": lg.TestScenario.Server,
+    "MultiStream": lg.TestScenario.MultiStream
+}
+
 
 def main():
     args = get_args()
@@ -96,7 +103,8 @@ def main():
 
     settings = lg.TestSettings()
     settings.scenario = scenario_map[args.scenario]
-    settings.FromConfig(args.mlperf_conf, "bert", args.scenario)
+    # mlperf.conf is automatically loaded by the loadgen
+    # settings.FromConfig(args.mlperf_conf, "bert", args.scenario)
     settings.FromConfig(args.user_conf, "bert", args.scenario)
 
     if args.accuracy:
@@ -117,7 +125,14 @@ def main():
 
     if args.network == "lon":
         from network_LON import app, set_args, main as app_main
-        set_args(args, settings, log_settings, args.audit_conf, args.sut_server, args.backend, args.max_examples)
+        set_args(
+            args,
+            settings,
+            log_settings,
+            args.audit_conf,
+            args.sut_server,
+            args.backend,
+            args.max_examples)
         app.run(app_main)
 
     elif args.network == "sut":
@@ -128,7 +143,12 @@ def main():
 
     else:
         print("Running LoadGen test...")
-        lg.StartTestWithLogSettings(sut.sut, sut.qsl.qsl, settings, log_settings, args.audit_conf)
+        lg.StartTestWithLogSettings(
+            sut.sut,
+            sut.qsl.qsl,
+            settings,
+            log_settings,
+            args.audit_conf)
         if args.accuracy and not os.environ.get("SKIP_VERIFY_ACCURACY"):
             cmd = "python3 {:}/accuracy-squad.py {}".format(
                 os.path.dirname(os.path.abspath(__file__)),