util: CI: refactor CI files (#250)

* util: CI: refactor CI files move all seperate files to /nfs/home/share/gem5_ci, copy gem5_ci folders could run CI on both node/open ci servers * util: CI: use https instead of git Change-Id: I669b20ceeef7a764526c114b28bd08abb66c296a * util: CI: delete aocc.sh Change-Id: Iaba0c54cc2f57bb51ab1644c7a29fdc8edd037a9
OpenXiangShan · Dec 31, 2024 · 7657cdc · 7657cdc
1 parent eaf1c14
commit 7657cdc
Show file tree

Hide file tree

Showing 3 changed files with 47 additions and 34 deletions.
diff --git a/.github/workflows/gem5-ideal-perf.yml b/.github/workflows/gem5-ideal-perf.yml
@@ -17,7 +17,7 @@ jobs:
         run: |
           export GEM5_HOME=$(pwd)
           cd ext/dramsim3
-          git clone git@github.com:umd-memsys/DRAMSim3.git DRAMsim3
+          git clone https://github.com/umd-memsys/DRAMsim3.git DRAMsim3
           cd DRAMsim3 && mkdir -p build
           cd build
           cmake ..
@@ -28,14 +28,21 @@ jobs:
       - name: XS-GEM5 - Run performance test
         # run 80% coverage spec06 checkpoints, 500+ checkpoints
         run: |
-          export GCBV_REF_SO="/nfs-nvme/home/share/zhenhao/ref-h/build/riscv64-nemu-interpreter-so"
+          # 根据机器名判断当前环境在小机房还是大机房
+          if [[ $(hostname) == open* ]]; then
+            export CHECKPOINT_ROOT="/nfs/home/share/jiaxiaoyu/simpoint_checkpoint_archive/spec06_rv64gcb_O3_20m_gcc12.2.0-intFpcOff-jeMalloc/zstd-checkpoint-0-0-0"
+          else
+            export CHECKPOINT_ROOT="/nfs/home/share/jiaxiaoyu/simpoint_checkpoint_zstd_format/spec06_rv64gcb_O3_20m_gcc12.2.0-intFpcOff-jeMalloc"
+          fi
+
+          export GCBV_REF_SO="/nfs/home/share/gem5_ci/ref/normal/riscv64-nemu-interpreter-so"
           export GCB_RESTORER=""
           export GEM5_HOME=$(pwd)
           mkdir -p $GEM5_HOME/util/xs_scripts/test
           cd $GEM5_HOME/util/xs_scripts/test
           bash ../parallel_sim.sh `realpath ../kmh_v3_ideal.sh` \
-            /nfs/home/yanyue/workspace/spec06_cpts/spec_0.8c_int.lst \
-            /nfs/home/share/jiaxiaoyu/simpoint_checkpoint_archive/spec06_rv64gcb_O3_20m_gcc12.2.0-intFpcOff-jeMalloc/zstd-checkpoint-0-0-0 \
+            /nfs/home/share/gem5_ci/spec06_cpts/spec_0.8c_int.lst \
+            $CHECKPOINT_ROOT \
             spec_all
       - name: Setup gem5_data_proc environment
         run: |
@@ -52,7 +59,7 @@ jobs:
           # 使用已有的数据spec_all生成测试报告
           bash example-scripts/gem5-score-ci.sh \
             $GEM5_HOME/util/xs_scripts/test/spec_all \
-            /nfs/home/yanyue/workspace/spec06_cpts/cluster-0-0.json \
+            /nfs/home/share/gem5_ci/spec06_cpts/cluster-0-0.json \
             > $GITHUB_WORKSPACE/score.txt
           # 提取最后42行score信息
           echo "### performance test result :rocket:" >> $GITHUB_STEP_SUMMARY

diff --git a/.github/workflows/gem5-perf.yml b/.github/workflows/gem5-perf.yml
@@ -17,7 +17,7 @@ jobs:
         run: |
           export GEM5_HOME=$(pwd)
           cd ext/dramsim3
-          git clone git@github.com:umd-memsys/DRAMSim3.git DRAMsim3
+          git clone https://github.com/umd-memsys/DRAMsim3.git DRAMsim3
           cd DRAMsim3 && mkdir -p build
           cd build
           cmake ..
@@ -28,14 +28,21 @@ jobs:
       - name: XS-GEM5 - Run performance test
         # run 80% coverage spec06 checkpoints, 500+ checkpoints
         run: |
-          export GCBV_REF_SO="/nfs-nvme/home/share/zhenhao/ref-h/build/riscv64-nemu-interpreter-so"
+          # 根据机器名判断当前环境在小机房还是大机房
+          if [[ $(hostname) == open* ]]; then
+            export CHECKPOINT_ROOT="/nfs/home/share/jiaxiaoyu/simpoint_checkpoint_archive/spec06_rv64gcb_O3_20m_gcc12.2.0-intFpcOff-jeMalloc/zstd-checkpoint-0-0-0"
+          else
+            export CHECKPOINT_ROOT="/nfs/home/share/jiaxiaoyu/simpoint_checkpoint_zstd_format/spec06_rv64gcb_O3_20m_gcc12.2.0-intFpcOff-jeMalloc"
+          fi
+
+          export GCBV_REF_SO="/nfs/home/share/gem5_ci/ref/normal/riscv64-nemu-interpreter-so"
           export GCB_RESTORER=""
           export GEM5_HOME=$(pwd)
           mkdir -p $GEM5_HOME/util/xs_scripts/test
           cd $GEM5_HOME/util/xs_scripts/test
           bash ../parallel_sim.sh `realpath ../kmh_6wide.sh` \
-            /nfs/home/yanyue/workspace/spec06_cpts/spec_0.8c_int.lst \
-            /nfs/home/share/jiaxiaoyu/simpoint_checkpoint_archive/spec06_rv64gcb_O3_20m_gcc12.2.0-intFpcOff-jeMalloc/zstd-checkpoint-0-0-0 \
+            /nfs/home/share/gem5_ci/spec06_cpts/spec_0.8c_int.lst \
+            $CHECKPOINT_ROOT \
             spec_all
       - name: Setup gem5_data_proc environment
         run: |
@@ -52,7 +59,7 @@ jobs:
           # 使用已有的数据spec_all生成测试报告
           bash example-scripts/gem5-score-ci.sh \
             $GEM5_HOME/util/xs_scripts/test/spec_all \
-            /nfs/home/yanyue/workspace/spec06_cpts/cluster-0-0.json \
+            /nfs/home/share/gem5_ci/spec06_cpts/cluster-0-0.json \
             > $GITHUB_WORKSPACE/score.txt
           # 提取最后42行score信息
           echo "### performance test result :rocket:" >> $GITHUB_STEP_SUMMARY

diff --git a/.github/workflows/gem5.yml b/.github/workflows/gem5.yml
@@ -17,7 +17,7 @@ jobs:
         run: |
           export GEM5_HOME=$(pwd)
           cd ext/dramsim3
-          git clone git@github.com:umd-memsys/DRAMSim3.git DRAMsim3
+          git clone https://github.com/umd-memsys/DRAMsim3.git DRAMsim3
           cd DRAMsim3 && mkdir -p build
           cd build
           cmake ..
@@ -41,7 +41,7 @@ jobs:
         run: |
           export GEM5_HOME=$(pwd)
           cd ext/dramsim3
-          git clone git@github.com:umd-memsys/DRAMSim3.git DRAMsim3
+          git clone https://github.com/umd-memsys/DRAMsim3.git DRAMsim3
           cd DRAMsim3 && mkdir -p build
           cd build
           cmake ..
@@ -51,12 +51,12 @@ jobs:
         run: CC=gcc CXX=g++ scons build/RISCV/gem5.opt --linker=gold -j64
       - name: XS-GEM5 - Test xiangshan.py simulation scripts
         run: |
-          export GCBV_REF_SO="/nfs-nvme/home/share/zhenhao/ref-h/build/riscv64-nemu-interpreter-so"
-          export GCB_RESTORER="/nfs/home/share/gem5_shared_tools/normal-gcb-restorer.bin"
+          export GCBV_REF_SO="/nfs/home/share/gem5_ci/ref/normal/riscv64-nemu-interpreter-so"
+          export GCB_RESTORER="/nfs/home/share/gem5_ci/tools/normal-gcb-restorer.bin"
           export GEM5_HOME=$(pwd)
           mkdir -p $GEM5_HOME/util/xs_scripts/test
           cd $GEM5_HOME/util/xs_scripts/test
-          bash ../kmh_6wide.sh /nfs/home/share/jiaxiaoyu/simpoint_checkpoint_archive/spec06_rv64gcb_O3_20m_gcc12.2.0-intFpcOff-jeMalloc/zstd-checkpoint-0-0-0/xalancbmk/133/_133_0.006424_.zstd
+          bash ../kmh_6wide.sh /nfs/home/share/gem5_ci/checkpoints/gcb_test.zstd
 
   new_sim_script_test_gcbv:
     runs-on: self-hosted
@@ -68,7 +68,7 @@ jobs:
         run: |
           export GEM5_HOME=$(pwd)
           cd ext/dramsim3
-          git clone git@github.com:umd-memsys/DRAMSim3.git DRAMsim3
+          git clone https://github.com/umd-memsys/DRAMsim3.git DRAMsim3
           cd DRAMsim3 && mkdir -p build
           cd build
           cmake ..
@@ -78,12 +78,12 @@ jobs:
         run: CC=gcc CXX=g++ scons build/RISCV/gem5.opt --linker=gold -j64
       - name: XS-GEM5 - Test xiangshan.py simulation scripts
         run: |
-          export GCBV_REF_SO="/nfs-nvme/home/share/zhenhao/ref-h/build/riscv64-nemu-interpreter-so"
-          export GCBV_RESTORER="/nfs/home/share/gem5_shared_tools/gcbv-restorer.bin"
+          export GCBV_REF_SO="/nfs/home/share/gem5_ci/ref/normal/riscv64-nemu-interpreter-so"
+          export GCBV_RESTORER="/nfs/home/share/gem5_ci/tools/gcbv-restorer.bin"
           export GEM5_HOME=$(pwd)
           mkdir -p $GEM5_HOME/util/xs_scripts/test_v
           cd $GEM5_HOME/util/xs_scripts/test_v
-          bash ../kmh_6wide_vector.sh /nfs/home/share/jiaxiaoyu/simpoint_checkpoint_archive/spec06_hmmer_libquantum_h264_rv64gcv_O3_20m_gcc14.0.0_1218-intFpcOff-ptMalloc/disdain-practical-toll-2023-12-25-11-26/zstd-checkpoint-0-0-0/hmmer_nph3/102/_102_0.0243576.zstd
+          bash ../kmh_6wide_vector.sh /nfs/home/share/gem5_ci/checkpoints/gcbv_test.zstd
 
   new_sim_script_test_gcb_multi_core:
     runs-on: self-hosted
@@ -93,16 +93,15 @@ jobs:
       - uses: actions/checkout@v2
       - name: Build GEM5 opt
         run: |
-          source /nfs-nvme/home/share/zyy/shared_apps/setenv_AOCC.sh
           CC=clang CXX=clang++ scons build/RISCV_CHI/gem5.opt -j 48 --gold-linker
       - name: XS-GEM5 - Test xiangshan.py simulation scripts
         run: |
-          export GCBV_MULTI_CORE_REF_SO="/nfs-nvme/home/share/zhenhao/ref-h/build-multi/riscv64-nemu-interpreter-so"
-          export GCB_MULTI_CORE_RESTORER="/nfs/home/share/gem5_shared_tools/gcb-2core-restorer.bin"
+          export GCBV_MULTI_CORE_REF_SO="/nfs/home/share/gem5_ci/ref/multi/riscv64-nemu-interpreter-so"
+          export GCB_MULTI_CORE_RESTORER="/nfs/home/share/gem5_ci/tools/gcb-2core-restorer.bin"
           export GEM5_HOME=$(pwd)
           mkdir -p $GEM5_HOME/util/xs_scripts/test_multi_core
           cd $GEM5_HOME/util/xs_scripts/test_multi_core
-          bash ../kmh-ruby-dual.sh /nfs-nvme/home/share/zyy/shared_payloads/bzip_checkpoint_with_sv39_without_rdtime_with_dualmtimecpt_1000M.gz
+          bash ../kmh-ruby-dual.sh /nfs/home/share/gem5_ci/checkpoints/multi_core_test.gz
 
   difftest_check:
     runs-on: self-hosted
@@ -114,7 +113,7 @@ jobs:
         run: |
           export GEM5_HOME=$(pwd)
           cd ext/dramsim3
-          git clone git@github.com:umd-memsys/DRAMSim3.git DRAMsim3
+          git clone https://github.com/umd-memsys/DRAMsim3.git DRAMsim3
           cd DRAMsim3 && mkdir -p build
           cd build
           cmake ..
@@ -124,12 +123,12 @@ jobs:
         run: CC=clang CXX=clang++ scons build/RISCV/gem5.opt -j 48 --gold-linker
       - name: difftest check
         run: |
-          export GCBV_REF_SO="/nfs-nvme/home/share/zhenhao/ref-h/build-error/riscv64-nemu-interpreter-so"
-          export GCB_RESTORER="/nfs-nvme/home/share/zhenhao/LibCheckpointAlpha/build/gcpt.bin"
+          export GCBV_REF_SO="/nfs/home/share/gem5_ci/ref/error/riscv64-nemu-interpreter-so"
+          export GCB_RESTORER="/nfs/home/share/gem5_ci/tools/normal-gcb-restorer.bin"
           export GEM5_HOME=$(pwd)
           mkdir -p $GEM5_HOME/util/xs_scripts/test
           cd $GEM5_HOME/util/xs_scripts/test
-          bash ../kmh_6wide.sh /nfs/home/share/jiaxiaoyu/simpoint_checkpoint_archive/spec06_rv64gcb_O3_20m_gcc12.2.0-intFpcOff-jeMalloc/zstd-checkpoint-0-0-0/xalancbmk/133/_133_0.006424_.zstd 2>log.txt || exit_code=$?
+          bash ../kmh_6wide.sh /nfs/home/share/gem5_ci/checkpoints/gcb_test.zstd 2>log.txt || exit_code=$?
           if [ ${exit_code} -eq 0 ]; then echo "Difftest is broken, it should report error!" exit 1; fi
           match=$(grep ".*Difftest failed!.*" log.txt -c)
           if [ ${match} -eq 0 ]; then echo "Difftest is broken, it should report at least one agnostic related difference!" exit 1; fi
@@ -144,7 +143,7 @@ jobs:
         run: |
           export GEM5_HOME=$(pwd)
           cd ext/dramsim3
-          git clone git@github.com:umd-memsys/DRAMSim3.git DRAMsim3
+          git clone https://github.com/umd-memsys/DRAMsim3.git DRAMsim3
           cd DRAMsim3 && mkdir -p build
           cd build
           cmake ..
@@ -154,12 +153,12 @@ jobs:
         run: CC=gcc CXX=g++ scons build/RISCV/gem5.opt --linker=gold -j64
       - name: XS-GEM5 - Test xiangshan.py simulation scripts
         run: |
-          export GCBV_REF_SO="/nfs-nvme/home/share/zhenhao/ref-h/build/riscv64-nemu-interpreter-so"
+          export GCBV_REF_SO="/nfs/home/share/gem5_ci/ref/normal/riscv64-nemu-interpreter-so"
           export GCB_RESTORER=""
           export GEM5_HOME=$(pwd)
           mkdir -p $GEM5_HOME/util/xs_scripts/test_l2tlb
           cd $GEM5_HOME/util/xs_scripts/test_l2tlb
-          bash ../kmh_6wide.sh  /nfs/home/share/yanyue/checkpoints/gcc13_gcb_403_61291.zstd
+          bash ../kmh_6wide.sh /nfs/home/share/gem5_ci/checkpoints/l2tlb_test.zstd
 
   new_sim_script_test_gcbh:
     runs-on: self-hosted
@@ -171,7 +170,7 @@ jobs:
         run: |
           export GEM5_HOME=$(pwd)
           cd ext/dramsim3
-          git clone git@github.com:umd-memsys/DRAMSim3.git DRAMsim3
+          git clone https://github.com/umd-memsys/DRAMsim3.git DRAMsim3
           cd DRAMsim3 && mkdir -p build
           cd build
           cmake ..
@@ -181,10 +180,10 @@ jobs:
         run: CC=gcc CXX=g++ scons build/RISCV/gem5.opt --linker=gold -j64
       - name: XS-GEM5 - Test xiangshan.py simulation scripts
         run: |
-          export GCBH_REF_SO="/nfs-nvme/home/share/zhenhao/ref-h/build-h/riscv64-nemu-interpreter-so"
-          export GCBH_RESTORER="/nfs-nvme/home/share/zhenhao/LibCheckpointAlpha/build/gcpt.bin"
+          export GCBH_REF_SO="/nfs/home/share/gem5_ci/ref/h/riscv64-nemu-interpreter-so"
+          export GCBH_RESTORER="/nfs/home/share/gem5_ci/tools/gcpt.bin"
           export GEM5_HOME=$(pwd)
           mkdir -p $GEM5_HOME/util/xs_scripts/test_h
           cd $GEM5_HOME/util/xs_scripts/test_h
-          bash ../kmh_6wide_h.sh /nfs/home/share/jiaxiaoyu/H-ext-checkpoint-uniform-zstd-nosmp/uniform/payload/820000000/_820000000_.zstd
+          bash ../kmh_6wide_h.sh /nfs/home/share/gem5_ci/checkpoints/gcbh_test.zstd