Skip to content

Commit

Permalink
Merge branch 'flat'
Browse files Browse the repository at this point in the history
  • Loading branch information
xieby1 committed Dec 8, 2024
2 parents e75a0c0 + 00b12d6 commit 2ec5eeb
Show file tree
Hide file tree
Showing 21 changed files with 200 additions and 173 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/runner.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@ jobs:
contents: write
steps:
- uses: actions/checkout@v4
- run: nix-build -A openblas -o result-openblas
- run: nix-build -A spec2006 --arg src $(ls -d /spec2006* | head -n1) -j29 -o result-spec2006
- run: nix-build -A openblas.cpt -o result-openblas
- run: nix-build -A spec2006.cpt --arg src $(ls -d /spec2006* | head -n1) -j29 -o result-spec2006
- run: |
git rev-parse HEAD > /tmp/rev
(date +%y%m%d%H%M%S && cat /tmp/rev && readlink result-openblas) | tr '\n' ',' > /tmp/openblas.txt
Expand Down
4 changes: 1 addition & 3 deletions benchmarks/openblas/default.nix
Original file line number Diff line number Diff line change
@@ -1,16 +1,14 @@
{ stdenv
, fetchFromGitHub

, dconfig
, traceDConfig
, riscv64-pkgs
, riscv64-cc
, riscv64-fortran
, riscv64-libc-static
}: let
riscv64-libfortran = riscv64-pkgs.gfortran.cc;
in stdenv.mkDerivation {
pname = traceDConfig dconfig "openblas";
pname = "openblas";
version = "0.3.28";
src = fetchFromGitHub {
owner = "OpenMathLib";
Expand Down
14 changes: 7 additions & 7 deletions benchmarks/spec2006/build-all.nix
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,12 @@
, fetchFromGitHub
, libxcrypt-legacy

, dconfig
, traceDConfig
, riscv64-pkgs
, riscv64-cc
, riscv64-fortran
, riscv64-libc-static
, src
, size
}:
let
customJemalloc = riscv64-pkgs.jemalloc.overrideAttrs (oldAttrs: {
Expand All @@ -34,11 +34,11 @@ let
hash = "sha256-qNxmM9Dmobr6fvTZapacu8jngcBPRbybwayTi7CZGd0=";
};
in stdenv.mkDerivation {
name = traceDConfig dconfig "spec2006exe";
name = "spec2006exe";
system = "x86_64-linux";

srcs = [
dconfig.src
src
CPU2006LiteWrapper
];
sourceRoot = ".";
Expand Down Expand Up @@ -103,8 +103,8 @@ in stdenv.mkDerivation {
pushd $WORK_DIR
mkdir -p run
if [ -d data/all/input ]; then cp -r data/all/input/* run/; fi
if [ -d data/${dconfig.size}/input ]; then cp -r data/${dconfig.size}/input/* run/; fi
if [ -f extra-data/${dconfig.size}.sh ]; then sh extra-data/${dconfig.size}.sh ; fi
if [ -d data/${size}/input ]; then cp -r data/${size}/input/* run/; fi
if [ -f extra-data/${size}.sh ]; then sh extra-data/${size}.sh ; fi
mkdir -p $out/$WORK_DIR/run/
cp -r run/* $out/$WORK_DIR/run/
Expand All @@ -113,7 +113,7 @@ in stdenv.mkDerivation {
# E.g.: 481.wrf/run-ref.sh
# before replace: [run-ref.h]: $APP > rsl.out.0000
# after replace: [run.sh]: ./481.wrf > rsl.out.0000
sed 's,\$APP,./'$WORK_DIR',' run-${dconfig.size}.sh > $out/$WORK_DIR/run/run-spec.sh
sed 's,\$APP,./'$WORK_DIR',' run-${size}.sh > $out/$WORK_DIR/run/run-spec.sh
popd
done
Expand Down
4 changes: 0 additions & 4 deletions benchmarks/spec2006/config.nix

This file was deleted.

12 changes: 6 additions & 6 deletions benchmarks/spec2006/default.nix
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
{ callPackage
, runCommand
, dconfig
{ runCommand

, callPackage
, src
, size ? "ref" # "ref" or "test"
}: let
testCases = [
"400.perlbench"
Expand Down Expand Up @@ -33,9 +35,7 @@
"482.sphinx3"
"483.xalancbmk"
];
build-all = callPackage ./build-all.nix {
dconfig = import ./config.nix // dconfig;
};
build-all = callPackage ./build-all.nix { inherit src size; };
in builtins.listToAttrs (
builtins.map (testcase: {
# change `.` to `_`, e.g. "403.gcc" to "403_gcc"
Expand Down
31 changes: 18 additions & 13 deletions builders/cptBuilder/1.profiling.nix
Original file line number Diff line number Diff line change
@@ -1,46 +1,51 @@
{ runCommand
, lib

, dconfig
, qemu
, nemu
, gcpt
, img
, workload ? "miao"
, intervals ? "20000000"
, simulator ? "qemu" # "qemu" or "nemu"
, profiling_log ? "profiling.log"
}:
let
name = "${lib.removeSuffix ".gcpt" gcpt.name}.1_profiling";
name = "${lib.removeSuffix ".gcpt" img.name}.1_profiling";

qemuCommand = [
"${qemu}/bin/qemu-system-riscv64"
"-bios ${gcpt}"
"-bios ${img}"
"-M nemu"
"-nographic"
"-m 8G"
"-smp 1"
"-cpu rv64,v=true,vlen=128,h=false,sv39=true,sv48=false,sv57=false,sv64=false"
"-plugin ${qemu}/lib/libprofiling.so,workload=${dconfig.workload},intervals=${toString dconfig.intervals},target=$out"
"-plugin ${qemu}/lib/libprofiling.so,workload=${workload},intervals=${intervals},target=$out"
"-icount shift=0,align=off,sleep=off"
];

nemuCommand = [
"${nemu}/bin/riscv64-nemu-interpreter"
"${gcpt}"
"${img}"
"-b"
"-D $out"
"-C ${name}"
"-w ${dconfig.workload}"
"-w ${workload}"
"--simpoint-profile"
"--cpt-interval ${toString dconfig.intervals}"
"--cpt-interval ${intervals}"
];

in runCommand name {} ''
in runCommand name {
passthru = { inherit qemu nemu img; };
} ''
mkdir -p $out
${if dconfig.simulator == "qemu" then ''
${if simulator == "qemu" then ''
echo ${builtins.toString qemuCommand}
${builtins.toString qemuCommand} | tee $out/${dconfig.profiling_log}
${builtins.toString qemuCommand} | tee $out/${profiling_log}
'' else ''
echo ${builtins.toString nemuCommand}
${builtins.toString nemuCommand} | tee $out/${dconfig.profiling_log}
cp $out/${name}/${dconfig.workload}/simpoint_bbv.gz $out/
${builtins.toString nemuCommand} | tee $out/${profiling_log}
cp $out/${name}/${workload}/simpoint_bbv.gz $out/
''}
''
5 changes: 3 additions & 2 deletions builders/cptBuilder/2.cluster.nix
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,9 @@
, maxK ? "30"
, simpoint
, stage1-profiling
, ...
}: runCommand "${lib.removeSuffix ".1_profiling" stage1-profiling.name}.2_cluster" {} (''
}: runCommand "${lib.removeSuffix ".1_profiling" stage1-profiling.name}.2_cluster" {
passthru = { inherit simpoint stage1-profiling; };
} (''
mkdir -p $out
'' + (builtins.toString [
"${simpoint}/bin/simpoint"
Expand Down
28 changes: 17 additions & 11 deletions builders/cptBuilder/3.checkpoint.nix
Original file line number Diff line number Diff line change
@@ -1,17 +1,21 @@
{ runCommand
, lib

, dconfig
, qemu
, nemu
, gcpt
, img
, stage2-cluster
, intervals ? "20000000"
, workload ? "miao"
, checkpoint_format ? "zstd" # gz or zstd, qemu only support zstd compressed
, simulator ? "qemu" # nemu or qemu
, checkpoint_log ? "checkpoint.log"
}:
let
qemuCommand = [
"${qemu}/bin/qemu-system-riscv64"
"-bios ${gcpt}"
"-M nemu,simpoint-path=${stage2-cluster},workload=.,cpt-interval=${toString dconfig.intervals},output-base-dir=$out,config-name=${dconfig.workload},checkpoint-mode=SimpointCheckpoint"
"-bios ${img}"
"-M nemu,simpoint-path=${stage2-cluster},workload=.,cpt-interval=${intervals},output-base-dir=$out,config-name=${workload},checkpoint-mode=SimpointCheckpoint"
"-nographic"
"-m 8G"
"-smp 1"
Expand All @@ -21,24 +25,26 @@ let

nemuCommand = [
"${nemu}/bin/riscv64-nemu-interpreter"
"${gcpt}"
"${img}"
"-b"
"-D $out"
"-C checkpoint"
"-w ."
"-S ${stage2-cluster}"
"--cpt-interval ${toString dconfig.intervals}"
"--checkpoint-format ${toString dconfig.checkpoint_format}"
"--cpt-interval ${intervals}"
"--checkpoint-format ${checkpoint_format}"
];

in runCommand "${lib.removeSuffix ".2_cluster" stage2-cluster.name}.3_checkpoint" {} ''
in runCommand "${lib.removeSuffix ".2_cluster" stage2-cluster.name}.3_checkpoint" {
passthru = { inherit qemu nemu img stage2-cluster; };
} ''
mkdir -p $out
${if dconfig.simulator == "qemu" then ''
${if simulator == "qemu" then ''
echo "Executing QEMU command: ${builtins.toString qemuCommand}"
${builtins.toString qemuCommand} | tee $out/${dconfig.checkpoint_log}
${builtins.toString qemuCommand} | tee $out/${checkpoint_log}
'' else ''
echo "Executing NEMU command: ${builtins.toString nemuCommand}"
${builtins.toString nemuCommand} | tee $out/${dconfig.checkpoint_log}
${builtins.toString nemuCommand} | tee $out/${checkpoint_log}
''}
''
26 changes: 6 additions & 20 deletions builders/cptBuilder/default.nix
Original file line number Diff line number Diff line change
@@ -1,25 +1,11 @@
{ callPackage

, imgBuilder
, ...
}@args: let
# TODO: move folders to cptBuilder/
qemu = callPackage ./qemu {};
nemu = callPackage ./nemu {};
gcpt = imgBuilder.gcpt;
stage1-profiling = callPackage ./1.profiling.nix {
inherit qemu nemu gcpt;
};
simpoint = callPackage ./simpoint {};
stage2-cluster = callPackage ./2.cluster.nix ({
inherit simpoint stage1-profiling;
} // args);
stage3-checkpoint = callPackage ./3.checkpoint.nix {
inherit qemu nemu gcpt stage2-cluster;
};
{ stage3-checkpoint }: let
stage2-cluster = stage3-checkpoint.stage2-cluster;
stage1-profiling = stage2-cluster.stage1-profiling;
in stage3-checkpoint.overrideAttrs (old: {
passthru = {
inherit qemu nemu simpoint;
inherit stage1-profiling stage2-cluster stage3-checkpoint;
qemu = stage1-profiling.qemu;
nemu = stage1-profiling.nemu;
simpoint = stage2-cluster.simpoint;
};
})
78 changes: 68 additions & 10 deletions builders/default.nix
Original file line number Diff line number Diff line change
@@ -1,10 +1,68 @@
{ callPackage

, benchmark
, ...
}@args: let
imgBuilder = callPackage ./imgBuilder { inherit benchmark; };
cptBuilder = callPackage ./cptBuilder ({ inherit imgBuilder; } // args);
in cptBuilder.overrideAttrs (old: {
passthru = { inherit benchmark imgBuilder cptBuilder; };
})
{ lib
, callPackage
}:
{ benchmark
, overlay ? (self: super: {})
}: let allPackages = self: super: with self; {
inherit benchmark;
gen_init_cpio = callPackage ./imgBuilder/linux/initramfs/base/gen_init_cpio {};
initramfs_base = callPackage ./imgBuilder/linux/initramfs/base {
inherit gen_init_cpio;
};

before_workload = callPackage ./imgBuilder/linux/initramfs/overlays/before_workload {};
nemu_trap = callPackage ./imgBuilder/linux/initramfs/overlays/nemu_trap {};
qemu_trap = callPackage ./imgBuilder/linux/initramfs/overlays/qemu_trap {};
initramfs_overlays = callPackage ./imgBuilder/linux/initramfs/overlays {
inherit before_workload qemu_trap nemu_trap;
benchmark-run = benchmark.run;
};

initramfs = callPackage ./imgBuilder/linux/initramfs {
inherit benchmark;
base = initramfs_base;
overlays = initramfs_overlays;
};

linux-common-build = callPackage ./imgBuilder/linux/common-build.nix {};
linux = callPackage ./imgBuilder/linux {
inherit initramfs;
common-build = linux-common-build;
};

dts = callPackage ./imgBuilder/opensbi/dts {};
opensbi-common-build = callPackage ./imgBuilder/opensbi/common-build.nix {
inherit dts;
};
opensbi = callPackage ./imgBuilder/opensbi {
inherit dts linux;
common-build = opensbi-common-build;
};
gcpt = callPackage ./imgBuilder/gcpt {
inherit opensbi;
};
img = callPackage ./imgBuilder {
inherit gcpt;
};

nemu = callPackage ./cptBuilder/nemu {};
qemu = callPackage ./cptBuilder/qemu {};
simpoint = callPackage ./cptBuilder/simpoint {};
stage1-profiling = callPackage ./cptBuilder/1.profiling.nix {
inherit qemu nemu img;
};
stage2-cluster = callPackage ./cptBuilder/2.cluster.nix {
inherit simpoint stage1-profiling;
};
stage3-checkpoint = callPackage ./cptBuilder/3.checkpoint.nix {
inherit qemu nemu img stage2-cluster;
};
cpt = callPackage ./cptBuilder {
inherit stage3-checkpoint;
};};
# refer to <nixpkgs>/pkgs/top-level/stage.nix
toFix = lib.foldl' (lib.flip lib.extends) (self: {}) [
allPackages
overlay
];
in lib.fix toFix
7 changes: 1 addition & 6 deletions builders/imgBuilder/default.nix
Original file line number Diff line number Diff line change
@@ -1,8 +1,3 @@
{ callPackage

, benchmark
}: let
gcpt = callPackage ./gcpt { inherit benchmark; };
in gcpt.overrideAttrs (old: {
{ gcpt }: gcpt.overrideAttrs (old: {
passthru = { inherit gcpt; };
})
10 changes: 4 additions & 6 deletions builders/imgBuilder/gcpt/default.nix
Original file line number Diff line number Diff line change
@@ -1,13 +1,11 @@
{ stdenv
, fetchFromGitHub
, callPackage

, riscv64-cc
, benchmark
}: let
opensbi = callPackage ../opensbi { inherit benchmark; };
in stdenv.mkDerivation {
name = "${benchmark.name}.gcpt";
, rmExt
, opensbi
}: stdenv.mkDerivation {
name = "${rmExt opensbi.name}.gcpt";

src = fetchFromGitHub {
owner = "OpenXiangShan";
Expand Down
Loading

0 comments on commit 2ec5eeb

Please sign in to comment.