Rebuild lotus and ffi when running on certain intel processors (Intel(R) Xeon(R) CPU E5-2673 v3)

tests/tools/testnet/Dockerfile

@@ -4,7 +4,7 @@ FROM ubuntu:20.10
 
 RUN apt update
 RUN apt-get install -y software-properties-common
-RUN apt install -y --no-install-recommends build-essential git jq python netcat curl wget pkg-config libgl-dev libglu-dev libglib2.0-dev libsm-dev libxrender-dev libfontconfig1-dev libxext-dev nvidia-opencl-dev ocl-icd-opencl-dev tmux
+RUN apt install -y --no-install-recommends build-essential git jq python netcat curl wget pkg-config libgl-dev libglu-dev libglib2.0-dev libsm-dev libxrender-dev libfontconfig1-dev libxext-dev nvidia-opencl-dev ocl-icd-opencl-dev libclang-dev
 RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
 
 #install node

tests/tools/testnet/bin/build_lotus.sh

@@ -1,17 +1,4 @@
 #!/usr/bin/env bash
-
-# USAGE:
-#
-# Option 1: Build and run tests using specific lotus Git SHA:
-#
-# > ./test-storage-and-retrieval-local-dev-net.sh --lotus-git-sha=492ffac913287c732cc5d54d0deddc44e41b642f
-#
-# Option 2: Build and run using binaries you've built previously (much faster)
-#
-# > cd $LOTUS_CHECKOUT_DIR && make clean deps debug lotus-shed fountain
-# > ./test-storage-and-retrieval-local-dev-net.sh --copy-binaries-from-dir=$LOTUS_CHECKOUT_DIR
-#
-
 set -Exo pipefail
 
 base_dir="/localnet.test"
@@ -26,7 +13,8 @@ deps=(printf paste jq python nc)
 #lotus_git_sha="009e14b679edf5f5d66481b0802427063ea47ee2" #calibration 18 aug
 #lotus_git_sha="4f45c623a5896637a9239d16b1836fdb635bb8f9" #master top 31 aug
 #lotus_git_sha="636810daa5e63a6ec132d78993d028a41f179276" #v0.8.1
-lotus_git_sha="a6b2180756db71574b385ef3c357f2b943252e78" #v0.10.0
+#lotus_git_sha="a6b2180756db71574b385ef3c357f2b943252e78" #v0.10.0
+lotus_git_sha="8340124786c244b21806e117e84b67fc6117686d" #v0.10.2
 
 copy_binaries_from_dir=""
 other_args=()

tests/tools/testnet/bin/build_lotus_ffi.sh

@@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+set -Exo pipefail
+
+base_dir="/localnet.test"
+build_log_path=$(mktemp)
+deps=(printf paste jq python nc)
+#lotus_git_sha="492ffac913287c732cc5d54d0deddc44e41b642f" #initial commit
+#lotus_git_sha="edcf899be1deb710811f8f9161fe82ef8cd4be91" #master branch top 19 jun
+#lotus_git_sha="bead3bf412954c02ac82e5367e6e9c498a8a897b" #interop branch top19 jun
+#lotus_git_sha="e5fdae45b78ac6dd6533b26a23ac263284831cbf" #master top 3 jul
+#lotus_git_sha="3f017688c7ae180d12d4ee28f732298299ee351f" #master top 13 jul
+#lotus_git_sha="b84030b3bdf19b953cbf330710453d20f3f3abfd" #calibration 20 jul
+#lotus_git_sha="009e14b679edf5f5d66481b0802427063ea47ee2" #calibration 18 aug
+#lotus_git_sha="4f45c623a5896637a9239d16b1836fdb635bb8f9" #master top 31 aug
+#lotus_git_sha="636810daa5e63a6ec132d78993d028a41f179276" #v0.8.1
+#lotus_git_sha="a6b2180756db71574b385ef3c357f2b943252e78" #v0.10.0
+lotus_git_sha="8340124786c244b21806e117e84b67fc6117686d" #v0.10.2
+
+copy_binaries_from_dir=""
+other_args=()
+
+# ensure that script dependencies are met
+#
+for dep in ${deps[@]}; do
+    if ! which "${dep}"; then
+        (>&2 echo "please install ${dep} before running this script")
+        exit 1
+    fi
+done
+
+
+# create some directories which we'll need later
+#
+rm -r "${base_dir}"
+
+mkdir -p "${base_dir}"
+mkdir -p "${base_dir}/scripts"
+mkdir -p "${base_dir}/bin"
+mkdir -p "${base_dir}/version"
+
+cat > "${base_dir}/version/build_commit.txt" <<EOF
+${lotus_git_sha}
+EOF
+
+cat > "${base_dir}/scripts/build_ffi.bash" <<EOF
+#!/usr/bin/env bash
+set -xe
+
+if [[ ! -z "${copy_binaries_from_dir}" ]]; then
+    pushd ${copy_binaries_from_dir}
+    cp lotus lotus-miner lotus-shed lotus-seed lotus-fountain ${base_dir}/bin/
+    popd
+fi
+
+if [[ ! -z "${lotus_git_sha}" ]]; then
+    git clone https://github.com/filecoin-project/lotus.git "${base_dir}/build"
+    pushd "${base_dir}/build" && git reset --hard "${lotus_git_sha}" && popd
+
+    SCRIPTDIR="\$( cd "\$( dirname "\${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+    pushd \$SCRIPTDIR/../build
+    pwd
+    source $HOME/.cargo/env
+    export RUSTFLAGS="-C target-cpu=native -g"
+    export FFI_BUILD_FROM_SOURCE=1
+    make clean deps debug lotus-shed lotus-fountain
+    cp lotus lotus-miner lotus-shed lotus-seed lotus-fountain ${base_dir}/bin/
+    popd
+fi
+EOF
+
+chmod +x "${base_dir}/scripts/build_ffi.bash"
+
+# build various lotus binaries
+#
+bash "${base_dir}/scripts/build_ffi.bash" 2>&1 | tee -a "${build_log_path}"
+
+if [ $? -eq 0 ]
+then
+  echo "built successfully"
+else
+  echo "failed to build: check ${build_log_path} for more details" >&2
+  exit 1
+fi
+

tests/tools/testnet/bin/entrypoint.sh

@@ -1,8 +1,24 @@
 #!/usr/bin/bash
 
 chmod +x "/testnet/start_testnet.sh"
+chmod +x "/testnet/build_lotus.sh"
+chmod +x "/testnet/init_testnet.sh"
 chmod +x "/testnet/set_env.sh"
 
+cpuName=$( lscpu | grep "Model name" | sed -r 's/Model name:\s{1,}(.*) @ .*z\s*/\1/g' )
+
+echo $cpuName
+
+IFS=$'\t'
+array=("Intel(R) Xeon(R) CPU E5-2673 v3")
+unset IFS
+
+if [[ "\t${array[@]}\t" =~ "\t${cpuName}\t" ]]; then
+    echo "rebuild ffi"
+    /testnet/build_lotus_ffi.sh
+    /testnet/init_testnet.sh
+fi
+
 /testnet/start_testnet.sh
 
 source /filecoin_miner/scripts/env-client.bash

tests/tools/testnet/bin/run_tests_ci.sh

@@ -6,7 +6,7 @@ echo 'waiting for testnetwork'
 i=0
 pollPeriod=15
 testCurlStatus=0
-timeoutPeriod=320
+timeoutPeriod=1800
 succesTestStatus=200
 
 while [[ $testCurlStatus != $succesTestStatus && $i -lt $timeoutPeriod ]]

tests/tools/testnet/docker-compose-no-image-build.yml

@@ -18,6 +18,8 @@ services:
 
     volumes:
       - ./credentials:/credentials:delegated
+      - ./bin:/testnet:delegated
+      - ./proxy:/testnet/proxy:delegated
       - filecoin_js_node_modules_volume:/src/node_modules
 
 

tests/tools/testnet/docker-compose-tests.yml

@@ -5,14 +5,16 @@ volumes:
 services:
   filecoin_js_infra:
     container_name: 'filecoin_js_infra'
-    image: smoothop/filecoin.js-testing-framework:v0.10.0
+    image: smoothop/filecoin.js-testing-framework:v0.10.2
     ports:
       - 8000:8000
       - 4502:4502
     entrypoint: /testnet/entrypoint.sh
 
     volumes:
       - ./credentials:/credentials:delegated
+      - ./bin:/testnet:delegated
+      - ./proxy:/testnet/proxy:delegated
       - filecoin_js_node_modules_volume:/src/node_modules
 
 

tests/tools/testnet/docker-compose.yml

@@ -9,14 +9,16 @@ volumes:
 services:
   filecoin_js_infra:
     container_name: 'filecoin_js_infra'
-    image: smoothop/filecoin.js-testing-framework:v0.10.0
+    image: smoothop/filecoin.js-testing-framework:v0.10.2
     ports:
       - 8000:8000
       - 4502:4502
     entrypoint: /testnet/entrypoint.sh
 
     volumes:
       - ./credentials:/credentials:delegated
+      - ./bin:/testnet:delegated
+      - ./proxy:/testnet/proxy:delegated
       - filecoin_js_node_modules_volume:/src/node_modules