Build magma without conda and push tarball

First step towards removing conda from magma builds.
This still relies on the python conda image to do the
builds, as that image already contains the required
dependencies and various versions of cuda.

Use a build script to build magma, instead of conda-build,
to produce the binary tarball without the rest of the conda
package.

Upload the package to downloads.pytorch.org instead of the
anaconda pytorch channel (TBD).

This will require an update to the "install_conda.sh" script
so that magma can be installed from that tarball directly.

Signed-off-by: Andrea Frittoli <[email protected]>

magma/Makefile

@@ -1,10 +1,11 @@
 SHELL=/usr/bin/env bash
 
+DOCKER_CMD ?= docker
 DESIRED_CUDA ?= 11.8
 PACKAGE_NAME ?= magma-cuda118
 CUDA_ARCH_LIST ?= -gencode arch=compute_50,code=sm_50 -gencode arch=compute_60,code=sm_60 -gencode arch=compute_70,code=sm_70 -gencode arch=compute_80,code=sm_80 -gencode arch=compute_86,code=sm_86 -gencode arch=compute_90,code=sm_90
 
-DOCKER_RUN = set -eou pipefail; docker run --rm -i \
+DOCKER_RUN = set -eou pipefail; ${DOCKER_CMD} run --rm -i \
 	-v $(shell git rev-parse --show-toplevel):/builder \
 	-w /builder \
 	-e DESIRED_CUDA=${DESIRED_CUDA} \

magma/build_magma.sh

@@ -2,21 +2,53 @@
 
 set -eou pipefail
 
-# Create a folder to be packaged
-PACKAGE_DIR=magma/${PACKAGE_NAME}
-PACKAGE_FILES=magma/package_files
-mkdir ${PACKAGE_DIR}
-cp ${PACKAGE_FILES}/build.sh ${PACKAGE_DIR}/build.sh
-cp ${PACKAGE_FILES}/meta.yaml ${PACKAGE_DIR}/meta.yaml
-cp ${PACKAGE_FILES}/thread_queue.patch ${PACKAGE_DIR}/thread_queue.patch
-cp ${PACKAGE_FILES}/cmakelists.patch ${PACKAGE_DIR}/cmakelists.patch
-cp ${PACKAGE_FILES}/getrf_shfl.patch ${PACKAGE_DIR}/getrf_shfl.patch
-cp ${PACKAGE_FILES}/getrf_nbparam.patch ${PACKAGE_DIR}/getrf_nbparam.patch
-cp ${PACKAGE_FILES}/CMake.patch ${PACKAGE_DIR}/CMake.patch
+# Environment variables
+# The script expects DESIRED_CUDA and PACKAGE_NAME to be set
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+MAGMA_VERSION=2.6.1
 
-conda install -yq conda-build conda-verify
-. ./conda/switch_cuda_version.sh "${DESIRED_CUDA}"
-(
-    set -x
-    conda build --output-folder magma/output "${PACKAGE_DIR}"
-)
+# Folders for the build
+PACKAGE_FILES=${ROOT_DIR}/magma/package_files # source patches and metadata
+PACKAGE_DIR=${ROOT_DIR}/magma/${PACKAGE_NAME} # build workspace
+PACKAGE_OUTPUT=${ROOT_DIR}/magma/output # where tarballs are stored
+PACKAGE_BUILD=${PACKAGE_DIR}/build # where the content of the tarball is prepared
+PACKAGE_RECIPE=${PACKAGE_BUILD}/info/recipe
+PACKAGE_LICENSE=${PACKAGE_BUILD}/info/license
+mkdir -p ${PACKAGE_DIR} ${PACKAGE_OUTPUT} ${PACKAGE_BUILD} ${PACKAGE_RECIPE} ${PACKAGE_LICENSE}
+
+# Fetch magma sources and verify checksum
+pushd ${PACKAGE_DIR}
+curl -LO http://icl.utk.edu/projectsfiles/magma/downloads/magma-${MAGMA_VERSION}.tar.gz
+tar zxf magma-${MAGMA_VERSION}.tar.gz
+sha256sum --check < ${PACKAGE_FILES}/magma-${MAGMA_VERSION}.sha256
+popd
+
+# Select cuda version
+. ${ROOT_DIR}/conda/switch_cuda_version.sh "${DESIRED_CUDA}"
+
+# Apply patches and build
+pushd ${PACKAGE_DIR}/magma-${MAGMA_VERSION}
+patch < ${PACKAGE_FILES}/CMake.patch
+patch < ${PACKAGE_FILES}/cmakelists.patch
+patch -p0 < ${PACKAGE_FILES}/thread_queue.patch
+patch -p1 < ${PACKAGE_FILES}/getrf_shfl.patch
+patch -p1 < ${PACKAGE_FILES}/getrf_nbparam.patch
+# The build.sh script expects to be executed from the sources root folder
+INSTALL_DIR=${PACKAGE_BUILD} ${PACKAGE_FILES}/build.sh
+popd
+
+# Package recipe, license and tarball
+# Folder and package name are backward compatible for the build workflow
+cp ${PACKAGE_FILES}/build.sh ${PACKAGE_RECIPE}/build.sh
+cp ${PACKAGE_FILES}/meta.yaml ${PACKAGE_RECIPE}/meta.yaml
+cp ${PACKAGE_FILES}/thread_queue.patch ${PACKAGE_RECIPE}/thread_queue.patch
+cp ${PACKAGE_FILES}/cmakelists.patch ${PACKAGE_RECIPE}/cmakelists.patch
+cp ${PACKAGE_FILES}/getrf_shfl.patch ${PACKAGE_RECIPE}/getrf_shfl.patch
+cp ${PACKAGE_FILES}/getrf_nbparam.patch ${PACKAGE_RECIPE}/getrf_nbparam.patch
+cp ${PACKAGE_FILES}/CMake.patch ${PACKAGE_RECIPE}/CMake.patch
+cp ${PACKAGE_FILES}/magma-${MAGMA_VERSION}.sha256 ${PACKAGE_RECIPE}/magma-${MAGMA_VERSION}.sha256
+cp ${PACKAGE_DIR}/magma-${MAGMA_VERSION}/COPYRIGHT ${PACKAGE_LICENSE}/COPYRIGHT
+pushd ${PACKAGE_BUILD}
+mkdir linux-64
+tar cjf ${PACKAGE_OUTPUT}/linux-64/${PACKAGE_NAME}-${MAGMA_VERSION}-1.tar.gz include lib info
+popd

magma/package_files/build.sh

@@ -1,7 +1,3 @@
-export CMAKE_LIBRARY_PATH=$PREFIX/lib:$PREFIX/include:$CMAKE_LIBRARY_PATH
-export CMAKE_PREFIX_PATH=$PREFIX
-export PATH=$PREFIX/bin:$PATH
-
 CUDA__VERSION=$(nvcc --version|sed -n 4p|cut -f5 -d" "|cut -f1 -d",")
 if [ "$CUDA__VERSION" != "$DESIRED_CUDA" ]; then
     echo "CUDA Version is not $DESIRED_CUDA. CUDA Version found: $CUDA__VERSION"
@@ -10,7 +6,7 @@ fi
 
 mkdir build
 cd build
-cmake .. -DUSE_FORTRAN=OFF -DGPU_TARGET="All" -DCMAKE_INSTALL_PREFIX=$PREFIX -DCUDA_ARCH_LIST="$CUDA_ARCH_LIST"
+cmake .. -DUSE_FORTRAN=OFF -DGPU_TARGET="All" -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" -DCUDA_ARCH_LIST="$CUDA_ARCH_LIST"
 make -j$(getconf _NPROCESSORS_CONF)
 make install
 cd ..

magma/package_files/magma-2.6.1.sha256

@@ -0,0 +1 @@
+6cd83808c6e8bc7a44028e05112b3ab4e579bcc73202ed14733f66661127e213  magma-2.6.1.tar.gz