upgrade pytorch and cuda versions (#141)

1 upgrade dependencies: pytorch version to 2.3.0 and cuda to 12.1
2 add dockerfiles for devel and wheel images
3 python package: deprecate support for python 3.7 and add python 3.11. Refer to: https://pytorch.org/blog/deprecation-cuda-python-support/

.github/workflows/ubuntu20.04-py3.8-cuda11.6-ci.yml → .github/workflows/glt-ci.yml

@@ -17,7 +17,7 @@ on:
       - '**.md'
 
 env:
-  IMAGE: graphlearn/graphlearn_for_pytorch:1.0.0-ubuntu20.04-py3.8-torch1.13-cuda11.6
+  IMAGE: graphlearn-torch-dev
   JOBNAME: glt-ci-${{ github.run_id }}
   DESTDIR: /mnt/graphlearn_for_pytorch
 
@@ -34,6 +34,11 @@ jobs:
         run: |
           git submodule update --init
 
+      - name: Build image
+        working-directory: ${{github.workspace}}
+        run: |
+          docker build -f dockerfiles/${IMAGE}.Dockerfile -t ${IMAGE} .
+
       - name: Launch CI container
         working-directory: ${{github.workspace}}
         run: |

.github/workflows/glt-v6d-ci.yml

@@ -17,7 +17,7 @@ on:
       - '**.md'
 
 env:
-  IMAGE: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-dev:latest
+  IMAGE: gs-graphlearn-torch-dev
   JOBNAME: glt-v6d-ci-${{ github.run_id }}
   DESTDIR: /home/graphscope/graphlearn-for-pytorch
 
@@ -34,6 +34,11 @@ jobs:
         run: |
           git submodule update --init
 
+      - name: Build image
+        working-directory: ${{github.workspace}}
+        run: |
+          docker build -f dockerfiles/${IMAGE}.Dockerfile -t ${IMAGE} .
+
       - name: Launch CI container
         working-directory: ${{github.workspace}}
         run: |

.github/workflows/manylinux-cuda11.7-cd.yml → .github/workflows/manylinux-cd.yml

@@ -6,7 +6,7 @@ on:
       - '*'
 
 env:
-  IMAGE: pytorch/manylinux-cuda117
+  IMAGE: graphlearn-torch-wheel
   JOBNAME: glt-cd-${{ github.run_id }}
   DESTDIR: /mnt/graphlearn_for_pytorch
 
@@ -22,6 +22,11 @@ jobs:
         working-directory: ${{github.workspace}}
         run: |
           git submodule update --init
+          
+      - name: Build image
+        working-directory: ${{github.workspace}}
+        run: |
+          docker build -f dockerfiles/${IMAGE}.Dockerfile -t ${IMAGE} .
 
       - name: Launch CD container
         working-directory: ${{github.workspace}}

.github/workflows/scripts/build.sh

@@ -6,7 +6,7 @@ GLT_ROOT_DIR=$(dirname $(dirname $(dirname $(dirname "$(realpath "$0")"))))
 GITHUB_REF=$1
 
 rm -rf /usr/local/cuda
-ln -s /usr/local/cuda-11.7 /usr/local/cuda
+ln -s /usr/local/cuda-12.1 /usr/local/cuda
 
 cd $GLT_ROOT_DIR
 
@@ -17,7 +17,7 @@ fi
 if [ -z "$GITHUB_TAG_REF" ]; then
   echo "Not on a tag, won't deploy to pypi"
 else
-  PYABIS="cp37-cp37m cp38-cp38 cp39-cp39 cp310-cp310"
+  PYABIS="cp38-cp38 cp39-cp39 cp310-cp310 cp311-cp311"
   for abi in $PYABIS; do
     PYABI=$abi bash .github/workflows/scripts/build_wheel.sh
   done

.github/workflows/scripts/build_wheel.sh

@@ -8,13 +8,6 @@ PYBIN=/opt/python/${PYABI}/bin
 cd $GLT_ROOT_DIR
 
 set -x
-
-${PYBIN}/pip install ninja
-${PYBIN}/pip install scipy
-${PYBIN}/pip install torch==1.13.0+cu117 torchvision==0.14.0+cu117 torchaudio==0.13.0 --extra-index-url https://download.pytorch.org/whl/cu117
-${PYBIN}/pip install torch_geometric
-${PYBIN}/pip install --no-index  torch_scatter torch_sparse -f https://data.pyg.org/whl/torch-1.13.0+cu117.html
-${PYBIN}/pip install auditwheel
 RELEASE=TRUE WITH_CUDA=ON ${PYBIN}/python setup.py bdist_wheel
 
 # Bundle external shared libraries into the wheels

CMakeLists.txt

@@ -1,6 +1,6 @@
 cmake_minimum_required(VERSION 3.10)
 project(GraphLearn-for-PyTorch)
-set(CMAKE_CXX_STANDARD 14)
+set(CMAKE_CXX_STANDARD 17) # C++17 or later compatible compiler is required to use PyTorch(2.3.0).
 
 option(DEBUG "Enable debug mode" OFF)
 option(BUILD_TESTS "Enable testing" ON)

dockerfiles/graphlearn-torch-dev.Dockerfile

@@ -0,0 +1,16 @@
+ARG GLT_CUDA_VERSION=12.1
+ARG GLT_TORCH_VERSION=2.3.0
+
+FROM nvidia/cuda:${GLT_CUDA_VERSION}.0-devel-ubuntu22.04
+
+RUN apt update && apt install python3-pip vim git cmake -y
+RUN ln -s python3 /usr/bin/python
+
+ARG GLT_TORCH_VERSION GLT_CUDA_VERSION
+
+RUN CUDA_WHEEL_VERSION=$(echo "$GLT_CUDA_VERSION" | sed 's/\.//g') \
+    && pip install torch==${GLT_TORCH_VERSION} --index-url https://download.pytorch.org/whl/cu${CUDA_WHEEL_VERSION} \
+    && pip install torch_geometric \
+    && pip install pyg_lib torch_scatter torch_sparse torch_cluster torch_spline_conv \
+    -f https://data.pyg.org/whl/torch-${GLT_TORCH_VERSION}+cu${CUDA_WHEEL_VERSION}.html
+

dockerfiles/graphlearn-torch-wheel.Dockerfile

@@ -0,0 +1,18 @@
+ARG GLT_TORCH_VERSION=2.3.0
+ARG CUDA_WHEEL_VERSION=121
+
+FROM pytorch/manylinux-cuda$CUDA_WHEEL_VERSION
+
+ARG GLT_TORCH_VERSION CUDA_WHEEL_VERSION
+ENV ABIS="cp38-cp38 cp39-cp39 cp310-cp310 cp311-cp311"
+
+RUN set -ex; \
+    for abi in $ABIS; do \
+        PYBIN=/opt/python/${abi}/bin; \
+        ${PYBIN}/pip install ninja scipy; \
+        ${PYBIN}/pip install torch==${GLT_TORCH_VERSION} --index-url https://download.pytorch.org/whl/cu${CUDA_WHEEL_VERSION}; \
+        ${PYBIN}/pip install torch_geometric; \
+        ${PYBIN}/pip install pyg_lib torch_scatter torch_sparse torch_cluster torch_spline_conv \
+            -f https://data.pyg.org/whl/torch-${GLT_TORCH_VERSION}+cu${CUDA_WHEEL_VERSION}.html; \
+        ${PYBIN}/pip install auditwheel; \
+    done

dockerfiles/gs-graphlearn-torch-dev.Dockerfile

@@ -0,0 +1,13 @@
+ARG REGISTRY=registry.cn-hongkong.aliyuncs.com
+ARG GLT_TORCH_VERSION=2.3.0
+ARG GS_VERSION=latest
+
+FROM $REGISTRY/graphscope/graphscope-dev:$GS_VERSION
+
+ARG GLT_TORCH_VERSION
+
+RUN pip install torch==${GLT_TORCH_VERSION} --index-url https://download.pytorch.org/whl/cpu \
+    && pip install torch_geometric \
+    && pip install pyg_lib torch_scatter torch_sparse torch_cluster torch_spline_conv \
+    -f https://data.pyg.org/whl/torch-${GLT_TORCH_VERSION}+cpu.html
+

scripts/build_wheel.sh

@@ -24,17 +24,10 @@ else
   VINEYARD_OPTION=$WITH_VINEYARD
 fi
 
-if [ "$VINEYARD_OPTION" = "ON" ]; then
-  # in graphscope environment
-  python3 -m pip install torch==1.13 --index-url https://download.pytorch.org/whl/cpu
-  python3 -m pip install torch_geometric ogb 
-  python3 -m pip install pyg_lib torch_scatter torch_sparse torch_cluster torch_spline_conv -f https://data.pyg.org/whl/torch-1.13.0+cpu.html
-fi
-
 # TODO(hongyi): build cpp with v6d
 # cmake -DWITH_CUDA=$CUDA_OPTION -DWITH_VINEYARD=$VINEYARD_OPTION .
 cmake -DWITH_CUDA=$CUDA_OPTION .
 make -j$CORES
 
 python3 setup.py bdist_wheel
-python3 -m pip install install dist/* --force-reinstall
+python3 -m pip install dist/* --force-reinstall

scripts/launch_container.sh

@@ -11,19 +11,6 @@ WITH_VINEYARD=${WITH_VINEYARD:-OFF}
 WITH_CUDA=${WITH_CUDA:-ON}
 MOUNT_VOLUME=${MOUNT_VOLUME:-TRUE}
 
-ret=0
-for i in $(seq 1 3); do
-  [ $i -gt 1 ] && echo "WARNING: pull image failed, will retry in $((i-1)) times later" && sleep 10
-  ret=0
-  docker pull $IMAGE_NAME && break || ret=$?
-done
-
-if [ $ret -ne 0 ]
-then
-  echo "ERROR: Pull Image $IMAGE_NAME failed, exit."
-  exit $ret
-fi
-
 docker_args="-itd --name $JOB_NAME --shm-size=1g -e WITH_VINEYARD=$WITH_VINEYARD -e WITH_CUDA=$WITH_CUDA"
 
 if [ "$WITH_CUDA" = "ON" ]