From 87c487a3109edd819945763d5aa2547057f00b83 Mon Sep 17 00:00:00 2001
From: sasha0552 <admin@sasha0552.org>
Date: Thu, 28 Dec 2023 09:25:11 +0000
Subject: [PATCH 01/23] Initial ROCm support

---
 .ci/configure.py                              | 44 ++++++++++++
 .github/workflows/build-iso-cuda.yml          |  5 +-
 .github/workflows/build-iso-rocm.yml          | 67 +++++++++++++++++++
 .gitignore                                    |  6 ++
 .gitignore.jinja2                             |  7 ++
 airootfs/root/customize_airootfs.sh           |  3 +
 ...es.sh => 0100-automatic-patches.sh.jinja2} |  2 +
 ...es.sh => 0100-koboldcpp-patches.sh.jinja2} |  2 +
 ...patches.sh => 0100-vllm-patches.sh.jinja2} |  2 +
 packages.x86_64 => packages.x86_64.jinja2     | 15 ++++-
 10 files changed, 149 insertions(+), 4 deletions(-)
 create mode 100644 .ci/configure.py
 create mode 100644 .github/workflows/build-iso-rocm.yml
 create mode 100644 .gitignore.jinja2
 rename airootfs/root/customize_airootfs/scripts/{0100-automatic-patches.sh => 0100-automatic-patches.sh.jinja2} (96%)
 rename airootfs/root/customize_airootfs/scripts/{0100-koboldcpp-patches.sh => 0100-koboldcpp-patches.sh.jinja2} (86%)
 rename airootfs/root/customize_airootfs/scripts/{0100-vllm-patches.sh => 0100-vllm-patches.sh.jinja2} (85%)
 rename packages.x86_64 => packages.x86_64.jinja2 (80%)

diff --git a/.ci/configure.py b/.ci/configure.py
new file mode 100644
index 0000000..b29e691
--- /dev/null
+++ b/.ci/configure.py
@@ -0,0 +1,44 @@
+import glob
+import jinja2
+import sys
+
+def render_template(filepath, **options):
+  if filepath.endswith(".jinja2"):
+    # read input file
+    with open(filepath, "r") as file:
+      template = jinja2.Template(file.read())
+
+    # render template
+    rendered = template.render(**options)
+
+    # write output file
+    with open(filepath[:-7], "w") as file:
+      file.write(rendered)
+
+def main():
+  # by default, use cuda
+  cuda = True
+  rocm = False
+
+  # enable rocm if specified
+  if len(sys.argv) == 2:
+    if sys.argv[1] == "rocm":
+      cuda = False
+      rocm = True
+
+  # list of rendered files
+  rendered = []
+
+  # render every file
+  for filepath in glob.glob("**/*.jinja2", recursive=True) + [".gitignore.jinja2"]:
+    # render file
+    render_template(filepath, CUDA=cuda, ROCm=rocm, rendered=rendered)
+
+    # add output file to rendered list
+    rendered.append(filepath[:-7])
+
+    # print status
+    print(f"File '{filepath}' rendered successfully")
+
+if __name__ == "__main__":
+  main()
diff --git a/.github/workflows/build-iso-cuda.yml b/.github/workflows/build-iso-cuda.yml
index 7b1e7f6..e609df6 100644
--- a/.github/workflows/build-iso-cuda.yml
+++ b/.github/workflows/build-iso-cuda.yml
@@ -35,7 +35,7 @@ jobs:
             pacman --sync --noconfirm --sysupgrade
 
             # Install required packages
-            pacman --sync --noconfirm --needed archiso patch
+            pacman --sync --noconfirm --needed archiso patch python python-jinja
 
             # Apply patch to archiso
             patch -p0 << 'EOF'
@@ -54,6 +54,9 @@ jobs:
              # export build artifacts for netboot
             EOF
 
+            # Configure to use CUDA
+            /workspace/.ci/configure.py cuda
+
             # Build image
             mkarchiso -v -m iso -w /workspace/work -o /workspace/out /workspace
 
diff --git a/.github/workflows/build-iso-rocm.yml b/.github/workflows/build-iso-rocm.yml
new file mode 100644
index 0000000..251e637
--- /dev/null
+++ b/.github/workflows/build-iso-rocm.yml
@@ -0,0 +1,67 @@
+name: Build ISO (ROCm)
+
+on:
+  - push
+  - pull_request
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Cleanup
+        uses: rokibhasansagar/slimhub_actions@main
+        with:
+          retain: "docker_imgcache,docker_buildkit,docker_imgcache"
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Build image
+        uses: addnab/docker-run-action@v3
+        with:
+          image: archlinux:latest
+          options: --privileged --volume ${{ github.workspace }}:/workspace
+          run: |
+            # Exit on error
+            set -eu
+
+            # Refresh package databases
+            pacman --sync --noconfirm --refresh
+
+            # Upgrade system
+            pacman --sync --noconfirm --sysupgrade
+
+            # Install required packages
+            pacman --sync --noconfirm --needed archiso patch python python-jinja
+
+            # Apply patch to archiso
+            patch -p0 << 'EOF'
+            --- /usr/bin/mkarchiso
+            +++ /usr/bin/mkarchiso
+            @@ -1227,6 +1227,10 @@
+                 if [[ -v cert_list ]]; then
+                     _cms_sign_artifact "${airootfs_image_filename}"
+                 fi
+            +
+            +    _msg_info 'Removing the pacstrap directory...'
+            +    rm -rf -- "${pacstrap_dir:?}/"
+            +    _msg_info 'Done!'
+             }
+
+             # export build artifacts for netboot
+            EOF
+
+            # Configure to use ROCm
+            /workspace/.ci/configure.py rocm
+
+            # Build image
+            mkarchiso -v -m iso -w /workspace/work -o /workspace/out /workspace
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: archiso-output
+          path: out/
diff --git a/.gitignore b/.gitignore
index 4cb60ed..1666f4e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,8 @@
 out/
 work/
+
+# rendered files
+packages.x86_64
+airootfs/root/customize_airootfs/scripts/0100-koboldcpp-patches.sh
+airootfs/root/customize_airootfs/scripts/0100-vllm-patches.sh
+airootfs/root/customize_airootfs/scripts/0100-automatic-patches.sh
diff --git a/.gitignore.jinja2 b/.gitignore.jinja2
new file mode 100644
index 0000000..e8b6e74
--- /dev/null
+++ b/.gitignore.jinja2
@@ -0,0 +1,7 @@
+out/
+work/
+
+# rendered files
+{% for file in rendered %}
+{{- file}}
+{% endfor %}
diff --git a/airootfs/root/customize_airootfs.sh b/airootfs/root/customize_airootfs.sh
index b2f7978..6c68b0e 100644
--- a/airootfs/root/customize_airootfs.sh
+++ b/airootfs/root/customize_airootfs.sh
@@ -7,6 +7,9 @@ mv /usr/lib/os-release.new /usr/lib/os-release
 # set user password
 echo "tori:tori" | chpasswd
 
+# remove any jinja2 files
+find -type f -name "*.jinja2" -print -delete
+
 # enter user directory
 cd "/home/tori"
 
diff --git a/airootfs/root/customize_airootfs/scripts/0100-automatic-patches.sh b/airootfs/root/customize_airootfs/scripts/0100-automatic-patches.sh.jinja2
similarity index 96%
rename from airootfs/root/customize_airootfs/scripts/0100-automatic-patches.sh
rename to airootfs/root/customize_airootfs/scripts/0100-automatic-patches.sh.jinja2
index f7f11cc..b5021ce 100644
--- a/airootfs/root/customize_airootfs/scripts/0100-automatic-patches.sh
+++ b/airootfs/root/customize_airootfs/scripts/0100-automatic-patches.sh.jinja2
@@ -14,6 +14,8 @@ pushd "automatic"
   sed -i 's/lambda: {"choices": theme.list_themes()}, refresh=theme.refresh_themes/{"choices": ["black-teal"]}/g' modules/shared.py
   sed -i 's/shared.opts.motd/False/g' modules/api/api.py
 
+{% if CUDA %}
   # drop pstate in idle
   patch -p1 < "$CUSTOMIZE_AIROOTFS/patches/0000-automatic-drop-pstate-in-idle.patch"
+{% endif %}
 popd
diff --git a/airootfs/root/customize_airootfs/scripts/0100-koboldcpp-patches.sh b/airootfs/root/customize_airootfs/scripts/0100-koboldcpp-patches.sh.jinja2
similarity index 86%
rename from airootfs/root/customize_airootfs/scripts/0100-koboldcpp-patches.sh
rename to airootfs/root/customize_airootfs/scripts/0100-koboldcpp-patches.sh.jinja2
index 2dc2267..fc1df65 100644
--- a/airootfs/root/customize_airootfs/scripts/0100-koboldcpp-patches.sh
+++ b/airootfs/root/customize_airootfs/scripts/0100-koboldcpp-patches.sh.jinja2
@@ -3,6 +3,8 @@ set -eu
 
 # koboldcpp patches
 pushd "koboldcpp"
+{% if CUDA %}
   # drop pstate in idle
   patch -p1 < "$CUSTOMIZE_AIROOTFS/patches/0000-koboldcpp-drop-pstate-in-idle.patch"
+{% endif %}
 popd
diff --git a/airootfs/root/customize_airootfs/scripts/0100-vllm-patches.sh b/airootfs/root/customize_airootfs/scripts/0100-vllm-patches.sh.jinja2
similarity index 85%
rename from airootfs/root/customize_airootfs/scripts/0100-vllm-patches.sh
rename to airootfs/root/customize_airootfs/scripts/0100-vllm-patches.sh.jinja2
index c1b18dc..abe8c07 100644
--- a/airootfs/root/customize_airootfs/scripts/0100-vllm-patches.sh
+++ b/airootfs/root/customize_airootfs/scripts/0100-vllm-patches.sh.jinja2
@@ -3,6 +3,8 @@ set -eu
 
 # vllm patches
 pushd "vllm"
+{% if CUDA %}
   # build for pascal
   patch -p1 < "$CUSTOMIZE_AIROOTFS/patches/0100-vllm-build-for-pascal.patch"
+{% endif %}
 popd
diff --git a/packages.x86_64 b/packages.x86_64.jinja2
similarity index 80%
rename from packages.x86_64
rename to packages.x86_64.jinja2
index 26fb240..e1e19a3 100644
--- a/packages.x86_64
+++ b/packages.x86_64.jinja2
@@ -3,7 +3,6 @@ base
 base-devel
 btop
 cmake
-cuda
 curl
 darkhttpd
 ffmpeg
@@ -22,8 +21,6 @@ nano
 ncdu
 neofetch
 nfs-utils
-nvidia
-nvidia-utils
 nvtop
 openssh
 progress
@@ -36,3 +33,15 @@ sudo
 syslinux
 wget
 zsh
+
+{% if CUDA %}
+# CUDA
+cuda
+nvidia
+nvidia-utils
+{% endif %}
+
+{% if ROCm %}
+# ROCm
+rocm-hip-sdk
+{% endif %}

From 9ddc1aa63a252ca844b7bfe759764bf7eab12d44 Mon Sep 17 00:00:00 2001
From: sasha0552 <admin@sasha0552.org>
Date: Thu, 28 Dec 2023 09:32:02 +0000
Subject: [PATCH 02/23] Update README.md [skip ci]

---
 README.md | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index 6b96f83..e640ad2 100644
--- a/README.md
+++ b/README.md
@@ -15,18 +15,16 @@ If you would like to see another AI-related project included in ToriLinux, pleas
 
 * Easy setup: just boot the ISO, and you will have a working setup for training and/or inferencing Large Language Models/Stable Diffusion/etc.
 * Fully offline training and/or inference.
-* Includes performance state switcher, which reduces GPU temperatures when inference is not running (only automatic & koboldcpp for now).
+* Includes performance state switcher, which reduces GPU temperatures when inference is not running (only NVIDIA, only automatic & koboldcpp for now).
 
 ## Usage
 
 To use ToriLinux:
 1. Install [Ventoy](https://ventoy.net/en/doc_start.html) on a USB drive.
-2. Download the latest ISO from [workflows](https://github.com/sasha0552/ToriLinux/actions?query=branch%3Amain) and copy it to the USB drive.
+2. Download the latest ISO from workflows ([NVIDIA](https://github.com/sasha0552/ToriLinux/actions/workflows/build-iso-cuda.yml?query=branch%3Amain) / [AMD](https://github.com/sasha0552/ToriLinux/actions/workflows/build-iso-rocm.yml?query=branch%3Amain)) and copy it to the USB drive.
 3. Boot from the USB drive (select it as the boot device in BIOS/UEFI).
 4. Log in with the username `tori` and password `tori`. You can also use [SSH](https://en.wikipedia.org/wiki/Secure_Shell).
 
-Please note that ToriLinux currently works only with NVIDIA GPUs. If you would like a ROCm (AMD GPUs) version, please open an issue, and I'll make one.
-
 ## Misc
 
 Note that you need pre-downloaded models on a local hard drive or NFS server, or enough RAM and internet connection to download models directly into RAM.

From aca10f163f4b8a6c7816b3867cb5b207193bd57e Mon Sep 17 00:00:00 2001
From: sasha0552 <admin@sasha0552.org>
Date: Thu, 28 Dec 2023 09:33:35 +0000
Subject: [PATCH 03/23] Fix CI

---
 .github/workflows/build-iso-cuda.yml | 4 +++-
 .github/workflows/build-iso-rocm.yml | 4 +++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-iso-cuda.yml b/.github/workflows/build-iso-cuda.yml
index e609df6..f4502b4 100644
--- a/.github/workflows/build-iso-cuda.yml
+++ b/.github/workflows/build-iso-cuda.yml
@@ -55,7 +55,9 @@ jobs:
             EOF
 
             # Configure to use CUDA
-            /workspace/.ci/configure.py cuda
+            pushd /workspace
+              python3 .ci/configure.py cuda
+            popd
 
             # Build image
             mkarchiso -v -m iso -w /workspace/work -o /workspace/out /workspace
diff --git a/.github/workflows/build-iso-rocm.yml b/.github/workflows/build-iso-rocm.yml
index 251e637..4656701 100644
--- a/.github/workflows/build-iso-rocm.yml
+++ b/.github/workflows/build-iso-rocm.yml
@@ -55,7 +55,9 @@ jobs:
             EOF
 
             # Configure to use ROCm
-            /workspace/.ci/configure.py rocm
+            pushd /workspace
+              python3 .ci/configure.py rocm
+            popd
 
             # Build image
             mkarchiso -v -m iso -w /workspace/work -o /workspace/out /workspace

From 530a2324c9ea83ddfc64200304edae11ea879958 Mon Sep 17 00:00:00 2001
From: sasha0552 <admin@sasha0552.org>
Date: Thu, 28 Dec 2023 09:54:30 +0000
Subject: [PATCH 04/23] Use ROCm instead of CUDA on supported projects

---
 .gitignore                                    |  5 +++++
 ...sh => 1000-axolotl-dependencies.sh.jinja2} |  2 ++
 ...sillytavern-extras-dependencies.sh.jinja2} | 11 +++++++++-
 ...t-generation-webui-dependencies.sh.jinja2} |  9 ++++++++-
 ...es.sh => 1000-vllm-dependencies.sh.jinja2} | 20 +++++++++++++++++--
 ...9999-cleanup.sh => 9999-cleanup.sh.jinja2} |  5 +++++
 6 files changed, 48 insertions(+), 4 deletions(-)
 rename airootfs/root/customize_airootfs/scripts/{1000-axolotl-dependencies.sh => 1000-axolotl-dependencies.sh.jinja2} (96%)
 rename airootfs/root/customize_airootfs/scripts/{1000-sillytavern-extras-dependencies.sh => 1000-sillytavern-extras-dependencies.sh.jinja2} (64%)
 rename airootfs/root/customize_airootfs/scripts/{1000-text-generation-webui-dependencies.sh => 1000-text-generation-webui-dependencies.sh.jinja2} (63%)
 rename airootfs/root/customize_airootfs/scripts/{1000-vllm-dependencies.sh => 1000-vllm-dependencies.sh.jinja2} (69%)
 rename airootfs/root/customize_airootfs/scripts/{9999-cleanup.sh => 9999-cleanup.sh.jinja2} (64%)

diff --git a/.gitignore b/.gitignore
index 1666f4e..f87df35 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,6 +3,11 @@ work/
 
 # rendered files
 packages.x86_64
+airootfs/root/customize_airootfs/scripts/1000-axolotl-dependencies.sh
 airootfs/root/customize_airootfs/scripts/0100-koboldcpp-patches.sh
+airootfs/root/customize_airootfs/scripts/1000-sillytavern-extras-dependencies.sh
+airootfs/root/customize_airootfs/scripts/1000-vllm-dependencies.sh
+airootfs/root/customize_airootfs/scripts/1000-text-generation-webui-dependencies.sh
 airootfs/root/customize_airootfs/scripts/0100-vllm-patches.sh
 airootfs/root/customize_airootfs/scripts/0100-automatic-patches.sh
+airootfs/root/customize_airootfs/scripts/9999-cleanup.sh
diff --git a/airootfs/root/customize_airootfs/scripts/1000-axolotl-dependencies.sh b/airootfs/root/customize_airootfs/scripts/1000-axolotl-dependencies.sh.jinja2
similarity index 96%
rename from airootfs/root/customize_airootfs/scripts/1000-axolotl-dependencies.sh
rename to airootfs/root/customize_airootfs/scripts/1000-axolotl-dependencies.sh.jinja2
index 129223f..5b15957 100644
--- a/airootfs/root/customize_airootfs/scripts/1000-axolotl-dependencies.sh
+++ b/airootfs/root/customize_airootfs/scripts/1000-axolotl-dependencies.sh.jinja2
@@ -3,6 +3,7 @@ set -eu
 
 # axolotl dependencies
 pushd "axolotl"
+{% if CUDA %}
   # disable package caching
   export PIP_NO_CACHE_DIR=0
 
@@ -33,4 +34,5 @@ pushd "axolotl"
     # downgrade flash-attn (https://github.com/OpenAccess-AI-Collective/axolotl/issues/911#issuecomment-1868546443)
     pip3 install flash-attn==2.3.2
   deactivate
+{% endif %}
 popd
diff --git a/airootfs/root/customize_airootfs/scripts/1000-sillytavern-extras-dependencies.sh b/airootfs/root/customize_airootfs/scripts/1000-sillytavern-extras-dependencies.sh.jinja2
similarity index 64%
rename from airootfs/root/customize_airootfs/scripts/1000-sillytavern-extras-dependencies.sh
rename to airootfs/root/customize_airootfs/scripts/1000-sillytavern-extras-dependencies.sh.jinja2
index 0613315..f27537a 100644
--- a/airootfs/root/customize_airootfs/scripts/1000-sillytavern-extras-dependencies.sh
+++ b/airootfs/root/customize_airootfs/scripts/1000-sillytavern-extras-dependencies.sh.jinja2
@@ -11,8 +11,17 @@ pushd "SillyTavern-Extras"
 
   # activate venv
   source venv/bin/activate
-    # install dependencies
+{% if CUDA %}
+    # install dependencies (cuda)
     pip3 install -r requirements.txt
+{% endif %}
+
+{% if ROCm %}
+    # install dependencies (rocm)
+    pip3 install -r requirements-rocm.txt
+{% endif %}
+
+    # install remaining dependencies
     pip3 install -r requirements-coqui.txt
     pip3 install -r requirements-rvc.txt
   deactivate
diff --git a/airootfs/root/customize_airootfs/scripts/1000-text-generation-webui-dependencies.sh b/airootfs/root/customize_airootfs/scripts/1000-text-generation-webui-dependencies.sh.jinja2
similarity index 63%
rename from airootfs/root/customize_airootfs/scripts/1000-text-generation-webui-dependencies.sh
rename to airootfs/root/customize_airootfs/scripts/1000-text-generation-webui-dependencies.sh.jinja2
index ab5001b..a59a017 100644
--- a/airootfs/root/customize_airootfs/scripts/1000-text-generation-webui-dependencies.sh
+++ b/airootfs/root/customize_airootfs/scripts/1000-text-generation-webui-dependencies.sh.jinja2
@@ -11,7 +11,14 @@ pushd "text-generation-webui"
 
   # activate venv
   source venv/bin/activate
-    # install dependencies
+{% if CUDA %}
+    # install dependencies (cuda)
     pip3 install -r requirements.txt
+{% endif %}
+
+{% if ROCm %}
+    # install dependencies (rocm)
+    pip3 install -r requirements_amd.txt
+{% endif %}
   deactivate
 popd
diff --git a/airootfs/root/customize_airootfs/scripts/1000-vllm-dependencies.sh b/airootfs/root/customize_airootfs/scripts/1000-vllm-dependencies.sh.jinja2
similarity index 69%
rename from airootfs/root/customize_airootfs/scripts/1000-vllm-dependencies.sh
rename to airootfs/root/customize_airootfs/scripts/1000-vllm-dependencies.sh.jinja2
index 6dce971..53d1396 100644
--- a/airootfs/root/customize_airootfs/scripts/1000-vllm-dependencies.sh
+++ b/airootfs/root/customize_airootfs/scripts/1000-vllm-dependencies.sh.jinja2
@@ -24,8 +24,17 @@ pushd "vllm"
 
   # activate venv
   source venv/bin/activate
-    # install dependencies
+{% if CUDA %}
+    # install dependencies (cuda)
     pip3 install -r requirements.txt
+{% endif %}
+
+{% if ROCm %}
+    # install dependencies (rocm)
+    pip3 install -r requirements-rocm.txt
+{% endif %}
+
+    # install dependencies (build)
     pip3 install -r requirements-build.txt
 
     # build native extension
@@ -40,7 +49,14 @@ pushd "vllm"
 
   # activate venv
   source venv/bin/activate
-    # install dependencies
+{% if CUDA %}
+    # install dependencies (cuda)
     pip3 install -r requirements.txt
+{% endif %}
+
+{% if ROCm %}
+    # install dependencies (rocm)
+    pip3 install -r requirements-rocm.txt
+{% endif %}
   deactivate
 popd
diff --git a/airootfs/root/customize_airootfs/scripts/9999-cleanup.sh b/airootfs/root/customize_airootfs/scripts/9999-cleanup.sh.jinja2
similarity index 64%
rename from airootfs/root/customize_airootfs/scripts/9999-cleanup.sh
rename to airootfs/root/customize_airootfs/scripts/9999-cleanup.sh.jinja2
index 4b6f6d5..ec5664d 100644
--- a/airootfs/root/customize_airootfs/scripts/9999-cleanup.sh
+++ b/airootfs/root/customize_airootfs/scripts/9999-cleanup.sh.jinja2
@@ -9,3 +9,8 @@ rm -fr /home/tori/.config/matplotlib
 
 # keras
 rm -fr /home/tori/.keras
+
+{% if ROCm %}
+# remove axolotl if rocm
+rm -fr /home/tori/axolotl
+{% endif %}

From c5afd19f2c7442d329fc4cffa04d02f3a94c8917 Mon Sep 17 00:00:00 2001
From: sasha0552 <admin@sasha0552.org>
Date: Thu, 28 Dec 2023 10:18:47 +0000
Subject: [PATCH 05/23] Disable ST-extras in ROCm for now

---
 .../scripts/1000-sillytavern-extras-dependencies.sh.jinja2      | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/airootfs/root/customize_airootfs/scripts/1000-sillytavern-extras-dependencies.sh.jinja2 b/airootfs/root/customize_airootfs/scripts/1000-sillytavern-extras-dependencies.sh.jinja2
index f27537a..411d7e9 100644
--- a/airootfs/root/customize_airootfs/scripts/1000-sillytavern-extras-dependencies.sh.jinja2
+++ b/airootfs/root/customize_airootfs/scripts/1000-sillytavern-extras-dependencies.sh.jinja2
@@ -3,6 +3,7 @@ set -eu
 
 # SillyTavern-Extras dependencies
 pushd "SillyTavern-Extras"
+{% if CUDA %}
   # disable package caching
   export PIP_NO_CACHE_DIR=0
 
@@ -25,4 +26,5 @@ pushd "SillyTavern-Extras"
     pip3 install -r requirements-coqui.txt
     pip3 install -r requirements-rvc.txt
   deactivate
+{% endif %}
 popd

From 5ee2c04d08ee0a519779afb993800cc921c05de1 Mon Sep 17 00:00:00 2001
From: sasha0552 <admin@sasha0552.org>
Date: Thu, 28 Dec 2023 11:01:19 +0000
Subject: [PATCH 06/23] vllm fixes for ROCm

---
 .gitignore                                        |  1 -
 ...l.patch => 0100-vllm-enable-other-archs.patch} |  8 +++++---
 .../scripts/0100-vllm-patches.sh                  |  8 ++++++++
 .../scripts/0100-vllm-patches.sh.jinja2           | 10 ----------
 .../scripts/1000-vllm-dependencies.sh.jinja2      | 15 ++++++++++++++-
 5 files changed, 27 insertions(+), 15 deletions(-)
 rename airootfs/root/customize_airootfs/patches/{0100-vllm-build-for-pascal.patch => 0100-vllm-enable-other-archs.patch} (69%)
 create mode 100644 airootfs/root/customize_airootfs/scripts/0100-vllm-patches.sh
 delete mode 100644 airootfs/root/customize_airootfs/scripts/0100-vllm-patches.sh.jinja2

diff --git a/.gitignore b/.gitignore
index f87df35..386b44d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,6 +8,5 @@ airootfs/root/customize_airootfs/scripts/0100-koboldcpp-patches.sh
 airootfs/root/customize_airootfs/scripts/1000-sillytavern-extras-dependencies.sh
 airootfs/root/customize_airootfs/scripts/1000-vllm-dependencies.sh
 airootfs/root/customize_airootfs/scripts/1000-text-generation-webui-dependencies.sh
-airootfs/root/customize_airootfs/scripts/0100-vllm-patches.sh
 airootfs/root/customize_airootfs/scripts/0100-automatic-patches.sh
 airootfs/root/customize_airootfs/scripts/9999-cleanup.sh
diff --git a/airootfs/root/customize_airootfs/patches/0100-vllm-build-for-pascal.patch b/airootfs/root/customize_airootfs/patches/0100-vllm-enable-other-archs.patch
similarity index 69%
rename from airootfs/root/customize_airootfs/patches/0100-vllm-build-for-pascal.patch
rename to airootfs/root/customize_airootfs/patches/0100-vllm-enable-other-archs.patch
index 5b300d1..530619d 100644
--- a/airootfs/root/customize_airootfs/patches/0100-vllm-build-for-pascal.patch
+++ b/airootfs/root/customize_airootfs/patches/0100-vllm-enable-other-archs.patch
@@ -1,14 +1,16 @@
 --- a/setup.py
 +++ b/setup.py
-@@ -15,7 +15,7 @@ ROOT_DIR = os.path.dirname(__file__)
+@@ -15,8 +15,8 @@ ROOT_DIR = os.path.dirname(__file__)
  MAIN_CUDA_VERSION = "12.1"
  
  # Supported NVIDIA GPU architectures.
 -NVIDIA_SUPPORTED_ARCHS = {"7.0", "7.5", "8.0", "8.6", "8.9", "9.0"}
-+NVIDIA_SUPPORTED_ARCHS = {"6.0", "6.1", "7.0", "7.5", "8.0", "8.6", "8.9", "9.0"}
- ROCM_SUPPORTED_ARCHS = {"gfx90a", "gfx908", "gfx906", "gfx1030", "gfx1100"}
+-ROCM_SUPPORTED_ARCHS = {"gfx90a", "gfx908", "gfx906", "gfx1030", "gfx1100"}
++NVIDIA_SUPPORTED_ARCHS = {"6.0", "6.1", "6.2", "7.0", "7.2", "7.5", "8.0", "8.6", "8.9", "9.0"}
++ROCM_SUPPORTED_ARCHS = {"gfx803", "gfx900", "gfx906", "gfx908", "gfx90a", "gfx1030", "gfx1100", "gfx1101", "gfx1102"}
  # SUPPORTED_ARCHS = NVIDIA_SUPPORTED_ARCHS.union(ROCM_SUPPORTED_ARCHS)
  
+ 
 @@ -146,9 +146,9 @@ if _is_cuda() and not compute_capabilities:
      device_count = torch.cuda.device_count()
      for i in range(device_count):
diff --git a/airootfs/root/customize_airootfs/scripts/0100-vllm-patches.sh b/airootfs/root/customize_airootfs/scripts/0100-vllm-patches.sh
new file mode 100644
index 0000000..61550e1
--- /dev/null
+++ b/airootfs/root/customize_airootfs/scripts/0100-vllm-patches.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+set -eu
+
+# vllm patches
+pushd "vllm"
+  # enable other architectures
+  patch -p1 < "$CUSTOMIZE_AIROOTFS/patches/0100-vllm-enable-other-archs.patch"
+popd
diff --git a/airootfs/root/customize_airootfs/scripts/0100-vllm-patches.sh.jinja2 b/airootfs/root/customize_airootfs/scripts/0100-vllm-patches.sh.jinja2
deleted file mode 100644
index abe8c07..0000000
--- a/airootfs/root/customize_airootfs/scripts/0100-vllm-patches.sh.jinja2
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/bin/bash
-set -eu
-
-# vllm patches
-pushd "vllm"
-{% if CUDA %}
-  # build for pascal
-  patch -p1 < "$CUSTOMIZE_AIROOTFS/patches/0100-vllm-build-for-pascal.patch"
-{% endif %}
-popd
diff --git a/airootfs/root/customize_airootfs/scripts/1000-vllm-dependencies.sh.jinja2 b/airootfs/root/customize_airootfs/scripts/1000-vllm-dependencies.sh.jinja2
index 53d1396..f35ec31 100644
--- a/airootfs/root/customize_airootfs/scripts/1000-vllm-dependencies.sh.jinja2
+++ b/airootfs/root/customize_airootfs/scripts/1000-vllm-dependencies.sh.jinja2
@@ -9,8 +9,9 @@ pushd "vllm"
   # limit the number of parallel jobs to avoid OOM
   export MAX_JOBS=1
 
+{% if CUDA %}
   # define supported architectures
-  export TORCH_CUDA_ARCH_LIST="6.0 6.1 7.0 7.5 8.0 8.6 8.9 9.0"
+  export TORCH_CUDA_ARCH_LIST="6.0 6.1 6.2 7.0 7.2 7.5 8.0 8.6 8.9 9.0 9.0+PTX"
 
   # cuda home directory
   export CUDA_HOME=/opt/cuda
@@ -18,6 +19,15 @@ pushd "vllm"
   # use gcc 12
   export CC=gcc-12
   export CXX=g++-12
+{% endif %}
+
+{% if ROCm %}
+  # define supported architectures
+  export TORCH_CUDA_ARCH_LIST="gfx803 gfx900 gfx906 gfx908 gfx90a gfx1030 gfx1100 gfx1101 gfx1102"
+
+  # rocm home directory
+  export ROCM_HOME=/opt/rocm
+{% endif %}
 
   # create venv
   python3 -m venv venv
@@ -58,5 +68,8 @@ pushd "vllm"
     # install dependencies (rocm)
     pip3 install -r requirements-rocm.txt
 {% endif %}
+
+    # install dependencies for openai api server
+    pip3 install accelerate
   deactivate
 popd

From 81b18810abc32285d24f2caa40b9a97f197cb237 Mon Sep 17 00:00:00 2001
From: sasha0552 <admin@sasha0552.org>
Date: Thu, 28 Dec 2023 11:54:16 +0000
Subject: [PATCH 07/23] Disable vllm for now

---
 .../customize_airootfs/scripts/1000-vllm-dependencies.sh.jinja2 | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/airootfs/root/customize_airootfs/scripts/1000-vllm-dependencies.sh.jinja2 b/airootfs/root/customize_airootfs/scripts/1000-vllm-dependencies.sh.jinja2
index f35ec31..e010b5c 100644
--- a/airootfs/root/customize_airootfs/scripts/1000-vllm-dependencies.sh.jinja2
+++ b/airootfs/root/customize_airootfs/scripts/1000-vllm-dependencies.sh.jinja2
@@ -3,6 +3,7 @@ set -eu
 
 # vllm dependencies
 pushd "vllm"
+{% if CUDA %}
   # disable package caching
   export PIP_NO_CACHE_DIR=0
 
@@ -72,4 +73,5 @@ pushd "vllm"
     # install dependencies for openai api server
     pip3 install accelerate
   deactivate
+{% endif %}
 popd

From 8b7d62f9978fe6135094a4e8add9da9bba910d68 Mon Sep 17 00:00:00 2001
From: sasha0552 <admin@sasha0552.org>
Date: Thu, 28 Dec 2023 11:56:10 +0000
Subject: [PATCH 08/23] Move automatic cleanup to dependencies step [skip ci]

---
 .../scripts/1000-automatic-dependencies.sh            |  6 ++++++
 .../scripts/2000-automatic-cleanup.sh                 | 11 -----------
 2 files changed, 6 insertions(+), 11 deletions(-)
 delete mode 100644 airootfs/root/customize_airootfs/scripts/2000-automatic-cleanup.sh

diff --git a/airootfs/root/customize_airootfs/scripts/1000-automatic-dependencies.sh b/airootfs/root/customize_airootfs/scripts/1000-automatic-dependencies.sh
index b68f97d..5dd48af 100644
--- a/airootfs/root/customize_airootfs/scripts/1000-automatic-dependencies.sh
+++ b/airootfs/root/customize_airootfs/scripts/1000-automatic-dependencies.sh
@@ -14,4 +14,10 @@ pushd "automatic"
     # install dependencies
     python3 launch.py --test
   deactivate
+
+  # remove installation config
+  rm config.json
+
+  # remove installation log
+  rm sdnext.log
 popd
diff --git a/airootfs/root/customize_airootfs/scripts/2000-automatic-cleanup.sh b/airootfs/root/customize_airootfs/scripts/2000-automatic-cleanup.sh
deleted file mode 100644
index 8e1ee2e..0000000
--- a/airootfs/root/customize_airootfs/scripts/2000-automatic-cleanup.sh
+++ /dev/null
@@ -1,11 +0,0 @@
-#!/bin/bash
-set -eu
-
-# automatic cleanup
-pushd "automatic"
-  # remove installation config
-  rm config.json
-
-  # remove installation log
-  rm sdnext.log
-popd

From 42fd0f117305e0595fc73a0be48b56655a0427bc Mon Sep 17 00:00:00 2001
From: sasha0552 <admin@sasha0552.org>
Date: Thu, 28 Dec 2023 12:27:17 +0000
Subject: [PATCH 09/23] Reenable st-extras, fix textgen-webui

---
 .../scripts/1000-sillytavern-extras-dependencies.sh.jinja2     | 2 +-
 .../scripts/1000-text-generation-webui-dependencies.sh.jinja2  | 3 +++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/airootfs/root/customize_airootfs/scripts/1000-sillytavern-extras-dependencies.sh.jinja2 b/airootfs/root/customize_airootfs/scripts/1000-sillytavern-extras-dependencies.sh.jinja2
index 411d7e9..e5ecd3b 100644
--- a/airootfs/root/customize_airootfs/scripts/1000-sillytavern-extras-dependencies.sh.jinja2
+++ b/airootfs/root/customize_airootfs/scripts/1000-sillytavern-extras-dependencies.sh.jinja2
@@ -3,7 +3,6 @@ set -eu
 
 # SillyTavern-Extras dependencies
 pushd "SillyTavern-Extras"
-{% if CUDA %}
   # disable package caching
   export PIP_NO_CACHE_DIR=0
 
@@ -22,6 +21,7 @@ pushd "SillyTavern-Extras"
     pip3 install -r requirements-rocm.txt
 {% endif %}
 
+{% if CUDA %}
     # install remaining dependencies
     pip3 install -r requirements-coqui.txt
     pip3 install -r requirements-rvc.txt
diff --git a/airootfs/root/customize_airootfs/scripts/1000-text-generation-webui-dependencies.sh.jinja2 b/airootfs/root/customize_airootfs/scripts/1000-text-generation-webui-dependencies.sh.jinja2
index a59a017..12638d0 100644
--- a/airootfs/root/customize_airootfs/scripts/1000-text-generation-webui-dependencies.sh.jinja2
+++ b/airootfs/root/customize_airootfs/scripts/1000-text-generation-webui-dependencies.sh.jinja2
@@ -17,6 +17,9 @@ pushd "text-generation-webui"
 {% endif %}
 
 {% if ROCm %}
+    # install pytorch
+    pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/rocm5.6
+
     # install dependencies (rocm)
     pip3 install -r requirements_amd.txt
 {% endif %}

From b4b0884ef47463558745e65d68c99c4fd37fd72e Mon Sep 17 00:00:00 2001
From: sasha0552 <admin@sasha0552.org>
Date: Thu, 28 Dec 2023 12:37:49 +0000
Subject: [PATCH 10/23] Remove nvidia-persistenced if ROCm [skip ci]

---
 .../root/customize_airootfs/scripts/9999-cleanup.sh.jinja2   | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/airootfs/root/customize_airootfs/scripts/9999-cleanup.sh.jinja2 b/airootfs/root/customize_airootfs/scripts/9999-cleanup.sh.jinja2
index ec5664d..d164f8e 100644
--- a/airootfs/root/customize_airootfs/scripts/9999-cleanup.sh.jinja2
+++ b/airootfs/root/customize_airootfs/scripts/9999-cleanup.sh.jinja2
@@ -10,6 +10,11 @@ rm -fr /home/tori/.config/matplotlib
 # keras
 rm -fr /home/tori/.keras
 
+{% if ROCm %}
+# remove nvidia-persistenced if rocm
+rm -f /etc/systemd/system/multi-user.target.wants/nvidia-persistenced.service
+{% endif %}
+
 {% if ROCm %}
 # remove axolotl if rocm
 rm -fr /home/tori/axolotl

From 626f9ccc679e4717bba54f1332292f197cb5b2b5 Mon Sep 17 00:00:00 2001
From: sasha0552 <admin@sasha0552.org>
Date: Thu, 28 Dec 2023 12:57:23 +0000
Subject: [PATCH 11/23] Fix ST-E dependencies building

---
 packages.x86_64.jinja2 | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/packages.x86_64.jinja2 b/packages.x86_64.jinja2
index e1e19a3..50f9592 100644
--- a/packages.x86_64.jinja2
+++ b/packages.x86_64.jinja2
@@ -44,4 +44,7 @@ nvidia-utils
 {% if ROCm %}
 # ROCm
 rocm-hip-sdk
+
+# Required for SillyTavern-Extras
+gtk3
 {% endif %}

From 7ffe8cfb98f2d7075b1226311510e8f69d99ceb5 Mon Sep 17 00:00:00 2001
From: sasha0552 <admin@sasha0552.org>
Date: Thu, 28 Dec 2023 13:09:34 +0000
Subject: [PATCH 12/23] Enable coqui & rvc for ROCm in STE

---
 .../scripts/1000-sillytavern-extras-dependencies.sh.jinja2      | 2 --
 1 file changed, 2 deletions(-)

diff --git a/airootfs/root/customize_airootfs/scripts/1000-sillytavern-extras-dependencies.sh.jinja2 b/airootfs/root/customize_airootfs/scripts/1000-sillytavern-extras-dependencies.sh.jinja2
index e5ecd3b..f27537a 100644
--- a/airootfs/root/customize_airootfs/scripts/1000-sillytavern-extras-dependencies.sh.jinja2
+++ b/airootfs/root/customize_airootfs/scripts/1000-sillytavern-extras-dependencies.sh.jinja2
@@ -21,10 +21,8 @@ pushd "SillyTavern-Extras"
     pip3 install -r requirements-rocm.txt
 {% endif %}
 
-{% if CUDA %}
     # install remaining dependencies
     pip3 install -r requirements-coqui.txt
     pip3 install -r requirements-rvc.txt
   deactivate
-{% endif %}
 popd

From 5985c87005e20d945654116b69087b2afaf5b348 Mon Sep 17 00:00:00 2001
From: sasha0552 <admin@sasha0552.org>
Date: Thu, 28 Dec 2023 13:13:59 +0000
Subject: [PATCH 13/23] Extract ROCm version from one_click.py (textgen)

---
 .../1000-text-generation-webui-dependencies.sh.jinja2        | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/airootfs/root/customize_airootfs/scripts/1000-text-generation-webui-dependencies.sh.jinja2 b/airootfs/root/customize_airootfs/scripts/1000-text-generation-webui-dependencies.sh.jinja2
index 12638d0..cf651ef 100644
--- a/airootfs/root/customize_airootfs/scripts/1000-text-generation-webui-dependencies.sh.jinja2
+++ b/airootfs/root/customize_airootfs/scripts/1000-text-generation-webui-dependencies.sh.jinja2
@@ -17,8 +17,11 @@ pushd "text-generation-webui"
 {% endif %}
 
 {% if ROCm %}
+    # extract pytorch version
+    index_url=$(grep -o 'https://download.pytorch.org/whl/rocm[0-9.]*' one_click.py)
+
     # install pytorch
-    pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/rocm5.6
+    pip3 install torch torchvision torchaudio --index-url "$index_url"
 
     # install dependencies (rocm)
     pip3 install -r requirements_amd.txt

From cb6473d92398c6b7e3f65481cf3e4025ebd4529c Mon Sep 17 00:00:00 2001
From: sasha0552 <admin@sasha0552.org>
Date: Thu, 28 Dec 2023 13:59:27 +0000
Subject: [PATCH 14/23] Fix removal of nvidia-persistenced on rocm systems

---
 .gitignore                                                   | 1 +
 .../{customize_airootfs.sh => customize_airootfs.sh.jinja2}  | 5 +++++
 .../root/customize_airootfs/scripts/9999-cleanup.sh.jinja2   | 5 -----
 3 files changed, 6 insertions(+), 5 deletions(-)
 rename airootfs/root/{customize_airootfs.sh => customize_airootfs.sh.jinja2} (82%)

diff --git a/.gitignore b/.gitignore
index 386b44d..4b28da9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,6 +3,7 @@ work/
 
 # rendered files
 packages.x86_64
+airootfs/root/customize_airootfs.sh
 airootfs/root/customize_airootfs/scripts/1000-axolotl-dependencies.sh
 airootfs/root/customize_airootfs/scripts/0100-koboldcpp-patches.sh
 airootfs/root/customize_airootfs/scripts/1000-sillytavern-extras-dependencies.sh
diff --git a/airootfs/root/customize_airootfs.sh b/airootfs/root/customize_airootfs.sh.jinja2
similarity index 82%
rename from airootfs/root/customize_airootfs.sh
rename to airootfs/root/customize_airootfs.sh.jinja2
index 6c68b0e..c6bb63d 100644
--- a/airootfs/root/customize_airootfs.sh
+++ b/airootfs/root/customize_airootfs.sh.jinja2
@@ -10,6 +10,11 @@ echo "tori:tori" | chpasswd
 # remove any jinja2 files
 find -type f -name "*.jinja2" -print -delete
 
+{% if ROCm %}
+# remove nvidia-persistenced if rocm
+rm -f /etc/systemd/system/multi-user.target.wants/nvidia-persistenced.service
+{% endif %}
+
 # enter user directory
 cd "/home/tori"
 
diff --git a/airootfs/root/customize_airootfs/scripts/9999-cleanup.sh.jinja2 b/airootfs/root/customize_airootfs/scripts/9999-cleanup.sh.jinja2
index d164f8e..ec5664d 100644
--- a/airootfs/root/customize_airootfs/scripts/9999-cleanup.sh.jinja2
+++ b/airootfs/root/customize_airootfs/scripts/9999-cleanup.sh.jinja2
@@ -10,11 +10,6 @@ rm -fr /home/tori/.config/matplotlib
 # keras
 rm -fr /home/tori/.keras
 
-{% if ROCm %}
-# remove nvidia-persistenced if rocm
-rm -f /etc/systemd/system/multi-user.target.wants/nvidia-persistenced.service
-{% endif %}
-
 {% if ROCm %}
 # remove axolotl if rocm
 rm -fr /home/tori/axolotl

From 93a6fc0d6a4a5e1a85e7f9ce53910baf0cf903d3 Mon Sep 17 00:00:00 2001
From: sasha0552 <admin@sasha0552.org>
Date: Fri, 29 Dec 2023 09:42:07 +0000
Subject: [PATCH 15/23] Disable everything except vllm; enable vllm

---
 ...-dependencies.sh => 1000-automatic-dependencies.sh.disabled} | 0
 ... => 1000-sillytavern-extras-dependencies.sh.disabled.jinja2} | 0
 ... 1000-text-generation-webui-dependencies.sh.disabled.jinja2} | 0
 .../customize_airootfs/scripts/1000-vllm-dependencies.sh.jinja2 | 2 --
 4 files changed, 2 deletions(-)
 rename airootfs/root/customize_airootfs/scripts/{1000-automatic-dependencies.sh => 1000-automatic-dependencies.sh.disabled} (100%)
 rename airootfs/root/customize_airootfs/scripts/{1000-sillytavern-extras-dependencies.sh.jinja2 => 1000-sillytavern-extras-dependencies.sh.disabled.jinja2} (100%)
 rename airootfs/root/customize_airootfs/scripts/{1000-text-generation-webui-dependencies.sh.jinja2 => 1000-text-generation-webui-dependencies.sh.disabled.jinja2} (100%)

diff --git a/airootfs/root/customize_airootfs/scripts/1000-automatic-dependencies.sh b/airootfs/root/customize_airootfs/scripts/1000-automatic-dependencies.sh.disabled
similarity index 100%
rename from airootfs/root/customize_airootfs/scripts/1000-automatic-dependencies.sh
rename to airootfs/root/customize_airootfs/scripts/1000-automatic-dependencies.sh.disabled
diff --git a/airootfs/root/customize_airootfs/scripts/1000-sillytavern-extras-dependencies.sh.jinja2 b/airootfs/root/customize_airootfs/scripts/1000-sillytavern-extras-dependencies.sh.disabled.jinja2
similarity index 100%
rename from airootfs/root/customize_airootfs/scripts/1000-sillytavern-extras-dependencies.sh.jinja2
rename to airootfs/root/customize_airootfs/scripts/1000-sillytavern-extras-dependencies.sh.disabled.jinja2
diff --git a/airootfs/root/customize_airootfs/scripts/1000-text-generation-webui-dependencies.sh.jinja2 b/airootfs/root/customize_airootfs/scripts/1000-text-generation-webui-dependencies.sh.disabled.jinja2
similarity index 100%
rename from airootfs/root/customize_airootfs/scripts/1000-text-generation-webui-dependencies.sh.jinja2
rename to airootfs/root/customize_airootfs/scripts/1000-text-generation-webui-dependencies.sh.disabled.jinja2
diff --git a/airootfs/root/customize_airootfs/scripts/1000-vllm-dependencies.sh.jinja2 b/airootfs/root/customize_airootfs/scripts/1000-vllm-dependencies.sh.jinja2
index e010b5c..f35ec31 100644
--- a/airootfs/root/customize_airootfs/scripts/1000-vllm-dependencies.sh.jinja2
+++ b/airootfs/root/customize_airootfs/scripts/1000-vllm-dependencies.sh.jinja2
@@ -3,7 +3,6 @@ set -eu
 
 # vllm dependencies
 pushd "vllm"
-{% if CUDA %}
   # disable package caching
   export PIP_NO_CACHE_DIR=0
 
@@ -73,5 +72,4 @@ pushd "vllm"
     # install dependencies for openai api server
     pip3 install accelerate
   deactivate
-{% endif %}
 popd

From 5683ce200dc6a6d175aa288e86f6ae8e10406bc9 Mon Sep 17 00:00:00 2001
From: sasha0552 <admin@sasha0552.org>
Date: Fri, 29 Dec 2023 09:59:18 +0000
Subject: [PATCH 16/23] Install pytorch before vllm dependencies

---
 .../scripts/1000-vllm-dependencies.sh.jinja2          | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/airootfs/root/customize_airootfs/scripts/1000-vllm-dependencies.sh.jinja2 b/airootfs/root/customize_airootfs/scripts/1000-vllm-dependencies.sh.jinja2
index f35ec31..8242189 100644
--- a/airootfs/root/customize_airootfs/scripts/1000-vllm-dependencies.sh.jinja2
+++ b/airootfs/root/customize_airootfs/scripts/1000-vllm-dependencies.sh.jinja2
@@ -34,6 +34,17 @@ pushd "vllm"
 
   # activate venv
   source venv/bin/activate
+{% if ROCm %}
+    # extract rocm version
+    rocm_version=$(grep -o 'rocm/pytorch:rocm[0-9.]*' Dockerfile.rocm | cut -d : -f 2)
+
+    # create pytorch url
+    index_url="https://download.pytorch.org/whl/$rocm_version"
+
+    # install pytorch
+    pip3 install torch torchvision torchaudio --index-url "$index_url"
+{% endif %}
+
 {% if CUDA %}
     # install dependencies (cuda)
     pip3 install -r requirements.txt

From c884065feedb8d56ccc9a0d9f9ebfa4e60fc6d51 Mon Sep 17 00:00:00 2001
From: sasha0552 <admin@sasha0552.org>
Date: Fri, 29 Dec 2023 10:12:09 +0000
Subject: [PATCH 17/23] Fix building

---
 .../scripts/1000-vllm-dependencies.sh.jinja2  | 26 +++++++++++--------
 1 file changed, 15 insertions(+), 11 deletions(-)

diff --git a/airootfs/root/customize_airootfs/scripts/1000-vllm-dependencies.sh.jinja2 b/airootfs/root/customize_airootfs/scripts/1000-vllm-dependencies.sh.jinja2
index 8242189..d1020e1 100644
--- a/airootfs/root/customize_airootfs/scripts/1000-vllm-dependencies.sh.jinja2
+++ b/airootfs/root/customize_airootfs/scripts/1000-vllm-dependencies.sh.jinja2
@@ -27,6 +27,15 @@ pushd "vllm"
 
   # rocm home directory
   export ROCM_HOME=/opt/rocm
+
+  # extract rocm version
+  #rocm_version=$(grep -o 'rocm/pytorch:rocm[0-9.]*' Dockerfile.rocm | cut -d : -f 2)
+
+  # TODO: rocm5.7 is not present in repo
+  rocm_version="rocm5.6"
+
+  # create pytorch url
+  index_url="https://download.pytorch.org/whl/$rocm_version"
 {% endif %}
 
   # create venv
@@ -34,23 +43,15 @@ pushd "vllm"
 
   # activate venv
   source venv/bin/activate
-{% if ROCm %}
-    # extract rocm version
-    rocm_version=$(grep -o 'rocm/pytorch:rocm[0-9.]*' Dockerfile.rocm | cut -d : -f 2)
-
-    # create pytorch url
-    index_url="https://download.pytorch.org/whl/$rocm_version"
-
-    # install pytorch
-    pip3 install torch torchvision torchaudio --index-url "$index_url"
-{% endif %}
-
 {% if CUDA %}
     # install dependencies (cuda)
     pip3 install -r requirements.txt
 {% endif %}
 
 {% if ROCm %}
+    # install pytorch
+    pip3 install torch torchvision torchaudio --index-url "$index_url"
+
     # install dependencies (rocm)
     pip3 install -r requirements-rocm.txt
 {% endif %}
@@ -76,6 +77,9 @@ pushd "vllm"
 {% endif %}
 
 {% if ROCm %}
+    # install pytorch
+    pip3 install torch torchvision torchaudio --index-url "$index_url"
+
     # install dependencies (rocm)
     pip3 install -r requirements-rocm.txt
 {% endif %}

From 33c3220af79b044f576456058ff1649b88cb6bd9 Mon Sep 17 00:00:00 2001
From: sasha0552 <admin@sasha0552.org>
Date: Fri, 29 Dec 2023 11:53:19 +0000
Subject: [PATCH 18/23] Reenable projects, disable vllm on rocm

---
 ...isabled => 1000-automatic-dependencies.sh} |  0
 ...sillytavern-extras-dependencies.sh.jinja2} |  0
 ...t-generation-webui-dependencies.sh.jinja2} |  0
 .../scripts/1000-vllm-dependencies.sh.jinja2  | 52 ++-----------------
 4 files changed, 5 insertions(+), 47 deletions(-)
 rename airootfs/root/customize_airootfs/scripts/{1000-automatic-dependencies.sh.disabled => 1000-automatic-dependencies.sh} (100%)
 rename airootfs/root/customize_airootfs/scripts/{1000-sillytavern-extras-dependencies.sh.disabled.jinja2 => 1000-sillytavern-extras-dependencies.sh.jinja2} (100%)
 rename airootfs/root/customize_airootfs/scripts/{1000-text-generation-webui-dependencies.sh.disabled.jinja2 => 1000-text-generation-webui-dependencies.sh.jinja2} (100%)

diff --git a/airootfs/root/customize_airootfs/scripts/1000-automatic-dependencies.sh.disabled b/airootfs/root/customize_airootfs/scripts/1000-automatic-dependencies.sh
similarity index 100%
rename from airootfs/root/customize_airootfs/scripts/1000-automatic-dependencies.sh.disabled
rename to airootfs/root/customize_airootfs/scripts/1000-automatic-dependencies.sh
diff --git a/airootfs/root/customize_airootfs/scripts/1000-sillytavern-extras-dependencies.sh.disabled.jinja2 b/airootfs/root/customize_airootfs/scripts/1000-sillytavern-extras-dependencies.sh.jinja2
similarity index 100%
rename from airootfs/root/customize_airootfs/scripts/1000-sillytavern-extras-dependencies.sh.disabled.jinja2
rename to airootfs/root/customize_airootfs/scripts/1000-sillytavern-extras-dependencies.sh.jinja2
diff --git a/airootfs/root/customize_airootfs/scripts/1000-text-generation-webui-dependencies.sh.disabled.jinja2 b/airootfs/root/customize_airootfs/scripts/1000-text-generation-webui-dependencies.sh.jinja2
similarity index 100%
rename from airootfs/root/customize_airootfs/scripts/1000-text-generation-webui-dependencies.sh.disabled.jinja2
rename to airootfs/root/customize_airootfs/scripts/1000-text-generation-webui-dependencies.sh.jinja2
diff --git a/airootfs/root/customize_airootfs/scripts/1000-vllm-dependencies.sh.jinja2 b/airootfs/root/customize_airootfs/scripts/1000-vllm-dependencies.sh.jinja2
index d1020e1..ceb2644 100644
--- a/airootfs/root/customize_airootfs/scripts/1000-vllm-dependencies.sh.jinja2
+++ b/airootfs/root/customize_airootfs/scripts/1000-vllm-dependencies.sh.jinja2
@@ -3,15 +3,15 @@ set -eu
 
 # vllm dependencies
 pushd "vllm"
+{% if CUDA %}
   # disable package caching
   export PIP_NO_CACHE_DIR=0
 
   # limit the number of parallel jobs to avoid OOM
   export MAX_JOBS=1
 
-{% if CUDA %}
   # define supported architectures
-  export TORCH_CUDA_ARCH_LIST="6.0 6.1 6.2 7.0 7.2 7.5 8.0 8.6 8.9 9.0 9.0+PTX"
+  export TORCH_CUDA_ARCH_LIST="6.0 6.1 7.0 7.5 8.0 8.6 8.9 9.0"
 
   # cuda home directory
   export CUDA_HOME=/opt/cuda
@@ -19,44 +19,14 @@ pushd "vllm"
   # use gcc 12
   export CC=gcc-12
   export CXX=g++-12
-{% endif %}
-
-{% if ROCm %}
-  # define supported architectures
-  export TORCH_CUDA_ARCH_LIST="gfx803 gfx900 gfx906 gfx908 gfx90a gfx1030 gfx1100 gfx1101 gfx1102"
-
-  # rocm home directory
-  export ROCM_HOME=/opt/rocm
-
-  # extract rocm version
-  #rocm_version=$(grep -o 'rocm/pytorch:rocm[0-9.]*' Dockerfile.rocm | cut -d : -f 2)
-
-  # TODO: rocm5.7 is not present in repo
-  rocm_version="rocm5.6"
-
-  # create pytorch url
-  index_url="https://download.pytorch.org/whl/$rocm_version"
-{% endif %}
 
   # create venv
   python3 -m venv venv
 
   # activate venv
   source venv/bin/activate
-{% if CUDA %}
-    # install dependencies (cuda)
+    # install dependencies
     pip3 install -r requirements.txt
-{% endif %}
-
-{% if ROCm %}
-    # install pytorch
-    pip3 install torch torchvision torchaudio --index-url "$index_url"
-
-    # install dependencies (rocm)
-    pip3 install -r requirements-rocm.txt
-{% endif %}
-
-    # install dependencies (build)
     pip3 install -r requirements-build.txt
 
     # build native extension
@@ -71,20 +41,8 @@ pushd "vllm"
 
   # activate venv
   source venv/bin/activate
-{% if CUDA %}
-    # install dependencies (cuda)
+    # install dependencies
     pip3 install -r requirements.txt
-{% endif %}
-
-{% if ROCm %}
-    # install pytorch
-    pip3 install torch torchvision torchaudio --index-url "$index_url"
-
-    # install dependencies (rocm)
-    pip3 install -r requirements-rocm.txt
-{% endif %}
-
-    # install dependencies for openai api server
-    pip3 install accelerate
   deactivate
+{% endif %}
 popd

From e5b6b5ffed3cc091f370d46bc4c3e2c7fb8440ca Mon Sep 17 00:00:00 2001
From: sasha0552 <admin@sasha0552.org>
Date: Fri, 29 Dec 2023 11:55:03 +0000
Subject: [PATCH 19/23] Use older vllm patch

---
 .../patches/0100-vllm-enable-other-archs.patch            | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/airootfs/root/customize_airootfs/patches/0100-vllm-enable-other-archs.patch b/airootfs/root/customize_airootfs/patches/0100-vllm-enable-other-archs.patch
index 530619d..5b300d1 100644
--- a/airootfs/root/customize_airootfs/patches/0100-vllm-enable-other-archs.patch
+++ b/airootfs/root/customize_airootfs/patches/0100-vllm-enable-other-archs.patch
@@ -1,16 +1,14 @@
 --- a/setup.py
 +++ b/setup.py
-@@ -15,8 +15,8 @@ ROOT_DIR = os.path.dirname(__file__)
+@@ -15,7 +15,7 @@ ROOT_DIR = os.path.dirname(__file__)
  MAIN_CUDA_VERSION = "12.1"
  
  # Supported NVIDIA GPU architectures.
 -NVIDIA_SUPPORTED_ARCHS = {"7.0", "7.5", "8.0", "8.6", "8.9", "9.0"}
--ROCM_SUPPORTED_ARCHS = {"gfx90a", "gfx908", "gfx906", "gfx1030", "gfx1100"}
-+NVIDIA_SUPPORTED_ARCHS = {"6.0", "6.1", "6.2", "7.0", "7.2", "7.5", "8.0", "8.6", "8.9", "9.0"}
-+ROCM_SUPPORTED_ARCHS = {"gfx803", "gfx900", "gfx906", "gfx908", "gfx90a", "gfx1030", "gfx1100", "gfx1101", "gfx1102"}
++NVIDIA_SUPPORTED_ARCHS = {"6.0", "6.1", "7.0", "7.5", "8.0", "8.6", "8.9", "9.0"}
+ ROCM_SUPPORTED_ARCHS = {"gfx90a", "gfx908", "gfx906", "gfx1030", "gfx1100"}
  # SUPPORTED_ARCHS = NVIDIA_SUPPORTED_ARCHS.union(ROCM_SUPPORTED_ARCHS)
  
- 
 @@ -146,9 +146,9 @@ if _is_cuda() and not compute_capabilities:
      device_count = torch.cuda.device_count()
      for i in range(device_count):

From 2e95de602c88c60b2a9867819b07e3c1958c8c61 Mon Sep 17 00:00:00 2001
From: sasha0552 <admin@sasha0552.org>
Date: Fri, 29 Dec 2023 11:58:34 +0000
Subject: [PATCH 20/23] Remove vllm if rocm

---
 .../root/customize_airootfs/scripts/9999-cleanup.sh.jinja2     | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/airootfs/root/customize_airootfs/scripts/9999-cleanup.sh.jinja2 b/airootfs/root/customize_airootfs/scripts/9999-cleanup.sh.jinja2
index ec5664d..498fcf5 100644
--- a/airootfs/root/customize_airootfs/scripts/9999-cleanup.sh.jinja2
+++ b/airootfs/root/customize_airootfs/scripts/9999-cleanup.sh.jinja2
@@ -13,4 +13,7 @@ rm -fr /home/tori/.keras
 {% if ROCm %}
 # remove axolotl if rocm
 rm -fr /home/tori/axolotl
+
+# remove vllm if rocm
+rm -fr /home/tori/vllm
 {% endif %}

From d008375adab8857014d8a7d7ad3a849cf1c320dc Mon Sep 17 00:00:00 2001
From: sasha0552 <admin@sasha0552.org>
Date: Fri, 29 Dec 2023 11:59:29 +0000
Subject: [PATCH 21/23] Update README.md [skip ci]

---
 README.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/README.md b/README.md
index e640ad2..a1b3cdd 100644
--- a/README.md
+++ b/README.md
@@ -28,3 +28,7 @@ To use ToriLinux:
 ## Misc
 
 Note that you need pre-downloaded models on a local hard drive or NFS server, or enough RAM and internet connection to download models directly into RAM.
+
+Note that following projects is not available on ROCm version:
+* [axolotl](https://github.com/OpenAccess-AI-Collective/axolotl)
+* [vllm](https://github.com/vllm-project/vllm)

From 838e17649d83cf395af5ac886ccea008a7717e83 Mon Sep 17 00:00:00 2001
From: sasha0552 <admin@sasha0552.org>
Date: Fri, 29 Dec 2023 12:02:59 +0000
Subject: [PATCH 22/23] Update repositories

---
 airootfs/home/tori/axolotl               | 2 +-
 airootfs/home/tori/llama.cpp             | 2 +-
 airootfs/home/tori/text-generation-webui | 2 +-
 airootfs/home/tori/vllm                  | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/airootfs/home/tori/axolotl b/airootfs/home/tori/axolotl
index 6ef46f8..41353d2 160000
--- a/airootfs/home/tori/axolotl
+++ b/airootfs/home/tori/axolotl
@@ -1 +1 @@
-Subproject commit 6ef46f8dcac84825c39fcea57917260abd33e9ac
+Subproject commit 41353d2ea04db3478d2f6f9069b7d0adb1f30ae8
diff --git a/airootfs/home/tori/llama.cpp b/airootfs/home/tori/llama.cpp
index b47879b..65e5f6d 160000
--- a/airootfs/home/tori/llama.cpp
+++ b/airootfs/home/tori/llama.cpp
@@ -1 +1 @@
-Subproject commit b47879b0dda43f2d26415e88b6840295817e552a
+Subproject commit 65e5f6dadbba4b496bba27f573e473c66b446496
diff --git a/airootfs/home/tori/text-generation-webui b/airootfs/home/tori/text-generation-webui
index 19d1374..3fd7073 160000
--- a/airootfs/home/tori/text-generation-webui
+++ b/airootfs/home/tori/text-generation-webui
@@ -1 +1 @@
-Subproject commit 19d13743a6c436474f80643c26eb1fb35e5446f5
+Subproject commit 3fd707380868ad6a2ba57aaed4c96799c2441d99
diff --git a/airootfs/home/tori/vllm b/airootfs/home/tori/vllm
index 4aaafdd..358c328 160000
--- a/airootfs/home/tori/vllm
+++ b/airootfs/home/tori/vllm
@@ -1 +1 @@
-Subproject commit 4aaafdd289f57a82513a7742155e4f1b796c8bdc
+Subproject commit 358c328d69f62c1fdbd6deaf3b872809734c37b2

From fe482df8bc881344c8ee63f866a80cc61ebf7623 Mon Sep 17 00:00:00 2001
From: sasha0552 <admin@sasha0552.org>
Date: Fri, 29 Dec 2023 13:31:47 +0000
Subject: [PATCH 23/23] Fix llama.cpp patch

---
 .../patches/0100-llamacpp-enable-prompt-cache.patch  | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/airootfs/root/customize_airootfs/patches/0100-llamacpp-enable-prompt-cache.patch b/airootfs/root/customize_airootfs/patches/0100-llamacpp-enable-prompt-cache.patch
index e6bca3c..a14d0d6 100644
--- a/airootfs/root/customize_airootfs/patches/0100-llamacpp-enable-prompt-cache.patch
+++ b/airootfs/root/customize_airootfs/patches/0100-llamacpp-enable-prompt-cache.patch
@@ -9,7 +9,7 @@
  
      uint32_t seed      = -1; // RNG seed
      int32_t  n_keep    =  0; // number of tokens to keep from initial prompt
-@@ -712,7 +712,7 @@ struct llama_server_context
+@@ -711,7 +711,7 @@ struct llama_server_context
          }
  
          slot->params.stream           = json_value(data, "stream",            false);
@@ -18,12 +18,12 @@
          slot->params.n_predict        = json_value(data, "n_predict",         default_params.n_predict);
          slot->sparams.top_k           = json_value(data, "top_k",             default_sparams.top_k);
          slot->sparams.top_p           = json_value(data, "top_p",             default_sparams.top_p);
-@@ -2439,7 +2439,7 @@ json oaicompat_completion_params_parse(
-     // Map OpenAI parameters to llama.cpp parameters
+@@ -2446,7 +2446,7 @@ json oaicompat_completion_params_parse(
+     llama_sampling_params default_sparams;
      llama_params["model"]             = json_value(body, "model", std::string("uknown"));
      llama_params["prompt"]            = format_chatml(body["messages"]); // OpenAI 'messages' to llama.cpp 'prompt'
 -    llama_params["cache_prompt"]      = json_value(body, "cache_prompt", false);
 +    llama_params["cache_prompt"]      = json_value(body, "cache_prompt", true);
-     llama_params["temperature"]       = json_value(body, "temperature", 0.8);
-     llama_params["top_k"]             = json_value(body, "top_k", 40);
-     llama_params["top_p"]             = json_value(body, "top_p", 0.95);
+     llama_params["temperature"]       = json_value(body, "temperature", 0.0);
+     llama_params["top_k"]             = json_value(body, "top_k", default_sparams.top_k);
+     llama_params["top_p"]             = json_value(body, "top_p", 1.0);