johnmmcgee · johnmmcgee · Nov 8, 2023 · Oct 5, 2023 · Oct 6, 2023 · Oct 15, 2023
diff --git a/Containerfile b/Containerfile
@@ -8,16 +8,19 @@ ARG COREOS_VERSION="${COREOS_VERSION:-stable}"
 
 COPY build*.sh /tmp
 COPY certs /tmp/certs
-ADD files/etc/nvidia-container-runtime/config-rootless.toml \
-    /tmp/ublue-os-ucore-nvidia/rpmbuild/SOURCES/config-rootless.toml
 ADD ublue-os-ucore-nvidia.spec \
-    /tmp/ublue-os-ucore-nvidia/ublue-os-ucore-nvidia.spec
+        /tmp/ublue-os-ucore-nvidia/ublue-os-ucore-nvidia.spec
+ADD files/usr/lib/systemd/system/ublue-nvctk-cdi.service \
+        /tmp/ublue-os-ucore-nvidia/rpmbuild/SOURCES/ublue-nvctk-cdi.service
+ADD files/usr/lib/systemd/system-preset/70-ublue-nvctk-cdi.preset \
+        /tmp/ublue-os-ucore-nvidia/rpmbuild/SOURCES/70-ublue-nvctk-cdi.preset
+
 
 RUN /tmp/build-prep.sh
 
-RUN /tmp/build-kmod-nvidia.sh
 RUN /tmp/build-ublue-nvidia.sh
-RUN /tmp/build-kmod-zfs.sh
+RUN /tmp/build-kmod-nvidia.sh
+RUN ZFS_MINOR_VERSION=2.1 /tmp/build-kmod-zfs.sh
 
 RUN for RPM in $(find /var/cache/akmods/ -type f -name \*.rpm); do \
         cp "${RPM}" /var/cache/rpms/kmods/; \

diff --git a/README.md b/README.md
@@ -9,8 +9,10 @@ A layer for adding extra kernel modules to your Fedora CoreOS image.
 Feel free to PR more kmod build scripts into this repo!
 
 - [nvidia](https://negativo17.org/nvidia-driver) - latest driver (currently version 535) built from negativo17's akmod package
-- ublue-os-ucore-nvidia - RPM with nvidia container runtime repo and selinux policy
-- [zfs](https://github.com/openzfs/zfs) - OpenZFS advanced file system and volume manager
+- ublue-os-ucore-nvidia - RPM with nvidia container toolkit repo and selinux policy
+    - [nvidia container selinux policy](https://github.com/NVIDIA/dgx-selinux/tree/master/src/nvidia-container-selinux) - uses RHEL9 policy as the closest match
+    - [nvidia-container-tookkit repo](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html#installing-with-yum-or-dnf) - version 1.14.2 (and newer) provide CDI for podman use of nvidia gpus
+- [zfs](https://github.com/openzfs/zfs) - OpenZFS advanced file system and volume manager *(currently pinned to 2.1 release series)*
 
 
 # Usage

diff --git a/build-kmod-nvidia.sh b/build-kmod-nvidia.sh
@@ -30,8 +30,6 @@ KERNEL_VERSION=${KERNEL_VERSION}
 RELEASE=${RELEASE}
 NVIDIA_AKMOD_VERSION=${NVIDIA_AKMOD_VERSION}
 EOF
-#NVIDIA_FULL_VERSION=${NVIDIA_FULL_VERSION}
-#NVIDIA_LIB_VERSION=${NVIDIA_LIB_VERSION}
 
 mv /var/cache/akmods/nvidia/*.rpm \
    /var/cache/rpms/kmods/nvidia/
diff --git a/build-kmod-zfs.sh b/build-kmod-zfs.sh
@@ -7,12 +7,16 @@ ARCH="$(rpm -E '%_arch')"
 KERNEL="$(rpm -q kernel --queryformat '%{VERSION}-%{RELEASE}.%{ARCH}')"
 RELEASE="$(rpm -E '%fedora')"
 
+# allow pinning to a specific release series (eg, 2.0.x or 2.1.x)
+ZFS_MINOR_VERSION="${ZFS_MINOR_VERSION:-}"
+
 cd /tmp
 
 # Use cURL to fetch the given URL, saving the response to `data.json`
 curl "https://api.github.com/repos/openzfs/zfs/releases" -o data.json
-ZFS_VERSION=$(jq -r '[ .[] | select(.prerelease==false and .draft==false) ][0].tag_name' data.json|cut -f2- -d-)
-ZFS_PREVIOUS=$(jq -r '[ .[] | select(.prerelease==false and .draft==false) ][1].tag_name' data.json|cut -f2- -d-)
+ZFS_VERSION=$(jq -r --arg ZMV "zfs-${ZFS_MINOR_VERSION}" '[ .[] | select(.prerelease==false and .draft==false) | select(.tag_name | startswith($ZMV))][0].tag_name' data.json|cut -f2- -d-)
+echo "ZFS_VERSION==$ZFS_VERSION"
+
 
 ### zfs specific build deps
 rpm-ostree install libtirpc-devel libblkid-devel libuuid-devel libudev-devel openssl-devel zlib-devel libaio-devel libattr-devel elfutils-libelf-devel python3-devel libffi-devel libcurl-devel

diff --git a/build-ublue-nvidia.sh b/build-ublue-nvidia.sh
@@ -6,9 +6,11 @@ set -oeux pipefail
 
 #install -D /etc/pki/akmods/certs/public_key.der /tmp/ublue-os-ucore-nvidia/rpmbuild/SOURCES/public_key.der
 
-curl -L https://nvidia.github.io/nvidia-docker/rhel9.0/nvidia-docker.repo \
-    -o /tmp/ublue-os-ucore-nvidia/rpmbuild/SOURCES/nvidia-container-runtime.repo 
-sed -i "s@gpgcheck=0@gpgcheck=1@" /tmp/ublue-os-ucore-nvidia/rpmbuild/SOURCES/nvidia-container-runtime.repo
+mkdir -p /tmp/ublue-os-ucore-nvidia/rpmbuild/SOURCES/
+
+curl -L https://nvidia.github.io/libnvidia-container/stable/rpm/nvidia-container-toolkit.repo \
+    -o /tmp/ublue-os-ucore-nvidia/rpmbuild/SOURCES/nvidia-container-toolkit.repo
+sed -i "s@gpgcheck=0@gpgcheck=1@" /tmp/ublue-os-ucore-nvidia/rpmbuild/SOURCES/nvidia-container-toolkit.repo
 
 curl -L https://raw.githubusercontent.com/NVIDIA/dgx-selinux/master/bin/RHEL9/nvidia-container.pp \
     -o /tmp/ublue-os-ucore-nvidia/rpmbuild/SOURCES/nvidia-container.pp

diff --git a/files/etc/nvidia-container-runtime/config-rootless.toml b/files/etc/nvidia-container-runtime/config-rootless.toml
diff --git a/files/usr/lib/systemd/system-preset/70-ublue-nvctk-cdi.preset b/files/usr/lib/systemd/system-preset/70-ublue-nvctk-cdi.preset
@@ -0,0 +1 @@
+enable ublue-nvctk-cdi.service
diff --git a/files/usr/lib/systemd/system/ublue-nvctk-cdi.service b/files/usr/lib/systemd/system/ublue-nvctk-cdi.service
@@ -0,0 +1,11 @@
+[Unit]
+Description=ublue nvidia container toolkit CDI auto-generation
+ConditionFileIsExecutable=/usr/bin/nvidia-ctk
+After=local-fs.target
+
+[Service]
+Type=oneshot
+ExecStart=/usr/bin/nvidia-ctk cdi generate --output=/etc/cdi/nvidia.yaml
+
+[Install]
+WantedBy=multi-user.target
diff --git a/ublue-os-ucore-nvidia.spec b/ublue-os-ucore-nvidia.spec
@@ -1,5 +1,5 @@
 Name:           ublue-os-ucore-nvidia
-Version:        0.1
+Version:        0.3
 Release:        1%{?dist}
 Summary:        Additional files for nvidia driver support on CoreOS
 
@@ -9,9 +9,10 @@ URL:            https://github.com/ublue-os/ucore-kmods
 BuildArch:      noarch
 Supplements:    mokutil policycoreutils
 
-Source0:        nvidia-container-runtime.repo
-Source1:        config-rootless.toml
-Source2:        nvidia-container.pp
+Source0:        nvidia-container-toolkit.repo
+Source1:        nvidia-container.pp
+Source2:        ublue-nvctk-cdi.service
+Source3:        70-ublue-nvctk-cdi.preset
 
 %description
 Adds various runtime files for nvidia support on Fedora CoreOS.
@@ -21,25 +22,34 @@ Adds various runtime files for nvidia support on Fedora CoreOS.
 
 
 %build
-install -Dm0644 %{SOURCE0} %{buildroot}%{_datadir}/ublue-os/%{_sysconfdir}/yum.repos.d/nvidia-container-runtime.repo
-install -Dm0644 %{SOURCE1} %{buildroot}%{_datadir}/ublue-os/%{_sysconfdir}/nvidia-container-runtime/config-rootless.toml
-install -Dm0644 %{SOURCE2} %{buildroot}%{_datadir}/ublue-os/%{_datadir}/selinux/packages/nvidia-container.pp
+install -Dm0644 %{SOURCE0} %{buildroot}%{_datadir}/ublue-os/%{_sysconfdir}/yum.repos.d/nvidia-container-toolkit.repo
+install -Dm0644 %{SOURCE1} %{buildroot}%{_datadir}/ublue-os/%{_datadir}/selinux/packages/nvidia-container.pp
+install -Dm0644 %{SOURCE2} %{buildroot}%{_datadir}/ublue-os/%{_unitdir}/ublue-nvctk-cdi.service
+install -Dm0644 %{SOURCE3} %{buildroot}%{_presetdir}/70-ublue-nvctk-cdi.preset
 
-sed -i 's@enabled=1@enabled=0@g' %{buildroot}%{_datadir}/ublue-os/%{_sysconfdir}/yum.repos.d/nvidia-container-runtime.repo
+sed -i 's@enabled=1@enabled=0@g' %{buildroot}%{_datadir}/ublue-os/%{_sysconfdir}/yum.repos.d/nvidia-container-toolkit.repo
 
-install -Dm0644 %{buildroot}%{_datadir}/ublue-os/%{_sysconfdir}/yum.repos.d/nvidia-container-runtime.repo     %{buildroot}%{_sysconfdir}/yum.repos.d/nvidia-container-runtime.repo
-install -Dm0644 %{buildroot}%{_datadir}/ublue-os/%{_sysconfdir}/nvidia-container-runtime/config-rootless.toml %{buildroot}%{_sysconfdir}/nvidia-container-runtime/config-rootless.toml
+install -Dm0644 %{buildroot}%{_datadir}/ublue-os/%{_sysconfdir}/yum.repos.d/nvidia-container-toolkit.repo     %{buildroot}%{_sysconfdir}/yum.repos.d/nvidia-container-toolkit.repo
 install -Dm0644 %{buildroot}%{_datadir}/ublue-os/%{_datadir}/selinux/packages/nvidia-container.pp             %{buildroot}%{_datadir}/selinux/packages/nvidia-container.pp
+install -Dm0644 %{buildroot}%{_datadir}/ublue-os/%{_unitdir}/ublue-nvctk-cdi.service                          %{buildroot}%{_unitdir}/ublue-nvctk-cdi.service
 
 %files
-%attr(0644,root,root) %{_datadir}/ublue-os/%{_sysconfdir}/yum.repos.d/nvidia-container-runtime.repo
-%attr(0644,root,root) %{_datadir}/ublue-os/%{_sysconfdir}/nvidia-container-runtime/config-rootless.toml
+%attr(0644,root,root) %{_datadir}/ublue-os/%{_sysconfdir}/yum.repos.d/nvidia-container-toolkit.repo
 %attr(0644,root,root) %{_datadir}/ublue-os/%{_datadir}/selinux/packages/nvidia-container.pp
-%attr(0644,root,root) %{_sysconfdir}/yum.repos.d/nvidia-container-runtime.repo
-%attr(0644,root,root) %{_sysconfdir}/nvidia-container-runtime/config-rootless.toml
+%attr(0644,root,root) %{_datadir}/ublue-os/%{_unitdir}/ublue-nvctk-cdi.service
+%attr(0644,root,root) %{_sysconfdir}/yum.repos.d/nvidia-container-toolkit.repo
 %attr(0644,root,root) %{_datadir}/selinux/packages/nvidia-container.pp
+%attr(0644,root,root) %{_unitdir}/ublue-nvctk-cdi.service
+%attr(0644,root,root) %{_presetdir}/70-ublue-nvctk-cdi.preset
 
 %changelog
+* Fri Oct 6 2023 Benjamin Sherman <[email protected]> - 0.3
+- add ublue-nvctk-cdi service to auto-generate NVIDIA CDI GPU definitions
+
+* Wed Oct 04 2023 Benjamin Sherman <[email protected]> - 0.2
+- use newer nvidia-container-toolkit repo
+- repo provides newer toolkit, no longer requires config.toml
+
 * Sat Aug 19 2023 Benjamin Sherman <[email protected]> - 0.1
 First release for Fedora CoreOS based on ublue-os-nvidia-addons includes:
 - nvidia-container-runtime repo