Reconfigure GPU passthrough on the test cluster

We need to demo GPU passthrough so this reconfigures the test cluster to pass
through GPUs for VMs. Note that this will trigger a machineconfig update on
the test cluster.

cluster-scope/overlays/nerc-ocp-test/machineconfigs/kustomization.yaml

@@ -5,3 +5,4 @@ resources:
   - configure-bond0
   - configure-sysctl-ip-forward
   - hostpath-provisioner-selinux-controller.yaml
+  - pci-passthrough

cluster-scope/overlays/nerc-ocp-test/machineconfigs/pci-passthrough/100-worker-iommu.yaml

@@ -0,0 +1,12 @@
+apiVersion: machineconfiguration.openshift.io/v1
+kind: MachineConfig
+metadata:
+  labels:
+    machineconfiguration.openshift.io/role: worker
+  name: 100-worker-iommu
+spec:
+  config:
+    ignition:
+      version: 3.2.0
+  kernelArguments:
+    - intel_iommu=on

cluster-scope/overlays/nerc-ocp-test/machineconfigs/pci-passthrough/100-worker-vfiopci.yaml

@@ -0,0 +1,25 @@
+# Generated by Butane; do not edit
+apiVersion: machineconfiguration.openshift.io/v1
+kind: MachineConfig
+metadata:
+  labels:
+    machineconfiguration.openshift.io/role: worker
+  name: 100-worker-vfiopci
+spec:
+  config:
+    ignition:
+      version: 3.4.0
+    storage:
+      files:
+        - contents:
+            compression: ""
+            source: data:,options%20vfio-pci%20ids%3D10de%3A1db6%2C10de%3A20b0%0A
+          mode: 420
+          overwrite: true
+          path: /etc/modprobe.d/vfio.conf
+        - contents:
+            compression: ""
+            source: data:,vfio-pci
+          mode: 420
+          overwrite: true
+          path: /etc/modules-load.d/vfio-pci.conf

cluster-scope/overlays/nerc-ocp-test/machineconfigs/pci-passthrough/Makefile

@@ -0,0 +1,10 @@
+MACHINECONFIGS = \
+	100-worker-vfiopci.yaml
+
+%.yaml: src/%.bu
+	butane -o $@ -d src $<
+
+all: $(MACHINECONFIGS)
+
+clean:
+	rm -f $(MACHINECONFIGS)

cluster-scope/overlays/nerc-ocp-test/machineconfigs/pci-passthrough/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - 100-worker-iommu.yaml
+  - 100-worker-vfiopci.yaml

cluster-scope/overlays/nerc-ocp-test/machineconfigs/pci-passthrough/src/100-worker-vfiopci.bu

@@ -0,0 +1,19 @@
+variant: openshift
+version: 4.15.0
+metadata:
+  name: 100-worker-vfiopci
+  labels:
+    machineconfiguration.openshift.io/role: worker
+storage:
+  files:
+  - path: /etc/modprobe.d/vfio.conf
+    mode: 0644
+    overwrite: true
+    contents:
+      inline: |
+        options vfio-pci ids=10de:1db6,10de:20b0
+  - path: /etc/modules-load.d/vfio-pci.conf
+    mode: 0644
+    overwrite: true
+    contents:
+      inline: vfio-pci

virt/overlays/nerc-ocp-test/hyperconverged.yaml

@@ -0,0 +1,12 @@
+apiVersion: hco.kubevirt.io/v1beta1
+kind: HyperConverged
+metadata:
+  name: kubevirt-hyperconverged
+  namespace: openshift-cnv
+spec:
+  permittedHostDevices:
+    pciHostDevices:
+    - pciDeviceSelector: "10DE:1DB6"
+      resourceName: "nvidia.com/GV100GL_Tesla_V100"
+    - pciDeviceSelector: "10DE:20B0"
+      resourceName: "nvidia.com/A100_SXM4_40GB"

virt/overlays/nerc-ocp-test/kustomization.yaml

@@ -3,3 +3,6 @@ kind: Kustomization
 namespace: openshift-cnv
 resources:
   - ../../base
+
+patches:
+- path: hyperconverged.yaml