From 98492a8bf9372bb201622b4f831c86e80d50643c Mon Sep 17 00:00:00 2001 From: Kaelem <62122480+kaelemc@users.noreply.github.com> Date: Wed, 10 Jul 2024 00:26:13 +1200 Subject: [PATCH] Add support for Cisco Catalyst 9000v (#225) * Add minimum nics arg to VM class & add dummy nic generation feature for Cat9kv support. * Add removal of .xml files from docker/ directory. * add copy of vswitch.xml file if present (for cat9kv). * Add initial support for Cisco Cat9kv * Remove image install process. * Remove ASIC image tagging * Correct hostname in bootstrap & startup configurations. * formatting * Fix directory creation for when vswitch.xml file is added via binds * Add supplement qemu PCI args for dummy nics * format with ruff add no-install-recommends --------- Co-authored-by: Roman Dodin --- cat9kv/Makefile | 12 ++ cat9kv/README.md | 95 ++++++++++++++ cat9kv/docker/Dockerfile | 31 +++++ cat9kv/docker/launch.py | 264 +++++++++++++++++++++++++++++++++++++++ common/vrnetlab.py | 52 +++++++- makefile.include | 3 +- 6 files changed, 454 insertions(+), 3 deletions(-) create mode 100644 cat9kv/Makefile create mode 100644 cat9kv/README.md create mode 100644 cat9kv/docker/Dockerfile create mode 100755 cat9kv/docker/launch.py diff --git a/cat9kv/Makefile b/cat9kv/Makefile new file mode 100644 index 00000000..4e4fd6d5 --- /dev/null +++ b/cat9kv/Makefile @@ -0,0 +1,12 @@ +VENDOR=Cisco +NAME=cat9kv +IMAGE_FORMAT=qcow2 +IMAGE_GLOB=*.qcow2 + +# match versions like: +# csr1000v-universalk9.16.03.01a.qcow2 +# csr1000v-universalk9.16.04.01.qcow2 +VERSION=$(shell echo $(IMAGE) | sed -e 's/.\+[^0-9]\([0-9]\+\.[0-9]\+\.[0-9]\+[a-z]\?\)\([^0-9].*\|$$\)/\1/') + +-include ../makefile-sanity.include +-include ../makefile.include \ No newline at end of file diff --git a/cat9kv/README.md b/cat9kv/README.md new file mode 100644 index 00000000..dc05ffbc --- /dev/null +++ b/cat9kv/README.md @@ -0,0 +1,95 @@ +# Cisco Catalyst 9000V + +This is the vrnetlab image for the Cisco Catalyst 9000v (cat9kv, c9000v). + +The Cat9kv emulates two types of ASICs that are found in the common Catalyst 9000 hardware platforms, either: + +- UADP (Cisco Unified Access Data Plane) +- Cisco Silicon One Q200 (referred to as Q200 for short) + +The Q200 is a newer ASIC, however doen't support as many features as the UADP ASIC emulation. + +> Insufficient RAM will not allow the node to boot correctly. + +Eight interfaces will always appear regardless if you have defined any links in the `*.clab.yaml` topology file. The Cat9kv requires 8 interfaces at minimum to boot, so dummy interfaces are created if there are an insufficient amount of interfaces (links) defined. + +## Building the image + +Copy the Cat9kv .qcow2 file in this directory and you can perform `make docker-image`. On average the image takes approxmiately ~4 minutes to build as an initial install process occurs. + +The UADP and Q200 use the same .qcow2 image. The default image created is the UADP image. + +To configure the Q200 image or enable a higher throughput dataplane for UADP; you must supply the relevant `vswitch.xml` file. You can place that file in this directory and build the image. + +> You can obtain a `vswitch.xml` file from the relevant CML node definiton file. + +Known working versions: + +- cat9kv-prd-17.12.01prd9.qcow2 (UADP & Q200) + +## Usage + +You can define the image easily and use it in a topolgy. As mentioned earlier no links are requried to be defined. + +```yaml +# topology.clab.yaml +name: mylab +topology: + nodes: + cat9kv: + kind: cisco_cat9kv + image: vrnetlab/vr-cat9kv: +``` + +You can also supply a vswitch.xml file using `binds`. Below is an example topology file. + +```yaml +# topology.clab.yaml +name: mylab +topology: + nodes: + cat9kv: + kind: cisco_cat9kv + image: vrnetlab/vr-cat9kv: + binds: + - /path/to/vswitch.xml:/vswitch.xml +``` + +### Interface naming + +Currently a maximum of 8 data-plane interfaces are supported. 9 interfaces total if including the management interface. + +- `eth0` - Node management interface +- `eth1` - First dataplane interface (GigabitEthernet1/0/1). +- `ethX` - Subsequent dataplane interfaces will count onwards from 1. For example, the third dataplane interface will be `eth3` + +You can also use interface aliases of `GigabitEthernet1/0/x` or `Gi1/0/x` + +### Environment Variables + +| Environment Variable | Default | +| --------------------- | ------------- | +| VCPU | 4 | +| RAM | 18432 | + +### Example + +```yaml +name: my-example-lab +topology: + nodes: + my-cat9kv: + kind: cisco_cat9kv + image: vrnetlab/vr-cat9kv:17.12.01 + env: + VCPU: 6 + RAM: 12288 +``` + +## System requirements + +| | UADP (Default)| Q200 | +| --------- | ------------- | ----- | +| vCPU | 4 | 4 | +| RAM (MB) | 18432 | 12288 | +| Disk (GB) | 4 | 4 | diff --git a/cat9kv/docker/Dockerfile b/cat9kv/docker/Dockerfile new file mode 100644 index 00000000..335c7dd1 --- /dev/null +++ b/cat9kv/docker/Dockerfile @@ -0,0 +1,31 @@ +FROM public.ecr.aws/docker/library/debian:bookworm-slim + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update -qy \ + && apt-get install -y --no-install-recommends \ + bridge-utils \ + iproute2 \ + socat \ + qemu-kvm \ + qemu-utils \ + python3 \ + tcpdump \ + inetutils-ping \ + ssh \ + telnet \ + procps \ + genisoimage \ + && rm -rf /var/lib/apt/lists/* + +ARG VERSION +ENV VERSION=${VERSION} +ARG IMAGE +COPY $IMAGE* / +COPY *.py / +# for vSwitch.xml file (specifies ASIC emulation parameters), won't throw error if vswitch.xml isn't present +COPY vswitch.xm[l] /img_dir/conf/ + +EXPOSE 22 161/udp 830 5000 10000-10099 +HEALTHCHECK CMD ["/healthcheck.py"] +ENTRYPOINT ["/launch.py"] diff --git a/cat9kv/docker/launch.py b/cat9kv/docker/launch.py new file mode 100755 index 00000000..179cd987 --- /dev/null +++ b/cat9kv/docker/launch.py @@ -0,0 +1,264 @@ +#!/usr/bin/env python3 + +import datetime +import logging +import os +import re +import signal +import subprocess +import sys + +import vrnetlab + +STARTUP_CONFIG_FILE = "/config/startup-config.cfg" + + +def handle_SIGCHLD(signal, frame): + os.waitpid(-1, os.WNOHANG) + + +def handle_SIGTERM(signal, frame): + sys.exit(0) + + +signal.signal(signal.SIGINT, handle_SIGTERM) +signal.signal(signal.SIGTERM, handle_SIGTERM) +signal.signal(signal.SIGCHLD, handle_SIGCHLD) + +TRACE_LEVEL_NUM = 9 +logging.addLevelName(TRACE_LEVEL_NUM, "TRACE") + + +def trace(self, message, *args, **kws): + # Yes, logger takes its '*args' as 'args'. + if self.isEnabledFor(TRACE_LEVEL_NUM): + self._log(TRACE_LEVEL_NUM, message, args, **kws) + + +logging.Logger.trace = trace + + +class cat9kv_vm(vrnetlab.VM): + def __init__(self, hostname, username, password, conn_mode, vcpu, ram): + disk_image = None + for e in sorted(os.listdir("/")): + if not disk_image and re.search(".qcow2$", e): + disk_image = "/" + e + if re.search(r"\.license$", e): + os.rename("/" + e, "/tftpboot/license.lic") + + self.license = False + if os.path.isfile("/tftpboot/license.lic"): + logger.info("License found") + self.license = True + + super().__init__( + username, + password, + disk_image=disk_image, + smp=f"cores={vcpu},threads=1,sockets=1", + ram=ram, + min_dp_nics=8, + ) + self.hostname = hostname + self.conn_mode = conn_mode + self.num_nics = 9 + self.nic_type = "virtio-net-pci" + + self.image_name = "config.img" + + self.qemu_args.extend( + [ + "-overcommit mem-lock=off", + f"-boot order=cd -cdrom /{self.image_name}", + ] + ) + + # create .img which is mounted for startup config and contains ASIC emulation in 'conf/vswitch.xml' dir. + self.create_boot_image() + + def create_boot_image(self): + """Creates a iso image with a bootstrap configuration""" + try: + os.makedirs("/img_dir/conf") + except: + self.logger.error( + "Unable to make '/img_dir'. Does the directory already exist?" + ) + + try: + os.popen("cp /vswitch.xml /img_dir/conf/") + except: + self.logger.debug("No vswitch.xml file provided.") + + with open("/img_dir/iosxe_config.txt", "w") as cfg_file: + cfg_file.write(f"hostname {self.hostname}\r\n") + cfg_file.write("end\r\n") + + genisoimage_args = [ + "genisoimage", + "-l", + "-o", + "/" + self.image_name, + "/img_dir", + ] + + self.logger.debug("Generating boot ISO") + subprocess.Popen(genisoimage_args) + + def bootstrap_spin(self): + """This function should be called periodically to do work.""" + + if self.spins > 300: + # too many spins with no result -> give up + self.stop() + self.start() + return + + (ridx, match, res) = self.tn.expect( + [ + b"Press RETURN to get started!", + b"IOSXEBOOT-4-FACTORY_RESET", + ], + 1, + ) + if match: # got a match! + if ridx == 0: # login + self.logger.debug("matched, Press RETURN to get started.") + + self.wait_write("", wait=None) + + # run main config! + self.bootstrap_config() + # add startup config if present + self.startup_config() + # close telnet connection + self.tn.close() + # startup time? + startup_time = datetime.datetime.now() - self.start_time + self.logger.info("Startup complete in: %s", startup_time) + # mark as running + self.running = True + return + elif ridx == 1: # IOSXEBOOT-4-FACTORY_RESET + self.logger.warning("Unexpected reload while running") + + # no match, if we saw some output from the router it's probably + # booting, so let's give it some more time + if res != b"": + self.logger.trace("OUTPUT: %s", res.decode()) + # reset spins if we saw some output + self.spins = 0 + + self.spins += 1 + + return + + def bootstrap_config(self): + """Do the actual bootstrap config""" + self.logger.info("applying bootstrap configuration") + + self.wait_write("", None) + self.wait_write("enable", wait=">") + self.wait_write("configure terminal", wait=">") + + self.wait_write(f"hostname {self.hostname}") + self.wait_write( + "username %s privilege 15 password %s" % (self.username, self.password) + ) + if int(self.version.split(".")[0]) >= 16: + self.wait_write("ip domain name example.com") + else: + self.wait_write("ip domain-name example.com") + self.wait_write("crypto key generate rsa modulus 2048") + + self.wait_write("no ip domain lookup") + + self.wait_write("interface GigabitEthernet0/0") + self.wait_write("ip address 10.0.0.15 255.255.255.0") + self.wait_write("no shut") + self.wait_write("exit") + + self.wait_write("restconf") + self.wait_write("netconf-yang") + self.wait_write("netconf max-sessions 16") + # I did not find any documentation about this, but is seems like a good idea!? + self.wait_write("netconf detailed-error") + self.wait_write("ip ssh server algorithm mac hmac-sha2-512") + self.wait_write("ip ssh maxstartups 128") + + self.wait_write("line vty 0 4") + self.wait_write("login local") + self.wait_write("transport input all") + self.wait_write("end") + self.wait_write("copy running-config startup-config") + self.wait_write("\r", "Destination") + + def startup_config(self): + """Load additional config provided by user.""" + + if not os.path.exists(STARTUP_CONFIG_FILE): + self.logger.trace(f"Startup config file {STARTUP_CONFIG_FILE} is not found") + return + + self.logger.trace(f"Startup config file {STARTUP_CONFIG_FILE} exists") + with open(STARTUP_CONFIG_FILE) as file: + config_lines = file.readlines() + config_lines = [line.rstrip() for line in config_lines] + self.logger.trace(f"Parsed startup config file {STARTUP_CONFIG_FILE}") + + self.logger.info(f"Writing lines from {STARTUP_CONFIG_FILE}") + + self.wait_write("configure terminal") + # Apply lines from file + for line in config_lines: + self.wait_write(line) + # End and Save + self.wait_write("end") + self.wait_write("copy running-config startup-config") + self.wait_write("\r", "Destination") + + +class cat9kv(vrnetlab.VR): + def __init__(self, hostname, username, password, conn_mode, vcpu, ram): + super(cat9kv, self).__init__(username, password) + self.vms = [cat9kv_vm(hostname, username, password, conn_mode, vcpu, ram)] + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser(description="") + parser.add_argument( + "--trace", action="store_true", help="enable trace level logging" + ) + parser.add_argument("--username", default="vrnetlab", help="Username") + parser.add_argument("--password", default="VR-netlab9", help="Password") + parser.add_argument("--hostname", default="cat9kv", help="Router hostname") + parser.add_argument( + "--connection-mode", + default="vrxcon", + help="Connection mode to use in the datapath", + ) + parser.add_argument("--vcpu", type=int, default=4, help="Allocated vCPUs") + parser.add_argument("--ram", type=int, default=18432, help="Allocaetd RAM in MB") + + args = parser.parse_args() + + LOG_FORMAT = "%(asctime)s: %(module)-10s %(levelname)-8s %(message)s" + logging.basicConfig(format=LOG_FORMAT) + logger = logging.getLogger() + + logger.setLevel(logging.DEBUG) + if args.trace: + logger.setLevel(1) + + vr = cat9kv( + args.hostname, + args.username, + args.password, + args.connection_mode, + args.vcpu, + args.ram, + ) + vr.start() diff --git a/common/vrnetlab.py b/common/vrnetlab.py index d4e35b42..3c0d7cd4 100644 --- a/common/vrnetlab.py +++ b/common/vrnetlab.py @@ -77,6 +77,7 @@ def __init__( provision_pci_bus=True, cpu="host", smp="1", + min_dp_nics=0, ): self.logger = logging.getLogger() @@ -106,6 +107,11 @@ def __init__( # "highest" provisioned nic num -- used for making sure we can allocate nics without needing # to have them allocated sequential from eth1 self.highest_provisioned_nic_num = 0 + + self.insuffucient_nics = False + # if an image needs minimum amount of dataplane nics to bootup, specify + if min_dp_nics: + self.min_nics = min_dp_nics # we setup pci bus by default self.provision_pci_bus = provision_pci_bus @@ -197,6 +203,9 @@ def start(self): cmd.extend(self.gen_mgmt()) # generate normal NICs cmd.extend(self.gen_nics()) + # generate dummy NICs + if self.insuffucient_nics: + cmd.extend(self.gen_dummy_nics()) self.logger.debug("qemu cmd: {}".format(" ".join(cmd))) @@ -315,8 +324,11 @@ def nic_provision_delay(self) -> None: f"number of provisioned data plane interfaces is {self.num_provisioned_nics}" ) + # no nics provisioned and/or not running from containerlab so we can bail if self.num_provisioned_nics == 0: - # no nics provisioned and/or not running from containerlab so we can bail + # unless the node has a minimum nic requirement + if self.min_nics: + self.insuffucient_nics = True return self.logger.debug("waiting for provisioned interfaces to appear...") @@ -346,8 +358,44 @@ def nic_provision_delay(self) -> None: f"highest allocated interface id determined to be: {self.highest_provisioned_nic_num}..." ) self.logger.debug("interfaces provisioned, continuing...") - return + break time.sleep(5) + + # check if we need to provision any more nics, do this after because they shouldn't interfere with the provisioned nics + if self.num_provisioned_nics < self.min_nics: + self.insuffucient_nics = True + + # if insuffucient amount of nics are defined in the topology file, generate dummmy nics so cat9kv can boot. + def gen_dummy_nics(self): + # calculate required num of nics to generate + nics = self.min_nics - self.num_provisioned_nics + + self.logger.debug(f"Insuffucient NICs defined. Generating {nics} dummy nics") + + res=[] + + pci_bus_ctr = self.num_provisioned_nics + + for i in range(0, nics): + # dummy interface naming + interface_name = f"dummy{str(i+self.num_provisioned_nics)}" + + # PCI bus counter is to ensure pci bus index starts from 1 + # and continuing in sequence regardles the eth index + pci_bus_ctr += 1 + + pci_bus = math.floor(pci_bus_ctr / self.nics_per_pci_bus) + 1 + addr = (pci_bus_ctr % self.nics_per_pci_bus) + 1 + + res.extend( + [ + "-device", + f"{self.nic_type},netdev={interface_name},id={interface_name},mac={gen_mac(i)},bus=pci.{pci_bus},addr=0x{addr}", + "-netdev", + f"tap,ifname={interface_name},id={interface_name},script=no,downscript=no", + ] + ) + return res def gen_nics(self): """Generate qemu args for the normal traffic carrying interface(s)""" diff --git a/makefile.include b/makefile.include index b78ae3b0..8d597467 100644 --- a/makefile.include +++ b/makefile.include @@ -13,7 +13,7 @@ docker-image: endif docker-clean-build: - -rm -f docker/*.qcow2* docker/*.tgz* docker/*.vmdk* docker/*.iso + -rm -f docker/*.qcow2* docker/*.tgz* docker/*.vmdk* docker/*.iso docker/*.xml docker-pre-build: ; @@ -25,6 +25,7 @@ docker-build-common: docker-clean-build docker-pre-build @if [ "$(IMAGE)" = "$(VERSION)" ]; then echo "ERROR: Incorrect version string ($(IMAGE)). The regexp for extracting version information is likely incorrect, check the regexp in the Makefile or open an issue at https://github.com/plajjan/vrnetlab/issues/new including the image file name you are using."; exit 1; fi @echo "Building docker image using $(IMAGE) as $(REGISTRY)vr-$(VR_NAME):$(VERSION)" cp ../common/* docker/ + @[ -f ./vswitch.xml ] && cp vswitch.xml docker/ || true $(MAKE) IMAGE=$$IMAGE docker-build-image-copy (cd docker; docker build --build-arg http_proxy=$(http_proxy) --build-arg HTTP_PROXY=$(HTTP_PROXY) --build-arg https_proxy=$(https_proxy) --build-arg HTTPS_PROXY=$(HTTPS_PROXY) --build-arg IMAGE=$(IMAGE) --build-arg VERSION=$(VERSION) -t $(REGISTRY)vr-$(VR_NAME):$(VERSION) .)