Skip to content

Commit

Permalink
Merge pull request #35 from datacenter/cilium_support
Browse files Browse the repository at this point in the history 
Cilium support with unit testing and new version.
  • Loading branch information
@samiib
samiib authored Feb 25, 2023
2 parents f2450dd + f52d0e1 commit c749ab4
Show file tree
Hide file tree
Showing 6 changed files with 172 additions and 29 deletions.
2 changes: 1 addition & 1 deletion app/buildah_build.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ export IMAGE_NAME="vkaci"
export IMAGE_NAME_INIT="vkaci-init"
if [ -z "$1" ]
then
export IMAGE_TAG="v1.0.2"
export IMAGE_TAG="v1.1.0"
else
export IMAGE_TAG="$1"
fi
Expand Down
66 changes: 49 additions & 17 deletions app/graph.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,7 @@ def __init__(self, env:VkaciEnvVariables, apic_methods:ApicMethodsResolve) -> No
self.bgp_info = {}
self.env = env
self.apic_methods = apic_methods
self.k8s_as = None

if self.env.tenant is not None and self.env.vrf is not None:
self.aci_vrf = 'uni/tn-' + self.env.tenant + '/ctx-' + self.env.vrf
Expand Down Expand Up @@ -244,49 +245,79 @@ def add_neighbour(self, node, neighbour):
logger.info("Added neighbour details %s to %s - %s", neighbour_adj_port + '-' + neighbour.id, neighbour_adj.sysName, switch)

def get_cluster_as(self):
''' Get the AS from K8s Configuration this assumes Calico is used'''
asn = 0
'''Returns the previously detected AS number'''
return self.k8s_as

def detect_cluster_as(self):
''' Detect the AS from K8s Configuration'''
asn = None
logger.debug("Detect Cluster AS")
# Try to get Cluster AS from Calico Config
try:
logger.debug("Try to detect Calico")
res = self.custom_obj.get_cluster_custom_object(
group="crd.projectcalico.org",
version="v1",
name="default",
plural="bgpconfigurations"
)
logger.info('Calico BGP Config Detected!')
logger.info("Try to detect Calico")
res = self.get_calico_custom_object()
asn = str(res['spec']['asNumber'])
logger.info('Calico BGP Config Detected!')
return asn
except Exception as e:
# The the CRD does not exists I get an 404 not found exeption
# If the CRD does not exists it returns a 404 not found exeption
pass
# Try to get Cluster AS from kube-rotuer Config
try:
logger.debug("Try to detect Kube-Router")
logger.info("Try to detect Kube-Router")
pods = self.get_pods(ns='kube-system')
kr_pod = False
for pod in pods:
if "kube-router" in pod:
#I just need one so I break immediately
# Only check the asn on the first kube-router pod found
kr_pod = self.v1.read_namespaced_pod(pod,'kube-system')
break
if kr_pod:
for arg in kr_pod.spec.containers[0].args:
if "--cluster-asn=" in arg:
asn = arg[14:]
logger.info('Kube-Router Detected! Cluster AS=%s',asn)
logger.info('Kube-Router Detected! Cluster AS=%s',asn)
return asn
except Exception as e:
pass
# Try to get Cluster AS from Cilium Config
try:
# VKACI only supports a single AS per Cluster. A set is used to ensure that
asn_set = set()
logger.info("Try to detect Cilium")
# Get all the CiliumBGPPeeringPolicies traverse them and the virtualRouters and add all the found ASN in the set
CiliumBGPPeeringPolicies = self.list_cilium_custom_objects()
for policy in CiliumBGPPeeringPolicies['items']:
for virtualrotuer in policy['spec']['virtualRouters']:
asn_set.add(str(virtualrotuer['localASN']))
if len(asn_set) == 1:
asn = asn_set.pop()
logger.info('Cilium Detected! Cluster AS=%s',asn)
return asn
elif len(asn_set) > 1:
logger.info('Cilium Detected! More than one AS is used, this is an unsupported configuration!')
except Exception as e:
pass
if asn == 0:
if asn is None:
logger.error("Can't detect K8s Cluster AS, BGP topology will not work corectly")
return asn

def get_calico_custom_object(self):
return self.custom_obj.get_cluster_custom_object(
group="crd.projectcalico.org",
version="v1",
name="default",
plural="bgpconfigurations"
)

def list_cilium_custom_objects(self):
return self.custom_obj.list_cluster_custom_object(group="cilium.io", version="v2alpha1", plural="ciliumbgppeeringpolicies")

def update_bgp_info(self, apic:Node):
'''Get the BGP information'''

# Get the K8s Cluster AS
k8s_as = self.get_cluster_as()
self.k8s_as = self.detect_cluster_as()
overlay_ip_to_switch = self.apic_methods.get_overlay_ip_to_switch_map(apic)
self.bgp_info = {}
vrf = self.env.tenant + ":" + self.env.vrf
Expand All @@ -305,7 +336,7 @@ def update_bgp_info(self, apic:Node):
self.bgp_info[leaf][route] = {}
self.bgp_info[leaf][route]['hosts'] = []
self.bgp_info[leaf][route]['k8s_route'] = True
if hop.tag == k8s_as:
if hop.tag == self.k8s_as:
#self.bgp_info[leaf][route]['ip'].add(next_hop)
host_name = ""
image = "node.svg"
Expand All @@ -329,6 +360,7 @@ def update_node(self, apic, node):
'''Gets a K8s node and populates it with the LLDP/CDP and BGP information'''
if 'mac' not in node:
logger.error("Could not resolved the mac address of node with ip %s", node['node_ip'] )
logger.error("This usually happnes if the Tenant/VRF config is wrong, I am configured to use '%s', is it correct?", self.aci_vrf)
exit()
#Find the mac to interface mapping
logger.info("Find the mac to interface mapping for Node %s with MAC %s", node['node_ip'], node['mac'])
Expand Down
2 changes: 1 addition & 1 deletion app/version.txt
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
Build: v1.0.2
Build: v1.1.0
5 changes: 3 additions & 2 deletions helm/Chart.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,14 @@ type: application
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)

version: 0.1.1
version: 1.1.0


# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to
# follow Semantic Versioning. They should reflect the version the application is using.
# It is recommended to use it with quotes.
appVersion: "v1.0.2"
appVersion: "v1.1.0"

maintainers:
- email: [email protected]
Expand Down
3 changes: 3 additions & 0 deletions helm/templates/clusterrole.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,9 @@ rules:
- apiGroups: ["crd.projectcalico.org"]
resources: ["bgpconfigurations"]
verbs: ["list","get"]
- apiGroups: ["cilium.io"]
resources: ["ciliumbgppeeringpolicies"]
verbs: ["list","get"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
Expand Down
123 changes: 115 additions & 8 deletions unit_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,33 @@
spec=client.V1PodSpec(
node_name="1234abc", containers=[]
)
),
client.V1Pod(
status=client.V1PodStatus(
host_ip="192.168.1.2", pod_ip="192.168.1.2"
),
metadata=client.V1ObjectMeta(
name="kube-router-xfgr", namespace="kube-system"
),
spec=client.V1PodSpec(
node_name="1234abc", containers=[client.V1Container(name="kube-router",
args=[
"--run-router=true",
"--run-firewall=true",
"--run-service-proxy=true",
"--bgp-graceful-restart=true",
"--bgp-holdtime=3s",
"--kubeconfig=/var/lib/kube-router/kubeconfig",
"--cluster-asn=56002",
"--advertise-external-ip",
"--advertise-loadbalancer-ip",
"--advertise-pod-cidr=true",
"--enable-ibgp=false",
"--enable-overlay=false",
"--enable-pod-egress=false",
"--override-nexthop=true"
])]
)
)
]

Expand Down Expand Up @@ -157,7 +184,7 @@ def get_overlay_ip_to_switch_map(self, apic:Node):
@patch('kubernetes.client.CoreV1Api.list_pod_for_all_namespaces', MagicMock(return_value=client.V1PodList(api_version="1", items=pods)))
@patch('kubernetes.client.CoreV1Api.list_service_for_all_namespaces', MagicMock(return_value=client.V1ServiceList(api_version="1", items=services)))
@patch('kubernetes.client.CoreV1Api.list_node', MagicMock(return_value=client.V1NodeList(api_version="1", items=nodes)))
@patch('kubernetes.client.CustomObjectsApi.get_cluster_custom_object', MagicMock(return_value={'spec': {'asNumber': 56001}}))
@patch('app.graph.VkaciBuilTopology.get_calico_custom_object', MagicMock(return_value={'spec': {'asNumber': 56001}}))
class TestVkaciGraph(unittest.TestCase):

vars = {"APIC_IPS": "192.168.25.192,192.168.1.2",
Expand Down Expand Up @@ -189,7 +216,8 @@ def test_valid_topology(self):
"""Test that a valid topology is created"""
# Arrange
expected = {'nodes': {'1234abc': {'node_ip': '192.168.1.2',
'pods': {'dateformat': {'ip': '192.158.1.3', 'ns': 'dockerimage', 'labels': {'guest': 'frontend'}}},
'pods': {'dateformat': {'ip': '192.158.1.3', 'ns': 'dockerimage', 'labels': {'guest': 'frontend'}},
'kube-router-xfgr': {'ip': '192.168.1.2', 'ns': 'kube-system', 'labels': {}}},
'bgp_peers': {'leaf-204': {'prefix_count': 2}}, 'neighbours': {'esxi4.cam.ciscolabs.com':
{'switches': {'leaf-204': {'vmxnic1-eth1/1'}}, 'Description': 'VMware version 123'}},
'labels': {'app': 'redis'}, 'mac': 'MOCKMO1C'}},
Expand Down Expand Up @@ -219,7 +247,11 @@ def test_valid_topology_cdpn(self):
'node_ip': '192.168.1.2',
'pods': {'dateformat': {'ip': '192.158.1.3',
'labels': {'guest': 'frontend'},
'ns': 'dockerimage'}}}},
'ns': 'dockerimage'},
'kube-router-xfgr': {'ip': '192.168.1.2',
'labels': {},
'ns': 'kube-system'}
}}},
'services': {'appx': [{'cluster_ip': '192.168.25.5',
'external_i_ps': ['192.168.5.1'],
'labels': {'app': 'guestbook'},
Expand Down Expand Up @@ -248,7 +280,11 @@ def test_valid_topology_no_neighbours(self):
'node_ip': '192.168.1.2',
'pods': {'dateformat': {'ip': '192.158.1.3',
'labels': {'guest': 'frontend'},
'ns': 'dockerimage'}}}},
'ns': 'dockerimage'},
'kube-router-xfgr': {'ip': '192.168.1.2',
'labels': {},
'ns': 'kube-system'}
}}},
'services': {'appx': [{'cluster_ip': '192.168.25.5',
'external_i_ps': ['192.168.5.1'],
'labels': {'app': 'guestbook'},
Expand Down Expand Up @@ -278,7 +314,11 @@ def test_valid_topology_no_desc_neighbour(self):
'node_ip': '192.168.1.2',
'pods': {'dateformat': {'ip': '192.158.1.3',
'labels': {'guest': 'frontend'},
'ns': 'dockerimage'}}}},
'ns': 'dockerimage'},
'kube-router-xfgr': {'ip': '192.168.1.2',
'labels': {},
'ns': 'kube-system'},
}}},
'services': {'appx': [{'cluster_ip': '192.168.25.5',
'external_i_ps': ['192.168.5.1'],
'labels': {'app': 'guestbook'},
Expand All @@ -305,7 +345,11 @@ def test_leaf_table(self):
'data': [{'data': [{'data': [{'data': [{'image': 'pod.svg',
'ip': '192.158.1.3',
'ns': 'dockerimage',
'value': 'dateformat'}],
'value': 'dateformat'},
{'image': 'pod.svg',
'ip': '192.168.1.2',
'ns': 'kube-system',
'value': 'kube-router-xfgr'}],
'image': 'node.svg',
'ip': '192.168.1.2',
'ns': '',
Expand Down Expand Up @@ -381,7 +425,10 @@ def test_pod_table(self):
# Arrange
expected = {'parent': 0, 'data': [{'value': 'leaf-204', 'ip': '', 'image': 'switch.png',
'data': [{'value': 'dateformat', 'ip': '192.158.1.3','ns': 'dockerimage', 'image': 'pod.svg',
'data': [{'value': 'guest', 'label_value': 'frontend', 'image': 'label.svg'}]}]}]}
'data': [{'value': 'guest', 'label_value': 'frontend', 'image': 'label.svg'}]},
{'value': 'kube-router-xfgr', 'ip': '192.168.1.2','ns': 'kube-system', 'image': 'pod.svg',
'data': []}
]}]}

build = VkaciBuilTopology(
VkaciEnvVariables(self.vars), ApicMethodsMock())
Expand Down Expand Up @@ -411,7 +458,67 @@ def test_services_table(self):
# Assert
self.assertDictEqual(result, expected)



def assert_cluster_as(self, expected):
build = VkaciBuilTopology(
VkaciEnvVariables(self.vars), ApicMethodsMock())
build.update()
asn = build.get_cluster_as()
self.assertEqual(asn, expected)


def test_calico_bgp_as_detection(self):
"""Test that the bgp AS is detected with calico"""
"""This is the default used on other tests but better be explicit so no one thinks it hasn't been tested"""
self.assert_cluster_as('56001')


@patch('kubernetes.client.CoreV1Api.read_namespaced_pod', MagicMock(return_value=pods[1]))
def test_kube_router_bgp_as_detection(self):
"""Test that the bgp AS is detected with kube-router"""
with patch('app.graph.VkaciBuilTopology.get_calico_custom_object', MagicMock(return_value={})):
self.assert_cluster_as('56002')


# AS numbers are intentionally repeated for testing.
cilium_policies = {
"items": [
{"spec": {"virtualRouters": [
{'localASN': 56003}, {'localASN': 56003}]}},
{"spec": {"virtualRouters": [{'localASN': 56003}]}}
]
}
@patch('kubernetes.client.CoreV1Api.read_namespaced_pod', MagicMock(return_value=None))
@patch('app.graph.VkaciBuilTopology.list_cilium_custom_objects', MagicMock(return_value=cilium_policies))
def test_cilium_bgp_as_detection(self):
"""Test that the bgp AS is detected with cilium"""
with patch('app.graph.VkaciBuilTopology.get_calico_custom_object', MagicMock(return_value={})):
self.assert_cluster_as('56003')


# Different AS numbers in Cilium is not supported.
invalid_cilium_policies = {
"items": [
{"spec": {"virtualRouters": [
{'localASN': 56003}, {'localASN': 56004}]}},
{"spec": {"virtualRouters": [{'localASN': 56005}]}}
]
}
@patch('kubernetes.client.CoreV1Api.read_namespaced_pod', MagicMock(return_value=None))
@patch('app.graph.VkaciBuilTopology.list_cilium_custom_objects', MagicMock(return_value=invalid_cilium_policies))
def test_invalid_cilium_bgp_as_detection(self):
"""Test that the bgp AS is not detected with invalid cilium config"""
with patch('app.graph.VkaciBuilTopology.get_calico_custom_object', MagicMock(return_value={})):
self.assert_cluster_as(None)


@patch('kubernetes.client.CoreV1Api.read_namespaced_pod', MagicMock(return_value=None))
@patch('app.graph.VkaciBuilTopology.list_cilium_custom_objects', MagicMock(return_value=[]))
def test_invalid_as_detection(self):
"""Test that the bgp AS is not detected with no valid config"""
with patch('app.graph.VkaciBuilTopology.get_calico_custom_object', MagicMock(return_value={})):
self.assert_cluster_as(None)


if __name__ == '__main__':
unittest.main()

0 comments on commit c749ab4

Please sign in to comment.