From c6b0a258e4d4a01470e97e108327b222bf8351a1 Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Thu, 5 Dec 2024 20:07:52 -0700 Subject: [PATCH] doc scratch --- design-docs/src/mdbook/src/SUMMARY.md | 73 +++++++++---------- .../mdbook/src/dataplane/design-session.md | 14 ++-- .../mdbook/src/dataplane/development-plan.md | 23 +++--- .../src/mdbook/src/dataplane/map-of-dpdk.md | 18 ++--- ...configuration-persistence-investigation.md | 2 +- .../dataplane/tasks/control-plane-dev-env.md | 9 +-- .../src/dataplane/tasks/core-pinning.md | 2 +- .../tasks/create-control-plane-image.md | 7 +- .../dataplane-control-plane-transport.md | 2 +- .../mdbook/src/dataplane/tasks/frr-plugin.md | 5 -- .../src/dataplane/tasks/pick-a-datastore.md | 32 ++++++++ .../src/dataplane/tasks/vpc-rate-limiting.md | 6 +- .../src/dataplane/tasks/zebra-plugin.md | 24 ++++++ .../src/mdbook/src/dataplane/tasks2/NAT44.md | 1 + .../dataplane/tasks2/NAT64-investigation.md | 1 + .../src/mdbook/src/dataplane/tasks2/NAT64.md | 1 + .../src/mdbook/src/dataplane/tasks2/NAT66.md | 1 + .../src/dataplane/tasks2/config-db-schema.md | 1 + ...configuration-persistence-investigation.md | 1 + .../dataplane/tasks2/control-plane-dev-env.md | 1 + .../src/dataplane/tasks2/core-pinning.md | 1 + .../tasks2/create-control-plane-image.md | 1 + .../dataplane-control-plane-protocol.md | 1 + .../dataplane-control-plane-transport.md | 1 + .../tasks2/dataplane-worker-lifecycle.md | 1 + .../tasks2/fault-tolerance-implementation.md | 1 + .../tasks2/fault-tolerance-validation.md | 1 + .../src/dataplane/tasks2/gateway-test-env.md | 1 + .../tasks2/identify-local-traffic.md | 1 + ...agement-plane-control-plane-interaction.md | 1 + .../management-plane-dataplane-interaction.md | 1 + .../one-control-plane-daemon-per-container.md | 1 + .../tasks2/performance-measurement.md | 1 + .../tasks2/programmatic-control-of-frr.md | 1 + .../tasks2/public-internet-access.md | 1 + .../tasks2/rate-limiting-investigation.md | 1 + .../src/dataplane/tasks2/route-manager.md | 1 + .../src/dataplane/tasks2/state-sync-design.md | 1 + .../mdbook/src/dataplane/tasks2/state-sync.md | 1 + .../src/dataplane/tasks2/telemetry-basic.md | 1 + .../dataplane/tasks2/telemetry-integration.md | 1 + .../tasks2/telemetry-investigation.md | 1 + .../src/dataplane/tasks2/underlay-routing.md | 1 + .../src/dataplane/tasks2/vpc-rate-limiting.md | 1 + .../src/dataplane/tasks2/vpc-routing.md | 1 + .../src/dataplane/tasks2/vxlan-tunnels.md | 1 + .../src/dataplane/tasks2/zebra-plugin.md | 1 + design-docs/src/mdbook/src/links.md | 4 + justfile | 1 + scripts/dpdk-sys.env | 2 +- 50 files changed, 172 insertions(+), 86 deletions(-) delete mode 100644 design-docs/src/mdbook/src/dataplane/tasks/frr-plugin.md create mode 100644 design-docs/src/mdbook/src/dataplane/tasks/pick-a-datastore.md create mode 100644 design-docs/src/mdbook/src/dataplane/tasks/zebra-plugin.md create mode 100644 design-docs/src/mdbook/src/dataplane/tasks2/NAT44.md create mode 100644 design-docs/src/mdbook/src/dataplane/tasks2/NAT64-investigation.md create mode 100644 design-docs/src/mdbook/src/dataplane/tasks2/NAT64.md create mode 100644 design-docs/src/mdbook/src/dataplane/tasks2/NAT66.md create mode 100644 design-docs/src/mdbook/src/dataplane/tasks2/config-db-schema.md create mode 100644 design-docs/src/mdbook/src/dataplane/tasks2/configuration-persistence-investigation.md create mode 100644 design-docs/src/mdbook/src/dataplane/tasks2/control-plane-dev-env.md create mode 100644 design-docs/src/mdbook/src/dataplane/tasks2/core-pinning.md create mode 100644 design-docs/src/mdbook/src/dataplane/tasks2/create-control-plane-image.md create mode 100644 design-docs/src/mdbook/src/dataplane/tasks2/dataplane-control-plane-protocol.md create mode 100644 design-docs/src/mdbook/src/dataplane/tasks2/dataplane-control-plane-transport.md create mode 100644 design-docs/src/mdbook/src/dataplane/tasks2/dataplane-worker-lifecycle.md create mode 100644 design-docs/src/mdbook/src/dataplane/tasks2/fault-tolerance-implementation.md create mode 100644 design-docs/src/mdbook/src/dataplane/tasks2/fault-tolerance-validation.md create mode 100644 design-docs/src/mdbook/src/dataplane/tasks2/gateway-test-env.md create mode 100644 design-docs/src/mdbook/src/dataplane/tasks2/identify-local-traffic.md create mode 100644 design-docs/src/mdbook/src/dataplane/tasks2/management-plane-control-plane-interaction.md create mode 100644 design-docs/src/mdbook/src/dataplane/tasks2/management-plane-dataplane-interaction.md create mode 100644 design-docs/src/mdbook/src/dataplane/tasks2/one-control-plane-daemon-per-container.md create mode 100644 design-docs/src/mdbook/src/dataplane/tasks2/performance-measurement.md create mode 100644 design-docs/src/mdbook/src/dataplane/tasks2/programmatic-control-of-frr.md create mode 100644 design-docs/src/mdbook/src/dataplane/tasks2/public-internet-access.md create mode 100644 design-docs/src/mdbook/src/dataplane/tasks2/rate-limiting-investigation.md create mode 100644 design-docs/src/mdbook/src/dataplane/tasks2/route-manager.md create mode 100644 design-docs/src/mdbook/src/dataplane/tasks2/state-sync-design.md create mode 100644 design-docs/src/mdbook/src/dataplane/tasks2/state-sync.md create mode 100644 design-docs/src/mdbook/src/dataplane/tasks2/telemetry-basic.md create mode 100644 design-docs/src/mdbook/src/dataplane/tasks2/telemetry-integration.md create mode 100644 design-docs/src/mdbook/src/dataplane/tasks2/telemetry-investigation.md create mode 100644 design-docs/src/mdbook/src/dataplane/tasks2/underlay-routing.md create mode 100644 design-docs/src/mdbook/src/dataplane/tasks2/vpc-rate-limiting.md create mode 100644 design-docs/src/mdbook/src/dataplane/tasks2/vpc-routing.md create mode 100644 design-docs/src/mdbook/src/dataplane/tasks2/vxlan-tunnels.md create mode 100644 design-docs/src/mdbook/src/dataplane/tasks2/zebra-plugin.md diff --git a/design-docs/src/mdbook/src/SUMMARY.md b/design-docs/src/mdbook/src/SUMMARY.md index 346ffa4d..3e1d2131 100644 --- a/design-docs/src/mdbook/src/SUMMARY.md +++ b/design-docs/src/mdbook/src/SUMMARY.md @@ -11,44 +11,39 @@ - [fake-nix](./build/fake-nix.md) - [Build dataplane](./build/just-cargo-build.md) - [Sterile builds](./build/sterile-build.md) - - [Design session](./dataplane/design-session.md) - - [Development Plan](./dataplane/development-plan.md) - - [Control plane dev-env](dataplane/tasks/control-plane-dev-env.md) - - [Create control plane image](dataplane/tasks/create-control-plane-image.md) - - [FRR Plugin](dataplane/tasks/frr-plugin.md) - - [Dataplane / Control plane transport](dataplane/tasks/dataplane-control-plane-transport.md) - - [Dataplane / Control plane protocol](dataplane/tasks/dataplane-control-plane-protocol.md) - - [Dataplane / Control plane reconcile](dataplane/tasks/dataplane-control-plane-reconcile.md) - - [Gateway test env](dataplane/tasks/gateway-test-env.md) - - [Identify local traffic](dataplane/tasks/identify-local-traffic.md) - - [Configuration Persistence Investigation](dataplane/tasks/configuration-persistence-investigation.md) - - [Route manager](dataplane/tasks/route-manager.md) - - [Dataplane worker lifecycle](dataplane/tasks/dataplane-worker-lifecycle.md) - - [Telemetry (investigation)](dataplane/tasks/telemetry-investigation.md) - - [Telemetry (basic)](dataplane/tasks/telemetry-basic.md) - - [Telemetry (integration)](dataplane/tasks/telemetry-integration.md) - - [Configuration database schema](dataplane/tasks/config-db-schema.md) - - [Management plane - dataplane interaction](dataplane/tasks/management-plane-dataplane-interaction.md) - - [VXLAN tunnels](dataplane/tasks/vxlan-tunnels.md) - - - [Underlay routing](dataplane/tasks/underlay-routing.md) - - [Management plane - dataplane interaction](dataplane/tasks/management-plane-dataplane-interaction.md) - - [Management plane - control plane interaction](dataplane/tasks/management-plane-control-plane-interaction.md) - - [VPC routing](dataplane/tasks/vpc-routing.md) - - [Rate limiting investigation](dataplane/tasks/rate-limiting-investigation.md) - - [VPC rate-limiting](dataplane/tasks/vpc-rate-limiting.md) - - [NAT44](dataplane/tasks/NAT44.md) - - [NAT66](dataplane/tasks/NAT66.md) - - [NAT64 (investigation)](dataplane/tasks/NAT64-investigation.md) - - [NAT64](dataplane/tasks/NAT64.md) - - [State sync (design)](dataplane/tasks/state-sync-design.md) - - [State sync (implementation)](dataplane/tasks/state-sync.md) - - [Public internet access](dataplane/tasks/public-internet-access.md) - - [Fault tolerance (implementation)](dataplane/tasks/fault-tolerance-implementation.md) - - [Fault tolerance (validation)](dataplane/tasks/fault-tolerance-validation.md) - - [Performance measurement](dataplane/tasks/performance-measurement.md) - - [Core pinning](dataplane/tasks/core-pinning.md) - - [One control plane daemon per container](dataplane/tasks/one-control-plane-daemon-per-container.md) - - [Programmatic Control of FRR](dataplane/tasks/programmatic-control-of-frr.md) + - [Configuration Persistence Investigation](./dataplane/tasks/configuration-persistence-investigation.md) + - [Configuration database schema](./dataplane/tasks/config-db-schema.md) + - [Control plane dev-env](./dataplane/tasks/control-plane-dev-env.md) + - [Core pinning](./dataplane/tasks/core-pinning.md) + - [Create control plane image](./dataplane/tasks/create-control-plane-image.md) + - [Dataplane / Control plane protocol](./dataplane/tasks/dataplane-control-plane-protocol.md) + - [Dataplane / Control plane transport](./dataplane/tasks/dataplane-control-plane-transport.md) + - [Dataplane worker lifecycle](./dataplane/tasks/dataplane-worker-lifecycle.md) + - [Fault tolerance (implementation)](./dataplane/tasks/fault-tolerance-implementation.md) + - [Fault tolerance (validation)](./dataplane/tasks/fault-tolerance-validation.md) + - [Gateway test env](./dataplane/tasks/gateway-test-env.md) + - [Identify local traffic](./dataplane/tasks/identify-local-traffic.md) + - [Management plane - control plane interaction](./dataplane/tasks/management-plane-control-plane-interaction.md) + - [Management plane - dataplane interaction](./dataplane/tasks/management-plane-dataplane-interaction.md) + - [NAT44](./dataplane/tasks/NAT44.md) + - [NAT64 (investigation)](./dataplane/tasks/NAT64-investigation.md) + - [NAT64](./dataplane/tasks/NAT64.md) + - [NAT66](./dataplane/tasks/NAT66.md) + - [One control plane daemon per container](./dataplane/tasks/one-control-plane-daemon-per-container.md) + - [Performance measurement](./dataplane/tasks/performance-measurement.md) + - [Programmatic Control of FRR](./dataplane/tasks/programmatic-control-of-frr.md) + - [Public internet access](./dataplane/tasks/public-internet-access.md) + - [Rate limiting investigation](./dataplane/tasks/rate-limiting-investigation.md) + - [Route manager](./dataplane/tasks/route-manager.md) + - [State sync (design)](./dataplane/tasks/state-sync-design.md) + - [State sync (implementation)](./dataplane/tasks/state-sync.md) + - [Telemetry (basic)](./dataplane/tasks/telemetry-basic.md) + - [Telemetry (integration)](./dataplane/tasks/telemetry-integration.md) + - [Telemetry (investigation)](./dataplane/tasks/telemetry-investigation.md) + - [Underlay routing](./dataplane/tasks/underlay-routing.md) + - [VPC rate-limiting](./dataplane/tasks/vpc-rate-limiting.md) + - [VPC routing](./dataplane/tasks/vpc-routing.md) + - [VXLAN tunnels](./dataplane/tasks/vxlan-tunnels.md) + - [Zebra Plugin](./dataplane/tasks/zebra-plugin.md) diff --git a/design-docs/src/mdbook/src/dataplane/design-session.md b/design-docs/src/mdbook/src/dataplane/design-session.md index 1e059775..1cc5a352 100644 --- a/design-docs/src/mdbook/src/dataplane/design-session.md +++ b/design-docs/src/mdbook/src/dataplane/design-session.md @@ -14,7 +14,7 @@ At a very high level, these are the _user facing_ features that we require to re ## User-facing features -
+
```plantuml @startdot @@ -149,7 +149,7 @@ skinparam linetype ortho "frr_agent": { "text": "FRR agent", "url": "#frr-agent" }, "zebra": { "text": "zebra", "url": "https://docs.frrouting.org/en/latest/zebra.html" }, "routing_daemons": { "text": "routing daemons", "url": "#routing-daemons" }, - "hh_plugin": { "text": "Hedgehog\\nplugin", "url": "#hedgehog-plugin" }, + "zebra_plugin": { "text": "Zebra\\nplugin", "url": "#zebra-plugin" }, "kernel": { "text": "kernel", "url": "https://en.wikipedia.org/wiki/Linux_kernel" }, "interface_manager": { "text": "interface manager", "url": "#interface-manager" }, "routing_manager": { "text": "routing manager", "url": "#routing-manager" }, @@ -186,7 +186,7 @@ $r(kernel) $r(control_plane) { $r(routing_daemons) $r(zebra) { - $r(hh_plugin) + $r(zebra_plugin) } $r(frr_agent) } @@ -210,7 +210,7 @@ rectangle sister_dataplane as "sister dataplane" { rectangle nics control_plane_interface -- dataplane_model -dataplane_workers <--> nics : dpdk +dataplane_workers <--> nics : [[ https://www.dpdk.org/ dpdk ]] frr_agent <--> routing_daemons frr_agent <--> zebra gateway_agent -- frr_agent @@ -223,7 +223,7 @@ dataplane_model <--> nat_manager dataplane_model <--> routing_manager management_plane_interface -- dataplane_model nat_manager <--> dataplane_workers -hh_plugin --- control_plane_interface : [[ https://en.wikipedia.org/wiki/Unix_domain_socket unix socket ]] +zebra_plugin --- control_plane_interface : [[ https://en.wikipedia.org/wiki/Unix_domain_socket unix socket ]] routing_daemons <-> zebra routing_manager <--> dataplane_workers state_sync <-> sister_state_sync : [[ https://en.wikipedia.org/wiki/Remote_direct_memory_access rdma]] @@ -284,11 +284,11 @@ Be afraid. Make Fredi fill in this section. But also be afraid.
-### Hedgehog Plugin +### Zebra Plugin This is a planned [zebra] plugin in the same spirit as [`fpm`](https://docs.frrouting.org/projects/dev-guide/en/latest/fpm.html#id1) or [`dataplane_fpm_nl`](https://docs.frrouting.org/projects/dev-guide/en/latest/fpm.html#dplane-fpm-nl). -The core idea is to have a plugin that can be dynamically loaded into `zebra` and will listen to the `zebra` event stream for updates. +The core idea is to have a plugin that can be dynamically loaded into [zebra] and will listen to the [zebra event stream](https://github.com/FRRouting/frr/blob/ee5a3456d34a756c70ad8856ab7be7bed75ee31c/zebra/zebra_dplane.h#L114-L217) for updates. The plugin will then take those updates and push them into the dataplane agent, allowing the dataplane to react to route updates.
diff --git a/design-docs/src/mdbook/src/dataplane/development-plan.md b/design-docs/src/mdbook/src/dataplane/development-plan.md index e6c36d01..d43ec74e 100644 --- a/design-docs/src/mdbook/src/dataplane/development-plan.md +++ b/design-docs/src/mdbook/src/dataplane/development-plan.md @@ -28,14 +28,13 @@ digraph g { cp_api_control_investigation [label=<programmatic control of frr
(investigation)
>, $urgent, href="$ptr/programmatic-control-of-frr.html", fontcolor=blue] cp_dev_env [label="control plane\ndev env", href="$ptr/control-plane-dev-env.html", fontcolor=blue] cp_image_creation [ label="Create control plane container image", href="$ptr/create-control-plane-image.html", fontcolor=blue] - dp_cp_reconciliation [ label="dp/cp reconcile", href="$ptr/dataplane-control-plane-reconcile.html", fontcolor=blue ] dp_dev_env [label="dataplane dev env", $completed, href="../../build/index.html", fontcolor=blue] dp_dp_state_sync [label="state sync\n(implementation)", $difficult, href="$ptr/state-sync.html", fontcolor=blue] dp_dp_state_sync_design [label="state sync\n(design)", $urgent, href="$ptr/state-sync-design.html", fontcolor=blue] dp_image_creation [label="dataplane image build", $completed] fault_tolerance [label="fault tolerance (implementation)", href="$ptr/fault-tolerance-implementation.html", fontcolor=blue] fault_tolerance_proof [label="fault tolerance (validation)", $difficult, href="$ptr/fault-tolerance-validation.html", fontcolor=blue] - frr_plugin_basic [ label="frr plugin\n(basic)", href="$ptr/frr-plugin.html", fontcolor=blue ] + zebra_plugin_basic [ label="zebra plugin\n(basic)", href="$ptr/zebra-plugin.html", fontcolor=blue ] frr_programmatic_control [label=<programmatic
control of frr
>, $difficult, href="$ptr/programmatic-control-of-frr.html", fontcolor=blue] gw_test_env [label="gateway test env", href="$ptr/gateway-test-env.html", fontcolor=blue] investigate_config_persist [ label=<configuration
persistence
(investigation)
>, $urgent, href="$ptr/configuration-persistence-investigation.html", fontcolor=blue ] @@ -44,8 +43,8 @@ digraph g { mp_dp_interaction [ label="management plane \ndataplane interaction", href="$ptr/management-plane-dataplane-interaction.html", fontcolor=blue] nat64_investigation [label=<NAT64 investigation>, $urgent, href="$ptr/NAT64-investigation.html", fontcolor=blue] performance_measurement [ label="measure performance", href="$ptr/performance-measurement.html", fontcolor=blue] - plugin_dp_proto [ label="plugin/dp protocol", $started, href="$ptr/dataplane-control-plane-protocol.html", fontcolor=blue] - plugin_dp_transport [ label="plugin/dp transport", $completed, href="$ptr/dataplane-control-plane-transport.html", fontcolor=blue] + plugin_dp_proto [ label="plugin/dataplane protocol", $started, href="$ptr/dataplane-control-plane-protocol.html", fontcolor=blue] + plugin_dp_transport [ label="plugin/dataplane transport", $completed, href="$ptr/dataplane-control-plane-transport.html", fontcolor=blue] public_internet_access [label="public internet access", href="$ptr/public-internet-access.html", fontcolor=blue] rate_limiting_investigation [label="rate limiting investigation", $completed] routing_manager [label="routing manager", href="$ptr/route-manager.html", fontcolor=blue] @@ -92,17 +91,16 @@ digraph g { cp_dev_env -> gw_test_env cp_image_creation -> cp_dev_env cp_image_creation -> separate_cp_containers - dp_cp_reconciliation -> frr_plugin_basic dp_dev_env -> gw_test_env dp_image_creation -> dp_dev_env - gw_test_env -> frr_plugin_basic - frr_plugin_basic -> routing_manager + gw_test_env -> zebra_plugin_basic + zebra_plugin_basic -> routing_manager config_db_schema -> mp_cp_interaction config_db_schema -> mp_dp_interaction - local_traffic_ident -> frr_plugin_basic + local_traffic_ident -> zebra_plugin_basic mp_dp_interaction -> vpc_routing - plugin_dp_proto -> dp_cp_reconciliation - plugin_dp_transport -> dp_cp_reconciliation + plugin_dp_proto -> zebra_plugin_basic + plugin_dp_transport -> zebra_plugin_basic routing_manager -> underlay_routing config_db_schema -> underlay_routing vpc_routing -> vpc_nat44 @@ -128,12 +126,13 @@ digraph g { } @enddot ``` +
> Graph of the engineering development plan. > Each node on the graph represents a task or required function. > No task can be _completed_ without all the other tasks which point to it. -> +> > * Tasks shown in orange are points of higher uncertainty and risk. > * Tasks shown in pink are points of expected higher difficulty. > * Tasks shown in gray are already completed. @@ -146,3 +145,5 @@ digraph g { > [!WARNING] > Tasks of high expected difficulty are different from tasks which we expect will be very time-consuming. + +{{#include ../links.md}} diff --git a/design-docs/src/mdbook/src/dataplane/map-of-dpdk.md b/design-docs/src/mdbook/src/dataplane/map-of-dpdk.md index 598cb8fb..48182b36 100644 --- a/design-docs/src/mdbook/src/dataplane/map-of-dpdk.md +++ b/design-docs/src/mdbook/src/dataplane/map-of-dpdk.md @@ -26,15 +26,15 @@ hide circle !endfunction !$doc_links = { - "flow_item": "https://doc.dpdk.org/guides-24.07/prog_guide/rte_flow.html#pattern-item", - "flow_item_template": "https://doc.dpdk.org/guides-24.07/prog_guide/rte_flow.html#pattern-templates", - "flow_action": "https://doc.dpdk.org/guides-24.07/prog_guide/rte_flow.html#actions", - "flow_action_template": "https://doc.dpdk.org/guides-24.07/prog_guide/rte_flow.html#actions-templates", - "flow_action_indirect": "https://doc.dpdk.org/guides-24.07/prog_guide/rte_flow.html#action-indirect", - "flow_action_indirect_list": "https://doc.dpdk.org/guides-24.07/prog_guide/rte_flow.html#action-indirect-list", - "switch_domain": "https://doc.dpdk.org/guides-24.07/prog_guide/switch_representation.html", - "flow_table": "https://doc.dpdk.org/guides-24.07/prog_guide/rte_flow.html#attribute-group", - "flow": "https://doc.dpdk.org/guides-24.07/prog_guide/rte_flow.html", + "flow_item": "https://doc.dpdk.org/guides-24.11/prog_guide/ethdev/flow_offload.html#pattern-item", + "flow_item_template": "https://doc.dpdk.org/guides-24.11/prog_guide/ethdev/flow_offload.html#pattern-templates", + "flow_action": "https://doc.dpdk.org/guides-24.11/prog_guide/ethdev/flow_offload.html#actions", + "flow_action_template": "https://doc.dpdk.org/guides-24.11/prog_guide/ethdev/flow_offload.html#actions-templates", + "flow_action_indirect": "https://doc.dpdk.org/guides-24.11/prog_guide/ethdev/flow_offload.html#action-indirect", + "flow_action_indirect_list": "https://doc.dpdk.org/guides-24.11/prog_guide/ethdev/flow_offload.html#action-indirect-list", + "switch_domain": "https://doc.dpdk.org/guides-24.11/prog_guide/ethdev/switch_representation.html", + "flow_table": "https://doc.dpdk.org/guides-24.11/prog_guide/ethdev/flow_offload.html#attribute-group", + "flow": "https://doc.dpdk.org/guides-24.11/prog_guide/ethdev/flow_offload.html", "hairpin_queue": "https://inbox.dpdk.org/dev/1565703468-55617-1-git-send-email-orika@mellanox.com/", "membuf": "https://doc.dpdk.org/guides/prog_guide/mbuf_lib.html", "mempool": "https://doc.dpdk.org/guides/prog_guide/mempool_lib.html", diff --git a/design-docs/src/mdbook/src/dataplane/tasks/configuration-persistence-investigation.md b/design-docs/src/mdbook/src/dataplane/tasks/configuration-persistence-investigation.md index 1ca5d887..46e4ceb5 100644 --- a/design-docs/src/mdbook/src/dataplane/tasks/configuration-persistence-investigation.md +++ b/design-docs/src/mdbook/src/dataplane/tasks/configuration-persistence-investigation.md @@ -12,7 +12,7 @@ Rather, this store needs to hold configuration data which is ## etcd -[`etcd`] is a reasonable choice because +[etcd] is a reasonable choice because 1. It is already in use in kubernetes and is therefore likely to be well-maintained and tested. 2. we are already using / integrating with kubernetes so any flaws in `etcd` are likely to impact us anyway. diff --git a/design-docs/src/mdbook/src/dataplane/tasks/control-plane-dev-env.md b/design-docs/src/mdbook/src/dataplane/tasks/control-plane-dev-env.md index ebc87385..2b8c569b 100644 --- a/design-docs/src/mdbook/src/dataplane/tasks/control-plane-dev-env.md +++ b/design-docs/src/mdbook/src/dataplane/tasks/control-plane-dev-env.md @@ -1,6 +1,6 @@ # Control plane dev-env -Create and document a development environment for the [`zebra`] [hedgehog plugin]. +Create and document a development environment for the [zebra] [hedgehog plugin]. Requirements: @@ -9,7 +9,6 @@ Requirements: - **REQUIRE**: CI runs tests in dev-env container or, - **IDEALLY**: tests run in a more minimal test-env container. -```yaml issue-meta -assign: - - @Fredi Raspall -``` +## Likely dispatch + +- [@Fredi-raspall] diff --git a/design-docs/src/mdbook/src/dataplane/tasks/core-pinning.md b/design-docs/src/mdbook/src/dataplane/tasks/core-pinning.md index ff981e1d..a8eaa4c8 100644 --- a/design-docs/src/mdbook/src/dataplane/tasks/core-pinning.md +++ b/design-docs/src/mdbook/src/dataplane/tasks/core-pinning.md @@ -1,4 +1,4 @@ # Core pinning > [!NOTE] -> I think we can punt on this till the last minute! +> I think we can punt on this until the last minute! diff --git a/design-docs/src/mdbook/src/dataplane/tasks/create-control-plane-image.md b/design-docs/src/mdbook/src/dataplane/tasks/create-control-plane-image.md index 3a92d3bd..2951d775 100644 --- a/design-docs/src/mdbook/src/dataplane/tasks/create-control-plane-image.md +++ b/design-docs/src/mdbook/src/dataplane/tasks/create-control-plane-image.md @@ -16,10 +16,9 @@ We need to generate a docker image to run our control plane. Both [@Fredi-raspall] and [@daniel-noland] have made some progress on this task and should sync up to get it over the line. -```yaml issue-meta -assign: - - @Fredi Raspall -``` +## Likely dispatch + +- [@Fredi-raspall] [Lua scripting]: https://docs.frrouting.org/en/latest/scripting.html diff --git a/design-docs/src/mdbook/src/dataplane/tasks/dataplane-control-plane-transport.md b/design-docs/src/mdbook/src/dataplane/tasks/dataplane-control-plane-transport.md index 9fcd97d1..bcf14741 100644 --- a/design-docs/src/mdbook/src/dataplane/tasks/dataplane-control-plane-transport.md +++ b/design-docs/src/mdbook/src/dataplane/tasks/dataplane-control-plane-transport.md @@ -2,7 +2,7 @@ It seems like we have all agreed on [unix domain sockets]. -## Likely assignment +## Likely dispatch * [@Fredi-raspall] * coordinate with: [@daniel-noland] diff --git a/design-docs/src/mdbook/src/dataplane/tasks/frr-plugin.md b/design-docs/src/mdbook/src/dataplane/tasks/frr-plugin.md deleted file mode 100644 index 3fa9e3ed..00000000 --- a/design-docs/src/mdbook/src/dataplane/tasks/frr-plugin.md +++ /dev/null @@ -1,5 +0,0 @@ -# FRR Plugin (basic) - -Just a feature marker for MVP FRR [plugin][hedgehog plugin]. - -{{#include ../../links.md}} diff --git a/design-docs/src/mdbook/src/dataplane/tasks/pick-a-datastore.md b/design-docs/src/mdbook/src/dataplane/tasks/pick-a-datastore.md new file mode 100644 index 00000000..6727c5e9 --- /dev/null +++ b/design-docs/src/mdbook/src/dataplane/tasks/pick-a-datastore.md @@ -0,0 +1,32 @@ +# Pick a data store + +We need to officially pick a data store for configuration information. + +This data store _is not_ intended for storing "fast" state. +Rather, this store needs to hold configuration data which is + +1. durable +2. atomic +3. strongly typed +4. immediately consistent + +[`etcd`] is a reasonable choice because + +1. It is already in use in kubernetes and is therefore likely to be well-maintained and tested. +2. we are already using / integrating with kubernetes so any flaws in `etcd` are likely to impact us anyway. + +I have used [`zookeeper`](https://zookeeper.apache.org/) in the past and *strongly recommend against it*. + +I would also consider [`consul`](https://github.com/hashicorp/consul) but [the license](https://github.com/hashicorp/consul/blob/main/LICENSE) is *_not_* acceptable. + +A newer entry in the space is [`nacos`](https://github.com/alibaba/nacos) but I think it is less well suited since it only seems to support eventual consistency. + +The remaining option I know of is [`rqlite`]. _I have not used it,_ but it seems to be a reasonable option. + +- has a supported [rust client](https://github.com/tomvoet/rqlite-rs) (and even a [sqlx](https://github.com/launchbadge/sqlx) client in the form of [sqlx-rqlite](https://crates.io/crates/sqlx-rqlite)) +- [weak](https://rqlite.io/docs/api/read-consistency/#weak), [linearizable](https://rqlite.io/docs/api/read-consistency/#linearizable), and [strong](https://rqlite.io/docs/api/read-consistency/#strong) consistency models supported +- [transactions](https://rqlite.io/docs/api/api/#transactions) (this seems less than ideal tho) + +Thus, I think the real choice is between [`etcd`] and [`rqlite`]. + +That choice comes down to how much we value the functionality of sqlite (multiple indexes, referential integrity, strong schema) vs. the upsides of [etcd] (watches, battle tested, and more widely used). diff --git a/design-docs/src/mdbook/src/dataplane/tasks/vpc-rate-limiting.md b/design-docs/src/mdbook/src/dataplane/tasks/vpc-rate-limiting.md index e846fc55..ebdf647b 100644 --- a/design-docs/src/mdbook/src/dataplane/tasks/vpc-rate-limiting.md +++ b/design-docs/src/mdbook/src/dataplane/tasks/vpc-rate-limiting.md @@ -2,5 +2,7 @@ Just rate limiting! -Explicitly not full QoS for the moment. -If we involve QoS in the MVP then we will have zero chance on this timeline. +Explicitly not full [QoS] for the moment. +If we involve [QoS] in the MVP then we will have zero chance on this timeline. + +{{#include ../../links.md}} diff --git a/design-docs/src/mdbook/src/dataplane/tasks/zebra-plugin.md b/design-docs/src/mdbook/src/dataplane/tasks/zebra-plugin.md new file mode 100644 index 00000000..92866b77 --- /dev/null +++ b/design-docs/src/mdbook/src/dataplane/tasks/zebra-plugin.md @@ -0,0 +1,24 @@ +# Zebra Plugin (basic) + +The dataplane and control plane need to communicate with each other regarding + +1. Full routing tables (for [state sync]) +2. route updates (i.e. differential updates) +3. route offloading status (including failures) +4. Address assignments, to ensure the dataplane can configure [local delivery](./identify-local-traffic.md) + +Keep in mind that route tables are, in general, notably more complex than a naive LPM trie, and may include like: + +1. [ECMP]/WCMP +2. [encapsulation rules](https://www.man7.org/linux/man-pages/man8/ip-route.8.html), +3. [nexthop groups](https://man7.org/linux/man-pages/man8/ip-nexthop.8.html), +4. multicast routes (this is unlikely to be important in the near term). + +We only expect to support basic IPv4 and IPv6 LPM routes in the near term, but feature evolution should be accounted for in the design. + +## Likely dispatch + +* [@Fredi-raspall] +* coordinate with: [@daniel-noland] + +{{#include ../../links.md}} diff --git a/design-docs/src/mdbook/src/dataplane/tasks2/NAT44.md b/design-docs/src/mdbook/src/dataplane/tasks2/NAT44.md new file mode 100644 index 00000000..290d06fa --- /dev/null +++ b/design-docs/src/mdbook/src/dataplane/tasks2/NAT44.md @@ -0,0 +1 @@ +# NAT44 diff --git a/design-docs/src/mdbook/src/dataplane/tasks2/NAT64-investigation.md b/design-docs/src/mdbook/src/dataplane/tasks2/NAT64-investigation.md new file mode 100644 index 00000000..80bcd2b7 --- /dev/null +++ b/design-docs/src/mdbook/src/dataplane/tasks2/NAT64-investigation.md @@ -0,0 +1 @@ +# NAT64 (investigation) diff --git a/design-docs/src/mdbook/src/dataplane/tasks2/NAT64.md b/design-docs/src/mdbook/src/dataplane/tasks2/NAT64.md new file mode 100644 index 00000000..1e6f90fd --- /dev/null +++ b/design-docs/src/mdbook/src/dataplane/tasks2/NAT64.md @@ -0,0 +1 @@ +# NAT64 diff --git a/design-docs/src/mdbook/src/dataplane/tasks2/NAT66.md b/design-docs/src/mdbook/src/dataplane/tasks2/NAT66.md new file mode 100644 index 00000000..d8689d08 --- /dev/null +++ b/design-docs/src/mdbook/src/dataplane/tasks2/NAT66.md @@ -0,0 +1 @@ +# NAT66 diff --git a/design-docs/src/mdbook/src/dataplane/tasks2/config-db-schema.md b/design-docs/src/mdbook/src/dataplane/tasks2/config-db-schema.md new file mode 100644 index 00000000..97a62115 --- /dev/null +++ b/design-docs/src/mdbook/src/dataplane/tasks2/config-db-schema.md @@ -0,0 +1 @@ +# Configuration database schema diff --git a/design-docs/src/mdbook/src/dataplane/tasks2/configuration-persistence-investigation.md b/design-docs/src/mdbook/src/dataplane/tasks2/configuration-persistence-investigation.md new file mode 100644 index 00000000..82450e4b --- /dev/null +++ b/design-docs/src/mdbook/src/dataplane/tasks2/configuration-persistence-investigation.md @@ -0,0 +1 @@ +# Configuration Persistence Investigation diff --git a/design-docs/src/mdbook/src/dataplane/tasks2/control-plane-dev-env.md b/design-docs/src/mdbook/src/dataplane/tasks2/control-plane-dev-env.md new file mode 100644 index 00000000..2dd156f2 --- /dev/null +++ b/design-docs/src/mdbook/src/dataplane/tasks2/control-plane-dev-env.md @@ -0,0 +1 @@ +# Control plane dev-env diff --git a/design-docs/src/mdbook/src/dataplane/tasks2/core-pinning.md b/design-docs/src/mdbook/src/dataplane/tasks2/core-pinning.md new file mode 100644 index 00000000..3e1ee591 --- /dev/null +++ b/design-docs/src/mdbook/src/dataplane/tasks2/core-pinning.md @@ -0,0 +1 @@ +# Core pinning diff --git a/design-docs/src/mdbook/src/dataplane/tasks2/create-control-plane-image.md b/design-docs/src/mdbook/src/dataplane/tasks2/create-control-plane-image.md new file mode 100644 index 00000000..cfa58297 --- /dev/null +++ b/design-docs/src/mdbook/src/dataplane/tasks2/create-control-plane-image.md @@ -0,0 +1 @@ +# Create control plane image diff --git a/design-docs/src/mdbook/src/dataplane/tasks2/dataplane-control-plane-protocol.md b/design-docs/src/mdbook/src/dataplane/tasks2/dataplane-control-plane-protocol.md new file mode 100644 index 00000000..b667c634 --- /dev/null +++ b/design-docs/src/mdbook/src/dataplane/tasks2/dataplane-control-plane-protocol.md @@ -0,0 +1 @@ +# Dataplane / Control plane protocol diff --git a/design-docs/src/mdbook/src/dataplane/tasks2/dataplane-control-plane-transport.md b/design-docs/src/mdbook/src/dataplane/tasks2/dataplane-control-plane-transport.md new file mode 100644 index 00000000..cd9b2be0 --- /dev/null +++ b/design-docs/src/mdbook/src/dataplane/tasks2/dataplane-control-plane-transport.md @@ -0,0 +1 @@ +# Dataplane / Control plane transport diff --git a/design-docs/src/mdbook/src/dataplane/tasks2/dataplane-worker-lifecycle.md b/design-docs/src/mdbook/src/dataplane/tasks2/dataplane-worker-lifecycle.md new file mode 100644 index 00000000..b58cf0c8 --- /dev/null +++ b/design-docs/src/mdbook/src/dataplane/tasks2/dataplane-worker-lifecycle.md @@ -0,0 +1 @@ +# Dataplane worker lifecycle diff --git a/design-docs/src/mdbook/src/dataplane/tasks2/fault-tolerance-implementation.md b/design-docs/src/mdbook/src/dataplane/tasks2/fault-tolerance-implementation.md new file mode 100644 index 00000000..b622363a --- /dev/null +++ b/design-docs/src/mdbook/src/dataplane/tasks2/fault-tolerance-implementation.md @@ -0,0 +1 @@ +# Fault tolerance (implementation) diff --git a/design-docs/src/mdbook/src/dataplane/tasks2/fault-tolerance-validation.md b/design-docs/src/mdbook/src/dataplane/tasks2/fault-tolerance-validation.md new file mode 100644 index 00000000..9d179316 --- /dev/null +++ b/design-docs/src/mdbook/src/dataplane/tasks2/fault-tolerance-validation.md @@ -0,0 +1 @@ +# Fault tolerance (validation) diff --git a/design-docs/src/mdbook/src/dataplane/tasks2/gateway-test-env.md b/design-docs/src/mdbook/src/dataplane/tasks2/gateway-test-env.md new file mode 100644 index 00000000..429844d3 --- /dev/null +++ b/design-docs/src/mdbook/src/dataplane/tasks2/gateway-test-env.md @@ -0,0 +1 @@ +# Gateway test env diff --git a/design-docs/src/mdbook/src/dataplane/tasks2/identify-local-traffic.md b/design-docs/src/mdbook/src/dataplane/tasks2/identify-local-traffic.md new file mode 100644 index 00000000..f679635c --- /dev/null +++ b/design-docs/src/mdbook/src/dataplane/tasks2/identify-local-traffic.md @@ -0,0 +1 @@ +# Identify local traffic diff --git a/design-docs/src/mdbook/src/dataplane/tasks2/management-plane-control-plane-interaction.md b/design-docs/src/mdbook/src/dataplane/tasks2/management-plane-control-plane-interaction.md new file mode 100644 index 00000000..2e78ef56 --- /dev/null +++ b/design-docs/src/mdbook/src/dataplane/tasks2/management-plane-control-plane-interaction.md @@ -0,0 +1 @@ +# Management plane - control plane interaction diff --git a/design-docs/src/mdbook/src/dataplane/tasks2/management-plane-dataplane-interaction.md b/design-docs/src/mdbook/src/dataplane/tasks2/management-plane-dataplane-interaction.md new file mode 100644 index 00000000..55a3d054 --- /dev/null +++ b/design-docs/src/mdbook/src/dataplane/tasks2/management-plane-dataplane-interaction.md @@ -0,0 +1 @@ +# Management plane - dataplane interaction diff --git a/design-docs/src/mdbook/src/dataplane/tasks2/one-control-plane-daemon-per-container.md b/design-docs/src/mdbook/src/dataplane/tasks2/one-control-plane-daemon-per-container.md new file mode 100644 index 00000000..b17e69d1 --- /dev/null +++ b/design-docs/src/mdbook/src/dataplane/tasks2/one-control-plane-daemon-per-container.md @@ -0,0 +1 @@ +# One control plane daemon per container diff --git a/design-docs/src/mdbook/src/dataplane/tasks2/performance-measurement.md b/design-docs/src/mdbook/src/dataplane/tasks2/performance-measurement.md new file mode 100644 index 00000000..a1cc571d --- /dev/null +++ b/design-docs/src/mdbook/src/dataplane/tasks2/performance-measurement.md @@ -0,0 +1 @@ +# Performance measurement diff --git a/design-docs/src/mdbook/src/dataplane/tasks2/programmatic-control-of-frr.md b/design-docs/src/mdbook/src/dataplane/tasks2/programmatic-control-of-frr.md new file mode 100644 index 00000000..476a2519 --- /dev/null +++ b/design-docs/src/mdbook/src/dataplane/tasks2/programmatic-control-of-frr.md @@ -0,0 +1 @@ +# Programmatic Control of FRR diff --git a/design-docs/src/mdbook/src/dataplane/tasks2/public-internet-access.md b/design-docs/src/mdbook/src/dataplane/tasks2/public-internet-access.md new file mode 100644 index 00000000..bbf78ff2 --- /dev/null +++ b/design-docs/src/mdbook/src/dataplane/tasks2/public-internet-access.md @@ -0,0 +1 @@ +# Public internet access diff --git a/design-docs/src/mdbook/src/dataplane/tasks2/rate-limiting-investigation.md b/design-docs/src/mdbook/src/dataplane/tasks2/rate-limiting-investigation.md new file mode 100644 index 00000000..adb4d38e --- /dev/null +++ b/design-docs/src/mdbook/src/dataplane/tasks2/rate-limiting-investigation.md @@ -0,0 +1 @@ +# Rate limiting investigation diff --git a/design-docs/src/mdbook/src/dataplane/tasks2/route-manager.md b/design-docs/src/mdbook/src/dataplane/tasks2/route-manager.md new file mode 100644 index 00000000..2b46a464 --- /dev/null +++ b/design-docs/src/mdbook/src/dataplane/tasks2/route-manager.md @@ -0,0 +1 @@ +# Route manager diff --git a/design-docs/src/mdbook/src/dataplane/tasks2/state-sync-design.md b/design-docs/src/mdbook/src/dataplane/tasks2/state-sync-design.md new file mode 100644 index 00000000..94a76488 --- /dev/null +++ b/design-docs/src/mdbook/src/dataplane/tasks2/state-sync-design.md @@ -0,0 +1 @@ +# State sync (design) diff --git a/design-docs/src/mdbook/src/dataplane/tasks2/state-sync.md b/design-docs/src/mdbook/src/dataplane/tasks2/state-sync.md new file mode 100644 index 00000000..1a1d4fd5 --- /dev/null +++ b/design-docs/src/mdbook/src/dataplane/tasks2/state-sync.md @@ -0,0 +1 @@ +# State sync (implementation) diff --git a/design-docs/src/mdbook/src/dataplane/tasks2/telemetry-basic.md b/design-docs/src/mdbook/src/dataplane/tasks2/telemetry-basic.md new file mode 100644 index 00000000..f8bed689 --- /dev/null +++ b/design-docs/src/mdbook/src/dataplane/tasks2/telemetry-basic.md @@ -0,0 +1 @@ +# Telemetry (basic) diff --git a/design-docs/src/mdbook/src/dataplane/tasks2/telemetry-integration.md b/design-docs/src/mdbook/src/dataplane/tasks2/telemetry-integration.md new file mode 100644 index 00000000..04c98a13 --- /dev/null +++ b/design-docs/src/mdbook/src/dataplane/tasks2/telemetry-integration.md @@ -0,0 +1 @@ +# Telemetry (integration) diff --git a/design-docs/src/mdbook/src/dataplane/tasks2/telemetry-investigation.md b/design-docs/src/mdbook/src/dataplane/tasks2/telemetry-investigation.md new file mode 100644 index 00000000..bbcfda10 --- /dev/null +++ b/design-docs/src/mdbook/src/dataplane/tasks2/telemetry-investigation.md @@ -0,0 +1 @@ +# Telemetry (investigation) diff --git a/design-docs/src/mdbook/src/dataplane/tasks2/underlay-routing.md b/design-docs/src/mdbook/src/dataplane/tasks2/underlay-routing.md new file mode 100644 index 00000000..a8fd2643 --- /dev/null +++ b/design-docs/src/mdbook/src/dataplane/tasks2/underlay-routing.md @@ -0,0 +1 @@ +# Underlay routing diff --git a/design-docs/src/mdbook/src/dataplane/tasks2/vpc-rate-limiting.md b/design-docs/src/mdbook/src/dataplane/tasks2/vpc-rate-limiting.md new file mode 100644 index 00000000..68ac8e50 --- /dev/null +++ b/design-docs/src/mdbook/src/dataplane/tasks2/vpc-rate-limiting.md @@ -0,0 +1 @@ +# VPC rate-limiting diff --git a/design-docs/src/mdbook/src/dataplane/tasks2/vpc-routing.md b/design-docs/src/mdbook/src/dataplane/tasks2/vpc-routing.md new file mode 100644 index 00000000..27dd3506 --- /dev/null +++ b/design-docs/src/mdbook/src/dataplane/tasks2/vpc-routing.md @@ -0,0 +1 @@ +# VPC routing diff --git a/design-docs/src/mdbook/src/dataplane/tasks2/vxlan-tunnels.md b/design-docs/src/mdbook/src/dataplane/tasks2/vxlan-tunnels.md new file mode 100644 index 00000000..4009af1d --- /dev/null +++ b/design-docs/src/mdbook/src/dataplane/tasks2/vxlan-tunnels.md @@ -0,0 +1 @@ +# VXLAN tunnels diff --git a/design-docs/src/mdbook/src/dataplane/tasks2/zebra-plugin.md b/design-docs/src/mdbook/src/dataplane/tasks2/zebra-plugin.md new file mode 100644 index 00000000..f6022e76 --- /dev/null +++ b/design-docs/src/mdbook/src/dataplane/tasks2/zebra-plugin.md @@ -0,0 +1 @@ +# Zebra Plugin diff --git a/design-docs/src/mdbook/src/links.md b/design-docs/src/mdbook/src/links.md index 67e17a9b..076cb541 100644 --- a/design-docs/src/mdbook/src/links.md +++ b/design-docs/src/mdbook/src/links.md @@ -24,6 +24,7 @@ [LACP]: https://en.wikipedia.org/wiki/Link_aggregation#Link_Aggregation_Control_Protocol [MySQL]: https://www.mysql.com/ [NAT]: https://en.wikipedia.org/wiki/Network_address_translation +[QoS]: https://en.wikipedia.org/wiki/Quality_of_service [TiDB]: https://www.pingcap.com/ [TiKV]: https://tikv.org/ [VXLAN]: https://en.wikipedia.org/wiki/Virtual_Extensible_LAN @@ -42,6 +43,8 @@ [cbindgen]: https://github.com/mozilla/cbindgen [distributed SQL]: https://en.wikipedia.org/wiki/Distributed_SQL [dpdk]: https://www.dpdk.org/ +[etcd]: https://github.com/coreos/etcd +[etherparse]: https://github.com/JulianSchmid/etherparse [frr]: https://frrouting.org/ [graphana]: https://grafana.com/ [kernel]: https://en.wikipedia.org/wiki/Linux_kernel @@ -51,6 +54,7 @@ [network address translation]: https://en.wikipedia.org/wiki/Network_address_translation [prometheus]: https://prometheus.io/ [protobuf]: https://protobuf.dev/ +[rqlite]: https://rqlite.io/ [rte lcores]: https://doc.dpdk.org/api/rte__lcore_8h.html [serde]: https://serde.rs/ [tracing]: https://docs.rs/tracing/latest/tracing/ diff --git a/justfile b/justfile index c2b809b6..5e4bd4a1 100644 --- a/justfile +++ b/justfile @@ -488,6 +488,7 @@ mdbook *args="build": --rm \ --init \ --volume "$(pwd):$(pwd)" \ + --env HOME=/tmp \ --user "$(id -u):$(id -g)" \ --mount type=bind,source=/tmp/doc-env,target=/tmp \ --workdir "$(pwd)" \ diff --git a/scripts/dpdk-sys.env b/scripts/dpdk-sys.env index 2132ad47..9cac607f 100644 --- a/scripts/dpdk-sys.env +++ b/scripts/dpdk-sys.env @@ -1,2 +1,2 @@ DPDK_SYS_BRANCH="main" -DPDK_SYS_COMMIT="362f54faf27e7b02148fe524492455e2f6762854" +DPDK_SYS_COMMIT="ae06e718b310c8ec8838d5198fc7cb5f9704125d"