Merge pull request #163 from netboxlabs/develop

🚚 release: v0.5.0

Makefile

@@ -1,12 +1,14 @@
-.PHONY: gen-diode-sdk-go gen-diode-sdk-python
-
+.PHONY: gen-diode-sdk-go
 gen-diode-sdk-go:
-	@cd diode-proto/ && buf format -w && buf generate --template buf.gen.go.yaml
-
-gen-diode-go-internal:
-	@cd diode-proto/ && buf format -w && buf generate --template buf.gen.go-internal.yaml
+	@cd diode-proto/ && buf format -w && buf generate --template buf.gen.sdk.go.yaml
 
+.PHONY: gen-diode-sdk-python
 gen-diode-sdk-python:
-	@cd diode-proto/ && buf format -w && buf generate --template buf.gen.py.yaml --include-imports
+	@cd diode-proto/ && buf format -w && buf generate --template buf.gen.sdk.py.yaml --include-imports
 	@find ../diode-sdk-python/netboxlabs/diode/sdk \( -name '*.py' -o -name '*.pyi' \) \
-	-exec sed -i '' 's/^from diode.v1/from netboxlabs.diode.sdk.diode.v1/; s/^from validate/from netboxlabs.diode.sdk.validate/' {} \;
+	-exec sed -i.bak -e 's/^from diode.v1/from netboxlabs.diode.sdk.diode.v1/' \
+	-e 's/^from validate/from netboxlabs.diode.sdk.validate/' {} \; -exec rm -f {}.bak \;
+
+.PHONY: gen-diode-server-go
+gen-diode-server-go:
+	@cd diode-proto/ && buf format -w && buf generate --template buf.gen.server.go.yaml

README.md

@@ -1,33 +1,56 @@
 # Diode
 
-Diode is a NetBox data ingestion service that greatly simplifies and enhances the process to add and update network data in NetBox, ensuring your network source of truth is always accurate and can be trusted to power your network automation pipelines. Our guiding principle in designing Diode has been to make it as easy as possible to get data into NetBox, removing as much burden as possible from the user while shifting that effort to technology. 
-
-To achieve this, Diode sits in front of NetBox and provides an API purpose built for ingestion of complex network data. Diode eliminates the need to preprocess data to make it conform to the strict object hierarchy imposed by the NetBox data model. This allows data to be sent to NetBox in a more freeform manner, in blocks that are intuitive for network engineers (such as by device or by interface) with much of the related information treated as attributes or properties of these components of interest. Then, Diode takes care of the heavy lifting, automatically transforming the data to align it with NetBox’s structured and comprehensive data model. Diode can even create placeholder objects to compensate for missing information, which means even fragmented information about the network can be captured in NetBox. 
+Diode is a NetBox data ingestion service that greatly simplifies and enhances the process to add and update network data
+in NetBox, ensuring your network source of truth is always accurate and can be trusted to power your network automation
+pipelines. Our guiding principle in designing Diode has been to make it as easy as possible to get data into NetBox,
+removing as much burden as possible from the user while shifting that effort to technology.
+
+To achieve this, Diode sits in front of NetBox and provides an API purpose built for ingestion of complex network data.
+Diode eliminates the need to preprocess data to make it conform to the strict object hierarchy imposed by the NetBox
+data model. This allows data to be sent to NetBox in a more freeform manner, in blocks that are intuitive for network
+engineers (such as by device or by interface) with much of the related information treated as attributes or properties
+of these components of interest. Then, Diode takes care of the heavy lifting, automatically transforming the data to
+align it with NetBox’s structured and comprehensive data model. Diode can even create placeholder objects to compensate
+for missing information, which means even fragmented information about the network can be captured in NetBox.
 
 ## Project status
 
-The Diode project is currently in the _Public Preview_ stage. Please see [NetBox Labs Product and Feature Lifecycle](https://docs.netboxlabs.com/product_feature_lifecycle/) for more details. We actively welcome feedback to help identify and prioritize bugs, new features and areas of improvement. 
+The Diode project is currently in the _Public Preview_ stage. Please
+see [NetBox Labs Product and Feature Lifecycle](https://docs.netboxlabs.com/product_feature_lifecycle/) for more
+details. We actively welcome feedback to help identify and prioritize bugs, new features and areas of improvement.
 
 ## Get started
 
-Diode runs as a sidecar service to NetBox and can run anywhere with network connectivity to NetBox, whether on the same host or elsewhere. The overall Diode service is delivered through three main components (and a fourth optional component):
+Diode runs as a sidecar service to NetBox and can run anywhere with network connectivity to NetBox, whether on the same
+host or elsewhere. The overall Diode service is delivered through three main components (and a fourth optional
+component):
 
 1. Diode plugin - see how to [install the Diode plugin](https://github.com/netboxlabs/diode-netbox-plugin)
-2. Diode server - see how to [run the Diode server](https://github.com/netboxlabs/diode/tree/develop/diode-server#readme)
-3. Diode SDK - see how to [install the Diode client SDK](https://github.com/netboxlabs/diode-sdk-python) and [download Diode Python script examples](https://github.com/netboxlabs/netbox-learning/tree/develop/diode)
-4. Diode agent (optional) - see how to [install and run the Diode NAPALM discovery agent](https://github.com/netboxlabs/diode-agent/tree/develop/diode-napalm-agent)
+2. Diode server - see how
+   to [run the Diode server](https://github.com/netboxlabs/diode/tree/develop/diode-server#readme)
+3. Diode SDK - see how
+   to [install the Diode Python client SDK](https://github.com/netboxlabs/diode-sdk-python), [download Diode Python script examples](https://github.com/netboxlabs/netbox-learning/tree/develop/diode)
+   and [use the Diode SDK Go](https://github.com/netboxlabs/diode-sdk-go)
+4. Diode agent (optional) - see how
+   to [install and run the Diode NAPALM discovery agent](https://github.com/netboxlabs/diode-agent/tree/develop/diode-napalm-agent)
 
 ## Related Projects
 
-- [diode-netbox-plugin](https://github.com/netboxlabs/diode-netbox-plugin) - The Diode NetBox plugin is a NetBox plugin and a required component of the Diode ingestion service.
-- [diode-sdk-python](https://github.com/netboxlabs/diode-sdk-python) - Diode SDK Python is a Python library for interacting with the Diode ingestion service utilizing gRPC.
-- [diode-agent](https://github.com/netboxlabs/diode-agent) - A collection of agents that leverage the Diode SDK to interact with the Diode server.
+- [diode-netbox-plugin](https://github.com/netboxlabs/diode-netbox-plugin) - The Diode NetBox plugin is a NetBox plugin
+  and a required component of the Diode ingestion service.
+- [diode-sdk-python](https://github.com/netboxlabs/diode-sdk-python) - Diode SDK Python is a Python library for
+  interacting with the Diode ingestion service utilizing gRPC.
+- [diode-sdk-go](https://github.com/netboxlabs/diode-sdk-go) - Diode SDK Go is a Go module for interacting with the
+  Diode ingestion service utilizing gRPC.
+- [diode-agent](https://github.com/netboxlabs/diode-agent) - A collection of agents that leverage the Diode SDK to
+  interact with the Diode server.
 
 ## License
 
 Distributed under the PolyForm Shield License 1.0.0 License. See [LICENSE.md](./LICENSE.md) for more information.
 
-Diode protocol buffers are distributed under the Apache 2.0 License. See [LICENSE.txt](./diode-proto/LICENSE.txt) for more information.
+Diode protocol buffers are distributed under the Apache 2.0 License. See [LICENSE.txt](./diode-proto/LICENSE.txt) for
+more information.
 
 ## Required Notice
 

diode-proto/buf.gen.go.yaml → diode-proto/buf.gen.sdk.go.yaml

@@ -1,15 +1,16 @@
-version: v1
+version: v2
+inputs:
+  - proto_file: diode/v1/ingester.proto
 plugins:
-  - plugin: go
+  - local: protoc-gen-go
     out: ../../diode-sdk-go/
     opt: module=github.com/netboxlabs/diode-sdk-go
-  - plugin: buf.build/grpc/go:v1.3.0
+  - remote: buf.build/grpc/go:v1.3.0
     out: ../../diode-sdk-go/
     opt: module=github.com/netboxlabs/diode-sdk-go
-  - plugin: buf.build/bufbuild/validate-go:v1.0.4
+  - remote: buf.build/bufbuild/validate-go:v1.0.4
     out: ../../diode-sdk-go/
     opt: module=github.com/netboxlabs/diode-sdk-go
-  - plugin: buf.build/community/pseudomuto-doc:v1.5.1
+  - remote: buf.build/community/pseudomuto-doc:v1.5.1
     out: ../docs/
-    opt:
-      - markdown,diode-proto.md
+    opt: markdown,diode-proto.md

diode-proto/buf.gen.sdk.py.yaml

@@ -0,0 +1,10 @@
+version: v2
+inputs:
+  - proto_file: diode/v1/ingester.proto
+plugins:
+  - remote: buf.build/protocolbuffers/python:v26.1
+    out: ../../diode-sdk-python/netboxlabs/diode/sdk/
+  - remote: buf.build/protocolbuffers/pyi:v26.1
+    out: ../../diode-sdk-python/netboxlabs/diode/sdk/
+  - remote: buf.build/grpc/python:v1.62.1
+    out: ../../diode-sdk-python/netboxlabs/diode/sdk/

diode-proto/buf.gen.server.go.yaml

@@ -0,0 +1,20 @@
+version: v2
+managed:
+  enabled: true
+  override:
+    - file_option: go_package
+      path: diode/v1/ingester.proto
+      value: github.com/netboxlabs/diode/diode-server/gen/diode/v1/diodepb
+    - file_option: go_package
+      path: diode/v1/reconciler.proto
+      value: github.com/netboxlabs/diode/diode-server/gen/diode/v1/reconcilerpb
+plugins:
+  - local: protoc-gen-go
+    out: ../
+    opt: module=github.com/netboxlabs/diode
+  - remote: buf.build/grpc/go:v1.3.0
+    out: ../
+    opt: module=github.com/netboxlabs/diode
+  - remote: buf.build/bufbuild/validate-go:v1.0.4
+    out: ../
+    opt: module=github.com/netboxlabs/diode

diode-proto/buf.lock

@@ -1,13 +1,9 @@
 # Generated by buf. DO NOT EDIT.
-version: v1
+version: v2
 deps:
-  - remote: buf.build
-    owner: envoyproxy
-    repository: protoc-gen-validate
+  - name: buf.build/envoyproxy/protoc-gen-validate
     commit: 71881f09a0c5420a9545a07987a86728
-    digest: shake256:d320bbf06653b1b2b45a1f95bfa82bf7b998221a777a042708c50d6f86a30d1a85b50c5704c597142d9b308280efe1295d39d76d1abea5f7046d3df4c8cc3cef
-  - remote: buf.build
-    owner: googleapis
-    repository: googleapis
+    digest: b5:e8a034a81f1d6218f712879269bedac768c9e39452d7a92f83d70923fcd6aa9eb02c81ff6ff337cd0dd9b9fe719d6c4459e655f662ae7112aaa1c2110992afd0
+  - name: buf.build/googleapis/googleapis
     commit: 7a6bc1e3207144b38e9066861e1de0ff
-    digest: shake256:d646836485c34192401253703c4e7ce899c826fceec060bf4b2a62c4749bd9976dc960833e134a1f814725e1ffd60b1bb3cf0335a7e99ef0e8cec34b070ffb66
+    digest: b5:6d05bde5ed4cd22531d7ca6467feb828d2dc45cc9de12ce3345fbddd64ddb1bf0db756558c32ca49e6bc7de4426ada8960d5590e8446854b81f5f36f0916dc48

diode-proto/buf.yaml

@@ -1,13 +1,19 @@
-version: v1
+version: v2
 deps:
-  - buf.build/googleapis/googleapis
   - buf.build/envoyproxy/protoc-gen-validate
-breaking:
-  use:
-    - FILE
+  - buf.build/googleapis/googleapis
 lint:
   use:
     - DEFAULT
   except:
+    - FIELD_NOT_REQUIRED
+    - PACKAGE_NO_IMPORT_CYCLE
     - RPC_REQUEST_STANDARD_NAME
     - RPC_RESPONSE_STANDARD_NAME
+  disallow_comment_ignores: true
+breaking:
+  use:
+    - FILE
+  except:
+    - EXTENSION_NO_DELETE
+    - FIELD_SAME_DEFAULT

diode-proto/diode/v1/ingester.proto

@@ -189,6 +189,116 @@ message Interface {
   repeated Tag tags = 14;
 }
 
+//A Cluster
+message Cluster {
+  string name = 1 [(validate.rules).string = {
+    min_len: 1
+    max_len: 100
+  }];
+  ClusterType type = 2;
+  ClusterGroup group = 3;
+  Site site = 4;
+  string status = 5 [(validate.rules).string = {
+    in: [
+      "offline",
+      "active",
+      "planned",
+      "staged",
+      "failed",
+      "decommissioning"
+    ]
+  }];
+  optional string description = 6 [(validate.rules).string = {max_len: 200}];
+  repeated Tag tags = 7;
+}
+
+//A Cluster Type
+message ClusterType {
+  string name = 1 [(validate.rules).string = {
+    min_len: 1
+    max_len: 100
+  }];
+  string slug = 2 [(validate.rules).string = {
+    min_len: 1
+    max_len: 100
+    pattern: "^[-a-zA-Z0-9_]+$"
+  }];
+  optional string description = 3 [(validate.rules).string = {max_len: 200}];
+  repeated Tag tags = 4;
+}
+
+//A Cluster Group
+message ClusterGroup {
+  string name = 1 [(validate.rules).string = {
+    min_len: 1
+    max_len: 100
+  }];
+  string slug = 2 [(validate.rules).string = {
+    min_len: 1
+    max_len: 100
+    pattern: "^[-a-zA-Z0-9_]+$"
+  }];
+  optional string description = 3 [(validate.rules).string = {max_len: 200}];
+  repeated Tag tags = 4;
+}
+
+//A Virtual Machine
+message VirtualMachine {
+  string name = 1 [(validate.rules).string = {max_len: 64}];
+  string status = 2 [(validate.rules).string = {
+    in: [
+      "offline",
+      "active",
+      "planned",
+      "staged",
+      "failed",
+      "decommissioning"
+    ]
+  }];
+  Site site = 3;
+  Cluster cluster = 4;
+  Role role = 5;
+  Device device = 6;
+  Platform platform = 7;
+  IPAddress primary_ip4 = 8;
+  IPAddress primary_ip6 = 9;
+  optional int32 vcpus = 10 [(validate.rules).int32 = {gte: 0}];
+  optional int32 memory = 11 [(validate.rules).int32 = {gte: 0}];
+  optional int32 disk = 12 [(validate.rules).int32 = {gte: 0}];
+  optional string description = 13 [(validate.rules).string = {max_len: 200}];
+  optional string comments = 14;
+  repeated Tag tags = 15;
+}
+
+//A Virtual Machine Interface
+message VMInterface {
+  VirtualMachine virtual_machine = 1 [(validate.rules).any.required = true];
+  string name = 2 [(validate.rules).string = {
+    min_len: 1
+    max_len: 64
+  }];
+  optional bool enabled = 3;
+  optional int32 mtu = 4 [(validate.rules).int32 = {
+    gte: 1
+    lte: 65536
+  }];
+  optional string mac_address = 5;
+  optional string description = 6 [(validate.rules).string = {max_len: 200}];
+  repeated Tag tags = 7;
+}
+
+//A Virtual Disk
+message VirtualDisk {
+  VirtualMachine virtual_machine = 1 [(validate.rules).any.required = true];
+  string name = 2 [(validate.rules).string = {
+    min_len: 1
+    max_len: 100
+  }];
+  int32 size = 3 [(validate.rules).int32 = {gte: 0}];
+  optional string description = 4 [(validate.rules).string = {max_len: 200}];
+  repeated Tag tags = 5;
+}
+
 // An IP address.
 message IPAddress {
   string address = 1 [(validate.rules).string.ip = true];
@@ -370,6 +480,12 @@ message Entity {
     Interface interface = 7;
     IPAddress ip_address = 9;
     Prefix prefix = 10;
+    ClusterGroup cluster_group = 11;
+    ClusterType cluster_type = 12;
+    Cluster cluster = 13;
+    VirtualMachine virtual_machine = 14;
+    VMInterface vminterface = 15;
+    VirtualDisk virtual_disk = 16;
   }
 
   // The timestamp of the data discovery at source