refine som examples and docs

BelWue · Feb 2, 2022 · f3efc9c · f3efc9c
1 parent 7ffe11f
commit f3efc9c
Show file tree

Hide file tree

Showing 7 changed files with 67 additions and 4 deletions.
diff --git a/CONFIGURATION.md b/CONFIGURATION.md
@@ -354,7 +354,7 @@ this can be configured using the fields parameter.
     fields: "SrcAddr,DstAddr,SamplerAddress"
 ```
 
-[any additional links](https://bwnet.belwue.de)
+[CryptoPan module](https://github.com/Yawning/cryptopan)
 [godoc](https://pkg.go.dev/github.com/bwNetFlow/flowpipeline/segments/modify/anonymize)
 [examples using this segment](https://github.com/search?q=%22segment%3A+anonymize%22+extension%3Ayml+repo%3AbwNetFlow%2Fflowpipeline%2Fexamples&type=Code)
 
@@ -529,6 +529,8 @@ sequence to export to different places.
 The `csv` segment provides an CSV output option. It uses stdout by default, but
 can be instructed to write to file using the filename parameter. The fields
 parameter can be used to limit which fields will be exported.
+If no filename is provided or empty, the output goes to stdout.
+By default all fields are exported. To reduce them, use a valid comma seperated list of fields.
 
 ```
 - segment: csv

diff --git a/examples/README.md b/examples/README.md
@@ -17,7 +17,7 @@ all inputs.
 This segment accesses local network interfaces using raw sockets, as for instance tcpdump does.
 
 Relevant examples are:
-* [./flowdump](https://github.com/bwNetFlow/flowpipeline/tree/master/examples/flowdump) -- create a tcpdump style view with custom filtering from CLI using local
+* [./flowdump/bpf.yml](https://github.com/bwNetFlow/flowpipeline/tree/master/examples/flowdump/bpf.yml) -- create a tcpdump style view with custom filtering from CLI using local
   interfaces
 
 
@@ -33,8 +33,8 @@ This segment accesses streams of flows generated by another pipeline using
 `kafkaproducer` or [goflow2](https://github.com/netsampler/goflow2).
 
 Relevant examples are:
-* [./kafkaflowdump](https://github.com/bwNetFlow/flowpipeline/tree/master/examples/kafkaflowdump) -- create a tcpdump style view with custom filtering from CLI
-* [./highlighted_flowdump](https://github.com/bwNetFlow/flowpipeline/tree/master/examples/highlighted_flowdump) -- create a tcpdump style view but use the filtering conditional to highlight desired flows instead of dropping undesired flows
+* [./flowdump/kafkaflowdump.yml](https://github.com/bwNetFlow/flowpipeline/tree/master/examples/flowdump/kafkaflowdump.yml) -- create a tcpdump style view with custom filtering from CLI
+* [./flowdump/highlight.yml](https://github.com/bwNetFlow/flowpipeline/tree/master/examples/flowdump/highlight.yml) -- create a tcpdump style view but use the filtering conditional to highlight desired flows instead of dropping undesired flows
 * [./enricher](https://github.com/bwNetFlow/flowpipeline/tree/master/examples/enricher) -- enrich flows with various bits of data and store them back in Kafka
 * [./reducer](https://github.com/bwNetFlow/flowpipeline/tree/master/examples/reducer) -- strip flows of fields and store them back in Kafka
 * [./splitter](https://github.com/bwNetFlow/flowpipeline/tree/master/examples/splitter) -- distribute flows to multiple Kafka topics based on a field

diff --git a/examples/flowdump/config.yml → examples/flowdump/bpf.yml b/examples/flowdump/config.yml → examples/flowdump/bpf.yml
diff --git a/examples/flowdump/csv.yml b/examples/flowdump/csv.yml
@@ -0,0 +1,22 @@
+---
+###############################################################################
+# Consume flow messages, it's best to use an enriched topic as flowdump
+# printing involves interface descriptions.
+- segment: kafkaconsumer
+  config:
+    server: kafka01.example.com:9093
+    topic: flow-messages-enriched
+    group: myuser-flowdump
+    user: myuser
+    pass: $KAFKA_SASL_PASS
+
+###############################################################################
+# CSV output with given fields. If no filename is configured output
+# is redirected to stdout.
+# 
+# Example list for fields may look like
+# "TimeFlowStart,TimeFlowEnd,Bytes,Packets,SrcAddr,SrcPort,FlowDirection,DstAddr,DstPort,Proto"
+- segment: csv
+  config:
+    filename: ""
+    fields: ""
diff --git a/examples/highlighted_flowdump/config.yml → examples/flowdump/highlight.yml b/examples/highlighted_flowdump/config.yml → examples/flowdump/highlight.yml
diff --git a/examples/flowdump/json.yml b/examples/flowdump/json.yml
@@ -0,0 +1,19 @@
+---
+###############################################################################
+# Consume flow messages, it's best to use an enriched topic as flowdump
+# printing involves interface descriptions.
+- segment: kafkaconsumer
+  config:
+    server: kafka01.example.com:9093
+    topic: flow-messages-enriched
+    group: myuser-flowdump
+    user: myuser
+    pass: $KAFKA_SASL_PASS
+
+###############################################################################
+# JSON output of flow messages. If no filename is configured output
+# is redirected to stdout.
+# 
+- segment: json
+  config:
+    filename: ""
diff --git a/examples/flowdump/kafkaflowdump.yml b/examples/flowdump/kafkaflowdump.yml
@@ -0,0 +1,20 @@
+---
+###############################################################################
+# Consume flow messages, it's best to use an enriched topic as flowdump
+# printing involves interface descriptions.
+- segment: kafkaconsumer
+  config:
+    server: kafka01.example.com:9093
+    topic: flow-messages-enriched
+    group: myuser-flowdump
+    user: myuser
+    pass: $KAFKA_SASL_PASS
+
+###############################################################################
+# tcpdump-style output of flows to stdout
+- segment: printflowdump
+  # the lines below are optional and set to default
+  config:
+    useprotoname: true
+    verbose: false
+    highlight: false