From f3efc9c95bc1181ee3b1b719889f1523c9613825 Mon Sep 17 00:00:00 2001 From: Georg Eisenhart Date: Wed, 2 Feb 2022 12:56:38 +0100 Subject: [PATCH] refine som examples and docs --- CONFIGURATION.md | 4 +++- examples/README.md | 6 ++--- examples/flowdump/{config.yml => bpf.yml} | 0 examples/flowdump/csv.yml | 22 +++++++++++++++++++ .../config.yml => flowdump/highlight.yml} | 0 examples/flowdump/json.yml | 19 ++++++++++++++++ examples/flowdump/kafkaflowdump.yml | 20 +++++++++++++++++ 7 files changed, 67 insertions(+), 4 deletions(-) rename examples/flowdump/{config.yml => bpf.yml} (100%) create mode 100644 examples/flowdump/csv.yml rename examples/{highlighted_flowdump/config.yml => flowdump/highlight.yml} (100%) create mode 100644 examples/flowdump/json.yml create mode 100644 examples/flowdump/kafkaflowdump.yml diff --git a/CONFIGURATION.md b/CONFIGURATION.md index 8015910..2a81ab4 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -354,7 +354,7 @@ this can be configured using the fields parameter. fields: "SrcAddr,DstAddr,SamplerAddress" ``` -[any additional links](https://bwnet.belwue.de) +[CryptoPan module](https://github.com/Yawning/cryptopan) [godoc](https://pkg.go.dev/github.com/bwNetFlow/flowpipeline/segments/modify/anonymize) [examples using this segment](https://github.com/search?q=%22segment%3A+anonymize%22+extension%3Ayml+repo%3AbwNetFlow%2Fflowpipeline%2Fexamples&type=Code) @@ -529,6 +529,8 @@ sequence to export to different places. The `csv` segment provides an CSV output option. It uses stdout by default, but can be instructed to write to file using the filename parameter. The fields parameter can be used to limit which fields will be exported. +If no filename is provided or empty, the output goes to stdout. +By default all fields are exported. To reduce them, use a valid comma seperated list of fields. ``` - segment: csv diff --git a/examples/README.md b/examples/README.md index e48d444..3ecc72d 100644 --- a/examples/README.md +++ b/examples/README.md @@ -17,7 +17,7 @@ all inputs. This segment accesses local network interfaces using raw sockets, as for instance tcpdump does. Relevant examples are: -* [./flowdump](https://github.com/bwNetFlow/flowpipeline/tree/master/examples/flowdump) -- create a tcpdump style view with custom filtering from CLI using local +* [./flowdump/bpf.yml](https://github.com/bwNetFlow/flowpipeline/tree/master/examples/flowdump/bpf.yml) -- create a tcpdump style view with custom filtering from CLI using local interfaces @@ -33,8 +33,8 @@ This segment accesses streams of flows generated by another pipeline using `kafkaproducer` or [goflow2](https://github.com/netsampler/goflow2). Relevant examples are: -* [./kafkaflowdump](https://github.com/bwNetFlow/flowpipeline/tree/master/examples/kafkaflowdump) -- create a tcpdump style view with custom filtering from CLI -* [./highlighted_flowdump](https://github.com/bwNetFlow/flowpipeline/tree/master/examples/highlighted_flowdump) -- create a tcpdump style view but use the filtering conditional to highlight desired flows instead of dropping undesired flows +* [./flowdump/kafkaflowdump.yml](https://github.com/bwNetFlow/flowpipeline/tree/master/examples/flowdump/kafkaflowdump.yml) -- create a tcpdump style view with custom filtering from CLI +* [./flowdump/highlight.yml](https://github.com/bwNetFlow/flowpipeline/tree/master/examples/flowdump/highlight.yml) -- create a tcpdump style view but use the filtering conditional to highlight desired flows instead of dropping undesired flows * [./enricher](https://github.com/bwNetFlow/flowpipeline/tree/master/examples/enricher) -- enrich flows with various bits of data and store them back in Kafka * [./reducer](https://github.com/bwNetFlow/flowpipeline/tree/master/examples/reducer) -- strip flows of fields and store them back in Kafka * [./splitter](https://github.com/bwNetFlow/flowpipeline/tree/master/examples/splitter) -- distribute flows to multiple Kafka topics based on a field diff --git a/examples/flowdump/config.yml b/examples/flowdump/bpf.yml similarity index 100% rename from examples/flowdump/config.yml rename to examples/flowdump/bpf.yml diff --git a/examples/flowdump/csv.yml b/examples/flowdump/csv.yml new file mode 100644 index 0000000..ae7030c --- /dev/null +++ b/examples/flowdump/csv.yml @@ -0,0 +1,22 @@ +--- +############################################################################### +# Consume flow messages, it's best to use an enriched topic as flowdump +# printing involves interface descriptions. +- segment: kafkaconsumer + config: + server: kafka01.example.com:9093 + topic: flow-messages-enriched + group: myuser-flowdump + user: myuser + pass: $KAFKA_SASL_PASS + +############################################################################### +# CSV output with given fields. If no filename is configured output +# is redirected to stdout. +# +# Example list for fields may look like +# "TimeFlowStart,TimeFlowEnd,Bytes,Packets,SrcAddr,SrcPort,FlowDirection,DstAddr,DstPort,Proto" +- segment: csv + config: + filename: "" + fields: "" \ No newline at end of file diff --git a/examples/highlighted_flowdump/config.yml b/examples/flowdump/highlight.yml similarity index 100% rename from examples/highlighted_flowdump/config.yml rename to examples/flowdump/highlight.yml diff --git a/examples/flowdump/json.yml b/examples/flowdump/json.yml new file mode 100644 index 0000000..2016141 --- /dev/null +++ b/examples/flowdump/json.yml @@ -0,0 +1,19 @@ +--- +############################################################################### +# Consume flow messages, it's best to use an enriched topic as flowdump +# printing involves interface descriptions. +- segment: kafkaconsumer + config: + server: kafka01.example.com:9093 + topic: flow-messages-enriched + group: myuser-flowdump + user: myuser + pass: $KAFKA_SASL_PASS + +############################################################################### +# JSON output of flow messages. If no filename is configured output +# is redirected to stdout. +# +- segment: json + config: + filename: "" \ No newline at end of file diff --git a/examples/flowdump/kafkaflowdump.yml b/examples/flowdump/kafkaflowdump.yml new file mode 100644 index 0000000..24c97dc --- /dev/null +++ b/examples/flowdump/kafkaflowdump.yml @@ -0,0 +1,20 @@ +--- +############################################################################### +# Consume flow messages, it's best to use an enriched topic as flowdump +# printing involves interface descriptions. +- segment: kafkaconsumer + config: + server: kafka01.example.com:9093 + topic: flow-messages-enriched + group: myuser-flowdump + user: myuser + pass: $KAFKA_SASL_PASS + +############################################################################### +# tcpdump-style output of flows to stdout +- segment: printflowdump + # the lines below are optional and set to default + config: + useprotoname: true + verbose: false + highlight: false \ No newline at end of file