Add support for setting take_over (#341)

The new optional Boolean parameter `take_over` for `filebeat::input`
defaults to `undef` but can be set to `true` to enable the "take over"
functionality when switching from the deprecated `log` input type to the
new `filestream` type. This allows Filebeat to take over files from the
`log` input without completely re-ingesting the files, which leads to
duplicated log entries. This feature is available in Filebeat 8.x.

This change adds very basic spec tests for the `filestream` input type
and the `take_over` logic, and updates the docs where appropriate.

README.md

@@ -398,7 +398,8 @@ to fully understand what these parameters do.
   - `syslog_host`: [String] Host to listen for syslog messages (default: localhost:5140)
   - `exclude_files`: [Array] Files that match any regex in the list are excluded from filebeat (default: [])
   - `encoding`: [String] The file encoding. (default: plain)
-  - `input_type`: [String] where filebeat reads the log from (default:log)
+  - `input_type`: [String] where filebeat reads the log from (default: filestream)
+  - `take_over` : [Boolean] Optionally enable [`take_over`](https://www.elastic.co/guide/en/beats/filebeat/8.11/filebeat-input-filestream.html#filebeat-input-filestream-take-over) when switchting from the deprecated input type `log` to the new input type `filestream`. This avoids re-ingesting already logfiles Filebeat already read when switching to `filestream`. This feature requires Filebeat 8.x.
   - `fields`: [Hash] Optional fields to add information to the output (default: {})
   - `fields_under_root`: [Boolean] Should the `fields` parameter fields be stored at the top level of indexed documents.
   - `ignore_older`: [String] Files older than this field will be ignored by filebeat (default: ignore nothing)

manifests/config.pp

@@ -68,7 +68,6 @@
     else {
       $filebeat_config = $filebeat_config_xpack
     }
-
   } else {
     $filebeat_config_temp = delete_undef_values({
         'shutdown_timeout'  => $filebeat::shutdown_timeout,

manifests/input.pp

@@ -19,6 +19,7 @@
   Boolean $cri_parse_flags                 = false,
   String $encoding                         = 'plain',
   String $input_type                       = $filebeat::params::default_input_type,
+  Optional[Boolean] $take_over             = undef,
   Hash $fields                             = {},
   Boolean $fields_under_root               = $filebeat::fields_under_root,
   Hash $ssl                                = {},

spec/defines/input_spec.rb

@@ -78,6 +78,58 @@
         }
       end
     end
+
+    context "with filestream input support on #{os}" do
+      let(:facts) { os_facts }
+
+      # Filestream
+      let(:title) { 'some-filestream' }
+
+      context "with take_over unset" do
+        let(:params) do
+          {
+            'input_type' => 'filestream',
+            'paths' => ['/var/log/foo.log'],
+          }
+        end
+
+        if os_facts[:kernel] == 'Linux'
+          it { is_expected.to compile }
+
+          it {
+            is_expected.to contain_file('filebeat-some-filestream').with(
+              notify: 'Service[filebeat]',
+            )
+            is_expected.to contain_file('filebeat-some-filestream').with_content(
+              %r{- type: filestream\n\s{2}id: some-filestream\n\s{2}paths:\n\s{2}- /var/log/foo.log},
+            )
+          }
+        end
+      end
+
+      context "with take_over => true" do
+        let(:params) do
+          {
+            'input_type' => 'filestream',
+            'paths' => ['/var/log/foo.log'],
+            'take_over' => true,
+          }
+        end
+
+        if os_facts[:kernel] == 'Linux'
+          it { is_expected.to compile }
+
+          it {
+            is_expected.to contain_file('filebeat-some-filestream').with(
+              notify: 'Service[filebeat]',
+            )
+            is_expected.to contain_file('filebeat-some-filestream').with_content(
+              %r{- type: filestream\n\s{2}id: some-filestream\n\s{2}take_over: true\n\s{2}paths:\n\s{2}- /var/log/foo.log},
+            )
+          }
+        end
+      end
+    end
   end
 
   on_supported_os.each do |os, os_facts|

templates/input.yml.erb

@@ -6,6 +6,9 @@
   <%- if @input_type =~ /(filestream|journald)/ -%>
   id: <%= @name %>
   <%- end -%>
+  <%- if @input_type == 'filestream' && @take_over != nil && @take_over == true -%>
+  take_over: true
+  <%- end -%>
   <%- if @input_type =~ /(tcp|udp)/ -%>
   host: <%= @host %>
     <%- if @ssl.length > 0 -%>
@@ -84,14 +87,14 @@
   <%- if @scan_frequency or @exclude_files -%>
     <%- if @input_type == 'filestream' -%>
   prospector:
-     scanner:
+    scanner:
       <%- if @scan_frequency -%>
-        check_interval: <%= @scan_frequency %>
+      check_interval: <%= @scan_frequency %>
       <%- end -%>
       <%- if @exclude_files.length > 0 -%>
-        exclude_files:
+      exclude_files:
         <%- @exclude_files.each do |exclude_file| -%>
-        - <%= exclude_file %>
+      - <%= exclude_file %>
         <%- end -%>
       <%- end -%>
     <%- else -%>