diff --git a/integration/logstash-plugins/README.md b/integration/logstash-plugins/README.md index 5270e038056..0f4b948be73 100644 --- a/integration/logstash-plugins/README.md +++ b/integration/logstash-plugins/README.md @@ -1,7 +1,5 @@ # Logstash Plugins for Interacting with Vespa -This repository contains plugins to both read from and write to Vespa by using Logstash. +Plugins to both read from and write to Vespa by using Logstash. -Check out the README.md files in the respective plugin directories for more information. - -This repository doesn't have issues enabled, but feel free to open related issues at https://github.com/vespa-engine/vespa +Check out the README.md files in the respective plugin directories for more information. \ No newline at end of file diff --git a/integration/logstash-plugins/logstash-input-vespa/CHANGELOG.md b/integration/logstash-plugins/logstash-input-vespa/CHANGELOG.md index 1486b9ae052..b8377ab356f 100644 --- a/integration/logstash-plugins/logstash-input-vespa/CHANGELOG.md +++ b/integration/logstash-plugins/logstash-input-vespa/CHANGELOG.md @@ -1,4 +1,6 @@ +## 0.2.0 +Added support for mTLS certificates, selector, page_size, backend_concurrency, timeout, from_timestamp, and to_timestamp + ## 0.1.0 Initial version. Can connect to an HTTP endpoint of Vespa and pull documents via Visit and use the continuation token - diff --git a/integration/logstash-plugins/logstash-input-vespa/Gemfile b/integration/logstash-plugins/logstash-input-vespa/Gemfile index 32cc6fbbf6d..37017aa74be 100644 --- a/integration/logstash-plugins/logstash-input-vespa/Gemfile +++ b/integration/logstash-plugins/logstash-input-vespa/Gemfile @@ -9,3 +9,11 @@ if Dir.exist?(logstash_path) && use_logstash_source gem 'logstash-core', :path => "#{logstash_path}/logstash-core" gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api" end + +group :development, :test do + gem 'logstash-devutils' + gem 'webmock' + gem 'rspec' +end + +gem 'minitar', '~> 1.0' diff --git a/integration/logstash-plugins/logstash-input-vespa/LICENSE b/integration/logstash-plugins/logstash-input-vespa/LICENSE deleted file mode 100644 index a80a3fd5323..00000000000 --- a/integration/logstash-plugins/logstash-input-vespa/LICENSE +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright 2020 Elastic and contributors - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/integration/logstash-plugins/logstash-input-vespa/README.md b/integration/logstash-plugins/logstash-input-vespa/README.md index 916ee2d9d47..c8714a74512 100644 --- a/integration/logstash-plugins/logstash-input-vespa/README.md +++ b/integration/logstash-plugins/logstash-input-vespa/README.md @@ -9,6 +9,17 @@ Download and unpack/install Logstash, then: bin/logstash-plugin install logstash-input-vespa ``` +## Development + +To run tests, you'll need to clone the Logstash branch you're developing the plugin for. See https://github.com/elastic/logstash + +Then: +``` +export LOGSTASH_PATH=/path/to/logstash/repository/clone +export LOGSTASH_SOURCE=1 +bundle exec rspec +``` + ## Usage Minimal Logstash config example: diff --git a/integration/logstash-plugins/logstash-input-vespa/lib/logstash/inputs/vespa.rb b/integration/logstash-plugins/logstash-input-vespa/lib/logstash/inputs/vespa.rb index d458e4e0a89..15f7c0923fb 100644 --- a/integration/logstash-plugins/logstash-input-vespa/lib/logstash/inputs/vespa.rb +++ b/integration/logstash-plugins/logstash-input-vespa/lib/logstash/inputs/vespa.rb @@ -1,4 +1,7 @@ # encoding: utf-8 + +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + require "logstash/inputs/base" require "logstash/namespace" require "net/http" @@ -22,6 +25,12 @@ class LogStash::Inputs::Vespa < LogStash::Inputs::Base # The cluster parameter to use in the request. config :cluster, :validate => :string, :required => true + # Maximum number of retries for failed HTTP requests + config :max_retries, :validate => :number, :default => 3 + + # Delay in seconds for the first retry attempt. We double this delay for each subsequent retry. + config :retry_delay, :validate => :number, :default => 1 + # Path to the client certificate file for mTLS. config :client_cert, :validate => :path @@ -85,6 +94,9 @@ def run(queue) uri.query = URI.encode_www_form(@uri_params) continuation = nil + retries = 0 + current_delay = @retry_delay + loop do response = fetch_documents_from_vespa(uri) # response should look like: @@ -120,28 +132,58 @@ def run(queue) end else - @logger.error("Failed to fetch documents from Vespa", :request => uri.to_s, + # Handle retriable status codes (5xx) + if (500..599).include?(response.code.to_i) && retries < (@max_retries - 1) + retries += 1 + @logger.warn("Retriable error from Vespa, retrying", + :response_code => response.code, + :retry_count => retries, + :max_retries => @max_retries, + :next_retry_delay => current_delay) + sleep(current_delay) + current_delay *= 2 + else + @logger.error("Failed to fetch documents from Vespa", :request => uri.to_s, :response_code => response.code, :response_message => response.message) - break # TODO retry? Only on certain codes? + break + end end # if response.is_a?(Net::HTTPSuccess) end # loop do end # def run def fetch_documents_from_vespa(uri) - http = Net::HTTP.new(uri.host, uri.port) - if uri.scheme == "https" - http.use_ssl = true - http.cert = @cert - http.key = @key - http.verify_mode = OpenSSL::SSL::VERIFY_PEER + retries = 0 + current_delay = @retry_delay # Start with the initial delay + + begin + http = Net::HTTP.new(uri.host, uri.port) + if uri.scheme == "https" + http.use_ssl = true + http.cert = @cert + http.key = @key + http.verify_mode = OpenSSL::SSL::VERIFY_PEER + end + + request = Net::HTTP::Get.new(uri.request_uri) + http.request(request) + rescue => e + retries += 1 + if retries < @max_retries + @logger.warn("Failed to make HTTP request to Vespa, retrying", + :error => e.message, + :retry_count => retries, + :max_retries => @max_retries, + :next_retry_delay => current_delay) + sleep(current_delay) + current_delay *= 2 # Double the delay for next retry + retry + else + @logger.error("Failed to make HTTP request to Vespa after #{@max_retries} attempts", + :error => e.message) + nil + end end - - request = Net::HTTP::Get.new(uri.request_uri) - http.request(request) - rescue => e - @logger.error("Failed to make HTTP request to Vespa", :error => e.message) - nil end # def fetch_documents_from_vespa def parse_response(response) diff --git a/integration/logstash-plugins/logstash-input-vespa/logstash-input-vespa.gemspec b/integration/logstash-plugins/logstash-input-vespa/logstash-input-vespa.gemspec index 8a9b4893dc7..5d779f7fe5f 100644 --- a/integration/logstash-plugins/logstash-input-vespa/logstash-input-vespa.gemspec +++ b/integration/logstash-plugins/logstash-input-vespa/logstash-input-vespa.gemspec @@ -1,6 +1,6 @@ Gem::Specification.new do |s| s.name = 'logstash-input-vespa' - s.version = '0.2.0' + s.version = '0.3.0' s.licenses = ['Apache-2.0'] s.summary = "Logstash input plugin reading from Vespa" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" @@ -18,8 +18,13 @@ Gem::Specification.new do |s| s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" } # Gem dependencies - s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99" + s.add_runtime_dependency "logstash-core", ">= 8.0.0" s.add_runtime_dependency 'logstash-codec-plain' s.add_runtime_dependency 'stud', '>= 0.0.22' + s.add_runtime_dependency 'logstash-codec-json' + + # Development dependencies s.add_development_dependency 'logstash-devutils' + s.add_development_dependency 'rspec', '~> 3.0' + s.add_development_dependency 'webmock', '~> 3.0' end diff --git a/integration/logstash-plugins/logstash-input-vespa/spec/inputs/vespa_spec.rb b/integration/logstash-plugins/logstash-input-vespa/spec/inputs/vespa_spec.rb new file mode 100644 index 00000000000..4864c8c492b --- /dev/null +++ b/integration/logstash-plugins/logstash-input-vespa/spec/inputs/vespa_spec.rb @@ -0,0 +1,116 @@ +# encoding: utf-8 +require "logstash/devutils/rspec/spec_helper" +require "logstash/inputs/vespa" +require "webmock/rspec" + +describe LogStash::Inputs::Vespa do + let(:config) do + { + "cluster" => "test-cluster", + "vespa_url" => "http://localhost:8080", + "retry_delay" => 0.1, # Small delay for faster tests + "max_retries" => 3 + } + end + + let(:plugin) { described_class.new(config) } + let(:queue) { Queue.new } + let(:base_uri) { "#{config['vespa_url']}/document/v1/" } + let(:uri_params) { "cluster=test-cluster&wantedDocumentCount=100&concurrency=1&timeout=180" } + + before do + plugin.register + allow(plugin).to receive(:sleep) # Mock sleep to speed up tests + end + + describe "#run" do + context "when server returns retriable errors" do + it "retries on 503 Service Unavailable" do + stub_request(:get, "#{base_uri}?#{uri_params}") + .to_return( + { status: 503, body: "Service Unavailable" }, + { status: 503, body: "Service Unavailable" }, + { status: 200, body: '{"documents": [], "documentCount": 0}' } + ) + + plugin.run(queue) + expect(a_request(:get, "#{base_uri}?#{uri_params}")).to have_been_made.times(3) + end + + it "retries on 502 Bad Gateway" do + stub_request(:get, "#{base_uri}?#{uri_params}") + .to_return( + { status: 502, body: "Bad Gateway" }, + { status: 200, body: '{"documents": [], "documentCount": 0}' } + ) + + plugin.run(queue) + expect(a_request(:get, "#{base_uri}?#{uri_params}")).to have_been_made.times(2) + end + + it "stops after max_retries attempts" do + stub_request(:get, "#{base_uri}?#{uri_params}") + .to_return(status: 503, body: "Service Unavailable").times(4) + + plugin.run(queue) + expect(a_request(:get, "#{base_uri}?#{uri_params}")).to have_been_made.times(config["max_retries"]) + end + end + + context "when server returns non-retriable errors" do + it "does not retry on 404 Not Found" do + stub_request(:get, "#{base_uri}?#{uri_params}") + .to_return(status: 404, body: "Not Found") + + plugin.run(queue) + expect(a_request(:get, "#{base_uri}?#{uri_params}")).to have_been_made.times(1) + end + + it "does not retry on 401 Unauthorized" do + stub_request(:get, "#{base_uri}?#{uri_params}") + .to_return(status: 401, body: "Unauthorized") + + plugin.run(queue) + expect(a_request(:get, "#{base_uri}?#{uri_params}")).to have_been_made.times(1) + end + end + + context "when server returns successful responses" do + it "processes documents and follows continuation tokens" do + + # First response with continuation token + first_response = { + "pathId" => "/document/v1/", + "documents" => [ + {"id" => "id:namespace:doctype::doc1", "fields" => {"field1" => "value1", "field2" => 7.0}}, + {"id" => "id:namespace:doctype::doc2", "fields" => {"field1" => "value2", "field2" => 8.0}} + ], + "documentCount" => 2, + "continuation" => "AAAAAA" + } + + # Second response without continuation (last page) + last_response = { + "pathId" => "/document/v1/", + "documents" => [ + {"id" => "id:namespace:doctype::doc3", "fields" => {"field1" => "value3", "field2" => 9.0}} + ], + "documentCount" => 1 + } + + # Stub the requests + stub_request(:get, "#{base_uri}?#{uri_params}") + .to_return(status: 200, body: first_response.to_json) + + stub_request(:get, "#{base_uri}?#{uri_params}&continuation=AAAAAA") + .to_return(status: 200, body: last_response.to_json) + + plugin.run(queue) + + expect(queue.size).to eq(3) # Total of 3 documents + expect(a_request(:get, "#{base_uri}?#{uri_params}")).to have_been_made.once + expect(a_request(:get, "#{base_uri}?#{uri_params}&continuation=AAAAAA")).to have_been_made.once + end + end + end +end \ No newline at end of file diff --git a/integration/logstash-plugins/logstash-output-vespa/LICENSE b/integration/logstash-plugins/logstash-output-vespa/LICENSE deleted file mode 100644 index a80a3fd5323..00000000000 --- a/integration/logstash-plugins/logstash-output-vespa/LICENSE +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright 2020 Elastic and contributors - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/integration/logstash-plugins/logstash-output-vespa/README.md b/integration/logstash-plugins/logstash-output-vespa/README.md index f02f2ef5f9f..d12d4e18c06 100644 --- a/integration/logstash-plugins/logstash-output-vespa/README.md +++ b/integration/logstash-plugins/logstash-output-vespa/README.md @@ -61,7 +61,9 @@ filter { columns => ["id", "description", ...] } - # remove fields that we don't need. Here you can do a lot more processing + # remove fields we don't need + # NOTE: the fields below are added by Logstash by default. You probably *need* this block + # otherwise Vespa will reject documents complaining that e.g. @timestamp is an unknown field mutate { remove_field => ["@timestamp", "@version", "event", "host", "log", "message"] } diff --git a/integration/logstash-plugins/logstash-output-vespa/VERSION b/integration/logstash-plugins/logstash-output-vespa/VERSION index 8f0916f768f..4b9fcbec101 100644 --- a/integration/logstash-plugins/logstash-output-vespa/VERSION +++ b/integration/logstash-plugins/logstash-output-vespa/VERSION @@ -1 +1 @@ -0.5.0 +0.5.1 diff --git a/integration/logstash-plugins/logstash-output-vespa/gradle/wrapper/gradle-wrapper.jar b/integration/logstash-plugins/logstash-output-vespa/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 00000000000..a4b76b9530d Binary files /dev/null and b/integration/logstash-plugins/logstash-output-vespa/gradle/wrapper/gradle-wrapper.jar differ diff --git a/integration/logstash-plugins/logstash-output-vespa/gradle/wrapper/gradle-wrapper.properties b/integration/logstash-plugins/logstash-output-vespa/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 00000000000..e2847c82004 --- /dev/null +++ b/integration/logstash-plugins/logstash-output-vespa/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,7 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip +networkTimeout=10000 +validateDistributionUrl=true +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/integration/logstash-plugins/logstash-output-vespa/gradlew b/integration/logstash-plugins/logstash-output-vespa/gradlew index 1aa94a42690..f5feea6d6b1 100755 --- a/integration/logstash-plugins/logstash-output-vespa/gradlew +++ b/integration/logstash-plugins/logstash-output-vespa/gradlew @@ -15,6 +15,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# SPDX-License-Identifier: Apache-2.0 +# ############################################################################## # @@ -55,7 +57,7 @@ # Darwin, MinGW, and NonStop. # # (3) This script is generated from the Groovy template -# https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt # within the Gradle project. # # You can find Gradle at https://github.com/gradle/gradle/. @@ -84,7 +86,8 @@ done # shellcheck disable=SC2034 APP_BASE_NAME=${0##*/} # Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) -APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit +APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s +' "$PWD" ) || exit # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD=maximum diff --git a/integration/logstash-plugins/logstash-output-vespa/gradlew.bat b/integration/logstash-plugins/logstash-output-vespa/gradlew.bat index 7101f8e4676..9b42019c791 100644 --- a/integration/logstash-plugins/logstash-output-vespa/gradlew.bat +++ b/integration/logstash-plugins/logstash-output-vespa/gradlew.bat @@ -13,6 +13,8 @@ @rem See the License for the specific language governing permissions and @rem limitations under the License. @rem +@rem SPDX-License-Identifier: Apache-2.0 +@rem @if "%DEBUG%"=="" @echo off @rem ########################################################################## diff --git a/integration/logstash-plugins/logstash-output-vespa/lib/logstash-output-vespa_feed_jars.rb b/integration/logstash-plugins/logstash-output-vespa/lib/logstash-output-vespa_feed_jars.rb index 6fd058bfc2a..ee4a9170c64 100644 --- a/integration/logstash-plugins/logstash-output-vespa/lib/logstash-output-vespa_feed_jars.rb +++ b/integration/logstash-plugins/logstash-output-vespa/lib/logstash-output-vespa_feed_jars.rb @@ -2,4 +2,4 @@ # encoding: utf-8 require 'jar_dependencies' -require_jar('org.logstashplugins', 'logstash-output-vespa_feed', '0.5.0') +require_jar('org.logstashplugins', 'logstash-output-vespa_feed', '0.5.1') diff --git a/integration/logstash-plugins/logstash-output-vespa/src/main/java/org/logstashplugins/DynamicOption.java b/integration/logstash-plugins/logstash-output-vespa/src/main/java/org/logstashplugins/DynamicOption.java index b11aaedafa7..774f4033383 100644 --- a/integration/logstash-plugins/logstash-output-vespa/src/main/java/org/logstashplugins/DynamicOption.java +++ b/integration/logstash-plugins/logstash-output-vespa/src/main/java/org/logstashplugins/DynamicOption.java @@ -1,3 +1,5 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + package org.logstashplugins; import java.util.regex.Matcher; diff --git a/integration/logstash-plugins/logstash-output-vespa/src/main/java/org/logstashplugins/VespaFeed.java b/integration/logstash-plugins/logstash-output-vespa/src/main/java/org/logstashplugins/VespaFeed.java index 2c032958a8a..188e414e823 100644 --- a/integration/logstash-plugins/logstash-output-vespa/src/main/java/org/logstashplugins/VespaFeed.java +++ b/integration/logstash-plugins/logstash-output-vespa/src/main/java/org/logstashplugins/VespaFeed.java @@ -1,3 +1,5 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + package org.logstashplugins; import ai.vespa.feed.client.*; @@ -327,7 +329,7 @@ public void awaitStop() throws InterruptedException { @Override public Collection> configSchema() { return List.of(VESPA_URL, CLIENT_CERT, CLIENT_KEY, OPERATION, CREATE, NAMESPACE, DOCUMENT_TYPE, ID_FIELD, - MAX_CONNECTIONS, MAX_STREAMS, MAX_RETRIES, OPERATION_TIMEOUT); + MAX_CONNECTIONS, MAX_STREAMS, MAX_RETRIES, OPERATION_TIMEOUT, GRACE_PERIOD, DOOM_PERIOD); } @Override