diff --git a/.env b/.env index 4ec13bdfed..6d124ef31a 100644 --- a/.env +++ b/.env @@ -1 +1 @@ -ELK_VERSION=7.15.1 +ELK_VERSION=8.0.0-alpha2 diff --git a/.github/workflows/scripts/run-tests-core.sh b/.github/workflows/scripts/run-tests-core.sh index 5757d162ee..bbc9a7aa30 100755 --- a/.github/workflows/scripts/run-tests-core.sh +++ b/.github/workflows/scripts/run-tests-core.sh @@ -31,7 +31,7 @@ curl -X POST -D- "http://${ip_kb}:5601/api/saved_objects/index-pattern" \ -H 'Content-Type: application/json' \ -H "kbn-version: ${ELK_VERSION}" \ -u elastic:testpasswd \ - -d '{"attributes":{"title":"logstash-*","timeFieldName":"@timestamp"}}' + -d '{"attributes":{"title":"logs-generic-default","timeFieldName":"@timestamp"}}' log 'Searching index pattern via Kibana API' response="$(curl "http://${ip_kb}:5601/api/saved_objects/_find?type=index-pattern" -s -u elastic:testpasswd)" @@ -66,7 +66,7 @@ curl -X POST "http://${ip_es}:9200/_refresh" -u elastic:testpasswd \ -s -w '\n' log 'Searching message in Elasticsearch' -response="$(curl "http://${ip_es}:9200/logstash-*/_count?q=message:dockerelk&pretty" -s -u elastic:testpasswd)" +response="$(curl "http://${ip_es}:9200/logs-generic-default/_count?q=message:dockerelk&pretty" -s -u elastic:testpasswd)" echo "$response" count="$(jq -rn --argjson data "${response}" '$data.count')" if (( count != 1 )); then diff --git a/.github/workflows/scripts/run-tests-logspout.sh b/.github/workflows/scripts/run-tests-logspout.sh index caf62bcbe3..ea1748b110 100755 --- a/.github/workflows/scripts/run-tests-logspout.sh +++ b/.github/workflows/scripts/run-tests-logspout.sh @@ -39,7 +39,7 @@ declare -i was_retried=0 # retry for max 60s (30*2s) for _ in $(seq 1 30); do - response="$(curl "http://${ip_es}:9200/logstash-*/_search?q=docker.image:%22docker-elk_logspout%22%20AND%20message:%22logspout%20gliderlabs%22~3&pretty" -s -u elastic:testpasswd)" + response="$(curl "http://${ip_es}:9200/logs-generic-default/_search?q=docker.image:%22docker-elk_logspout%22%20AND%20message:%22logspout%20gliderlabs%22~3&pretty" -s -u elastic:testpasswd)" set +u # prevent "unbound variable" if assigned value is not an integer count="$(jq -rn --argjson data "${response}" '$data.hits.total.value')" diff --git a/.github/workflows/update.yml b/.github/workflows/update.yml index ed342524c5..633a7c4a03 100644 --- a/.github/workflows/update.yml +++ b/.github/workflows/update.yml @@ -12,13 +12,13 @@ jobs: strategy: matrix: release: + - 8.x - 7.x - - 6.x include: - - release: 7.x + - release: 8.x branch: main - - release: 6.x - branch: release-6.x + - release: 7.x + branch: release-7.x steps: - uses: actions/setup-node@v2 diff --git a/README.md b/README.md index 2cc3ab1891..667b1d6144 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Elastic stack (ELK) on Docker -[![Elastic Stack version](https://img.shields.io/badge/Elastic%20Stack-7.15.1-00bfb3?style=flat&logo=elastic-stack)](https://www.elastic.co/blog/category/releases) +[![Elastic Stack version](https://img.shields.io/badge/Elastic%20Stack-8.0.0--alpha2-00bfb3?style=flat&logo=elastic-stack)](https://www.elastic.co/blog/category/releases) [![Build Status](https://github.com/deviantony/docker-elk/workflows/CI/badge.svg?branch=main)](https://github.com/deviantony/docker-elk/actions?query=workflow%3ACI+branch%3Amain) [![Join the chat at https://gitter.im/deviantony/docker-elk](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/deviantony/docker-elk?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) @@ -22,7 +22,7 @@ Based on the official Docker images from Elastic: Other available stack variants: -* [`tls`](https://github.com/deviantony/docker-elk/tree/tls): TLS encryption enabled in Elasticsearch. +* [`tls`](https://github.com/deviantony/docker-elk/tree/tls): TLS encryption enabled in Elasticsearch * [`searchguard`](https://github.com/deviantony/docker-elk/tree/searchguard): Search Guard support --- @@ -123,7 +123,7 @@ instructions from the [documentation][mac-filesharing] to add more locations. ### Version selection This repository tries to stay aligned with the latest version of the Elastic stack. The `main` branch tracks the current -major version (7.x). +major version (8.x). To use a different version of the core Elastic components, simply change the version number inside the `.env` file. If you are upgrading an existing stack, please carefully read the note in the next section. @@ -133,6 +133,7 @@ performing a stack upgrade.** Older major versions are also supported on separate branches: +* [`release-7.x`](https://github.com/deviantony/docker-elk/tree/release-7.x): 7.x series * [`release-6.x`](https://github.com/deviantony/docker-elk/tree/release-6.x): 6.x series * [`release-5.x`](https://github.com/deviantony/docker-elk/tree/release-5.x): 5.x series (End-Of-Life) @@ -190,9 +191,8 @@ users][builtin-users] instead for increased security. 1. Replace usernames and passwords in configuration files - Use the `kibana_system` user (`kibana` for releases <7.8.0) inside the Kibana configuration file - (`kibana/config/kibana.yml`) and the `logstash_system` user inside the Logstash configuration file - (`logstash/config/logstash.yml`) in place of the existing `elastic` user. + Use the `kibana_system` user inside the Kibana configuration file (`kibana/config/kibana.yml`) in place of the + existing `elastic` user. Replace the password for the `elastic` user inside the Logstash pipeline file (`logstash/pipeline/logstash.conf`). @@ -244,8 +244,9 @@ When Kibana launches for the first time, it is not configured with any index pat the Kibana web UI.* Navigate to the _Discover_ view of Kibana from the left sidebar. You will be prompted to create an index pattern. Enter -`logstash-*` to match Logstash indices then, on the next page, select `@timestamp` as the time filter field. Finally, -click _Create index pattern_ and return to the _Discover_ view to inspect your log entries. +`logs-generic-default` to match the data stream backing Logstash indices then, on the next page, select `@timestamp` as +the time filter field. Finally, click _Create index pattern_ and return to the _Discover_ view to inspect your log +entries. Refer to [Connect Kibana with Elasticsearch][connect-kibana] and [Creating an index pattern][index-pattern] for detailed instructions about the index pattern configuration. @@ -257,9 +258,9 @@ Create an index pattern via the Kibana API: ```console $ curl -XPOST -D- 'http://localhost:5601/api/saved_objects/index-pattern' \ -H 'Content-Type: application/json' \ - -H 'kbn-version: 7.15.1' \ + -H 'kbn-version: 8.0.0-alpha2' \ -u elastic: \ - -d '{"attributes":{"title":"logstash-*","timeFieldName":"@timestamp"}}' + -d '{"attributes":{"title":"logs-generic-default","timeFieldName":"@timestamp"}}' ``` The created pattern will automatically be marked as the default index pattern as soon as the Kibana UI is opened for the diff --git a/docker-stack.yml b/docker-stack.yml index 9a40ee3f37..877e11e42a 100644 --- a/docker-stack.yml +++ b/docker-stack.yml @@ -3,7 +3,7 @@ version: '3.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:7.15.1 + image: docker.elastic.co/elasticsearch/elasticsearch:8.0.0-alpha2 ports: - "9200:9200" - "9300:9300" @@ -25,7 +25,7 @@ services: replicas: 1 logstash: - image: docker.elastic.co/logstash/logstash:7.15.1 + image: docker.elastic.co/logstash/logstash:8.0.0-alpha2 ports: - "5044:5044" - "5000:5000" @@ -44,7 +44,7 @@ services: replicas: 1 kibana: - image: docker.elastic.co/kibana/kibana:7.15.1 + image: docker.elastic.co/kibana/kibana:8.0.0-alpha2 ports: - "5601:5601" configs: diff --git a/extensions/enterprise-search/README.md b/extensions/enterprise-search/README.md index d055c8852f..71fd789dec 100644 --- a/extensions/enterprise-search/README.md +++ b/extensions/enterprise-search/README.md @@ -57,6 +57,17 @@ add the following setting: xpack.security.authc.api_key.enabled: true ``` +### Configure the Enterprise Search host in Kibana + +Kibana acts as the [management interface][enterprisesearch-ui] to Enterprise Search. + +To enable the management experience for Enterprise Search, modify the Kibana configuration file in +[`kibana/config/kibana.yml`][config-kbn] and add the following setting: + +```yaml +enterpriseSearch.host: http://enterprise-search:3002 +``` + ### Start the server To include Enterprise Search in the stack, run Docker Compose from the root of the repository with an additional command @@ -129,6 +140,8 @@ Docker container: [Running Enterprise Search Using Docker][enterprisesearch-dock [enterprisesearch-config]: https://www.elastic.co/guide/en/enterprise-search/current/configuration.html [enterprisesearch-docker]: https://www.elastic.co/guide/en/enterprise-search/current/docker.html [enterprisesearch-docs]: https://www.elastic.co/guide/en/enterprise-search/current/index.html +[enterprisesearch-ui]: https://www.elastic.co/guide/en/enterprise-search/current/user-interfaces.html [es-security]: https://www.elastic.co/guide/en/elasticsearch/reference/current/security-settings.html#api-key-service-settings [config-es]: ../../elasticsearch/config/elasticsearch.yml +[config-kbn]: ../../kibana/config/kibana.yml diff --git a/extensions/enterprise-search/config/enterprise-search.yml b/extensions/enterprise-search/config/enterprise-search.yml index 891b510a53..eb94457a72 100644 --- a/extensions/enterprise-search/config/enterprise-search.yml +++ b/extensions/enterprise-search/config/enterprise-search.yml @@ -15,8 +15,9 @@ secret_management.encryption_keys: # IP address Enterprise Search listens on ent_search.listen_host: 0.0.0.0 -# URL at which users reach Enterprise Search +# URL at which users reach Enterprise Search / Kibana ent_search.external_url: http://localhost:3002 +kibana.host: http://localhost:5601 # Elasticsearch URL and credentials elasticsearch.host: http://elasticsearch:9200 diff --git a/logstash/config/logstash.yml b/logstash/config/logstash.yml index a48c35ff58..47722ea7f3 100644 --- a/logstash/config/logstash.yml +++ b/logstash/config/logstash.yml @@ -3,10 +3,3 @@ ## https://github.com/elastic/logstash/blob/master/docker/data/logstash/config/logstash-full.yml # http.host: "0.0.0.0" -xpack.monitoring.elasticsearch.hosts: [ "http://elasticsearch:9200" ] - -## X-Pack security credentials -# -xpack.monitoring.enabled: true -xpack.monitoring.elasticsearch.username: elastic -xpack.monitoring.elasticsearch.password: changeme diff --git a/logstash/pipeline/logstash.conf b/logstash/pipeline/logstash.conf index 7d5918ba88..94ce71a6ce 100644 --- a/logstash/pipeline/logstash.conf +++ b/logstash/pipeline/logstash.conf @@ -10,11 +10,22 @@ input { ## Add your filters / logstash plugins configuration here +filter { + # Both the `beats` and `tcp` inputs inject a top-level [host] field, + # which string format is incompatible with the structured top-level + # [host] field reserved by the Elastic Common Schema (ECS). + # Ref. https://www.elastic.co/guide/en/ecs/current/ecs-host.html + if [host] and ![host][name] { + mutate { + rename => { "[host]" => "[host][name]" } + } + } +} + output { elasticsearch { hosts => "elasticsearch:9200" user => "elastic" password => "changeme" - ecs_compatibility => disabled } }