diff --git a/helpers/config.py b/helpers/config.py
index c73821e..6b5454a 100644
--- a/helpers/config.py
+++ b/helpers/config.py
@@ -32,8 +32,8 @@ class Config(with_metaclass(Singleton)):
DEFAULT_PROXY_PORT = "8080"
DEFAULT_NGINX_PORT = "80"
DEFAULT_NGINX_HTTPS_PORT = "443"
- KOBO_DOCKER_BRANCH = '2.020.44'
- KOBO_INSTALL_VERSION = '3.3.0'
+ KOBO_DOCKER_BRANCH = 'add-wale-support'
+ KOBO_INSTALL_VERSION = '3.4.0'
def __init__(self):
self.__config = self.read_config()
@@ -286,7 +286,23 @@ def get_config_template(cls):
"postgres_profile": "Mixed",
"postgres_max_connections": "100",
"postgres_hard_drive_type": "hdd",
- "postgres_settings_content": "",
+ "postgres_settings_content": "\n".join([
+ "# Memory Configuration",
+ "shared_buffers = 512MB",
+ "effective_cache_size = 2GB",
+ "work_mem = 10MB",
+ "maintenance_work_mem = 128MB",
+ "",
+ "# Checkpoint Related Configuration",
+ "min_wal_size = 512MB",
+ "max_wal_size = 2GB",
+ "checkpoint_completion_target = 0.9",
+ "wal_buffers = 15MB",
+ "",
+ "# Network Related Configuration",
+ "listen_addresses = '*'",
+ "max_connections = 100",
+ ]),
"custom_secret_keys": Config.FALSE,
"enketo_api_token": binascii.hexlify(os.urandom(60)).decode("utf-8"),
"enketo_encryption_key": binascii.hexlify(os.urandom(60)).decode("utf-8"),
@@ -295,6 +311,7 @@ def get_config_template(cls):
# we want to keep the same value when users upgrade.
"enketo_less_secure_encryption_key": 'this $3cr3t key is crackable',
"use_backup": Config.FALSE,
+ "use_wal_e": Config.FALSE,
"kobocat_media_schedule": "0 0 * * 0",
"mongo_backup_schedule": "0 1 * * 0",
"postgres_backup_schedule": "0 2 * * 0",
@@ -748,6 +765,29 @@ def __questions_backup(self):
if self.backend_questions and not self.frontend_questions:
self.__questions_aws()
+ # Prompting user whether they want to use WAL-E for
+ # continuous archiving - only if they are using aws for backups
+ if self.aws:
+ if self.primary_backend or not self.multi_servers:
+ CLI.colored_print(
+ "Do you want to use WAL-E for continuous archiving of PostgreSQL backups?",
+ CLI.COLOR_SUCCESS,
+ )
+ CLI.colored_print("\t1) Yes")
+ CLI.colored_print("\t2) No")
+ self.__config["use_wal_e"] = CLI.get_response(
+ [Config.TRUE, Config.FALSE],
+ self.__config.get("use_wal_e", Config.FALSE))
+
+ if self.__config['use_wal_e'] == Config.TRUE:
+ self.__config['backup_from_primary'] = Config.TRUE
+ else:
+ # WAL-E cannot run on secondary
+ self.__config['use_wal_e'] = Config.FALSE
+ else:
+ # WAL-E is only supported with AWS
+ self.__config['use_wal_e'] = Config.FALSE
+
schedule_regex_pattern = (r"^((((\d+(,\d+)*)|(\d+-\d+)|(\*(\/\d+)?)))"
r"(\s+(((\d+(,\d+)*)|(\d+\-\d+)|(\*(\/\d+)?)))){4})$")
CLI.colored_print("╔═════════════════════════════════════════════════════════════════╗",
@@ -774,15 +814,21 @@ def __questions_backup(self):
"0 0 * * 0"))
if self.backend_questions:
-
- if self.primary_backend:
- CLI.colored_print("Run PostgreSQL backup from primary backend server?",
- CLI.COLOR_SUCCESS)
- CLI.colored_print("\t1) Yes")
- CLI.colored_print("\t2) No")
- self.__config["backup_from_primary"] = CLI.get_response(
- [Config.TRUE, Config.FALSE],
- self.__config.get("backup_from_primary", Config.TRUE))
+ if self.__config['use_wal_e'] == Config.TRUE:
+ self.__config['backup_from_primary'] = Config.TRUE
+ else:
+ if self.primary_backend:
+ CLI.colored_print(
+ "Run PostgreSQL backup from primary backend server?",
+ CLI.COLOR_SUCCESS
+ )
+ CLI.colored_print("\t1) Yes")
+ CLI.colored_print("\t2) No")
+ self.__config["backup_from_primary"] = CLI.get_response(
+ [Config.TRUE, Config.FALSE],
+ self.__config.get("backup_from_primary", Config.TRUE))
+ else:
+ self.__config["backup_from_primary"] = Config.FALSE
backup_from_primary = self.__config["backup_from_primary"] == Config.TRUE
if (not self.multi_servers or
@@ -1305,6 +1351,28 @@ def __questions_postgres(self):
# Stop container
docker_command = ['docker', 'stop', '-t', '0', 'pgconfig_container']
CLI.run_command(docker_command)
+ else:
+ # Forcing the default settings to remain even if there
+ # is an existing value in .run.conf. Without this,
+ # the value for `postgres_settings_content` would not update
+ default_postgres_settings_content = "\n".join([
+ "# Memory Configuration",
+ "shared_buffers = 512MB",
+ "effective_cache_size = 2GB",
+ "work_mem = 10MB",
+ "maintenance_work_mem = 128MB",
+ "",
+ "# Checkpoint Related Configuration",
+ "min_wal_size = 512MB",
+ "max_wal_size = 2GB",
+ "checkpoint_completion_target = 0.9",
+ "wal_buffers = 15MB",
+ "",
+ "# Network Related Configuration",
+ "listen_addresses = '*'",
+ "max_connections = 100",
+ ])
+ self.__config["postgres_settings_content"] = default_postgres_settings_content
def __questions_ports(self):
"""
diff --git a/helpers/template.py b/helpers/template.py
index 098f080..3286e7a 100644
--- a/helpers/template.py
+++ b/helpers/template.py
@@ -215,6 +215,7 @@ def _get_value(property_, true_value="", false_value="#",
"REDIS_MAIN_PORT": config.get("redis_main_port", "6739"),
"REDIS_CACHE_PORT": config.get("redis_cache_port", "6380"),
"USE_BACKUP": "" if config.get("use_backup") == Config.TRUE else "#",
+ "USE_WAL_E": _get_value("use_wal_e"),
"USE_AWS_BACKUP": "" if config_object.aws and
config.get("use_backup") == Config.TRUE and
config.get("aws_backup_bucket_name") != "" else "#",
diff --git a/readme.md b/readme.md
index 8e6295f..13479fb 100644
--- a/readme.md
+++ b/readme.md
@@ -109,7 +109,8 @@ User can choose between 2 types of installations:
|MongoDB user's username| **kobo** | ✓ | ✓ |
|MongoDB user's password| **Autogenerate** | ✓ | ✓ |
|Redis password4| **Autogenerate** | ✓ | ✓ |
-|Use AWS storage| **No** | ✓ | ✓ (frontend only) |
+|Use AWS storage5| **No** | ✓ | ✓ |
+|Use WAL-E PostgreSQL backups6 | **No** | ✓ | ✓ (backend only) |
|uWGI workers| **start: 2, max: 4** | ✓ | ✓ (frontend only) |
|uWGI memory limit| **128 MB** | ✓ | ✓ (frontend only) |
|uWGI harakiri timeout | **120s** | ✓ | ✓ (frontend only) |
@@ -130,6 +131,10 @@ User can choose between 2 types of installations:
4) _Redis password is optional but **strongly** recommended_
+5) _If AWS storage is selected, credentials must be provided if backups are activated_
+
+6) _WAL-E backups for PostgreSQL are only available when using AWS storage_
+
ℹ Intercom App ID [must now](https://github.com/kobotoolbox/kpi/pull/2285) be configured through "Per user settings" in the Django admin interface of KPI.
## Requirements
diff --git a/templates/kobo-docker/docker-compose.backend.primary.override.yml.tpl b/templates/kobo-docker/docker-compose.backend.primary.override.yml.tpl
index be9ee07..f7a8277 100644
--- a/templates/kobo-docker/docker-compose.backend.primary.override.yml.tpl
+++ b/templates/kobo-docker/docker-compose.backend.primary.override.yml.tpl
@@ -4,8 +4,8 @@ version: '2.2'
services:
postgres:
- ${OVERRIDE_POSTGRES_SETTINGS}volumes:
- ${OVERRIDE_POSTGRES_SETTINGS} - ../kobo-env/postgres/primary/postgres.conf:/kobo-docker-scripts/primary/postgres.conf
+ volumes:
+ - ../kobo-env/postgres/primary/postgres.conf:/kobo-docker-scripts/primary/postgres.conf
${POSTGRES_BACKUP_FROM_SECONDARY}environment:
${POSTGRES_BACKUP_FROM_SECONDARY} - POSTGRES_BACKUP_FROM_SECONDARY=True
${EXPOSE_BACKEND_PORTS}ports:
diff --git a/templates/kobo-docker/docker-compose.backend.secondary.override.yml.tpl b/templates/kobo-docker/docker-compose.backend.secondary.override.yml.tpl
index 0560cde..d839393 100644
--- a/templates/kobo-docker/docker-compose.backend.secondary.override.yml.tpl
+++ b/templates/kobo-docker/docker-compose.backend.secondary.override.yml.tpl
@@ -6,8 +6,8 @@ services:
extends:
file: docker-compose.backend.template.yml
service: postgres
- ${OVERRIDE_POSTGRES_SETTINGS}volumes:
- ${OVERRIDE_POSTGRES_SETTINGS} - ../kobo-env/postgres/secondary/postgres.conf:/kobo-docker-scripts/secondary/postgres.conf
+ volumes:
+ - ../kobo-env/postgres/secondary/postgres.conf:/kobo-docker-scripts/secondary/postgres.conf
ports:
- ${POSTGRES_PORT}:5432
${ADD_BACKEND_EXTRA_HOSTS}extra_hosts:
diff --git a/templates/kobo-env/envfiles/databases.txt.tpl b/templates/kobo-env/envfiles/databases.txt.tpl
index 2206ef5..789bfa4 100644
--- a/templates/kobo-env/envfiles/databases.txt.tpl
+++ b/templates/kobo-env/envfiles/databases.txt.tpl
@@ -42,6 +42,9 @@ KOBO_POSTGRES_PRIMARY_ENDPOINT=primary.postgres.${PRIVATE_DOMAIN_NAME}
# Default Postgres backup schedule is weekly at 02:00 AM UTC on Sunday.
${USE_BACKUP}POSTGRES_BACKUP_SCHEDULE=${POSTGRES_BACKUP_SCHEDULE}
+# WAL-E archiving and backup support
+${USE_WAL_E}USE_WAL_E=1
+
#--------------------------------------------------------------------------------
# REDIS
#--------------------------------------------------------------------------------
diff --git a/templates/kobo-env/postgres/primary/postgres.conf.tpl b/templates/kobo-env/postgres/primary/postgres.conf.tpl
index f5078fb..141c8df 100644
--- a/templates/kobo-env/postgres/primary/postgres.conf.tpl
+++ b/templates/kobo-env/postgres/primary/postgres.conf.tpl
@@ -14,3 +14,7 @@
# Total Memory (RAM): ${POSTGRES_RAM}GB
${POSTGRES_SETTINGS}
+
+${USE_WAL_E}archive_mode = on
+${USE_WAL_E}archive_command = 'envdir $$PGDATA/wal-e.d/env wal-e wal-push %p'
+${USE_WAL_E}archive_timeout = 60