From c4d4a5ec4fd44921617ab64ffb075a48f8eb2671 Mon Sep 17 00:00:00 2001 From: Joshua Beretta Date: Thu, 1 Oct 2020 16:02:01 +0200 Subject: [PATCH 01/12] Adding support for wal-e. Changes made to config.py, template.py and primary/postgres.conf.tpl --- helpers/config.py | 64 +++++++++++++------ helpers/template.py | 1 + .../postgres/primary/postgres.conf.tpl | 4 ++ 3 files changed, 49 insertions(+), 20 deletions(-) diff --git a/helpers/config.py b/helpers/config.py index bcf078b..6a1a935 100644 --- a/helpers/config.py +++ b/helpers/config.py @@ -295,6 +295,7 @@ def get_config_template(cls): # we want to keep the same value when users upgrade. "enketo_less_secure_encryption_key": 'this $3cr3t key is crackable', "use_backup": Config.FALSE, + "use_wal_e": Config.FALSE, "kobocat_media_schedule": "0 0 * * 0", "mongo_backup_schedule": "0 1 * * 0", "postgres_backup_schedule": "0 2 * * 0", @@ -748,6 +749,27 @@ def __questions_backup(self): if self.backend_questions and not self.frontend_questions: self.__questions_aws() + # Prompting user whether they want to use WAL-E for + # continuous archiving + if self.primary_backend or not self.multi_servers: + CLI.colored_print( + "Do you want to use WAL-E for continuous archiving of backups?", + CLI.COLOR_SUCCESS) + CLI.colored_print("\t1) Yes") + CLI.colored_print("\t2) No") + self.__config["use_wal_e"] = CLI.get_response( + [Config.TRUE, Config.FALSE], + self.__config.get("use_wal_e", Config.FALSE)) + + if self.__config.get('use_wal_e') == Config.TRUE: + # Forcing postgres backup schedule and settings + self.__config["postgres_backup_schedule"] == '' + self.__config["postgres_settings"] = Config.TRUE + # Not entirely sure if this is what you were asking for + self.__config["backup_from_primary"] = Config.TRUE + + # ------------------------------------------------------------ # + schedule_regex_pattern = (r"^((((\d+(,\d+)*)|(\d+-\d+)|(\*(\/\d+)?)))" r"(\s+(((\d+(,\d+)*)|(\d+\-\d+)|(\*(\/\d+)?)))){4})$") CLI.colored_print("╔═════════════════════════════════════════════════════════════════╗", @@ -774,26 +796,28 @@ def __questions_backup(self): "0 0 * * 0")) if self.backend_questions: - - if self.primary_backend: - CLI.colored_print("Run PostgreSQL backup from primary backend server?", - CLI.COLOR_SUCCESS) - CLI.colored_print("\t1) Yes") - CLI.colored_print("\t2) No") - self.__config["backup_from_primary"] = CLI.get_response( - [Config.TRUE, Config.FALSE], - self.__config.get("backup_from_primary", Config.TRUE)) - - backup_from_primary = self.__config["backup_from_primary"] == Config.TRUE - if (not self.multi_servers or - (self.primary_backend and backup_from_primary) or - (self.secondary_backend and not backup_from_primary)): - CLI.colored_print("PostgreSQL backup schedule?", CLI.COLOR_SUCCESS) - self.__config["postgres_backup_schedule"] = CLI.get_response( - "~{}".format(schedule_regex_pattern), - self.__config.get( - "postgres_backup_schedule", - "0 2 * * 0")) + + # Only asking these Postgres config questions if WAL-E not being used + if self.__config['use_wal_e'] == Config.FALSE: + if self.primary_backend: + CLI.colored_print("Run PostgreSQL backup from primary backend server?", + CLI.COLOR_SUCCESS) + CLI.colored_print("\t1) Yes") + CLI.colored_print("\t2) No") + self.__config["backup_from_primary"] = CLI.get_response( + [Config.TRUE, Config.FALSE], + self.__config.get("backup_from_primary", Config.TRUE)) + + backup_from_primary = self.__config["backup_from_primary"] == Config.TRUE + if (not self.multi_servers or + (self.primary_backend and backup_from_primary) or + (self.secondary_backend and not backup_from_primary)): + CLI.colored_print("PostgreSQL backup schedule?", CLI.COLOR_SUCCESS) + self.__config["postgres_backup_schedule"] = CLI.get_response( + "~{}".format(schedule_regex_pattern), + self.__config.get( + "postgres_backup_schedule", + "0 2 * * 0")) if self.primary_backend or not self.multi_servers: diff --git a/helpers/template.py b/helpers/template.py index 098f080..f842b5b 100644 --- a/helpers/template.py +++ b/helpers/template.py @@ -215,6 +215,7 @@ def _get_value(property_, true_value="", false_value="#", "REDIS_MAIN_PORT": config.get("redis_main_port", "6739"), "REDIS_CACHE_PORT": config.get("redis_cache_port", "6380"), "USE_BACKUP": "" if config.get("use_backup") == Config.TRUE else "#", + "USE_WAL_E": "" if config.get("use_wal_e") == Config.TRUE else "#", "USE_AWS_BACKUP": "" if config_object.aws and config.get("use_backup") == Config.TRUE and config.get("aws_backup_bucket_name") != "" else "#", diff --git a/templates/kobo-env/postgres/primary/postgres.conf.tpl b/templates/kobo-env/postgres/primary/postgres.conf.tpl index f5078fb..7f3465e 100644 --- a/templates/kobo-env/postgres/primary/postgres.conf.tpl +++ b/templates/kobo-env/postgres/primary/postgres.conf.tpl @@ -14,3 +14,7 @@ # Total Memory (RAM): ${POSTGRES_RAM}GB ${POSTGRES_SETTINGS} + +${USE_WAL_E}archive_mode = on +${USE_WAL_E}archive_command = 'envdir /var/lib/postgresql/data/wal-e.d/env wal-e wal-push %p' +${USE_WAL_E}archive_timeout = 60 \ No newline at end of file From ca01d85baec81cba0a1ac35d179bd2457fb9b55d Mon Sep 17 00:00:00 2001 From: Joshua Beretta Date: Fri, 2 Oct 2020 11:30:57 +0200 Subject: [PATCH 02/12] Adjusting config.py to have default postgres_settings_content and updating the primary and secondary docker-compose templates to mount postgres.conf by default --- helpers/config.py | 45 ++++++++++++++++--- ...r-compose.backend.primary.override.yml.tpl | 4 +- ...compose.backend.secondary.override.yml.tpl | 4 +- 3 files changed, 44 insertions(+), 9 deletions(-) diff --git a/helpers/config.py b/helpers/config.py index 6a1a935..9e6f847 100644 --- a/helpers/config.py +++ b/helpers/config.py @@ -286,7 +286,23 @@ def get_config_template(cls): "postgres_profile": "Mixed", "postgres_max_connections": "100", "postgres_hard_drive_type": "hdd", - "postgres_settings_content": "", + "postgres_settings_content": "\n".join([ + "# Memory Configuration", + "shared_buffers = 512MB", + "effective_cache_size = 2GB", + "work_mem = 10MB", + "maintenance_work_mem = 128MB", + "", + "# Checkpoint Related Configuration", + "min_wal_size = 512MB", + "max_wal_size = 2GB", + "checkpoint_completion_target = 0.9", + "wal_buffers = 15MB", + "", + "# Network Related Configuration", + "listen_addresses = '*'", + "max_connections = 100", + ]), "custom_secret_keys": Config.FALSE, "enketo_api_token": binascii.hexlify(os.urandom(60)).decode("utf-8"), "enketo_encryption_key": binascii.hexlify(os.urandom(60)).decode("utf-8"), @@ -753,7 +769,7 @@ def __questions_backup(self): # continuous archiving if self.primary_backend or not self.multi_servers: CLI.colored_print( - "Do you want to use WAL-E for continuous archiving of backups?", + "Do you want to use WAL-E for continuous archiving of PostgreSQL backups?", CLI.COLOR_SUCCESS) CLI.colored_print("\t1) Yes") CLI.colored_print("\t2) No") @@ -764,12 +780,9 @@ def __questions_backup(self): if self.__config.get('use_wal_e') == Config.TRUE: # Forcing postgres backup schedule and settings self.__config["postgres_backup_schedule"] == '' - self.__config["postgres_settings"] = Config.TRUE # Not entirely sure if this is what you were asking for self.__config["backup_from_primary"] = Config.TRUE - # ------------------------------------------------------------ # - schedule_regex_pattern = (r"^((((\d+(,\d+)*)|(\d+-\d+)|(\*(\/\d+)?)))" r"(\s+(((\d+(,\d+)*)|(\d+\-\d+)|(\*(\/\d+)?)))){4})$") CLI.colored_print("╔═════════════════════════════════════════════════════════════════╗", @@ -1329,6 +1342,28 @@ def __questions_postgres(self): # Stop container docker_command = ['docker', 'stop', '-t', '0', 'pgconfig_container'] CLI.run_command(docker_command) + else: + # Forcing the default settings to remain even if there + # is an existing value in .run.conf. Without this, + # the value for `postgres_settings_content` would not update + default_postgres_settings_content = "\n".join([ + "# Memory Configuration", + "shared_buffers = 512MB", + "effective_cache_size = 2GB", + "work_mem = 10MB", + "maintenance_work_mem = 128MB", + "", + "# Checkpoint Related Configuration", + "min_wal_size = 512MB", + "max_wal_size = 2GB", + "checkpoint_completion_target = 0.9", + "wal_buffers = 15MB", + "", + "# Network Related Configuration", + "listen_addresses = '*'", + "max_connections = 100", + ]) + self.__config["postgres_settings_content"] = default_postgres_settings_content def __questions_ports(self): """ diff --git a/templates/kobo-docker/docker-compose.backend.primary.override.yml.tpl b/templates/kobo-docker/docker-compose.backend.primary.override.yml.tpl index be9ee07..f7a8277 100644 --- a/templates/kobo-docker/docker-compose.backend.primary.override.yml.tpl +++ b/templates/kobo-docker/docker-compose.backend.primary.override.yml.tpl @@ -4,8 +4,8 @@ version: '2.2' services: postgres: - ${OVERRIDE_POSTGRES_SETTINGS}volumes: - ${OVERRIDE_POSTGRES_SETTINGS} - ../kobo-env/postgres/primary/postgres.conf:/kobo-docker-scripts/primary/postgres.conf + volumes: + - ../kobo-env/postgres/primary/postgres.conf:/kobo-docker-scripts/primary/postgres.conf ${POSTGRES_BACKUP_FROM_SECONDARY}environment: ${POSTGRES_BACKUP_FROM_SECONDARY} - POSTGRES_BACKUP_FROM_SECONDARY=True ${EXPOSE_BACKEND_PORTS}ports: diff --git a/templates/kobo-docker/docker-compose.backend.secondary.override.yml.tpl b/templates/kobo-docker/docker-compose.backend.secondary.override.yml.tpl index 0560cde..d839393 100644 --- a/templates/kobo-docker/docker-compose.backend.secondary.override.yml.tpl +++ b/templates/kobo-docker/docker-compose.backend.secondary.override.yml.tpl @@ -6,8 +6,8 @@ services: extends: file: docker-compose.backend.template.yml service: postgres - ${OVERRIDE_POSTGRES_SETTINGS}volumes: - ${OVERRIDE_POSTGRES_SETTINGS} - ../kobo-env/postgres/secondary/postgres.conf:/kobo-docker-scripts/secondary/postgres.conf + volumes: + - ../kobo-env/postgres/secondary/postgres.conf:/kobo-docker-scripts/secondary/postgres.conf ports: - ${POSTGRES_PORT}:5432 ${ADD_BACKEND_EXTRA_HOSTS}extra_hosts: From 20d136edbef848121dd61bd6621fa2743c43891d Mon Sep 17 00:00:00 2001 From: Joshua Beretta Date: Fri, 2 Oct 2020 12:21:12 +0200 Subject: [PATCH 03/12] Added USE_WAL_E_BACKUP variable and changed function of USE_WAL_E --- helpers/template.py | 3 ++- templates/kobo-env/postgres/primary/postgres.conf.tpl | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/helpers/template.py b/helpers/template.py index f842b5b..db42b24 100644 --- a/helpers/template.py +++ b/helpers/template.py @@ -215,7 +215,8 @@ def _get_value(property_, true_value="", false_value="#", "REDIS_MAIN_PORT": config.get("redis_main_port", "6739"), "REDIS_CACHE_PORT": config.get("redis_cache_port", "6380"), "USE_BACKUP": "" if config.get("use_backup") == Config.TRUE else "#", - "USE_WAL_E": "" if config.get("use_wal_e") == Config.TRUE else "#", + "USE_WAL_E_BACKUP": "" if config.get("use_wal_e") == Config.TRUE else "#", + "USE_WAL_E": config.get("use_wal_e"), "USE_AWS_BACKUP": "" if config_object.aws and config.get("use_backup") == Config.TRUE and config.get("aws_backup_bucket_name") != "" else "#", diff --git a/templates/kobo-env/postgres/primary/postgres.conf.tpl b/templates/kobo-env/postgres/primary/postgres.conf.tpl index 7f3465e..11584dd 100644 --- a/templates/kobo-env/postgres/primary/postgres.conf.tpl +++ b/templates/kobo-env/postgres/primary/postgres.conf.tpl @@ -15,6 +15,6 @@ ${POSTGRES_SETTINGS} -${USE_WAL_E}archive_mode = on -${USE_WAL_E}archive_command = 'envdir /var/lib/postgresql/data/wal-e.d/env wal-e wal-push %p' -${USE_WAL_E}archive_timeout = 60 \ No newline at end of file +${USE_WAL_E_BACKUP}archive_mode = on +${USE_WAL_E_BACKUP}archive_command = 'envdir /var/lib/postgresql/data/wal-e.d/env wal-e wal-push %p' +${USE_WAL_E_BACKUP}archive_timeout = 60 \ No newline at end of file From 7d8eac1bece61db65090a397753f7cec130a74ad Mon Sep 17 00:00:00 2001 From: Joshua Beretta Date: Wed, 7 Oct 2020 17:22:00 +0200 Subject: [PATCH 04/12] using _get_value() and $PGDATA based on PR comments --- helpers/template.py | 4 +++- templates/kobo-env/postgres/primary/postgres.conf.tpl | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/helpers/template.py b/helpers/template.py index db42b24..a95a169 100644 --- a/helpers/template.py +++ b/helpers/template.py @@ -215,7 +215,9 @@ def _get_value(property_, true_value="", false_value="#", "REDIS_MAIN_PORT": config.get("redis_main_port", "6739"), "REDIS_CACHE_PORT": config.get("redis_cache_port", "6380"), "USE_BACKUP": "" if config.get("use_backup") == Config.TRUE else "#", - "USE_WAL_E_BACKUP": "" if config.get("use_wal_e") == Config.TRUE else "#", + "USE_WAL_E_BACKUP": _get_value("use_wal_e", true_value="", + false_value="#", + comparison_value=Config.TRUE), "USE_WAL_E": config.get("use_wal_e"), "USE_AWS_BACKUP": "" if config_object.aws and config.get("use_backup") == Config.TRUE and diff --git a/templates/kobo-env/postgres/primary/postgres.conf.tpl b/templates/kobo-env/postgres/primary/postgres.conf.tpl index 11584dd..d69d6cc 100644 --- a/templates/kobo-env/postgres/primary/postgres.conf.tpl +++ b/templates/kobo-env/postgres/primary/postgres.conf.tpl @@ -16,5 +16,5 @@ ${POSTGRES_SETTINGS} ${USE_WAL_E_BACKUP}archive_mode = on -${USE_WAL_E_BACKUP}archive_command = 'envdir /var/lib/postgresql/data/wal-e.d/env wal-e wal-push %p' +${USE_WAL_E_BACKUP}archive_command = 'envdir $$PGDATA/wal-e.d/env wal-e wal-push %p' ${USE_WAL_E_BACKUP}archive_timeout = 60 \ No newline at end of file From 2d023f4e423ebfd7602a6fff56994272664a0d93 Mon Sep 17 00:00:00 2001 From: Joshua Beretta Date: Fri, 9 Oct 2020 16:07:24 +0200 Subject: [PATCH 05/12] minor change to `USE_WAL_E_BACKUP` in helper/template.py from PR comments --- helpers/template.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/helpers/template.py b/helpers/template.py index a95a169..9b5f792 100644 --- a/helpers/template.py +++ b/helpers/template.py @@ -215,9 +215,7 @@ def _get_value(property_, true_value="", false_value="#", "REDIS_MAIN_PORT": config.get("redis_main_port", "6739"), "REDIS_CACHE_PORT": config.get("redis_cache_port", "6380"), "USE_BACKUP": "" if config.get("use_backup") == Config.TRUE else "#", - "USE_WAL_E_BACKUP": _get_value("use_wal_e", true_value="", - false_value="#", - comparison_value=Config.TRUE), + "USE_WAL_E_BACKUP": _get_value("use_wal_e"), "USE_WAL_E": config.get("use_wal_e"), "USE_AWS_BACKUP": "" if config_object.aws and config.get("use_backup") == Config.TRUE and From 8399714dd2f3c75005996ed0092bc8dc28a4761e Mon Sep 17 00:00:00 2001 From: Joshua Beretta Date: Wed, 14 Oct 2020 16:59:10 +0200 Subject: [PATCH 06/12] version bump to 3.4.0 and change kobo-docker branch in config --- helpers/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helpers/config.py b/helpers/config.py index 9e6f847..50c8f89 100644 --- a/helpers/config.py +++ b/helpers/config.py @@ -32,8 +32,8 @@ class Config(with_metaclass(Singleton)): DEFAULT_PROXY_PORT = "8080" DEFAULT_NGINX_PORT = "80" DEFAULT_NGINX_HTTPS_PORT = "443" - KOBO_DOCKER_BRANCH = '2.020.40a' - KOBO_INSTALL_VERSION = '3.3.0' + KOBO_DOCKER_BRANCH = 'add-wale-support' + KOBO_INSTALL_VERSION = '3.4.0' def __init__(self): self.__config = self.read_config() From 34c434d6bf9c95f9c5d59bde2a1f7dc4035d169d Mon Sep 17 00:00:00 2001 From: Joshua Beretta Date: Wed, 21 Oct 2020 15:02:12 -0700 Subject: [PATCH 07/12] Updating wal-e question to be dependant on aws response and roling back postgres cron dependance on wal-e response --- helpers/config.py | 73 +++++++++++++++++++++++------------------------ 1 file changed, 36 insertions(+), 37 deletions(-) diff --git a/helpers/config.py b/helpers/config.py index 50c8f89..1a25326 100644 --- a/helpers/config.py +++ b/helpers/config.py @@ -766,22 +766,23 @@ def __questions_backup(self): self.__questions_aws() # Prompting user whether they want to use WAL-E for - # continuous archiving - if self.primary_backend or not self.multi_servers: - CLI.colored_print( - "Do you want to use WAL-E for continuous archiving of PostgreSQL backups?", - CLI.COLOR_SUCCESS) - CLI.colored_print("\t1) Yes") - CLI.colored_print("\t2) No") - self.__config["use_wal_e"] = CLI.get_response( - [Config.TRUE, Config.FALSE], - self.__config.get("use_wal_e", Config.FALSE)) - - if self.__config.get('use_wal_e') == Config.TRUE: - # Forcing postgres backup schedule and settings - self.__config["postgres_backup_schedule"] == '' - # Not entirely sure if this is what you were asking for - self.__config["backup_from_primary"] = Config.TRUE + # continuous archiving - only if they are using aws for backups + if self.aws: + if self.primary_backend or not self.multi_servers: + CLI.colored_print( + "Do you want to use WAL-E for continuous archiving of PostgreSQL backups?", + CLI.COLOR_SUCCESS) + CLI.colored_print("\t1) Yes") + CLI.colored_print("\t2) No") + self.__config["use_wal_e"] = CLI.get_response( + [Config.TRUE, Config.FALSE], + self.__config.get("use_wal_e", Config.FALSE)) + + if self.__config.get('use_wal_e') == Config.TRUE: + # Forcing postgres backup schedule and settings + self.__config["postgres_backup_schedule"] == '' + # Not entirely sure if this is what you were asking for + self.__config["backup_from_primary"] = Config.TRUE schedule_regex_pattern = (r"^((((\d+(,\d+)*)|(\d+-\d+)|(\*(\/\d+)?)))" r"(\s+(((\d+(,\d+)*)|(\d+\-\d+)|(\*(\/\d+)?)))){4})$") @@ -810,27 +811,25 @@ def __questions_backup(self): if self.backend_questions: - # Only asking these Postgres config questions if WAL-E not being used - if self.__config['use_wal_e'] == Config.FALSE: - if self.primary_backend: - CLI.colored_print("Run PostgreSQL backup from primary backend server?", - CLI.COLOR_SUCCESS) - CLI.colored_print("\t1) Yes") - CLI.colored_print("\t2) No") - self.__config["backup_from_primary"] = CLI.get_response( - [Config.TRUE, Config.FALSE], - self.__config.get("backup_from_primary", Config.TRUE)) - - backup_from_primary = self.__config["backup_from_primary"] == Config.TRUE - if (not self.multi_servers or - (self.primary_backend and backup_from_primary) or - (self.secondary_backend and not backup_from_primary)): - CLI.colored_print("PostgreSQL backup schedule?", CLI.COLOR_SUCCESS) - self.__config["postgres_backup_schedule"] = CLI.get_response( - "~{}".format(schedule_regex_pattern), - self.__config.get( - "postgres_backup_schedule", - "0 2 * * 0")) + if self.primary_backend: + CLI.colored_print("Run PostgreSQL backup from primary backend server?", + CLI.COLOR_SUCCESS) + CLI.colored_print("\t1) Yes") + CLI.colored_print("\t2) No") + self.__config["backup_from_primary"] = CLI.get_response( + [Config.TRUE, Config.FALSE], + self.__config.get("backup_from_primary", Config.TRUE)) + + backup_from_primary = self.__config["backup_from_primary"] == Config.TRUE + if (not self.multi_servers or + (self.primary_backend and backup_from_primary) or + (self.secondary_backend and not backup_from_primary)): + CLI.colored_print("PostgreSQL backup schedule?", CLI.COLOR_SUCCESS) + self.__config["postgres_backup_schedule"] = CLI.get_response( + "~{}".format(schedule_regex_pattern), + self.__config.get( + "postgres_backup_schedule", + "0 2 * * 0")) if self.primary_backend or not self.multi_servers: From 308f9adb7376219db58b58b99630de7f5f2d7a09 Mon Sep 17 00:00:00 2001 From: Joshua Beretta Date: Wed, 21 Oct 2020 15:14:13 -0700 Subject: [PATCH 08/12] Updating readme with WAL-E --- readme.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/readme.md b/readme.md index 8e6295f..1a5c0a5 100644 --- a/readme.md +++ b/readme.md @@ -110,6 +110,7 @@ User can choose between 2 types of installations: |MongoDB user's password| **Autogenerate** | ✓ | ✓ | |Redis password4| **Autogenerate** | ✓ | ✓ | |Use AWS storage| **No** | ✓ | ✓ (frontend only) | +|Use WAL-E PostgreSQL backups5 | **No** | ✓ | ✓ (frontend only) | |uWGI workers| **start: 2, max: 4** | ✓ | ✓ (frontend only) | |uWGI memory limit| **128 MB** | ✓ | ✓ (frontend only) | |uWGI harakiri timeout | **120s** | ✓ | ✓ (frontend only) | @@ -130,6 +131,8 @@ User can choose between 2 types of installations: 4) _Redis password is optional but **strongly** recommended_ +5) _WAL-E backups for PostgreSQL are only available when using AWS storage_ + ℹ Intercom App ID [must now](https://github.com/kobotoolbox/kpi/pull/2285) be configured through "Per user settings" in the Django admin interface of KPI. ## Requirements From 3308fb735856d29003c92c4eb8afdb057a083523 Mon Sep 17 00:00:00 2001 From: Joshua Beretta Date: Wed, 28 Oct 2020 17:08:15 -0700 Subject: [PATCH 09/12] updaing config.py to correctly handle primary or secondary backend options with wal-e --- helpers/config.py | 27 +++++++++++++++++---------- readme.md | 8 +++++--- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/helpers/config.py b/helpers/config.py index 1a25326..be33e5e 100644 --- a/helpers/config.py +++ b/helpers/config.py @@ -770,8 +770,9 @@ def __questions_backup(self): if self.aws: if self.primary_backend or not self.multi_servers: CLI.colored_print( - "Do you want to use WAL-E for continuous archiving of PostgreSQL backups?", - CLI.COLOR_SUCCESS) + "Do you want to use WAL-E for continuous archiving of PostgreSQL backups?", + CLI.COLOR_SUCCESS, + ) CLI.colored_print("\t1) Yes") CLI.colored_print("\t2) No") self.__config["use_wal_e"] = CLI.get_response( @@ -811,14 +812,20 @@ def __questions_backup(self): if self.backend_questions: - if self.primary_backend: - CLI.colored_print("Run PostgreSQL backup from primary backend server?", - CLI.COLOR_SUCCESS) - CLI.colored_print("\t1) Yes") - CLI.colored_print("\t2) No") - self.__config["backup_from_primary"] = CLI.get_response( - [Config.TRUE, Config.FALSE], - self.__config.get("backup_from_primary", Config.TRUE)) + if self.__config['use_wal_e'] == Config.TRUE: + self.__config["backup_from_primary"] = Config.FALSE + else: + if self.primary_backend: + CLI.colored_print("Run PostgreSQL backup from primary backend server?", + CLI.COLOR_SUCCESS) + CLI.colored_print("\t1) Yes") + CLI.colored_print("\t2) No") + self.__config["backup_from_primary"] = CLI.get_response( + [Config.TRUE, Config.FALSE], + self.__config.get("backup_from_primary", Config.TRUE)) + else: + print('I got here') + self.__config["backup_from_primary"] = Config.FALSE backup_from_primary = self.__config["backup_from_primary"] == Config.TRUE if (not self.multi_servers or diff --git a/readme.md b/readme.md index 1a5c0a5..13479fb 100644 --- a/readme.md +++ b/readme.md @@ -109,8 +109,8 @@ User can choose between 2 types of installations: |MongoDB user's username| **kobo** | ✓ | ✓ | |MongoDB user's password| **Autogenerate** | ✓ | ✓ | |Redis password4| **Autogenerate** | ✓ | ✓ | -|Use AWS storage| **No** | ✓ | ✓ (frontend only) | -|Use WAL-E PostgreSQL backups5 | **No** | ✓ | ✓ (frontend only) | +|Use AWS storage5| **No** | ✓ | ✓ | +|Use WAL-E PostgreSQL backups6 | **No** | ✓ | ✓ (backend only) | |uWGI workers| **start: 2, max: 4** | ✓ | ✓ (frontend only) | |uWGI memory limit| **128 MB** | ✓ | ✓ (frontend only) | |uWGI harakiri timeout | **120s** | ✓ | ✓ (frontend only) | @@ -131,7 +131,9 @@ User can choose between 2 types of installations: 4) _Redis password is optional but **strongly** recommended_ -5) _WAL-E backups for PostgreSQL are only available when using AWS storage_ +5) _If AWS storage is selected, credentials must be provided if backups are activated_ + +6) _WAL-E backups for PostgreSQL are only available when using AWS storage_ ℹ Intercom App ID [must now](https://github.com/kobotoolbox/kpi/pull/2285) be configured through "Per user settings" in the Django admin interface of KPI. From 8ddf5df948eec5ba4bec79d1c5f0b3a9adbb027a Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Thu, 29 Oct 2020 14:42:47 -0400 Subject: [PATCH 10/12] Force WAL-E backup to False on secondary backend or on local storage --- helpers/config.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/helpers/config.py b/helpers/config.py index be33e5e..6b5454a 100644 --- a/helpers/config.py +++ b/helpers/config.py @@ -779,11 +779,14 @@ def __questions_backup(self): [Config.TRUE, Config.FALSE], self.__config.get("use_wal_e", Config.FALSE)) - if self.__config.get('use_wal_e') == Config.TRUE: - # Forcing postgres backup schedule and settings - self.__config["postgres_backup_schedule"] == '' - # Not entirely sure if this is what you were asking for - self.__config["backup_from_primary"] = Config.TRUE + if self.__config['use_wal_e'] == Config.TRUE: + self.__config['backup_from_primary'] = Config.TRUE + else: + # WAL-E cannot run on secondary + self.__config['use_wal_e'] = Config.FALSE + else: + # WAL-E is only supported with AWS + self.__config['use_wal_e'] = Config.FALSE schedule_regex_pattern = (r"^((((\d+(,\d+)*)|(\d+-\d+)|(\*(\/\d+)?)))" r"(\s+(((\d+(,\d+)*)|(\d+\-\d+)|(\*(\/\d+)?)))){4})$") @@ -811,20 +814,20 @@ def __questions_backup(self): "0 0 * * 0")) if self.backend_questions: - if self.__config['use_wal_e'] == Config.TRUE: - self.__config["backup_from_primary"] = Config.FALSE + self.__config['backup_from_primary'] = Config.TRUE else: if self.primary_backend: - CLI.colored_print("Run PostgreSQL backup from primary backend server?", - CLI.COLOR_SUCCESS) + CLI.colored_print( + "Run PostgreSQL backup from primary backend server?", + CLI.COLOR_SUCCESS + ) CLI.colored_print("\t1) Yes") CLI.colored_print("\t2) No") self.__config["backup_from_primary"] = CLI.get_response( [Config.TRUE, Config.FALSE], self.__config.get("backup_from_primary", Config.TRUE)) else: - print('I got here') self.__config["backup_from_primary"] = Config.FALSE backup_from_primary = self.__config["backup_from_primary"] == Config.TRUE From a6c46360699fe4db5932696582cadf8112cb01d7 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Thu, 29 Oct 2020 15:14:39 -0400 Subject: [PATCH 11/12] Pass "USE_WAL_E" to PostgreSQL container, remove redundant "USE_WAL_E_BACKUP" variable --- helpers/template.py | 3 +-- templates/kobo-env/envfiles/databases.txt.tpl | 3 +++ templates/kobo-env/postgres/primary/postgres.conf.tpl | 6 +++--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/helpers/template.py b/helpers/template.py index 9b5f792..3286e7a 100644 --- a/helpers/template.py +++ b/helpers/template.py @@ -215,8 +215,7 @@ def _get_value(property_, true_value="", false_value="#", "REDIS_MAIN_PORT": config.get("redis_main_port", "6739"), "REDIS_CACHE_PORT": config.get("redis_cache_port", "6380"), "USE_BACKUP": "" if config.get("use_backup") == Config.TRUE else "#", - "USE_WAL_E_BACKUP": _get_value("use_wal_e"), - "USE_WAL_E": config.get("use_wal_e"), + "USE_WAL_E": _get_value("use_wal_e"), "USE_AWS_BACKUP": "" if config_object.aws and config.get("use_backup") == Config.TRUE and config.get("aws_backup_bucket_name") != "" else "#", diff --git a/templates/kobo-env/envfiles/databases.txt.tpl b/templates/kobo-env/envfiles/databases.txt.tpl index 2206ef5..c04b565 100644 --- a/templates/kobo-env/envfiles/databases.txt.tpl +++ b/templates/kobo-env/envfiles/databases.txt.tpl @@ -42,6 +42,9 @@ KOBO_POSTGRES_PRIMARY_ENDPOINT=primary.postgres.${PRIVATE_DOMAIN_NAME} # Default Postgres backup schedule is weekly at 02:00 AM UTC on Sunday. ${USE_BACKUP}POSTGRES_BACKUP_SCHEDULE=${POSTGRES_BACKUP_SCHEDULE} +# WAL-E archiving and backup support +{USE_WAL_E_BACKUP}USE_WAL_E=1 + #-------------------------------------------------------------------------------- # REDIS #-------------------------------------------------------------------------------- diff --git a/templates/kobo-env/postgres/primary/postgres.conf.tpl b/templates/kobo-env/postgres/primary/postgres.conf.tpl index d69d6cc..141c8df 100644 --- a/templates/kobo-env/postgres/primary/postgres.conf.tpl +++ b/templates/kobo-env/postgres/primary/postgres.conf.tpl @@ -15,6 +15,6 @@ ${POSTGRES_SETTINGS} -${USE_WAL_E_BACKUP}archive_mode = on -${USE_WAL_E_BACKUP}archive_command = 'envdir $$PGDATA/wal-e.d/env wal-e wal-push %p' -${USE_WAL_E_BACKUP}archive_timeout = 60 \ No newline at end of file +${USE_WAL_E}archive_mode = on +${USE_WAL_E}archive_command = 'envdir $$PGDATA/wal-e.d/env wal-e wal-push %p' +${USE_WAL_E}archive_timeout = 60 From 70d83f3d7bca0f98913d7f3d45ecfc3a340c1d96 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Fri, 30 Oct 2020 12:20:24 -0400 Subject: [PATCH 12/12] Fixed typo in databases.txt to comment out (or not) "USE_WAL_E=1" --- templates/kobo-env/envfiles/databases.txt.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/kobo-env/envfiles/databases.txt.tpl b/templates/kobo-env/envfiles/databases.txt.tpl index c04b565..789bfa4 100644 --- a/templates/kobo-env/envfiles/databases.txt.tpl +++ b/templates/kobo-env/envfiles/databases.txt.tpl @@ -43,7 +43,7 @@ KOBO_POSTGRES_PRIMARY_ENDPOINT=primary.postgres.${PRIVATE_DOMAIN_NAME} ${USE_BACKUP}POSTGRES_BACKUP_SCHEDULE=${POSTGRES_BACKUP_SCHEDULE} # WAL-E archiving and backup support -{USE_WAL_E_BACKUP}USE_WAL_E=1 +${USE_WAL_E}USE_WAL_E=1 #-------------------------------------------------------------------------------- # REDIS