Merge pull request #122 from kobotoolbox/add-wal-e-support

Add wal e support

helpers/config.py

@@ -32,8 +32,8 @@ class Config(with_metaclass(Singleton)):
     DEFAULT_PROXY_PORT = "8080"
     DEFAULT_NGINX_PORT = "80"
     DEFAULT_NGINX_HTTPS_PORT = "443"
-    KOBO_DOCKER_BRANCH = '2.020.44'
-    KOBO_INSTALL_VERSION = '3.3.0'
+    KOBO_DOCKER_BRANCH = 'add-wale-support'
+    KOBO_INSTALL_VERSION = '3.4.0'
 
     def __init__(self):
         self.__config = self.read_config()
@@ -286,7 +286,23 @@ def get_config_template(cls):
             "postgres_profile": "Mixed",
             "postgres_max_connections": "100",
             "postgres_hard_drive_type": "hdd",
-            "postgres_settings_content": "",
+            "postgres_settings_content": "\n".join([
+                "# Memory Configuration",
+                "shared_buffers = 512MB",
+                "effective_cache_size = 2GB",
+                "work_mem = 10MB",
+                "maintenance_work_mem = 128MB",
+                "",
+                "# Checkpoint Related Configuration",
+                "min_wal_size = 512MB",
+                "max_wal_size = 2GB",
+                "checkpoint_completion_target = 0.9",
+                "wal_buffers = 15MB",
+                "",
+                "# Network Related Configuration",
+                "listen_addresses = '*'",
+                "max_connections = 100",
+            ]),
             "custom_secret_keys": Config.FALSE,
             "enketo_api_token": binascii.hexlify(os.urandom(60)).decode("utf-8"),
             "enketo_encryption_key": binascii.hexlify(os.urandom(60)).decode("utf-8"),
@@ -295,6 +311,7 @@ def get_config_template(cls):
             # we want to keep the same value when users upgrade.
             "enketo_less_secure_encryption_key": 'this $3cr3t key is crackable',
             "use_backup": Config.FALSE,
+            "use_wal_e": Config.FALSE,
             "kobocat_media_schedule": "0 0 * * 0",
             "mongo_backup_schedule": "0 1 * * 0",
             "postgres_backup_schedule": "0 2 * * 0",
@@ -748,6 +765,29 @@ def __questions_backup(self):
                     if self.backend_questions and not self.frontend_questions:
                         self.__questions_aws()
 
+                    # Prompting user whether they want to use WAL-E for
+                    # continuous archiving - only if they are using aws for backups
+                    if self.aws:
+                        if self.primary_backend or not self.multi_servers:
+                            CLI.colored_print(
+                                "Do you want to use WAL-E for continuous archiving of PostgreSQL backups?",
+                                CLI.COLOR_SUCCESS,
+                            )
+                            CLI.colored_print("\t1) Yes")
+                            CLI.colored_print("\t2) No")
+                            self.__config["use_wal_e"] = CLI.get_response(
+                                [Config.TRUE, Config.FALSE],
+                                self.__config.get("use_wal_e", Config.FALSE))
+
+                            if self.__config['use_wal_e'] == Config.TRUE:
+                                self.__config['backup_from_primary'] = Config.TRUE
+                        else:
+                            # WAL-E cannot run on secondary
+                            self.__config['use_wal_e'] = Config.FALSE
+                    else:
+                        # WAL-E is only supported with AWS
+                        self.__config['use_wal_e'] = Config.FALSE
+
                     schedule_regex_pattern = (r"^((((\d+(,\d+)*)|(\d+-\d+)|(\*(\/\d+)?)))"
                                               r"(\s+(((\d+(,\d+)*)|(\d+\-\d+)|(\*(\/\d+)?)))){4})$")
                     CLI.colored_print("╔═════════════════════════════════════════════════════════════════╗",
@@ -774,15 +814,21 @@ def __questions_backup(self):
                                 "0 0 * * 0"))
 
                     if self.backend_questions:
-
-                        if self.primary_backend:
-                            CLI.colored_print("Run PostgreSQL backup from primary backend server?",
-                                              CLI.COLOR_SUCCESS)
-                            CLI.colored_print("\t1) Yes")
-                            CLI.colored_print("\t2) No")
-                            self.__config["backup_from_primary"] = CLI.get_response(
-                                [Config.TRUE, Config.FALSE],
-                                self.__config.get("backup_from_primary", Config.TRUE))
+                        if self.__config['use_wal_e'] == Config.TRUE:
+                            self.__config['backup_from_primary'] = Config.TRUE
+                        else:
+                            if self.primary_backend:
+                                CLI.colored_print(
+                                    "Run PostgreSQL backup from primary backend server?",
+                                    CLI.COLOR_SUCCESS
+                                )
+                                CLI.colored_print("\t1) Yes")
+                                CLI.colored_print("\t2) No")
+                                self.__config["backup_from_primary"] = CLI.get_response(
+                                    [Config.TRUE, Config.FALSE],
+                                    self.__config.get("backup_from_primary", Config.TRUE))
+                            else:
+                                self.__config["backup_from_primary"] = Config.FALSE
 
                         backup_from_primary = self.__config["backup_from_primary"] == Config.TRUE
                         if (not self.multi_servers or
@@ -1305,6 +1351,28 @@ def __questions_postgres(self):
                 # Stop container
                 docker_command = ['docker', 'stop', '-t', '0', 'pgconfig_container']
                 CLI.run_command(docker_command)
+            else:
+                # Forcing the default settings to remain even if there
+                # is an existing value in .run.conf. Without this,
+                # the value for `postgres_settings_content` would not update
+                default_postgres_settings_content = "\n".join([
+                                                        "# Memory Configuration",
+                                                        "shared_buffers = 512MB",
+                                                        "effective_cache_size = 2GB",
+                                                        "work_mem = 10MB",
+                                                        "maintenance_work_mem = 128MB",
+                                                        "",
+                                                        "# Checkpoint Related Configuration",
+                                                        "min_wal_size = 512MB",
+                                                        "max_wal_size = 2GB",
+                                                        "checkpoint_completion_target = 0.9",
+                                                        "wal_buffers = 15MB",
+                                                        "",
+                                                        "# Network Related Configuration",
+                                                        "listen_addresses = '*'",
+                                                        "max_connections = 100",
+                                                    ])
+                self.__config["postgres_settings_content"] = default_postgres_settings_content
 
     def __questions_ports(self):
         """

helpers/template.py

@@ -215,6 +215,7 @@ def _get_value(property_, true_value="", false_value="#",
             "REDIS_MAIN_PORT": config.get("redis_main_port", "6739"),
             "REDIS_CACHE_PORT": config.get("redis_cache_port", "6380"),
             "USE_BACKUP": "" if config.get("use_backup") == Config.TRUE else "#",
+            "USE_WAL_E": _get_value("use_wal_e"),
             "USE_AWS_BACKUP": "" if config_object.aws and
                                     config.get("use_backup") == Config.TRUE and
                                     config.get("aws_backup_bucket_name") != "" else "#",

readme.md

@@ -109,7 +109,8 @@ User can choose between 2 types of installations:
 |MongoDB user's username|  **kobo**  | ✓ | ✓ |
 |MongoDB user's password|  **Autogenerate**  | ✓ | ✓ |
 |Redis password<sup>4</sup>|  **Autogenerate**  | ✓ | ✓ |
-|Use AWS storage|  **No**  | ✓ | ✓ (frontend only) |
+|Use AWS storage<sup>5</sup>|  **No**  | ✓ | ✓ |
+|Use WAL-E PostgreSQL backups<sup>6</sup> |  **No**  | ✓ | ✓ (backend only) |
 |uWGI workers|  **start: 2, max: 4**  | ✓ | ✓ (frontend only) |
 |uWGI memory limit|  **128 MB**  | ✓ | ✓ (frontend only) |
 |uWGI harakiri timeout |  **120s**  | ✓ | ✓ (frontend only) |
@@ -130,6 +131,10 @@ User can choose between 2 types of installations:
 
 <sup>4)</sup> _Redis password is optional but **strongly** recommended_
 
+<sup>5)</sup> _If AWS storage is selected, credentials must be provided if backups are activated_
+
+<sup>6)</sup> _WAL-E backups for PostgreSQL are only available when using AWS storage_
+
 ℹ  Intercom App ID [must now](https://github.com/kobotoolbox/kpi/pull/2285) be configured through "Per user settings" in the Django admin interface of KPI.
 
 ## Requirements

templates/kobo-docker/docker-compose.backend.primary.override.yml.tpl

@@ -4,8 +4,8 @@ version: '2.2'
 services:
 
   postgres:
-    ${OVERRIDE_POSTGRES_SETTINGS}volumes:
-    ${OVERRIDE_POSTGRES_SETTINGS}  - ../kobo-env/postgres/primary/postgres.conf:/kobo-docker-scripts/primary/postgres.conf
+    volumes:
+      - ../kobo-env/postgres/primary/postgres.conf:/kobo-docker-scripts/primary/postgres.conf
     ${POSTGRES_BACKUP_FROM_SECONDARY}environment:
     ${POSTGRES_BACKUP_FROM_SECONDARY}  - POSTGRES_BACKUP_FROM_SECONDARY=True
     ${EXPOSE_BACKEND_PORTS}ports:

templates/kobo-docker/docker-compose.backend.secondary.override.yml.tpl

@@ -6,8 +6,8 @@ services:
     extends:
       file: docker-compose.backend.template.yml
       service: postgres
-    ${OVERRIDE_POSTGRES_SETTINGS}volumes:
-    ${OVERRIDE_POSTGRES_SETTINGS}  - ../kobo-env/postgres/secondary/postgres.conf:/kobo-docker-scripts/secondary/postgres.conf
+    volumes:
+      - ../kobo-env/postgres/secondary/postgres.conf:/kobo-docker-scripts/secondary/postgres.conf
     ports:
       - ${POSTGRES_PORT}:5432
     ${ADD_BACKEND_EXTRA_HOSTS}extra_hosts:

templates/kobo-env/envfiles/databases.txt.tpl

@@ -42,6 +42,9 @@ KOBO_POSTGRES_PRIMARY_ENDPOINT=primary.postgres.${PRIVATE_DOMAIN_NAME}
 # Default Postgres backup schedule is weekly at 02:00 AM UTC on Sunday.
 ${USE_BACKUP}POSTGRES_BACKUP_SCHEDULE=${POSTGRES_BACKUP_SCHEDULE}
 
+# WAL-E archiving and backup support
+${USE_WAL_E}USE_WAL_E=1
+
 #--------------------------------------------------------------------------------
 # REDIS
 #--------------------------------------------------------------------------------

templates/kobo-env/postgres/primary/postgres.conf.tpl

@@ -14,3 +14,7 @@
 # Total Memory (RAM): ${POSTGRES_RAM}GB
 
 ${POSTGRES_SETTINGS}
+
+${USE_WAL_E}archive_mode = on
+${USE_WAL_E}archive_command = 'envdir $$PGDATA/wal-e.d/env wal-e wal-push %p'
+${USE_WAL_E}archive_timeout = 60