Integration tests!

.travis.yml

@@ -10,6 +10,7 @@ before_install:
   - curl -L https://github.com/docker/compose/releases/download/1.11.2/docker-compose-`uname -s`-`uname -m` > docker-compose
   - chmod +x docker-compose
   - sudo mv docker-compose /usr/local/bin
-  
+
 script:
   - docker-compose build
+  - test/integration/run_tests.sh

Readme.md

@@ -4,28 +4,108 @@
 Dockerized for fun and profit.
 
 ## Prerequisites
-- [Docker](https://www.docker.com/)
+- Install [Docker](https://docs.docker.com/engine/installation/)
 
-## Setup
-- Create a `.env` file at the repo root with all the required secrets.
+## Quick start
+Create a `.env` file at the repo root and add all the required secrets.
 ```
-cp .env.template .env # now modify .env
+cp .env.template .env
 ```
-- Run:
+
+Start all the basic services
+
+**Note:** Depending on how you installed docker, you might have to run the docker commands through `sudo`
 ```
-docker-compose up
+docker-compose up --build -d
 ```
+
 Monitor the output to make sure nothing failed.
-- Restore from backup:
+```
+docker-compose logs -f
+```
+
+Now you need to initialise the database. Pick one of the following
+options.
+
+### Run the web installer
+
+Remove LocalSettings.php
+```
+docker-compose exec php rm /srv/mediawiki/LocalSettings.php
+```
+
+Go to http://localhost:8080 and complete the web installation. The
+database user is `metakgp_user` and the database host is
+`mysql-docker`. All the other configuration should be exactly the same
+as your `.env` file.
+
+After completing the installation, download the generated
+`LocalSettings.php` file and move it into place.
+```
+docker cp <path to downloaded LocalSettings.php> $(docker-compose ps -q php):/srv/mediawiki
+```
+
+Create the tables necessary for extensions.
+```
+docker-compose exec php php /srv/mediawiki/maintenance/update.php
+```
+
+Reload http://localhost:8080, you should see the main page.
+
+### Restore from backup
 ```
 ./scripts/restore-from-backup.sh <path to backup>
 ```
-- Go to localhost:8080 and gaze upon its wonder.
+
+Go to http://localhost:8080, you should see the main page.
 
 ## Development
-Make sure that your changes are actually being picked up. If you don't mind *deleting all docker volumes*, you can run `./scripts/clean-build.sh`.
+
+### Compose configuration
+`docker-compose` supports
+[multiple configuration files](https://docs.docker.com/compose/extends/#understanding-multiple-compose-files).
+`docker-compose.yml` is the base config, and
+`docker-compose.override.yml` is the default override. This is set up
+so that while developing, you can just use `docker-compose <command>`,
+and it will work.
+
+For production, we want to run some additional services (like backups),
+so we need to specify `docker-compose.prod.yml` as an _additional_
+override.
+
+For integration tests, we want to make sure that volumes created
+during integration tests don't overwrite volumes being used for
+development. We use `test/integration/docker-compose.test.yml` as the
+override instead.
+
+Overrides can be applied by using the `-f` option. See
+`test/integration/run_tests.sh` for an example.
+```
+docker-compose -f docker-compose.yml -f <another compose file>
+```
+
+### Volumes
+We use [Docker volumes](https://docs.docker.com/engine/tutorials/dockervolumes/)
+to persist data between container rebuilds (eg. mysql database), and
+to share data between containers (eg. mediawiki volume shared between
+nginx and php).
+
+When rebuilding/restarting containers, keep in mind that volumes are
+not automatically recreated. If a volume already exists, it will be
+attached to the new container. If you want a "clean" build, you need
+to make sure any existing volumes are removed.
+
+List volumes:
+```
+docker-compose volume ls
+```
+
+Remove the volumes you want to recreate:
+```
+docker-compose volume rm <volume name>
+```
 
 ## Todo
-- Enable remaining extensions
+- Enable VisualEditor
 - Restore images, peqp
 - Measure performance

docker-compose.override.yml

@@ -0,0 +1,21 @@
+version: '2.1'
+
+services:
+  mysql:
+    volumes:
+      - db-volume:/var/lib/mysql
+  nginx:
+    volumes:
+      - mediawiki-volume:/srv/mediawiki
+    ports:
+      - "${SERVER_PORT:-8080}:80"
+  php:
+    volumes:
+      - mediawiki-volume:/srv/mediawiki
+  mediawiki:
+    volumes:
+      - mediawiki-volume:/srv/mediawiki
+
+volumes:
+  mediawiki-volume:
+  db-volume:

docker-compose.prod.yml

@@ -0,0 +1,15 @@
+version: '2.1'
+
+services:
+  backup:
+    build: './backup'
+    links:
+      - mysql:mysql-docker
+    environment:
+      - MYSQL_PASSWORD=$MYSQL_PASSWORD
+      - DROPBOX_ACCESS_TOKEN=$DROPBOX_ACCESS_TOKEN
+    volumes:
+      - mediawiki-volume:/srv/mediawiki
+
+volumes:
+  mediawiki-volume:

docker-compose.yml

@@ -8,22 +8,14 @@ services:
       - MYSQL_DATABASE=metakgp_wiki_db
       - MYSQL_USER=metakgp_user
       - MYSQL_PASSWORD=${MYSQL_PASSWORD}
-    volumes:
-      - db-volume:/var/lib/mysql
   nginx:
     build: './nginx'
     links:
       - php:php-docker
-    volumes:
-      - mediawiki-volume:/srv/mediawiki
-    ports:
-      - "${SERVER_PORT:-8080}:80"
   php:
     build: './php'
     links:
       - mysql:mysql-docker
-    volumes:
-      - mediawiki-volume:/srv/mediawiki
     environment:
       - MAILGUN_PASSWORD=$MAILGUN_PASSWORD
       - RECAPTCHA_SECRET_KEY=$RECAPTCHA_SECRET_KEY
@@ -33,18 +25,3 @@ services:
       - SERVER_NAME=$SERVER_NAME
   mediawiki:
     build: './mediawiki'
-    volumes:
-      - mediawiki-volume:/srv/mediawiki
-  backup:
-    build: './backup'
-    links:
-      - mysql:mysql-docker
-    volumes:
-      - mediawiki-volume:/srv/mediawiki
-    environment:
-      - MYSQL_PASSWORD=$MYSQL_PASSWORD
-      - DROPBOX_ACCESS_TOKEN=$DROPBOX_ACCESS_TOKEN
-
-volumes:
-  mediawiki-volume:
-  db-volume:

test/integration/.env

@@ -0,0 +1,8 @@
+MYSQL_PASSWORD=db_password
+MAILGUN_PASSWORD=
+RECAPTCHA_SECRET_KEY=
+WG_SECRET_KEY=
+SITE_UPGRADE_KEY=
+SERVER_PORT=
+SERVER_NAME=
+DROPBOX_ACCESS_TOKEN=

test/integration/docker-compose.test.yml

@@ -0,0 +1,21 @@
+version: '2.1'
+
+services:
+  mysql:
+    volumes:
+      - db-test-volume:/var/lib/mysql
+  nginx:
+    volumes:
+      - mediawiki-test-volume:/srv/mediawiki
+    ports:
+      - "80"
+  php:
+    volumes:
+      - mediawiki-test-volume:/srv/mediawiki
+  mediawiki:
+    volumes:
+      - mediawiki-test-volume:/srv/mediawiki
+
+volumes:
+  mediawiki-test-volume:
+  db-test-volume:

test/integration/run_tests.sh

@@ -0,0 +1,98 @@
+#!/usr/bin/env bash
+
+RED='\033[1;31m'
+GREEN='\033[1;32m'
+YELLOW='\033[1;33m'
+NC='\033[0m' # No Color
+
+set -e
+
+REPO_ROOT=$(git rev-parse --show-toplevel)
+TEST_ROOT="$REPO_ROOT/test/integration"
+DOCKER_CONFIG="-f $REPO_ROOT/docker-compose.yml -f $TEST_ROOT/docker-compose.test.yml"
+DOCKER_COMPOSE="docker-compose $DOCKER_CONFIG"
+WIKI="/srv/mediawiki"
+
+cd $TEST_ROOT
+source .env
+
+function info {
+    echo -e "${YELLOW}$1${NC}"
+}
+
+function error {
+    echo -e "${RED}$1${NC}"
+    exit 1
+}
+
+function cleanup {
+    EXIT_CODE=$?
+
+    if [[ $EXIT_CODE != 0 && $($DOCKER_COMPOSE top) != "" ]]; then
+        info "Dumping logs"
+        $DOCKER_COMPOSE logs
+    fi
+
+    info "Cleaning up"
+    $DOCKER_COMPOSE down --volumes
+
+    if [[ $EXIT_CODE == 0 ]]; then
+        echo -e "${GREEN}All tests passed.${NC}"
+    else
+        echo -e "${RED}Test failure(s)!${NC}"
+    fi
+
+    exit $EXIT_CODE
+}
+
+info "Making sure no services are running"
+if [[ $($DOCKER_COMPOSE top) != "" ]]; then
+    error "Cannot run integration tests while services are running. Run 'docker-compose down' and try again."
+fi
+
+# About to start tests; make sure we clean up afterwards
+trap cleanup EXIT
+
+info "Starting integration test services"
+$DOCKER_COMPOSE up --build -d 1>/dev/null
+info "Waiting for mysql to initialize"
+for i in {1..24}; do
+    if [[ $($DOCKER_COMPOSE logs mysql | grep "ready for connections") != "" ]]; then
+        sleep 5 # to be extra-sure that mysql is ready
+        break
+    fi
+    sleep 5
+    if [[ $i == 24 ]]; then
+        error "mysql failed to initialise within 120 seconds"
+    fi
+done
+
+# Find the random port that nginx is mapped to
+NGINX_ADDR=$(docker-compose port nginx 80)
+
+info "Initializing database"
+# Move LocalSettings.php out of the way otherwise the installer complains
+$DOCKER_COMPOSE exec -T php mv $WIKI/LocalSettings.php $WIKI/LocalSettings.php.bak
+$DOCKER_COMPOSE exec -T php php $WIKI/maintenance/install.php \
+    --confpath /tmp \
+    --dbname metakgp_wiki_db \
+    --dbserver mysql-docker \
+    --dbuser metakgp_user \
+    --dbpass $MYSQL_PASSWORD \
+    --installdbuser metakgp_user \
+    --installdbpass $MYSQL_PASSWORD \
+    --pass admin_password \
+    --scriptpath "" \
+    Metakgp Test Wiki \
+    Admin
+
+# Move LocalSettings.php back in place
+$DOCKER_COMPOSE exec -T php mv $WIKI/LocalSettings.php.bak $WIKI/LocalSettings.php
+
+# Sample test only; more comprehensive tests to follow
+CURL_OUTPUT=$(curl -sSL $NGINX_ADDR)
+if [[ $CURL_OUTPUT != *"Powered by MediaWiki"* ]]; then
+    error "Main page failed to load properly: "$CURL_OUTPUT
+fi
+
+info "Tests complete"