Merge pull request #69 from TACC/sharrell-docker

Basic docker-compose integration

.gitignore

@@ -1,3 +1,4 @@
+services-conf/nginx.conf
 dist/
 rabbitmq-c/
 tacc_stats.egg-info/
@@ -27,6 +28,7 @@ tacc_stats/version.py
 tacc_stats/cfg.py
 *egg*
 #*
+*.swp
 tacc_stats/taccstats
 Makefile.in
 /ar-lib
@@ -50,4 +52,4 @@ Makefile.in
 /stamp-h1
 /ltmain.sh
 /texinfo.tex
-tacc_stats/pickler/*lock
+tacc_stats/pickler/*lock

Dockerfile

@@ -0,0 +1,31 @@
+# pull official base image
+FROM python:3.6.15
+
+RUN useradd -ms /bin/bash hpcperfstats
+WORKDIR /home/hpcperfstats
+
+# run as root
+RUN apt-get update && apt-get upgrade -y
+RUN apt-get install netcat supervisor -y
+
+
+ENV PYTHONDONTWRITEBYTECODE 1
+ENV PYTHONUNBUFFERED 1
+ENV PATH $PATH:/home/hpcperfstats/.local/bin
+
+# install dependencies
+RUN pip install --upgrade pip
+COPY --chown=hpcperfstats:hpcperfstats ./requirements.txt .
+RUN pip install -r requirements.txt
+
+
+# copy project
+COPY --chown=hpcperfstats:hpcperfstats . .
+# This includes the tacc_stats.ini
+#COPY --chown=hpcperfstats:hpcperfstats ./tacc_stats.ini .
+
+
+RUN pip install .
+
+ADD services-conf/supervisord.conf /etc/supervisor/conf.d/supervisord.conf
+

docker-compose.yml

@@ -0,0 +1,61 @@
+services:
+  web:
+    build: ./
+    image: hpcperfstats
+    command: /home/hpcperfstats/services-conf/django_startup.sh
+    env_file:
+      - ./.env.dev
+
+  pipeline:
+    image: hpcperfstats
+    depends_on:
+      - web
+    command: /home/hpcperfstats/services-conf/supervisor_startup.sh
+    volumes:
+      - hpcperfstatsdata:/hpcperfstats/
+    env_file:
+      - ./.env.dev
+
+  proxy:
+    restart: always
+    image: nginx:1-alpine
+    ports:
+      - 80:80
+      - 443:443
+    volumes:
+      - ./services-conf/nginx.conf:/etc/nginx/conf.d/default.conf
+      - /etc/letsencrypt/:/etc/letsencrypt/
+
+  db:
+    image: timescale/timescaledb:latest-pg15
+    volumes:
+      - postgres_data:/var/lib/postgresql/data/
+    environment:
+      - POSTGRES_USER=hpcperfstats
+      - POSTGRES_PASSWORD=hpcperfstats
+
+  rabbitmq:
+    image: rabbitmq:4-alpine
+    ports:
+        - 5672:5672
+    # Admin Console
+    #    - 15672:15672
+    volumes:
+        - rabbitmq_messages:/var/lib/rabbitmq/
+        - /var/log/rabbitmq:/var/log/rabbitmq
+
+networks:
+  default:
+    driver: bridge
+
+volumes:
+      postgres_data:
+
+      rabbitmq_messages:
+
+      hpcperfstatsdata:
+        driver: local
+        driver_opts:
+          type: none
+          device: /home/sharrell/compose_test/hpcperfstats_data/
+          o: bind

docker-instructions.txt

@@ -0,0 +1,32 @@
+# From a base rocky image
+
+dnf install docker git podman-compose
+
+# May need to use an ssh git address if you plan to commit to the repo
+git clone https://github.com/TACC/tacc_stats.git 
+
+cd tacc_stats
+
+git checkout sharrell-docker
+
+# Create env.dev file, contents:
+DEBUG=1
+SECRET_KEY=foo
+PATH=$PATH:/home/hpcperfstats/.local/bin
+# end contents
+
+# In service-conf, nginx.conf needs to be chosen, to make things easier for development, use the nossl config, if its production use the ssl version, you can copy the nginx-nossl.conf to nginx.conf and build.
+
+cp services-conf/nginx-nossl.conf services-conf/nginx.conf
+
+docker compose up --build -d
+
+# to see the startup
+docker compose logs 
+
+#you should be able to get to the website at this point, will error if no data
+
+
+
+
+

requirements.txt

@@ -0,0 +1,77 @@
+agavepy==0.9.5
+ansible-cmdb==1.31
+asgiref==3.4.1
+attrs==21.2.0
+backports.ssl-match-hostname==3.7.0.1
+bokeh==2.3.3
+certifi==2021.10.8
+cffi==1.15.1
+charset-normalizer==2.0.9
+cheroot==8.5.2
+CherryPy==18.6.1
+cloudpickle==2.0.0
+configparser==5.2.0
+cryptography==40.0.2
+cycler==0.11.0
+Django==3.1.14
+future==0.18.2
+gitdb==4.0.9
+GitPython==3.1.20
+idna==3.3
+importlib-metadata==4.8.2
+importlib-resources==5.4.0
+iniconfig==1.1.1
+jaraco.classes==3.2.1
+jaraco.collections==3.4.0
+jaraco.functools==3.4.0
+jaraco.text==3.6.0
+Jinja2==3.0.3
+jsonxs==0.6
+kiwisolver==1.3.1
+Mako==1.1.6
+MarkupSafe==2.0.1
+matplotlib==3.3.4
+more-itertools==8.12.0
+mysql==0.0.3
+mysql-connector-python==8.0.33
+mysqlclient==2.1.1
+numpy==1.19.5
+packaging==21.3
+pandas==1.1.5
+pgcopy==1.5.0
+pika==1.2.0
+Pillow==8.4.0
+pip==21.3.1
+pluggy==1.0.0
+portend==3.0.0
+protobuf==3.19.1
+psycopg2==2.9.2
+psycopg2-binary==2.9.2
+py==1.11.0
+pycparser==2.21
+PyMySQL==1.0.2
+pyparsing==3.0.6
+pytest==6.2.5
+python-dateutil==2.8.2
+python-hostlist==1.21
+python-memcached==1.59
+pytz==2021.3
+PyYAML==6.0
+requests==2.26.0
+requests-toolbelt==0.9.1
+setuptools==39.2.0
+six==1.16.0
+smmap==5.0.0
+sqlparse==0.4.2
+tempora==4.1.2
+termcolor==1.1.0
+toml==0.10.2
+tornado==6.1
+typing==3.7.4.3
+typing_extensions==4.0.1
+urllib3==1.26.7
+ushlex==0.99.1
+websocket-client==0.53.0
+zc.lockfile==2.0
+zipp==3.6.0
+gunicorn

services-conf/django_startup.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+
+echo "Waiting for postgres..."
+
+while ! nc -z db 5432; do
+  sleep 0.1
+done
+
+echo "PostgreSQL started"
+
+# detect if the tables are existing and create if not
+/usr/local/bin/python3 tacc_stats/site/manage.py makemigrations
+/usr/local/bin/python3 tacc_stats/site/manage.py migrate
+
+# then run this (gunicorn later)
+/usr/local/bin/gunicorn tacc_stats.site.tacc_stats_site.wsgi --bind 0.0.0.0:8000  --env DJANGO_SETTINGS_MODULE=tacc_stats.site.tacc_stats_site.settings -u hpcperfstats
+

services-conf/nginx-nossl.conf

@@ -0,0 +1,17 @@
+upstream tacc_stats {
+    server web:8000;
+}
+
+
+server {
+    listen 80;    
+
+    client_max_body_size 14M;
+    location / {
+        proxy_pass http://web:8000;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header Host $host;
+        proxy_redirect off;
+    } 
+}   
+

services-conf/nginx-withssl.conf

@@ -0,0 +1,35 @@
+upstream django_server {
+    server web:8000;
+}
+
+
+server {
+    listen 80;
+    location / {
+         rewrite ^ https://$host$request_uri? permanent;
+    }   
+}
+
+server {
+        listen 443 ssl;
+
+        ssl_certificate /etc/letsencrypt/live/stats.stampede3.tacc.utexas.edu/fullchain.pem;
+        ssl_certificate_key /etc/letsencrypt/live/stats.stampede3.tacc.utexas.edu/privkey.pem;
+
+        location / {
+            proxy_pass         http://web:8000;
+            proxy_redirect     off;
+            proxy_http_version 1.1;
+            proxy_cache_bypass $http_upgrade;
+            proxy_set_header   Upgrade $http_upgrade;
+            proxy_set_header   Connection keep-alive;
+            proxy_set_header   Host $host;
+            proxy_set_header   X-Real-IP $remote_addr;
+            proxy_set_header   X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header   X-Forwarded-Proto $scheme;
+            proxy_set_header   X-Forwarded-Host nginx;
+            proxy_buffer_size           128k;
+            proxy_buffers               4 256k;
+            proxy_busy_buffers_size     256k;
+        }
+}

services-conf/supervisor_startup.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+
+
+chmod -c 755 /hpcperfstats/
+# make directories if they are not there
+mkdir -pv /hpcperfstats/accounting
+mkdir -pv /hpcperfstats/archive
+mkdir -pv /hpcperfstats/daily_archive
+mkdir -pv /hpcperfstats/logs
+chown -R hpcperfstats:hpcperfstats /hpcperfstats/* 
+
+/usr/bin/supervisord
+
+
+

services-conf/supervisord.conf

@@ -0,0 +1,30 @@
+[supervisord]
+nodaemon=true
+logfile=/dev/fd/1
+logfile_maxbytes=0
+
+[program:hpcperfstats-rabbitmq-listener]
+command=/usr/local/bin/python3 /home/hpcperfstats/tacc_stats/listend.py
+user=hpcperfstats
+autorestart=true
+startretries=20
+stdout_redirect=true
+stdout_logfile=/dev/fd/1
+stdout_logfile_maxbytes=0
+stderr_redirect=true
+stderr_logfile=/dev/fd/1
+stderr_logfile_maxbytes=0
+
+
+[program:sync_acct]
+command=/usr/local/bin/python3 /home/hpcperfstats/tacc_stats/dbload/sync_acct.py
+user=hpcperfstats
+autorestart=true
+startretries=20
+stdout_redirect=true
+stdout_logfile=/dev/fd/1
+stdout_logfile_maxbytes=0
+stderr_redirect=true
+stderr_logfile=/dev/fd/1
+stderr_logfile_maxbytes=0
+

setup.py

@@ -29,10 +29,10 @@
 config = ConfigParser()
 config.read("tacc_stats.ini")
 
-with open("tacc_stats/cfg.py", 'w') as fd:
-    for s in config.sections():
-        for key, val in dict(config.items(s)).items():
-            fd.write(key + " = " + "\"" + val + "\"" + '\n')
+#with open("tacc_stats/cfg.py", 'w') as fd:
+#    for s in config.sections():
+#        for key, val in dict(config.items(s)).items():
+#            fd.write(key + " = " + "\"" + val + "\"" + '\n')
 
 setup(
     name = DISTNAME,
@@ -49,7 +49,7 @@
     include_package_data = True,
     scripts = scripts,
     install_requires = ['argparse','numpy', 'psycopg2-binary', 'pandas', 'pgcopy',
-                        'bokeh', 'django==3.2.25', 'python-hostlist', 'PyMySQL', 'mod_wsgi',
+                        'bokeh', 'django==3.1.14', 'python-hostlist', 'PyMySQL',
                         'mysql-connector-python', 'python-memcached', 'pika', 'mysqlclient'],
     platforms = 'any',
     classifiers = [