Kubernetes Staging Environment (#242)

busy_beaver/__init__.py

@@ -5,18 +5,21 @@
 from sentry_sdk.integrations.flask import FlaskIntegration
 from sentry_sdk.integrations.sqlalchemy import SqlalchemyIntegration
 
-from .config import IN_PRODUCTION, LOGGING_CONFIG, SENTRY_DSN
+from .config import LOGGING_CONFIG, SENTRY_DSN, SENTRY_ENV_FILTER
 
 # observability
 load_dict_config(LOGGING_CONFIG)
 logger = logging.getLogger(__name__)
-if IN_PRODUCTION and SENTRY_DSN:
-    integrations = [FlaskIntegration(), SqlalchemyIntegration()]
-    sentry_sdk.init(dsn=SENTRY_DSN, integrations=integrations)
+if SENTRY_DSN:
+    sentry_sdk.init(
+        dsn=SENTRY_DSN,
+        environment=SENTRY_ENV_FILTER,
+        integrations=[FlaskIntegration(), SqlalchemyIntegration()],
+    )
 
-logger.info("[BusyBeaver] Configure Integrations")
+logger.info("Configure Integrations")
 from . import clients  # noqa isort:skip
 
-logger.info("[BusyBeaver] Starting Server")
+logger.info("Starting Server")
 from .app import create_app  # noqa isort:skip
 from . import models  # noqa isort:skip

busy_beaver/apps/debug/api/healthcheck.py

@@ -11,5 +11,11 @@
 @healthcheck_bp.route("/healthcheck", methods=["GET"])
 @talisman(force_https=False)
 def health_check():
+    return jsonify({"ping": "pong"})
+
+
+@healthcheck_bp.route("/healthcheck-logged", methods=["GET"])
+@talisman(force_https=False)
+def health_check_logged():
     logger.info("Hit healthcheck")
     return jsonify({"ping": "pong"})

busy_beaver/apps/retweeter/task.py

@@ -18,8 +18,8 @@
 
 @click.option("--channel_name", required=True, help="Slack channel")
 @click.option("--workspace", required=True, help="Slack workspace ID")
-@twitter_bp.cli.command("post_tweets_to_slack", help="Find new tweets to post to Slack")
-def post_new_tweets_to_slack(channel_name: str, workspace: str):
+@twitter_bp.cli.command("poll_twitter", help="Find new tweets to post to Slack")
+def poll_twitter(channel_name: str, workspace: str):
     # TODO add logging and times
     slack_installation = SlackInstallation.query.filter_by(
         workspace_id=workspace

busy_beaver/apps/slack_integration/api/README.md

@@ -3,6 +3,6 @@
 Installation of the application into other Slack workspaces
 is enabled via [OAuth2](https://api.slack.com/docs/oauth).
 
-Point the Redirect URL to: https://busybeaver.sivji.com/slack/oauth`
+Point the Redirect URL to: https://app.busybeaverbot.com/slack/oauth`
 
 We are using the Slack Event Subscription API and Slash Commands.

busy_beaver/config.py

@@ -1,18 +1,16 @@
 import os
 
 IN_PRODUCTION = os.getenv("IN_PRODUCTION", False)
+SENTRY_ENV_FILTER = "production" if IN_PRODUCTION else "staging"
 TASK_QUEUE_MAX_RETRIES = 1
 
 SECRET_KEY = os.getenv("SECRET_KEY", "abcdef").encode("utf-8")
 
-# app urls
-PROD_BASE_URI = "https://busybeaver.sivji.com"
-DEV_BASE_URI = os.getenv("NGROK_BASE_URI", None)
-APP_URI = PROD_BASE_URI if IN_PRODUCTION else DEV_BASE_URI
-
 # app constraints
 FULL_INSTALLATION_WORKSPACE_IDS = [
     "T093FC1RC",  # ChiPy Workspace -- https://chipy.slack.com -- prod env
+    "TPDB2AV4K",  # Production instance (home workspace)
+    "TKT910ZU0",  # Staging instance (staging workspace)
     "T5G0FCMNW",  # SivBots -- https://sivbots.slack.com -- dev env
 ]
 

deployment/README.md

@@ -0,0 +1,59 @@
+# Deployment
+
+Material related to deploying Busy Beaver.
+
+## Current Deployment Workflow
+
+Busy Beaver has been packaged up as a Helm chart.
+
+### Prerequestites
+
+- Install Helm
+- Use Helm to set up `nginx`, `cert-manager`, `redis`, fluent-bit
+- Add `busybeaver-staging` Secret to cluster
+
+### Installing Busy Beaver
+
+```console
+helm install busybeaver-staging ./busybeaver/
+helm upgrade busybeaver-staging ./busybeaver/
+```
+
+### Secrets Format
+
+All data values need to be `base64` encoded.
+
+```yaml
+apiVersion: v1
+kind: Secret
+metadata:
+  name: busybeaver-staging
+type: Opaque
+data:
+  db-uri:
+  cache-uri:
+  sentry-dsn:
+  secret-key:
+  slack-client-id:
+  slack-client-secret:
+  slack-botuser-oauth-token:
+  slack-signing-secret:
+  meetup-api-key:
+  github-client-id:
+  github-client-secret:
+  github-oauth-token:
+  github-signing-secret:
+  twitter-access-token-secret:
+  twitter-access-token:
+  twitter-consumer-key:
+  twitter-consumer-secret:
+```
+
+### Notes
+
+- if staging database gets deleted and we have to start again
+  - will need to set up app for distribution and install it via OAuth
+- TODO (need to clean this up)
+  - [ ] what accounts did we need to create?
+    - [ ] what are those accounts?
+    - [ ] where do we store credentials?

deployment/busybeaver/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

deployment/busybeaver/Chart.yaml

@@ -0,0 +1,22 @@
+apiVersion: v2
+name: busybeaver
+description: Community engagement Slackbot
+home: https://github.com/busy-beaver-dev/busy-beaver
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+version: 0.0.2
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application.
+appVersion: 1.3.3
+
+keywords:
+  - community-engagement
+  - slackbot
+
+maintainers:
+  - name: Aly Sivji
+    email: [email protected]
+    url: https://alysivji.github.io

deployment/busybeaver/templates/NOTES.txt

@@ -0,0 +1,8 @@
+Thank you for installing {{ .Chart.Name }}.
+
+Your release is named {{ .Release.Name }}.
+
+To learn more about the release, try:
+
+  $ helm status {{ .Release.Name }}
+  $ helm get all {{ .Release.Name }}

deployment/busybeaver/templates/_helpers.tpl

@@ -0,0 +1,151 @@
+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "busybeaver.name" -}}
+{{- default .Chart.Name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "busybeaver.fullname" -}}
+{{- $name := default .Chart.Name -}}
+{{- if contains $name .Release.Name -}}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "busybeaver.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Common labels
+*/}}
+{{- define "busybeaver.labels" -}}
+{{ include "busybeaver.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+helm.sh/chart: {{ include "busybeaver.chart" . }}
+{{- end -}}
+
+{{/*
+Selector labels
+*/}}
+{{- define "busybeaver.selectorLabels" -}}
+app: webapp
+environment: {{ .Values.environment }}
+app.kubernetes.io/name: {{ include "busybeaver.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end -}}
+
+{{/*
+Environment Variables
+*/}}
+{{- define "busybeaver.env_vars" }}
+- name: PYTHONPATH
+  value: .
+- name: FLASK_APP
+  value: /app/busy_beaver/__init__.py
+- name: SENTRY_DSN
+  valueFrom:
+    secretKeyRef:
+      name: {{ .Values.secretName }}
+      key: sentry-dsn
+# infrastructure
+- name: DATABASE_URI
+  valueFrom:
+    secretKeyRef:
+      name: {{ .Values.secretName }}
+      key: db-uri
+- name: REDIS_URI
+  valueFrom:
+    secretKeyRef:
+      name: {{ .Values.secretName }}
+      key: cache-uri
+# integrations
+# oauth lib
+- name: OAUTHLIB_INSECURE_TRANSPORT
+  value: "1"
+- name: OAUTHLIB_RELAX_TOKEN_SCOPE
+  value: "1"
+# slack
+- name: SLACK_CLIENT_ID
+  valueFrom:
+    secretKeyRef:
+      name: {{ .Values.secretName }}
+      key: slack-client-id
+- name: SLACK_CLIENT_SECRET
+  valueFrom:
+    secretKeyRef:
+      name: {{ .Values.secretName }}
+      key: slack-client-secret
+- name: SLACK_BOTUSER_OAUTH_TOKEN
+  valueFrom:
+    secretKeyRef:
+      name: {{ .Values.secretName }}
+      key: slack-botuser-oauth-token
+- name: SLACK_SIGNING_SECRET
+  valueFrom:
+    secretKeyRef:
+      name: {{ .Values.secretName }}
+      key: slack-signing-secret
+# meetup
+- name: MEETUP_API_KEY
+  valueFrom:
+    secretKeyRef:
+      name: {{ .Values.secretName }}
+      key: meetup-api-key
+# github
+- name: GITHUB_APP_CLIENT_ID
+  valueFrom:
+    secretKeyRef:
+      name: {{ .Values.secretName }}
+      key: github-client-id
+- name: GITHUB_APP_CLIENT_SECRET
+  valueFrom:
+    secretKeyRef:
+      name: {{ .Values.secretName }}
+      key: github-client-secret
+- name: GITHUB_OAUTH_TOKEN
+  valueFrom:
+    secretKeyRef:
+      name: {{ .Values.secretName }}
+      key: github-oauth-token
+- name: GITHUB_SIGNING_SECRET
+  valueFrom:
+    secretKeyRef:
+      name: {{ .Values.secretName }}
+      key: github-signing-secret
+# twitter
+- name: TWITTER_ACCESS_TOKEN_SECRET
+  valueFrom:
+    secretKeyRef:
+      name: {{ .Values.secretName }}
+      key: twitter-access-token-secret
+- name: TWITTER_ACCESS_TOKEN
+  valueFrom:
+    secretKeyRef:
+      name: {{ .Values.secretName }}
+      key: twitter-access-token
+- name: TWITTER_CONSUMER_KEY
+  valueFrom:
+    secretKeyRef:
+      name: {{ .Values.secretName }}
+      key: twitter-consumer-key
+- name: TWITTER_CONSUMER_SECRET
+  valueFrom:
+    secretKeyRef:
+      name: {{ .Values.secretName }}
+      key: twitter-consumer-secret
+{{- end }}

deployment/busybeaver/templates/cronjob--poll-twitter.yaml

@@ -0,0 +1,26 @@
+apiVersion: batch/v1beta1
+kind: CronJob
+metadata:
+  name: "{{ include "busybeaver.fullname" . }}--cron--poll-twitter"
+  labels:
+    type: poll-twitter
+    {{- include "busybeaver.labels" . | nindent 4 }}
+spec:
+  schedule: "*/13 * * * *"
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          restartPolicy: Never
+          containers:
+          - name: busybeaver--poll-twitter
+            image: {{ .Values.image.repository }}:{{ .Values.image.version }}
+            imagePullPolicy: {{ .Values.image.pullPolicy }}
+            command: ["flask"]
+            args:
+            - "poll_twitter"
+            - "--workspace"
+            - {{ .Values.workspaceId | quote }}
+            - "--channel_name"
+            - "at-chicagopython"
+            env: {{- include "busybeaver.env_vars" . | indent 12 }}