Merge branch 'unstable' into websockets

# Conflicts: # Makefile # contentcuration/contentcuration/frontend/shared/data/applyRemoteChanges.js # contentcuration/contentcuration/frontend/shared/data/resources.js # contentcuration/contentcuration/frontend/shared/data/serverSync.js # contentcuration/contentcuration/settings.py # contentcuration/contentcuration/viewsets/base.py # contentcuration/contentcuration/viewsets/channel.py # contentcuration/contentcuration/viewsets/sync/endpoint.py # requirements-dev.txt # requirements.in # requirements.txt
learningequality · Mar 29, 2024 · b0af9fd · b0af9fd
2 parents 99edf3f + 6e3be57
commit b0af9fd
Show file tree

Hide file tree

Showing 534 changed files with 29,267 additions and 11,490 deletions.
diff --git a/.docker/README.md b/.docker/README.md
@@ -0,0 +1,17 @@
+## What is this directory?
+This directory is a space for mounting directories to docker containers, allowing the mounts to be specified in committed code, but the contents of the mounts to remain ignored by git.
+
+### postgres
+The `postgres` directory is mounted to `/docker-entrypoint-initdb.d`. Any `.sh` or `.sql` files will be executed when the container is first started with a new data volume. You may read more regarding this functionality on the [Docker Hub page](https://hub.docker.com/_/postgres), under _Initialization scripts_.
+
+When running docker services through the Makefile commands, it specifies a docker-compose project name that depends on the name of the current git branch. This causes the volumes to change when the branch changes, which is helpful when switching between many branches that might have incompatible database schema changes. The downside is that whenever you start a new branch, you'll have to re-initialize the database again, like with `yarn run devsetup`. Creating a SQL dump from an existing, initialized database and placing it in this directory will allow you to skip this step.
+
+To create a SQL dump of your preferred database data useful for local testing, run `make .docker/postgres/init.sql` while the docker postgres container is running.
+
+> Note: you will likely need to run `make migrate` to ensure your database schema is up-to-date when using this technique.
+
+#### pgpass
+Stores the postgres authentication for the docker service for scripting access without manually providing a password, created by `make .docker/pgpass`
+
+### minio
+The `minio` directory is mounted to `/data`, since it isn't necessarily useful to have this data isolated based off the current git branch.
diff --git a/.editorconfig b/.editorconfig
@@ -1,5 +1,8 @@
 root = true
 
+[*]
+max_line_length = 100
+
 [*.js]
 indent_size = 2
 

diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml
@@ -1 +1,8 @@
 blank_issues_enabled: false
+contact_links:
+  - name: Studio GitHub Discussions
+    url: https://github.com/learningequality/studio/discussions
+    about: Please ask general questions about contributing to Studio or report development server issues here.
+  - name: Learning Equality Community Forum
+    url: https://community.learningequality.org/
+    about: Ask and answer questions about Learning Equality's products and tools, share your experiences using Kolibri, and connect with users around the world.
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -7,10 +7,30 @@ updates:
   - package-ecosystem: "pip"
     directory: "/"
     schedule:
-      interval: "daily"
+      interval: "weekly"
+      day: "wednesday"
+      time: "00:00"
 
   # Maintain dependencies for Javascript
   - package-ecosystem: "npm"
     directory: "/"
     schedule:
-      interval: "daily"
+      interval: "weekly"
+      day: "wednesday"
+      time: "00:00"
+    groups:
+      babel:
+        patterns:
+          - "@babel/*"
+
+  # Maintain dependencies for Github Actions
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+      day: "wednesday"
+      time: "00:00"
+    groups:
+      github:
+        patterns:
+          - "actions/*"
diff --git a/.github/workflows/containerbuild.yml b/.github/workflows/containerbuild.yml
@@ -0,0 +1,105 @@
+name: Container Build
+
+on:
+  push:
+    branches:
+    - unstable
+    - hotfixes
+    - master
+    tags:
+    - 'v*'
+  pull_request:
+
+jobs:
+  pre_postgres:
+    name: Path match check - postgres
+    runs-on: ubuntu-latest
+    # Map a step output to a job output
+    outputs:
+      should_skip: ${{ steps.skip_check.outputs.should_skip }}
+    steps:
+      - id: skip_check
+        uses: fkirc/skip-duplicate-actions@master
+        with:
+          skip_after_successful_duplicate: false
+          github_token: ${{ github.token }}
+          paths: '["docker/Dockerfile.postgres.dev", ".github/workflows/containerbuild.yml"]'
+
+  build_and_push_postgres:
+    name: Postgres - build and push Docker image to GitHub Container Registry
+    needs: pre_postgres
+    if: ${{ needs.pre_postgres.outputs.should_skip != 'true' }}
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout codebase
+      uses: actions/checkout@v4
+
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v3
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+
+    - name: Log in to Docker Hub
+      if: github.event_name != 'pull_request'
+      uses: docker/login-action@v3
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Extract metadata (tags, labels) for Docker
+      id: meta
+      uses: docker/metadata-action@v5
+      with:
+        images: ghcr.io/learningequality/postgres
+      env:
+        DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,index
+
+    - name: Build and push Docker image
+      uses: docker/build-push-action@v5
+      with:
+        context: ./docker
+        file: ./docker/Dockerfile.postgres.dev
+        platforms: linux/amd64,linux/arm64
+        push: ${{ github.event_name != 'pull_request' }}
+        tags: ${{ steps.meta.outputs.tags }}
+        labels: ${{ steps.meta.outputs.labels }}
+        annotations: ${{ steps.meta.outputs.annotations }}
+
+  pre_nginx:
+    name: Path match check - nginx
+    runs-on: ubuntu-latest
+    # Map a step output to a job output
+    outputs:
+      should_skip: ${{ steps.skip_check.outputs.should_skip }}
+    steps:
+      - id: skip_check
+        uses: fkirc/skip-duplicate-actions@master
+        with:
+          skip_after_successful_duplicate: false
+          github_token: ${{ github.token }}
+          paths: '["k8s/images/nginx/*", ".github/workflows/containerbuild.yml"]'
+
+  build_nginx:
+    name: nginx - test build of nginx Docker image
+    needs: pre_nginx
+    if: ${{ needs.pre_nginx.outputs.should_skip != 'true' }}
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout codebase
+      uses: actions/checkout@v4
+
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v3
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+
+    - name: Build Docker image
+      uses: docker/build-push-action@v5
+      with:
+        context: ./
+        file: ./k8s/images/nginx/Dockerfile
+        platforms: linux/amd64
+        push: false
diff --git a/.github/workflows/deploytest.yml b/.github/workflows/deploytest.yml
@@ -27,13 +27,13 @@ jobs:
     if: ${{ needs.pre_job.outputs.should_skip != 'true' }}
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Use Node.js
-      uses: actions/setup-node@v1
+      uses: actions/setup-node@v4
       with:
         node-version: '16.x'
     - name: Cache Node.js modules
-      uses: actions/cache@v2
+      uses: actions/cache@v4
       with:
         path: '**/node_modules'
         key: ${{ runner.OS }}-node-${{ hashFiles('**/yarn.lock') }}
@@ -51,13 +51,13 @@ jobs:
     if: ${{ needs.pre_job.outputs.should_skip != 'true' }}
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python 3.9
-      uses: actions/setup-python@v2
+    - uses: actions/checkout@v4
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v5
       with:
-        python-version: 3.9
+        python-version: '3.10'
     - name: pip cache
-      uses: actions/cache@v2
+      uses: actions/cache@v4
       with:
         path: ~/.cache/pip
         key: ${{ runner.os }}-pyprod-${{ hashFiles('requirements.txt') }}
@@ -69,11 +69,11 @@ jobs:
         pip install pip-tools
         pip-sync requirements.txt
     - name: Use Node.js
-      uses: actions/setup-node@v1
+      uses: actions/setup-node@v4
       with:
         node-version: '16.x'
     - name: Cache Node.js modules
-      uses: actions/cache@v2
+      uses: actions/cache@v4
       with:
         path: '**/node_modules'
         key: ${{ runner.OS }}-node-${{ hashFiles('**/yarn.lock') }}

diff --git a/.github/workflows/frontendlint.yml b/.github/workflows/frontendlint.yml
@@ -27,13 +27,13 @@ jobs:
     if: ${{ needs.pre_job.outputs.should_skip != 'true' }}
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Use Node.js
-      uses: actions/setup-node@v1
+      uses: actions/setup-node@v4
       with:
         node-version: '16.x'
     - name: Cache Node.js modules
-      uses: actions/cache@v2
+      uses: actions/cache@v4
       with:
         path: '**/node_modules'
         key: ${{ runner.OS }}-node-${{ hashFiles('**/yarn.lock') }}
@@ -49,7 +49,7 @@ jobs:
       if: github.event.pull_request && github.event.pull_request.head.repo.full_name == github.repository
       id: git-check
       run: echo ::set-output name=modified::$(git diff-index --name-only HEAD)
-    - uses: tibdex/github-app-token@v1
+    - uses: tibdex/github-app-token@v2
       if: github.event.pull_request && github.event.pull_request.head.repo.full_name == github.repository && steps.git-check.outputs.modified != ''
       id: generate-token
       with:

diff --git a/.github/workflows/frontendtest.yml b/.github/workflows/frontendtest.yml
@@ -27,13 +27,13 @@ jobs:
     if: ${{ needs.pre_job.outputs.should_skip != 'true' }}
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Use Node.js
-      uses: actions/setup-node@v1
+      uses: actions/setup-node@v4
       with:
         node-version: '16.x'
     - name: Cache Node.js modules
-      uses: actions/cache@v2
+      uses: actions/cache@v4
       with:
         path: '**/node_modules'
         key: ${{ runner.OS }}-node-${{ hashFiles('**/yarn.lock') }}

diff --git a/.github/workflows/notify_team_new_comment.yml b/.github/workflows/notify_team_new_comment.yml
@@ -0,0 +1,35 @@
+name: Send a slack notification when a contributor comments on issue
+
+on:
+  issue_comment:
+    types: [created]
+
+jobs:
+  contributor_issue_comment:
+    name: Contributor issue comment
+
+    if: >-
+      ${{
+        !github.event.issue.pull_request &&
+        github.event.comment.author_association != 'MEMBER' &&
+        github.event.comment.author_association != 'OWNER'
+      }}
+
+    runs-on: ubuntu-latest
+    steps:
+      - name: Escape title double quotes
+        id: escape_title
+        run: |
+          title='${{ github.event.issue.title }}'
+          echo "ISSUE_TITLE=${title//\"/\\\"}" >> "$GITHUB_OUTPUT"
+
+      - name: Send message to Slack channel
+        env:
+            SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+            SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
+        uses: slackapi/[email protected]
+        with:
+          payload: |
+            {
+              "text": "*[Studio] New comment on issue: <${{ github.event.issue.html_url }}#issuecomment-${{ github.event.comment.id }}|${{ steps.escape_title.outputs.ISSUE_TITLE }} by ${{ github.event.comment.user.login }}>*"
+            }
diff --git a/.github/workflows/pythontest.yml b/.github/workflows/pythontest.yml
@@ -61,7 +61,7 @@ jobs:
           # Maps port 6379 on service container to the host
           - 6379:6379
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Set up minio
       run: |
         docker run -d -p 9000:9000 --name minio \
@@ -70,12 +70,12 @@ jobs:
                     -v /tmp/minio_data:/data \
                     -v /tmp/minio_config:/root/.minio \
                     minio/minio server /data
-    - name: Set up Python 3.9
-      uses: actions/setup-python@v2
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v5
       with:
-        python-version: 3.9
+        python-version: '3.10'
     - name: pip cache
-      uses: actions/cache@v2
+      uses: actions/cache@v4
       with:
         path: ~/.cache/pip
         key: ${{ runner.os }}-pytest-${{ hashFiles('requirements.txt', 'requirements-dev.txt') }}

diff --git a/.gitignore b/.gitignore
@@ -29,6 +29,8 @@ var/
 # IntelliJ IDE, except project config
 .idea/*
 !.idea/studio.iml
+# ignore future updates to run configuration
+.run/devserver.run.xml
 
 # PyInstaller
 #  Usually these files are written by a python script from a template
@@ -95,8 +97,11 @@ contentcuration/csvs/
 # Ignore the TAGS file generated by some editors
 TAGS
 
-# Ignore Vagrant-created files
-/.vagrant/
+# Services
+.vagrant/
+.docker/minio/*
+.docker/postgres/*
+.docker/pgpass
 
 # Ignore test files
 /contentcuration/contentcuration/proxy_settings.py

diff --git a/.run/devserver.run.xml b/.run/devserver.run.xml
@@ -0,0 +1,24 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="devserver" type="PythonConfigurationType" factoryName="Python">
+    <module name="studio" />
+    <option name="INTERPRETER_OPTIONS" value="" />
+    <option name="PARENT_ENVS" value="true" />
+    <envs>
+      <env name="PYTHONUNBUFFERED" value="1" />
+    </envs>
+    <option name="SDK_HOME" value="$ModuleSdkPath$" />
+    <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
+    <option name="IS_MODULE_SDK" value="true" />
+    <option name="ADD_CONTENT_ROOTS" value="false" />
+    <option name="ADD_SOURCE_ROOTS" value="false" />
+    <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
+    <option name="SCRIPT_NAME" value="contentcuration/manage.py" />
+    <option name="PARAMETERS" value="runserver --settings=contentcuration.dev_settings 0.0.0.0:8080" />
+    <option name="SHOW_COMMAND_LINE" value="false" />
+    <option name="EMULATE_TERMINAL" value="false" />
+    <option name="MODULE_MODE" value="false" />
+    <option name="REDIRECT_INPUT" value="false" />
+    <option name="INPUT_FILE" value="" />
+    <method v="2" />
+  </configuration>
+</component>