[devOps] import/erase/deploy workflows refactored

.github/workflows/deploy.yml

@@ -3,14 +3,28 @@ name: Deploy DSpace
 
 on:
   workflow_dispatch:
+    inputs:
+      INSTANCE:
+        required: true
+        default: '*'
+        type: choice
+        options:
+          - '*'
+          - '5'
+          - '8'
+      IMPORT:
+        required: true
+        default: true
+        type: boolean
 
 jobs:
   deploy:
+    if: github.event.inputs.INSTANCE == '*' || github.event.inputs.INSTANCE == '5'
     runs-on: dspace-dep-1
+    timeout-minutes: 5
     steps:
-      - uses: actions/checkout@v3
-        with:
-          submodules: false
+      - uses: actions/checkout@v4
+
       - name: deploy to dev-5
         run: |
           cd $GITHUB_WORKSPACE/build-scripts/run/
@@ -40,15 +54,32 @@ jobs:
           export ENVFILE=$(pwd)/.env.dev-5
           ./start.sh dspace-5
 
+  pre-deploy-import:
+    if: github.event.inputs.INSTANCE == '*' || github.event.inputs.INSTANCE == '8'
+    uses: dataquest-dev/dspace-angular/.github/workflows/erase_db.yml@dtq-dev
+    with:
+      INSTANCE: '8'
+
+  deploy-import:
+    if: github.event.inputs.INSTANCE == '*' || github.event.inputs.INSTANCE == '8'
+    needs: pre-deploy-import
+    runs-on: dspace-dep-1
+    timeout-minutes: 5
+    env:
+      INSTANCE: '8'
+    steps:
+      - uses: actions/checkout@v4
+
       - name: deploy dspace-import on dev-5
         working-directory: build-scripts/run/
         run: |
-          ENVFILE=/opt/dspace-envs/.env.dspace.imported.dev-5 ./start.sh dspace-5-import
-          sleep 120
-          echo "done with start.sh, going to request import"
-          curl -H "Accept: application/vnd.github.everest-preview+json" \
-          -H "Authorization: token ${{ secrets.DEPLOY_DEV5_GH_ACTION_DISPATCH }}" \
-          --request POST \
-          https://api.github.com/repos/dataquest-dev/\
-          dspace-angular/actions/workflows/import.yml/dispatches \
-          --data "{\"ref\":\"refs/heads/dtq-dev\"}"
+          ENVFILE=/opt/dspace-envs/.env.dspace.imported.dev-5 ./start.sh dspace-$INSTANCE
+          /bin/bash -c 'while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' http://dev-5.pc:8$INSTANCE/server/api)" != "200" ]]; do sleep 5; done'
+
+  post-deploy-import:
+    if: (github.event.inputs.INSTANCE == '*' || github.event.inputs.INSTANCE == '8') && github.event.inputs.IMPORT != false
+    needs: deploy-import
+    uses: dataquest-dev/dspace-angular/.github/workflows/import.yml@dtq-dev
+    secrets: inherit
+    with:
+      INSTANCE: '8'

.github/workflows/erase_db.yml

@@ -1,6 +1,11 @@
 name: Erase database
 
 on:
+  workflow_call:
+    inputs:
+      INSTANCE:
+        required: true
+        type: string
   workflow_dispatch:
     inputs:
       INSTANCE:
@@ -13,24 +18,23 @@ on:
 
 
 jobs:
-  restart_db:
+  recreate_db:
+    runs-on: dspace-dep-1
+    timeout-minutes: 5
     env:
       INSTANCE: ${{ inputs.INSTANCE }}
-    runs-on: dspace-dep-1
+      NAME: dspace-${{ inputs.INSTANCE }}
     steps:
-      - uses: actions/checkout@v3
-        with:
-          submodules: false
+
       - name: stop and remove containers
         run: |
-          docker stop dspacedb$INSTANCE dspace$INSTANCE dspace-angular$INSTANCE
-          docker rm dspacedb$INSTANCE dspace$INSTANCE dspace-angular$INSTANCE
+          docker stop dspacesolr$INSTANCE dspacedb$INSTANCE dspace$INSTANCE dspace-angular$INSTANCE || true
+          docker rm   dspacesolr$INSTANCE dspacedb$INSTANCE dspace$INSTANCE dspace-angular$INSTANCE || true
+
       - name: remove volumes
         if: '!cancelled()'
         run: |
-          docker volume rm dspace-${{env.INSTANCE}}_pgdata dspace-${{env.INSTANCE}}_assetstore
-      - name: redeploy
-        if: '!cancelled()'
-        run: |
-          curl -H "Accept: application/vnd.github.everest-preview+json" -H "Authorization: token ${{ secrets.DEPLOY_DEV5_GH_ACTION_DISPATCH }}" --request POST https://api.github.com/repos/dataquest-dev/dspace-angular/actions/workflows/deploy.yml/dispatches --data "{\"ref\":\"refs/heads/dtq-dev\"}"
-
+          # be sure to have INSTANCE set
+          if [[ "x${{ env.NAME }}" != "dspace-" ]]; then
+            docker volume rm $(docker volume ls --filter name="${{ env.NAME }}_" -q) || true
+          fi;

.github/workflows/import.yml

@@ -1,32 +1,55 @@
 name: Import DSpace v5
 
 on:
+  workflow_call:
+    inputs:
+      INSTANCE:
+        required: true
+        type: string
   workflow_dispatch:
+    inputs:
+      INSTANCE:
+        required: true
+        default: '8'
+        type: choice
+        options:
+          - '8'
 
 jobs:
-  deploy:
+  import:
     runs-on: dspace-dep-1
     steps:
+
       - uses: actions/checkout@v4
         with:
           repository: dataquest-dev/dspace-python-api
           path: dspace-python-api
           ref: 'refactor_jm'
 
+      - name: info
+        run: |
+          docker ps -a
+
       - name: deploy dspace-import on dev-5
         id: import
         working-directory: dspace-python-api/scripts
         run: |
           docker stop dspace-import-db5 || true
           DB5PORT=15432
-          cid=$(docker run -d --name dspace-import-db5 -v $pwd:/dq/scripts -v $(pwd)/../input/dump:/dq/dump \
+          # create otherwise it will be created with root owner
+          mkdir -p $(pwd)/../input/dump || true
+          cid=$(docker run -d --rm --name dspace-import-db5 -v $pwd:/dq/scripts -v $(pwd)/../input/dump:/dq/dump \
             -p 127.0.0.1:$DB5PORT:5432 -e POSTGRES_DB=empty -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=dspace postgres /bin/bash -c "cd /dq/scripts && ./init.dspacedb5.sh")
           echo "cid=$cid" >> $GITHUB_OUTPUT
           sleep 10
           cd ../src
           # cleanup resume
           rm __temp/resume/*.json || true
-          echo python repo_import.py --resume=false
+          # python repo_import.py --resume=false \
+          #   --config=backend.endpoint=http://dev-5.pc:88/server/api/ \
+          #   --config=db_dspace_7.port=5438 \
+          #   --config=db_dspace_5.port=$DB5PORT \
+          #   --config=db_utilities_5.port=$DB5PORT
 
       - name: cleanup
         run: |

build-scripts/run/start.sh

@@ -1,29 +1,43 @@
 #!/bin/bash
-echo "testing existence of envfile"
 if [[ "x$ENVFILE" == "x" ]]; then
     export ENVFILE=$(pwd)/envs/.default
-    echo "setting up new envfile"
+    echo "Using default envfile"
 fi
 
+PROJECT=${1:-unnamed_dspace}
+
+echo "Using envfile: [$ENVFILE] for project: [$PROJECT]"
+
 source $ENVFILE
 
 # docker-compose does not pull those that have `build` section?!
+echo "====="
 docker pull $DSPACE_UI_IMAGE
 
 pushd ../..
+echo "====="
 docker compose --env-file $ENVFILE -f docker/docker-compose.yml -f docker/docker-compose-rest.yml pull
-docker compose --env-file $ENVFILE -p ${1:-unnamed_dspace} -f docker/docker-compose.yml -f docker/docker-compose-rest.yml up -d --no-build
+docker compose --env-file $ENVFILE -p $PROJECT -f docker/docker-compose.yml -f docker/docker-compose-rest.yml up -d --no-build --remove-orphans
 popd
 
 # Create admin user
 # set DOCKER_OWNER to match our image (see cli.yml)
 pushd ../..
-#docker compose --env-file $ENVFILE -p ${1:-unnamed_dspace} -f docker/matomo-w-db.yml pull
-#docker compose --env-file $ENVFILE -p ${1:-unnamed_dspace} -f docker/matomo-w-db.yml up -d --no-build
-docker compose --env-file $ENVFILE -p ${1:-unnamed_dspace} -f docker/cli.yml run --rm dspace-cli create-administrator -e [email protected] -f admin -l user -p admin -c en -o dataquest
-docker compose --env-file $ENVFILE -p ${1:-unnamed_dspace} -f docker/cli.yml run --rm dspace-cli user --add -m user@test.edu -g meno -s priezvisko -l en -p user -o dataquest
-docker compose --env-file $ENVFILE -p ${1:-unnamed_dspace} -f docker/cli.yml run --rm dspace-cli version
-popd
+echo "====="
+#docker compose --env-file $ENVFILE -p $PROJECT -f docker/matomo-w-db.yml pull
+#docker compose --env-file $ENVFILE -p $PROJECT -f docker/matomo-w-db.yml up -d --no-build
+docker compose --env-file $ENVFILE -p $PROJECT -f docker/cli.yml run --rm dspace-cli create-administrator -e test@test.edu -f admin -l user -p admin -c en -o dataquest
+docker compose --env-file $ENVFILE -p $PROJECT -f docker/cli.yml run --rm dspace-cli user --add -m [email protected] -g meno -s priezvisko -l en -p user -o dataquest
+docker compose --env-file $ENVFILE -p $PROJECT -f docker/cli.yml run --rm dspace-cli version
 
+echo "====="
+echo "Logs"
+docker compose --env-file $ENVFILE -p $PROJECT -f docker/docker-compose.yml -f docker/docker-compose-rest.yml logs -n 50 || true
+popd
 
+echo "====="
+echo "Copy assetstore"
 docker cp assetstore dspace${INSTANCE}:/dspace/
+
+echo "====="
+echo "Finished start.sh"