Merge to v4 2024 10 03 (#66)

chore: add turborepo cache feat: add python prepare and cache action, add save-always option to cache action chore: add link to detailed description for glob search files action chore: publish-to-public pnpm docker chore: add append option for npmrc action merge v4-beta to v4 --------- Co-authored-by: Korenevskiy Denis <[email protected]>
milaboratory · Oct 3, 2024 · dfd44f2 · dfd44f2
1 parent 2d153ec
commit dfd44f2
Show file tree

Hide file tree

Showing 20 changed files with 435 additions and 252 deletions.
diff --git a/.github/workflows/node-docker-simple-fast-pnpm.yaml b/.github/workflows/node-docker-simple-fast-pnpm.yaml
@@ -74,7 +74,7 @@ on:
         required: false
         default: 'v1'
 
-      hashfiles-search-path:
+      cache-hashfiles-search-path:
         description: |
           Hashfiles search path for pnpm-lock.yaml
         type: string
@@ -86,11 +86,36 @@ on:
       #
       pnpm-version:
         description: |
-          Version of pnpm to install
+          Version of pnpm to install.
         type: string
         required: false
         default: 'latest'
 
+      pnpm-recursive-build:
+        description: |
+          Run pnpm build with -r flag.
+        type: boolean
+        required: true
+
+      pnpm-recursive-tests:
+        description: |
+          Run pnpm tests with -r flag.
+        type: boolean
+        required: true
+
+      pnpm-build-args:
+        description: |
+          Additional pnpm build arguments to pass
+          for none --recursive pnpm invocations.
+        type: string
+        required: false
+
+      pnpm-tests-args:
+        description: |
+          Additional pnpm tests arguments to pass
+          for none --recursive pnpm invocations.
+        type: string
+        required: false
       #
       # Build control
       #
@@ -127,49 +152,30 @@ on:
       #
       # NPM registry settings
       #
-      registry-url:
-        description: |
-          The npm registry url to set up for auth
-          and publication of the node js package.
-        type: string
-        required: false
-        default: 'https://registry.npmjs.org'
-
-      scope:
+      npmrc-config:
         description: |
-          Scope for authenticating against npm registries.
+          Generates a .npmrc file based on provided configuration.
         type: string
-        required: false
-        default: 'milaboratories'
+        required: true
 
-      always-auth:
+      #
+      # Distribution control
+      #
+      publish-to-public:
         description: |
-          Set always-auth in npmrc.
+          Whether or not the npm package should be published as public.
         type: string
         required: false
         default: 'false'
-
       #
       # AWS ECR settings
       #
-      aws-iam-role-to-assume:
-        description: |
-          The ARN of a IAM role to assume and configure the Actions environment with
-          assumed role credentials.
-        type: string
-        required: false
       aws-region:
         description: |
           The AWS region
         type: string
         required: false
         default: 'eu-central-1'
-      aws-s3-turborepo-cache-bucket:
-        description: |
-          Name of the Turborepo Cache s3 bucket
-        type: string
-        required: false
-        default: 'milab-euce1-prod-data-s3-turborepo-cache'
       #
       # Platforma Docker settings
       #
@@ -223,6 +229,15 @@ on:
         type: boolean
         required: false
         default: true
+      #
+      # Debug
+      #
+      interactive-debug:
+        description: |
+          Enable interactive debugging session
+        type: string
+        default: ''
+        required: false
 
       #
       # Notifications
@@ -339,7 +354,7 @@ jobs:
 
       - uses: aws-actions/configure-aws-credentials@v4
         with:
-          role-to-assume: ${{ inputs.aws-iam-role-to-assume || env.AWS_CI_IAM_MONOREPO_SIMPLE_ROLE }}
+          role-to-assume: ${{ env.AWS_CI_IAM_MONOREPO_SIMPLE_ROLE }}
           aws-region: ${{ inputs.aws-region }}
 
       - id: login-ecr
@@ -348,43 +363,44 @@ jobs:
           mask-password: 'true'
 
       - uses: milaboratory/github-ci/actions/turborepo/cache@v4
-        with:
-          storage-provider: 's3'
-          storage-path: ${{ inputs.aws-s3-turborepo-cache-bucket }}
 
-      - uses: milaboratory/github-ci-internal/blocks/node/build-and-test-pnpm@v4
+      - uses: milaboratory/github-ci/blocks/node/build-and-test-pnpm@v4
         env:
+          NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          NPMJS_TOKEN: ${{ env.NPMJS_TOKEN  }}
           PNPM_PKG_VERSION: ${{ fromJSON(steps.npm-pkg-metadata.outputs.data).pnpm-pkg-version }}
         with:
           build-script-name: ${{ inputs.build-script-name }}
           tests: 'false'
           test-script-name: ${{ inputs.test-script-name }}
+          pnpm-recursive-build: ${{ inputs.pnpm-recursive-build }}
+          pnpm-recursive-tests: ${{ inputs.pnpm-recursive-tests }}
+          pnpm-build-args: ${{ inputs.pnpm-build-args }}
+          pnpm-tests-args: ${{ inputs.pnpm-tests-args }}
           node-version: ${{ inputs.node-version }}
           cache-version: ${{ inputs.cache-version }}
-          hashfiles-search-path: ${{ inputs.hashfiles-search-path }}
+          cache-hashfiles-search-path: ${{ inputs.cache-hashfiles-search-path }}
           pnpm-version: ${{ env.PNPM_PKG_VERSION || inputs.pnpm-version }}
-          registry-url: ${{ inputs.registry-url }}
-          scope: ${{ inputs.scope }}
-          always-auth: ${{ inputs.always-auth }}
-          npm-auth-token: ${{ env.NPMJS_TOKEN || secrets.GITHUB_TOKEN }}
+          npmrc-config: ${{ inputs.npmrc-config }}
 
       - id: test
         uses: milaboratory/github-ci/blocks/pl/linux/test-pl-docker-pnpm@v4
         env:
           PL_ADDRESS: "http://127.0.0.1:6345"
           PL_TEST_USER: ${{ env.PL_CI_TEST_USER }}
           PL_TEST_PASSWORD: ${{ env.PL_CI_TEST_PASSWORD }}
+          NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          NPMJS_TOKEN: ${{ env.NPMJS_TOKEN  }}
           PNPM_PKG_VERSION: ${{ fromJSON(steps.npm-pkg-metadata.outputs.data).pnpm-pkg-version }}
         with:
-          test-name: ${{ inputs.test-script-name }}
+          test-script-name: ${{ inputs.test-script-name }}
+          pnpm-recursive-tests: ${{ inputs.pnpm-recursive-tests }}
+          pnpm-tests-args: ${{ inputs.pnpm-tests-args }}
           node-version: ${{ inputs.node-version }}
           cache-version: ${{ inputs.cache-version }}
           pnpm-version: ${{ env.PNPM_PKG_VERSION || inputs.pnpm-version }}
-          hashfiles-search-path: ${{ inputs.hashfiles-search-path }}
-          registry-url: ${{ inputs.registry-url }}
-          scope: ${{ inputs.scope }}
-          always-auth: ${{ inputs.always-auth }}
-          npm-auth-token: ${{ secrets.GITHUB_TOKEN }}
+          cache-hashfiles-search-path: ${{ inputs.cache-hashfiles-search-path }}
+          npmrc-config: ${{ inputs.npmrc-config }}
           pl-auth-enabled: ${{ inputs.pl-auth-enabled }}
           pl-docker-registry: ${{ format('{0}/{1}', steps.login-ecr.outputs.registry, 'pl') }}
           pl-docker-tag: ${{ inputs.pl-docker-tag }}
@@ -394,37 +410,29 @@ jobs:
           pl-log-commit-summary: ${{ inputs.pl-log-commit-summary }}
           pl-log-resource-data: ${{ inputs.pl-log-resource-data }}
 
-      - name: Search for npmrc_ci
-        uses: milaboratory/github-ci/actions/shell@v4
-        with:
-          run: |
-            NPMRC_CI="${GITHUB_WORKSPACE}/.npmrc_ci"
-            TARGET_NPMRC="${GITHUB_WORKSPACE}/.npmrc"
-            
-            if [ -f "${NPMRC_CI}" ];then
-              echo "Found npmrc_ci, rewriting .npmrc..."
-              cat "${NPMRC_CI}" > "${TARGET_NPMRC}"
-            else
-              echo "npmrc_ci not found, skipping..."
-            fi
-
-      - name: Checking for changesets on a pull_request
-        uses: milaboratory/github-ci/actions/shell@v4
-        if: github.event_name == 'pull_request'
+      - name: Create Release Pull Request and Publish npm package (Public)
+        if: steps.context.outputs.is-release == 'true' && inputs.publish-to-public == 'true'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          NPMJS_TOKEN: ${{ env.NPMJS_TOKEN }}
+        uses: changesets/action@v1
         with:
-          run: |
-            changeset status --since=main
+          commit: 'chore: update versions'
+          title: 'chore: update versions'
+          publish: pnpm -r publish --no-git-checks --access public
 
       - name: Create Release Pull Request and Publish npm package
-        if: steps.context.outputs.is-release == 'true'
+        if: steps.context.outputs.is-release == 'true' && inputs.publish-to-public == 'false'
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          NODE_AUTH_TOKEN: ${{ env.NPMJS_TOKEN || secrets.GITHUB_TOKEN }}
+          NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          NPMJS_TOKEN: ${{ env.NPMJS_TOKEN }}
         uses: changesets/action@v1
         with:
           commit: 'chore: update versions'
           title: 'chore: update versions'
-          publish: pnpm -r publish --no-git-checks --access public
+          publish: pnpm -r publish --no-git-checks
 
     outputs:
       # Even constant outputs become initialized only after job start.

diff --git a/actions/docker/pl-compose/action.yaml b/actions/docker/pl-compose/action.yaml
@@ -65,6 +65,16 @@ inputs:
       Whether or not to enable authentication in the Platforma.
     required: false
     default: 'true'
+  pl-data-create-bucket:
+    description: |
+      Try to automatically create buckets for storages if they not exist.
+    required: false
+    default: 'true'
+  pl-data-hardlink-mode:
+    description: |
+      Wether to use hardlinks when copying data between storages and within storage.
+    required: false
+    default: 'auto'
 
 runs:
   using: "composite"
@@ -83,6 +93,8 @@ runs:
         PL_LOG_RESOURCE_DATA: ${{ inputs.pl-log-resource-data }}
         PL_SLEEP_TIME: ${{ inputs.pl-sleep-time }}
         PL_AUTH_ENABLED: ${{ inputs.pl-auth-enabled }}
+        PL_DATA_CREATE_BUCKET: ${{ inputs.pl-data-create-bucket }}
+        PL_DATA_HARDLINK_MODE: ${{ inputs.pl-data-hardlink-mode }}
       with:
         dump-stdout: false
         run: |

diff --git a/actions/docker/pl-compose/docker-compose.yaml b/actions/docker/pl-compose/docker-compose.yaml
@@ -10,64 +10,82 @@ services:
     environment:
       MINIO_ROOT_USER: testuser
       MINIO_ROOT_PASSWORD: testpassword
-
   platforma:
     image: ${PL_DOCKER_REGISTRY}:${PL_DOCKER_TAG}
     ports:
       - ${PL_API_PORT:-6345}:${PL_API_PORT:-6345}
     tmpfs: [ /tmp ]
-
     logging:
       driver: json-file
       options:
         max-size: "100m"
         max-file: "4"
-
     environment:
+      # dump platforma's configuration to stdout
+      - "PL_DUMP_CONFIG_BEFORE_RUN=true"
+      # license settings
       - "MI_LICENSE=${MI_LICENSE:-}"
       - "PL_LICENSE=${PL_LICENSE:-}"
       - "PL_LICENSE_FILE=${PL_LICENSE_FILE:+/storage/license/${PL_LICENSE_FILE}}"
+      # network settings
       - "PL_API_ADDR=0.0.0.0:${PL_API_PORT:-6345}"
+      - "PL_GRPC_TLS_ENABLED=${PL_GRPC_TLS_ENABLED:-false}"
+      - "PL_GRPC_TLS_AUTH_MODE=${PL_GRPC_TLS_AUTH_MODE:-}"
+      # logging settings
       - "PL_LOG_LEVEL=${PL_LOG_LEVEL:-info}"
       - "PL_LOG_DESTINATION=${PL_LOG_DESTINATION:-stdout}"
       - "PL_LOG_COMMIT_SUMMARY=${PL_LOG_COMMIT_SUMMARY:-false}"
       - "PL_LOG_RESOURCE_DATA=${PL_LOG_RESOURCE_DATA:-false}"
+      # auth settings
       - "PL_AUTH_ENABLED=${PL_AUTH_ENABLED:-false}"
-      - "PL_DUMP_CONFIG_BEFORE_RUN=yes"
-      - "PLC_FILE_PRIMARY_S3_ENDPOINT=http://minio:9000/"
-      - "PLC_FILE_PRIMARY_S3_KEY=static:testuser"
-      - "PLC_FILE_PRIMARY_S3_SECRET=static:testpassword"
-      - "PLC_FILE_PRIMARY_S3_PRESIGN_ENDPOINT=${PLC_FILE_PRIMARY_S3_PRESIGN_ENDPOINT:-http://localhost:9000}"
+      # data controller: general settings
+      - "PL_DATA_CREATE_BUCKET=${PL_DATA_CREATE_BUCKET:-true}"
+      - "PL_DATA_HARDLINK_MODE=${PL_DATA_HARDLINK_MODE:-auto}"
+      # data controller: primary storage
+      - "PL_DATA_PRIMARY_TYPE=${PL_DATA_PRIMARY_TYPE:-S3}"
+      - "PL_DATA_PRIMARY_S3_ENDPOINT=${PL_DATA_PRIMARY_S3_ENDPOINT:-http://minio:9000/}"
+      - "PL_DATA_PRIMARY_S3_PRESIGN_ENDPOINT=${PL_DATA_PRIMARY_S3_PRESIGN_ENDPOINT:-${PL_DATA_PRIMARY_S3_ENDPOINT:-http://localhost:9000/}}"
+      - "PL_DATA_PRIMARY_S3_KEY_PREFIX=${PL_DATA_PRIMARY_S3_KEY_PREFIX:-}"
+      - "PL_DATA_PRIMARY_S3_REGION=${PL_DATA_PRIMARY_S3_REGION:-eu-central-1}"
+      - "PL_DATA_PRIMARY_S3_BUCKET=${PL_DATA_PRIMARY_S3_BUCKET:-platforma-primary-bucket}"
+      - "PL_DATA_PRIMARY_S3_KEY=${PL_DATA_PRIMARY_S3_KEY:-static:testuser}"
+      - "PL_DATA_PRIMARY_S3_SECRET=${PL_DATA_PRIMARY_S3_SECRET:-static:testpassword}"
+      # data controller: library storage
+      - "PL_DATA_LIBRARY_TYPE=${PL_DATA_LIBRARY_TYPE:-FS}"
     volumes:
-      - pl_test_assets:/storage/controllers/file/library
-      - pl_log:/storage/log
-      - pl_license:/storage/license
-
+      - platforma-file-primary:/storage/controllers/data/primary
+      - platforma-file-library:/storage/controllers/data/library
+      - platforma-file-work:/storage/controllers/data/work
+      - platforma-packages:/storage/controllers/packageLoader
+      - platforma-db:/storage/rocksdb
+      - platforma-log:/storage/log
+      - platforma-license:/storage/license
     restart: always
 
 volumes:
-  host_tmp:
+  host-tmp:
     driver: local
     driver_opts:
       type: none
       o: bind
       device: ${TMPDIR:-/tmp}
-
-  pl_test_assets:
+  platforma-packages:
+  platforma-file-work:
+  platforma-file-primary:
+  platforma-file-library:
     driver: local
     driver_opts:
       type: none
       o: bind
       device: ${PL_TEST_ASSETS_DIR}
-
-  pl_log:
+  platforma-db:
+  platforma-log:
     driver: local
     driver_opts:
       type: none
       o: bind
       device: ${PL_LOG_DIR}
-
-  pl_license:
+  platforma-license:
     driver: local
     driver_opts:
       type: none