From 8d0cbad8b6c68c14d849af4a2c0fd4c7f3aec5c8 Mon Sep 17 00:00:00 2001 From: Kristaps Fabians Geikins Date: Tue, 10 Dec 2024 09:05:18 +0000 Subject: [PATCH] feat(server): multi region blob support (#3653) Co-authored-by: Iain Sproat <68657+iainsproat@users.noreply.github.com> --- .circleci/config.yml | 4 +- .circleci/multiregion.test-ci.json | 16 ++ README.md | 8 + packages/server/db/migrations.ts | 1 + .../activitystream/repositories/index.ts | 2 +- .../automate/graph/resolvers/automate.ts | 2 +- packages/server/modules/automate/index.ts | 2 +- .../server/modules/automate/rest/logStream.ts | 2 +- .../blobstorage/clients/objectStorage.ts | 59 ++++++ .../modules/blobstorage/domain/operations.ts | 12 +- .../blobstorage/domain/storageOperations.ts | 23 +++ .../blobstorage/graph/resolvers/index.ts | 2 +- packages/server/modules/blobstorage/index.ts | 57 ++++-- .../modules/blobstorage/objectStorage.ts | 172 ------------------ .../modules/blobstorage/repositories/blobs.ts | 157 ++++++++++++++++ .../blobstorage/services/management.ts | 6 +- .../tests/blobstorage.integration.spec.js | 95 ++++------ ...lobstorage.spec.js => blobstorage.spec.ts} | 115 ++++++++---- .../tests/{helpers.js => helpers.ts} | 25 ++- .../modules/cli/commands/db/helpers/index.ts | 2 +- .../modules/cli/commands/download/commit.ts | 2 +- .../modules/cli/commands/download/project.ts | 2 +- .../comments/graph/resolvers/comments.ts | 2 +- .../modules/core/graph/resolvers/branches.ts | 2 +- .../modules/core/graph/resolvers/commits.ts | 2 +- .../modules/core/graph/resolvers/common.ts | 2 +- .../modules/core/graph/resolvers/models.ts | 2 +- .../modules/core/graph/resolvers/objects.ts | 2 +- .../modules/core/graph/resolvers/projects.ts | 2 +- .../modules/core/graph/resolvers/versions.ts | 2 +- .../server/modules/core/rest/diffDownload.ts | 2 +- .../server/modules/core/rest/diffUpload.ts | 2 +- packages/server/modules/core/rest/download.ts | 2 +- packages/server/modules/core/rest/upload.ts | 2 +- .../core/tests/integration/subs.graph.spec.ts | 2 +- .../graph/resolvers/fileUploads.ts | 2 +- packages/server/modules/fileuploads/index.ts | 2 +- .../modules/gendo/graph/resolvers/index.ts | 16 +- packages/server/modules/gendo/rest/index.ts | 21 +-- .../modules/multiregion/domain/operations.ts | 11 +- .../multiregion/graph/resolvers/index.ts | 9 +- packages/server/modules/multiregion/index.ts | 14 +- .../modules/multiregion/regionConfig.ts | 16 +- .../multiregion/services/management.ts | 12 ++ .../tests/e2e/serverAdmin.graph.spec.ts | 27 ++- .../multiregion/utils/blobStorageSelector.ts | 124 +++++++++++++ .../multiregion/{ => utils}/dbSelector.ts | 55 ++---- .../multiregion/utils/regionSelector.ts | 74 ++++++++ .../services/handlers/mentionedInComment.ts | 2 +- packages/server/modules/previews/index.ts | 2 +- .../server/modules/previews/resultListener.ts | 2 +- .../webhooks/graph/resolvers/webhooks.ts | 2 +- packages/server/modules/webhooks/index.ts | 2 +- .../workspaces/graph/resolvers/regions.ts | 2 +- .../workspaces/graph/resolvers/workspaces.ts | 2 +- .../modules/workspaces/services/projects.ts | 2 +- .../workspaces/tests/helpers/creation.ts | 2 +- packages/server/multiregion.example.json | 10 +- packages/server/multiregion.test.example.json | 12 +- packages/server/readme.md | 6 +- packages/server/test/hooks.ts | 56 ++++-- packages/server/test/mocks/global.ts | 8 +- .../test/speckle-helpers/branchHelper.ts | 2 +- .../test/speckle-helpers/commitHelper.ts | 2 +- .../src/environment/multiRegionConfig.ts | 41 +++-- 65 files changed, 878 insertions(+), 452 deletions(-) create mode 100644 packages/server/modules/blobstorage/clients/objectStorage.ts create mode 100644 packages/server/modules/blobstorage/domain/storageOperations.ts delete mode 100644 packages/server/modules/blobstorage/objectStorage.ts create mode 100644 packages/server/modules/blobstorage/repositories/blobs.ts rename packages/server/modules/blobstorage/tests/{blobstorage.spec.js => blobstorage.spec.ts} (77%) rename packages/server/modules/blobstorage/tests/{helpers.js => helpers.ts} (64%) create mode 100644 packages/server/modules/multiregion/utils/blobStorageSelector.ts rename packages/server/modules/multiregion/{ => utils}/dbSelector.ts (82%) create mode 100644 packages/server/modules/multiregion/utils/regionSelector.ts diff --git a/.circleci/config.yml b/.circleci/config.yml index f94131d7b0..9a7eda562c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -597,7 +597,9 @@ jobs: POSTGRES_USER: speckle command: -c 'max_connections=1000' -c 'port=5433' -c 'wal_level=logical' - image: 'minio/minio' - command: server /data --console-address ":9001" + command: server /data --console-address ":9001" --address "0.0.0.0:9000" + - image: 'minio/minio' + command: server /data --console-address ":9021" --address "0.0.0.0:9020" environment: # Same as test-server: NODE_ENV: test diff --git a/.circleci/multiregion.test-ci.json b/.circleci/multiregion.test-ci.json index bbf5d3142b..3d5a9ec1c3 100644 --- a/.circleci/multiregion.test-ci.json +++ b/.circleci/multiregion.test-ci.json @@ -2,12 +2,28 @@ "main": { "postgres": { "connectionUri": "postgresql://speckle:speckle@127.0.0.1:5432/speckle2_test" + }, + "blobStorage": { + "accessKey": "minioadmin", + "secretKey": "minioadmin", + "bucket": "speckle-server", + "createBucketIfNotExists": true, + "endpoint": "http://127.0.0.1:9000", + "s3Region": "us-east-1" } }, "regions": { "region1": { "postgres": { "connectionUri": "postgresql://speckle:speckle@127.0.0.1:5433/speckle2_test" + }, + "blobStorage": { + "accessKey": "minioadmin", + "secretKey": "minioadmin", + "bucket": "speckle-server", + "createBucketIfNotExists": true, + "endpoint": "http://127.0.0.1:9020", + "s3Region": "us-east-1" } } } diff --git a/README.md b/README.md index 68bd40db63..0a9ef1fc47 100644 --- a/README.md +++ b/README.md @@ -99,6 +99,14 @@ EMAIL_PORT="1025" The web portal is available at `localhost:1080` and it's listening for mail on port `1025`. +### Minio (S3 storage) + +Default credentials are: `minioadmin:minioadmin` +Main storage Web UI: [http://localhost:9001/](http://localhost:9001/) +Region1 storage Web UI: [http://localhost:9021/](http://localhost:9021/) + +You can use the web UI to validate uploaded blobs + # Contributing Please make sure you read the [contribution guidelines](https://github.com/specklesystems/speckle-server/blob/main/CONTRIBUTING.md) for an overview of the best practices we try to follow. diff --git a/packages/server/db/migrations.ts b/packages/server/db/migrations.ts index f93e418a54..bab6e69f19 100644 --- a/packages/server/db/migrations.ts +++ b/packages/server/db/migrations.ts @@ -7,5 +7,6 @@ export const migrateDbToLatest = async (params: { db: Knex; region: string }) => await db.migrate.latest() } catch (err: unknown) { logger.error({ err, region }, 'Error migrating db to latest for region "{region}".') + throw err } } diff --git a/packages/server/modules/activitystream/repositories/index.ts b/packages/server/modules/activitystream/repositories/index.ts index 049eef771c..3f4d8819cc 100644 --- a/packages/server/modules/activitystream/repositories/index.ts +++ b/packages/server/modules/activitystream/repositories/index.ts @@ -26,7 +26,7 @@ import { Knex } from 'knex' import { getStreamFactory } from '@/modules/core/repositories/streams' import { getUserFactory } from '@/modules/core/repositories/users' import { getServerInfoFactory } from '@/modules/core/repositories/server' -import { getProjectDbClient } from '@/modules/multiregion/dbSelector' +import { getProjectDbClient } from '@/modules/multiregion/utils/dbSelector' const tables = { streamActivity: (db: Knex) => diff --git a/packages/server/modules/automate/graph/resolvers/automate.ts b/packages/server/modules/automate/graph/resolvers/automate.ts index f72fd21ad5..9d79aad217 100644 --- a/packages/server/modules/automate/graph/resolvers/automate.ts +++ b/packages/server/modules/automate/graph/resolvers/automate.ts @@ -125,7 +125,7 @@ import { storeUserServerAppTokenFactory } from '@/modules/core/repositories/tokens' import { getEventBus } from '@/modules/shared/services/eventBus' -import { getProjectDbClient } from '@/modules/multiregion/dbSelector' +import { getProjectDbClient } from '@/modules/multiregion/utils/dbSelector' const { FF_AUTOMATE_MODULE_ENABLED } = getFeatureFlags() diff --git a/packages/server/modules/automate/index.ts b/packages/server/modules/automate/index.ts index 1928de46a7..8570076ce9 100644 --- a/packages/server/modules/automate/index.ts +++ b/packages/server/modules/automate/index.ts @@ -45,7 +45,7 @@ import { storeTokenScopesFactory, storeUserServerAppTokenFactory } from '@/modules/core/repositories/tokens' -import { getProjectDbClient } from '@/modules/multiregion/dbSelector' +import { getProjectDbClient } from '@/modules/multiregion/utils/dbSelector' import { ProjectAutomationsUpdatedMessageType, ProjectTriggeredAutomationsStatusUpdatedMessageType diff --git a/packages/server/modules/automate/rest/logStream.ts b/packages/server/modules/automate/rest/logStream.ts index 51534e76f3..5d5fdea153 100644 --- a/packages/server/modules/automate/rest/logStream.ts +++ b/packages/server/modules/automate/rest/logStream.ts @@ -4,7 +4,7 @@ import { ExecutionEngineFailedResponseError } from '@/modules/automate/errors/ex import { getAutomationRunWithTokenFactory } from '@/modules/automate/repositories/automations' import { corsMiddleware } from '@/modules/core/configs/cors' import { getStreamFactory } from '@/modules/core/repositories/streams' -import { getProjectDbClient } from '@/modules/multiregion/dbSelector' +import { getProjectDbClient } from '@/modules/multiregion/utils/dbSelector' import { validateRequiredStreamFactory, validateResourceAccess, diff --git a/packages/server/modules/blobstorage/clients/objectStorage.ts b/packages/server/modules/blobstorage/clients/objectStorage.ts new file mode 100644 index 0000000000..252d52afef --- /dev/null +++ b/packages/server/modules/blobstorage/clients/objectStorage.ts @@ -0,0 +1,59 @@ +import { + getS3AccessKey, + getS3BucketName, + getS3Endpoint, + getS3Region, + getS3SecretKey +} from '@/modules/shared/helpers/envHelper' +import { S3Client, S3ClientConfig } from '@aws-sdk/client-s3' +import { Optional } from '@speckle/shared' + +export type ObjectStorage = { + client: S3Client + bucket: string +} + +export type GetObjectStorageParams = { + credentials: S3ClientConfig['credentials'] + endpoint: S3ClientConfig['endpoint'] + region: S3ClientConfig['region'] + bucket: string +} + +/** + * Get object storage client + */ +export const getObjectStorage = (params: GetObjectStorageParams): ObjectStorage => { + const { bucket, credentials, endpoint, region } = params + + const config: S3ClientConfig = { + credentials, + endpoint, + region, + forcePathStyle: true + } + const client = new S3Client(config) + return { client, bucket } +} + +let mainObjectStorage: Optional = undefined + +/** + * Get main object storage client + */ +export const getMainObjectStorage = (): ObjectStorage => { + if (mainObjectStorage) return mainObjectStorage + + const mainParams: GetObjectStorageParams = { + credentials: { + accessKeyId: getS3AccessKey(), + secretAccessKey: getS3SecretKey() + }, + endpoint: getS3Endpoint(), + region: getS3Region(), + bucket: getS3BucketName() + } + + mainObjectStorage = getObjectStorage(mainParams) + return mainObjectStorage +} diff --git a/packages/server/modules/blobstorage/domain/operations.ts b/packages/server/modules/blobstorage/domain/operations.ts index 71d0438814..8b90d3c012 100644 --- a/packages/server/modules/blobstorage/domain/operations.ts +++ b/packages/server/modules/blobstorage/domain/operations.ts @@ -4,6 +4,7 @@ import { } from '@/modules/blobstorage/domain/types' import { MaybeNullOrUndefined, Nullable } from '@speckle/shared' import type { Readable } from 'stream' +import { StoreFileStream } from '@/modules/blobstorage/domain/storageOperations' export type GetBlobs = (params: { streamId?: MaybeNullOrUndefined @@ -33,11 +34,11 @@ export type GetBlobMetadataCollection = (params: { }) => Promise<{ blobs: BlobStorageItem[]; cursor: Nullable }> export type UploadFileStream = ( - params1: { + streamData: { streamId: string userId: string | undefined }, - params2: { + blobData: { blobId: string fileName: string fileType: string | undefined @@ -45,9 +46,4 @@ export type UploadFileStream = ( } ) => Promise<{ blobId: string; fileName: string; fileHash: string }> -type FileStream = string | Blob | Readable | Uint8Array | Buffer - -export type StoreFileStream = (args: { - objectKey: string - fileStream: FileStream -}) => Promise<{ fileHash: string }> +export { StoreFileStream } diff --git a/packages/server/modules/blobstorage/domain/storageOperations.ts b/packages/server/modules/blobstorage/domain/storageOperations.ts new file mode 100644 index 0000000000..abe636ed60 --- /dev/null +++ b/packages/server/modules/blobstorage/domain/storageOperations.ts @@ -0,0 +1,23 @@ +import type stream from 'stream' +import type { Readable } from 'stream' + +export type GetObjectStream = (params: { + objectKey: string +}) => Promise + +export type GetObjectAttributes = (params: { objectKey: string }) => Promise<{ + fileSize: number +}> + +type FileStream = string | Blob | Readable | Uint8Array | Buffer + +export type StoreFileStream = (args: { + objectKey: string + fileStream: FileStream +}) => Promise<{ fileHash: string }> + +export type DeleteObject = (params: { objectKey: string }) => Promise + +export type EnsureStorageAccess = (params: { + createBucketIfNotExists: boolean +}) => Promise diff --git a/packages/server/modules/blobstorage/graph/resolvers/index.ts b/packages/server/modules/blobstorage/graph/resolvers/index.ts index 633e3149e4..23faf86d5f 100644 --- a/packages/server/modules/blobstorage/graph/resolvers/index.ts +++ b/packages/server/modules/blobstorage/graph/resolvers/index.ts @@ -12,7 +12,7 @@ import { StreamBlobsArgs } from '@/modules/core/graph/generated/graphql' import { StreamGraphQLReturn } from '@/modules/core/helpers/graphTypes' -import { getProjectDbClient } from '@/modules/multiregion/dbSelector' +import { getProjectDbClient } from '@/modules/multiregion/utils/dbSelector' import { BadRequestError, NotFoundError, diff --git a/packages/server/modules/blobstorage/index.ts b/packages/server/modules/blobstorage/index.ts index 60b6e177fc..dd46613f47 100644 --- a/packages/server/modules/blobstorage/index.ts +++ b/packages/server/modules/blobstorage/index.ts @@ -6,13 +6,6 @@ import { streamWritePermissionsPipelineFactory, streamReadPermissionsPipelineFactory } from '@/modules/shared/authz' -import { - ensureStorageAccess, - storeFileStream, - getObjectStream, - deleteObject, - getObjectAttributes -} from '@/modules/blobstorage/objectStorage' import crs from 'crypto-random-string' import { authMiddlewareCreator } from '@/modules/shared/middleware' import { isArray } from 'lodash' @@ -42,12 +35,24 @@ import { fullyDeleteBlobFactory } from '@/modules/blobstorage/services/management' import { getRolesFactory } from '@/modules/shared/repositories/roles' -import { adminOverrideEnabled } from '@/modules/shared/helpers/envHelper' +import { + adminOverrideEnabled, + createS3Bucket +} from '@/modules/shared/helpers/envHelper' import { getStreamFactory } from '@/modules/core/repositories/streams' import { Request, Response } from 'express' import { ensureError } from '@speckle/shared' import { SpeckleModule } from '@/modules/shared/helpers/typeHelper' -import { getProjectDbClient } from '@/modules/multiregion/dbSelector' +import { getProjectDbClient } from '@/modules/multiregion/utils/dbSelector' +import { + deleteObjectFactory, + ensureStorageAccessFactory, + getObjectAttributesFactory, + getObjectStreamFactory, + storeFileStreamFactory +} from '@/modules/blobstorage/repositories/blobs' +import { getMainObjectStorage } from '@/modules/blobstorage/clients/objectStorage' +import { getProjectObjectStorage } from '@/modules/multiregion/utils/blobStorageSelector' const ensureConditions = async () => { if (process.env.DISABLE_FILE_UPLOADS) { @@ -55,7 +60,11 @@ const ensureConditions = async () => { return } else { moduleLogger.info('📦 Init BlobStorage module') - await ensureStorageAccess() + const storage = getMainObjectStorage() + const ensureStorageAccess = ensureStorageAccessFactory({ storage }) + await ensureStorageAccess({ + createBucketIfNotExists: createS3Bucket() + }) } if (!process.env.S3_BUCKET) { @@ -125,8 +134,12 @@ export const init: SpeckleModule['init'] = async (app) => { limits: { fileSize: getFileSizeLimit() } }) - const projectDb = await getProjectDbClient({ projectId: streamId }) + const [projectDb, projectStorage] = await Promise.all([ + getProjectDbClient({ projectId: streamId }), + getProjectObjectStorage({ projectId: streamId }) + ]) + const storeFileStream = storeFileStreamFactory({ storage: projectStorage }) const updateBlob = updateBlobFactory({ db: projectDb }) const getBlobMetadata = getBlobMetadataFactory({ db: projectDb }) @@ -146,6 +159,11 @@ export const init: SpeckleModule['init'] = async (app) => { updateBlob }) + const getObjectAttributes = getObjectAttributesFactory({ + storage: projectStorage + }) + const deleteObject = deleteObjectFactory({ storage: projectStorage }) + busboy.on('file', (formKey, file, info) => { const { filename: fileName } = info const fileType = fileName?.split('.')?.pop()?.toLowerCase() @@ -275,9 +293,15 @@ export const init: SpeckleModule['init'] = async (app) => { }, async (req, res) => { errorHandler(req, res, async (req, res) => { - const projectDb = await getProjectDbClient({ projectId: req.params.streamId }) + const streamId = req.params.streamId + const [projectDb, projectStorage] = await Promise.all([ + getProjectDbClient({ projectId: streamId }), + getProjectObjectStorage({ projectId: streamId }) + ]) + const getBlobMetadata = getBlobMetadataFactory({ db: projectDb }) const getFileStream = getFileStreamFactory({ getBlobMetadata }) + const getObjectStream = getObjectStreamFactory({ storage: projectStorage }) const { fileName } = await getBlobMetadata({ streamId: req.params.streamId, @@ -304,12 +328,19 @@ export const init: SpeckleModule['init'] = async (app) => { }, async (req, res) => { errorHandler(req, res, async (req, res) => { - const projectDb = await getProjectDbClient({ projectId: req.params.streamId }) + const streamId = req.params.streamId + const [projectDb, projectStorage] = await Promise.all([ + getProjectDbClient({ projectId: streamId }), + getProjectObjectStorage({ projectId: streamId }) + ]) + const getBlobMetadata = getBlobMetadataFactory({ db: projectDb }) const deleteBlob = fullyDeleteBlobFactory({ getBlobMetadata, deleteBlob: deleteBlobFactory({ db: projectDb }) }) + const deleteObject = deleteObjectFactory({ storage: projectStorage }) + await deleteBlob({ streamId: req.params.streamId, blobId: req.params.blobId, diff --git a/packages/server/modules/blobstorage/objectStorage.ts b/packages/server/modules/blobstorage/objectStorage.ts deleted file mode 100644 index 8ae08dfe28..0000000000 --- a/packages/server/modules/blobstorage/objectStorage.ts +++ /dev/null @@ -1,172 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ -import { - NotFoundError, - EnvironmentResourceError, - BadRequestError -} from '@/modules/shared/errors' -import { - S3Client, - GetObjectCommand, - HeadBucketCommand, - DeleteObjectCommand, - CreateBucketCommand, - S3ServiceException, - S3ClientConfig, - ServiceOutputTypes -} from '@aws-sdk/client-s3' -import { Upload } from '@aws-sdk/lib-storage' -import { - getS3AccessKey, - getS3SecretKey, - getS3Endpoint, - getS3Region, - getS3BucketName, - createS3Bucket -} from '@/modules/shared/helpers/envHelper' -import { ensureError, Nullable } from '@speckle/shared' -import { get } from 'lodash' -import type { Command } from '@aws-sdk/smithy-client' -import type stream from 'stream' -import { StoreFileStream } from '@/modules/blobstorage/domain/operations' - -let s3Config: Nullable = null - -const getS3Config = () => { - if (!s3Config) { - s3Config = { - credentials: { - accessKeyId: getS3AccessKey(), - secretAccessKey: getS3SecretKey() - }, - endpoint: getS3Endpoint(), - forcePathStyle: true, - // s3ForcePathStyle: true, - // signatureVersion: 'v4', - region: getS3Region() - } - } - return s3Config -} - -let storageBucket: Nullable = null - -const getStorageBucket = () => { - if (!storageBucket) { - storageBucket = getS3BucketName() - } - return storageBucket -} - -const getObjectStorage = () => ({ - client: new S3Client(getS3Config()), - Bucket: getStorageBucket(), - createBucket: createS3Bucket() -}) - -const sendCommand = async ( - command: (Bucket: string) => Command -) => { - const { client, Bucket } = getObjectStorage() - try { - const ret = await client.send(command(Bucket)) - return ret - } catch (err) { - if (err instanceof S3ServiceException && get(err, 'Code') === 'NoSuchKey') - throw new NotFoundError(err.message) - throw err - } -} - -export const getObjectStream = async ({ objectKey }: { objectKey: string }) => { - const data = await sendCommand( - (Bucket) => new GetObjectCommand({ Bucket, Key: objectKey }) - ) - - // TODO: Apparently not always stream.Readable according to types, but in practice this works - return data.Body as stream.Readable -} - -export const getObjectAttributes = async ({ objectKey }: { objectKey: string }) => { - const data = await sendCommand( - (Bucket) => new GetObjectCommand({ Bucket, Key: objectKey }) - ) - return { fileSize: data.ContentLength || 0 } -} - -export const storeFileStream: StoreFileStream = async ({ objectKey, fileStream }) => { - const { client, Bucket } = getObjectStorage() - const parallelUploads3 = new Upload({ - client, - params: { Bucket, Key: objectKey, Body: fileStream }, - tags: [ - /*...*/ - ], // optional tags - queueSize: 4, // optional concurrency configuration - partSize: 1024 * 1024 * 5, // optional size of each part, in bytes, at least 5MB - leavePartsOnError: false // optional manually handle dropped parts - }) - - const data = await parallelUploads3.done() - // the ETag is a hash of the object. Could be used to dedupe stuff... - - if (!data || !('ETag' in data) || !data.ETag) { - throw new BadRequestError('No ETag in response') - } - - const fileHash = data.ETag.replaceAll('"', '') - return { fileHash } -} - -export const deleteObject = async ({ objectKey }: { objectKey: string }) => { - await sendCommand((Bucket) => new DeleteObjectCommand({ Bucket, Key: objectKey })) -} - -// No idea what the actual error type is, too difficult to figure out -type EnsureStorageAccessError = Error & { - statusCode?: number - $metadata?: { httpStatusCode?: number } -} - -const isExpectedEnsureStorageAccessError = ( - err: unknown -): err is EnsureStorageAccessError => - err instanceof Error && ('statusCode' in err || '$metadata' in err) - -export const ensureStorageAccess = async () => { - const { client, Bucket, createBucket } = getObjectStorage() - try { - await client.send(new HeadBucketCommand({ Bucket })) - return - } catch (err) { - if ( - isExpectedEnsureStorageAccessError(err) && - (err.statusCode === 403 || err['$metadata']?.httpStatusCode === 403) - ) { - throw new EnvironmentResourceError("Access denied to S3 bucket '{bucket}'", { - cause: err, - info: { bucket: Bucket } - }) - } - if (createBucket) { - try { - await client.send(new CreateBucketCommand({ Bucket })) - } catch (err) { - throw new EnvironmentResourceError( - "Can't open S3 bucket '{bucket}', and have failed to create it.", - { - cause: ensureError(err), - info: { bucket: Bucket } - } - ) - } - } else { - throw new EnvironmentResourceError( - "Can't open S3 bucket '{bucket}', and the Speckle server configuration has disabled creation of the bucket.", - { - cause: ensureError(err), - info: { bucket: Bucket } - } - ) - } - } -} diff --git a/packages/server/modules/blobstorage/repositories/blobs.ts b/packages/server/modules/blobstorage/repositories/blobs.ts new file mode 100644 index 0000000000..475403e1cc --- /dev/null +++ b/packages/server/modules/blobstorage/repositories/blobs.ts @@ -0,0 +1,157 @@ +import { ObjectStorage } from '@/modules/blobstorage/clients/objectStorage' +import { + DeleteObject, + EnsureStorageAccess, + GetObjectAttributes, + GetObjectStream, + StoreFileStream +} from '@/modules/blobstorage/domain/storageOperations' +import { + BadRequestError, + EnvironmentResourceError, + NotFoundError +} from '@/modules/shared/errors' +import { + CreateBucketCommand, + DeleteObjectCommand, + GetObjectCommand, + HeadBucketCommand, + S3ServiceException, + ServiceOutputTypes +} from '@aws-sdk/client-s3' +import { Upload } from '@aws-sdk/lib-storage' +import type { Command } from '@aws-sdk/smithy-client' +import { ensureError } from '@speckle/shared' +import { get } from 'lodash' +import type stream from 'stream' + +const sendCommand = async ( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + command: Command, + storage: ObjectStorage +) => { + const { client } = storage + try { + const ret = await client.send(command) + return ret + } catch (err) { + if (err instanceof S3ServiceException && get(err, 'Code') === 'NoSuchKey') + throw new NotFoundError(err.message) + throw err + } +} + +export const getObjectStreamFactory = + (deps: { storage: ObjectStorage }): GetObjectStream => + async ({ objectKey }) => { + const { storage } = deps + const data = await sendCommand( + new GetObjectCommand({ Bucket: storage.bucket, Key: objectKey }), + storage + ) + + // Apparently not always stream.Readable according to types, but in practice it always is + return data.Body as stream.Readable + } + +export const getObjectAttributesFactory = + (deps: { storage: ObjectStorage }): GetObjectAttributes => + async ({ objectKey }) => { + const { storage } = deps + const data = await sendCommand( + new GetObjectCommand({ Bucket: storage.bucket, Key: objectKey }), + storage + ) + + return { fileSize: data.ContentLength || 0 } + } + +export const storeFileStreamFactory = + (deps: { storage: ObjectStorage }): StoreFileStream => + async ({ objectKey, fileStream }) => { + const { + storage: { client, bucket: Bucket } + } = deps + + const upload = new Upload({ + client, + params: { Bucket, Key: objectKey, Body: fileStream }, + tags: [ + /*...*/ + ], // optional tags + queueSize: 4, // optional concurrency configuration + partSize: 1024 * 1024 * 5, // optional size of each part, in bytes, at least 5MB + leavePartsOnError: false // optional manually handle dropped parts + }) + + const data = await upload.done() + // the ETag is a hash of the object. Could be used to dedupe stuff... + + if (!data || !('ETag' in data) || !data.ETag) { + throw new BadRequestError('No ETag in response') + } + + const fileHash = data.ETag.replaceAll('"', '') + return { fileHash } + } + +export const deleteObjectFactory = + (deps: { storage: ObjectStorage }): DeleteObject => + async ({ objectKey }) => { + await sendCommand( + new DeleteObjectCommand({ Bucket: deps.storage.bucket, Key: objectKey }), + deps.storage + ) + } + +// No idea what the actual error type is, too difficult to figure out +type EnsureStorageAccessError = Error & { + statusCode?: number + $metadata?: { httpStatusCode?: number } +} + +const isExpectedEnsureStorageAccessError = ( + err: unknown +): err is EnsureStorageAccessError => + err instanceof Error && ('statusCode' in err || '$metadata' in err) + +export const ensureStorageAccessFactory = + (deps: { storage: ObjectStorage }): EnsureStorageAccess => + async ({ createBucketIfNotExists }) => { + const { client, bucket: Bucket } = deps.storage + try { + await client.send(new HeadBucketCommand({ Bucket })) + return + } catch (err) { + if ( + isExpectedEnsureStorageAccessError(err) && + (err.statusCode === 403 || err['$metadata']?.httpStatusCode === 403) + ) { + throw new EnvironmentResourceError("Access denied to S3 bucket '{bucket}'", { + cause: err, + info: { bucket: Bucket } + }) + } + if (createBucketIfNotExists) { + try { + await client.send(new CreateBucketCommand({ Bucket })) + } catch (err) { + throw new EnvironmentResourceError( + "Can't open S3 bucket '{bucket}', and have failed to create it.", + { + cause: ensureError(err), + info: { bucket: Bucket } + } + ) + } + } else { + throw new EnvironmentResourceError( + "Can't open S3 bucket '{bucket}', and the Speckle server configuration has disabled creation of the bucket.", + { + cause: ensureError(err), + info: { bucket: Bucket } + } + ) + } + } + } diff --git a/packages/server/modules/blobstorage/services/management.ts b/packages/server/modules/blobstorage/services/management.ts index 522fa4307d..c5a626e969 100644 --- a/packages/server/modules/blobstorage/services/management.ts +++ b/packages/server/modules/blobstorage/services/management.ts @@ -22,9 +22,9 @@ export const uploadFileStreamFactory = updateBlob: UpdateBlob storeFileStream: StoreFileStream }): UploadFileStream => - async (params1, params2) => { - const { streamId, userId } = params1 - const { blobId, fileName, fileType, fileStream } = params2 + async (streamData, blobData) => { + const { streamId, userId } = streamData + const { blobId, fileName, fileType, fileStream } = blobData if (streamId.length !== 10) throw new BadRequestError('The stream id has to be of length 10') diff --git a/packages/server/modules/blobstorage/tests/blobstorage.integration.spec.js b/packages/server/modules/blobstorage/tests/blobstorage.integration.spec.js index 29b994c182..e81dba53f5 100644 --- a/packages/server/modules/blobstorage/tests/blobstorage.integration.spec.js +++ b/packages/server/modules/blobstorage/tests/blobstorage.integration.spec.js @@ -1,40 +1,15 @@ const { Buffer } = require('node:buffer') const request = require('supertest') const expect = require('chai').expect -const { beforeEachContext } = require('@/test/hooks') +const { beforeEachContext, getMainTestRegionKeyIfMultiRegion } = require('@/test/hooks') const { Scopes } = require('@/modules/core/helpers/mainConstants') -const { - getStreamFactory, - createStreamFactory -} = require('@/modules/core/repositories/streams') const { db } = require('@/db/knex') const { - legacyCreateStreamFactory, - createStreamReturnRecordFactory -} = require('@/modules/core/services/streams/management') -const { - inviteUsersToProjectFactory -} = require('@/modules/serverinvites/services/projectInviteManagement') -const { - createAndSendInviteFactory -} = require('@/modules/serverinvites/services/creation') -const { - findUserByTargetFactory, - insertInviteAndDeleteOldFactory, deleteServerOnlyInvitesFactory, updateAllInviteTargetsFactory } = require('@/modules/serverinvites/repositories/serverInvites') + const { - collectAndValidateCoreTargetsFactory -} = require('@/modules/serverinvites/services/coreResourceCollection') -const { - buildCoreInviteEmailContentsFactory -} = require('@/modules/serverinvites/services/coreEmailContents') -const { getEventBus } = require('@/modules/shared/services/eventBus') -const { createBranchFactory } = require('@/modules/core/repositories/branches') -const { ProjectsEmitter } = require('@/modules/core/events/projectsEmitter') -const { - getUsersFactory, getUserFactory, storeUserFactory, countAdminUsersFactory, @@ -68,38 +43,12 @@ const { storeTokenResourceAccessDefinitionsFactory } = require('@/modules/core/repositories/tokens') const { getServerInfoFactory } = require('@/modules/core/repositories/server') +const { createTestStream } = require('@/test/speckle-helpers/streamHelper') +const { waitForRegionUser } = require('@/test/speckle-helpers/regions') +const { createTestWorkspace } = require('@/modules/workspaces/tests/helpers/creation') +const { faker } = require('@faker-js/faker') const getServerInfo = getServerInfoFactory({ db }) -const getUser = getUserFactory({ db }) -const getUsers = getUsersFactory({ db }) -const getStream = getStreamFactory({ db }) -const createStream = legacyCreateStreamFactory({ - createStreamReturnRecord: createStreamReturnRecordFactory({ - inviteUsersToProject: inviteUsersToProjectFactory({ - createAndSendInvite: createAndSendInviteFactory({ - findUserByTarget: findUserByTargetFactory({ db }), - insertInviteAndDeleteOld: insertInviteAndDeleteOldFactory({ db }), - collectAndValidateResourceTargets: collectAndValidateCoreTargetsFactory({ - getStream - }), - buildInviteEmailContents: buildCoreInviteEmailContentsFactory({ - getStream - }), - emitEvent: ({ eventName, payload }) => - getEventBus().emit({ - eventName, - payload - }), - getUser, - getServerInfo - }), - getUsers - }), - createStream: createStreamFactory({ db }), - createBranch: createBranchFactory({ db }), - projectsEventsEmitter: ProjectsEmitter.emit - }) -}) const findEmail = findEmailFactory({ db }) const requestNewEmailVerification = requestNewEmailVerificationFactory({ @@ -144,10 +93,30 @@ describe('Blobs integration @blobstorage', () => { email: 'barron@bubble.bobble', password: 'bubblesAreMyBlobs' } + const workspace = { + name: 'Anutha Blob Test Workspace #1', + ownerId: '', + id: '', + slug: '' + } + + const createStreamForTest = async () => { + const stream = { + name: faker.company.name(), + isPublic: false, + workspaceId: workspace.id + } + await createTestStream(stream, user) + return stream.id + } before(async () => { ;({ app } = await beforeEachContext()) user.id = await createUser(user) + await waitForRegionUser(user.id) + await createTestWorkspace(workspace, user, { + regionKey: getMainTestRegionKeyIfMultiRegion() + }) ;({ token } = await createToken({ userId: user.id, name: 'test token', @@ -155,7 +124,7 @@ describe('Blobs integration @blobstorage', () => { })) }) it('Uploads from multipart upload', async () => { - const streamId = await createStream({ ownerId: user.id }) + const streamId = await createStreamForTest() const response = await request(app) .post(`/api/stream/${streamId}/blob`) .set('Authorization', `Bearer ${token}`) @@ -169,7 +138,7 @@ describe('Blobs integration @blobstorage', () => { }) it('Errors for too big files, file is deleted', async () => { - const streamId = await createStream({ ownerId: user.id }) + const streamId = await createStreamForTest() const response = await request(app) .post(`/api/stream/${streamId}/blob`) .set('Authorization', `Bearer ${token}`) @@ -185,7 +154,7 @@ describe('Blobs integration @blobstorage', () => { }) it('Gets blob metadata', async () => { - const streamId = await createStream({ ownerId: user.id }) + const streamId = await createStreamForTest() const response = await request(app) .post(`/api/stream/${streamId}/blob`) .set('Authorization', `Bearer ${token}`) @@ -203,7 +172,7 @@ describe('Blobs integration @blobstorage', () => { }) it('Deletes blob and object metadata', async () => { - const streamId = await createStream({ ownerId: user.id }) + const streamId = await createStreamForTest() const response = await request(app) .post(`/api/stream/${streamId}/blob`) .set('Authorization', `Bearer ${token}`) @@ -229,7 +198,7 @@ describe('Blobs integration @blobstorage', () => { }) it('Gets uploaded blob data', async () => { - const streamId = await createStream({ ownerId: user.id }) + const streamId = await createStreamForTest() const response = await request(app) .post(`/api/stream/${streamId}/blob`) .set('Authorization', `Bearer ${token}`) @@ -248,7 +217,7 @@ describe('Blobs integration @blobstorage', () => { }) it('Returns 400 for bad form data', async () => { - const streamId = await createStream({ ownerId: user.id }) + const streamId = await createStreamForTest() const response = await request(app) .post(`/api/stream/${streamId}/blob`) .set('Authorization', `Bearer ${token}`) diff --git a/packages/server/modules/blobstorage/tests/blobstorage.spec.js b/packages/server/modules/blobstorage/tests/blobstorage.spec.ts similarity index 77% rename from packages/server/modules/blobstorage/tests/blobstorage.spec.js rename to packages/server/modules/blobstorage/tests/blobstorage.spec.ts index cf6f869103..31de237ea2 100644 --- a/packages/server/modules/blobstorage/tests/blobstorage.spec.js +++ b/packages/server/modules/blobstorage/tests/blobstorage.spec.ts @@ -1,37 +1,62 @@ -const expect = require('chai').expect -const { beforeEachContext } = require('@/test/hooks') -const { NotFoundError, BadRequestError } = require('@/modules/shared/errors') -const { range } = require('lodash') -const { fakeIdGenerator, createBlobs } = require('@/modules/blobstorage/tests/helpers') -const { +import { beforeEachContext } from '@/test/hooks' +import { NotFoundError, BadRequestError } from '@/modules/shared/errors' +import { range } from 'lodash' +import { fakeIdGenerator, createBlobs } from '@/modules/blobstorage/tests/helpers' +import { uploadFileStreamFactory, getFileStreamFactory, markUploadSuccessFactory, markUploadOverFileSizeLimitFactory, fullyDeleteBlobFactory -} = require('@/modules/blobstorage/services/management') -const { +} from '@/modules/blobstorage/services/management' +import { upsertBlobFactory, updateBlobFactory, getBlobMetadataFactory, getBlobMetadataCollectionFactory, blobCollectionSummaryFactory, deleteBlobFactory -} = require('@/modules/blobstorage/repositories') -const { db } = require('@/db/knex') -const { cursorFromRows, decodeCursor } = require('@/modules/blobstorage/helpers/db') -const { createTestStream } = require('@/test/speckle-helpers/streamHelper') -const cryptoRandomString = require('crypto-random-string') -const { createTestUser } = require('@/test/authHelper') -const { storeFileStream } = require('@/modules/blobstorage/objectStorage') -const fakeFileStreamStore = (fakeHash) => async () => ({ fileHash: fakeHash }) +} from '@/modules/blobstorage/repositories' +import { db } from '@/db/knex' +import { cursorFromRows, decodeCursor } from '@/modules/blobstorage/helpers/db' +import { BasicTestStream, createTestStream } from '@/test/speckle-helpers/streamHelper' +import cryptoRandomString from 'crypto-random-string' +import { BasicTestUser, createTestUser } from '@/test/authHelper' +import { storeFileStreamFactory } from '@/modules/blobstorage/repositories/blobs' +import { getMainObjectStorage } from '@/modules/blobstorage/clients/objectStorage' +import { expect } from 'chai' +import { UploadFileStream } from '@/modules/blobstorage/domain/operations' +import { BlobStorageItem } from '@/modules/blobstorage/domain/types' +import { + BasicTestWorkspace, + createTestWorkspace +} from '@/modules/workspaces/tests/helpers/creation' +import { waitForRegionUser } from '@/test/speckle-helpers/regions' +import { getProjectObjectStorage } from '@/modules/multiregion/utils/blobStorageSelector' + +type UploadFileStreamStreamData = Parameters[0] +type UploadFileStreamBlobData = Parameters[1] + +const buildUploadFileStream = async (params: { streamId: string | null }) => { + const { streamId } = params + + const storage = streamId + ? await getProjectObjectStorage({ projectId: streamId }) + : getMainObjectStorage() + const storeFileStream = storeFileStreamFactory({ storage }) + const uploadFileStream = uploadFileStreamFactory({ + upsertBlob, + updateBlob, + storeFileStream + }) + + return uploadFileStream +} + +const fakeFileStreamStore = (fakeHash: string) => async () => ({ fileHash: fakeHash }) const upsertBlob = upsertBlobFactory({ db }) const updateBlob = updateBlobFactory({ db }) -const uploadFileStream = uploadFileStreamFactory({ - upsertBlob, - updateBlob, - storeFileStream -}) + const getBlobMetadata = getBlobMetadataFactory({ db }) const getBlobMetadataCollection = getBlobMetadataCollectionFactory({ db }) const blobCollectionSummary = blobCollectionSummaryFactory({ db }) @@ -52,21 +77,29 @@ describe('Blob storage @blobstorage', () => { }) describe('Upload file stream', () => { - const data = [ + const invalidData: Array< + [ + caseName: string, + streamData: UploadFileStreamStreamData, + blobData: UploadFileStreamBlobData + ] + > = [ [ 'stream', { streamId: 'a'.padStart(1, 'a'), userId: 'a'.padStart(10, 'b') }, - { blobId: 'a'.padStart(10, 'c') } + { blobId: 'a'.padStart(10, 'c') } as UploadFileStreamBlobData ], [ 'user', { streamId: 'a'.padStart(10, 'a'), userId: 'a'.padStart(1, 'b') }, - { blobId: 'a'.padStart(10, 'c') } + { blobId: 'a'.padStart(10, 'c') } as UploadFileStreamBlobData ] ] - data.map(([caseName, streamData, blobData]) => + invalidData.map(([caseName, streamData, blobData]) => it(`Should throw if ${caseName} id length is incorrect`, async () => { + const uploadFileStream = await buildUploadFileStream({ streamId: null }) + try { await uploadFileStream(streamData, blobData) } catch (err) { @@ -91,34 +124,42 @@ describe('Blob storage @blobstorage', () => { const blobData = await uploadFileStream( { streamId, userId }, - { blobId, fileName, fileType: '.something', fileStream: null } + { blobId, fileName, fileType: '.something', fileStream: Buffer.from('') } ) expect(blobData).to.deep.equal({ blobId, fileName, fileHash }) }) }) describe('Get blob metadata', () => { - const testUser1 = { + const testUser1: BasicTestUser = { name: 'Blob Test User #1', email: 'testUser1@gmailll.com', id: '' } - const testStream1 = { + const testStream1: BasicTestStream = { name: 'Blob Test Stream #1', isPublic: false, ownerId: '', id: '' } - /** - * @type {import('@/modules/blobstorage/domain/types').BlobStorageItem} - */ - let testStreamBlob1 + const testWorkspace1: BasicTestWorkspace = { + name: 'Blob Test Workspace #1', + ownerId: '', + id: '', + slug: '' + } + + let testStreamBlob1: BlobStorageItem before(async () => { // Insert blob await createTestUser(testUser1) + await waitForRegionUser(testUser1) + await createTestWorkspace(testWorkspace1, testUser1) + + testStream1.workspaceId = testWorkspace1.id await createTestStream(testStream1, testUser1) testStreamBlob1 = await upsertBlob({ id: cryptoRandomString({ length: 10 }), @@ -140,7 +181,7 @@ describe('Blob storage @blobstorage', () => { }) it('when no streamId throws ResourceMismatch', async () => { try { - await getBlobMetadata({ streamId: null, blobId: 'bar' }) + await getBlobMetadata({ streamId: null as unknown as string, blobId: 'bar' }) throw new Error('This should have failed') } catch (err) { if (!(err instanceof BadRequestError)) throw err @@ -162,10 +203,10 @@ describe('Blob storage @blobstorage', () => { describe('cursorFromRows', () => { it('returns base64 encoded date ISO string', () => { const cursorTarget = 'foo' - const rowItem = {} + const rowItem: Record = {} const cursorValue = new Date() rowItem[cursorTarget] = cursorValue - const createdCursor = cursorFromRows([rowItem], cursorTarget) + const createdCursor = cursorFromRows([rowItem], cursorTarget)! expect(Buffer.from(createdCursor, 'base64').toString()).to.equal( cursorValue.toISOString() @@ -173,11 +214,11 @@ describe('Blob storage @blobstorage', () => { }) it('return null if rows is null or empty array', () => { expect(cursorFromRows([], 'cursorTarget')).to.be.null - expect(cursorFromRows(null, 'cursorTarget')).to.be.null + expect(cursorFromRows(null as unknown as [], 'cursorTarget')).to.be.null }) it("throws if the cursor target doesn't find a date object", () => { try { - cursorFromRows([{}], 'cursorTarget') + cursorFromRows([{}], 'cursorTarget' as never) throw new Error('This should have thrown') } catch (err) { if (!(err instanceof BadRequestError)) throw err diff --git a/packages/server/modules/blobstorage/tests/helpers.js b/packages/server/modules/blobstorage/tests/helpers.ts similarity index 64% rename from packages/server/modules/blobstorage/tests/helpers.js rename to packages/server/modules/blobstorage/tests/helpers.ts index 23f86620b4..cb18cf7ca3 100644 --- a/packages/server/modules/blobstorage/tests/helpers.js +++ b/packages/server/modules/blobstorage/tests/helpers.ts @@ -1,11 +1,21 @@ /* istanbul ignore file */ -const crs = require('crypto-random-string') -const { range } = require('lodash') -const { knex } = require('@/db/knex') +import crs from 'crypto-random-string' +import { range } from 'lodash' +import { knex } from '@/db/knex' + const BlobStorage = () => knex('blob_storage') -const fakeIdGenerator = () => crs({ length: 10 }) -const createBlobs = async ({ streamId, number, fileSize = 1 }) => +export const fakeIdGenerator = () => crs({ length: 10 }) + +export const createBlobs = async ({ + streamId, + number, + fileSize = 1 +}: { + streamId: string + number: number + fileSize?: number +}) => await Promise.all( range(number).map(async (num) => { const id = fakeIdGenerator() @@ -24,8 +34,3 @@ const createBlobs = async ({ streamId, number, fileSize = 1 }) => return dbFile }) ) - -module.exports = { - fakeIdGenerator, - createBlobs -} diff --git a/packages/server/modules/cli/commands/db/helpers/index.ts b/packages/server/modules/cli/commands/db/helpers/index.ts index b16ef7f2f6..81c67effc3 100644 --- a/packages/server/modules/cli/commands/db/helpers/index.ts +++ b/packages/server/modules/cli/commands/db/helpers/index.ts @@ -1,4 +1,4 @@ -import { getAllRegisteredDbClients } from '@/modules/multiregion/dbSelector' +import { getAllRegisteredDbClients } from '@/modules/multiregion/utils/dbSelector' export type CommonDbArgs = { regionKey?: string diff --git a/packages/server/modules/cli/commands/download/commit.ts b/packages/server/modules/cli/commands/download/commit.ts index e1507dff13..2a7868f1c5 100644 --- a/packages/server/modules/cli/commands/download/commit.ts +++ b/packages/server/modules/cli/commands/download/commit.ts @@ -60,7 +60,7 @@ import { saveActivityFactory } from '@/modules/activitystream/repositories' import { publish } from '@/modules/shared/utils/subscriptions' import { getUserFactory } from '@/modules/core/repositories/users' import { createObjectFactory } from '@/modules/core/services/objects/management' -import { getProjectDbClient } from '@/modules/multiregion/dbSelector' +import { getProjectDbClient } from '@/modules/multiregion/utils/dbSelector' import { db, mainDb } from '@/db/knex' const command: CommandModule< diff --git a/packages/server/modules/cli/commands/download/project.ts b/packages/server/modules/cli/commands/download/project.ts index af3191ff21..1c7f386b79 100644 --- a/packages/server/modules/cli/commands/download/project.ts +++ b/packages/server/modules/cli/commands/download/project.ts @@ -69,7 +69,7 @@ import { addBranchCreatedActivityFactory } from '@/modules/activitystream/servic import { authorizeResolver } from '@/modules/shared' import { Roles } from '@speckle/shared' import { getDefaultRegionFactory } from '@/modules/workspaces/repositories/regions' -import { getDb } from '@/modules/multiregion/dbSelector' +import { getDb } from '@/modules/multiregion/utils/dbSelector' import { createNewProjectFactory } from '@/modules/core/services/projects' import { deleteProjectFactory, diff --git a/packages/server/modules/comments/graph/resolvers/comments.ts b/packages/server/modules/comments/graph/resolvers/comments.ts index 6a2bc8d572..2a9dfd805b 100644 --- a/packages/server/modules/comments/graph/resolvers/comments.ts +++ b/packages/server/modules/comments/graph/resolvers/comments.ts @@ -97,7 +97,7 @@ import { import { getStreamObjectsFactory } from '@/modules/core/repositories/objects' import { getStreamFactory } from '@/modules/core/repositories/streams' import { saveActivityFactory } from '@/modules/activitystream/repositories' -import { getProjectDbClient } from '@/modules/multiregion/dbSelector' +import { getProjectDbClient } from '@/modules/multiregion/utils/dbSelector' import { Knex } from 'knex' // We can use the main DB for these diff --git a/packages/server/modules/core/graph/resolvers/branches.ts b/packages/server/modules/core/graph/resolvers/branches.ts index b6b9aa001e..40eb9de674 100644 --- a/packages/server/modules/core/graph/resolvers/branches.ts +++ b/packages/server/modules/core/graph/resolvers/branches.ts @@ -31,7 +31,7 @@ import { Resolvers } from '@/modules/core/graph/generated/graphql' import { getPaginatedStreamBranchesFactory } from '@/modules/core/services/branch/retrieval' import { saveActivityFactory } from '@/modules/activitystream/repositories' import { filteredSubscribe, publish } from '@/modules/shared/utils/subscriptions' -import { getProjectDbClient } from '@/modules/multiregion/dbSelector' +import { getProjectDbClient } from '@/modules/multiregion/utils/dbSelector' export = { Query: {}, diff --git a/packages/server/modules/core/graph/resolvers/commits.ts b/packages/server/modules/core/graph/resolvers/commits.ts index 7d8fb589e6..2d627822a4 100644 --- a/packages/server/modules/core/graph/resolvers/commits.ts +++ b/packages/server/modules/core/graph/resolvers/commits.ts @@ -80,7 +80,7 @@ import { CommitGraphQLReturn } from '@/modules/core/helpers/graphTypes' import { getProjectDbClient, getRegisteredDbClients -} from '@/modules/multiregion/dbSelector' +} from '@/modules/multiregion/utils/dbSelector' import { LegacyUserCommit } from '@/modules/core/domain/commits/types' const getStreams = getStreamsFactory({ db }) diff --git a/packages/server/modules/core/graph/resolvers/common.ts b/packages/server/modules/core/graph/resolvers/common.ts index ad63dbf99d..c2a24db389 100644 --- a/packages/server/modules/core/graph/resolvers/common.ts +++ b/packages/server/modules/core/graph/resolvers/common.ts @@ -1,7 +1,7 @@ import { mainDb } from '@/db/knex' import { getBlobsFactory } from '@/modules/blobstorage/repositories' import { Resolvers } from '@/modules/core/graph/generated/graphql' -import { getProjectDbClient } from '@/modules/multiregion/dbSelector' +import { getProjectDbClient } from '@/modules/multiregion/utils/dbSelector' import { isNonNullable } from '@speckle/shared' import { keyBy } from 'lodash' diff --git a/packages/server/modules/core/graph/resolvers/models.ts b/packages/server/modules/core/graph/resolvers/models.ts index f222be86f7..e596405707 100644 --- a/packages/server/modules/core/graph/resolvers/models.ts +++ b/packages/server/modules/core/graph/resolvers/models.ts @@ -64,7 +64,7 @@ import { saveActivityFactory } from '@/modules/activitystream/repositories' import { getProjectDbClient, getRegisteredRegionClients -} from '@/modules/multiregion/dbSelector' +} from '@/modules/multiregion/utils/dbSelector' export = { User: { diff --git a/packages/server/modules/core/graph/resolvers/objects.ts b/packages/server/modules/core/graph/resolvers/objects.ts index c17d68c026..abd21400d2 100644 --- a/packages/server/modules/core/graph/resolvers/objects.ts +++ b/packages/server/modules/core/graph/resolvers/objects.ts @@ -9,7 +9,7 @@ import { storeObjectsIfNotFoundFactory } from '@/modules/core/repositories/objects' import { createObjectsFactory } from '@/modules/core/services/objects/management' -import { getProjectDbClient } from '@/modules/multiregion/dbSelector' +import { getProjectDbClient } from '@/modules/multiregion/utils/dbSelector' type GetObjectChildrenQueryParams = Parameters< ReturnType diff --git a/packages/server/modules/core/graph/resolvers/projects.ts b/packages/server/modules/core/graph/resolvers/projects.ts index 04b0bb4510..fc6bc83ae4 100644 --- a/packages/server/modules/core/graph/resolvers/projects.ts +++ b/packages/server/modules/core/graph/resolvers/projects.ts @@ -69,7 +69,7 @@ import { } from '@/modules/core/services/streams/management' import { createOnboardingStreamFactory } from '@/modules/core/services/streams/onboarding' import { getOnboardingBaseProjectFactory } from '@/modules/cross-server-sync/services/onboardingProject' -import { getProjectDbClient } from '@/modules/multiregion/dbSelector' +import { getProjectDbClient } from '@/modules/multiregion/utils/dbSelector' import { deleteAllResourceInvitesFactory, findUserByTargetFactory, diff --git a/packages/server/modules/core/graph/resolvers/versions.ts b/packages/server/modules/core/graph/resolvers/versions.ts index 1f89adf0d6..dc7fdcf2ca 100644 --- a/packages/server/modules/core/graph/resolvers/versions.ts +++ b/packages/server/modules/core/graph/resolvers/versions.ts @@ -56,7 +56,7 @@ import { } from '@/modules/activitystream/services/commitActivity' import { getObjectFactory } from '@/modules/core/repositories/objects' import { saveActivityFactory } from '@/modules/activitystream/repositories' -import { getProjectDbClient } from '@/modules/multiregion/dbSelector' +import { getProjectDbClient } from '@/modules/multiregion/utils/dbSelector' export = { Project: { diff --git a/packages/server/modules/core/rest/diffDownload.ts b/packages/server/modules/core/rest/diffDownload.ts index 3f45dacc1a..ceeb49564d 100644 --- a/packages/server/modules/core/rest/diffDownload.ts +++ b/packages/server/modules/core/rest/diffDownload.ts @@ -8,7 +8,7 @@ import { db } from '@/db/knex' import { validatePermissionsReadStreamFactory } from '@/modules/core/services/streams/auth' import { getStreamFactory } from '@/modules/core/repositories/streams' import { authorizeResolver, validateScopes } from '@/modules/shared' -import { getProjectDbClient } from '@/modules/multiregion/dbSelector' +import { getProjectDbClient } from '@/modules/multiregion/utils/dbSelector' export default (app: Application) => { const validatePermissionsReadStream = validatePermissionsReadStreamFactory({ diff --git a/packages/server/modules/core/rest/diffUpload.ts b/packages/server/modules/core/rest/diffUpload.ts index a6e85486a6..4b5e1d8089 100644 --- a/packages/server/modules/core/rest/diffUpload.ts +++ b/packages/server/modules/core/rest/diffUpload.ts @@ -5,7 +5,7 @@ import type { Application } from 'express' import { hasObjectsFactory } from '@/modules/core/repositories/objects' import { validatePermissionsWriteStreamFactory } from '@/modules/core/services/streams/auth' import { authorizeResolver, validateScopes } from '@/modules/shared' -import { getProjectDbClient } from '@/modules/multiregion/dbSelector' +import { getProjectDbClient } from '@/modules/multiregion/utils/dbSelector' export default (app: Application) => { const validatePermissionsWriteStream = validatePermissionsWriteStreamFactory({ diff --git a/packages/server/modules/core/rest/download.ts b/packages/server/modules/core/rest/download.ts index 6e2b662d2a..154be5ceaf 100644 --- a/packages/server/modules/core/rest/download.ts +++ b/packages/server/modules/core/rest/download.ts @@ -13,7 +13,7 @@ import { validatePermissionsReadStreamFactory } from '@/modules/core/services/st import { getStreamFactory } from '@/modules/core/repositories/streams' import { validateScopes, authorizeResolver } from '@/modules/shared' import type express from 'express' -import { getProjectDbClient } from '@/modules/multiregion/dbSelector' +import { getProjectDbClient } from '@/modules/multiregion/utils/dbSelector' export default (app: express.Express) => { const validatePermissionsReadStream = validatePermissionsReadStreamFactory({ diff --git a/packages/server/modules/core/rest/upload.ts b/packages/server/modules/core/rest/upload.ts index 2e59c11ab3..23a6890022 100644 --- a/packages/server/modules/core/rest/upload.ts +++ b/packages/server/modules/core/rest/upload.ts @@ -19,7 +19,7 @@ import { } from '@/modules/core/repositories/objects' import { validatePermissionsWriteStreamFactory } from '@/modules/core/services/streams/auth' import { authorizeResolver, validateScopes } from '@/modules/shared' -import { getProjectDbClient } from '@/modules/multiregion/dbSelector' +import { getProjectDbClient } from '@/modules/multiregion/utils/dbSelector' const MAX_FILE_SIZE = maximumObjectUploadFileSizeMb() * 1024 * 1024 const { FF_NO_CLOSURE_WRITES } = getFeatureFlags() diff --git a/packages/server/modules/core/tests/integration/subs.graph.spec.ts b/packages/server/modules/core/tests/integration/subs.graph.spec.ts index 854abbc147..a31c73a4ee 100644 --- a/packages/server/modules/core/tests/integration/subs.graph.spec.ts +++ b/packages/server/modules/core/tests/integration/subs.graph.spec.ts @@ -62,7 +62,7 @@ import { deleteStreamAndNotifyFactory, updateStreamAndNotifyFactory } from '@/modules/core/services/streams/management' -import { getProjectDbClient } from '@/modules/multiregion/dbSelector' +import { getProjectDbClient } from '@/modules/multiregion/utils/dbSelector' import { deleteAllResourceInvitesFactory } from '@/modules/serverinvites/repositories/serverInvites' import { authorizeResolver } from '@/modules/shared' import { publish } from '@/modules/shared/utils/subscriptions' diff --git a/packages/server/modules/fileuploads/graph/resolvers/fileUploads.ts b/packages/server/modules/fileuploads/graph/resolvers/fileUploads.ts index 25c336f446..e2b21d5aa9 100644 --- a/packages/server/modules/fileuploads/graph/resolvers/fileUploads.ts +++ b/packages/server/modules/fileuploads/graph/resolvers/fileUploads.ts @@ -11,7 +11,7 @@ import { FileImportSubscriptions, filteredSubscribe } from '@/modules/shared/utils/subscriptions' -import { getProjectDbClient } from '@/modules/multiregion/dbSelector' +import { getProjectDbClient } from '@/modules/multiregion/utils/dbSelector' export = { Stream: { diff --git a/packages/server/modules/fileuploads/index.ts b/packages/server/modules/fileuploads/index.ts index e0ef3a1397..ae05222690 100644 --- a/packages/server/modules/fileuploads/index.ts +++ b/packages/server/modules/fileuploads/index.ts @@ -22,7 +22,7 @@ import { getStreamFactory } from '@/modules/core/repositories/streams' import { addBranchCreatedActivityFactory } from '@/modules/activitystream/services/branchActivity' import { saveActivityFactory } from '@/modules/activitystream/repositories' import { getPort } from '@/modules/shared/helpers/envHelper' -import { getProjectDbClient } from '@/modules/multiregion/dbSelector' +import { getProjectDbClient } from '@/modules/multiregion/utils/dbSelector' import { listenFor } from '@/modules/core/utils/dbNotificationListener' export const init: SpeckleModule['init'] = async (app, isInitial) => { diff --git a/packages/server/modules/gendo/graph/resolvers/index.ts b/packages/server/modules/gendo/graph/resolvers/index.ts index 9a8db63948..86bbc51e5a 100644 --- a/packages/server/modules/gendo/graph/resolvers/index.ts +++ b/packages/server/modules/gendo/graph/resolvers/index.ts @@ -17,7 +17,6 @@ import { updateBlobFactory, upsertBlobFactory } from '@/modules/blobstorage/repositories' -import { storeFileStream } from '@/modules/blobstorage/objectStorage' import { getLatestVersionRenderRequestsFactory, getUserCreditsFactory, @@ -25,7 +24,7 @@ import { storeRenderFactory, upsertUserCreditsFactory } from '@/modules/gendo/repositories' -import { getProjectDbClient } from '@/modules/multiregion/dbSelector' +import { getProjectDbClient } from '@/modules/multiregion/utils/dbSelector' import { requestNewImageGenerationFactory } from '@/modules/gendo/clients/gendo' import { getUserGendoAiCreditsFactory, @@ -39,6 +38,8 @@ import { getServerOrigin, getFeatureFlags } from '@/modules/shared/helpers/envHelper' +import { getProjectObjectStorage } from '@/modules/multiregion/utils/blobStorageSelector' +import { storeFileStreamFactory } from '@/modules/blobstorage/repositories/blobs' const upsertUserCredits = upsertUserCreditsFactory({ db }) const getUserGendoAiCredits = getUserGendoAiCreditsFactory({ @@ -95,9 +96,13 @@ export = FF_GENDOAI_MODULE_ENABLED const userId = ctx.userId! - const projectDb = await getProjectDbClient({ - projectId: args.input.projectId - }) + const projectId = args.input.projectId + const [projectDb, projectStorage] = await Promise.all([ + getProjectDbClient({ + projectId + }), + getProjectObjectStorage({ projectId }) + ]) await useUserGendoAiCreditsFactory({ getUserGendoAiCredits, @@ -111,6 +116,7 @@ export = FF_GENDOAI_MODULE_ENABLED token: getGendoAIKey() }) + const storeFileStream = storeFileStreamFactory({ storage: projectStorage }) const createRenderRequest = createRenderRequestFactory({ uploadFileStream: uploadFileStreamFactory({ storeFileStream, diff --git a/packages/server/modules/gendo/rest/index.ts b/packages/server/modules/gendo/rest/index.ts index 2e9c5611c1..41c31f1c19 100644 --- a/packages/server/modules/gendo/rest/index.ts +++ b/packages/server/modules/gendo/rest/index.ts @@ -6,24 +6,16 @@ import { updateRenderRecordFactory } from '@/modules/gendo/repositories' import { uploadFileStreamFactory } from '@/modules/blobstorage/services/management' -import { storeFileStream } from '@/modules/blobstorage/objectStorage' import { updateBlobFactory, upsertBlobFactory } from '@/modules/blobstorage/repositories' import { publish } from '@/modules/shared/utils/subscriptions' -import { getProjectDbClient } from '@/modules/multiregion/dbSelector' +import { getProjectDbClient } from '@/modules/multiregion/utils/dbSelector' import { createHmac, timingSafeEqual } from 'node:crypto' import { getGendoAIKey } from '@/modules/shared/helpers/envHelper' -//Validate payload -// function validatePayload(req, res, next) { -// if (req.get(sigHeaderName)) { -// //Extract Signature header -// -// } - -// return next(); -// } +import { getProjectObjectStorage } from '@/modules/multiregion/utils/blobStorageSelector' +import { storeFileStreamFactory } from '@/modules/blobstorage/repositories/blobs' export default function (app: express.Express) { // const responseToken = getGendoAIResponseKey() @@ -46,8 +38,13 @@ export default function (app: express.Express) { const status = payload.status const gendoGenerationId = payload.generationId - const projectDb = await getProjectDbClient({ projectId: req.params.projectId }) + const projectId = req.params.projectId + const [projectDb, projectStorage] = await Promise.all([ + getProjectDbClient({ projectId }), + getProjectObjectStorage({ projectId }) + ]) + const storeFileStream = storeFileStreamFactory({ storage: projectStorage }) const updateRenderRequest = updateRenderRequestFactory({ getRenderByGenerationId: getRenderByGenerationIdFactory({ db: projectDb }), uploadFileStream: uploadFileStreamFactory({ diff --git a/packages/server/modules/multiregion/domain/operations.ts b/packages/server/modules/multiregion/domain/operations.ts index 55e7e4b708..53382a7ef2 100644 --- a/packages/server/modules/multiregion/domain/operations.ts +++ b/packages/server/modules/multiregion/domain/operations.ts @@ -7,6 +7,7 @@ import { import { UpdateServerRegionInput } from '@/modules/core/graph/generated/graphql' import { InsertableRegionRecord } from '@/modules/multiregion/helpers/types' import { Optional } from '@speckle/shared' +import { ObjectStorage } from '@/modules/blobstorage/clients/objectStorage' export type GetRegions = () => Promise export type GetRegion = (params: { key: string }) => Promise> @@ -22,7 +23,7 @@ export type GetAvailableRegionConfig = () => Promise export type GetAvailableRegionKeys = () => Promise export type GetFreeRegionKeys = () => Promise -export type InitializeRegion = (args: { regionKey: string }) => Promise +export type InitializeRegion = (args: { regionKey: string }) => Promise export type CreateAndValidateNewRegion = (params: { region: InsertableRegionRecord @@ -54,3 +55,11 @@ export type AsyncRegionKeyStore = (args: RegionKeyStoreArgs) => Promise export type UpdateAndValidateRegion = (params: { input: UpdateServerRegionInput }) => Promise + +export type GetProjectObjectStorage = (args: { + projectId: string +}) => Promise + +export type GetRegionObjectStorage = (args: { + regionKey: string +}) => Promise diff --git a/packages/server/modules/multiregion/graph/resolvers/index.ts b/packages/server/modules/multiregion/graph/resolvers/index.ts index f607166d5a..561f19bb0a 100644 --- a/packages/server/modules/multiregion/graph/resolvers/index.ts +++ b/packages/server/modules/multiregion/graph/resolvers/index.ts @@ -1,6 +1,6 @@ import { db } from '@/db/knex' import { Resolvers } from '@/modules/core/graph/generated/graphql' -import { initializeRegion } from '@/modules/multiregion/dbSelector' +import { initializeRegion as initializeDb } from '@/modules/multiregion/utils/dbSelector' import { getAvailableRegionConfig } from '@/modules/multiregion/regionConfig' import { getRegionFactory, @@ -14,8 +14,10 @@ import { } from '@/modules/multiregion/services/config' import { createAndValidateNewRegionFactory, + initializeRegionClients, updateAndValidateRegionFactory } from '@/modules/multiregion/services/management' +import { initializeRegion as initializeBlobStorage } from '@/modules/multiregion/utils/blobStorageSelector' export default { ServerMultiRegionConfiguration: { @@ -44,7 +46,10 @@ export default { }), getRegion: getRegionFactory({ db }), storeRegion: storeRegionFactory({ db }), - initializeRegion + initializeRegion: initializeRegionClients({ + initializeDb, + initializeBlobStorage + }) }) return await createAndValidateNewRegion({ region: args.input }) }, diff --git a/packages/server/modules/multiregion/index.ts b/packages/server/modules/multiregion/index.ts index ca3d2c7ff5..5575116cbe 100644 --- a/packages/server/modules/multiregion/index.ts +++ b/packages/server/modules/multiregion/index.ts @@ -1,7 +1,11 @@ import { moduleLogger } from '@/logging/logging' -import { initializeRegisteredRegionClients } from '@/modules/multiregion/dbSelector' +import { initializeRegisteredRegionClients as initDb } from '@/modules/multiregion/utils/dbSelector' import { isMultiRegionEnabled } from '@/modules/multiregion/helpers' import { SpeckleModule } from '@/modules/shared/helpers/typeHelper' +import { + initializeRegisteredRegionClients as initBlobs, + isMultiRegionBlobStorageEnabled +} from '@/modules/multiregion/utils/blobStorageSelector' const multiRegion: SpeckleModule = { async init() { @@ -13,7 +17,13 @@ const multiRegion: SpeckleModule = { moduleLogger.info('🌍 Init multiRegion module') // Init registered region clients - await initializeRegisteredRegionClients() + await initDb() + + const isBlobStorageEnabled = isMultiRegionBlobStorageEnabled() + if (isBlobStorageEnabled) { + moduleLogger.info('🌍 Init multiRegion blob storage') + await initBlobs() + } } } diff --git a/packages/server/modules/multiregion/regionConfig.ts b/packages/server/modules/multiregion/regionConfig.ts index ff28f8b846..f162c26e33 100644 --- a/packages/server/modules/multiregion/regionConfig.ts +++ b/packages/server/modules/multiregion/regionConfig.ts @@ -17,7 +17,21 @@ import { let multiRegionConfig: Optional = undefined const getMultiRegionConfig = async (): Promise => { - const emptyReturn = () => ({ main: { postgres: { connectionUri: '' } }, regions: {} }) + // Only for non region enabled dev envs + const emptyReturn = (): MultiRegionConfig => ({ + main: { + postgres: { connectionUri: '' }, + blobStorage: { + accessKey: '', + secretKey: '', + endpoint: '', + s3Region: '', + bucket: '', + createBucketIfNotExists: true + } + }, + regions: {} + }) if (isDevOrTestEnv() && !isMultiRegionEnabled()) { return emptyReturn() diff --git a/packages/server/modules/multiregion/services/management.ts b/packages/server/modules/multiregion/services/management.ts index 480b6c4af5..46a5a8635a 100644 --- a/packages/server/modules/multiregion/services/management.ts +++ b/packages/server/modules/multiregion/services/management.ts @@ -68,3 +68,15 @@ export const updateAndValidateRegionFactory = return await deps.updateRegion({ regionKey: input.key, region: update }) } + +export const initializeRegionClients = + (deps: { + initializeDb: InitializeRegion + initializeBlobStorage: InitializeRegion + }): InitializeRegion => + async ({ regionKey }) => { + await Promise.all([ + deps.initializeDb({ regionKey }), + deps.initializeBlobStorage({ regionKey }) + ]) + } diff --git a/packages/server/modules/multiregion/tests/e2e/serverAdmin.graph.spec.ts b/packages/server/modules/multiregion/tests/e2e/serverAdmin.graph.spec.ts index a9bccde5e1..54123761ec 100644 --- a/packages/server/modules/multiregion/tests/e2e/serverAdmin.graph.spec.ts +++ b/packages/server/modules/multiregion/tests/e2e/serverAdmin.graph.spec.ts @@ -1,3 +1,4 @@ +import { ObjectStorage } from '@/modules/blobstorage/clients/objectStorage' import { DataRegionsConfig } from '@/modules/multiregion/domain/types' import { isMultiRegionEnabled } from '@/modules/multiregion/helpers' import { BasicTestUser, createTestUser } from '@/test/authHelper' @@ -15,7 +16,11 @@ import { TestApolloServer } from '@/test/graphqlHelper' import { beforeEachContext, getRegionKeys } from '@/test/hooks' -import { MultiRegionConfigMock, MultiRegionDbSelectorMock } from '@/test/mocks/global' +import { + MultiRegionBlobStorageSelectorMock, + MultiRegionConfigMock, + MultiRegionDbSelectorMock +} from '@/test/mocks/global' import { truncateRegionsSafely } from '@/test/speckle-helpers/regions' import { Roles } from '@speckle/shared' import { expect } from 'chai' @@ -35,11 +40,27 @@ isEnabled [fakeRegionKey1]: { postgres: { connectionUri: 'postgres://user:password@uswest1:port/dbname' + }, + blobStorage: { + accessKey: '', + secretKey: '', + s3Region: '', + bucket: '', + endpoint: '', + createBucketIfNotExists: false } }, [fakeRegionKey2]: { postgres: { connectionUri: 'postgres://user:password@eueast3:port/dbname' + }, + blobStorage: { + accessKey: '', + secretKey: '', + s3Region: '', + bucket: '', + endpoint: '', + createBucketIfNotExists: false } } } @@ -52,6 +73,9 @@ isEnabled MultiRegionDbSelectorMock.mockFunction('initializeRegion', async () => Promise.resolve() ) + MultiRegionBlobStorageSelectorMock.mockFunction('initializeRegion', async () => + Promise.resolve(undefined as unknown as ObjectStorage) + ) await beforeEachContext() testAdminUser = await createTestUser({ role: Roles.Server.Admin }) @@ -62,6 +86,7 @@ isEnabled after(() => { MultiRegionConfigMock.resetMockedFunctions() MultiRegionDbSelectorMock.resetMockedFunctions() + MultiRegionBlobStorageSelectorMock.resetMockedFunctions() }) describe('server config', () => { diff --git a/packages/server/modules/multiregion/utils/blobStorageSelector.ts b/packages/server/modules/multiregion/utils/blobStorageSelector.ts new file mode 100644 index 0000000000..59e44a832c --- /dev/null +++ b/packages/server/modules/multiregion/utils/blobStorageSelector.ts @@ -0,0 +1,124 @@ +import { + getMainObjectStorage, + getObjectStorage, + ObjectStorage +} from '@/modules/blobstorage/clients/objectStorage' +import { ensureStorageAccessFactory } from '@/modules/blobstorage/repositories/blobs' +import { + GetProjectObjectStorage, + GetRegionObjectStorage +} from '@/modules/multiregion/domain/operations' +import { getAvailableRegionConfig } from '@/modules/multiregion/regionConfig' +import { + getProjectRegionKey, + getRegisteredRegionConfig, + getRegisteredRegionConfigs +} from '@/modules/multiregion/utils/regionSelector' +import { MisconfiguredEnvironmentError } from '@/modules/shared/errors' +import { getFeatureFlags } from '@/modules/shared/helpers/envHelper' +import { Optional } from '@speckle/shared' +import { BlobStorageConfig } from '@speckle/shared/dist/commonjs/environment/multiRegionConfig' + +type RegionStorageClients = { + [regionKey: string]: ObjectStorage +} + +let initializedClients: Optional = undefined + +export const isMultiRegionBlobStorageEnabled = () => + !!getFeatureFlags().FF_WORKSPACES_MULTI_REGION_BLOB_STORAGE_ENABLED + +export const initializeRegion = async (params: { + regionKey: string + /** + * As an optimization measure (when doing this in batch), you can pass in the config which would + * otherwise be resolved from scratch + */ + config?: BlobStorageConfig +}) => { + if (!isMultiRegionBlobStorageEnabled()) return getMainObjectStorage() + + const { regionKey } = params + let config = params.config + if (!config) { + // getAvailableRegionConfig allows getting configs that may not be registered yet + const regionConfigs = await getAvailableRegionConfig() + config = regionConfigs[regionKey].blobStorage + if (!config) throw new Error(`RegionKey ${regionKey} not available in config`) + } + + const storage = getObjectStorage({ + credentials: { + accessKeyId: config.accessKey, + secretAccessKey: config.secretKey + }, + endpoint: config.endpoint, + region: config.s3Region, + bucket: config.bucket + }) + + // ensure it works + const ensure = ensureStorageAccessFactory({ storage }) + await ensure({ createBucketIfNotExists: config.createBucketIfNotExists }) + + // Only add, if clients already initialized + if (initializedClients) { + initializedClients[regionKey] = storage + } + + return storage +} + +/** + * Idempotently initialize registered region clients + */ +export const initializeRegisteredRegionClients = + async (): Promise => { + const configs = await getRegisteredRegionConfigs() + + const newRet: RegionStorageClients = Object.fromEntries( + await Promise.all( + Object.entries(configs).map(async ([regionKey, { blobStorage: config }]) => { + return [regionKey, await initializeRegion({ regionKey, config })] + }) + ) + ) + initializedClients = newRet + + return newRet + } + +export const getRegisteredRegionClients = async (): Promise => { + if (!initializedClients) { + initializedClients = await initializeRegisteredRegionClients() + } + return initializedClients +} + +export const getRegionObjectStorage: GetRegionObjectStorage = async ({ regionKey }) => { + if (!isMultiRegionBlobStorageEnabled()) return getMainObjectStorage() + + const clients = await getRegisteredRegionClients() + let storage = clients[regionKey] + if (!storage) { + // Region may have been initialized in a different server instance + const config = await getRegisteredRegionConfig({ regionKey }) + if (config) { + storage = await initializeRegion({ regionKey, config: config.blobStorage }) + } + } + if (!storage) { + throw new MisconfiguredEnvironmentError( + `Region ${regionKey} blobStorage region not found` + ) + } + + return storage +} + +export const getProjectObjectStorage: GetProjectObjectStorage = async ({ + projectId +}) => { + const regionKey = await getProjectRegionKey({ projectId }) + return regionKey ? getRegionObjectStorage({ regionKey }) : getMainObjectStorage() +} diff --git a/packages/server/modules/multiregion/dbSelector.ts b/packages/server/modules/multiregion/utils/dbSelector.ts similarity index 82% rename from packages/server/modules/multiregion/dbSelector.ts rename to packages/server/modules/multiregion/utils/dbSelector.ts index 89a3a8407f..e04bef528c 100644 --- a/packages/server/modules/multiregion/dbSelector.ts +++ b/packages/server/modules/multiregion/utils/dbSelector.ts @@ -1,21 +1,12 @@ import { db } from '@/db/knex' -import { isMultiRegionEnabled } from '@/modules/multiregion/helpers' -import { - getRegionKeyFromCacheFactory, - getRegionKeyFromStorageFactory, - inMemoryRegionKeyStoreFactory, - writeRegionKeyToCacheFactory -} from '@/modules/multiregion/repositories/projectRegion' import { GetProjectDb, getProjectDbClientFactory, - getProjectRegionKeyFactory, GetRegionDb } from '@/modules/multiregion/services/projectRegion' -import { getGenericRedis } from '@/modules/shared/redis/redis' import { Knex } from 'knex' -import { getRegionFactory, getRegionsFactory } from '@/modules/multiregion/repositories' -import { DatabaseError, MisconfiguredEnvironmentError } from '@/modules/shared/errors' +import { getRegionFactory } from '@/modules/multiregion/repositories' +import { DatabaseError } from '@/modules/shared/errors' import { configureClient } from '@/knexfile' import { InitializeRegion } from '@/modules/multiregion/domain/operations' import { @@ -25,6 +16,11 @@ import { import { ensureError, MaybeNullOrUndefined } from '@speckle/shared' import { isDevOrTestEnv, isTestEnv } from '@/modules/shared/helpers/envHelper' import { migrateDbToLatest } from '@/db/migrations' +import { + getProjectRegionKey, + getRegisteredRegionConfigs +} from '@/modules/multiregion/utils/regionSelector' +import { mapValues } from 'lodash' let getter: GetProjectDb | undefined = undefined @@ -62,22 +58,6 @@ export const getDb = async ({ const initializeDbGetter = async (): Promise => { const getDefaultDb = () => db - - // if multi region is not enabled, lets fall back to the main Db ALWAYS - if (!isMultiRegionEnabled()) return async () => getDefaultDb() - - const { getRegionKey, writeRegion } = inMemoryRegionKeyStoreFactory() - - const redis = getGenericRedis() - - const getProjectRegionKey = getProjectRegionKeyFactory({ - getRegionKeyFromMemory: getRegionKey, - writeRegionToMemory: writeRegion, - getRegionKeyFromCache: getRegionKeyFromCacheFactory({ redis }), - writeRegionKeyToCache: writeRegionKeyToCacheFactory({ redis }), - getRegionKeyFromStorage: getRegionKeyFromStorageFactory({ db }) - }) - return getProjectDbClientFactory({ getDefaultDb, getRegionDb, @@ -98,20 +78,9 @@ let registeredRegionClients: RegionClients | undefined = undefined * Idempotently initialize registered region (in db) Knex clients */ export const initializeRegisteredRegionClients = async (): Promise => { - const configuredRegions = await getRegionsFactory({ db })() - if (!configuredRegions.length) return {} - // init knex clients - const regionConfigs = await getAvailableRegionConfig() - const ret = Object.fromEntries( - configuredRegions.map((region) => { - if (!(region.key in regionConfigs)) - throw new MisconfiguredEnvironmentError( - `Missing region config for ${region.key} region` - ) - return [region.key, configureClient(regionConfigs[region.key]).public] - }) - ) + const configs = await getRegisteredRegionConfigs() + const ret = mapValues(configs, (config) => configureClient(config).public) // run migrations await Promise.all( @@ -159,7 +128,7 @@ export const getAllRegisteredDbClients = async (): Promise< } /** - * Idempotently initialize region + * Idempotently initialize region db */ export const initializeRegion: InitializeRegion = async ({ regionKey }) => { const regionConfigs = await getAvailableRegionConfig() @@ -189,8 +158,8 @@ export const initializeRegion: InitializeRegion = async ({ regionKey }) => { sslmode }) - // pushing to the singleton object here, its only not available - // if this is being triggered from init, and in that case its gonna be set after anyway + // pushing to the singleton object here, only if its not available + // if this is being triggered from init, its gonna be set after anyway if (registeredRegionClients) { registeredRegionClients[regionKey] = regionDb.public } diff --git a/packages/server/modules/multiregion/utils/regionSelector.ts b/packages/server/modules/multiregion/utils/regionSelector.ts new file mode 100644 index 0000000000..e12248c96d --- /dev/null +++ b/packages/server/modules/multiregion/utils/regionSelector.ts @@ -0,0 +1,74 @@ +import { mainDb } from '@/db/knex' +import { isMultiRegionEnabled } from '@/modules/multiregion/helpers' +import { getAvailableRegionConfig } from '@/modules/multiregion/regionConfig' +import { getRegionsFactory } from '@/modules/multiregion/repositories' +import { + getRegionKeyFromCacheFactory, + getRegionKeyFromStorageFactory, + inMemoryRegionKeyStoreFactory, + writeRegionKeyToCacheFactory +} from '@/modules/multiregion/repositories/projectRegion' +import { + GetProjectRegionKey, + getProjectRegionKeyFactory +} from '@/modules/multiregion/services/projectRegion' +import { MisconfiguredEnvironmentError } from '@/modules/shared/errors' +import { getGenericRedis } from '@/modules/shared/redis/redis' +import { Optional } from '@speckle/shared' +import { DataRegionsConfig } from '@speckle/shared/dist/commonjs/environment/multiRegionConfig' + +export const getRegisteredRegionConfigs = async () => { + const registeredKeys = (await getRegionsFactory({ db: mainDb })()).map((r) => r.key) + if (!registeredKeys.length) return {} + + const availableConfigs = await getAvailableRegionConfig() + const result: DataRegionsConfig = {} + + for (const key of registeredKeys) { + const config = availableConfigs[key] + if (!config) { + throw new MisconfiguredEnvironmentError(`Missing region config for ${key} region`) + } + + result[key] = config + } + + return result +} + +export const getRegisteredRegionConfig = async (params: { regionKey: string }) => { + const availableConfigs = await getRegisteredRegionConfigs() + const config = availableConfigs[params.regionKey] + if (!config) return undefined + + return config +} + +let cachedProjectRegionKeyResolver: Optional = undefined + +const buildProjectRegionKeyResolver = async (): Promise => { + // if multi region is not enabled, lets fall back to the main region ALWAYS + if (!isMultiRegionEnabled()) return async () => null + + const { getRegionKey, writeRegion } = inMemoryRegionKeyStoreFactory() + + const redis = getGenericRedis() + + const getProjectRegionKey = getProjectRegionKeyFactory({ + getRegionKeyFromMemory: getRegionKey, + writeRegionToMemory: writeRegion, + getRegionKeyFromCache: getRegionKeyFromCacheFactory({ redis }), + writeRegionKeyToCache: writeRegionKeyToCacheFactory({ redis }), + getRegionKeyFromStorage: getRegionKeyFromStorageFactory({ db: mainDb }) + }) + + return getProjectRegionKey +} + +export const getProjectRegionKey: GetProjectRegionKey = async ({ projectId }) => { + if (!cachedProjectRegionKeyResolver) { + cachedProjectRegionKeyResolver = await buildProjectRegionKeyResolver() + } + + return await cachedProjectRegionKeyResolver({ projectId }) +} diff --git a/packages/server/modules/notifications/services/handlers/mentionedInComment.ts b/packages/server/modules/notifications/services/handlers/mentionedInComment.ts index 8aa7ce7e00..d5d1ca9359 100644 --- a/packages/server/modules/notifications/services/handlers/mentionedInComment.ts +++ b/packages/server/modules/notifications/services/handlers/mentionedInComment.ts @@ -17,7 +17,7 @@ import { renderEmail } from '@/modules/emails/services/emailRendering' import { sendEmail } from '@/modules/emails/services/sending' -import { getProjectDbClient } from '@/modules/multiregion/dbSelector' +import { getProjectDbClient } from '@/modules/multiregion/utils/dbSelector' import { NotificationValidationError } from '@/modules/notifications/errors' import { NotificationHandler, diff --git a/packages/server/modules/previews/index.ts b/packages/server/modules/previews/index.ts index 5c331ae684..f6899b95b8 100644 --- a/packages/server/modules/previews/index.ts +++ b/packages/server/modules/previews/index.ts @@ -27,7 +27,7 @@ import { getStreamFactory } from '@/modules/core/repositories/streams' import { getPaginatedBranchCommitsItemsByNameFactory } from '@/modules/core/services/commit/retrieval' import { getStreamBranchByNameFactory } from '@/modules/core/repositories/branches' import { getFormattedObjectFactory } from '@/modules/core/repositories/objects' -import { getProjectDbClient } from '@/modules/multiregion/dbSelector' +import { getProjectDbClient } from '@/modules/multiregion/utils/dbSelector' import { listenFor } from '@/modules/core/utils/dbNotificationListener' const httpErrorImage = (httpErrorCode: number) => diff --git a/packages/server/modules/previews/resultListener.ts b/packages/server/modules/previews/resultListener.ts index 48580a5aad..7fd0f20b32 100644 --- a/packages/server/modules/previews/resultListener.ts +++ b/packages/server/modules/previews/resultListener.ts @@ -2,7 +2,7 @@ import { ProjectSubscriptions } from '@/modules/shared/utils/subscriptions' import { MessageType } from '@/modules/core/utils/dbNotificationListener' import { getObjectCommitsWithStreamIdsFactory } from '@/modules/core/repositories/commits' import { publish } from '@/modules/shared/utils/subscriptions' -import { getProjectDbClient } from '@/modules/multiregion/dbSelector' +import { getProjectDbClient } from '@/modules/multiregion/utils/dbSelector' const payloadRegexp = /^([\w\d]+):([\w\d]+):([\w\d]+)$/i diff --git a/packages/server/modules/webhooks/graph/resolvers/webhooks.ts b/packages/server/modules/webhooks/graph/resolvers/webhooks.ts index b0aad1607c..4a2188abdd 100644 --- a/packages/server/modules/webhooks/graph/resolvers/webhooks.ts +++ b/packages/server/modules/webhooks/graph/resolvers/webhooks.ts @@ -18,7 +18,7 @@ import { } from '@/modules/webhooks/repositories/webhooks' import { ForbiddenError } from '@/modules/shared/errors' import { TokenResourceIdentifier } from '@/modules/core/domain/tokens/types' -import { getProjectDbClient } from '@/modules/multiregion/dbSelector' +import { getProjectDbClient } from '@/modules/multiregion/utils/dbSelector' const streamWebhooksResolver = async ( parent: { id: string }, diff --git a/packages/server/modules/webhooks/index.ts b/packages/server/modules/webhooks/index.ts index 0ca35a563f..8cb439d628 100644 --- a/packages/server/modules/webhooks/index.ts +++ b/packages/server/modules/webhooks/index.ts @@ -9,7 +9,7 @@ import { import { cleanOrphanedWebhookConfigsFactory } from '@/modules/webhooks/repositories/cleanup' import { Knex } from 'knex' import { db } from '@/db/knex' -import { getRegisteredDbClients } from '@/modules/multiregion/dbSelector' +import { getRegisteredDbClients } from '@/modules/multiregion/utils/dbSelector' const scheduleWebhookCleanupFactory = ({ db }: { db: Knex }) => { const scheduleExecution = scheduleExecutionFactory({ diff --git a/packages/server/modules/workspaces/graph/resolvers/regions.ts b/packages/server/modules/workspaces/graph/resolvers/regions.ts index 8ffcdd5ee8..a80df9e942 100644 --- a/packages/server/modules/workspaces/graph/resolvers/regions.ts +++ b/packages/server/modules/workspaces/graph/resolvers/regions.ts @@ -2,7 +2,7 @@ import { db } from '@/db/knex' import { Resolvers } from '@/modules/core/graph/generated/graphql' import { getWorkspacePlanFactory } from '@/modules/gatekeeper/repositories/billing' import { canWorkspaceUseRegionsFactory } from '@/modules/gatekeeper/services/featureAuthorization' -import { getDb } from '@/modules/multiregion/dbSelector' +import { getDb } from '@/modules/multiregion/utils/dbSelector' import { getRegionsFactory } from '@/modules/multiregion/repositories' import { authorizeResolver } from '@/modules/shared' import { diff --git a/packages/server/modules/workspaces/graph/resolvers/workspaces.ts b/packages/server/modules/workspaces/graph/resolvers/workspaces.ts index 6055834117..4f456a28ff 100644 --- a/packages/server/modules/workspaces/graph/resolvers/workspaces.ts +++ b/packages/server/modules/workspaces/graph/resolvers/workspaces.ts @@ -163,7 +163,7 @@ import { isRateLimitBreached } from '@/modules/core/services/ratelimiter' import { RateLimitError } from '@/modules/core/errors/ratelimit' -import { getRegionDb } from '@/modules/multiregion/dbSelector' +import { getRegionDb } from '@/modules/multiregion/utils/dbSelector' import { listUserExpiredSsoSessionsFactory, listWorkspaceSsoMembershipsByUserEmailFactory diff --git a/packages/server/modules/workspaces/services/projects.ts b/packages/server/modules/workspaces/services/projects.ts index 2d7282fc59..886a28c99c 100644 --- a/packages/server/modules/workspaces/services/projects.ts +++ b/packages/server/modules/workspaces/services/projects.ts @@ -34,7 +34,7 @@ import { } from '@/modules/core/domain/streams/operations' import { ProjectNotFoundError } from '@/modules/core/errors/projects' import { WorkspaceProjectCreateInput } from '@/test/graphql/generated/graphql' -import { getDb } from '@/modules/multiregion/dbSelector' +import { getDb } from '@/modules/multiregion/utils/dbSelector' import { createNewProjectFactory } from '@/modules/core/services/projects' import { deleteProjectFactory, diff --git a/packages/server/modules/workspaces/tests/helpers/creation.ts b/packages/server/modules/workspaces/tests/helpers/creation.ts index 368011dd5a..977941bc52 100644 --- a/packages/server/modules/workspaces/tests/helpers/creation.ts +++ b/packages/server/modules/workspaces/tests/helpers/creation.ts @@ -73,7 +73,7 @@ import { getDefaultRegionFactory, upsertRegionAssignmentFactory } from '@/modules/workspaces/repositories/regions' -import { getDb } from '@/modules/multiregion/dbSelector' +import { getDb } from '@/modules/multiregion/utils/dbSelector' const { FF_WORKSPACES_MODULE_ENABLED } = getFeatureFlags() diff --git a/packages/server/multiregion.example.json b/packages/server/multiregion.example.json index fc84bf17cb..77e82f94ef 100644 --- a/packages/server/multiregion.example.json +++ b/packages/server/multiregion.example.json @@ -8,8 +8,9 @@ "accessKey": "minioadmin", "secretKey": "minioadmin", "bucket": "speckle-server", - "createBucketIfNotExists": "true", - "endpoint": "http://127.0.0.1:9000" + "createBucketIfNotExists": true, + "endpoint": "http://127.0.0.1:9000", + "s3Region": "us-east-1" } }, "regions": { @@ -22,8 +23,9 @@ "accessKey": "minioadmin", "secretKey": "minioadmin", "bucket": "speckle-server", - "createBucketIfNotExists": "true", - "endpoint": "http://127.0.0.1:9020" + "createBucketIfNotExists": true, + "endpoint": "http://127.0.0.1:9020", + "s3Region": "us-east-1" } } } diff --git a/packages/server/multiregion.test.example.json b/packages/server/multiregion.test.example.json index 3f68d3acdf..0eff189562 100644 --- a/packages/server/multiregion.test.example.json +++ b/packages/server/multiregion.test.example.json @@ -8,8 +8,9 @@ "accessKey": "minioadmin", "secretKey": "minioadmin", "bucket": "speckle-server", - "createBucketIfNotExists": "true", - "endpoint": "http://127.0.0.1:9000" + "createBucketIfNotExists": true, + "endpoint": "http://127.0.0.1:9000", + "s3Region": "us-east-1" } }, "regions": { @@ -21,9 +22,10 @@ "blobStorage": { "accessKey": "minioadmin", "secretKey": "minioadmin", - "bucket": "speckle-server", - "createBucketIfNotExists": "true", - "endpoint": "http://127.0.0.1:9020" + "bucket": "test-speckle-server", + "createBucketIfNotExists": true, + "endpoint": "http://127.0.0.1:9020", + "s3Region": "us-east-1" } } } diff --git a/packages/server/readme.md b/packages/server/readme.md index 78afe2fcdb..02dc160eca 100644 --- a/packages/server/readme.md +++ b/packages/server/readme.md @@ -90,10 +90,10 @@ For non-authenticated api exploration, you can use the Graphql Playground which To run all tests, simply run `yarn test`. The recommended extensions for the workspace include a test explorer, that can run individual tests. -If you really want to run specific tests from a terminal, use the `mocha --grep @subset` syntax. For example: +If you really want to run specific tests from a terminal, use the `yarn test --grep @subset` syntax. For example: -- `mocha --grep @auth --watch` to run tests pertaning to the auth module only in watch mode. -- `mocha --grep @core-streams --watch` to run tests pertaining to stream related services. +- `yarn test --grep="@auth" --watch` to run tests pertaning to the auth module only in watch mode. +- `yarn test --grep="@core-streams" --watch` to run tests pertaining to stream related services. It's suggested to just run tests from the VSCode test explorer, however. diff --git a/packages/server/test/hooks.ts b/packages/server/test/hooks.ts index ab4597e01e..0995b66ede 100644 --- a/packages/server/test/hooks.ts +++ b/packages/server/test/hooks.ts @@ -17,7 +17,14 @@ import { once } from 'events' import type http from 'http' import type express from 'express' import type net from 'net' -import { MaybeAsync, MaybeNullOrUndefined, Optional, wait } from '@speckle/shared' +import { + ensureError, + MaybeAsync, + MaybeNullOrUndefined, + Nullable, + Optional, + wait +} from '@speckle/shared' import * as mocha from 'mocha' import { getAvailableRegionKeysFactory, @@ -34,7 +41,7 @@ import { import { getRegisteredRegionClients, initializeRegion -} from '@/modules/multiregion/dbSelector' +} from '@/modules/multiregion/utils/dbSelector' import { Knex } from 'knex' import { isMultiRegionTestMode } from '@/test/speckle-helpers/regions' import { isMultiRegionEnabled } from '@/modules/multiregion/helpers' @@ -43,6 +50,7 @@ import { ApolloServer } from '@apollo/server' import { ReadinessHandler } from '@/healthchecks/health' import { set } from 'lodash' import { fixStackTrace } from '@/test/speckle-helpers/error' +import { EnvironmentResourceError } from '@/modules/shared/errors' // why is server config only created once!???? // because its done in a migration, to not override existing configs @@ -106,7 +114,7 @@ const ensureAivenExtrasFactory = (deps: { db: Knex }) => async () => { const setupDatabases = async () => { // First reset main db const db = mainDb - const resetMainDb = resetSchemaFactory({ db }) + const resetMainDb = resetSchemaFactory({ db, regionKey: null }) await resetMainDb() const getAvailableRegionKeys = getAvailableRegionKeysFactory({ @@ -138,8 +146,8 @@ const setupDatabases = async () => { regionClients = await getRegisteredRegionClients() // Reset each region DB client (re-run all migrations and setup) - for (const client of Object.values(regionClients)) { - const reset = resetSchemaFactory({ db: client }) + for (const [regionKey, db] of Object.entries(regionClients)) { + const reset = resetSchemaFactory({ db, regionKey }) await reset() } @@ -240,18 +248,32 @@ const truncateTablesFactory = (deps: { db: Knex }) => async (tableNames?: string } } -const resetSchemaFactory = (deps: { db: Knex }) => async () => { - const resetPubSub = resetPubSubFactory(deps) - const truncate = truncateTablesFactory(deps) - - await unlockFactory(deps)() - await resetPubSub() - await truncate() // otherwise some rollbacks will fail - - // Reset schema - await deps.db.migrate.rollback() - await deps.db.migrate.latest() -} +const resetSchemaFactory = + (deps: { db: Knex; regionKey: Nullable }) => async () => { + const { regionKey } = deps + + const resetPubSub = resetPubSubFactory(deps) + const truncate = truncateTablesFactory(deps) + + await unlockFactory(deps)() + await resetPubSub() + await truncate() // otherwise some rollbacks will fail + + // Reset schema + try { + await deps.db.migrate.rollback() + await deps.db.migrate.latest() + } catch (e) { + throw new EnvironmentResourceError( + `Failed to reset schema for ${ + regionKey ? 'region ' + regionKey + ' ' : 'main DB' + }`, + { + cause: ensureError(e) + } + ) + } + } export const truncateTables = async ( tableNames?: string[], diff --git a/packages/server/test/mocks/global.ts b/packages/server/test/mocks/global.ts index d80a63e08c..f0b6ca3d4c 100644 --- a/packages/server/test/mocks/global.ts +++ b/packages/server/test/mocks/global.ts @@ -13,8 +13,12 @@ export const CommentsRepositoryMock = mockRequireModule< >(['@/modules/comments/repositories/comments']) export const MultiRegionDbSelectorMock = mockRequireModule< - typeof import('@/modules/multiregion/dbSelector') ->(['@/modules/multiregion/dbSelector']) + typeof import('@/modules/multiregion/utils/dbSelector') +>(['@/modules/multiregion/utils/dbSelector']) + +export const MultiRegionBlobStorageSelectorMock = mockRequireModule< + typeof import('@/modules/multiregion/utils/blobStorageSelector') +>(['@/modules/multiregion/utils/blobStorageSelector']) export const MultiRegionConfigMock = mockRequireModule< typeof import('@/modules/multiregion/regionConfig') diff --git a/packages/server/test/speckle-helpers/branchHelper.ts b/packages/server/test/speckle-helpers/branchHelper.ts index 8a1a22099b..f21f7b9be1 100644 --- a/packages/server/test/speckle-helpers/branchHelper.ts +++ b/packages/server/test/speckle-helpers/branchHelper.ts @@ -6,7 +6,7 @@ import { getStreamBranchByNameFactory } from '@/modules/core/repositories/branches' import { createBranchAndNotifyFactory } from '@/modules/core/services/branch/management' -import { getProjectDbClient } from '@/modules/multiregion/dbSelector' +import { getProjectDbClient } from '@/modules/multiregion/utils/dbSelector' import { publish } from '@/modules/shared/utils/subscriptions' import { BasicTestUser } from '@/test/authHelper' import { BasicTestStream } from '@/test/speckle-helpers/streamHelper' diff --git a/packages/server/test/speckle-helpers/commitHelper.ts b/packages/server/test/speckle-helpers/commitHelper.ts index 33789b3b4f..87b4e4f011 100644 --- a/packages/server/test/speckle-helpers/commitHelper.ts +++ b/packages/server/test/speckle-helpers/commitHelper.ts @@ -23,7 +23,7 @@ import { createCommitByBranchNameFactory } from '@/modules/core/services/commit/management' import { createObjectFactory } from '@/modules/core/services/objects/management' -import { getProjectDbClient } from '@/modules/multiregion/dbSelector' +import { getProjectDbClient } from '@/modules/multiregion/utils/dbSelector' import { publish } from '@/modules/shared/utils/subscriptions' import { BasicTestUser } from '@/test/authHelper' import { BasicTestStream } from '@/test/speckle-helpers/streamHelper' diff --git a/packages/shared/src/environment/multiRegionConfig.ts b/packages/shared/src/environment/multiRegionConfig.ts index d0874be6af..27c504165a 100644 --- a/packages/shared/src/environment/multiRegionConfig.ts +++ b/packages/shared/src/environment/multiRegionConfig.ts @@ -2,8 +2,11 @@ import { z } from 'zod' import fs from 'node:fs/promises' import { Knex, knex } from 'knex' import { Logger } from 'pino' +import { getFeatureFlags } from './index.js' -export const regionConfigSchema = z.object({ +const useV1Config = !getFeatureFlags().FF_WORKSPACES_MULTI_REGION_BLOB_STORAGE_ENABLED + +const regionConfigSchemaV1 = z.object({ postgres: z.object({ connectionUri: z .string() @@ -21,17 +24,25 @@ export const regionConfigSchema = z.object({ .describe('Public TLS ("CA") certificate for the Postgres server') .optional() }) - //TODO - add the rest of the config when blob storage is implemented - // blobStorage: z - // .object({ - // endpoint: z.string().url(), - // accessKey: z.string(), - // secretKey: z.string(), - // bucket: z.string() - // }) }) -export const multiRegionConfigSchema = z.object({ +const regionConfigSchema = regionConfigSchemaV1.extend({ + blobStorage: z.object({ + endpoint: z.string().url(), + accessKey: z.string(), + secretKey: z.string(), + bucket: z.string(), + createBucketIfNotExists: z.boolean(), + s3Region: z.string() + }) +}) + +const multiRegionConfigV1Schema = z.object({ + main: regionConfigSchemaV1, + regions: z.record(z.string(), regionConfigSchemaV1) +}) + +const multiRegionConfigSchema = z.object({ main: regionConfigSchema, regions: z.record(z.string(), regionConfigSchema) }) @@ -40,6 +51,7 @@ export type MultiRegionConfig = z.infer export type MainRegionConfig = MultiRegionConfig['main'] export type DataRegionsConfig = MultiRegionConfig['regions'] export type RegionServerConfig = z.infer +export type BlobStorageConfig = RegionServerConfig['blobStorage'] export const loadMultiRegionsConfig = async ({ path @@ -63,13 +75,16 @@ export const loadMultiRegionsConfig = async ({ throw new Error(`Multi-region config file at path '${path}' is not valid JSON`) } - const multiRegionConfigFileResult = multiRegionConfigSchema.safeParse(parsedJson) // This will throw if the config is invalid + const schema = useV1Config ? multiRegionConfigV1Schema : multiRegionConfigSchema + const multiRegionConfigFileResult = schema.safeParse(parsedJson) // This will throw if the config is invalid if (!multiRegionConfigFileResult.success) throw new Error( `Multi-region config file at path '${path}' does not fit the schema: ${multiRegionConfigFileResult.error}` ) - return multiRegionConfigFileResult.data + // Type assertion should be fine cause the FF should be temporary AND v1 logic should not even + // try to access the extra blobStorage fields anyway + return multiRegionConfigFileResult.data as MultiRegionConfig } export type KnexConfigArgs = { @@ -133,7 +148,7 @@ export const createKnexConfig = ({ } export const configureKnexClient = ( - config: RegionServerConfig, + config: Pick, configArgs: KnexConfigArgs ): { public: Knex; private?: Knex } => { const knexConfig = createKnexConfig({