diff --git a/README.md b/README.md index 13bc028..07df8b3 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,7 @@ Please manually create dataset, handle permission on Google Cloud, and setup rel Run the following script to create big query tables under the dataset specified in the environment variable. ``` -./node_modules/.bin/babel-node db/createBqTables.ts --extensions .ts,.js +./node_modules/.bin/babel-node db/setBqTables.ts --extensions .ts,.js ``` --- diff --git a/bq/events.ts b/bq/events.ts index 0499ab3..645c476 100644 --- a/bq/events.ts +++ b/bq/events.ts @@ -7,6 +7,7 @@ import { convert } from 'zoq'; export const TABLE = 'events'; const eventBatchSchema = z.object({ + userId: z.string().nullable(), createdAt: z.date().describe('Time of the event batch is sent'), text: z.string().describe('Message text').nullable(), messageSource: z.enum(['user', 'group', 'room']).nullable(), diff --git a/db/createBqTables.ts b/db/createBqTables.ts deleted file mode 100644 index 7a4a4b3..0000000 --- a/db/createBqTables.ts +++ /dev/null @@ -1,28 +0,0 @@ -/** - * BigQuery client and schema - */ -import 'dotenv/config'; -import { BigQuery } from '@google-cloud/bigquery'; -import * as bq from '../bq/events'; - -const bqDataset = new BigQuery().dataset( - process.env.BIGQUERY_ANALYTICS_DATASET || '' -); - -async function loadSchema() { - const [table] = await bqDataset.createTable(bq.TABLE, { - schema: bq.SCHEMA, - timePartitioning: { - type: 'MONTH', - field: 'createdAt', - }, - }); - - console.log(`Table ${table.id} created under dataset ${table.dataset.id}.`); - console.log('You may need to wait up to 2 minutes before seeing it in GCS console.'); -} - -loadSchema().catch((e) => { - console.error('[createBqTables]', e); - process.exit(1); -}); diff --git a/db/setBqTables.ts b/db/setBqTables.ts new file mode 100644 index 0000000..1b9d16f --- /dev/null +++ b/db/setBqTables.ts @@ -0,0 +1,38 @@ +/** + * BigQuery client and schema + */ +import 'dotenv/config'; +import { BigQuery } from '@google-cloud/bigquery'; +import type { TableSchema } from '@google-cloud/bigquery'; +import * as bqEvents from '../bq/events'; + +const bqDataset = new BigQuery().dataset( + process.env.BIGQUERY_ANALYTICS_DATASET || '' +); + +async function loadSchema({TABLE, SCHEMA}: {TABLE: string, SCHEMA: TableSchema[]}) { + const table = bqDataset.table(TABLE); + + if(await table.exists()) { + const [result] = await table.setMetadata({schema: SCHEMA}); + + console.log(`Table ${table.id} fields updated to:`, result.schema.fields); + return; + } + + const [createdTable] = await bqDataset.createTable(TABLE, { + schema: SCHEMA, + timePartitioning: { + type: 'MONTH', + field: 'createdAt', + }, + }); + + console.log(`Table ${table.id} created under dataset ${createdTable.dataset.id}.`); + console.log('You may need to wait up to 2 minutes before seeing it in GCS console.'); +} + +loadSchema(bqEvents).catch((e) => { + console.error('[setBqTables]', e); + process.exit(1); +});