Deduplicate pages from notion (#88)

* Support caching pages

* Add tests for notion/cache

* Add clearPages function

* Improve caching logic

* Update cache for setPageProperty calls

* Improve cache expiration strategy

* Improve test coverage

__tests__/notion/cache.test.ts

@@ -0,0 +1,61 @@
+import { PageObjectResponse } from '@notionhq/client/build/src/api-endpoints.js'
+import { getPage, putPage, clearPages, cleanupPages } from '../../src/notion/cache.js'
+
+const page: PageObjectResponse = {
+  object: 'page',
+  parent: {
+    type: 'database_id',
+    database_id: ''
+  },
+  properties: undefined,
+  icon: {
+    type: 'emoji',
+    emoji: '😀'
+  },
+  cover: {
+    type: 'external',
+    external: {
+      url: ''
+    }
+  },
+  created_by: {
+    id: '',
+    object: 'user'
+  },
+  last_edited_by: {
+    id: '',
+    object: 'user'
+  },
+  id: '',
+  created_time: '',
+  last_edited_time: '',
+  archived: false,
+  url: ''
+}
+
+describe('cached pages operations', () => {
+  it('put, get, clear, and get', () => {
+    clearPages() // reset cache
+
+    putPage(page)
+    expect(getPage(page.id)).toBe(page)
+    clearPages()
+    expect(getPage(page.id)).toBe(null)
+  })
+
+  it('cleanup', () => {
+    clearPages() // reset cache
+
+    putPage(page)
+    expect(getPage(page.id)).toBe(page)
+
+    // cleanupPages decrements ttl
+    // and removes pages with ttl === 0
+    // so the page should be removed
+    // after two calls to cleanupPages
+    cleanupPages()
+    expect(getPage(page.id)).toBe(page)
+    cleanupPages()
+    expect(getPage(page.id)).toBe(null)
+  })
+})

src/notion/cache.ts

@@ -0,0 +1,38 @@
+import { PageObjectResponse } from "@notionhq/client/build/src/api-endpoints"
+
+let cachedPages: Record<string, {
+  page: PageObjectResponse,
+  ttl: number
+}> = {}
+
+export function getPage(pageId: string) {
+  if (pageId in cachedPages) {
+    const { page, ttl } = cachedPages[pageId]
+    if (ttl === 0) return null
+    return page
+  }
+  return null
+}
+
+export function putPage(page: PageObjectResponse) {
+  cachedPages[page.id] = {
+    page,
+    ttl: 2 // count down to 0
+  }
+}
+
+// cleanupPages removes pages with ttl === 0
+// and decrements ttl for all other pages
+export function cleanupPages() {
+  for (const pageId in cachedPages) {
+    if (cachedPages[pageId].ttl === 0) {
+      delete cachedPages[pageId]
+    } else {
+      cachedPages[pageId].ttl -= 1
+    }
+  }
+}
+
+export function clearPages() {
+  cachedPages = {}
+}

src/notion/notion.ts

@@ -9,6 +9,7 @@ import {
   PartialPageObjectResponse,
 } from '@notionhq/client/build/src/api-endpoints.js'
 import cache from '../utils/cache.js'
+import * as notionCache from './cache.js'
 
 export const notion = new Client({ auth: process.env.NOTION_KEY })
 
@@ -57,8 +58,6 @@ export async function getNewPagesFromDatabase(
   databaseId: string,
   events: Set<string> = null,
 ): Promise<Array<PageObjectResponse>> {
-  // TODO: use an in-memory store to avoid duplicates
-
   let filter = null
   if (eventsContainsOnly(events, 'create', 'update')) {
     filter = {
@@ -85,9 +84,20 @@ export async function getNewPagesFromDatabase(
     throw error
   }
 
-  return pages.filter((page): page is PageObjectResponse => {
-    return isFullPage(page)
+  const pageResponse = pages.filter((page): page is PageObjectResponse => {
+    if (!isFullPage(page)) return false
+
+    const cachedPage = notionCache.getPage(page.id)
+    if (cachedPage !== null && cachedPage.last_edited_time === page.last_edited_time) {
+      // page hasn't changed from last query
+      return false
+    }
+
+    notionCache.putPage(page)
+    return true
   })
+
+  return pageResponse
 }
 
 export async function getPageProperty(pageId: string, propertyID: string) {
@@ -104,7 +114,7 @@ export async function setPageProperty(
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
   value: any,
 ) {
-  await notion.pages.update({
+  const page = await notion.pages.update({
     page_id: pageId,
     properties: {
       [propertyName]: {
@@ -114,4 +124,8 @@ export async function setPageProperty(
       },
     },
   })
+
+  if (isFullPage(page)) {
+    notionCache.putPage(page)
+  }
 }

src/runner/runner.ts

@@ -6,6 +6,7 @@ import { Configuration, ConfigurationIndex, Job } from '../utils/config.js'
 import { evaluate } from '../expression/expr.js'
 import { getIntFromEnv } from '../utils/env_setting.js'
 import { getLogger } from '../utils/logger.js'
+import * as notionCache from '../notion/cache.js'
 
 // exec is a function that executes a bash command
 const exec = util.promisify(child_process.exec)
@@ -64,6 +65,8 @@ export async function runWorkflow(index: ConfigurationIndex): Promise<number> {
     const wait = await runWorkflowForDB(databaseId, value.on, value.configs)
     if (wait > 0) return wait
   }
+
+  notionCache.cleanupPages()
   return 0
 }