Merge pull request #82 from clocklimited/feature/mongo-7

feat: refactor to upgrade to mongodb driver 6.9
clocklimited · Nov 25, 2024 · b43bea1 · b43bea1
2 parents c4d8e8c + 4460d3d
commit b43bea1
Show file tree

Hide file tree

Showing 7 changed files with 182 additions and 173 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,27 @@
 # CHANGELOG
 
+## Version 14.0.0
+
+A major upgrade to the MongoDB driver from 3.7.3 to 6.9.0.
+
+This means MongoDB server versions below 3.6 are **no longer supported**.
+(3.6 support is _deprecated_ in this version).
+
+Additionally, the native GridFS driver removes the `md5` and `contentType` attributes from records.
+These are now stored under `metadata.{md5,contentType}`. No consumer application changes are required - only if other
+supporting applications are interacting directly with the Darkroom database.
+
+This means a database update is required for migrations from v13.0.0 to v14.0.0.
+
+To do this, run the following mongo shell commands in order:
+
+```js
+
+db['fs.files'].updateMany({ md5: { $exists: true } }, { $rename: { 'md5': 'metadata.md5' } })
+db['fs.files'].updateMany({ contentType: { $exists: true } }, { $rename: { 'contentType': 'metadata.contentType' } })
+db['fs.files'].updateMany({ filename: '' }, [{ $set: { 'filename': '$metadata.md5' }] })
+```
+
 ## Version 13.0.0
 
 Updates response code on restricted uploads file types from 403 to 415.

diff --git a/lib/backends/MongoBackend.js b/lib/backends/MongoBackend.js
@@ -1,5 +1,4 @@
 const mongo = require('mongodb')
-const GridStream = require('@appveen/gridfs-stream')
 const tmp = require('tmp')
 const { PassThrough } = require('stream')
 const crypto = require('crypto')
@@ -11,24 +10,26 @@ const Backend = require('./Backend')
 class MongoBackend extends Backend {
   setup(cb) {
     const client = new mongo.MongoClient(this.config.databaseUri)
-    client.connect((error) => {
-      if (error) return cb(error)
+    client
+      .connect()
+      .then(() => {
+        const db = client.db(this.config.databaseName)
 
-      const db = client.db(this.config.databaseName)
+        const gfs = new mongo.GridFSBucket(db)
 
-      const gfsStream = new GridStream(db, mongo)
-      const gfs = new mongo.GridFSBucket(db)
+        this._db = db
+        this._gfs = gfs
 
-      this._db = db
-      this._gfsStream = gfsStream
-      this._gfs = gfs
-
-      cb()
-    })
+        cb()
+      })
+      .catch(cb)
   }
 
   clean(cb) {
-    this._db.dropDatabase(cb)
+    this._db
+      .dropDatabase()
+      .then(() => cb())
+      .catch(cb)
   }
 
   isHealthy(cb) {
@@ -37,42 +38,44 @@ class MongoBackend extends Backend {
     this._db
       .collection('users')
       .find({})
-      .toArray((error) => {
-        if (error) {
-          return cb(error, false)
-        }
-        cb(null, true)
-      })
+      .toArray()
+      .then(() => cb(null, true))
+      .catch((error) => cb(error, false))
   }
 
   _getReadStream(query, id) {
     const stream = new PassThrough()
-    this._gfsStream.findOne(query, (error, file) => {
-      if (file === null) return stream.emit('notFound', id)
-      if (error) return stream.emit('error', error)
-      const readStream = this._gfsStream.createReadStream({ _id: file._id })
-      readStream.pipe(stream)
-      stream.emit('meta', {
-        type: file.contentType,
-        size: file.length,
-        lastModified: file.uploadDate,
-        originalId: file.metadata.originalId
-      })
+    this._gfs
+      .find(query)
+      .limit(1)
+      .toArray()
+      .then((files) => files[0])
+      .then((file) => {
+        if (!file) return stream.emit('notFound', id)
+        const readStream = this._gfs.openDownloadStream(file._id)
+        readStream.pipe(stream)
+        stream.emit('meta', {
+          type: file.metadata.contentType,
+          size: file.length,
+          lastModified: file.uploadDate,
+          originalId: file.metadata.originalId
+        })
 
-      readStream.on('error', (error) => {
-        if (error.code === 'ENOENT') {
-          stream.emit('notFound', id)
-        } else {
-          stream.emit('error', error)
-        }
+        readStream.on('error', (error) => {
+          if (error.code === 'ENOENT') {
+            stream.emit('notFound', id)
+          } else {
+            stream.emit('error', error)
+          }
+        })
       })
-    })
+      .catch((error) => stream.emit('error', error))
 
     return stream
   }
 
   createDataReadStream(id) {
-    return this._getReadStream({ md5: id }, id)
+    return this._getReadStream({ 'metadata.md5': id }, id)
   }
 
   createCacheReadStream(id) {
@@ -111,34 +114,36 @@ class MongoBackend extends Backend {
         return passThrough.emit('error', error)
       }
 
-      this._gfsStream.files.countDocuments({ md5: md5 }, (error, count) => {
-        if (error) {
-          tmpFile.removeCallback()
-          return passThrough.emit('error', error)
-        }
-        if (count > 0) {
-          tmpFile.removeCallback()
-          passThrough.emit('done', { id: md5, type: uploadType, size: size })
-        } else {
-          const mongoWriteStream = this._gfsStream.createWriteStream({
-            _id: '',
-            mode: 'w',
-            content_type: uploadType,
-            metadata: { type: 'data' }
-          })
-
-          mongoWriteStream.on('close', () => {
+      this._gfs
+        .find({ 'metadata.md5': md5 })
+        .limit(1)
+        .toArray()
+        .then((files) => files.length)
+        .then((count) => {
+          if (count > 0) {
             tmpFile.removeCallback()
-            passThrough.emit('done', {
-              id: md5,
-              type: uploadType,
-              size: size
+            passThrough.emit('done', { id: md5, type: uploadType, size: size })
+          } else {
+            const mongoWriteStream = this._gfs.openUploadStream(md5, {
+              metadata: { type: 'data', contentType: uploadType, md5 }
             })
-          })
 
-          fs.createReadStream(tmpFile.name).pipe(mongoWriteStream)
-        }
-      })
+            mongoWriteStream.on('close', () => {
+              tmpFile.removeCallback()
+              passThrough.emit('done', {
+                id: md5,
+                type: uploadType,
+                size: size
+              })
+            })
+
+            fs.createReadStream(tmpFile.name).pipe(mongoWriteStream)
+          }
+        })
+        .catch((error) => {
+          tmpFile.removeCallback()
+          return passThrough.emit('error', error)
+        })
     })
 
     passThrough.pipe(typeDetectorStream)
@@ -158,8 +163,8 @@ class MongoBackend extends Backend {
   async deleteData(id) {
     const deletes = []
     const cursor = this._gfs.find({
-      md5: id,
-      'metadata.type': 'data'
+      'metadata.type': 'data',
+      'metadata.md5': id
     })
 
     while (await cursor.hasNext()) {
@@ -199,14 +204,11 @@ class MongoBackend extends Backend {
     })
 
     typeDetectorStream.on('detect', (type) => {
-      const mongoWriteStream = this._gfsStream.createWriteStream({
-        filename: id,
-        mode: 'w',
-        content_type: type,
-        metadata: { type: 'cache', originalId: originalId }
+      const mongoWriteStream = this._gfs.openUploadStream(id, {
+        metadata: { type: 'cache', originalId: originalId, contentType: type }
       })
 
-      mongoWriteStream.on('close', (file) => {
+      mongoWriteStream.on('finish', () => {
         if (size === 0) {
           const error = new Error('Zero size')
           error.name = 'SizeError'
@@ -215,7 +217,7 @@ class MongoBackend extends Backend {
 
         passThrough.emit('done', {
           id: id,
-          size: file.length,
+          size: mongoWriteStream.gridFSFile.length,
           type: type
         })
       })

diff --git a/package.json b/package.json
@@ -5,7 +5,6 @@
   "description": "Darkroom API for image manipulation",
   "main": "app.js",
   "dependencies": {
-    "@appveen/gridfs-stream": "^1.0.0",
     "@clocklimited/darkroom": "^8.1.1",
     "@serby/logger": "^3.1.0",
     "async": "^3.2.0",
@@ -21,7 +20,7 @@
     "lodash.clone": "^4.0.1",
     "mkdirp": "^1.0.4",
     "mmmagic": "^0.5.3",
-    "mongodb": "3.7.3",
+    "mongodb": "6.9.0",
     "morgan": "^1.10.0",
     "mv": "^2.0.3",
     "response-time": "^2.3.2",

diff --git a/support/migration/fs-to-s3.js b/support/migration/fs-to-s3.js
@@ -161,7 +161,7 @@ async function migrateImages() {
       const params = {
         Bucket: config.bucket,
         Key: `${file.type}/${file.id}`,
-        ContentType: file.contentType,
+        ContentType: file.metadata.contentType,
         ContentLength: file.size,
         Metadata: { type: file.type },
         Body: readStream
@@ -180,7 +180,7 @@ async function migrateImages() {
       retries: 5,
       onFailedAttempt: (error) => {
         console.log(
-          `File #${i} ${file.md5} failed attempt #${error.attemptNumber}. ${error.retriesLeft} retries left.`
+          `File #${i} ${file.metadata.md5} failed attempt #${error.attemptNumber}. ${error.retriesLeft} retries left.`
         )
       }
     })

diff --git a/support/migration/mongodb-to-s3.js b/support/migration/mongodb-to-s3.js
@@ -1,6 +1,5 @@
 /* eslint-disable no-console */
 const mongo = require('mongodb')
-const GridStream = require('@appveen/gridfs-stream')
 const AWS = require('aws-sdk')
 const pLimit = require('p-limit')
 const pRetry = require('p-retry')
@@ -16,7 +15,6 @@ async function migrateImages() {
   await client.connect()
 
   const db = client.db(config.databaseName)
-  const gfsStream = new GridStream(db, mongo)
   const gfs = new mongo.GridFSBucket(db)
 
   AWS.config.update({
@@ -46,7 +44,7 @@ async function migrateImages() {
       const s3Object = await s3
         .headObject({
           Bucket: config.bucket,
-          Key: `${file.metadata.type}/${file.md5}`
+          Key: `${file.metadata.type}/${file.metadata.md5}`
         })
         .promise()
       return s3Object !== null
@@ -83,8 +81,8 @@ async function migrateImages() {
       const [nextGridFsFile] = await getFileAtIndex(mid + 1)
 
       console.log({
-        gridFsFile: gridFsFile && gridFsFile.md5,
-        nextGridFsFile: nextGridFsFile && nextGridFsFile.md5
+        gridFsFile: gridFsFile && gridFsFile.metadata.md5,
+        nextGridFsFile: nextGridFsFile && nextGridFsFile.metadata.md5
       })
       const fileInS3 = await hasFileInS3(gridFsFile)
       const nextFileInS3 = nextGridFsFile
@@ -120,13 +118,13 @@ async function migrateImages() {
 
   const migrateFile = (file, i) =>
     new Promise((resolve, reject) => {
-      console.log(`Migrating #${i} ${file.md5}...`)
-      const readStream = gfsStream.createReadStream({ _id: file._id })
+      console.log(`Migrating #${i} ${file.metadata.md5}...`)
+      const readStream = gfs.openDownloadStream(file._id)
 
       const params = {
         Bucket: config.bucket,
-        Key: `${file.metadata.type}/${file.md5}`,
-        ContentType: file.contentType,
+        Key: `${file.metadata.type}/${file.metadata.md5}`,
+        ContentType: file.metadata.contentType,
         ContentLength: file.length,
         Metadata: { type: file.metadata.type },
         Body: readStream
@@ -145,7 +143,7 @@ async function migrateImages() {
       retries: 5,
       onFailedAttempt: (error) => {
         console.log(
-          `File #${i} ${file.md5} failed attempt #${error.attemptNumber}. ${error.retriesLeft} retries left.`
+          `File #${i} ${file.metadata.md5} failed attempt #${error.attemptNumber}. ${error.retriesLeft} retries left.`
         )
       }
     })

diff --git a/test/lib/backends/mongo.test.js b/test/lib/backends/mongo.test.js
@@ -31,10 +31,12 @@ describe('Mongo Backend using: ' + getConfig().databaseUri, function () {
       writeStream.on('done', function (file) {
         backend._db
           .collection('fs.files')
-          .findOne({ md5: file.id }, function (err, data) {
+          .findOne({ 'metadata.md5': file.id })
+          .then((data) => {
             assert.strictEqual(data.metadata.type, 'data')
             done()
           })
+          .catch(done)
       })
       writeStream.write('hello')
       writeStream.end()
@@ -45,10 +47,12 @@ describe('Mongo Backend using: ' + getConfig().databaseUri, function () {
       writeStream.on('done', function (file) {
         backend._db
           .collection('fs.files')
-          .findOne({ filename: file.id }, function (err, data) {
+          .findOne({ filename: file.id })
+          .then((data) => {
             assert.strictEqual(data.metadata.type, 'cache')
             done()
           })
+          .catch(done)
       })
       writeStream.write('hello')
       writeStream.end()