chore: adds batched image upload

src/prisma/prismaservice/src/dobbelOmega/migrateImages.ts

@@ -1,4 +1,5 @@
 import { vevenIdToPnId, type IdMapper } from './IdMapper'
+import manifest from '@/src/logger'
 import { imageSizes, imageStoreLocation } from '@/src/seedImages'
 import { v4 as uuid } from 'uuid'
 import { writeFile, mkdir } from 'fs/promises'
@@ -70,22 +71,30 @@ export default async function migrateImages(
         }
     })
 
+    // Find what the profile collection is on veven
+    const vevenProfileCollection = await vevenPrisma.imageGroups.findFirstOrThrow({
+        where: {
+            name: 'Profilbilder',
+        },
+    })
+
+    manifest.info(`Before filter: ${images.length} images`)
     const imagesWithCollection = images.map(image => {
         let collectionId = vevenIdToPnId(migrateImageCollectionIdMap, image.ImageGroupId)
         if (image.Ombul.length) {
             collectionId = ombulCollection.id
-        } else if (image.UserId) {
-            collectionId = profileCollection.id
         } else if (!collectionId) {
             collectionId = garbageCollection.id
+        } else if (image.ImageGroupId === vevenProfileCollection.id) {
+            collectionId = profileCollection.id
         }
         return {
             ...image,
             collectionId,
         }
     }).filter(image => {
         //Apply limits
-        if (!limits.numberOffFullImageCollections) return true
+        if (limits.numberOffFullImageCollections === null) return true
         if (image.Ombul.length) return true
         if (image.Articles.length) return true
         if (image.Events.length) return true
@@ -94,32 +103,9 @@ export default async function migrateImages(
         if (image.ImageGroupId && image.ImageGroupId < limits.numberOffFullImageCollections) return true
         return false
     })
+    manifest.info(`After filter: ${imagesWithCollection.length} images`)
 
-    const imagesWithCollectionAndFs = await Promise.all(imagesWithCollection.map(async (image) => {
-        const ext = image.originalName.split('.').pop() || ''
-        const fsLocationDefaultOldVev = `${process.env.VEVEN_STORE_URL}/image/default/${image.name}`
-            + `?url=/store/images/${image.name}.${ext}`
-        const fsLocationMediumOldVev = `${process.env.VEVEN_STORE_URL}/image/resize/${imageSizes.medium}/`
-            + `${imageSizes.medium}/${image.name}?url=/store/images/${image.name}.${ext}`
-        const fsLocationSmallOldVev = `${process.env.VEVEN_STORE_URL}/image/resize/${imageSizes.small}/`
-            + `${imageSizes.small}/${image.name}?url=/store/images/${image.name}.${ext}`
-        const [fsLocationSmallSize, fsLocationMediumSize, fsLocation] = await Promise.all([
-            fetchImageAndUploadToStore(fsLocationSmallOldVev),
-            fetchImageAndUploadToStore(fsLocationMediumOldVev),
-            fetchImageAndUploadToStore(fsLocationDefaultOldVev)
-        ])
-        if (!fsLocation || !fsLocationMediumSize || !fsLocationSmallSize) {
-            console.error(`Failed to fetch image from ${fsLocationDefaultOldVev}`)
-            return null
-        }
-
-        return {
-            ...image,
-            fsLocationSmallSize,
-            fsLocationMediumSize,
-            fsLocation
-        }
-    }))
+    const imagesWithCollectionAndFs = await fetchAllImagesAndUploadToStore(imagesWithCollection)
 
     //correct names if there are duplicates
     const namesTaken: { name: string, times: number }[] = []
@@ -173,6 +159,51 @@ export default async function migrateImages(
     return migrateImageIdMap
 }
 
+async function fetchAllImagesAndUploadToStore<X extends {
+    originalName: string,
+    name: string,
+    id: number
+}>(images: X[]): Promise<((X & { fsLocation: string, fsLocationSmallSize: string, fsLocationMediumSize: string }) | null)[]> {
+    const ret: ((X & { fsLocation: string, fsLocationSmallSize: string, fsLocationMediumSize: string }) | null)[] = []
+    let imageCounter = 1
+    const batchSize = 1200
+    const imageBatches = images.reduce((acc, image) => {
+        if (acc[acc.length - 1].length >= batchSize) {
+            acc.push([image])
+        } else {
+            acc[acc.length - 1].push(image)
+        }
+        return acc
+    }, [[]] as X[][])
+    for (const imageBatch of imageBatches) {
+        await Promise.all(imageBatch.map(async image => {
+            manifest.info(`Migrating image number ${imageCounter++} of ${images.length}`)
+            const ext = image.originalName.split('.').pop() || ''
+            const fsLocationDefaultOldVev = `${process.env.VEVEN_STORE_URL}/image/default/${image.name}`
+                + `?url=/store/images/${image.name}.${ext}`
+            const fsLocationMediumOldVev = `${process.env.VEVEN_STORE_URL}/image/resize/${imageSizes.medium}/`
+                + `${imageSizes.medium}/${image.name}?url=/store/images/${image.name}.${ext}`
+            const fsLocationSmallOldVev = `${process.env.VEVEN_STORE_URL}/image/resize/${imageSizes.small}/`
+                + `${imageSizes.small}/${image.name}?url=/store/images/${image.name}.${ext}`
+            const fsLocation = await fetchImageAndUploadToStore(fsLocationDefaultOldVev)
+            const fsLocationMediumSize = await fetchImageAndUploadToStore(fsLocationMediumOldVev)
+            const fsLocationSmallSize = await fetchImageAndUploadToStore(fsLocationSmallOldVev)
+            if (!fsLocation || !fsLocationMediumSize || !fsLocationSmallSize) {
+                console.error(`Failed to fetch image from ${fsLocationDefaultOldVev}`)
+                ret.push(null)
+            } else {
+                ret.push({
+                    ...image,
+                    fsLocationSmallSize,
+                    fsLocationMediumSize,
+                    fsLocation
+                })
+            }
+        }))
+    }
+    return ret
+}
+
 /**
  * fetches an image from the Veven store and uploads it to the PN store (does NOT store the image
  * in the database, only the file on the server). Note that veven will often fail when we make a