offset validation

.gitignore

@@ -13,3 +13,4 @@ node_modules
 video-sync
 batch.sh
 *.csv
+.vscode/

README.md

@@ -48,7 +48,7 @@ $ video-sync -h # help page
 
 - `-f, --forceOffset` use the estimated offset as the final offset, no synching
 
-- `x, --exclusiveDirection=<ahead|behind>` only search the matching frame offset in one direction. 'ahead' means that the source video scene comes *before* the destination video scene. (requires algorithm=matching-scene)
+- `-x, --exclusiveDirection=<ahead|behind>` only search the matching frame offset in one direction. 'ahead' means that the source video scene comes *before* the destination video scene. (requires algorithm=matching-scene)
 
 - `-g, --algorithm=<simple|matching-scene>` [default: matching-scene] search algorithm to use for video syncing
 

src/index.js

@@ -16,6 +16,7 @@ const { ALGORITHMS, calcOffset } = require(`../util/calc-offset`)
 const { calculateOffset } = require(`../util/find-offset-new`)
 const merge = require(`../util/merge-tracks`)
 const tracks = require(`../util/tracks`)
+const { validateOffset } = require('../util/warping')
 
 class VideoSyncCommand extends Command {
   async run() {
@@ -237,15 +238,36 @@ class VideoSyncCommand extends Command {
     }
 
     // check if one of the videos is warped
-    //TODO import and test
+    let videoWarped = false
+    const offsetValidationSpinner = ora(`Checking if found offset applies to the whole video...`).start();
+    try {
+      videoWarped = ! await validateOffset(args.destination, args.source, videoOffset)
+    } catch (err) {
+      console.error(`Error while checking if found offset applies to the whole video:`, err)
+    }
 
+    // log warning about warped video
+    if (videoWarped && flags.confirm) {
+      offsetValidationSpinner.warn(`Syncing the tracks might not work well because one of the videos appears to be warped.`)
+    } else if (!videoWarped) {
+      offsetValidationSpinner.succeed(`Offset is valid.`)
+    } else {
+      offsetValidationSpinner.stop()
+    }
+
     let continueWithMerging = answers.output !== undefined && (selectedTracks.audio.length > 0 || selectedTracks.subs.length > 0)
 
     if (continueWithMerging && (!flags.confirm && flags.algorithm === `ssim` && confidence < 0.6)) {
       continueWithMerging = (await inquirer.prompt([{
         type: `confirm`,
         name: `continue`,
-        message: `Syncing confidence is very low (${confidence}). Do you want to continue?`,
+        message: `Syncing confidence is very low (${confidence}). Do you want to continue anyway?`,
+      }])).continue
+    } else if (continueWithMerging && videoWarped && !flags.confirm) {
+      continueWithMerging = (await inquirer.prompt([{
+        type: `confirm`,
+        name: `continue`,
+        message: `It seems like one of the videos might be warped (slightly sped up or slowed down). This might make synchronization impossible. Do you want to continue anyway?`,
       }])).continue
     }
 

util/calc-offset.js

@@ -190,6 +190,8 @@ async function getVideoInfo(vid1, vid2) {
 
   let vid1Data = await probe(vid1)
   let vid2Data = await probe(vid2)
+  console.log(`vid1Data:`, vid1Data)
+  console.log(`vid2Data:`, vid2Data)
 
   console.debug(`Video 1: width: ${vid1Data.streams[0].width}, height: ${vid1Data.streams[0].height}`)
   console.debug(`Video 2: width: ${vid2Data.streams[0].width}, height: ${vid2Data.streams[0].height}`)

util/find-offset-new.js

@@ -277,7 +277,6 @@ async function calculateOffset(video1, video2, options) {
       (sceneComparison.preSceneChangeFrameSimilarity > 0.9 && sceneComparison.postSceneChangeFrameSimilarity > 0.9 && (await sceneComparison).deltaOfDeltas < 0.1)
     ) {
       // matching scene found
-      console.info(`Found matching scene after ${ms(Date.now() - startTime)}`);
 
       // remove tmp folder
       await fs.rm(`tmp`, {
@@ -289,7 +288,7 @@ async function calculateOffset(video1, video2, options) {
         videoOffset: video1SceneChange.preSceneChangeFrame.offset - sceneComparison.video2SceneChange.preSceneChangeFrame.offset,
         confidence: 1,
       }
-      spinner.succeed(`Source video is approx. ${Math.abs(result.videoOffset)} ms ${result.videoOffset > 0 ? `ahead` : `behind`} destination video (confidence ${result.confidence.toFixed(5)}).`)
+      spinner.succeed(`Source video is approx. ${Math.abs(result.videoOffset)} ms ${result.videoOffset > 0 ? `ahead` : `behind`} destination video. Took ${ms(Date.now() - startTime)}`)
       return result
 
     } else {

util/warping.js

@@ -4,11 +4,18 @@ const { performance } = require(`perf_hooks`)
 const bmp = require(`bmp-js`)
 const ssim = require(`ssim.js`).default
 const ora = require('ora');
+const resizeImg = require('resize-img')
 const { getVideoInfo } = require("./calc-offset")
 
 async function findClosestFrame(destinationVideo, sourceVideo, destinationTimestamp, offset, radius, stepSize) {
 
-  let framesDir = await fs.mkdtemp(`tmp/frames`)
+  // create the tmp folder if it doesn't exist yet
+  try {
+    await fs.access(`tmp`)
+  } catch (err) {
+    await fs.mkdir(`tmp`)
+  }
+  const framesDir = await fs.mkdtemp(`tmp/frames`)
 
   let seekPosition = destinationTimestamp / 1000.0
   let destinationFrame
@@ -41,12 +48,33 @@ async function findClosestFrame(destinationVideo, sourceVideo, destinationTimest
   }
 
   // extract frame, check similarity, delete it
-  for (const currentOffset = offset-radius; currentOffset <= offset+radius; currentOffset += stepSize) {
+  for (let currentOffset = offset-radius; currentOffset <= offset+radius; currentOffset += stepSize) {
 
     seekPosition = (destinationTimestamp + currentOffset) / 1000.0
     fullOutputPath = `${framesDir}/screenshot_${performance.now()*10000000000000}.bmp`  
 
-    const frameData = await extractFrame(sourceVideo, seekPosition, cropValueSource, fullOutputPath)
+    let frameData = await extractFrame(sourceVideo, seekPosition, cropValueSource, fullOutputPath)
+
+    //TODO instead of only checking image dimensions at the end, do it at the start and then generate the frames from the smaller video
+    //!!! make sure to use the negated offset when swapping source and destination! 
+    if (destinationFrame.width !== frameData.width || destinationFrame.height !== frameData.height) {
+      // make sure to always downsize
+      if (destinationFrame.width*destinationFrame.height >= frameData.width*frameData.height) {
+        // resize destination frame once
+        destinationFrame.data = bmp.decode(await resizeImg(await fs.readFile(destinationFrame.path), {
+          format: `bmp`,
+          width: frameData.width,
+          height: frameData.height,
+        }));
+      } else {
+        // resize every source frame
+        frameData = bmp.decode(await resizeImg(await fs.readFile(fullOutputPath), {
+          format: `bmp`,
+          width: destinationFrame.data.width,
+          height: destinationFrame.data.height,
+        }));
+      }
+    }
 
     similarity = ssim(frameData, destinationFrame.data).mssim;
 
@@ -59,7 +87,11 @@ async function findClosestFrame(destinationVideo, sourceVideo, destinationTimest
 
   }
 
-  await fs.unlink(framesDir)
+  // remove tmp folder
+  await fs.rm(`tmp`, {
+    recursive: true,
+    force: true,
+  })
 
   return mostSimilarFrame
 
@@ -100,12 +132,14 @@ async function validateOffset(destinationVideo, sourceVideo, offsetToTest) {
 
   }
 
-  const offsetDelta = Math.max(mostSimilarFrameOffsets) - Math.min(mostSimilarFrameOffsets)
+  console.debug(`mostSimilarFrameOffsets:`, mostSimilarFrameOffsets)
 
+  const offsetDelta = Math.abs(Math.max(...mostSimilarFrameOffsets) - Math.min(...mostSimilarFrameOffsets))
   if (offsetDelta > 250) {
     return false
   }
 
   return true
 
 }
+module.exports.validateOffset = validateOffset