Add support for libvmaf v2.3.1 output (#9)

* Add support for libvmaf v2.3.1 output

With version v2.3.1 of libvmaf there have been introduced changes to
field names in libvmaf generated output (per frame metrics anf pooled
metrics). With this change set we make sure that JSON unmarshaling works
as expected for changed field names.

* Change back static ffmpeg url to upstream

.github/workflows/ci.yaml

@@ -52,14 +52,14 @@ jobs:
         id: cache-ffmpeg
         with:
           path: ${{ env.FFMPEG_DEST_DIR }}
-          key: ${{ runner.os }}-static-ffmpeg-v5.0.1
+          key: ${{ runner.os }}-static-ffmpeg-v5.1.1
 
       # Only run this expensive step if we have a cache miss
       - if: steps.cache-ffmpeg.outputs.cache-hit != 'true'
         name: Install static ffmpeg
         run: |
-          curl --connect-timeout 10 -Lv -o ffmpeg.tar.xz https://web.archive.org/web/20220502225332/https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz
-          echo "07149022696e10e528f48f402c3b8570  ffmpeg.tar.xz" | md5sum --check -
+          curl --connect-timeout 10 -Lv -o ffmpeg.tar.xz https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz
+          echo "4cbbe32169c4ec79a0969d5c92cbcaff  ffmpeg.tar.xz" | md5sum --check -
           mkdir -p "$FFMPEG_DEST_DIR"
           tar -C "$FFMPEG_DEST_DIR" -xf ffmpeg.tar.xz --strip-components=1
 

internal/vqm/frame_test.go

@@ -15,7 +15,7 @@ import (
 )
 
 var (
-	metricsFile = "../../testdata/vqm/ffmpeg_vmaf.json"
+	metricsFile = "../../testdata/vqm/libvmaf_v2.3.1.json"
 	// Expected count of metrics from metricsFile.
 	wantMetricCount       = 10
 	frameMetricsFile      = "../../testdata/vqm/frame_metrics.json"

internal/vqm/vqm.go

@@ -189,15 +189,75 @@ type frame struct {
 }
 
 type metric struct {
-	VMAF    float64 `json:"vmaf"`
-	PSNR    float64 `json:"psnr"`
-	MS_SSIM float64 `json:"ms_ssim"`
+	VMAF    float64
+	PSNR    float64
+	MS_SSIM float64
+}
+
+// UnmarshalJSON implements json.Unmarshaler interface for metric.
+//
+// A custom unmarshaler is needed to work around lack of stability around libvmaf measured
+// VQ metric field names in output.
+func (m *metric) UnmarshalJSON(b []byte) error {
+	// Ignore "null" as per convention.
+	if string(b) == "null" {
+		return nil
+	}
+
+	// Unmarshal JSON blob into map so that it includes all fields.
+	raw := make(map[string]float64)
+	if err := json.Unmarshal(b, &raw); err != nil {
+		return err
+	}
+
+	// Pull out required fields and in some cases their aliases (due to libvmaf changing
+	// field names at will)
+	for k, v := range raw {
+		switch k {
+		case "vmaf":
+			m.VMAF = v
+		case "psnr", "psnr_y":
+			m.PSNR = v
+		case "ms_ssim", "float_ms_ssim":
+			m.MS_SSIM = v
+		}
+	}
+
+	return nil
 }
 
 type pooledMetrics struct {
-	VMAF    pMetric `json:"vmaf"`
-	PSNR    pMetric `json:"psnr"`
-	MS_SSIM pMetric `json:"ms_ssim"`
+	VMAF    pMetric
+	PSNR    pMetric
+	MS_SSIM pMetric
+}
+
+// UnmarshalJSON implements json.Unmarshaler interface for pooledMetrics.
+//
+// A custom unmarshaler is needed to work around lack of stability around libvmaf measured
+// VQ metric field names in output.
+func (p *pooledMetrics) UnmarshalJSON(b []byte) error {
+	// Ignore "null" as per convention.
+	if string(b) == "null" {
+		return nil
+	}
+
+	raw := make(map[string]pMetric)
+	if err := json.Unmarshal(b, &raw); err != nil {
+		return err
+	}
+
+	for k, v := range raw {
+		switch k {
+		case "vmaf":
+			p.VMAF = v
+		case "psnr", "psnr_y":
+			p.PSNR = v
+		case "ms_ssim", "float_ms_ssim":
+			p.MS_SSIM = v
+		}
+	}
+	return nil
 }
 
 type pMetric struct {

internal/vqm/vqm_test.go

@@ -5,6 +5,8 @@
 package vqm
 
 import (
+	"encoding/json"
+	"os"
 	"testing"
 
 	"github.com/evolution-gaming/ease/internal/tools"
@@ -126,3 +128,79 @@ func TestFfmpegVMAF_Negative(t *testing.T) {
 		}
 	})
 }
+
+// Different libvmaf versions will generate slightly different outputs. Have to support
+// and test accordingly.
+func Test_ffmpegVMAFResult_UnmarshalVersions(t *testing.T) {
+	tests := map[string]struct {
+		resultFile string
+	}{
+		"libvmaf v2.3.0": {
+			resultFile: "../../testdata/vqm/libvmaf_v2.3.0.json",
+		},
+		"libvmaf v2.3.1": {
+			resultFile: "../../testdata/vqm/libvmaf_v2.3.1.json",
+		},
+	}
+
+	for name, tt := range tests {
+		t.Run(name, func(t *testing.T) {
+			jsonDoc, err := os.ReadFile(tt.resultFile)
+			if err != nil {
+				t.Fatalf("Unexpected error reading %s: %s", tt.resultFile, err)
+			}
+
+			res := &ffmpegVMAFResult{}
+			if err := json.Unmarshal(jsonDoc, res); err != nil {
+				t.Fatalf("Unexpected error from unmarshaling: %s", err)
+			}
+
+			// Check that per-frame VQM values were properly unmarshalled (should not be 0).
+			for i, v := range res.Frames {
+				if v.Metrics.MS_SSIM == 0 || v.Metrics.PSNR == 0 || v.Metrics.VMAF == 0 {
+					t.Errorf("Unexpected metric value for frame %v", i)
+				}
+			}
+
+			// Check that pooled metric values were properly unmarshalled (should not be 0).
+			if res.PooledMetrics.MS_SSIM.Min == 0 {
+				t.Error("Unexpected value 0 for pooled metric MS_SSIM.Min")
+			}
+			if res.PooledMetrics.MS_SSIM.Max == 0 {
+				t.Error("Unexpected value 0 for pooled metric MS_SSIM.Max")
+			}
+			if res.PooledMetrics.MS_SSIM.Mean == 0 {
+				t.Error("Unexpected value 0 for pooled metric MS_SSIM.Mean")
+			}
+			if res.PooledMetrics.MS_SSIM.HarmonicMean == 0 {
+				t.Error("Unexpected value 0 for pooled metric MS_SSIM.HarmonicMean")
+			}
+
+			if res.PooledMetrics.PSNR.Min == 0 {
+				t.Error("Unexpected value 0 for pooled metric PSNR.Min")
+			}
+			if res.PooledMetrics.PSNR.Max == 0 {
+				t.Error("Unexpected value 0 for pooled metric PSNR.Max")
+			}
+			if res.PooledMetrics.PSNR.Mean == 0 {
+				t.Error("Unexpected value 0 for pooled metric PSNR.Mean")
+			}
+			if res.PooledMetrics.PSNR.HarmonicMean == 0 {
+				t.Error("Unexpected value 0 for pooled metric PSNR.HarmonicMean")
+			}
+
+			if res.PooledMetrics.VMAF.Min == 0 {
+				t.Error("Unexpected value 0 for pooled metric VMAF.Min")
+			}
+			if res.PooledMetrics.VMAF.Max == 0 {
+				t.Error("Unexpected value 0 for pooled metric VMAF.Max")
+			}
+			if res.PooledMetrics.VMAF.Mean == 0 {
+				t.Error("Unexpected value 0 for pooled metric VMAF.Mean")
+			}
+			if res.PooledMetrics.VMAF.HarmonicMean == 0 {
+				t.Error("Unexpected value 0 for pooled metric VMAF.HarmonicMean")
+			}
+		})
+	}
+}