[O2B-1325] GAQ Summary API

lib/database/repositories/QcFlagRepository.js

@@ -15,6 +15,39 @@ const { Op } = require('sequelize');
 const { models: { QcFlag } } = require('..');
 const Repository = require('./Repository');
 
+const GAQ_PERIODS_VIEW = `
+        SELECT
+            data_pass_id,
+            run_number,
+            timestamp AS \`from\`,
+            NTH_VALUE(timestamp, 2) OVER (
+                PARTITION BY data_pass_id,
+                run_number
+                ORDER BY ap.timestamp
+                ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING
+            ) AS \`to\`
+        FROM (
+                (
+                    SELECT data_pass_id,
+                        run_number,
+                        COALESCE(UNIX_TIMESTAMP(qcfep.\`from\`), 0) AS timestamp
+                    FROM quality_control_flag_effective_periods AS qcfep
+                        INNER JOIN quality_control_flags AS qcf ON qcf.id = qcfep.flag_id
+                        INNER JOIN data_pass_quality_control_flag AS dpqcf ON dpqcf.quality_control_flag_id = qcf.id
+                )
+                UNION
+                (
+                    SELECT data_pass_id,
+                        run_number,
+                        UNIX_TIMESTAMP(COALESCE(qcfep.\`to\`, NOW())) AS timestamp
+                    FROM quality_control_flag_effective_periods AS qcfep
+                        INNER JOIN quality_control_flags AS qcf ON qcf.id = qcfep.flag_id
+                        INNER JOIN data_pass_quality_control_flag AS dpqcf ON dpqcf.quality_control_flag_id = qcf.id
+                )
+                ORDER BY timestamp
+            ) AS ap
+    `;
+
 /**
  * @typedef GaqPeriod
  *
@@ -25,6 +58,17 @@ const Repository = require('./Repository');
  * @property {number[]} contributingFlagIds IDs of QC flags which together define global aggregated quality in the period [from, to]
  */
 
+/**
+ * @typedef RunGaqSubSummary aggregation of QC flags information by QcFlagType property `bad`
+ *
+ * @property {number} runNumber
+ * @property {number} bad
+ * @property {number} effectiveRunCoverage
+ * @property {number[]} flagsIds
+ * @property {number[]} verifiedFlagsIds
+ * @property {number} mcReproducible
+ */
+
 /**
  * Sequelize implementation of the QcFlagRepository
  */
@@ -44,39 +88,6 @@ class QcFlagRepository extends Repository {
      * @return {Promise<GaqPeriod[]>} Resolves with the GAQ periods
      */
     async findGaqPeriods(dataPassId, runNumber) {
-        const gaqPeriodsSubQuery = `
-                SELECT 
-                    data_pass_id,
-                    run_number,
-                    timestamp AS \`from\`,
-                    NTH_VALUE(timestamp, 2) OVER (
-                        PARTITION BY data_pass_id,
-                        run_number
-                        ORDER BY ap.timestamp
-                        ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING
-                    ) AS \`to\`
-                FROM (
-                        (
-                            SELECT data_pass_id,
-                                run_number,
-                                COALESCE(UNIX_TIMESTAMP(qcfep.\`from\`), 0) AS timestamp
-                            FROM quality_control_flag_effective_periods AS qcfep
-                                INNER JOIN quality_control_flags AS qcf ON qcf.id = qcfep.flag_id
-                                INNER JOIN data_pass_quality_control_flag AS dpqcf ON dpqcf.quality_control_flag_id = qcf.id
-                        )
-                        UNION
-                        (
-                            SELECT data_pass_id,
-                                run_number,
-                                UNIX_TIMESTAMP(COALESCE(qcfep.\`to\`, NOW())) AS timestamp
-                            FROM quality_control_flag_effective_periods AS qcfep
-                                INNER JOIN quality_control_flags AS qcf ON qcf.id = qcfep.flag_id
-                                INNER JOIN data_pass_quality_control_flag AS dpqcf ON dpqcf.quality_control_flag_id = qcf.id
-                        )
-                        ORDER BY timestamp
-                    ) AS ap
-            `;
-
         const query = `
             SELECT
                 gaq_periods.data_pass_id AS dataPassId,
@@ -89,7 +100,7 @@ class QcFlagRepository extends Repository {
             INNER JOIN quality_control_flag_effective_periods AS qcfep
                 ON qcf.id = qcfep.flag_id
             INNER JOIN data_pass_quality_control_flag AS dpqcf ON dpqcf.quality_control_flag_id = qcf.id
-            INNER JOIN (${gaqPeriodsSubQuery}) AS gaq_periods ON gaq_periods.data_pass_id = dpqcf.data_pass_id
+            INNER JOIN (${GAQ_PERIODS_VIEW}) AS gaq_periods ON gaq_periods.data_pass_id = dpqcf.data_pass_id
             INNER JOIN global_aggregated_quality_detectors AS gaqd
                 ON gaqd.data_pass_id = gaq_periods.data_pass_id
                     AND gaqd.run_number = gaq_periods.run_number
@@ -123,6 +134,118 @@ class QcFlagRepository extends Repository {
         }));
     }
 
+    /**
+     * Get GAQ sub-summaries for given data pass
+     *
+     * @param {number} dataPassId id of data pass id
+     * @return {Promise<RunGaqSubSummary[]>} Resolves with the GAQ sub-summaries
+     */
+    async getRunGaqSubSummaries(dataPassId) {
+        const effectivePeriodsWithTypeSubQuery = `
+            SELECT
+                gaq_periods.data_pass_id AS dataPassId,
+                gaq_periods.run_number AS runNumber,
+                IF(gaq_periods.\`from\` = 0, null, gaq_periods.\`from\`) AS \`from\`,
+                IF(gaq_periods.\`to\` = UNIX_TIMESTAMP(NOW()), null, gaq_periods.\`to\`) AS \`to\`,
+                SUM(qcft.bad) >= 1 AS bad,
+                SUM(qcft.bad) = SUM(qcft.monte_carlo_reproducible) AND SUM(qcft.monte_carlo_reproducible) AS mcReproducible,
+                GROUP_CONCAT( DISTINCT qcfv.flag_id ) AS verifiedFlagsList,
+                GROUP_CONCAT( DISTINCT qcf.id ) AS flagsList
+
+            FROM quality_control_flags AS qcf
+            INNER JOIN quality_control_flag_types AS qcft
+                ON qcft.id = qcf.flag_type_id
+            LEFT JOIN quality_control_flag_verifications AS qcfv
+                ON qcfv.flag_id = qcf.id
+            INNER JOIN quality_control_flag_effective_periods AS qcfep
+                ON qcf.id = qcfep.flag_id
+            INNER JOIN data_pass_quality_control_flag AS dpqcf
+                ON dpqcf.quality_control_flag_id = qcf.id
+            INNER JOIN (${GAQ_PERIODS_VIEW}) AS gaq_periods
+                ON gaq_periods.data_pass_id = dpqcf.data_pass_id
+            INNER JOIN global_aggregated_quality_detectors AS gaqd
+                ON gaqd.data_pass_id = gaq_periods.data_pass_id
+                    AND gaqd.run_number = gaq_periods.run_number
+                    AND gaqd.detector_id = qcf.detector_id
+                    AND gaq_periods.run_number = qcf.run_number
+                        AND (qcfep.\`from\` IS NULL OR UNIX_TIMESTAMP(qcfep.\`from\`) <= gaq_periods.\`from\`)
+                        AND (qcfep.\`to\`   IS NULL OR gaq_periods.\`to\` <= UNIX_TIMESTAMP(qcfep.\`to\`))
+
+            GROUP BY
+                gaq_periods.data_pass_id,
+                gaq_periods.run_number,
+                gaq_periods.\`from\`,
+                gaq_periods.\`to\`
+            `;
+
+        const query = `
+            SELECT
+                effectivePeriods.runNumber,
+                effectivePeriods.dataPassId,
+                effectivePeriods.bad,
+                SUM(effectivePeriods.mcReproducible) > 0 AS mcReproducible,
+                GROUP_CONCAT(effectivePeriods.verifiedFlagsList) AS verifiedFlagsList,
+                GROUP_CONCAT(effectivePeriods.flagsList) AS flagsList,
+
+                IF(
+                    (
+                        COALESCE(run.time_trg_end,   run.time_o2_end  ) IS NULL
+                     OR COALESCE(run.time_trg_start, run.time_o2_start) IS NULL
+                    ),
+                    IF(
+                        SUM(
+                            COALESCE(effectivePeriods.\`to\`  , 0)
+                          + COALESCE(effectivePeriods.\`from\`, 0)
+                        ) = 0,
+                        1,
+                        null
+                    ),
+                    SUM(
+                        COALESCE(
+                            effectivePeriods.\`to\`,
+                            UNIX_TIMESTAMP(run.time_trg_end),
+                            UNIX_TIMESTAMP(run.time_o2_end)
+                        )
+                      - COALESCE(
+                            effectivePeriods.\`from\`,
+                            UNIX_TIMESTAMP(run.time_trg_start),
+                            UNIX_TIMESTAMP(run.time_o2_start)
+                        )
+                    ) / (
+                        UNIX_TIMESTAMP(COALESCE(run.time_trg_end,   run.time_o2_end))
+                      - UNIX_TIMESTAMP(COALESCE(run.time_trg_start, run.time_o2_start))
+                    )
+                ) AS effectiveRunCoverage
+
+            FROM (${effectivePeriodsWithTypeSubQuery}) AS effectivePeriods
+            INNER JOIN runs AS run ON run.run_number = effectivePeriods.runNumber
+
+            WHERE effectivePeriods.dataPassId = :dataPassId
+
+            GROUP BY
+                effectivePeriods.dataPassId,
+                effectivePeriods.runNumber,
+                effectivePeriods.bad
+        `;
+
+        const [rows] = await this.model.sequelize.query(query, { replacements: { dataPassId } });
+        return rows.map(({
+            runNumber,
+            bad,
+            effectiveRunCoverage,
+            mcReproducible,
+            flagsList,
+            verifiedFlagsList,
+        }) => ({
+            runNumber,
+            bad,
+            effectiveRunCoverage,
+            mcReproducible: Boolean(mcReproducible),
+            flagsIds: [...new Set(flagsList.split(','))],
+            verifiedFlagsIds: verifiedFlagsList ? [...new Set(verifiedFlagsList.split(','))] : [],
+        }));
+    }
+
     /**
      * Find all QC flags created before and after given one for the same run, detector, data/simulation pass.
      * Flags are orted by createdAt property in ascedning manner

lib/database/seeders/20240716124000-gaq-detectors.js

@@ -21,6 +21,15 @@ module.exports = {
                 updated_at: '2024-07-16 12:45:00',
             },
 
+            {
+                run_number: 54,
+                data_pass_id: 3,
+                detector_id: 4,
+
+                created_at: '2024-07-16 12:45:00',
+                updated_at: '2024-07-16 12:45:00',
+            },
+
             {
                 run_number: 106,
                 data_pass_id: 1,

lib/server/controllers/qcFlag.controller.js

@@ -273,6 +273,28 @@
     }
 };
 
+// eslint-disable-next-line valid-jsdoc
+/**
+ * Get GAQ summary for given data pass
+ */
+const getGaqSummaryHandler = async (request, response) => {
+    const validatedDTO = await dtoValidator(
+        DtoFactory.queryOnly(Joi.object({ dataPassId: Joi.number().required() })),
+        request,
+        response,
+    );
+    if (validatedDTO) {
+        try {
+            const { dataPassId } = validatedDTO.query;
+
+            const data = await qcFlagService.getGaqSummary(dataPassId);
+            response.json({ data });
+        } catch (error) {
+            updateExpressResponseFromNativeError(response, error);
+        }
+    }
+};
+
 exports.QcFlagController = {
     getQcFlagByIdHandler,
     listQcFlagsPerDataPassHandler,
@@ -282,4 +304,5 @@
     verifyQcFlagHandler,
     getQcFlagsSummaryHandler,
     getGaqQcFlagsHandler,
+    getGaqSummaryHandler,
 };

lib/server/routers/qcFlag.router.js

@@ -27,6 +27,14 @@ exports.qcFlagsRouter = {
             path: 'summary',
             method: 'get',
             controller: QcFlagController.getQcFlagsSummaryHandler,
+
+            children: [
+                {
+                    path: 'gaq',
+                    method: 'get',
+                    controller: QcFlagController.getGaqSummaryHandler,
+                },
+            ],
         },
         {
             path: 'perDataPass',

lib/server/services/qualityControlFlag/QcFlagService.js

@@ -93,6 +93,19 @@ const QC_SUMMARY_PROPERTIES = {
  * @property {QcFlag[]} contributingFlags
  */
 
+/**
+ * @typedef RunGaqSummary
+ * @property {number} badEffectiveRunCoverage - fraction of run's data, which aggregated quality is bad
+ * @property {number} explicitlyNotBadEffectiveRunCoverage - fraction of run's data, which aggregated quality is explicitly good
+ * @property {number} missingVerificationsCount - number of not verified QC flags which are not discarded
+ * @property {boolean} mcReproducible - states whether some aggregation of QC flags is Limited Acceptance MC Reproducible
+ */
+
+/**
+ * @typedef GaqSummary aggregated global quality summaries for given data pass
+ * @type {Object<number, RunGaqSummary>} runNumber to RunGaqSummary mapping
+ */
+
 /**
  * Quality control flags service
  */
@@ -586,6 +599,68 @@ class QcFlagService {
         }));
     }
 
+    /**
+     * Get GAQ summary
+     *
+     * @param {number} dataPassId id of data pass id
+     * @return {Promise<GaqSummary[]>} Resolves with the GAQ Summary
+     */
+    async getGaqSummary(dataPassId) {
+        await getOneDataPassOrFail({ id: dataPassId });
+        const runGaqSubSummaries = await QcFlagRepository.getRunGaqSubSummaries(dataPassId);
+
+        const summary = {};
+        const flagsAndVerifications = {};
+
+        // Fold list of subSummaries into one summary
+        for (const subSummary of runGaqSubSummaries) {
+            const {
+                runNumber,
+                bad,
+                effectiveRunCoverage,
+                flagsIds,
+                verifiedFlagsIds,
+                mcReproducible,
+            } = subSummary;
+
+            if (!summary[runNumber]) {
+                summary[runNumber] = { [QC_SUMMARY_PROPERTIES.mcReproducible]: false };
+            }
+            if (!flagsAndVerifications[runNumber]) {
+                flagsAndVerifications[runNumber] = {};
+            }
+
+            const runSummary = summary[runNumber];
+
+            const distinctRunFlagsIds = flagsAndVerifications[runNumber]?.distinctFlagsIds ?? [];
+            const distinctRunVerifiedFlagsIds = flagsAndVerifications[runNumber]?.distinctVerifiedFlagsIds ?? [];
+
+            flagsAndVerifications[runNumber] = {
+                distinctFlagsIds: new Set([...distinctRunFlagsIds, ...flagsIds]),
+                distinctVerifiedFlagsIds: new Set([...distinctRunVerifiedFlagsIds, ...verifiedFlagsIds]),
+            };
+
+            if (bad) {
+                runSummary[QC_SUMMARY_PROPERTIES.badEffectiveRunCoverage] = parseFloat(effectiveRunCoverage);
+                runSummary[QC_SUMMARY_PROPERTIES.mcReproducible] = mcReproducible;
+            } else {
+                runSummary[QC_SUMMARY_PROPERTIES.explicitlyNotBadEffectiveRunCoverage] = parseFloat(effectiveRunCoverage);
+            }
+            if (runSummary[QC_SUMMARY_PROPERTIES.badEffectiveRunCoverage] === undefined) {
+                runSummary[QC_SUMMARY_PROPERTIES.badEffectiveRunCoverage] = 0;
+            }
+            if (runSummary[QC_SUMMARY_PROPERTIES.explicitlyNotBadEffectiveRunCoverage] === undefined) {
+                runSummary[QC_SUMMARY_PROPERTIES.explicitlyNotBadEffectiveRunCoverage] = 0;
+            }
+        }
+
+        for (const [runNumber, { distinctFlagsIds, distinctVerifiedFlagsIds }] of Object.entries(flagsAndVerifications)) {
+            summary[runNumber][QC_SUMMARY_PROPERTIES.missingVerificationsCount] = distinctFlagsIds.size - distinctVerifiedFlagsIds.size;
+        }
+
+        return summary;
+    }
+
     /**
      * Return a paginated list of QC flags related to a given simulation pass, run and dpl detector
      *