From 03ca36e69e72db50a09db9c2ee213a2137c7a897 Mon Sep 17 00:00:00 2001 From: xsalonx Date: Tue, 3 Dec 2024 12:26:54 +0100 Subject: [PATCH 01/16] use tf_timestamps --- .../20241127123000-create-gaq-views.js | 137 ++++++++++++++++++ 1 file changed, 137 insertions(+) create mode 100644 lib/database/migrations/20241127123000-create-gaq-views.js diff --git a/lib/database/migrations/20241127123000-create-gaq-views.js b/lib/database/migrations/20241127123000-create-gaq-views.js new file mode 100644 index 0000000000..29197f84e0 --- /dev/null +++ b/lib/database/migrations/20241127123000-create-gaq-views.js @@ -0,0 +1,137 @@ +'use strict'; + +const GAQ_PERIODS_TIMESTAMPS_VIEW_NAME = 'gaq_periods_timestamps'; +const CREATE_GAQ_PERIODS_TIMESTAMPS_VIEW = ` +CREATE OR REPLACE VIEW ${GAQ_PERIODS_TIMESTAMPS_VIEW_NAME} AS + SELECT + data_pass_id, + run_number, + timestamp AS \`from\`, + NTH_VALUE(timestamp, 2) OVER ( + PARTITION BY data_pass_id, + run_number + ORDER BY ap.timestamp + ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING + ) AS \`to\` + FROM ( + -- Two selects for runs' timestamps (in case QC flag's eff. period doesn't start at run's start or end at run's end ) + ( + SELECT gaqd.data_pass_id, + gaqd.run_number, + COALESCE(UNIX_TIMESTAMP(first_tf_timestamp), UNIX_TIMESTAMP(time_start), 0) AS timestamp + FROM global_aggregated_quality_detectors AS gaqd + INNER JOIN runs as r + ON gaqd.run_number = r.run_number + ) + UNION + ( + SELECT gaqd.data_pass_id, + gaqd.run_number, + UNIX_TIMESTAMP(COALESCE(last_tf_timestamp, time_end, NOW())) AS timestamp + FROM global_aggregated_quality_detectors AS gaqd + INNER JOIN runs as r + ON gaqd.run_number = r.run_number + ) + UNION + -- Two selectes for timestamps of QC flags' effective periods + ( + SELECT gaqd.data_pass_id, + gaqd.run_number, + COALESCE(UNIX_TIMESTAMP(qcfep.\`from\`), 0) AS timestamp + FROM quality_control_flag_effective_periods AS qcfep + INNER JOIN quality_control_flags AS qcf ON qcf.id = qcfep.flag_id + INNER JOIN data_pass_quality_control_flag AS dpqcf ON dpqcf.quality_control_flag_id = qcf.id + -- Only flags of detectors which are defined in global_aggregated_quality_detectors + -- should be taken into account for calculation of gaq_effective_periods + INNER JOIN global_aggregated_quality_detectors AS gaqd + ON gaqd.data_pass_id = dpqcf.data_pass_id + AND gaqd.run_number = qcf.run_number + AND gaqd.detector_id = qcf.detector_id + ) + UNION + ( + SELECT gaqd.data_pass_id, + gaqd.run_number, + UNIX_TIMESTAMP(COALESCE(qcfep.\`to\`, NOW())) AS timestamp + FROM quality_control_flag_effective_periods AS qcfep + INNER JOIN quality_control_flags AS qcf ON qcf.id = qcfep.flag_id + INNER JOIN data_pass_quality_control_flag AS dpqcf ON dpqcf.quality_control_flag_id = qcf.id + -- Only flags of detectors which are defined in global_aggregated_quality_detectors + -- should be taken into account for calculation of gaq_effective_periods + INNER JOIN global_aggregated_quality_detectors AS gaqd + ON gaqd.data_pass_id = dpqcf.data_pass_id + AND gaqd.run_number = qcf.run_number + AND gaqd.detector_id = qcf.detector_id + ) + ORDER BY timestamp + ) AS ap + `; + +const DROP_GAQ_PERIODS_TIMESTAMPS_VIEW = `DROP VIEW ${GAQ_PERIODS_TIMESTAMPS_VIEW_NAME}`; + +const GAQ_PERIODS_VIEW_NAME = 'gaq_periods'; +const CREATE_GAQ_PERIODS_VIEW = ` +CREATE OR REPLACE VIEW ${GAQ_PERIODS_VIEW_NAME} AS +SELECT + gaq_periods_timestamps.data_pass_id AS dataPassId, + gaq_periods_timestamps.run_number AS runNumber, + IF(gaq_periods_timestamps.\`from\` = 0, null, gaq_periods_timestamps.\`from\`) AS \`from\`, + IF(gaq_periods_timestamps.\`to\` = UNIX_TIMESTAMP(NOW()), null, gaq_periods_timestamps.\`to\`) AS \`to\`, + IF(COUNT( DISTINCT gaqd.detector_id ) > COUNT( DISTINCT qcfep.flag_id ), + null, + SUM(qcft.bad) >= 1 + ) AS bad, + IF(COUNT( DISTINCT gaqd.detector_id ) > COUNT( DISTINCT qcfep.flag_id ), + null, + SUM(IF(qcft.monte_carlo_reproducible, false, qcft.bad)) >= 1 + ) AS badWhenMcReproducibleAsNotBad, + SUM(qcft.bad) = SUM(qcft.monte_carlo_reproducible) AND SUM(qcft.monte_carlo_reproducible) AS mcReproducible, + GROUP_CONCAT( DISTINCT qcfv.flag_id ) AS verifiedFlagsList, + GROUP_CONCAT( DISTINCT qcfep.flag_id ) AS flagsList + +FROM ${GAQ_PERIODS_TIMESTAMPS_VIEW_NAME} AS gaq_periods_timestamps +INNER JOIN global_aggregated_quality_detectors AS gaqd + ON gaqd.data_pass_id = gaq_periods_timestamps.data_pass_id + AND gaqd.run_number = gaq_periods_timestamps.run_number + +LEFT JOIN ( + data_pass_quality_control_flag AS dpqcf + INNER JOIN quality_control_flags AS qcf + ON dpqcf.quality_control_flag_id = qcf.id + INNER JOIN quality_control_flag_types AS qcft + ON qcft.id = qcf.flag_type_id + INNER JOIN quality_control_flag_effective_periods AS qcfep + ON qcf.id = qcfep.flag_id + LEFT JOIN quality_control_flag_verifications AS qcfv + ON qcfv.flag_id = qcf.id +) + ON gaq_periods_timestamps.data_pass_id = dpqcf.data_pass_id + AND qcf.run_number = gaq_periods_timestamps.run_number + AND gaqd.detector_id = qcf.detector_id + AND gaq_periods_timestamps.run_number = qcf.run_number + AND (qcfep.\`from\` IS NULL OR UNIX_TIMESTAMP(qcfep.\`from\`) <= gaq_periods_timestamps.\`from\`) + AND (qcfep.\`to\` IS NULL OR gaq_periods_timestamps.\`to\` <= UNIX_TIMESTAMP(qcfep.\`to\`)) + +WHERE gaq_periods_timestamps.\`to\` IS NOT null + +GROUP BY + gaq_periods_timestamps.data_pass_id, + gaq_periods_timestamps.run_number, + IF(gaq_periods_timestamps.\`from\` = 0, null, gaq_periods_timestamps.\`from\`), + IF(gaq_periods_timestamps.\`to\` = UNIX_TIMESTAMP(NOW()), null, gaq_periods_timestamps.\`to\`) +`; + +const DROP_GAQ_PERIODS_VIEW = `DROP VIEW ${GAQ_PERIODS_VIEW_NAME}`; + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + up: async (queryInterface) => queryInterface.sequelize.transaction(async (transaction) => { + await queryInterface.sequelize.query(CREATE_GAQ_PERIODS_TIMESTAMPS_VIEW, { transaction }); + await queryInterface.sequelize.query(CREATE_GAQ_PERIODS_VIEW, { transaction }); + }), + + down: async (queryInterface) => queryInterface.sequelize.transaction(async (transaction) => { + await queryInterface.sequelize.query(DROP_GAQ_PERIODS_VIEW, { transaction }); + await queryInterface.sequelize.query(DROP_GAQ_PERIODS_TIMESTAMPS_VIEW, { transaction }); + }), +}; From 24360410608ddaaa6a922c0fffe0e52a193e5cbb Mon Sep 17 00:00:00 2001 From: xsalonx Date: Tue, 3 Dec 2024 12:39:18 +0100 Subject: [PATCH 02/16] use miliseconds --- .../migrations/20241127123000-create-gaq-views.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/database/migrations/20241127123000-create-gaq-views.js b/lib/database/migrations/20241127123000-create-gaq-views.js index 29197f84e0..d9f3ac0bd0 100644 --- a/lib/database/migrations/20241127123000-create-gaq-views.js +++ b/lib/database/migrations/20241127123000-create-gaq-views.js @@ -27,7 +27,7 @@ CREATE OR REPLACE VIEW ${GAQ_PERIODS_TIMESTAMPS_VIEW_NAME} AS ( SELECT gaqd.data_pass_id, gaqd.run_number, - UNIX_TIMESTAMP(COALESCE(last_tf_timestamp, time_end, NOW())) AS timestamp + UNIX_TIMESTAMP(COALESCE(last_tf_timestamp, time_end, NOW(3))) AS timestamp FROM global_aggregated_quality_detectors AS gaqd INNER JOIN runs as r ON gaqd.run_number = r.run_number @@ -52,7 +52,7 @@ CREATE OR REPLACE VIEW ${GAQ_PERIODS_TIMESTAMPS_VIEW_NAME} AS ( SELECT gaqd.data_pass_id, gaqd.run_number, - UNIX_TIMESTAMP(COALESCE(qcfep.\`to\`, NOW())) AS timestamp + UNIX_TIMESTAMP(COALESCE(qcfep.\`to\`, NOW(3))) AS timestamp FROM quality_control_flag_effective_periods AS qcfep INNER JOIN quality_control_flags AS qcf ON qcf.id = qcfep.flag_id INNER JOIN data_pass_quality_control_flag AS dpqcf ON dpqcf.quality_control_flag_id = qcf.id @@ -76,7 +76,7 @@ SELECT gaq_periods_timestamps.data_pass_id AS dataPassId, gaq_periods_timestamps.run_number AS runNumber, IF(gaq_periods_timestamps.\`from\` = 0, null, gaq_periods_timestamps.\`from\`) AS \`from\`, - IF(gaq_periods_timestamps.\`to\` = UNIX_TIMESTAMP(NOW()), null, gaq_periods_timestamps.\`to\`) AS \`to\`, + IF(gaq_periods_timestamps.\`to\` = UNIX_TIMESTAMP(NOW(3)), null, gaq_periods_timestamps.\`to\`) AS \`to\`, IF(COUNT( DISTINCT gaqd.detector_id ) > COUNT( DISTINCT qcfep.flag_id ), null, SUM(qcft.bad) >= 1 @@ -118,7 +118,7 @@ GROUP BY gaq_periods_timestamps.data_pass_id, gaq_periods_timestamps.run_number, IF(gaq_periods_timestamps.\`from\` = 0, null, gaq_periods_timestamps.\`from\`), - IF(gaq_periods_timestamps.\`to\` = UNIX_TIMESTAMP(NOW()), null, gaq_periods_timestamps.\`to\`) + IF(gaq_periods_timestamps.\`to\` = UNIX_TIMESTAMP(NOW(3)), null, gaq_periods_timestamps.\`to\`) `; const DROP_GAQ_PERIODS_VIEW = `DROP VIEW ${GAQ_PERIODS_VIEW_NAME}`; From 06268f2635ca05157594aa9dc2e163c1ba259996 Mon Sep 17 00:00:00 2001 From: xsalonx Date: Tue, 3 Dec 2024 12:56:43 +0100 Subject: [PATCH 03/16] remove names --- .../migrations/20241127123000-create-gaq-views.js | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/lib/database/migrations/20241127123000-create-gaq-views.js b/lib/database/migrations/20241127123000-create-gaq-views.js index d9f3ac0bd0..dc22d886a9 100644 --- a/lib/database/migrations/20241127123000-create-gaq-views.js +++ b/lib/database/migrations/20241127123000-create-gaq-views.js @@ -1,8 +1,7 @@ 'use strict'; -const GAQ_PERIODS_TIMESTAMPS_VIEW_NAME = 'gaq_periods_timestamps'; const CREATE_GAQ_PERIODS_TIMESTAMPS_VIEW = ` -CREATE OR REPLACE VIEW ${GAQ_PERIODS_TIMESTAMPS_VIEW_NAME} AS +CREATE OR REPLACE VIEW gaq_periods_timestamps AS SELECT data_pass_id, run_number, @@ -67,11 +66,10 @@ CREATE OR REPLACE VIEW ${GAQ_PERIODS_TIMESTAMPS_VIEW_NAME} AS ) AS ap `; -const DROP_GAQ_PERIODS_TIMESTAMPS_VIEW = `DROP VIEW ${GAQ_PERIODS_TIMESTAMPS_VIEW_NAME}`; +const DROP_GAQ_PERIODS_TIMESTAMPS_VIEW = 'DROP VIEW gaq_periods_timestamps'; -const GAQ_PERIODS_VIEW_NAME = 'gaq_periods'; const CREATE_GAQ_PERIODS_VIEW = ` -CREATE OR REPLACE VIEW ${GAQ_PERIODS_VIEW_NAME} AS +CREATE OR REPLACE VIEW gaq_periods AS SELECT gaq_periods_timestamps.data_pass_id AS dataPassId, gaq_periods_timestamps.run_number AS runNumber, @@ -89,7 +87,7 @@ SELECT GROUP_CONCAT( DISTINCT qcfv.flag_id ) AS verifiedFlagsList, GROUP_CONCAT( DISTINCT qcfep.flag_id ) AS flagsList -FROM ${GAQ_PERIODS_TIMESTAMPS_VIEW_NAME} AS gaq_periods_timestamps +FROM gaq_periods_timestamps INNER JOIN global_aggregated_quality_detectors AS gaqd ON gaqd.data_pass_id = gaq_periods_timestamps.data_pass_id AND gaqd.run_number = gaq_periods_timestamps.run_number @@ -121,7 +119,7 @@ GROUP BY IF(gaq_periods_timestamps.\`to\` = UNIX_TIMESTAMP(NOW(3)), null, gaq_periods_timestamps.\`to\`) `; -const DROP_GAQ_PERIODS_VIEW = `DROP VIEW ${GAQ_PERIODS_VIEW_NAME}`; +const DROP_GAQ_PERIODS_VIEW = 'DROP VIEW gaq_periods'; /** @type {import('sequelize-cli').Migration} */ module.exports = { From d6e7fcb333442f81a1c7eae822f10b9f960bc022 Mon Sep 17 00:00:00 2001 From: xsalonx Date: Tue, 3 Dec 2024 14:41:11 +0100 Subject: [PATCH 04/16] use views --- lib/database/repositories/QcFlagRepository.js | 149 +++--------------- 1 file changed, 22 insertions(+), 127 deletions(-) diff --git a/lib/database/repositories/QcFlagRepository.js b/lib/database/repositories/QcFlagRepository.js index 04b94b12bf..990e4ac282 100644 --- a/lib/database/repositories/QcFlagRepository.js +++ b/lib/database/repositories/QcFlagRepository.js @@ -15,51 +15,6 @@ const { Op } = require('sequelize'); const { models: { QcFlag } } = require('..'); const Repository = require('./Repository'); -const GAQ_PERIODS_VIEW = ` - SELECT - data_pass_id, - run_number, - timestamp AS \`from\`, - NTH_VALUE(timestamp, 2) OVER ( - PARTITION BY data_pass_id, - run_number - ORDER BY ap.timestamp - ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING - ) AS \`to\` - FROM ( - ( - SELECT gaqd.data_pass_id, - gaqd.run_number, - COALESCE(UNIX_TIMESTAMP(qcfep.\`from\`), 0) AS timestamp - FROM quality_control_flag_effective_periods AS qcfep - INNER JOIN quality_control_flags AS qcf ON qcf.id = qcfep.flag_id - INNER JOIN data_pass_quality_control_flag AS dpqcf ON dpqcf.quality_control_flag_id = qcf.id - -- Only flags of detectors which are defined in global_aggregated_quality_detectors - -- should be taken into account for calculation of gaq_effective_periods - INNER JOIN global_aggregated_quality_detectors AS gaqd - ON gaqd.data_pass_id = dpqcf.data_pass_id - AND gaqd.run_number = qcf.run_number - AND gaqd.detector_id = qcf.detector_id - ) - UNION - ( - SELECT gaqd.data_pass_id, - gaqd.run_number, - UNIX_TIMESTAMP(COALESCE(qcfep.\`to\`, NOW())) AS timestamp - FROM quality_control_flag_effective_periods AS qcfep - INNER JOIN quality_control_flags AS qcf ON qcf.id = qcfep.flag_id - INNER JOIN data_pass_quality_control_flag AS dpqcf ON dpqcf.quality_control_flag_id = qcf.id - -- Only flags of detectors which are defined in global_aggregated_quality_detectors - -- should be taken into account for calculation of gaq_effective_periods - INNER JOIN global_aggregated_quality_detectors AS gaqd - ON gaqd.data_pass_id = dpqcf.data_pass_id - AND gaqd.run_number = qcf.run_number - AND gaqd.detector_id = qcf.detector_id - ) - ORDER BY timestamp - ) AS ap - `; - /** * @typedef GaqPeriod * @@ -101,33 +56,10 @@ class QcFlagRepository extends Repository { */ async findGaqPeriods(dataPassId, runNumber) { const query = ` - SELECT - gaq_periods.data_pass_id AS dataPassId, - gaq_periods.run_number AS runNumber, - IF(gaq_periods.\`from\` = 0, null, gaq_periods.\`from\` * 1000) AS \`from\`, - IF(gaq_periods.\`to\` = UNIX_TIMESTAMP(NOW()), null, gaq_periods.\`to\` * 1000) AS \`to\`, - group_concat(qcf.id) AS contributingFlagIds - - FROM quality_control_flags AS qcf - INNER JOIN quality_control_flag_effective_periods AS qcfep - ON qcf.id = qcfep.flag_id - INNER JOIN data_pass_quality_control_flag AS dpqcf ON dpqcf.quality_control_flag_id = qcf.id - INNER JOIN (${GAQ_PERIODS_VIEW}) AS gaq_periods ON gaq_periods.data_pass_id = dpqcf.data_pass_id - INNER JOIN global_aggregated_quality_detectors AS gaqd - ON gaqd.data_pass_id = gaq_periods.data_pass_id - AND gaqd.run_number = gaq_periods.run_number - AND gaqd.detector_id = qcf.detector_id - AND gaq_periods.run_number = qcf.run_number - AND (qcfep.\`from\` IS NULL OR UNIX_TIMESTAMP(qcfep.\`from\`) <= gaq_periods.\`from\`) - AND (qcfep.\`to\` IS NULL OR gaq_periods.\`to\` <= UNIX_TIMESTAMP(qcfep.\`to\`)) - - WHERE gaq_periods.data_pass_id = ${dataPassId} - ${runNumber ? `AND gaq_periods.run_number = ${runNumber}` : ''} - - GROUP BY gaq_periods.run_number, - gaq_periods.data_pass_id, - gaq_periods.\`from\`, - gaq_periods.\`to\`; + SELECT * FROM gaq_periods + WHERE IF(gaq_periods.\`to\` = UNIX_TIMESTAMP(NOW()), null, gaq_periods.\`to\`) IS NOT NULL + AND gaq_periods.dataPassId = ${dataPassId} + ${runNumber ? `AND gaq_periods.runNumber = ${runNumber}` : ''} `; const [rows] = await this.model.sequelize.query(query); @@ -136,14 +68,14 @@ class QcFlagRepository extends Repository { runNumber, from, to, - contributingFlagIds, + flagsList, }) => ({ dataPassId, runNumber, - from, - to, - contributingFlagIds: contributingFlagIds.split(',').map((id) => parseInt(id, 10)), - })); + from: from * 1000, // Change unix seconds to miliseconds + to: to * 1000, + contributingFlagIds: flagsList ? flagsList.split(',').map((id) => parseInt(id, 10)) : [], + })).filter(({ contributingFlagIds }) => contributingFlagIds.length > 0); } /** @@ -156,56 +88,20 @@ class QcFlagRepository extends Repository { * @return {Promise} Resolves with the GAQ sub-summaries */ async getRunGaqSubSummaries(dataPassId, { mcReproducibleAsNotBad = false } = {}) { - const effectivePeriodsWithTypeSubQuery = ` - SELECT - gaq_periods.data_pass_id AS dataPassId, - gaq_periods.run_number AS runNumber, - IF(gaq_periods.\`from\` = 0, null, gaq_periods.\`from\`) AS \`from\`, - IF(gaq_periods.\`to\` = UNIX_TIMESTAMP(NOW()), null, gaq_periods.\`to\`) AS \`to\`, - SUM(IF(qcft.monte_carlo_reproducible AND :mcReproducibleAsNotBad, false, qcft.bad)) >= 1 AS bad, - SUM(qcft.bad) = SUM(qcft.monte_carlo_reproducible) AND SUM(qcft.monte_carlo_reproducible) AS mcReproducible, - GROUP_CONCAT( DISTINCT qcfv.flag_id ) AS verifiedFlagsList, - GROUP_CONCAT( DISTINCT qcf.id ) AS flagsList - - FROM quality_control_flags AS qcf - INNER JOIN quality_control_flag_types AS qcft - ON qcft.id = qcf.flag_type_id - LEFT JOIN quality_control_flag_verifications AS qcfv - ON qcfv.flag_id = qcf.id - INNER JOIN quality_control_flag_effective_periods AS qcfep - ON qcf.id = qcfep.flag_id - INNER JOIN data_pass_quality_control_flag AS dpqcf - ON dpqcf.quality_control_flag_id = qcf.id - INNER JOIN (${GAQ_PERIODS_VIEW}) AS gaq_periods - ON gaq_periods.data_pass_id = dpqcf.data_pass_id - INNER JOIN global_aggregated_quality_detectors AS gaqd - ON gaqd.data_pass_id = gaq_periods.data_pass_id - AND gaqd.run_number = gaq_periods.run_number - AND gaqd.detector_id = qcf.detector_id - AND gaq_periods.run_number = qcf.run_number - AND (qcfep.\`from\` IS NULL OR UNIX_TIMESTAMP(qcfep.\`from\`) <= gaq_periods.\`from\`) - AND (qcfep.\`to\` IS NULL OR gaq_periods.\`to\` <= UNIX_TIMESTAMP(qcfep.\`to\`)) - - GROUP BY - gaq_periods.data_pass_id, - gaq_periods.run_number, - gaq_periods.\`from\`, - gaq_periods.\`to\` - `; - const query = ` SELECT effectivePeriods.runNumber, effectivePeriods.dataPassId, effectivePeriods.bad, + effectivePeriods.badWhenMcReproducibleAsNotBad, SUM(effectivePeriods.mcReproducible) > 0 AS mcReproducible, GROUP_CONCAT(effectivePeriods.verifiedFlagsList) AS verifiedFlagsList, GROUP_CONCAT(effectivePeriods.flagsList) AS flagsList, IF( ( - COALESCE(run.time_trg_end, run.time_o2_end ) IS NULL - OR COALESCE(run.time_trg_start, run.time_o2_start) IS NULL + run.time_end IS NULL + OR run.time_start IS NULL ), IF( SUM( @@ -218,21 +114,19 @@ class QcFlagRepository extends Repository { SUM( COALESCE( effectivePeriods.\`to\`, - UNIX_TIMESTAMP(run.time_trg_end), - UNIX_TIMESTAMP(run.time_o2_end) + UNIX_TIMESTAMP(run.time_end) ) - COALESCE( effectivePeriods.\`from\`, - UNIX_TIMESTAMP(run.time_trg_start), - UNIX_TIMESTAMP(run.time_o2_start) + UNIX_TIMESTAMP(run.time_start) ) ) / ( - UNIX_TIMESTAMP(COALESCE(run.time_trg_end, run.time_o2_end)) - - UNIX_TIMESTAMP(COALESCE(run.time_trg_start, run.time_o2_start)) + UNIX_TIMESTAMP(run.time_end) + - UNIX_TIMESTAMP(run.time_start) ) ) AS effectiveRunCoverage - FROM (${effectivePeriodsWithTypeSubQuery}) AS effectivePeriods + FROM gaq_periods AS effectivePeriods INNER JOIN runs AS run ON run.run_number = effectivePeriods.runNumber WHERE effectivePeriods.dataPassId = :dataPassId @@ -243,22 +137,23 @@ class QcFlagRepository extends Repository { effectivePeriods.bad `; - const [rows] = await this.model.sequelize.query(query, { replacements: { dataPassId, mcReproducibleAsNotBad } }); + const [rows] = await this.model.sequelize.query(query, { replacements: { dataPassId } }); return rows.map(({ runNumber, bad, + badWhenMcReproducibleAsNotBad, effectiveRunCoverage, mcReproducible, flagsList, verifiedFlagsList, }) => ({ runNumber, - bad, + bad: mcReproducibleAsNotBad ? badWhenMcReproducibleAsNotBad : bad, effectiveRunCoverage: parseFloat(effectiveRunCoverage) || null, mcReproducible: Boolean(mcReproducible), - flagsIds: [...new Set(flagsList.split(','))], + flagsIds: flagsList ? [...new Set(flagsList.split(','))] : [], verifiedFlagsIds: verifiedFlagsList ? [...new Set(verifiedFlagsList.split(','))] : [], - })); + })).filter(({ bad }) => bad !== null); } /** From 1658d1d11394da07b88a92180229f35be4a8f70b Mon Sep 17 00:00:00 2001 From: xsalonx Date: Tue, 3 Dec 2024 18:42:36 +0100 Subject: [PATCH 05/16] fix tests, preserve previous functionalties --- lib/database/repositories/QcFlagRepository.js | 2 +- .../qualityControlFlag/QcFlagService.js | 42 ++++++++++++++----- .../qualityControlFlag/QcFlagService.test.js | 11 ++--- 3 files changed, 38 insertions(+), 17 deletions(-) diff --git a/lib/database/repositories/QcFlagRepository.js b/lib/database/repositories/QcFlagRepository.js index 990e4ac282..f8e8b849ee 100644 --- a/lib/database/repositories/QcFlagRepository.js +++ b/lib/database/repositories/QcFlagRepository.js @@ -153,7 +153,7 @@ class QcFlagRepository extends Repository { mcReproducible: Boolean(mcReproducible), flagsIds: flagsList ? [...new Set(flagsList.split(','))] : [], verifiedFlagsIds: verifiedFlagsList ? [...new Set(verifiedFlagsList.split(','))] : [], - })).filter(({ bad }) => bad !== null); + })); } /** diff --git a/lib/server/services/qualityControlFlag/QcFlagService.js b/lib/server/services/qualityControlFlag/QcFlagService.js index 11de154656..646623568e 100644 --- a/lib/server/services/qualityControlFlag/QcFlagService.js +++ b/lib/server/services/qualityControlFlag/QcFlagService.js @@ -688,11 +688,18 @@ class QcFlagService { distinctVerifiedFlagsIds: new Set([...distinctRunVerifiedFlagsIds, ...verifiedFlagsIds]), }; - this._mergeIntoSummaryUnit(runSummary, subSummary); + if (subSummary.bad !== null) { + this._mergeIntoSummaryUnit(runSummary, subSummary); + } } for (const [runNumber, { distinctFlagsIds, distinctVerifiedFlagsIds }] of Object.entries(flagsAndVerifications)) { summary[runNumber][QC_SUMMARY_PROPERTIES.missingVerificationsCount] = distinctFlagsIds.size - distinctVerifiedFlagsIds.size; + if (!summary[runNumber][QC_SUMMARY_PROPERTIES.badEffectiveRunCoverage] && + !summary[runNumber][QC_SUMMARY_PROPERTIES.explicitlyNotBadEffectiveRunCoverage] + ) { + delete summary[runNumber]; + } } return summary; @@ -713,20 +720,33 @@ class QcFlagService { mcReproducible, } = partialSummaryUnit; + const { badEffectiveRunCoverage, explicitlyNotBadEffectiveRunCoverage } = summaryUnit; + if (bad) { - summaryUnit[QC_SUMMARY_PROPERTIES.badEffectiveRunCoverage] = effectiveRunCoverage; - summaryUnit[QC_SUMMARY_PROPERTIES.mcReproducible] = - mcReproducible || summaryUnit[QC_SUMMARY_PROPERTIES.mcReproducible]; - } else { - summaryUnit[QC_SUMMARY_PROPERTIES.explicitlyNotBadEffectiveRunCoverage] = effectiveRunCoverage; + if (effectiveRunCoverage !== null && badEffectiveRunCoverage !== null) { + summaryUnit.badEffectiveRunCoverage = + (badEffectiveRunCoverage ?? 0) + effectiveRunCoverage; + } else { + summaryUnit.badEffectiveRunCoverage = null; + } + summaryUnit[QC_SUMMARY_PROPERTIES.mcReproducible] = - mcReproducible || summaryUnit[QC_SUMMARY_PROPERTIES.mcReproducible]; + mcReproducible || summaryUnit.mcReproducible; + } else if (bad !== null) { + if (effectiveRunCoverage !== null && explicitlyNotBadEffectiveRunCoverage !== null) { + summaryUnit.explicitlyNotBadEffectiveRunCoverage = + (explicitlyNotBadEffectiveRunCoverage ?? 0) + effectiveRunCoverage; + } else { + summaryUnit.explicitlyNotBadEffectiveRunCoverage = null; + } + summaryUnit.mcReproducible = + mcReproducible || summaryUnit.mcReproducible; } - if (summaryUnit[QC_SUMMARY_PROPERTIES.badEffectiveRunCoverage] === undefined) { - summaryUnit[QC_SUMMARY_PROPERTIES.badEffectiveRunCoverage] = 0; + if (summaryUnit.badEffectiveRunCoverage === undefined) { + summaryUnit.badEffectiveRunCoverage = 0; } - if (summaryUnit[QC_SUMMARY_PROPERTIES.explicitlyNotBadEffectiveRunCoverage] === undefined) { - summaryUnit[QC_SUMMARY_PROPERTIES.explicitlyNotBadEffectiveRunCoverage] = 0; + if (summaryUnit.explicitlyNotBadEffectiveRunCoverage === undefined) { + summaryUnit.explicitlyNotBadEffectiveRunCoverage = 0; } } diff --git a/test/lib/server/services/qualityControlFlag/QcFlagService.test.js b/test/lib/server/services/qualityControlFlag/QcFlagService.test.js index 92d32903d2..edde35dabc 100644 --- a/test/lib/server/services/qualityControlFlag/QcFlagService.test.js +++ b/test/lib/server/services/qualityControlFlag/QcFlagService.test.js @@ -1447,19 +1447,20 @@ module.exports = () => { const timeTrgEnd = t('22:00:00'); const gaqSubSummaries = [ - { from: t('06:00:00'), to: t('10:00:00'), bad: true, mcReproducible: false }, + { from: t('06:00:00'), to: t('10:00:00'), bad: null, mcReproducible: false }, { from: t('10:00:00'), to: t('12:00:00'), bad: true, mcReproducible: false }, { from: t('12:00:00'), to: t('13:00:00'), bad: true, mcReproducible: true }, - { from: t('13:00:00'), to: t('14:00:00'), bad: true, mcReproducible: true }, + { from: t('13:00:00'), to: t('14:00:00'), bad: null, mcReproducible: true }, { from: t('14:00:00'), to: t('16:00:00'), bad: true, mcReproducible: false }, - { from: t('18:00:00'), to: t('20:00:00'), bad: false, mcReproducible: false }, - { from: t('20:00:00'), to: t('22:00:00'), bad: false, mcReproducible: false }, + { from: t('16:00:00'), to: t('18:00:00'), bad: null, mcReproducible: false }, + { from: t('18:00:00'), to: t('20:00:00'), bad: null, mcReproducible: false }, + { from: t('20:00:00'), to: t('22:00:00'), bad: null, mcReproducible: false }, ]; const expectedGaqSummary = gaqSubSummaries.reduce((acc, { from, to, bad, mcReproducible }) => { if (bad) { acc.badEffectiveRunCoverage += to - from; - } else { + } else if (bad !== null) { acc.explicitlyNotBadEffectiveRunCoverage += to - from; } acc.mcReproducible = acc.mcReproducible || mcReproducible; From 068d48c11bb4b9c37c8aff421ebeb2759ea70915 Mon Sep 17 00:00:00 2001 From: xsalonx Date: Tue, 3 Dec 2024 18:45:35 +0100 Subject: [PATCH 06/16] fix --- lib/database/repositories/QcFlagRepository.js | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/lib/database/repositories/QcFlagRepository.js b/lib/database/repositories/QcFlagRepository.js index f8e8b849ee..88c8a5f634 100644 --- a/lib/database/repositories/QcFlagRepository.js +++ b/lib/database/repositories/QcFlagRepository.js @@ -90,13 +90,13 @@ class QcFlagRepository extends Repository { async getRunGaqSubSummaries(dataPassId, { mcReproducibleAsNotBad = false } = {}) { const query = ` SELECT - effectivePeriods.runNumber, - effectivePeriods.dataPassId, - effectivePeriods.bad, - effectivePeriods.badWhenMcReproducibleAsNotBad, - SUM(effectivePeriods.mcReproducible) > 0 AS mcReproducible, - GROUP_CONCAT(effectivePeriods.verifiedFlagsList) AS verifiedFlagsList, - GROUP_CONCAT(effectivePeriods.flagsList) AS flagsList, + gaq_periods.runNumber, + gaq_periods.dataPassId, + gaq_periods.bad, + gaq_periods.badWhenMcReproducibleAsNotBad, + SUM(gaq_periods.mcReproducible) > 0 AS mcReproducible, + GROUP_CONCAT(gaq_periods.verifiedFlagsList) AS verifiedFlagsList, + GROUP_CONCAT(gaq_periods.flagsList) AS flagsList, IF( ( @@ -105,19 +105,19 @@ class QcFlagRepository extends Repository { ), IF( SUM( - COALESCE(effectivePeriods.\`to\` , 0) - + COALESCE(effectivePeriods.\`from\`, 0) + COALESCE(gaq_periods.\`to\` , 0) + + COALESCE(gaq_periods.\`from\`, 0) ) = 0, 1, null ), SUM( COALESCE( - effectivePeriods.\`to\`, + gaq_periods.\`to\`, UNIX_TIMESTAMP(run.time_end) ) - COALESCE( - effectivePeriods.\`from\`, + gaq_periods.\`from\`, UNIX_TIMESTAMP(run.time_start) ) ) / ( @@ -126,15 +126,15 @@ class QcFlagRepository extends Repository { ) ) AS effectiveRunCoverage - FROM gaq_periods AS effectivePeriods - INNER JOIN runs AS run ON run.run_number = effectivePeriods.runNumber + FROM gaq_periods + INNER JOIN runs AS run ON run.run_number = gaq_periods.runNumber - WHERE effectivePeriods.dataPassId = :dataPassId + WHERE gaq_periods.dataPassId = :dataPassId GROUP BY - effectivePeriods.dataPassId, - effectivePeriods.runNumber, - effectivePeriods.bad + gaq_periods.dataPassId, + gaq_periods.runNumber, + gaq_periods.bad `; const [rows] = await this.model.sequelize.query(query, { replacements: { dataPassId } }); From 4ee19e7876cc494af41532aa9d65717adba67f76 Mon Sep 17 00:00:00 2001 From: xsalonx Date: Tue, 3 Dec 2024 23:33:13 +0100 Subject: [PATCH 07/16] fix test --- .../seeders/20240404100811-qc-flags.js | 103 ++++++++++++++++++ .../qualityControlFlag/QcFlagService.test.js | 2 +- .../usecases/run/GetAllRunsUseCase.test.js | 4 +- 3 files changed, 106 insertions(+), 3 deletions(-) diff --git a/lib/database/seeders/20240404100811-qc-flags.js b/lib/database/seeders/20240404100811-qc-flags.js index a92578eb29..52b0eb0cb2 100644 --- a/lib/database/seeders/20240404100811-qc-flags.js +++ b/lib/database/seeders/20240404100811-qc-flags.js @@ -127,6 +127,65 @@ module.exports = { updated_at: '2024-02-13 11:58:20', }, + /** Flags for runNumber: 56, LHC22a_apass1, FT0 */ + { + id: 10, + from: '2019-08-08 20:00:00', + to: '2019-08-08 21:00:00', + comment: 'Some qc comment 10', + + // Associations + created_by_id: 2, + flag_type_id: 3, // Good + run_number: 56, + detector_id: 7, // FT0 + created_at: '2024-02-13 11:58:20', + updated_at: '2024-02-13 11:58:20', + }, + { + id: 13, + from: '2019-08-08 20:00:00', + to: '2019-08-08 20:30:00', + comment: 'Some qc comment 10', + + // Associations + created_by_id: 2, + flag_type_id: 5, // Good + run_number: 56, + detector_id: 7, // FT0 + created_at: '2024-02-13 11:58:20', + updated_at: '2024-02-13 11:58:20', + }, + // For ITS + { + id: 11, + from: '2019-08-08 20:00:00', + to: '2019-08-08 21:00:00', + comment: 'Some qc comment 11', + + // Associations + created_by_id: 2, + flag_type_id: 3, // Good + run_number: 56, + detector_id: 4, // ITS + created_at: '2024-02-13 11:58:20', + updated_at: '2024-02-13 11:58:20', + }, + { + id: 12, + from: '2019-08-08 20:30:00', + to: '2019-08-08 21:00:00', + comment: 'Some qc comment 12', + + // Associations + created_by_id: 2, + flag_type_id: 5, // Lim. Acc. MC Reproducible + run_number: 56, + detector_id: 4, // ITS + created_at: '2024-02-13 11:58:20', + updated_at: '2024-02-13 11:58:20', + }, + /** Synchronous */ // Run : 56, FT0 @@ -214,6 +273,33 @@ module.exports = { to: '2019-08-09 14:00:00', }, + { + id: 10, + flag_id: 10, + from: '2019-08-08 20:30:00', + to: '2019-08-08 21:00:00', + }, + { + id: 13, + flag_id: 13, + from: '2019-08-08 20:00:00', + to: '2019-08-08 20:30:00', + }, + + { + id: 11, + flag_id: 11, + from: '2019-08-08 20:00:00', + to: '2019-08-08 20:30:00', + }, + + { + id: 12, + flag_id: 12, + from: '2019-08-08 20:30:00', + to: '2019-08-08 21:00:00', + }, + /** Synchronous */ // Run : 56, FT0 { @@ -260,6 +346,23 @@ module.exports = { data_pass_id: 2, // LHC22b_apass2 quality_control_flag_id: 4, }, + + { + data_pass_id: 3, // LHC22a_apass1 + quality_control_flag_id: 10, + }, + { + data_pass_id: 3, // LHC22a_apass1 + quality_control_flag_id: 13, + }, + { + data_pass_id: 3, // LHC22a_apass1 + quality_control_flag_id: 11, + }, + { + data_pass_id: 3, // LHC22a_apass1 + quality_control_flag_id: 12, + }, ], { transaction }), queryInterface.bulkInsert('simulation_pass_quality_control_flag', [ diff --git a/test/lib/server/services/qualityControlFlag/QcFlagService.test.js b/test/lib/server/services/qualityControlFlag/QcFlagService.test.js index edde35dabc..4db5889e07 100644 --- a/test/lib/server/services/qualityControlFlag/QcFlagService.test.js +++ b/test/lib/server/services/qualityControlFlag/QcFlagService.test.js @@ -238,7 +238,7 @@ module.exports = () => { }); it('should successfully get empty QC flag summary for data pass', async () => { - expect(await qcFlagService.getQcFlagsSummary({ dataPassId: 3 })).to.be.eql({}); + expect(await qcFlagService.getQcFlagsSummary({ dataPassId: 4 })).to.be.eql({}); }); it('should successfully get non-empty QC flag summary for simulation pass', async () => { diff --git a/test/lib/usecases/run/GetAllRunsUseCase.test.js b/test/lib/usecases/run/GetAllRunsUseCase.test.js index 652691499b..169f850373 100644 --- a/test/lib/usecases/run/GetAllRunsUseCase.test.js +++ b/test/lib/usecases/run/GetAllRunsUseCase.test.js @@ -677,14 +677,14 @@ module.exports = () => { } it('should successfully filter by GAQ notBadFraction', async () => { - const dataPassIds = [1]; + const dataPassIds = [3]; { const { runs } = await new GetAllRunsUseCase().execute({ query: { filter: { dataPassIds, gaq: { notBadFraction: { '<': 0.8 } }, } } }); expect(runs).to.be.an('array'); - expect(runs.map(({ runNumber }) => runNumber)).to.have.all.members([106]); + expect(runs.map(({ runNumber }) => runNumber)).to.have.all.members([56]); } { const { runs } = await new GetAllRunsUseCase().execute({ query: { filter: { From 10be130e5d53ce5365ab9032dbc4d77075ee85f8 Mon Sep 17 00:00:00 2001 From: xsalonx Date: Tue, 3 Dec 2024 23:41:34 +0100 Subject: [PATCH 08/16] fix test --- test/api/qcFlags.test.js | 6 ++++++ test/api/runs.test.js | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/test/api/qcFlags.test.js b/test/api/qcFlags.test.js index 776c14868a..5d03610e3f 100644 --- a/test/api/qcFlags.test.js +++ b/test/api/qcFlags.test.js @@ -193,6 +193,12 @@ module.exports = () => { badEffectiveRunCoverage: 1, explicitlyNotBadEffectiveRunCoverage: 0, }, + 56: { + badEffectiveRunCoverage: 1, + explicitlyNotBadEffectiveRunCoverage: 0, + mcReproducible: true, + missingVerificationsCount: 4, + }, }); }); diff --git a/test/api/runs.test.js b/test/api/runs.test.js index 8b1ccbc36e..39692429ba 100644 --- a/test/api/runs.test.js +++ b/test/api/runs.test.js @@ -378,7 +378,7 @@ module.exports = () => { } it('should successfully filter by GAQ notBadFraction', async () => { - const dataPassId = 1; + const dataPassId = 3; { const response = await request(server).get(`/api/runs?filter[dataPassIds][]=${dataPassId}&filter[gaq][notBadFraction][<]=0.8`); @@ -386,7 +386,7 @@ module.exports = () => { const { data: runs } = response.body; expect(runs).to.be.an('array'); - expect(runs.map(({ runNumber }) => runNumber)).to.have.all.members([106]); + expect(runs.map(({ runNumber }) => runNumber)).to.have.all.members([56]); } { const response = await request(server).get(`/api/runs?filter[dataPassIds][]=${dataPassId}` + From a7a7b5455ddfea68aef20079f183a0ddb74bc5af Mon Sep 17 00:00:00 2001 From: xsalonx Date: Tue, 3 Dec 2024 23:43:14 +0100 Subject: [PATCH 09/16] fix test --- .../runs/runsPerDataPass.overview.test.js | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/test/public/runs/runsPerDataPass.overview.test.js b/test/public/runs/runsPerDataPass.overview.test.js index 990ca1d3bf..820cf20f18 100644 --- a/test/public/runs/runsPerDataPass.overview.test.js +++ b/test/public/runs/runsPerDataPass.overview.test.js @@ -165,12 +165,17 @@ module.exports = () => { }); await page.waitForSelector('tr#row106 .column-CPV a .icon'); - await expectInnerText(page, '#row106-globalAggregatedQuality', '67MC.R'); - expect(await getPopoverInnerText(await page.waitForSelector('#row106-globalAggregatedQuality .popover-trigger'))) - .to.be.equal('Missing 3 verifications'); + await expectInnerText(page, '#row106-globalAggregatedQuality', 'GAQ'); + + await navigateToRunsPerDataPass(page, { lhcPeriodId: 1, dataPassId: 3 }, { epectedRowsCount: 4 }); + await expectInnerText(page, '#row56-globalAggregatedQuality', '0MC.R'); + expect(await getPopoverInnerText(await page.waitForSelector('#row56-globalAggregatedQuality .popover-trigger'))) + .to.be.equal('Missing 4 verifications'); }); it('should switch mcReproducibleAsNotBad', async () => { + await navigateToRunsPerDataPass(page, { lhcPeriodId: 2, dataPassId: 1 }, { epectedRowsCount: 3 }); + await pressElement(page, '#mcReproducibleAsNotBadToggle input', true); await waitForTableLength(page, 3); await expectInnerText(page, 'tr#row106 .column-CPV a', '89'); @@ -425,21 +430,21 @@ module.exports = () => { } it('should successfully apply gaqNotBadFraction filters', async () => { - await navigateToRunsPerDataPass(page, { lhcPeriodId: 2, dataPassId: 1 }, { epectedRowsCount: 3 }); + await navigateToRunsPerDataPass(page, { lhcPeriodId: 1, dataPassId: 3 }, { epectedRowsCount: 4 }); await pressElement(page, '#openFilterToggle', true); const popoverSelector = await getPopoverSelector(await page.waitForSelector('.globalAggregatedQuality-filter .popover-trigger')); await pressElement(page, `${popoverSelector} #gaqNotBadFraction-dropdown-option-le`, true); await fillInput(page, '#gaqNotBadFraction-value-input', '80'); - await expectColumnValues(page, 'runNumber', ['106']); + await expectColumnValues(page, 'runNumber', ['56']); await pressElement(page, '#mcReproducibleAsNotBadToggle input', true); await expectColumnValues(page, 'runNumber', []); await pressElement(page, '#openFilterToggle', true); await pressElement(page, '#reset-filters', true); - await expectColumnValues(page, 'runNumber', ['108', '107', '106']); + await expectColumnValues(page, 'runNumber', ['105', '56', '54', '49']); }); it('should successfully apply muInelasticInteractionRate filters', async () => { From 1ee8babb269abb09cf3959ec67e01b60233b697c Mon Sep 17 00:00:00 2001 From: xsalonx Date: Wed, 4 Dec 2024 00:29:48 +0100 Subject: [PATCH 10/16] fix seeders --- lib/database/repositories/QcFlagRepository.js | 2 +- lib/database/seeders/20200713103855-runs.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/database/repositories/QcFlagRepository.js b/lib/database/repositories/QcFlagRepository.js index 88c8a5f634..3a4a252591 100644 --- a/lib/database/repositories/QcFlagRepository.js +++ b/lib/database/repositories/QcFlagRepository.js @@ -57,7 +57,7 @@ class QcFlagRepository extends Repository { async findGaqPeriods(dataPassId, runNumber) { const query = ` SELECT * FROM gaq_periods - WHERE IF(gaq_periods.\`to\` = UNIX_TIMESTAMP(NOW()), null, gaq_periods.\`to\`) IS NOT NULL + WHERE IF(gaq_periods.\`to\` = UNIX_TIMESTAMP(NOW(3)), null, gaq_periods.\`to\`) IS NOT NULL AND gaq_periods.dataPassId = ${dataPassId} ${runNumber ? `AND gaq_periods.runNumber = ${runNumber}` : ''} `; diff --git a/lib/database/seeders/20200713103855-runs.js b/lib/database/seeders/20200713103855-runs.js index 4a63975a52..e194969794 100644 --- a/lib/database/seeders/20200713103855-runs.js +++ b/lib/database/seeders/20200713103855-runs.js @@ -2654,7 +2654,7 @@ module.exports = { time_o2_end: '2019-08-09 14:00:00', time_trg_start: '2019-08-08 13:00:00', time_trg_end: '2019-08-09 14:00:00', - first_tf_timestamp: '2019-08-09 13:00:00', + first_tf_timestamp: '2019-08-08 13:00:00', run_type_id: 12, run_quality: 'good', n_detectors: 15, From 3fb54a50e8679597570decec39e544651e4539a8 Mon Sep 17 00:00:00 2001 From: xsalonx Date: Wed, 11 Dec 2024 10:31:53 +0100 Subject: [PATCH 11/16] use timeframe timestamps --- lib/server/services/qualityControlFlag/QcFlagService.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/server/services/qualityControlFlag/QcFlagService.js b/lib/server/services/qualityControlFlag/QcFlagService.js index 646623568e..69e9f2b1d1 100644 --- a/lib/server/services/qualityControlFlag/QcFlagService.js +++ b/lib/server/services/qualityControlFlag/QcFlagService.js @@ -265,13 +265,13 @@ class QcFlagService { sequelize.literal(` IF( ( - COALESCE(run.time_trg_end, run.time_o2_end ) IS NULL - OR COALESCE(run.time_trg_start, run.time_o2_start) IS NULL + COALESCE(run.first_tf_timestamp, run.time_start ) IS NULL + OR COALESCE(run.last_tf_timestamp , run.time_end) IS NULL ), IF( SUM( COALESCE(UNIX_TIMESTAMP(effectivePeriods.\`to\` ), 0) - + COALESCE(UNIX_TIMESTAMP(effectivePeriods.\`from\`), 0) + COALESCE(UNIX_TIMESTAMP(effectivePeriods.\`from\`), 0) ) = 0, 1, null From db84fcca003cb70e3c4ceda12d757a3889b0927a Mon Sep 17 00:00:00 2001 From: xsalonx Date: Wed, 11 Dec 2024 13:45:51 +0100 Subject: [PATCH 12/16] ref --- .../20241127123000-create-gaq-views.js | 123 +++++++++++------- .../qualityControlFlag/QcFlagService.js | 2 +- 2 files changed, 74 insertions(+), 51 deletions(-) diff --git a/lib/database/migrations/20241127123000-create-gaq-views.js b/lib/database/migrations/20241127123000-create-gaq-views.js index dc22d886a9..e53bbf10df 100644 --- a/lib/database/migrations/20241127123000-create-gaq-views.js +++ b/lib/database/migrations/20241127123000-create-gaq-views.js @@ -1,7 +1,60 @@ 'use strict'; +const SELECT_RUNS_FROM_TIMESTAMPS_FOR_GAQ_PERIODS = ` + SELECT gaqd.data_pass_id, + gaqd.run_number, + COALESCE(UNIX_TIMESTAMP(first_tf_timestamp), UNIX_TIMESTAMP(time_start), 0) AS ord_timestamp, + COALESCE(UNIX_TIMESTAMP(first_tf_timestamp), UNIX_TIMESTAMP(time_start)) AS timestamp + FROM global_aggregated_quality_detectors AS gaqd + INNER JOIN runs as r + ON gaqd.run_number = r.run_number +`; + +const SELECT_RUNS_TO_TIMESTAMPS_FOR_GAQ_PERIODS = ` + SELECT gaqd.data_pass_id, + gaqd.run_number, + UNIX_TIMESTAMP(COALESCE(last_tf_timestamp, time_end, NOW(3))) AS ord_timestamp, + UNIX_TIMESTAMP(COALESCE(last_tf_timestamp, time_end)) AS timestamp + FROM global_aggregated_quality_detectors AS gaqd + INNER JOIN runs as r + ON gaqd.run_number = r.run_number +`; + +const SELECT_QCF_EFFECTIVE_PERIODS_FROM_TIMESTAMPS_FOR_GAQ_PERIODS = ` + SELECT gaqd.data_pass_id, + gaqd.run_number, + COALESCE(UNIX_TIMESTAMP(qcfep.\`from\`), 0) AS ord_timestamp, + UNIX_TIMESTAMP(qcfep.\`from\`) AS timestamp + FROM quality_control_flag_effective_periods AS qcfep + INNER JOIN quality_control_flags AS qcf ON qcf.id = qcfep.flag_id + INNER JOIN data_pass_quality_control_flag AS dpqcf ON dpqcf.quality_control_flag_id = qcf.id + -- Only flags of detectors which are defined in global_aggregated_quality_detectors + -- should be taken into account for calculation of gaq_effective_periods + INNER JOIN global_aggregated_quality_detectors AS gaqd + ON gaqd.data_pass_id = dpqcf.data_pass_id + AND gaqd.run_number = qcf.run_number + AND gaqd.detector_id = qcf.detector_id +`; + +const SELECT_QCF_EFFECTIVE_PERIODS_TO_TIMESTAMPS_FOR_GAQ_PERIODS = ` + SELECT gaqd.data_pass_id, + gaqd.run_number, + UNIX_TIMESTAMP(COALESCE(qcfep.\`to\`, NOW(3))) AS ord_timestamp, + UNIX_TIMESTAMP(COALESCE(qcfep.\`to\`)) AS timestamp + FROM quality_control_flag_effective_periods AS qcfep + INNER JOIN quality_control_flags AS qcf ON qcf.id = qcfep.flag_id + INNER JOIN data_pass_quality_control_flag AS dpqcf ON dpqcf.quality_control_flag_id = qcf.id + -- Only flags of detectors which are defined in global_aggregated_quality_detectors + -- should be taken into account for calculation of gaq_effective_periods + INNER JOIN global_aggregated_quality_detectors AS gaqd + ON gaqd.data_pass_id = dpqcf.data_pass_id + AND gaqd.run_number = qcf.run_number + AND gaqd.detector_id = qcf.detector_id +`; + const CREATE_GAQ_PERIODS_TIMESTAMPS_VIEW = ` CREATE OR REPLACE VIEW gaq_periods_timestamps AS + SELECT * FROM ( SELECT data_pass_id, run_number, @@ -9,61 +62,24 @@ CREATE OR REPLACE VIEW gaq_periods_timestamps AS NTH_VALUE(timestamp, 2) OVER ( PARTITION BY data_pass_id, run_number - ORDER BY ap.timestamp + ORDER BY ap.ord_timestamp ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING ) AS \`to\` FROM ( -- Two selects for runs' timestamps (in case QC flag's eff. period doesn't start at run's start or end at run's end ) - ( - SELECT gaqd.data_pass_id, - gaqd.run_number, - COALESCE(UNIX_TIMESTAMP(first_tf_timestamp), UNIX_TIMESTAMP(time_start), 0) AS timestamp - FROM global_aggregated_quality_detectors AS gaqd - INNER JOIN runs as r - ON gaqd.run_number = r.run_number - ) + ( ${SELECT_RUNS_FROM_TIMESTAMPS_FOR_GAQ_PERIODS} ) UNION - ( - SELECT gaqd.data_pass_id, - gaqd.run_number, - UNIX_TIMESTAMP(COALESCE(last_tf_timestamp, time_end, NOW(3))) AS timestamp - FROM global_aggregated_quality_detectors AS gaqd - INNER JOIN runs as r - ON gaqd.run_number = r.run_number - ) + ( ${SELECT_RUNS_TO_TIMESTAMPS_FOR_GAQ_PERIODS} ) UNION -- Two selectes for timestamps of QC flags' effective periods - ( - SELECT gaqd.data_pass_id, - gaqd.run_number, - COALESCE(UNIX_TIMESTAMP(qcfep.\`from\`), 0) AS timestamp - FROM quality_control_flag_effective_periods AS qcfep - INNER JOIN quality_control_flags AS qcf ON qcf.id = qcfep.flag_id - INNER JOIN data_pass_quality_control_flag AS dpqcf ON dpqcf.quality_control_flag_id = qcf.id - -- Only flags of detectors which are defined in global_aggregated_quality_detectors - -- should be taken into account for calculation of gaq_effective_periods - INNER JOIN global_aggregated_quality_detectors AS gaqd - ON gaqd.data_pass_id = dpqcf.data_pass_id - AND gaqd.run_number = qcf.run_number - AND gaqd.detector_id = qcf.detector_id - ) + ( ${SELECT_QCF_EFFECTIVE_PERIODS_FROM_TIMESTAMPS_FOR_GAQ_PERIODS} ) UNION - ( - SELECT gaqd.data_pass_id, - gaqd.run_number, - UNIX_TIMESTAMP(COALESCE(qcfep.\`to\`, NOW(3))) AS timestamp - FROM quality_control_flag_effective_periods AS qcfep - INNER JOIN quality_control_flags AS qcf ON qcf.id = qcfep.flag_id - INNER JOIN data_pass_quality_control_flag AS dpqcf ON dpqcf.quality_control_flag_id = qcf.id - -- Only flags of detectors which are defined in global_aggregated_quality_detectors - -- should be taken into account for calculation of gaq_effective_periods - INNER JOIN global_aggregated_quality_detectors AS gaqd - ON gaqd.data_pass_id = dpqcf.data_pass_id - AND gaqd.run_number = qcf.run_number - AND gaqd.detector_id = qcf.detector_id - ) - ORDER BY timestamp + ( ${SELECT_QCF_EFFECTIVE_PERIODS_TO_TIMESTAMPS_FOR_GAQ_PERIODS} ) + + ORDER BY ord_timestamp ) AS ap + ) AS gaq_periods_with_last_nullish_row + WHERE gaq_periods_with_last_nullish_row.\`to\` IS NOT NULL `; const DROP_GAQ_PERIODS_TIMESTAMPS_VIEW = 'DROP VIEW gaq_periods_timestamps'; @@ -73,8 +89,12 @@ CREATE OR REPLACE VIEW gaq_periods AS SELECT gaq_periods_timestamps.data_pass_id AS dataPassId, gaq_periods_timestamps.run_number AS runNumber, - IF(gaq_periods_timestamps.\`from\` = 0, null, gaq_periods_timestamps.\`from\`) AS \`from\`, - IF(gaq_periods_timestamps.\`to\` = UNIX_TIMESTAMP(NOW(3)), null, gaq_periods_timestamps.\`to\`) AS \`to\`, + -- IF(gaq_periods_timestamps.\`from\` = 0, null, gaq_periods_timestamps.\`from\`) AS \`from\`, + -- IF(gaq_periods_timestamps.\`to\` = UNIX_TIMESTAMP(NOW(3)), null, gaq_periods_timestamps.\`to\`) AS \`to\`, + + gaq_periods_timestamps.\`from\` AS \`from\`, + gaq_periods_timestamps.\`to\` AS \`to\`, + IF(COUNT( DISTINCT gaqd.detector_id ) > COUNT( DISTINCT qcfep.flag_id ), null, SUM(qcft.bad) >= 1 @@ -115,8 +135,11 @@ WHERE gaq_periods_timestamps.\`to\` IS NOT null GROUP BY gaq_periods_timestamps.data_pass_id, gaq_periods_timestamps.run_number, - IF(gaq_periods_timestamps.\`from\` = 0, null, gaq_periods_timestamps.\`from\`), - IF(gaq_periods_timestamps.\`to\` = UNIX_TIMESTAMP(NOW(3)), null, gaq_periods_timestamps.\`to\`) + -- IF(gaq_periods_timestamps.\`from\` = 0, null, gaq_periods_timestamps.\`from\`), + -- IF(gaq_periods_timestamps.\`to\` = UNIX_TIMESTAMP(NOW(3)), null, gaq_periods_timestamps.\`to\`) + + gaq_periods_timestamps.\`from\`, + gaq_periods_timestamps.\`to\` `; const DROP_GAQ_PERIODS_VIEW = 'DROP VIEW gaq_periods'; diff --git a/lib/server/services/qualityControlFlag/QcFlagService.js b/lib/server/services/qualityControlFlag/QcFlagService.js index bbce637228..a8d9055648 100644 --- a/lib/server/services/qualityControlFlag/QcFlagService.js +++ b/lib/server/services/qualityControlFlag/QcFlagService.js @@ -271,7 +271,7 @@ class QcFlagService { IF( SUM( COALESCE(UNIX_TIMESTAMP(effectivePeriods.\`to\` ), 0) - COALESCE(UNIX_TIMESTAMP(effectivePeriods.\`from\`), 0) + + COALESCE(UNIX_TIMESTAMP(effectivePeriods.\`from\`), 0) ) = 0, 1, null From 4199b3ff42d80e7f0a48c6e4bb98b61312ac9a39 Mon Sep 17 00:00:00 2001 From: xsalonx Date: Wed, 11 Dec 2024 13:52:34 +0100 Subject: [PATCH 13/16] cleanup --- .../20241127123000-create-gaq-views.js | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/lib/database/migrations/20241127123000-create-gaq-views.js b/lib/database/migrations/20241127123000-create-gaq-views.js index e53bbf10df..eb230b5d7f 100644 --- a/lib/database/migrations/20241127123000-create-gaq-views.js +++ b/lib/database/migrations/20241127123000-create-gaq-views.js @@ -3,7 +3,7 @@ const SELECT_RUNS_FROM_TIMESTAMPS_FOR_GAQ_PERIODS = ` SELECT gaqd.data_pass_id, gaqd.run_number, - COALESCE(UNIX_TIMESTAMP(first_tf_timestamp), UNIX_TIMESTAMP(time_start), 0) AS ord_timestamp, + COALESCE(UNIX_TIMESTAMP(first_tf_timestamp), UNIX_TIMESTAMP(time_start), 0) AS ordering_timestamp, COALESCE(UNIX_TIMESTAMP(first_tf_timestamp), UNIX_TIMESTAMP(time_start)) AS timestamp FROM global_aggregated_quality_detectors AS gaqd INNER JOIN runs as r @@ -13,7 +13,7 @@ const SELECT_RUNS_FROM_TIMESTAMPS_FOR_GAQ_PERIODS = ` const SELECT_RUNS_TO_TIMESTAMPS_FOR_GAQ_PERIODS = ` SELECT gaqd.data_pass_id, gaqd.run_number, - UNIX_TIMESTAMP(COALESCE(last_tf_timestamp, time_end, NOW(3))) AS ord_timestamp, + UNIX_TIMESTAMP(COALESCE(last_tf_timestamp, time_end, NOW(3))) AS ordering_timestamp, UNIX_TIMESTAMP(COALESCE(last_tf_timestamp, time_end)) AS timestamp FROM global_aggregated_quality_detectors AS gaqd INNER JOIN runs as r @@ -23,7 +23,7 @@ const SELECT_RUNS_TO_TIMESTAMPS_FOR_GAQ_PERIODS = ` const SELECT_QCF_EFFECTIVE_PERIODS_FROM_TIMESTAMPS_FOR_GAQ_PERIODS = ` SELECT gaqd.data_pass_id, gaqd.run_number, - COALESCE(UNIX_TIMESTAMP(qcfep.\`from\`), 0) AS ord_timestamp, + COALESCE(UNIX_TIMESTAMP(qcfep.\`from\`), 0) AS ordering_timestamp, UNIX_TIMESTAMP(qcfep.\`from\`) AS timestamp FROM quality_control_flag_effective_periods AS qcfep INNER JOIN quality_control_flags AS qcf ON qcf.id = qcfep.flag_id @@ -39,7 +39,7 @@ const SELECT_QCF_EFFECTIVE_PERIODS_FROM_TIMESTAMPS_FOR_GAQ_PERIODS = ` const SELECT_QCF_EFFECTIVE_PERIODS_TO_TIMESTAMPS_FOR_GAQ_PERIODS = ` SELECT gaqd.data_pass_id, gaqd.run_number, - UNIX_TIMESTAMP(COALESCE(qcfep.\`to\`, NOW(3))) AS ord_timestamp, + UNIX_TIMESTAMP(COALESCE(qcfep.\`to\`, NOW(3))) AS ordering_timestamp, UNIX_TIMESTAMP(COALESCE(qcfep.\`to\`)) AS timestamp FROM quality_control_flag_effective_periods AS qcfep INNER JOIN quality_control_flags AS qcf ON qcf.id = qcfep.flag_id @@ -62,7 +62,7 @@ CREATE OR REPLACE VIEW gaq_periods_timestamps AS NTH_VALUE(timestamp, 2) OVER ( PARTITION BY data_pass_id, run_number - ORDER BY ap.ord_timestamp + ORDER BY ap.ordering_timestamp ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING ) AS \`to\` FROM ( @@ -76,7 +76,7 @@ CREATE OR REPLACE VIEW gaq_periods_timestamps AS UNION ( ${SELECT_QCF_EFFECTIVE_PERIODS_TO_TIMESTAMPS_FOR_GAQ_PERIODS} ) - ORDER BY ord_timestamp + ORDER BY ordering_timestamp ) AS ap ) AS gaq_periods_with_last_nullish_row WHERE gaq_periods_with_last_nullish_row.\`to\` IS NOT NULL @@ -89,9 +89,6 @@ CREATE OR REPLACE VIEW gaq_periods AS SELECT gaq_periods_timestamps.data_pass_id AS dataPassId, gaq_periods_timestamps.run_number AS runNumber, - -- IF(gaq_periods_timestamps.\`from\` = 0, null, gaq_periods_timestamps.\`from\`) AS \`from\`, - -- IF(gaq_periods_timestamps.\`to\` = UNIX_TIMESTAMP(NOW(3)), null, gaq_periods_timestamps.\`to\`) AS \`to\`, - gaq_periods_timestamps.\`from\` AS \`from\`, gaq_periods_timestamps.\`to\` AS \`to\`, @@ -135,9 +132,6 @@ WHERE gaq_periods_timestamps.\`to\` IS NOT null GROUP BY gaq_periods_timestamps.data_pass_id, gaq_periods_timestamps.run_number, - -- IF(gaq_periods_timestamps.\`from\` = 0, null, gaq_periods_timestamps.\`from\`), - -- IF(gaq_periods_timestamps.\`to\` = UNIX_TIMESTAMP(NOW(3)), null, gaq_periods_timestamps.\`to\`) - gaq_periods_timestamps.\`from\`, gaq_periods_timestamps.\`to\` `; From 53f093aa306ff464ed1e4df088cfaf530d2d6afc Mon Sep 17 00:00:00 2001 From: xsalonx Date: Wed, 11 Dec 2024 16:13:40 +0100 Subject: [PATCH 14/16] a --- lib/database/repositories/QcFlagRepository.js | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/database/repositories/QcFlagRepository.js b/lib/database/repositories/QcFlagRepository.js index 3a4a252591..5e675c12db 100644 --- a/lib/database/repositories/QcFlagRepository.js +++ b/lib/database/repositories/QcFlagRepository.js @@ -104,10 +104,7 @@ class QcFlagRepository extends Repository { OR run.time_start IS NULL ), IF( - SUM( - COALESCE(gaq_periods.\`to\` , 0) - + COALESCE(gaq_periods.\`from\`, 0) - ) = 0, + gaq_periods.\`to\` IS NULL AND gaq_periods.\`from\` IS NULL, 1, null ), From dc33aa6c11755037aef5ba8e3844e79c92b0ea1d Mon Sep 17 00:00:00 2001 From: xsalonx Date: Wed, 11 Dec 2024 17:02:19 +0100 Subject: [PATCH 15/16] a --- .../migrations/20241127123000-create-gaq-views.js | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/database/migrations/20241127123000-create-gaq-views.js b/lib/database/migrations/20241127123000-create-gaq-views.js index cd95e513bf..b3424ad3be 100644 --- a/lib/database/migrations/20241127123000-create-gaq-views.js +++ b/lib/database/migrations/20241127123000-create-gaq-views.js @@ -1,7 +1,8 @@ 'use strict'; const SELECT_RUNS_FROM_TIMESTAMPS_FOR_GAQ_PERIODS = ` - SELECT gaqd.data_pass_id, + SELECT + gaqd.data_pass_id, gaqd.run_number, COALESCE(UNIX_TIMESTAMP(first_tf_timestamp), UNIX_TIMESTAMP(time_start), 0) AS ordering_timestamp, UNIX_TIMESTAMP(COALESCE(first_tf_timestamp, time_start)) AS timestamp @@ -11,7 +12,8 @@ const SELECT_RUNS_FROM_TIMESTAMPS_FOR_GAQ_PERIODS = ` `; const SELECT_RUNS_TO_TIMESTAMPS_FOR_GAQ_PERIODS = ` - SELECT gaqd.data_pass_id, + SELECT + gaqd.data_pass_id, gaqd.run_number, UNIX_TIMESTAMP(COALESCE(last_tf_timestamp, time_end, NOW(3))) AS ordering_timestamp, UNIX_TIMESTAMP(COALESCE(last_tf_timestamp, time_end)) AS timestamp @@ -21,7 +23,8 @@ const SELECT_RUNS_TO_TIMESTAMPS_FOR_GAQ_PERIODS = ` `; const SELECT_QCF_EFFECTIVE_PERIODS_FROM_TIMESTAMPS_FOR_GAQ_PERIODS = ` - SELECT gaqd.data_pass_id, + SELECT + gaqd.data_pass_id, gaqd.run_number, COALESCE(UNIX_TIMESTAMP(qcfep.\`from\`), 0) AS ordering_timestamp, UNIX_TIMESTAMP(qcfep.\`from\`) AS timestamp @@ -37,7 +40,8 @@ const SELECT_QCF_EFFECTIVE_PERIODS_FROM_TIMESTAMPS_FOR_GAQ_PERIODS = ` `; const SELECT_QCF_EFFECTIVE_PERIODS_TO_TIMESTAMPS_FOR_GAQ_PERIODS = ` - SELECT gaqd.data_pass_id, + SELECT + gaqd.data_pass_id, gaqd.run_number, UNIX_TIMESTAMP(COALESCE(qcfep.\`to\`, NOW(3))) AS ordering_timestamp, UNIX_TIMESTAMP(COALESCE(qcfep.\`to\`)) AS timestamp From 86cb1fc469db3427d15bc7061d56145129e78e47 Mon Sep 17 00:00:00 2001 From: xsalonx Date: Wed, 11 Dec 2024 17:05:54 +0100 Subject: [PATCH 16/16] use correct field --- .../migrations/20241127123000-create-gaq-views.js | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/database/migrations/20241127123000-create-gaq-views.js b/lib/database/migrations/20241127123000-create-gaq-views.js index b3424ad3be..f3c7162966 100644 --- a/lib/database/migrations/20241127123000-create-gaq-views.js +++ b/lib/database/migrations/20241127123000-create-gaq-views.js @@ -68,7 +68,13 @@ CREATE OR REPLACE VIEW gaq_periods_timestamps AS run_number ORDER BY ap.ordering_timestamp ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING - ) AS \`to\` + ) AS \`to\`, + NTH_VALUE(ordering_timestamp, 2) OVER ( + PARTITION BY data_pass_id, + run_number + ORDER BY ap.ordering_timestamp + ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING + ) AS \`to_ordering_timestamp\` FROM ( -- Two selects for runs' timestamps (in case QC flag's eff. period doesn't start at run's start or end at run's end ) ( ${SELECT_RUNS_FROM_TIMESTAMPS_FOR_GAQ_PERIODS} ) @@ -83,7 +89,7 @@ CREATE OR REPLACE VIEW gaq_periods_timestamps AS ORDER BY ordering_timestamp ) AS ap ) AS gaq_periods_with_last_nullish_row - WHERE gaq_periods_with_last_nullish_row.\`to\` IS NOT NULL + WHERE gaq_periods_with_last_nullish_row.\`to_ordering_timestamp\` IS NOT NULL `; const DROP_GAQ_PERIODS_TIMESTAMPS_VIEW = 'DROP VIEW gaq_periods_timestamps';