Skip to content

Commit

Permalink
[O2B-1195] Don't transition runs & envs created less than 30s ago (#1465
Browse files Browse the repository at this point in the history
)

* [O2B-1195] Don't transition runs & envs created less than 30s ago

* Remove console.log
  • Loading branch information
martinboulais authored Mar 28, 2024
1 parent ea82633 commit d75cc4e
Show file tree
Hide file tree
Showing 3 changed files with 23 additions and 13 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -14,24 +14,25 @@
const { sequelize } = require('../../../database/index.js');
const { utilities: { TransactionHelper } } = require('../../../database');
const { EnvironmentHistoryItemRepository } = require('../../../database/repositories/index.js');
const { timestampToMysql } = require('../../utilities/timestampToMysql.js');

/**
* Consider all the environments that are currently active but not in the given list of ids to be lost, and transition them to error
*
* @param {string[]} activeEnvironmentIds the ids of active environments (those environments will not be transitioned to error)
* @param {number} modificationTimeWindow the time we have (in MINUTES), after the creation of an environment, to purge them. If an environment
* has been created more than this amount of minutes before NOW, it is NOT purged
* @param {Period} modificationTimePeriod environments created outside this time period will not be updated
* @return {Promise<number[]>} resolve once all inactive environments has been transitioned to error with their ids
* @deprecated
*/
exports.transitionToErrorLostEnvironments = async (activeEnvironmentIds, modificationTimeWindow) => {
exports.transitionToErrorLostEnvironments = async (activeEnvironmentIds, modificationTimePeriod) => {
// The select query is adapted for the insert
let selectQuery = `
SELECT e.id
FROM environments_history_items ehi
INNER JOIN environments e ON ehi.environment_id = e.id
GROUP BY e.id, e.updated_at
HAVING e.updated_at > date_sub(now(), INTERVAL ${modificationTimeWindow} MINUTE)
GROUP BY e.id, e.updated_at, e.created_at
HAVING e.updated_at >= '${timestampToMysql(modificationTimePeriod.from)}'
AND e.created_at < '${timestampToMysql(modificationTimePeriod.to)}'
AND group_concat(ehi.status) NOT LIKE '%ERROR%'
AND group_concat(ehi.status) NOT LIKE '%DESTROYED%'
`;
Expand Down
15 changes: 11 additions & 4 deletions lib/server/services/housekeeping/handleLostRunsAndEnvironments.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ const { transitionToErrorLostEnvironments } = require('../environment/transition
const { setO2StopOfLostRuns } = require('../run/setO2StopOfLostRuns.js');
const { ServicesConfig } = require('../../../config/index.js');

const MILLISECONDS_IN_ONE_DAY = 24 * 60 * 60 * 1000;

/**
* Handle lost environments and runs
*
Expand Down Expand Up @@ -33,11 +35,16 @@ exports.handleLostRunsAndEnvironments = async () => {
}
}

// Maximum amount of minutes after run start/environment creation to purge them
const modificationTime = 2880;
// Environments and runs created outside this time window (timestamps in ms) will not be updated
const modificationPeriod = {
// Don't update runs and environments lost more than 48 hours ago
from: Date.now() - MILLISECONDS_IN_ONE_DAY * 2,
// Don't update runs and environments created less than 30 seconds ago
to: Date.now() - 1000 * 30,
};

const transitionedEnvironments = await transitionToErrorLostEnvironments(environmentIdsToKeep, modificationTime);
const endedRuns = await setO2StopOfLostRuns(runNumbersToKeep, modificationTime);
const transitionedEnvironments = await transitionToErrorLostEnvironments(environmentIdsToKeep, modificationPeriod);
const endedRuns = await setO2StopOfLostRuns(runNumbersToKeep, modificationPeriod);

return { transitionedEnvironments, endedRuns };
} else {
Expand Down
10 changes: 6 additions & 4 deletions lib/server/services/run/setO2StopOfLostRuns.js
Original file line number Diff line number Diff line change
Expand Up @@ -12,28 +12,30 @@
*/

const { sequelize } = require('../../../database/index.js');
const { timestampToMysql } = require('../../utilities/timestampToMysql.js');

/**
* Consider all the runs that are still running but not in the given list of run numbers to be lost, and set their stop time to now
*
* @param {number[]} runNumbersOfRunningRuns the list of run numbers corresponding to runs that are actually running
* @param {number} modificationTimeWindow the time we have (in MINUTES), after the start of a run, to purge them. If a run started more than this
* amount of minutes before NOW, it is NOT purged
* @param {Period} modificationTimePeriod runs started outside this period will not be updated
* @return {Promise<number[]>} resolve once the lost runs has been marked as ended with the list of ended runs numbers
* @deprecated
*/
exports.setO2StopOfLostRuns = async (runNumbersOfRunningRuns, modificationTimeWindow) => {
exports.setO2StopOfLostRuns = async (runNumbersOfRunningRuns, modificationTimePeriod) => {
let fetchQuery = `
SELECT run_number
FROM runs
WHERE time_o2_end IS NULL
AND time_trg_end IS NULL
AND COALESCE(time_trg_start, time_o2_start) IS NOT NULL
AND COALESCE(time_trg_start, time_o2_start) > date_sub(now(), INTERVAL ${modificationTimeWindow} MINUTE)
AND COALESCE(time_trg_start, time_o2_start) >= '${timestampToMysql(modificationTimePeriod.from)}'
AND COALESCE(time_trg_start, time_o2_start) < '${timestampToMysql(modificationTimePeriod.to)}'
`;
if (runNumbersOfRunningRuns.length > 0) {
fetchQuery += ` AND run_number NOT IN (${runNumbersOfRunningRuns.join(',')})`;
}

const [runs] = await sequelize.query(fetchQuery, { raw: true });
const runNumbers = runs.map(({ run_number }) => run_number);

Expand Down

0 comments on commit d75cc4e

Please sign in to comment.