Skip to content

[O2B-1401] Store runs and QC flags timestamps with ms #1795

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
martinboulais merged 9 commits into from
Nov 27, 2024
Merged

[O2B-1401] Store runs and QC flags timestamps with ms #1795

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
2a89441
[O2B-1401] Store runs and QC flags timestamps with ms
martinboulais Nov 5, 2024
e0ec8d0
Merge branch 'main' into mboulais/O2B-1401/store-runs-and-qcflags-tim…
martinboulais Nov 6, 2024
9be843a
Make time_start and time_end to DATETIME(3) too
martinboulais Nov 7, 2024
1b40073
Fix services tests
martinboulais Nov 7, 2024
4b94bda
Fix more tests
martinboulais Nov 7, 2024
8cbb834
Fix linter
martinboulais Nov 7, 2024
86d5bbd
Merge branch 'main' into mboulais/O2B-1401/store-runs-and-qcflags-tim…
martinboulais Nov 25, 2024
cb5ad03
Merge branch 'main' into mboulais/O2B-1401/store-runs-and-qcflags-tim…
martinboulais Nov 25, 2024
033b79a
Merge branch 'main' into mboulais/O2B-1401/store-runs-and-qcflags-tim…
martinboulais Nov 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
266 changes: 266 additions & 0 deletions lib/database/migrations/20241105090940-store-run-and-qcflags-timestamps-in-ms.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,266 @@
'use strict';

const TIME_START_END_TO_DATETIME_3 = `
ALTER TABLE runs
CHANGE COLUMN time_start time_start DATETIME(3) AS (COALESCE(time_trg_start, time_trg_end)) VIRTUAL,
CHANGE COLUMN time_end time_end DATETIME(3) AS (COALESCE(time_trg_end, time_o2_end)) VIRTUAL;
`;

const TIME_START_END_TO_DATETIME = `
ALTER TABLE runs
CHANGE COLUMN time_start time_start DATETIME AS (COALESCE(time_trg_start, time_trg_end)) VIRTUAL,
CHANGE COLUMN time_end time_end DATETIME AS (COALESCE(time_trg_end, time_o2_end)) VIRTUAL;
`;

/** @type {import('sequelize-cli').Migration} */
module.exports = {
up: async (queryInterface, Sequelize) => queryInterface.sequelize.transaction(async (transaction) => {
await queryInterface.changeColumn(
'runs',
'time_o2_start',
{
type: Sequelize.DATE(3),
},
{ transaction },
);
await queryInterface.changeColumn(
'runs',
'time_trg_start',
{
type: Sequelize.DATE(3),
},
{ transaction },
);
await queryInterface.changeColumn(
'runs',
'first_tf_timestamp',
{
type: Sequelize.DATE(3),
},
{ transaction },
);
await queryInterface.changeColumn(
'runs',
'last_tf_timestamp',
{
type: Sequelize.DATE(3),
},
{ transaction },
);
await queryInterface.changeColumn(
'runs',
'time_trg_end',
{
type: Sequelize.DATE(3),
},
{ transaction },
);
await queryInterface.changeColumn(
'runs',
'time_o2_end',
{
type: Sequelize.DATE(3),
},
{ transaction },
);
await queryInterface.changeColumn(
'quality_control_flags',
'from',
{
type: Sequelize.DATE(3),
},
{ transaction },
);
await queryInterface.changeColumn(
'quality_control_flags',
'to',
{
type: Sequelize.DATE(3),
},
{ transaction },
);
await queryInterface.changeColumn(
'quality_control_flags',
'created_at',
{
type: Sequelize.DATE(3),
allowNull: false,
defaultValue: Sequelize.literal('CURRENT_TIMESTAMP'),
},
{ transaction },
);
await queryInterface.changeColumn(
'quality_control_flags',
'updated_at',
{
type: Sequelize.DATE(3),
allowNull: false,
defaultValue: Sequelize.literal('CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP'),
},
{ transaction },
);
await queryInterface.changeColumn(
'quality_control_flag_effective_periods',
'from',
{
type: Sequelize.DATE(3),
},
{ transaction },
);
await queryInterface.changeColumn(
'quality_control_flag_effective_periods',
'to',
{
type: Sequelize.DATE(3),
},
{ transaction },
);
await queryInterface.changeColumn(
'quality_control_flag_effective_periods',
'created_at',
{
type: Sequelize.DATE(3),
allowNull: false,
defaultValue: Sequelize.literal('CURRENT_TIMESTAMP'),
},
{ transaction },
);
await queryInterface.changeColumn(
'quality_control_flag_effective_periods',
'updated_at',
{
type: Sequelize.DATE(3),
allowNull: false,
defaultValue: Sequelize.literal('CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP'),
},
{ transaction },
);

await queryInterface.sequelize.query(TIME_START_END_TO_DATETIME_3, { transaction });
}),

down: async (queryInterface, Sequelize) => queryInterface.sequelize.transaction(async (transaction) => {
await queryInterface.changeColumn(
'runs',
'time_o2_start',
{
type: Sequelize.DATE,
},
{ transaction },
);
await queryInterface.changeColumn(
'runs',
'time_trg_start',
{
type: Sequelize.DATE,
},
{ transaction },
);
await queryInterface.changeColumn(
'runs',
'first_tf_timestamp',
{
type: Sequelize.DATE,
},
{ transaction },
);
await queryInterface.changeColumn(
'runs',
'last_tf_timestamp',
{
type: Sequelize.DATE,
},
{ transaction },
);
await queryInterface.changeColumn(
'runs',
'time_trg_end',
{
type: Sequelize.DATE,
},
{ transaction },
);
await queryInterface.changeColumn(
'runs',
'time_o2_end',
{
type: Sequelize.DATE,
},
{ transaction },
);
await queryInterface.changeColumn(
'quality_control_flags',
'from',
{
type: Sequelize.DATE,
},
{ transaction },
);
await queryInterface.changeColumn(
'quality_control_flags',
'to',
{
type: Sequelize.DATE,
},
{ transaction },
);
await queryInterface.changeColumn(
'quality_control_flags',
'created_at',
{
type: Sequelize.DATE,
allowNull: false,
defaultValue: Sequelize.literal('CURRENT_TIMESTAMP'),
},
{ transaction },
);
await queryInterface.changeColumn(
'quality_control_flags',
'updated_at',
{
type: Sequelize.DATE,
allowNull: false,
defaultValue: Sequelize.literal('CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP'),
},
{ transaction },
);
await queryInterface.changeColumn(
'quality_control_flag_effective_periods',
'from',
{
type: Sequelize.DATE,
},
{ transaction },
);
await queryInterface.changeColumn(
'quality_control_flag_effective_periods',
'to',
{
type: Sequelize.DATE,
},
{ transaction },
);
await queryInterface.changeColumn(
'quality_control_flag_effective_periods',
'created_at',
{
type: Sequelize.DATE,
allowNull: false,
defaultValue: Sequelize.literal('CURRENT_TIMESTAMP'),
},
{ transaction },
);
await queryInterface.changeColumn(
'quality_control_flag_effective_periods',
'updated_at',
{
type: Sequelize.DATE,
allowNull: false,
defaultValue: Sequelize.literal('CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP'),
},
{ transaction },
);

await queryInterface.sequelize.query(TIME_START_END_TO_DATETIME, { transaction });
}),
};
4 changes: 2 additions & 2 deletions lib/database/repositories/EnvironmentHistoryItemRepository.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,8 @@ const getHistoryDistributionQuery = ({ from, to }) => `
FROM environments e
INNER JOIN environments_history_items AS ehi
ON e.id = ehi.environment_id
WHERE e.updated_at >= '${timestampToMysql(from)}'
AND e.created_at < '${timestampToMysql(to)}'
WHERE e.updated_at >= '${timestampToMysql(from, true)}'
AND e.created_at < '${timestampToMysql(to, true)}'
GROUP BY e.id
HAVING FIND_IN_SET('DESTROYED', statusHistory) > 0 OR FIND_IN_SET('ERROR', statusHistory) > 0
) AS statusHistoryByEnvironmentId
Expand Down
4 changes: 2 additions & 2 deletions lib/server/services/run/setO2StopOfLostRuns.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,8 @@ exports.setO2StopOfLostRuns = async (runNumbersOfRunningRuns, modificationTimePe
WHERE time_o2_end IS NULL
AND time_trg_end IS NULL
AND COALESCE(time_trg_start, time_o2_start) IS NOT NULL
AND COALESCE(time_trg_start, time_o2_start) >= '${timestampToMysql(modificationTimePeriod.from)}'
AND COALESCE(time_trg_start, time_o2_start) < '${timestampToMysql(modificationTimePeriod.to)}'
AND COALESCE(time_trg_start, time_o2_start) >= '${timestampToMysql(modificationTimePeriod.from, true)}'
AND COALESCE(time_trg_start, time_o2_start) < '${timestampToMysql(modificationTimePeriod.to, true)}'
`;
if (runNumbersOfRunningRuns.length > 0) {
fetchQuery += ` AND run_number NOT IN (${runNumbersOfRunningRuns.join(',')})`;
Expand Down
7 changes: 6 additions & 1 deletion lib/server/utilities/timestampToMysql.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,11 @@
* Convert a UNIX timestamp (in ms) to a MySQL date expression
*
* @param {number} timestamp the timestamp to convert
* @param {boolean} [milliseconds=false] if true, milliseconds will be stored in database
* @return {string} the resulting SQL date
*/
exports.timestampToMysql = (timestamp) => new Date(timestamp).toISOString().slice(0, 19).replace('T', ' ');
exports.timestampToMysql = (timestamp, milliseconds = false) => {
let dateIso = new Date(timestamp).toISOString();
dateIso = dateIso.slice(0, milliseconds ? 23 : 19);
return dateIso.replace('T', ' ');
};
12 changes: 6 additions & 6 deletions test/api/qcFlags.test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ module.exports = () => {
1: {
missingVerificationsCount: 3,
mcReproducible: true,
badEffectiveRunCoverage: 0.3333,
badEffectiveRunCoverage: 0.3333333,
explicitlyNotBadEffectiveRunCoverage: 0,
},
16: {
Expand All @@ -99,8 +99,8 @@ module.exports = () => {
1: {
missingVerificationsCount: 3,
mcReproducible: true,
badEffectiveRunCoverage: 0.1111,
explicitlyNotBadEffectiveRunCoverage: 0.2222,
badEffectiveRunCoverage: 0.1111111,
explicitlyNotBadEffectiveRunCoverage: 0.2222222,
},
16: {
badEffectiveRunCoverage: 0,
Expand All @@ -121,7 +121,7 @@ module.exports = () => {
1: {
missingVerificationsCount: 1,
mcReproducible: false,
badEffectiveRunCoverage: 0.7222,
badEffectiveRunCoverage: 0.7222222,
explicitlyNotBadEffectiveRunCoverage: 0,
},
},
Expand All @@ -138,8 +138,8 @@ module.exports = () => {
7: {
missingVerificationsCount: 1,
mcReproducible: false,
badEffectiveRunCoverage: 0.1667,
explicitlyNotBadEffectiveRunCoverage: 0.8333,
badEffectiveRunCoverage: 0.1666667,
explicitlyNotBadEffectiveRunCoverage: 0.8333333,
},

// ITS
Expand Down
16 changes: 8 additions & 8 deletions test/lib/server/services/qualityControlFlag/QcFlagService.test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,7 @@ module.exports = () => {
1: {
missingVerificationsCount: 3,
mcReproducible: true,
badEffectiveRunCoverage: 0.3333,
badEffectiveRunCoverage: 0.3333333,
explicitlyNotBadEffectiveRunCoverage: 0,
},
16: {
Expand All @@ -169,8 +169,8 @@ module.exports = () => {
1: {
missingVerificationsCount: 3,
mcReproducible: true,
badEffectiveRunCoverage: 0.1111,
explicitlyNotBadEffectiveRunCoverage: 0.2222,
badEffectiveRunCoverage: 0.1111111,
explicitlyNotBadEffectiveRunCoverage: 0.2222222,
},
16: {
badEffectiveRunCoverage: 0,
Expand Down Expand Up @@ -213,7 +213,7 @@ module.exports = () => {
1: {
missingVerificationsCount: 0,
mcReproducible: false,
badEffectiveRunCoverage: 0.0769,
badEffectiveRunCoverage: 0.0769231,
explicitlyNotBadEffectiveRunCoverage: 0,
},
},
Expand All @@ -230,7 +230,7 @@ module.exports = () => {
1: {
missingVerificationsCount: 0,
mcReproducible: false,
badEffectiveRunCoverage: 0.0769,
badEffectiveRunCoverage: 0.0769231,
explicitlyNotBadEffectiveRunCoverage: 0,
},
},
Expand All @@ -247,7 +247,7 @@ module.exports = () => {
1: {
missingVerificationsCount: 1,
mcReproducible: false,
badEffectiveRunCoverage: 0.7222,
badEffectiveRunCoverage: 0.7222222,
explicitlyNotBadEffectiveRunCoverage: 0,
},
},
Expand All @@ -265,8 +265,8 @@ module.exports = () => {
7: {
missingVerificationsCount: 1,
mcReproducible: false,
badEffectiveRunCoverage: 0.1667,
explicitlyNotBadEffectiveRunCoverage: 0.8333,
badEffectiveRunCoverage: 0.1666667,
explicitlyNotBadEffectiveRunCoverage: 0.8333333,
},

// ITS
Expand Down
Loading