diff --git a/lib/application.js b/lib/application.js index 733fab3514..ee3ed8d0bd 100644 --- a/lib/application.js +++ b/lib/application.js @@ -15,12 +15,11 @@ const { webUiServer } = require('./server'); const { gRPCServer } = require('./server/index.js'); const { GRPCConfig, ServicesConfig } = require('./config'); -const { monalisa: monalisaConfig, ccdb: ccdbConfig } = ServicesConfig; +const { userCertificate, monalisa: monalisaConfig, ccdb: ccdbConfig, enableHousekeeping } = ServicesConfig; const { handleLostRunsAndEnvironments } = require('./server/services/housekeeping/handleLostRunsAndEnvironments.js'); const { isInTestMode } = require('./utilities/env-utils.js'); const { ScheduledProcessesManager } = require('./server/services/ScheduledProcessesManager.js'); const { MonAlisaSynchronizer } = require('./server/externalServicesSynchronization/monalisa/MonAlisaSynchronizer'); -const { createMonAlisaClient } = require('./server/externalServicesSynchronization/monalisa/MonAlisaClient'); const { LogManager } = require('@aliceo2/web-ui'); const { Kafka, logLevel } = require('kafkajs'); const { KafkaConfig } = require('./config/index.js'); @@ -28,6 +27,9 @@ const { AliEcsSynchronizer } = require('./server/kafka/AliEcsSynchronizer.js'); const { environmentService } = require('./server/services/environment/EnvironmentService.js'); const { runService } = require('./server/services/run/RunService.js'); const { CcdbSynchronizer } = require('./server/externalServicesSynchronization/ccdb/CcdbSynchronizer.js'); +const { promises: fs } = require('fs'); +const { MonAlisaClient } = require('./server/externalServicesSynchronization/monalisa/MonAlisaClient.js'); +const https = require('https'); /** * Bookkeeping Application @@ -78,15 +80,34 @@ class BookkeepingApplication { await this.aliEcsSynchronizer.start(); } - if (monalisaConfig.enableSynchronization) { - const monAlisaSynchronizer = await this.createMonAlisaSynchronizer(); - this.scheduledProcessesManager.schedule( - () => monAlisaSynchronizer.synchronize(), - { - wait: 10 * 1000, - every: monalisaConfig.synchronizationPeriod, - }, - ); + if (monalisaConfig.enableSynchronization || enableHousekeeping) { + const pfxCertificateBytes = await fs.readFile(userCertificate.path); + const certificatePassphrase = userCertificate.passphrase; + + const httpAgent = pfxCertificateBytes && certificatePassphrase + ? new https.Agent({ pfx: pfxCertificateBytes, passphrase: certificatePassphrase }) + : undefined; + + if (monalisaConfig.enableSynchronization) { + const monAlisaSynchronizer = await this.createMonAlisaSynchronizer(httpAgent); + this.scheduledProcessesManager.schedule( + () => monAlisaSynchronizer.synchronize(), + { + wait: 10 * 1000, + every: monalisaConfig.synchronizationPeriod, + }, + ); + } + + if (enableHousekeeping) { + this.scheduledProcessesManager.schedule( + () => this.housekeeping(httpAgent), + { + wait: 30 * 1000, + every: 30 * 1000, + }, + ); + } } if (ccdbConfig.enableSynchronization) { @@ -100,16 +121,6 @@ class BookkeepingApplication { }, ); } - - if (ServicesConfig.enableHousekeeping) { - this.scheduledProcessesManager.schedule( - () => this.housekeeping(), - { - wait: 30 * 1000, - every: 30 * 1000, - }, - ); - } } catch (error) { this._logger.errorMessage(`Error while starting: ${error}`); return this.stop(); @@ -121,27 +132,28 @@ class BookkeepingApplication { /** * Instantiate MonAlisa synchronizer with global configuration * + * @param {Agent} agent the HTTP agent to be used by synchronizer * @return {Promise} resolves with MonAlisaSynchronizer instance */ - async createMonAlisaSynchronizer() { - return new MonAlisaSynchronizer(await createMonAlisaClient({ + async createMonAlisaSynchronizer(agent) { + return new MonAlisaSynchronizer(new MonAlisaClient({ dataPassesUrl: monalisaConfig.dataPassesUrl, dataPassDetailsUrl: monalisaConfig.dataPassDetailsUrl, simulationPassesUrl: monalisaConfig.simulationPassesUrl, yearLowerLimit: monalisaConfig.dataPassesYearLowerLimit, - userCertificatePath: monalisaConfig.userCertificate.path, - certificatePassphrase: monalisaConfig.userCertificate.passphrase, + agent, })); } /** * Housekeeping method, it wraps @see handleLostRunsAndEnvironments and logs its results * + * @param {Agent} httpAgent agent to be used by the HTTP client * @return {Promise} promise */ - async housekeeping() { + async housekeeping(httpAgent) { try { - const { transitionedEnvironments, endedRuns } = await handleLostRunsAndEnvironments(); + const { transitionedEnvironments, endedRuns } = await handleLostRunsAndEnvironments(httpAgent); const subMessages = []; if (transitionedEnvironments.length > 0) { subMessages.push(`environments (${transitionedEnvironments.join(', ')})`); diff --git a/lib/config/services.js b/lib/config/services.js index 052f4a79b8..9e535881a9 100644 --- a/lib/config/services.js +++ b/lib/config/services.js @@ -12,8 +12,10 @@ */ const { - MONALISA_CERTIFICATE_PATH, - MONALISA_CERTIFICATE_PASSPHRASE, + USER_CERTIFICATE_PATH, + USER_CERTIFICATE_PASSPHRASE, + MONALISA_CERTIFICATE_PATH, // DEPRECATED + MONALISA_CERTIFICATE_PASSPHRASE, // DEPRECATED DATA_PASSES_YEAR_LOWER_LIMIT, MONALISA_DATA_PASSES_URL, MONALISA_DATA_PASS_DETAILS_URL, @@ -27,6 +29,10 @@ const { exports.services = { enableHousekeeping: process.env?.ENABLE_HOUSEKEEPING?.toLowerCase() === 'true', + userCertificate: { + path: USER_CERTIFICATE_PATH ?? MONALISA_CERTIFICATE_PATH, + passphrase: USER_CERTIFICATE_PASSPHRASE ?? MONALISA_CERTIFICATE_PASSPHRASE, + }, aliEcsGui: { url: process.env?.ALI_ECS_GUI_URL || null, token: process.env?.ALI_ECS_GUI_TOKEN || null, @@ -45,12 +51,7 @@ exports.services = { aliFlpIndex: { url: process.env?.ALI_FLP_INDEX_URL || null, }, - monalisa: { - userCertificate: { - path: MONALISA_CERTIFICATE_PATH, - passphrase: MONALISA_CERTIFICATE_PASSPHRASE, - }, dataPassesYearLowerLimit: Number(DATA_PASSES_YEAR_LOWER_LIMIT) || 2022, dataPassesUrl: MONALISA_DATA_PASSES_URL, diff --git a/lib/server/externalServicesSynchronization/monalisa/MonAlisaClient.js b/lib/server/externalServicesSynchronization/monalisa/MonAlisaClient.js index 4c528a0bb3..b0b5e0d554 100644 --- a/lib/server/externalServicesSynchronization/monalisa/MonAlisaClient.js +++ b/lib/server/externalServicesSynchronization/monalisa/MonAlisaClient.js @@ -13,9 +13,7 @@ * or submit itself to any jurisdiction. */ -const https = require('https'); const { extractLhcPeriod } = require('../../utilities/extractLhcPeriod'); -const fs = require('fs').promises; const VALID_DATA_PASS_NAME_REGEX = /^LHC\d\d[a-zA-Z]+_([a-z]pass|skimming|skimmed)[^\s]*$/; @@ -62,20 +60,17 @@ class MonAlisaClient { * @param {object} configuration configuration of MonAlisa client * @param {URL|string} [configuration.dataPassesUrl] url to fetch data passes * @param {URL|string} [configuration.dataPassDetailsUrl] url to fetch data pass version details - * @param {string|string[]|Buffer|Buffer[]|Object[]} [configuration.pfxCertificateBytes] PFX or PKCS12 encoded private key - * and certificate chain @see tls.createSecureContext([options]) - * @param {string} [configuration.certificatePassphrase] passphrase to the certificate + * @param {URL|string} [configuration.simulationPassesUrl] url to fetch simulation passes * @param {number} [configuration.yearLowerLimit] indicates how old data are accepted, * year of lhc period that given data pass belongs must be greater or equal - * @param {URL|string} [configuration.simulationPassesUrl] url to fetch simulation passes + * @param {Agent} agent HTTPs agent to use when sending requests */ constructor({ dataPassesUrl, dataPassDetailsUrl, - pfxCertificateBytes, - certificatePassphrase, - yearLowerLimit, simulationPassesUrl, + yearLowerLimit, + agent, } = {}) { this.dataPassesUrl = dataPassesUrl ? new URL(dataPassesUrl) : null; this.dataPassDetailsUrl = dataPassDetailsUrl ? new URL(dataPassDetailsUrl) : null; @@ -88,8 +83,7 @@ class MonAlisaClient { Accept: 'application/json;charset=utf-8', Connection: 'keep-alive', }, - agent: pfxCertificateBytes && certificatePassphrase - ? new https.Agent({ pfx: pfxCertificateBytes, passphrase: certificatePassphrase }) : undefined, + agent, }; } @@ -106,7 +100,7 @@ class MonAlisaClient { /** * Fetch data passes versions from MonAlisa - * @return {string} payload raw data acquired from MonAlisa: csv with data passes properties + * @return {Promise} payload raw data acquired from MonAlisa: csv with data passes properties * @private */ async _fetchDataPassesVersions() { @@ -283,30 +277,3 @@ class MonAlisaClient { } exports.MonAlisaClient = MonAlisaClient; - -/** - * Create a new instance of MonAlisaClient - * - * @param {object} configuration configuration of MonAlisa client - * @param {URL|string} [configuration.dataPassesUrl] url to fetch data passes - * @param {URL|string} [configuration.dataPassDetailsUrl] url to fetch data pass version details - * @param {number} [configuration.yearLowerLimit] indicates how old data are accepted, - * @param {string} [configuration.userCertificatePath] path to PKCS12 certificate - * @param {string} [configuration.certificatePassphrase] passphrase to the certificate - * @return {Promise} resolved with instance configured with environment variables - */ -exports.createMonAlisaClient = async ({ - dataPassesUrl, - dataPassDetailsUrl, - simulationPassesUrl, - yearLowerLimit, - userCertificatePath, - certificatePassphrase, -}) => new MonAlisaClient({ - dataPassesUrl, - dataPassDetailsUrl, - simulationPassesUrl, - yearLowerLimit, - pfxCertificateBytes: await fs.readFile(userCertificatePath), - certificatePassphrase, -}); diff --git a/lib/server/services/housekeeping/handleLostRunsAndEnvironments.js b/lib/server/services/housekeeping/handleLostRunsAndEnvironments.js index 31724c30b3..6b27794aa5 100644 --- a/lib/server/services/housekeeping/handleLostRunsAndEnvironments.js +++ b/lib/server/services/housekeeping/handleLostRunsAndEnvironments.js @@ -12,16 +12,21 @@ const MILLISECONDS_IN_ONE_DAY = 24 * 60 * 60 * 1000; * mark them as gone to error. For all runs that do not have timeO2Stop or timeTrgStop, check through AliECS GUI if they are still running, and * if not, mark them as stopped NOW * + * @param {Agent} [httpAgent] agent to be used when sending HTTP requests * @return {Promise<{transitionedEnvironments: number[], endedRuns: []}>} resolve with the list of environment ids and run numbers that were lost * @deprecated */ -exports.handleLostRunsAndEnvironments = async () => { +exports.handleLostRunsAndEnvironments = async (httpAgent) => { // TODO remove with node 18 const { default: fetch } = await import('node-fetch'); - const existingEnvironmentsResponse = await fetch(buildUrl( - `${ServicesConfig.aliEcsGui.url}/api/core/environments`, - { token: ServicesConfig.aliEcsGui.token }, - )); + + const existingEnvironmentsResponse = await fetch( + buildUrl( + `${ServicesConfig.aliEcsGui.url}/api/core/environments`, + { token: ServicesConfig.aliEcsGui.token }, + ), + { agent: httpAgent }, + ); if (existingEnvironmentsResponse.ok) { const { environments } = await existingEnvironmentsResponse.json();