From b9a277c87604602c2437c939a5a52091a5aad1cb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 3 Dec 2024 01:23:19 +0000 Subject: [PATCH 01/12] Bump mongoose from 5.13.20 to 8.8.3 Bumps [mongoose](https://github.com/Automattic/mongoose) from 5.13.20 to 8.8.3. - [Release notes](https://github.com/Automattic/mongoose/releases) - [Changelog](https://github.com/Automattic/mongoose/blob/master/CHANGELOG.md) - [Commits](https://github.com/Automattic/mongoose/compare/5.13.20...8.8.3) --- updated-dependencies: - dependency-name: mongoose dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index bb68e73..9fac14a 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,7 @@ "express": "4.20.0", "iotagent-node-lib": "https://github.com/telefonicaid/iotagent-node-lib.git#master", "logops": "2.1.2", - "mongoose": "5.13.20", + "mongoose": "8.8.3", "revalidator": "~0.3.1", "underscore": "~1.12.1" }, From 7b0f2ea0bfecf0b01255c96b644d49f504a4adf7 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Wed, 4 Dec 2024 11:21:31 +0100 Subject: [PATCH 02/12] use mongoose 8 instead of 5 --- lib/model/dbConn.js | 190 +++++++++--------------------- lib/services/configurationData.js | 57 ++++++--- lib/services/protocolData.js | 113 +++++++++--------- lib/services/protocols.js | 28 ++++- 4 files changed, 176 insertions(+), 212 deletions(-) diff --git a/lib/model/dbConn.js b/lib/model/dbConn.js index 08e2963..8fe016e 100644 --- a/lib/model/dbConn.js +++ b/lib/model/dbConn.js @@ -29,7 +29,6 @@ const mongoose = require('mongoose'); const config = require('../utils/commonConfig'); const constants = require('../utils/constants'); -const async = require('async'); const errors = require('../errors'); let defaultDb; const DEFAULT_DB_NAME = 'iotagent-manager'; @@ -41,167 +40,84 @@ function loadModels() { require('./Configuration').load(defaultDb); } +// Delay function for nodejs16 +function delay(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + /** * Creates a new connection to the Mongo DB. * * @this Reference to the dbConn module itself. */ -function init(logger, host, db, port, username, password, options, callback) { - let url; +async function init(logger, host, db, port, username, password, options) { let credentials = ''; - let retries = 0; - let lastError; - const maxRetries = - (config.getConfig().mongodb && config.getConfig().mongodb.retries) || constants.DEFAULT_MONGODB_RETRIES; - if (username && password) { - credentials = username + ':' + password + '@'; - } - - function addPort(item) { - return item + ':' + port; + credentials = `${username}:${password}@`; } - function commaConcat(previous, current, currentIndex) { - if (currentIndex !== 0) { - previous += ','; - } - - previous += current; - - return previous; - } - - const hosts = host.split(',').map(addPort).reduce(commaConcat, ''); - - url = 'mongodb://' + credentials + hosts + '/' + db; + const hosts = host + .split(',') + .map((item) => `${item}:${port}`) + .join(','); + let url = `mongodb://${credentials}${hosts}/${db}`; if (options.replicaSet) { - url += '?replicaSet=' + options.replicaSet.rs_name; + url += `?replicaSet=${options.replicaSet.rs_name}`; } - /* eslint-disable-next-line no-unused-vars */ - function createConnectionHandler(error, results) { - if (defaultDb) { - logger.info('Successfully connected to MongoDB.'); - module.exports.db = defaultDb; - loadModels(); - } else { - logger.error('MONGODB-002: Error found after [%d] attempts: %s', retries, error || lastError); - } + const maxRetries = config.getConfig().mongodb?.retries || constants.DEFAULT_MONGODB_RETRIES; + const retryTime = config.getConfig().mongodb?.retryTime || constants.DEFAULT_MONGODB_RETRY_TIME; - callback(error); - } + for (let attempt = 1; attempt <= maxRetries; attempt++) { + try { + logger.info(`Attempt ${attempt}: Connecting to MongoDB at ${url}`); + defaultDb = await mongoose.connect(url, options); + logger.info('Successfully connected to MongoDB.'); - function retryCheck() { - return !defaultDb && retries < maxRetries; - } + // Register events + mongoose.connection.on('error', (err) => logger.error('Mongo Driver error:', err)); + mongoose.connection.on('connected', () => logger.debug('Mongo Driver connected')); + mongoose.connection.on('disconnected', () => logger.debug('Mongo Driver disconnected')); + mongoose.connection.on('reconnectFailed', () => { + logger.error('MONGODB-004: MongoDB connection was lost'); + process.exit(1); + }); - function connectionAttempt(url, options, callback) { - logger.info( - 'Attempting to connect to MongoDB instance with url %j and options %j. Attempt %d', - url, - options, - retries - ); - // just to avoid warnings with recent mongoose versions - options.useNewUrlParser = true; - options.useUnifiedTopology = true; - mongoose.set('useCreateIndex', true); - /* eslint-disable-next-line no-unused-vars */ - const candidateDb = mongoose.createConnection(url, options, function (error, result) { - if (error) { - logger.error('MONGODB-001: Error trying to connect to MongoDB: %s', error); - lastError = error; + loadModels(); + break; // End loop if connection was OK + } catch (error) { + logger.error(`Attempt ${attempt} failed: ${error}`); + if (attempt < maxRetries) { + logger.info(`Retrying in ${retryTime} seconds...`); + await delay(retryTime * 1000); } else { - defaultDb = candidateDb; - - defaultDb.on('error', function (error) { - logger.error('Mongo Driver error: %j', error); - }); - /* eslint-disable-next-line no-unused-vars */ - defaultDb.on('connecting', function (error) { - logger.debug('Mongo Driver connecting'); - }); - defaultDb.on('connected', function () { - logger.debug('Mongo Driver connected'); - }); - defaultDb.on('reconnected', function () { - logger.debug('Mongo Driver reconnected'); - }); - defaultDb.on('disconnected', function () { - logger.debug('Mongo Driver disconnected'); - }); - defaultDb.on('reconnectFailed', function () { - logger.error('MONGODB-004: MongoDB connection was lost'); - process.exit(1); - }); - defaultDb.on('disconnecting', function () { - logger.debug('Mongo Driver disconnecting'); - }); - defaultDb.on('open', function () { - logger.debug('Mongo Driver open'); - }); - defaultDb.on('close', function () { - logger.debug('Mongo Driver close'); - }); + throw new Error(`MONGODB-002: Connection failed after ${maxRetries} attempts.`); } - - callback(); - }); - } - - function tryCreateConnection(callback) { - const attempt = async.apply(connectionAttempt, url, options, callback); - const seconds = - (config.getConfig().mongodb && config.getConfig().mongodb.retryTime) || - constants.DEFAULT_MONGODB_RETRY_TIME; - - retries++; - - if (retries === 1) { - logger.info('First connection attempt'); - attempt(); - } else { - logger.info('Waiting %d seconds before attempting again.', seconds); - setTimeout(attempt, seconds * 1000); } } - - defaultDb = null; - async.whilst(retryCheck, tryCreateConnection, createConnectionHandler); } -function configureDb(logger, callback) { +async function configureDb(logger) { const currentConfig = config.getConfig(); - - if (!currentConfig.mongodb || !currentConfig.mongodb.host) { + if (!currentConfig.mongodb?.host) { logger.fatal('No host found for MongoDB driver.'); - callback(new errors.BadConfiguration('No host found for MongoDB driver')); - } else { - let dbName = currentConfig.mongodb.db; - const port = currentConfig.mongodb.port || 27017; - const options = {}; - - if (!currentConfig.mongodb.db) { - dbName = DEFAULT_DB_NAME; - } - - if (currentConfig.mongodb.replicaSet) { - options.replset = { rs_name: currentConfig.mongodb.replicaSet }; - } - - init( - logger, - config.getConfig().mongodb.host, - dbName, - port, - currentConfig.username, - currentConfig.password, - options, - callback - ); + throw new errors.BadConfiguration('No host found for MongoDB driver'); } + + const dbName = currentConfig.mongodb.db || DEFAULT_DB_NAME; + const port = currentConfig.mongodb.port || 27017; + const options = currentConfig.mongodb.replicaSet ? { replicaSet: currentConfig.mongodb.replicaSet } : {}; + + await init( + logger, + currentConfig.mongodb.host, + dbName, + port, + currentConfig.username, + currentConfig.password, + options + ); } exports.configureDb = configureDb; diff --git a/lib/services/configurationData.js b/lib/services/configurationData.js index d9173f3..7d083dd 100644 --- a/lib/services/configurationData.js +++ b/lib/services/configurationData.js @@ -70,7 +70,7 @@ function createGetWithFields(fields) { logger.debug('Looking for configuration with params %j and query %j', fields, queryObj); const query = Configuration.model.find(queryObj); - + const query2 = Configuration.model.countDocuments(queryObj); if (limit) { query.limit(parseInt(limit, 10)); } @@ -79,21 +79,39 @@ function createGetWithFields(fields) { query.skip(parseInt(offset, 10)); } - async.series( - [query.exec.bind(query), Configuration.model.countDocuments.bind(Configuration.model, queryObj)], - function (error, results) { - if (!error && results && results.length === 2) { - callback(null, { - services: results[0], - count: results[1] - }); - } else if (error) { - callback(new errors.InternalDbError(error)); - } else { - callback(new errors.DeviceGroupNotFound(fields, arguments)); - } + function func1(callback) { + query + .exec({}) + .then((res) => { + callback(null, res); + }) + .catch((error) => { + callback(error); + }); + } + function func2(callback) { + query2 + .exec({}) + .then((res) => { + callback(null, res); + }) + .catch((error) => { + callback(error); + }); + } + + async.series([func1, func2], function (error, results) { + if (!error && results && results.length === 2) { + callback(null, { + services: results[0], + count: results[1] + }); + } else if (error) { + callback(new errors.InternalDbError(error)); + } else { + callback(new errors.DeviceGroupNotFound(fields, arguments)); } - ); + }); }; } @@ -143,7 +161,14 @@ function save(theLogger, protocol, description, iotagent, resource, configuratio } } theLogger.debug('Saving Configuration %j translated to %j ', configuration, configurationObj); - configurationObj.save(callback); + configurationObj + .save({}) + .then((res) => { + callback(null, res); + }) + .catch((error) => { + callback(error); + }); } exports.get = iotagentLib.alarms.intercept( diff --git a/lib/services/protocolData.js b/lib/services/protocolData.js index 0c69742..05b7609 100644 --- a/lib/services/protocolData.js +++ b/lib/services/protocolData.js @@ -51,30 +51,27 @@ function processConfiguration(logger, protocol, description, iotagent, resource, } function cleanConfigurations(logger, protocol, iotagent, resource, callback) { - Configuration.model.deleteMany( - { - protocol, - iotagent, - resource - }, - /* eslint-disable-next-line no-unused-vars */ - function (error, commandResult) { - if (error) { - logger.error('MONGODB-003: Internal MongoDB Error removing services from protocol: %s', error); - - callback(new errors.InternalDbError(error)); - } else { - logger.debug( - 'Configurations for Protocol [%s][%s][%s] successfully removed.', - protocol, - iotagent, - resource - ); - - callback(null); - } - } - ); + const queryObj = { + protocol, + iotagent, + resource + }; + const query = Configuration.model.deleteMany(queryObj); + query + .exec({}) + .then((commandResult) => { + logger.debug( + 'Configurations for Protocol [%s][%s][%s] successfully removed.', + protocol, + iotagent, + resource + ); + callback(null); + }) + .catch((error) => { + logger.error('MONGODB-003: Internal MongoDB Error removing services from protocol: %s', error); + callback(new errors.InternalDbError(error)); + }); } function getProtocol(protocol, callback) { @@ -82,35 +79,35 @@ function getProtocol(protocol, callback) { protocol }; const query = Protocol.model.find(condition).sort(); - - query.exec(function (error, protocolFound) { - if (!error && protocolFound && protocolFound.length === 1) { - callback(null, protocolFound[0]); - } else if (error) { + query + .exec({}) + .then((protocolFound) => { + if (protocolFound && protocolFound.length === 1) { + callback(null, protocolFound[0]); + } else { + const resource = 'n/a'; + callback(new errors.ProtocolNotFound(resource, protocol)); + } + }) + .catch((error) => { callback(error); - } else { - const resource = 'n/a'; - callback(new errors.ProtocolNotFound(resource, protocol)); - } - }); + }); } function listProtocol(callback) { const condition = {}; - function toObject(o) { return o.toObject(); } - const query = Protocol.model.find(condition).sort(); - - query.exec(function (error, protocols) { - if (error) { - callback(error); - } else { + query + .exec({}) + .then((protocols) => { callback(null, protocols.map(toObject)); - } - }); + }) + .catch((error) => { + callback(error); + }); } function save(logger, newProtocol, callback) { @@ -147,9 +144,17 @@ function save(logger, newProtocol, callback) { ) ); } - - actions.push(protocolObj.save.bind(protocolObj)); - + function func1(callback) { + protocolObj + .save({}) + .then((res) => { + callback(null, res); + }) + .catch((error) => { + callback(error); + }); + } + actions.push(func1); async.series(actions, callback); } }); @@ -166,15 +171,17 @@ function removeProtocol(logger, id, callback) { if (error) { callback(error); } else { - Protocol.model.deleteOne(condition, function (error, results) { - if (error) { - logger.debug('Internal MongoDB Error getting device: %s', error); - callback(new errors.InternalDbError(error)); - } else { + const query = Protocol.model.deleteOne(condition); + query + .exec({}) + .then((results) => { logger.debug('Protocol [%s] successfully removed with results [%j]', id, results); callback(null); - } - }); + }) + .catch((error) => { + logger.debug('Internal MongoDB Error getting device: %s', error); + callback(new errors.InternalDbError(error)); + }); } }); } diff --git a/lib/services/protocols.js b/lib/services/protocols.js index c447ca3..053b37d 100644 --- a/lib/services/protocols.js +++ b/lib/services/protocols.js @@ -34,7 +34,7 @@ function readProtocolList(req, res, next) { logger.debug('List protocols request with args %j', JSON.stringify(req.query)); const query = Protocol.model.find(condition).sort(); - + const query2 = Protocol.model.countDocuments(condition); if (req.query && req.query.limit) { query.limit(parseInt(req.query.limit, 10)); } @@ -42,11 +42,27 @@ function readProtocolList(req, res, next) { if (req.query && req.query.offset) { query.skip(parseInt(req.query.offset, 10)); } - - async.series([query.exec.bind(query), Protocol.model.countDocuments.bind(Protocol.model, condition)], function ( - error, - results - ) { + function func1(callback) { + query + .exec({}) + .then((res) => { + callback(null, res); + }) + .catch((error) => { + callback(error); + }); + } + function func2(callback) { + query2 + .exec({}) + .then((res) => { + callback(null, res); + }) + .catch((error) => { + callback(error); + }); + } + async.series([func1, func2], function (error, results) { if (error) { logger.error('error: %j', error); next(error); From 26d9db6624dedd7e6f37bd7d799855fb8c30b7ae Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Wed, 4 Dec 2024 12:43:30 +0100 Subject: [PATCH 03/12] fix linter --- lib/model/dbConn.js | 3 ++- lib/services/protocolData.js | 7 +++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/model/dbConn.js b/lib/model/dbConn.js index 8fe016e..925e39c 100644 --- a/lib/model/dbConn.js +++ b/lib/model/dbConn.js @@ -68,7 +68,7 @@ async function init(logger, host, db, port, username, password, options) { const maxRetries = config.getConfig().mongodb?.retries || constants.DEFAULT_MONGODB_RETRIES; const retryTime = config.getConfig().mongodb?.retryTime || constants.DEFAULT_MONGODB_RETRY_TIME; - + /* eslint-disable no-await-in-loop */ for (let attempt = 1; attempt <= maxRetries; attempt++) { try { logger.info(`Attempt ${attempt}: Connecting to MongoDB at ${url}`); @@ -96,6 +96,7 @@ async function init(logger, host, db, port, username, password, options) { } } } + /* eslint-enable no-await-in-loop */ } async function configureDb(logger) { diff --git a/lib/services/protocolData.js b/lib/services/protocolData.js index 05b7609..d30b518 100644 --- a/lib/services/protocolData.js +++ b/lib/services/protocolData.js @@ -61,10 +61,11 @@ function cleanConfigurations(logger, protocol, iotagent, resource, callback) { .exec({}) .then((commandResult) => { logger.debug( - 'Configurations for Protocol [%s][%s][%s] successfully removed.', + 'Configurations for Protocol [%s][%s][%s] successfully removed %s', protocol, iotagent, - resource + resource, + commandResult ); callback(null); }) @@ -144,6 +145,7 @@ function save(logger, newProtocol, callback) { ) ); } + /* eslint-disable no-inner-declarations */ function func1(callback) { protocolObj .save({}) @@ -154,6 +156,7 @@ function save(logger, newProtocol, callback) { callback(error); }); } + /* eslint-enable no-inner-declarations */ actions.push(func1); async.series(actions, callback); } From a611f16db75843b1e4db46309e9e916155460a6f Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Thu, 5 Dec 2024 15:57:17 +0100 Subject: [PATCH 04/12] remove conn --- lib/model/Configuration.js | 4 ++-- lib/model/Protocol.js | 4 ++-- lib/model/dbConn.js | 8 +++----- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/lib/model/Configuration.js b/lib/model/Configuration.js index c639a84..2e64842 100644 --- a/lib/model/Configuration.js +++ b/lib/model/Configuration.js @@ -69,9 +69,9 @@ const Configuration = new Schema({ transport: String }); -function load(db) { +function load() { Configuration.index({ apikey: 1, resource: 1, protocol: 1 }, { unique: true }); - module.exports.model = db.model('Configuration', Configuration); + module.exports.model = mongoose.model('Configuration', Configuration); module.exports.internalSchema = Configuration; } diff --git a/lib/model/Protocol.js b/lib/model/Protocol.js index 11e0d20..a5bad2b 100644 --- a/lib/model/Protocol.js +++ b/lib/model/Protocol.js @@ -31,9 +31,9 @@ const Protocol = new Schema({ description: String }); -function load(db) { +function load() { Protocol.index({ protocol: 1, resource: 1 }, { unique: true }); - module.exports.model = db.model('Protocol', Protocol); + module.exports.model = mongoose.model('Protocol', Protocol); module.exports.internalSchema = Protocol; } diff --git a/lib/model/dbConn.js b/lib/model/dbConn.js index 925e39c..66fc71f 100644 --- a/lib/model/dbConn.js +++ b/lib/model/dbConn.js @@ -30,14 +30,13 @@ const mongoose = require('mongoose'); const config = require('../utils/commonConfig'); const constants = require('../utils/constants'); const errors = require('../errors'); -let defaultDb; const DEFAULT_DB_NAME = 'iotagent-manager'; mongoose.Promise = global.Promise; // not including this causes DeprecationWarning function loadModels() { - require('./Protocol').load(defaultDb); - require('./Configuration').load(defaultDb); + require('./Protocol').load(); + require('./Configuration').load(); } // Delay function for nodejs16 @@ -72,7 +71,7 @@ async function init(logger, host, db, port, username, password, options) { for (let attempt = 1; attempt <= maxRetries; attempt++) { try { logger.info(`Attempt ${attempt}: Connecting to MongoDB at ${url}`); - defaultDb = await mongoose.connect(url, options); + await mongoose.connect(url, options); logger.info('Successfully connected to MongoDB.'); // Register events @@ -122,5 +121,4 @@ async function configureDb(logger) { } exports.configureDb = configureDb; -exports.db = defaultDb; exports.DEFAULT_DB_NAME = DEFAULT_DB_NAME; From 5ab3b5ff7e6554438b77cafec8fe0fc0fa600306 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Mon, 9 Dec 2024 13:12:47 +0100 Subject: [PATCH 05/12] upgrade to 8.8.4 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 9fac14a..2cdd52b 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,7 @@ "express": "4.20.0", "iotagent-node-lib": "https://github.com/telefonicaid/iotagent-node-lib.git#master", "logops": "2.1.2", - "mongoose": "8.8.3", + "mongoose": "8.8.4", "revalidator": "~0.3.1", "underscore": "~1.12.1" }, From 26a2aebda19ff4675c777a18d6f59ee5d40ebf15 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Tue, 10 Dec 2024 09:36:38 +0100 Subject: [PATCH 06/12] rename sub callbacks --- lib/services/configurationData.js | 18 +++++++++--------- lib/services/protocolData.js | 8 ++++---- lib/services/protocols.js | 10 +++++----- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/lib/services/configurationData.js b/lib/services/configurationData.js index 7d083dd..639705c 100644 --- a/lib/services/configurationData.js +++ b/lib/services/configurationData.js @@ -70,7 +70,7 @@ function createGetWithFields(fields) { logger.debug('Looking for configuration with params %j and query %j', fields, queryObj); const query = Configuration.model.find(queryObj); - const query2 = Configuration.model.countDocuments(queryObj); + const queryCount = Configuration.model.countDocuments(queryObj); if (limit) { query.limit(parseInt(limit, 10)); } @@ -79,28 +79,28 @@ function createGetWithFields(fields) { query.skip(parseInt(offset, 10)); } - function func1(callback) { + function funcQuery(cb) { query .exec({}) .then((res) => { - callback(null, res); + cb(null, res); }) .catch((error) => { - callback(error); + cb(error); }); } - function func2(callback) { - query2 + function funcQueryCount(cb) { + queryCount .exec({}) .then((res) => { - callback(null, res); + cb(null, res); }) .catch((error) => { - callback(error); + cb(error); }); } - async.series([func1, func2], function (error, results) { + async.series([funcQuery, funcQueryCount], function (error, results) { if (!error && results && results.length === 2) { callback(null, { services: results[0], diff --git a/lib/services/protocolData.js b/lib/services/protocolData.js index d30b518..b8bca2d 100644 --- a/lib/services/protocolData.js +++ b/lib/services/protocolData.js @@ -146,18 +146,18 @@ function save(logger, newProtocol, callback) { ); } /* eslint-disable no-inner-declarations */ - function func1(callback) { + function funcSave(cb) { protocolObj .save({}) .then((res) => { - callback(null, res); + cb(null, res); }) .catch((error) => { - callback(error); + cb(error); }); } /* eslint-enable no-inner-declarations */ - actions.push(func1); + actions.push(funcSave); async.series(actions, callback); } }); diff --git a/lib/services/protocols.js b/lib/services/protocols.js index 053b37d..4ac8f0b 100644 --- a/lib/services/protocols.js +++ b/lib/services/protocols.js @@ -34,7 +34,7 @@ function readProtocolList(req, res, next) { logger.debug('List protocols request with args %j', JSON.stringify(req.query)); const query = Protocol.model.find(condition).sort(); - const query2 = Protocol.model.countDocuments(condition); + const queryCount = Protocol.model.countDocuments(condition); if (req.query && req.query.limit) { query.limit(parseInt(req.query.limit, 10)); } @@ -42,7 +42,7 @@ function readProtocolList(req, res, next) { if (req.query && req.query.offset) { query.skip(parseInt(req.query.offset, 10)); } - function func1(callback) { + function funcQuery(callback) { query .exec({}) .then((res) => { @@ -52,8 +52,8 @@ function readProtocolList(req, res, next) { callback(error); }); } - function func2(callback) { - query2 + function funcQueryCount(callback) { + queryCount .exec({}) .then((res) => { callback(null, res); @@ -62,7 +62,7 @@ function readProtocolList(req, res, next) { callback(error); }); } - async.series([func1, func2], function (error, results) { + async.series([funcQuery, funcQueryCount], function (error, results) { if (error) { logger.error('error: %j', error); next(error); From 8964e4b6032e37be99f122f469be4db66db1d889 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Tue, 10 Dec 2024 15:16:42 +0100 Subject: [PATCH 07/12] remove node 14 from ci due to mongoose --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7f6197d..853e5cb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,7 +64,6 @@ jobs: strategy: matrix: node-version: - - 14.x - 16.x - 18.x steps: From aa220f394f123ba8780f46298d6ba0e8bdfecd7c Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Tue, 10 Dec 2024 17:58:03 +0100 Subject: [PATCH 08/12] add missed callback to db init --- lib/model/dbConn.js | 95 ++++++++++++++++++++++++++++++--------------- 1 file changed, 64 insertions(+), 31 deletions(-) diff --git a/lib/model/dbConn.js b/lib/model/dbConn.js index 66fc71f..99c696a 100644 --- a/lib/model/dbConn.js +++ b/lib/model/dbConn.js @@ -31,6 +31,7 @@ const config = require('../utils/commonConfig'); const constants = require('../utils/constants'); const errors = require('../errors'); const DEFAULT_DB_NAME = 'iotagent-manager'; +//let defaultDb; mongoose.Promise = global.Promise; // not including this causes DeprecationWarning @@ -49,8 +50,10 @@ function delay(ms) { * * @this Reference to the dbConn module itself. */ -async function init(logger, host, db, port, username, password, options) { +function init(logger, host, db, port, username, password, options, callback) { let credentials = ''; + let retries = 0; + let lastError; if (username && password) { credentials = `${username}:${password}@`; } @@ -67,38 +70,66 @@ async function init(logger, host, db, port, username, password, options) { const maxRetries = config.getConfig().mongodb?.retries || constants.DEFAULT_MONGODB_RETRIES; const retryTime = config.getConfig().mongodb?.retryTime || constants.DEFAULT_MONGODB_RETRY_TIME; - /* eslint-disable no-await-in-loop */ - for (let attempt = 1; attempt <= maxRetries; attempt++) { - try { - logger.info(`Attempt ${attempt}: Connecting to MongoDB at ${url}`); - await mongoose.connect(url, options); - logger.info('Successfully connected to MongoDB.'); - - // Register events - mongoose.connection.on('error', (err) => logger.error('Mongo Driver error:', err)); - mongoose.connection.on('connected', () => logger.debug('Mongo Driver connected')); - mongoose.connection.on('disconnected', () => logger.debug('Mongo Driver disconnected')); - mongoose.connection.on('reconnectFailed', () => { - logger.error('MONGODB-004: MongoDB connection was lost'); - process.exit(1); - }); + // /* eslint-disable no-await-in-loop */ + // for (let attempt = 1; attempt <= maxRetries; attempt++) { + // try { + // logger.info(`Attempt ${attempt}: Connecting to MongoDB at ${url}`); + // mongoose.connect(url, options); + // //logger.info('Successfully connected to MongoDB.'); + + // // Register events + // mongoose.connection.on('error', (err) => logger.error('Mongo Driver error:', err)); + // mongoose.connection.on('connected', () => logger.debug('Mongo Driver connected')); + // mongoose.connection.on('disconnected', () => logger.debug('Mongo Driver disconnected')); + // mongoose.connection.on('reconnectFailed', () => { + // logger.error('MONGODB-004: MongoDB connection was lost'); + // process.exit(1); + // }); + // //module.exports.db = defaultDb; + // loadModels(); + // break; // End loop if connection was OK + // } catch (error) { + // logger.error(`Attempt ${attempt} failed: ${error}`); + // if (attempt < maxRetries) { + // logger.info(`Retrying in ${retryTime} seconds...`); + // delay(retryTime * 1000); + // } else { + // throw new Error(`MONGODB-002: Connection failed after ${maxRetries} attempts.`); + // } + // } + // } + // /* eslint-enable no-await-in-loop */ + function connectionAttempt(callback) { + logger.info(context, `Attempting to connect to MongoDB at ${url}. Attempt ${retries + 1}`); - loadModels(); - break; // End loop if connection was OK - } catch (error) { - logger.error(`Attempt ${attempt} failed: ${error}`); - if (attempt < maxRetries) { - logger.info(`Retrying in ${retryTime} seconds...`); - await delay(retryTime * 1000); - } else { - throw new Error(`MONGODB-002: Connection failed after ${maxRetries} attempts.`); - } - } + mongoose + .connect(url, options) + .then(() => { + //defaultDb = connection.connection; + defaultDb = mongoose.connection; + logger.info(context, 'Successfully connected to MongoDB.'); + loadModels(); + callback(); + }) + .catch((err) => { + logger.error(context, `MONGODB-001: Error trying to connect to MongoDB: ${err}`); + //lastError = err; + retries++; + if (retries < maxRetries) { + //const retryTime = config.getConfig().mongodb?.retryTime || constants.DEFAULT_MONGODB_RETRY_TIME; + logger.info(context, `Retrying in ${retryTime} seconds...`); + setTimeout(() => connectionAttempt(callback), retryTime * 1000); + } else { + logger.error(context, `MONGODB-002: Failed to connect after ${maxRetries} attempts.`); + callback(err); + } + }); } - /* eslint-enable no-await-in-loop */ + + connectionAttempt(callback); } -async function configureDb(logger) { +function configureDb(logger, callback) { const currentConfig = config.getConfig(); if (!currentConfig.mongodb?.host) { logger.fatal('No host found for MongoDB driver.'); @@ -109,16 +140,18 @@ async function configureDb(logger) { const port = currentConfig.mongodb.port || 27017; const options = currentConfig.mongodb.replicaSet ? { replicaSet: currentConfig.mongodb.replicaSet } : {}; - await init( + init( logger, currentConfig.mongodb.host, dbName, port, currentConfig.username, currentConfig.password, - options + options, + callback ); } exports.configureDb = configureDb; +//exports.db = defaultDb; exports.DEFAULT_DB_NAME = DEFAULT_DB_NAME; From 5c86cc8def1b2b3221f8da565b0bd45f972c1774 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Wed, 11 Dec 2024 11:11:10 +0100 Subject: [PATCH 09/12] update CNR --- CHANGES_NEXT_RELEASE | 3 +- lib/model/dbConn.js | 139 +++++++++++++++++++++++++------------------ package.json | 2 +- 3 files changed, 85 insertions(+), 59 deletions(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index f2bd912..e44ed29 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -2,4 +2,5 @@ - Deprecated: /iot/services API routes - Upgrade body-parser dep from 1.19.0 to 1.20.3 - Upgrade express from 4.19.2 to 4.20.0 -- Upgrade mongoose dep from 5.13.20 to 5.13.22 +- Upgrade mongodb devdep from 4.17.1 to 4.17.2 +- Upgrade mongoose dep from 5.13.22 to 8.4.4 (solving vulnerability CVE-2024-53900) (#351) diff --git a/lib/model/dbConn.js b/lib/model/dbConn.js index 99c696a..658d848 100644 --- a/lib/model/dbConn.js +++ b/lib/model/dbConn.js @@ -29,9 +29,10 @@ const mongoose = require('mongoose'); const config = require('../utils/commonConfig'); const constants = require('../utils/constants'); +const iotagentLib = require('iotagent-node-lib'); const errors = require('../errors'); const DEFAULT_DB_NAME = 'iotagent-manager'; -//let defaultDb; +let defaultDb; mongoose.Promise = global.Promise; // not including this causes DeprecationWarning @@ -40,87 +41,95 @@ function loadModels() { require('./Configuration').load(); } -// Delay function for nodejs16 -function delay(ms) { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - /** * Creates a new connection to the Mongo DB. * * @this Reference to the dbConn module itself. */ function init(logger, host, db, port, username, password, options, callback) { - let credentials = ''; + let url; let retries = 0; let lastError; - if (username && password) { - credentials = `${username}:${password}@`; + const maxRetries = config.getConfig().mongodb?.retries || constants.DEFAULT_MONGODB_RETRIES; + + function addPort(item) { + return `${item}:${port}`; } - const hosts = host - .split(',') - .map((item) => `${item}:${port}`) - .join(','); - let url = `mongodb://${credentials}${hosts}/${db}`; + url = 'mongodb://'; - if (options.replicaSet) { - url += `?replicaSet=${options.replicaSet.rs_name}`; + if (options.auth) { + url += `${encodeURIComponent(options.auth.user)}:${encodeURIComponent(options.auth.password)}@`; + } + + const hosts = host.split(',').map(addPort).join(','); + url += `${hosts}/${db}`; + + if (options.extraArgs) { + const query = new URLSearchParams(options.extraArgs).toString(); + if (query) { + url += `?${query}`; + } + delete options.extraArgs; } - const maxRetries = config.getConfig().mongodb?.retries || constants.DEFAULT_MONGODB_RETRIES; - const retryTime = config.getConfig().mongodb?.retryTime || constants.DEFAULT_MONGODB_RETRY_TIME; - // /* eslint-disable no-await-in-loop */ - // for (let attempt = 1; attempt <= maxRetries; attempt++) { - // try { - // logger.info(`Attempt ${attempt}: Connecting to MongoDB at ${url}`); - // mongoose.connect(url, options); - // //logger.info('Successfully connected to MongoDB.'); - - // // Register events - // mongoose.connection.on('error', (err) => logger.error('Mongo Driver error:', err)); - // mongoose.connection.on('connected', () => logger.debug('Mongo Driver connected')); - // mongoose.connection.on('disconnected', () => logger.debug('Mongo Driver disconnected')); - // mongoose.connection.on('reconnectFailed', () => { - // logger.error('MONGODB-004: MongoDB connection was lost'); - // process.exit(1); - // }); - // //module.exports.db = defaultDb; - // loadModels(); - // break; // End loop if connection was OK - // } catch (error) { - // logger.error(`Attempt ${attempt} failed: ${error}`); - // if (attempt < maxRetries) { - // logger.info(`Retrying in ${retryTime} seconds...`); - // delay(retryTime * 1000); - // } else { - // throw new Error(`MONGODB-002: Connection failed after ${maxRetries} attempts.`); - // } - // } - // } - // /* eslint-enable no-await-in-loop */ function connectionAttempt(callback) { - logger.info(context, `Attempting to connect to MongoDB at ${url}. Attempt ${retries + 1}`); + logger.info(`Attempting to connect to MongoDB at ${url}. Attempt ${retries + 1}`); mongoose .connect(url, options) .then(() => { - //defaultDb = connection.connection; defaultDb = mongoose.connection; - logger.info(context, 'Successfully connected to MongoDB.'); + logger.info('Successfully connected to MongoDB.'); loadModels(); + defaultDb.on('error', function (error) { + logger.error('Mongo Driver error: %j', error); + lastError = error; + iotagentLib.alarms.raise(iotagentLib.constants.MONGO_ALARM, error); + }); + /* eslint-disable-next-line no-unused-vars */ + defaultDb.on('connecting', function (error) { + logger.debug('Mongo Driver connecting'); + }); + defaultDb.on('connected', function () { + logger.debug('Mongo Driver connected'); + }); + defaultDb.on('reconnected', function () { + logger.debug('Mongo Driver reconnected'); + }); + defaultDb.on('disconnected', function () { + logger.debug('Mongo Driver disconnected'); + }); + defaultDb.on('reconnectFailed', function () { + logger.error('MONGODB-004: MongoDB connection was lost'); + process.exit(1); + }); + defaultDb.on('disconnecting', function () { + logger.debug('Mongo Driver disconnecting'); + }); + defaultDb.on('open', function () { + logger.debug('Mongo Driver open'); + }); + defaultDb.on('close', function () { + logger.debug('Mongo Driver close'); + }); callback(); }) .catch((err) => { - logger.error(context, `MONGODB-001: Error trying to connect to MongoDB: ${err}`); - //lastError = err; + logger.error(`MONGODB-001: Error trying to connect to MongoDB: ${err}`); + lastError = err; retries++; if (retries < maxRetries) { - //const retryTime = config.getConfig().mongodb?.retryTime || constants.DEFAULT_MONGODB_RETRY_TIME; - logger.info(context, `Retrying in ${retryTime} seconds...`); + const retryTime = config.getConfig().mongodb?.retryTime || constants.DEFAULT_MONGODB_RETRY_TIME; + logger.info(`Retrying in ${retryTime} seconds...`); setTimeout(() => connectionAttempt(callback), retryTime * 1000); } else { - logger.error(context, `MONGODB-002: Failed to connect after ${maxRetries} attempts.`); + logger.error( + context, + 'MONGODB-002: Error to connect found after %d attempts: %s', + retries, + lastError + ); callback(err); } }); @@ -138,7 +147,23 @@ function configureDb(logger, callback) { const dbName = currentConfig.mongodb.db || DEFAULT_DB_NAME; const port = currentConfig.mongodb.port || 27017; - const options = currentConfig.mongodb.replicaSet ? { replicaSet: currentConfig.mongodb.replicaSet } : {}; + const options = {}; + + if (currentConfig.mongodb.replicaSet) options.replicaSet = currentConfig.mongodb.replicaSet; + if (currentConfig.mongodb.ssl) options.ssl = currentConfig.mongodb.ssl; + if (currentConfig.mongodb.extraArgs) options.extraArgs = currentConfig.mongodb.extraArgs; + if (currentConfig.mongodb.user && currentConfig.mongodb.password) { + options.auth = { + user: currentConfig.mongodb.user, + password: currentConfig.mongodb.password + }; + if (currentConfig.mongodb.authSource) { + options.extraArgs = { + ...options.extraArgs, + authSource: currentConfig.mongodb.authSource + }; + } + } init( logger, @@ -153,5 +178,5 @@ function configureDb(logger, callback) { } exports.configureDb = configureDb; -//exports.db = defaultDb; +exports.db = defaultDb; exports.DEFAULT_DB_NAME = DEFAULT_DB_NAME; diff --git a/package.json b/package.json index 2cdd52b..0c2bca3 100644 --- a/package.json +++ b/package.json @@ -45,7 +45,7 @@ "husky": "~4.2.5", "lint-staged": "~10.2.11", "mocha": "8.0.1", - "mongodb": "4.17.0", + "mongodb": "4.17.2", "nock": "13.0.3", "nyc": "~15.1.0", "prettier": "~2.0.5", From 0d918677450fa8cb3e6b5654090cef539951f53b Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Wed, 11 Dec 2024 11:13:24 +0100 Subject: [PATCH 10/12] fix linter --- lib/model/dbConn.js | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/lib/model/dbConn.js b/lib/model/dbConn.js index 658d848..63dc9cf 100644 --- a/lib/model/dbConn.js +++ b/lib/model/dbConn.js @@ -124,12 +124,7 @@ function init(logger, host, db, port, username, password, options, callback) { logger.info(`Retrying in ${retryTime} seconds...`); setTimeout(() => connectionAttempt(callback), retryTime * 1000); } else { - logger.error( - context, - 'MONGODB-002: Error to connect found after %d attempts: %s', - retries, - lastError - ); + logger.error('MONGODB-002: Error to connect found after %d attempts: %s', retries, lastError); callback(err); } }); From 240c3b790d4f93e17e38a1bc04ebd04b0287d578 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Mon, 16 Dec 2024 08:06:59 +0100 Subject: [PATCH 11/12] Update CHANGES_NEXT_RELEASE MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fermín Galán Márquez --- CHANGES_NEXT_RELEASE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index e44ed29..2782178 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -3,4 +3,4 @@ - Upgrade body-parser dep from 1.19.0 to 1.20.3 - Upgrade express from 4.19.2 to 4.20.0 - Upgrade mongodb devdep from 4.17.1 to 4.17.2 -- Upgrade mongoose dep from 5.13.22 to 8.4.4 (solving vulnerability CVE-2024-53900) (#351) +- Upgrade mongoose dep from 5.13.20 to 8.4.4 (solving vulnerability CVE-2024-53900) (#351) From 4122afbf06fa478ce51c10b02c5e1140b73c1887 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Mon, 16 Dec 2024 08:07:24 +0100 Subject: [PATCH 12/12] Update CHANGES_NEXT_RELEASE MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fermín Galán Márquez --- CHANGES_NEXT_RELEASE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 2782178..a10a14c 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -2,5 +2,5 @@ - Deprecated: /iot/services API routes - Upgrade body-parser dep from 1.19.0 to 1.20.3 - Upgrade express from 4.19.2 to 4.20.0 -- Upgrade mongodb devdep from 4.17.1 to 4.17.2 +- Upgrade mongodb devdep from 4.17.0 to 4.17.2 - Upgrade mongoose dep from 5.13.20 to 8.4.4 (solving vulnerability CVE-2024-53900) (#351)