diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 40809a1a9..1582bfc6a 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,3 +1,5 @@ +- Add: store last measure in device (by id, apikey, service and subservice) and new API field storeLastMeasure at group and device levels (#1669) +- Add: IOTA_STORE_LAST_MEASURE env var to set default store last measure behaviour at instance level (#1669) - Upgrade express dep from 4.19.2 to 4.20.0 - Upgrade mongodb devdep from 4.17.1 to 4.17.2 - Upgrade mongoose dep from 5.13.20 to 8.4.4 (solving vulnerability CVE-2024-53900) (#1674) diff --git a/config.js b/config.js index 88b4e610d..d9ad2e7a4 100644 --- a/config.js +++ b/config.js @@ -77,7 +77,8 @@ var config = { providerUrl: 'http://192.168.56.1:4041', deviceRegistrationDuration: 'P1M', defaultType: 'Thing', - expressLimit: '1Mb' + expressLimit: '1Mb', + storeLastMeasure: false }; module.exports = config; diff --git a/doc/admin.md b/doc/admin.md index b82c27f88..7f7b687f3 100644 --- a/doc/admin.md +++ b/doc/admin.md @@ -125,9 +125,9 @@ allowing the computer to interpret the rest of the data with more clarity and de ``` Under mixed mode, **NGSI v2** payloads are used for context broker communications by default, but this payload may also -be switched to **NGSI LD** at group or device provisioning time using the `ngsiVersion` field in the -provisioning API. The `ngsiVersion` field switch may be added at either group or device level, with the device level -overriding the group setting. +be switched to **NGSI LD** at group or device provisioning time using the `ngsiVersion` field in the provisioning API. +The `ngsiVersion` field switch may be added at either group or device level, with the device level overriding the group +setting. #### `server` @@ -306,7 +306,8 @@ added `agentPath`: #### `types` -This parameter includes additional groups configuration as described into the [Config group API](api.md#config-group-api) section. +This parameter includes additional groups configuration as described into the +[Config group API](api.md#config-group-api) section. #### `service` @@ -415,7 +416,33 @@ IotAgents, as all Express applications that use the body-parser middleware, have size that the application will handle. This default limit for ioiotagnets are 1Mb. So, if your IotAgent receives a request with a body that exceeds this limit, the application will throw a “Error: Request entity too large”. -The 1Mb default can be changed setting the `expressLimit` configuration parameter (or equivalente `IOTA_EXPRESS_LIMIT` environment variable). +The 1Mb default can be changed setting the `expressLimit` configuration parameter (or equivalente `IOTA_EXPRESS_LIMIT` +environment variable). + +#### `storeLastMeasure` + +If this flag is activated, last measure arrived to Device IoTAgent without be processed will be stored in Device under +`lastMeasure` field (composed of sub-fields `timestamp` and `measure` for the measure itself, in multi-measure format). This flag is overwritten by `storeLastMeasure` flag in group or device. This flag +is disabled by default. + +For example in a device document stored in MongoDB will be extended with a subdocument named lastMeasure like this: + +```json +{ + "lastMeasure": { + "timestamp": "2025-01-09T10:35:33.079Z", + "measure": [ + [ + { + "name": "level", + "type": "Text", + "value": 33 + } + ] + ] + } +} +``` ### Configuration using environment variables @@ -479,6 +506,7 @@ overrides. | IOTA_DEFAULT_ENTITY_NAME_CONJUNCTION | `defaultEntityNameConjunction` | | IOTA_RELAX_TEMPLATE_VALIDATION | `relaxTemplateValidation` | | IOTA_EXPRESS_LIMIT | `expressLimit` | +| IOTA_STORE_LAST_MEASURE | `storeLastMeasure` | Note: diff --git a/doc/api.md b/doc/api.md index 15807bf9d..30d05f06e 100644 --- a/doc/api.md +++ b/doc/api.md @@ -1777,6 +1777,7 @@ Config group is represented by a JSON object with the following fields: | `payloadType` | ✓ | string | | optional string value used to switch between **IoTAgent**, **NGSI-v2** and **NGSI-LD** measure payloads types. Possible values are: `iotagent`, `ngsiv2` or `ngsild`. The default is `iotagent`. | | `transport` | ✓ | `string` | | Transport protocol used by the group of devices to send updates, for the IoT Agents with multiple transport protocols. | | `endpoint` | ✓ | `string` | | Endpoint where the group of device is going to receive commands, if any. | +| `storeLastMeasure` | ✓ | `boolean` | | Store in device last measure received. See more info [in this section](admin.md#storelastmeasure). False by default | ### Config group operations @@ -1998,6 +1999,8 @@ the API resource fields and the same fields in the database model. | `ngsiVersion` | ✓ | `string` | | string value used in mixed mode to switch between **NGSI-v2** and **NGSI-LD** payloads. The default is `v2`. When not running in mixed mode, this field is ignored. | | `payloadType` | ✓ | `string` | | optional string value used to switch between **IoTAgent**, **NGSI-v2** and **NGSI-LD** measure payloads types. Possible values are: `iotagent`, `ngsiv2` or `ngsild`. The default is `iotagent`. | +| `storeLastMeasure` | ✓ | `boolean` | | Store in device last measure received. See more info [in this section](admin.md#storelastmeasure). False by default. | + ### Device operations #### Retrieve devices /iot/devices `GET /iot/devices` diff --git a/lib/commonConfig.js b/lib/commonConfig.js index d16908615..d10c26a9c 100644 --- a/lib/commonConfig.js +++ b/lib/commonConfig.js @@ -157,7 +157,8 @@ function processEnvironmentVariables() { 'IOTA_FALLBACK_PATH', 'IOTA_LD_SUPPORT_NULL', 'IOTA_LD_SUPPORT_DATASET_ID', - 'IOTA_EXPRESS_LIMIT' + 'IOTA_EXPRESS_LIMIT', + 'IOTA_STORE_LAST_MEASURE' ]; const iotamVariables = [ 'IOTA_IOTAM_URL', @@ -474,6 +475,11 @@ function processEnvironmentVariables() { } else { config.expressLimit = config.expressLimit ? config.expressLimit : '1mb'; } + if (process.env.IOTA_STORE_LAST_MEASURE) { + config.storeLastMeasure = process.env.IOTA_STORE_LAST_MEASURE === 'true'; + } else { + config.storeLastMeasure = config.storeLastMeasure === true; + } } function setConfig(newConfig) { @@ -503,7 +509,8 @@ function getConfigForTypeInformation() { multiCore: config.multiCore, relaxTemplateValidation: config.relaxTemplateValidation, defaultEntityNameConjunction: config.defaultEntityNameConjunction, - defaultType: config.defaultType + defaultType: config.defaultType, + storeLastMeasure: config.storeLastMeasure }; return conf; } diff --git a/lib/model/Device.js b/lib/model/Device.js index a6611fe73..8513eba04 100644 --- a/lib/model/Device.js +++ b/lib/model/Device.js @@ -53,7 +53,9 @@ const Device = new Schema({ autoprovision: Boolean, explicitAttrs: Group.ExplicitAttrsType, ngsiVersion: String, - payloadType: String + payloadType: String, + storeLastMeasure: Boolean, + lastMeasure: Object }); function load() { diff --git a/lib/model/Group.js b/lib/model/Group.js index cf710be26..83b2e1a3f 100644 --- a/lib/model/Group.js +++ b/lib/model/Group.js @@ -65,7 +65,8 @@ const Group = new Schema({ defaultEntityNameConjunction: String, ngsiVersion: String, entityNameExp: String, - payloadType: String + payloadType: String, + storeLastMeasure: Boolean }); function load() { diff --git a/lib/services/common/iotManagerService.js b/lib/services/common/iotManagerService.js index b56fe9858..911ed3e26 100644 --- a/lib/services/common/iotManagerService.js +++ b/lib/services/common/iotManagerService.js @@ -63,7 +63,8 @@ function register(callback) { entityNameExp: service.entityNameExp, payloadType: service.payloadType, endpoint: service.endpoint, - transport: service.transport + transport: service.transport, + storeLastMeasure: service.storeLastMeasure }; } diff --git a/lib/services/devices/deviceRegistryMemory.js b/lib/services/devices/deviceRegistryMemory.js index f191692a2..3a81aec49 100644 --- a/lib/services/devices/deviceRegistryMemory.js +++ b/lib/services/devices/deviceRegistryMemory.js @@ -210,8 +210,40 @@ function getDevicesByAttribute(name, value, service, subservice, callback) { } } +function storeLastMeasure(measure, typeInformation, callback) { + if ( + typeInformation && + typeInformation.id && + typeInformation.apikey && + typeInformation.service && + typeInformation.subservice + ) { + getDevice( + typeInformation.id, + typeInformation.apikey, + typeInformation.service, + typeInformation.subservice, + function (error, data) { + if (!error) { + if (data) { + data.lastMeasure = measure; + } + if (callback) { + callback(null, data); + } + } else { + callback(error, null); + } + } + ); + } else { + callback(null, null); + } +} + exports.getDevicesByAttribute = getDevicesByAttribute; exports.store = storeDevice; +exports.storeLastMeasure = storeLastMeasure; exports.update = update; exports.remove = removeDevice; exports.list = listDevices; diff --git a/lib/services/devices/deviceRegistryMongoDB.js b/lib/services/devices/deviceRegistryMongoDB.js index 4086f1933..6d013df2e 100644 --- a/lib/services/devices/deviceRegistryMongoDB.js +++ b/lib/services/devices/deviceRegistryMongoDB.js @@ -57,7 +57,8 @@ const attributeList = [ 'explicitAttrs', 'ngsiVersion', 'subscriptions', - 'payloadType' + 'payloadType', + 'storeLastMeasure' ]; /** @@ -222,7 +223,7 @@ function findOneInMongoDB(queryParams, id, callback) { * @param {String} subservice Division inside the service (optional). */ function getDeviceById(id, apikey, service, subservice, callback) { - let queryParams = { + const queryParams = { id, service, subservice @@ -254,9 +255,9 @@ function getDevice(id, apikey, service, subservice, callback) { function getByNameAndType(name, type, service, servicepath, callback) { context = fillService(context, { service, subservice: servicepath }); - let optionsQuery = { - name: name, - service: service, + const optionsQuery = { + name, + service, subservice: servicepath }; if (type) { @@ -319,6 +320,7 @@ function update(previousDevice, device, callback) { data.timestamp = device.timestamp; data.subscriptions = device.subscriptions; data.payloadType = device.payloadType; + data.storeLastMeasure = device.storeLastMeasure; /* eslint-disable-next-line new-cap */ const deviceObj = new Device.model(data); @@ -392,8 +394,51 @@ function getDevicesByAttribute(name, value, service, subservice, callback) { }); } +function storeLastMeasure(measure, typeInformation, callback) { + if ( + typeInformation && + typeInformation.id && + typeInformation.apikey && + typeInformation.service && + typeInformation.subservice + ) { + getDevice( + typeInformation.id, + typeInformation.apikey, + typeInformation.service, + typeInformation.subservice, + function (error, data) { + if (error) { + callback(error); + } else { + data.lastMeasure = { timestamp: new Date().toISOString(), measure }; + /* eslint-disable-next-line new-cap */ + const deviceObj = new Device.model(data); + deviceObj.isNew = false; + deviceObj + .save({}) + .then((deviceDao) => { + callback(null, deviceDao.toObject()); + }) + .catch((error) => { + logger.debug( + fillService(context, deviceObj), + 'Error storing device information: %s', + error + ); + callback(new errors.InternalDbError(error)); + }); + } + } + ); + } else { + callback(null, null); + } +} + exports.getDevicesByAttribute = alarmsInt(constants.MONGO_ALARM, getDevicesByAttribute); exports.store = alarmsInt(constants.MONGO_ALARM, storeDevice); +exports.storeLastMeasure = alarmsInt(constants.MONGO_ALARM, storeLastMeasure); exports.update = alarmsInt(constants.MONGO_ALARM, update); exports.remove = alarmsInt(constants.MONGO_ALARM, removeDevice); exports.list = alarmsInt(constants.MONGO_ALARM, listDevices); diff --git a/lib/services/devices/deviceService.js b/lib/services/devices/deviceService.js index 03be2722c..c3af24053 100644 --- a/lib/services/devices/deviceService.js +++ b/lib/services/devices/deviceService.js @@ -179,6 +179,9 @@ function mergeDeviceWithConfiguration(fields, defaults, deviceData, configuratio if (configuration && configuration.payloadType !== undefined && deviceData.payloadType === undefined) { deviceData.payloadType = configuration.payloadType; } + if (configuration && configuration.storeLastMeasure !== undefined && deviceData.storeLastMeasure === undefined) { + deviceData.storeLastMeasure = configuration.storeLastMeasure; + } logger.debug(context, 'deviceData after merge with conf: %j', deviceData); callback(null, deviceData); } @@ -285,7 +288,7 @@ function registerDevice(deviceObj, callback) { let attrList = pluginUtils.getIdTypeServSubServiceFromDevice(deviceData); attrList = deviceData.staticAttributes ? attrList.concat(deviceData.staticAttributes) : attrList; attrList = configuration.staticAttributes ? attrList.concat(configuration.staticAttributes) : attrList; - let ctxt = expressionPlugin.extractContext(attrList); + const ctxt = expressionPlugin.extractContext(attrList); try { entityName = expressionPlugin.applyExpression(configuration.entityNameExp, ctxt, deviceData); } catch (e) { @@ -616,7 +619,7 @@ function findOrCreate(deviceId, apikey, group, callback) { } else if (error.name === 'DEVICE_NOT_FOUND') { const newDevice = { id: deviceId, - apikey: apikey, + apikey, service: group.service, subservice: group.subservice, type: group.type @@ -692,6 +695,10 @@ function retrieveDevice(deviceId, apiKey, callback) { } } +function storeLastMeasure(measure, typeInformation, callback) { + config.getRegistry().storeLastMeasure(measure, typeInformation, callback); +} + exports.listDevices = intoTrans(context, checkRegistry)(listDevices); exports.listDevicesWithType = intoTrans(context, checkRegistry)(listDevicesWithType); exports.getDevice = intoTrans(context, checkRegistry)(getDevice); @@ -708,4 +715,5 @@ exports.retrieveDevice = intoTrans(context, checkRegistry)(retrieveDevice); exports.mergeDeviceWithConfiguration = mergeDeviceWithConfiguration; exports.findOrCreate = findOrCreate; exports.findConfigurationGroup = findConfigurationGroup; +exports.storeLastMeasure = storeLastMeasure; exports.init = init; diff --git a/lib/services/devices/devices-NGSI-v2.js b/lib/services/devices/devices-NGSI-v2.js index f5942467f..52f4ab72f 100644 --- a/lib/services/devices/devices-NGSI-v2.js +++ b/lib/services/devices/devices-NGSI-v2.js @@ -285,7 +285,9 @@ function updateRegisterDeviceNgsi2(deviceObj, previousDevice, entityInfoUpdated, if ('transport' in newDevice && newDevice.transport !== undefined) { oldDevice.transport = newDevice.transport; } - + if ('storeLastMeasure' in newDevice && newDevice.storeLastMeasure !== undefined) { + oldDevice.storeLastMeasure = newDevice.storeLastMeasure; + } callback(null, oldDevice); } else { callback(new errors.DeviceNotFound(newDevice.id, newDevice)); diff --git a/lib/services/groups/groupRegistryMongoDB.js b/lib/services/groups/groupRegistryMongoDB.js index 4cd8c4fdb..20e408744 100644 --- a/lib/services/groups/groupRegistryMongoDB.js +++ b/lib/services/groups/groupRegistryMongoDB.js @@ -61,7 +61,8 @@ const attributeList = [ 'defaultEntityNameConjunction', 'ngsiVersion', 'entityNameExp', - 'payloadType' + 'payloadType', + 'storeLastMeasure' ]; function createGroup(group, callback) { diff --git a/lib/services/ngsi/ngsiService.js b/lib/services/ngsi/ngsiService.js index 4b57c53c1..3f2619f55 100644 --- a/lib/services/ngsi/ngsiService.js +++ b/lib/services/ngsi/ngsiService.js @@ -27,6 +27,7 @@ const async = require('async'); const apply = async.apply; const statsRegistry = require('../stats/statsRegistry'); +const deviceService = require('../devices/deviceService'); const intoTrans = require('../common/domain').intoTrans; const fillService = require('./../common/domain').fillService; const errors = require('../../errors'); @@ -68,8 +69,16 @@ function init() { * @param {String} token User token to identify against the PEP Proxies (optional). */ function sendUpdateValue(entityName, attributes, typeInformation, token, callback) { + // check config about store last measure const newCallback = statsRegistry.withStats('updateEntityRequestsOk', 'updateEntityRequestsError', callback); - entityHandler.sendUpdateValue(entityName, attributes, typeInformation, token, newCallback); + if (typeInformation.storeLastMeasure) { + logger.debug(context, 'StoreLastMeasure for %j', typeInformation); + deviceService.storeLastMeasure(attributes, typeInformation, function () { + return entityHandler.sendUpdateValue(entityName, attributes, typeInformation, token, newCallback); + }); + } else { + entityHandler.sendUpdateValue(entityName, attributes, typeInformation, token, newCallback); + } } /** diff --git a/lib/services/northBound/deviceProvisioningServer.js b/lib/services/northBound/deviceProvisioningServer.js index 2099cba45..bc5d468a6 100644 --- a/lib/services/northBound/deviceProvisioningServer.js +++ b/lib/services/northBound/deviceProvisioningServer.js @@ -64,7 +64,9 @@ const provisioningAPITranslation = { explicitAttrs: 'explicitAttrs', ngsiVersion: 'ngsiVersion', entityNameExp: 'entityNameExp', - payloadType: 'payloadType' + payloadType: 'payloadType', + storeLastMeasure: 'storeLastMeasure', + lastMeasure: 'lastMeasure' }; /** @@ -143,7 +145,9 @@ function handleProvision(req, res, next) { autoprovision: body.autoprovision, explicitAttrs: body.explicitAttrs, ngsiVersion: body.ngsiVersion, - payloadType: body.payloadType + payloadType: body.payloadType, + storeLastMeasure: body.storeLastMeasure, + lastMeasure: body.lastMeasure }); } @@ -220,7 +224,9 @@ function toProvisioningAPIFormat(device) { autoprovision: device.autoprovision, explicitAttrs: device.explicitAttrs, ngsiVersion: device.ngsiVersion, - payloadType: device.payloadType + payloadType: device.payloadType, + storeLastMeasure: device.storeLastMeasure, + lastMeasure: device.lastMeasure }; } diff --git a/lib/templates/updateDevice.json b/lib/templates/updateDevice.json index 3b6c0835a..9a27f32a7 100644 --- a/lib/templates/updateDevice.json +++ b/lib/templates/updateDevice.json @@ -194,6 +194,10 @@ "payloadType": { "description": "Payload type allowed for measures for this device", "type": "string" + }, + "storeLastMeasure": { + "description": "Store last measure", + "type": "boolean" } } } diff --git a/lib/templates/updateDeviceLax.json b/lib/templates/updateDeviceLax.json index a40bf59ed..32e73ff2c 100644 --- a/lib/templates/updateDeviceLax.json +++ b/lib/templates/updateDeviceLax.json @@ -146,6 +146,10 @@ "payloadType": { "description": "Payload type allowed for measures for this device", "type": "string" + }, + "storeLastMeasure": { + "description": "Store last measure", + "type": "boolean" } } }