From ab2283ef35f443aa2027c28ca4e4484611135cf8 Mon Sep 17 00:00:00 2001 From: paulo-ocean Date: Fri, 6 Dec 2024 09:42:52 +0000 Subject: [PATCH 01/11] refactor connections rate, per minute not second --- .env.example | 2 +- docs/dockerDeployment.md | 2 +- docs/env.md | 2 +- scripts/ocean-node-quickstart.sh | 2 +- src/components/core/handler/handler.ts | 16 ++++++++-------- src/test/unit/networking.test.ts | 8 ++++---- src/utils/config.ts | 18 +++++++++--------- src/utils/constants.ts | 14 ++++++++------ 8 files changed, 33 insertions(+), 31 deletions(-) diff --git a/.env.example b/.env.example index 264e0dfdc..67a9312fe 100644 --- a/.env.example +++ b/.env.example @@ -27,7 +27,7 @@ export INDEXER_INTERVAL= export ALLOWED_ADMINS= export DASHBOARD=true export RATE_DENY_LIST= -export MAX_REQ_PER_SECOND= +export MAX_REQ_PER_MINUTE= export MAX_CHECKSUM_LENGTH= export LOG_LEVEL= export HTTP_API_PORT= diff --git a/docs/dockerDeployment.md b/docs/dockerDeployment.md index ef1d675d9..dfa3d3393 100644 --- a/docs/dockerDeployment.md +++ b/docs/dockerDeployment.md @@ -99,7 +99,7 @@ services: # INDEXER_INTERVAL: '' DASHBOARD: 'true' # RATE_DENY_LIST: '' - # MAX_REQ_PER_SECOND: '' + # MAX_REQ_PER_MINUTE: '' # MAX_CHECKSUM_LENGTH: '' # LOG_LEVEL: '' HTTP_API_PORT: '8000' diff --git a/docs/env.md b/docs/env.md index 9c718177c..2ed6937a5 100644 --- a/docs/env.md +++ b/docs/env.md @@ -23,7 +23,7 @@ Environmental variables are also tracked in `ENVIRONMENT_VARIABLES` within `src/ - `ALLOWED_ADMINS`: Sets the public address of accounts which have access to admin endpoints e.g. shutting down the node. Example: `"[\"0x967da4048cD07aB37855c090aAF366e4ce1b9F48\",\"0x388C818CA8B9251b393131C08a736A67ccB19297\"]"` - `DASHBOARD`: If `false` the dashboard will not run. If not set or `true` the dashboard will start with the node. Example: `false` - `RATE_DENY_LIST`: Blocked list of IPs and peer IDs. Example: `"{ \"peers\": [\"16Uiu2HAkuYfgjXoGcSSLSpRPD6XtUgV71t5RqmTmcqdbmrWY9MJo\"], \"ips\": [\"127.0.0.1\"] }"` -- `MAX_REQ_PER_SECOND`: Number of requests per second allowed by the same client. Example: `3` +- `MAX_REQ_PER_MINUTE`: Number of requests per minute allowed by the same client. Example: `30` - `MAX_CHECKSUM_LENGTH`: Define the maximum length for a file if checksum is required (Mb). Example: `10` - `IS_BOOTSTRAP`: Is this node to be used as bootstrap node or not. Default is `false`. diff --git a/scripts/ocean-node-quickstart.sh b/scripts/ocean-node-quickstart.sh index 4c3085338..e2fd601f0 100755 --- a/scripts/ocean-node-quickstart.sh +++ b/scripts/ocean-node-quickstart.sh @@ -155,7 +155,7 @@ services: # INDEXER_INTERVAL: '' DASHBOARD: 'true' # RATE_DENY_LIST: '' -# MAX_REQ_PER_SECOND: '' +# MAX_REQ_PER_MINUTE: '' # MAX_CHECKSUM_LENGTH: '' # LOG_LEVEL: '' HTTP_API_PORT: '$HTTP_API_PORT' diff --git a/src/components/core/handler/handler.ts b/src/components/core/handler/handler.ts index 93aafdc94..179ab0d78 100644 --- a/src/components/core/handler/handler.ts +++ b/src/components/core/handler/handler.ts @@ -38,7 +38,7 @@ export abstract class Handler implements ICommandHandler { // TODO LOG, implement all handlers async checkRateLimit(): Promise { - const ratePerSecond = (await getConfiguration()).rateLimit + const ratePerMinute = (await getConfiguration()).rateLimit const caller: string | string[] = this.getOceanNode().getRemoteCaller() const requestTime = new Date().getTime() let isOK = true @@ -49,7 +49,7 @@ export abstract class Handler implements ICommandHandler { const updatedRequestData = self.checkRequestData( remoteCaller, requestTime, - ratePerSecond + ratePerMinute ) isOK = updatedRequestData.valid self.requestMap.set(remoteCaller, updatedRequestData.updatedRequestData) @@ -105,18 +105,18 @@ export abstract class Handler implements ICommandHandler { /** * Checks if the request is within the rate limit defined * @param remote remote endpoint (ip or peer identifier) - * @param ratePerSecond number of calls per second allowed + * @param ratePerMinute number of calls per minute allowed (per ip or peer identifier) * @returns updated request data */ checkRequestData( remote: string, currentTime: number, - ratePerSecond: number + ratePerMinute: number ): RequestDataCheck { const requestData: RequestLimiter = this.requestMap.get(remote) - const diffSeconds = (currentTime - requestData.lastRequestTime) / 1000 - // more than 1 sec difference means no problem - if (diffSeconds >= 1) { + const diffMinutes = ((currentTime - requestData.lastRequestTime) / 1000) * 60 + // more than 1 minute difference means no problem + if (diffMinutes >= 1) { // its fine requestData.lastRequestTime = currentTime requestData.numRequests = 1 @@ -128,7 +128,7 @@ export abstract class Handler implements ICommandHandler { // requests in the same interval of 1 second requestData.numRequests++ return { - valid: requestData.numRequests <= ratePerSecond, + valid: requestData.numRequests <= ratePerMinute, updatedRequestData: requestData } } diff --git a/src/test/unit/networking.test.ts b/src/test/unit/networking.test.ts index 36e4a2a19..b2e6faac6 100644 --- a/src/test/unit/networking.test.ts +++ b/src/test/unit/networking.test.ts @@ -1,5 +1,5 @@ import { - DEFAULT_RATE_LIMIT_PER_SECOND, + DEFAULT_RATE_LIMIT_PER_MINUTE, ENVIRONMENT_VARIABLES, PROTOCOL_COMMANDS, getConfiguration @@ -105,7 +105,7 @@ describe('Test rate limitations and deny list defaults', () => { // const node: OceanNode = OceanNode.getInstance() before(async () => { envOverrides = buildEnvOverrideConfig( - [ENVIRONMENT_VARIABLES.RATE_DENY_LIST, ENVIRONMENT_VARIABLES.MAX_REQ_PER_SECOND], + [ENVIRONMENT_VARIABLES.RATE_DENY_LIST, ENVIRONMENT_VARIABLES.MAX_REQ_PER_MINUTE], [undefined, undefined] ) await setupEnvironment(null, envOverrides) @@ -115,7 +115,7 @@ describe('Test rate limitations and deny list defaults', () => { const config = await getConfiguration(true) expect(config.denyList.ips).to.be.length(0) expect(config.denyList.peers).to.be.length(0) - expect(config.rateLimit).to.be.equal(DEFAULT_RATE_LIMIT_PER_SECOND) + expect(config.rateLimit).to.be.equal(DEFAULT_RATE_LIMIT_PER_MINUTE) }) // put it back @@ -132,7 +132,7 @@ describe('Test rate limitations and deny list settings', () => { [ ENVIRONMENT_VARIABLES.PRIVATE_KEY, ENVIRONMENT_VARIABLES.RATE_DENY_LIST, - ENVIRONMENT_VARIABLES.MAX_REQ_PER_SECOND + ENVIRONMENT_VARIABLES.MAX_REQ_PER_MINUTE ], [ '0xcb345bd2b11264d523ddaf383094e2675c420a17511c3102a53817f13474a7ff', diff --git a/src/utils/config.ts b/src/utils/config.ts index c96b4f3ff..4ce1f352c 100644 --- a/src/utils/config.ts +++ b/src/utils/config.ts @@ -9,7 +9,7 @@ import { C2DClusterType } from '../@types/C2D.js' import { createFromPrivKey } from '@libp2p/peer-id-factory' import { keys } from '@libp2p/crypto' import { - DEFAULT_RATE_LIMIT_PER_SECOND, + DEFAULT_RATE_LIMIT_PER_MINUTE, ENVIRONMENT_VARIABLES, EnvVariable, hexStringToByteArray @@ -428,19 +428,19 @@ function logMissingVariableWithDefault(envVariable: EnvVariable) { } // have a rate limit for handler calls function getRateLimit(isStartup: boolean = false) { - if (!existsEnvironmentVariable(ENVIRONMENT_VARIABLES.MAX_REQ_PER_SECOND)) { + if (!existsEnvironmentVariable(ENVIRONMENT_VARIABLES.MAX_REQ_PER_MINUTE)) { if (isStartup) { - logMissingVariableWithDefault(ENVIRONMENT_VARIABLES.MAX_REQ_PER_SECOND) + logMissingVariableWithDefault(ENVIRONMENT_VARIABLES.MAX_REQ_PER_MINUTE) } - return DEFAULT_RATE_LIMIT_PER_SECOND + return DEFAULT_RATE_LIMIT_PER_MINUTE } else { try { - return getIntEnvValue(process.env.MAX_REQ_PER_SECOND, DEFAULT_RATE_LIMIT_PER_SECOND) + return getIntEnvValue(process.env.MAX_REQ_PER_MINUTE, DEFAULT_RATE_LIMIT_PER_MINUTE) } catch (err) { CONFIG_LOGGER.error( - `Invalid "${ENVIRONMENT_VARIABLES.MAX_REQ_PER_SECOND.name}" env variable...` + `Invalid "${ENVIRONMENT_VARIABLES.MAX_REQ_PER_MINUTE.name}" env variable...` ) - return DEFAULT_RATE_LIMIT_PER_SECOND + return DEFAULT_RATE_LIMIT_PER_MINUTE } } } @@ -549,7 +549,7 @@ async function getEnvConfig(isStartup?: boolean): Promise { ), dhtMaxInboundStreams: getIntEnvValue(process.env.P2P_dhtMaxInboundStreams, 500), dhtMaxOutboundStreams: getIntEnvValue(process.env.P2P_dhtMaxOutboundStreams, 500), - enableDHTServer: getBoolEnvValue(process.env.P2P_ENABLE_DHT_SERVER, false), + enableDHTServer: getBoolEnvValue('P2P_ENABLE_DHT_SERVER', false), mDNSInterval: getIntEnvValue(process.env.P2P_mDNSInterval, 20e3), // 20 seconds connectionsMaxParallelDials: getIntEnvValue( process.env.P2P_connectionsMaxParallelDials, @@ -617,7 +617,7 @@ async function getEnvConfig(isStartup?: boolean): Promise { isStartup, knownUnsafeURLs ), - isBootstrap: getBoolEnvValue(process.env.IS_BOOTSTRAP, false) + isBootstrap: getBoolEnvValue('IS_BOOTSTRAP', false) } if (!previousConfiguration) { diff --git a/src/utils/constants.ts b/src/utils/constants.ts index 2bf91f761..c141ddd86 100644 --- a/src/utils/constants.ts +++ b/src/utils/constants.ts @@ -266,10 +266,10 @@ export const ENVIRONMENT_VARIABLES: Record = { value: process.env.DASHBOARD, required: false }, - MAX_REQ_PER_SECOND: { - // rate limit per second - name: 'MAX_REQ_PER_SECOND', - value: process.env.MAX_REQ_PER_SECOND, + MAX_REQ_PER_MINUTE: { + // rate limit per minute + name: 'MAX_REQ_PER_MINUTE', + value: process.env.MAX_REQ_PER_MINUTE, required: false }, RATE_DENY_LIST: { @@ -325,8 +325,10 @@ export const ENVIRONMENT_VARIABLES: Record = { } } -// default to 3 requests per second (configurable) -export const DEFAULT_RATE_LIMIT_PER_SECOND = 3 +// default to 30 requests per minute (configurable), per ip/peer +export const DEFAULT_RATE_LIMIT_PER_MINUTE = 30 +// max connections per minute (configurable), all connections +export const DEFAULT_INCOMING_CONNECTIONS_PER_MINUTE = 60 * 2 // 120 requests per minute // Typesense's maximum limit to send 250 hits at a time export const TYPESENSE_HITS_CAP = 250 export const DDO_IDENTIFIER_PREFIX = 'did:op:' From f8e264e0f0d92e1f8ac57e45959a60c20bfd673e Mon Sep 17 00:00:00 2001 From: paulo-ocean Date: Fri, 6 Dec 2024 12:39:20 +0000 Subject: [PATCH 02/11] add global connection slimit, http request --- src/@types/OceanNode.ts | 3 +- src/components/httpRoutes/requestValidator.ts | 59 ++++++++++++++++--- src/utils/config.ts | 31 +++++++++- src/utils/constants.ts | 10 +++- 4 files changed, 91 insertions(+), 12 deletions(-) diff --git a/src/@types/OceanNode.ts b/src/@types/OceanNode.ts index f221a30d8..d213939ff 100644 --- a/src/@types/OceanNode.ts +++ b/src/@types/OceanNode.ts @@ -91,7 +91,8 @@ export interface OceanNodeConfig { assetPurgatoryUrl: string allowedAdmins?: string[] codeHash?: string - rateLimit?: number + rateLimit?: number // per request ip or peer + maxConnections?: number // global, regardless of client address(es) denyList?: DenyList unsafeURLs?: string[] isBootstrap?: boolean diff --git a/src/components/httpRoutes/requestValidator.ts b/src/components/httpRoutes/requestValidator.ts index 6fb6e553b..cfee6566e 100644 --- a/src/components/httpRoutes/requestValidator.ts +++ b/src/components/httpRoutes/requestValidator.ts @@ -1,6 +1,9 @@ import { Request, Response } from 'express' -import { getConfiguration } from '../../utils/index.js' +import { getConfiguration } from '../../utils/config.js' import { HTTP_LOGGER } from '../../utils/logging/common.js' +import { OceanNodeConfig } from '../../@types/OceanNode.js' +import { RequestLimiter } from '../core/handler/handler.js' +import { DEFAULT_MAX_CONNECTIONS_PER_MINUTE } from '../../utils/constants.js' // TODO we should group common stuff, // we have multiple similar validation interfaces @@ -9,23 +12,65 @@ export interface CommonValidation { error?: string } -// midleware to valid client addresses against a denylist +// hold data about last request made +const connectionsData: RequestLimiter = { + lastRequestTime: Date.now(), + requester: '', + numRequests: 0 +} + +// midleware to validate client addresses against a denylist +// it also checks the global rate limit export const requestValidator = async function (req: Request, res: Response, next: any) { // Perform the validations. const requestIP = req.headers['x-forwarded-for'] || req.socket.remoteAddress - const validation = await checkIP(requestIP) + + // grab request time + const requestTime = Date.now() + if (requestTime - connectionsData.lastRequestTime > 1000 * 60) { + // last one was more than 1 minute ago? reset counter + connectionsData.numRequests = 0 + } + // always increment counter + connectionsData.numRequests += 1 + // update time and requester information + connectionsData.lastRequestTime = requestTime + connectionsData.requester = requestIP + + const configuration = await getConfiguration() + + // check if IP is allowed or denied + const ipValidation = await checkIP(requestIP, configuration) // Validation failed, or an error occurred during the external request. - if (!validation.valid) { - res.status(403).send(validation.error) + if (!ipValidation.valid) { + res.status(403).send(ipValidation.error) + return + } + // check global rate limits (not ip related) + const requestRateValidation = checkConnectionsRateLimit(configuration) + if (!requestRateValidation.valid) { + res.status(403).send(requestRateValidation.error) return } // Validation passed. next() } -async function checkIP(requestIP: string | string[]): Promise { +function checkConnectionsRateLimit(configuration: OceanNodeConfig): CommonValidation { + const connectionLimits = + configuration.maxConnections || DEFAULT_MAX_CONNECTIONS_PER_MINUTE + const ok = connectionsData.numRequests <= connectionLimits + return { + valid: ok, + error: ok ? '' : 'Unauthorized request. Rate limit exceeded!' + } +} + +function checkIP( + requestIP: string | string[], + configuration: OceanNodeConfig +): CommonValidation { let onDenyList = false - const configuration = await getConfiguration() if (!Array.isArray(requestIP)) { onDenyList = configuration.denyList?.ips.includes(requestIP) } else { diff --git a/src/utils/config.ts b/src/utils/config.ts index 4ce1f352c..eaf0c7bc4 100644 --- a/src/utils/config.ts +++ b/src/utils/config.ts @@ -14,7 +14,11 @@ import { EnvVariable, hexStringToByteArray } from '../utils/index.js' -import { defaultBootstrapAddresses, knownUnsafeURLs } from '../utils/constants.js' +import { + DEFAULT_MAX_CONNECTIONS_PER_MINUTE, + defaultBootstrapAddresses, + knownUnsafeURLs +} from '../utils/constants.js' import { LOG_LEVELS_STR, GENERIC_EMOJIS, getLoggerLevelEmoji } from './logging/Logger.js' import { RPCS } from '../@types/blockchain' @@ -426,7 +430,7 @@ function logMissingVariableWithDefault(envVariable: EnvVariable) { true ) } -// have a rate limit for handler calls +// have a rate limit for handler calls (per IP address or peer id) function getRateLimit(isStartup: boolean = false) { if (!existsEnvironmentVariable(ENVIRONMENT_VARIABLES.MAX_REQ_PER_MINUTE)) { if (isStartup) { @@ -445,6 +449,28 @@ function getRateLimit(isStartup: boolean = false) { } } +// Global requests limit +function getConnectionsLimit(isStartup: boolean = false) { + if (!existsEnvironmentVariable(ENVIRONMENT_VARIABLES.MAX_CONNECTIONS_PER_MINUTE)) { + if (isStartup) { + logMissingVariableWithDefault(ENVIRONMENT_VARIABLES.MAX_CONNECTIONS_PER_MINUTE) + } + return DEFAULT_RATE_LIMIT_PER_MINUTE + } else { + try { + return getIntEnvValue( + process.env.MAX_CONNECTIONS_PER_MINUTE, + DEFAULT_MAX_CONNECTIONS_PER_MINUTE + ) + } catch (err) { + CONFIG_LOGGER.error( + `Invalid "${ENVIRONMENT_VARIABLES.MAX_CONNECTIONS_PER_MINUTE.name}" env variable...` + ) + return DEFAULT_MAX_CONNECTIONS_PER_MINUTE + } + } +} + // get blocked ips and peer ids function getDenyList(isStartup: boolean = false): DenyList { const defaultDenyList: DenyList = { @@ -611,6 +637,7 @@ async function getEnvConfig(isStartup?: boolean): Promise { assetPurgatoryUrl: getEnvValue(process.env.ASSET_PURGATORY_URL, ''), allowedAdmins: getAllowedAdmins(isStartup), rateLimit: getRateLimit(isStartup), + maxConnections: getConnectionsLimit(isStartup), denyList: getDenyList(isStartup), unsafeURLs: readListFromEnvVariable( ENVIRONMENT_VARIABLES.UNSAFE_URLS, diff --git a/src/utils/constants.ts b/src/utils/constants.ts index c141ddd86..f104eaf23 100644 --- a/src/utils/constants.ts +++ b/src/utils/constants.ts @@ -267,11 +267,17 @@ export const ENVIRONMENT_VARIABLES: Record = { required: false }, MAX_REQ_PER_MINUTE: { - // rate limit per minute + // rate limit per minute (MAX requests per minute for a given IP or peer ID) name: 'MAX_REQ_PER_MINUTE', value: process.env.MAX_REQ_PER_MINUTE, required: false }, + MAX_CONNECTIONS_PER_MINUTE: { + // rate connections limit per minute (MAX requests per minute that the node will process) + name: 'MAX_CONNECTIONS_PER_MINUTE', + value: process.env.MAX_CONNECTIONS_PER_MINUTE, + required: false + }, RATE_DENY_LIST: { // rate limit / deny list (peers and ips) name: 'RATE_DENY_LIST', @@ -328,7 +334,7 @@ export const ENVIRONMENT_VARIABLES: Record = { // default to 30 requests per minute (configurable), per ip/peer export const DEFAULT_RATE_LIMIT_PER_MINUTE = 30 // max connections per minute (configurable), all connections -export const DEFAULT_INCOMING_CONNECTIONS_PER_MINUTE = 60 * 2 // 120 requests per minute +export const DEFAULT_MAX_CONNECTIONS_PER_MINUTE = 60 * 2 // 120 requests per minute // Typesense's maximum limit to send 250 hits at a time export const TYPESENSE_HITS_CAP = 250 export const DDO_IDENTIFIER_PREFIX = 'did:op:' From 67d3209792b63121cc1910f8db44e73bb6970592 Mon Sep 17 00:00:00 2001 From: paulo-ocean Date: Mon, 9 Dec 2024 11:07:53 +0000 Subject: [PATCH 03/11] refactor function + add checks on p2p as well --- src/components/P2P/handleProtocolCommands.ts | 36 +++++++++++++++++-- src/components/httpRoutes/requestValidator.ts | 14 +++++--- src/utils/constants.ts | 2 ++ 3 files changed, 46 insertions(+), 6 deletions(-) diff --git a/src/components/P2P/handleProtocolCommands.ts b/src/components/P2P/handleProtocolCommands.ts index 924b47f05..1034dfb77 100644 --- a/src/components/P2P/handleProtocolCommands.ts +++ b/src/components/P2P/handleProtocolCommands.ts @@ -7,8 +7,17 @@ import { Command } from '../../@types/commands.js' import { P2PCommandResponse } from '../../@types/OceanNode' import { GENERIC_EMOJIS, LOG_LEVELS_STR } from '../../utils/logging/Logger.js' import StreamConcat from 'stream-concat' -import { Handler } from '../core/handler/handler.js' +import { Handler, RequestLimiter } from '../core/handler/handler.js' import { getConfiguration } from '../../utils/index.js' +import { checkConnectionsRateLimit } from '../httpRoutes/requestValidator.js' +import { CONNECTIONS_RATE_INTERVAL } from '../../utils/constants.js' + +// hold data about last request made +const connectionsData: RequestLimiter = { + lastRequestTime: Date.now(), + requester: '', + numRequests: 0 +} export class ReadableString extends Readable { private sent = false @@ -60,7 +69,9 @@ export async function handleProtocolCommands(otherPeerConnection: any) { return status } - const denyList = await (await getConfiguration()).denyList + const configuration = await getConfiguration() + // check deny list configs + const { denyList } = configuration if (denyList.peers.length > 0) { if (denyList.peers.includes(remotePeer.toString())) { P2P_LOGGER.error(`Incoming request denied to peer: ${remotePeer}`) @@ -79,6 +90,27 @@ export async function handleProtocolCommands(otherPeerConnection: any) { return } } + // check connections rate limit + const requestTime = Date.now() + if (requestTime - connectionsData.lastRequestTime > CONNECTIONS_RATE_INTERVAL) { + // last one was more than 1 minute ago? reset counter + connectionsData.numRequests = 0 + } + // always increment counter + connectionsData.numRequests += 1 + // update time and requester information + connectionsData.lastRequestTime = requestTime + connectionsData.requester = remoteAddr + + // check global rate limits (not ip related) + const requestRateValidation = checkConnectionsRateLimit(configuration, connectionsData) + if (!requestRateValidation.valid) { + await closeStreamConnection(otherPeerConnection.connection, remotePeer) + P2P_LOGGER.warn( + `Incoming request denied to peer: ${remotePeer} due to rate limit exceeded!` + ) + return + } try { // eslint-disable-next-line no-unreachable-loop diff --git a/src/components/httpRoutes/requestValidator.ts b/src/components/httpRoutes/requestValidator.ts index cfee6566e..d46a81080 100644 --- a/src/components/httpRoutes/requestValidator.ts +++ b/src/components/httpRoutes/requestValidator.ts @@ -3,7 +3,10 @@ import { getConfiguration } from '../../utils/config.js' import { HTTP_LOGGER } from '../../utils/logging/common.js' import { OceanNodeConfig } from '../../@types/OceanNode.js' import { RequestLimiter } from '../core/handler/handler.js' -import { DEFAULT_MAX_CONNECTIONS_PER_MINUTE } from '../../utils/constants.js' +import { + CONNECTIONS_RATE_INTERVAL, + DEFAULT_MAX_CONNECTIONS_PER_MINUTE +} from '../../utils/constants.js' // TODO we should group common stuff, // we have multiple similar validation interfaces @@ -27,7 +30,7 @@ export const requestValidator = async function (req: Request, res: Response, nex // grab request time const requestTime = Date.now() - if (requestTime - connectionsData.lastRequestTime > 1000 * 60) { + if (requestTime - connectionsData.lastRequestTime > CONNECTIONS_RATE_INTERVAL) { // last one was more than 1 minute ago? reset counter connectionsData.numRequests = 0 } @@ -47,7 +50,7 @@ export const requestValidator = async function (req: Request, res: Response, nex return } // check global rate limits (not ip related) - const requestRateValidation = checkConnectionsRateLimit(configuration) + const requestRateValidation = checkConnectionsRateLimit(configuration, connectionsData) if (!requestRateValidation.valid) { res.status(403).send(requestRateValidation.error) return @@ -56,7 +59,10 @@ export const requestValidator = async function (req: Request, res: Response, nex next() } -function checkConnectionsRateLimit(configuration: OceanNodeConfig): CommonValidation { +export function checkConnectionsRateLimit( + configuration: OceanNodeConfig, + connectionsData: RequestLimiter +): CommonValidation { const connectionLimits = configuration.maxConnections || DEFAULT_MAX_CONNECTIONS_PER_MINUTE const ok = connectionsData.numRequests <= connectionLimits diff --git a/src/utils/constants.ts b/src/utils/constants.ts index f104eaf23..8d69af33f 100644 --- a/src/utils/constants.ts +++ b/src/utils/constants.ts @@ -335,6 +335,8 @@ export const ENVIRONMENT_VARIABLES: Record = { export const DEFAULT_RATE_LIMIT_PER_MINUTE = 30 // max connections per minute (configurable), all connections export const DEFAULT_MAX_CONNECTIONS_PER_MINUTE = 60 * 2 // 120 requests per minute +// 1 minute +export const CONNECTIONS_RATE_INTERVAL = 60 * 1000 // Typesense's maximum limit to send 250 hits at a time export const TYPESENSE_HITS_CAP = 250 export const DDO_IDENTIFIER_PREFIX = 'did:op:' From 94d9fdb8b71f8931acf3278ff2733ce9951534b8 Mon Sep 17 00:00:00 2001 From: paulo-ocean Date: Mon, 9 Dec 2024 13:34:14 +0000 Subject: [PATCH 04/11] clear history after certain threshold --- src/components/core/handler/handler.ts | 6 ++++++ src/utils/constants.ts | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/components/core/handler/handler.ts b/src/components/core/handler/handler.ts index 179ab0d78..cbf0d5a8c 100644 --- a/src/components/core/handler/handler.ts +++ b/src/components/core/handler/handler.ts @@ -9,6 +9,7 @@ import { import { getConfiguration } from '../../../utils/index.js' import { CORE_LOGGER } from '../../../utils/logging/common.js' import { ReadableString } from '../../P2P/handlers.js' +import { CONNECTION_HISTORY_DELETE_THRESHOLD } from '../../../utils/constants.js' export interface RequestLimiter { requester: string | string[] // IP address or peer ID @@ -43,6 +44,11 @@ export abstract class Handler implements ICommandHandler { const requestTime = new Date().getTime() let isOK = true + // we have to clear this from time to time, so it does not grow forever + if (this.requestMap.size > CONNECTION_HISTORY_DELETE_THRESHOLD) { + this.requestMap.clear() + } + const self = this // common stuff const updateRequestData = function (remoteCaller: string) { diff --git a/src/utils/constants.ts b/src/utils/constants.ts index 8d69af33f..be06225ee 100644 --- a/src/utils/constants.ts +++ b/src/utils/constants.ts @@ -330,7 +330,7 @@ export const ENVIRONMENT_VARIABLES: Record = { required: false } } - +export const CONNECTION_HISTORY_DELETE_THRESHOLD = 300 // default to 30 requests per minute (configurable), per ip/peer export const DEFAULT_RATE_LIMIT_PER_MINUTE = 30 // max connections per minute (configurable), all connections From 1ac49e4ddcf7ac732a385cf0ac194b44004a564a Mon Sep 17 00:00:00 2001 From: paulo-ocean Date: Mon, 9 Dec 2024 14:54:59 +0000 Subject: [PATCH 05/11] add reference in docs --- docs/env.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/env.md b/docs/env.md index 2ed6937a5..d23b46512 100644 --- a/docs/env.md +++ b/docs/env.md @@ -23,7 +23,8 @@ Environmental variables are also tracked in `ENVIRONMENT_VARIABLES` within `src/ - `ALLOWED_ADMINS`: Sets the public address of accounts which have access to admin endpoints e.g. shutting down the node. Example: `"[\"0x967da4048cD07aB37855c090aAF366e4ce1b9F48\",\"0x388C818CA8B9251b393131C08a736A67ccB19297\"]"` - `DASHBOARD`: If `false` the dashboard will not run. If not set or `true` the dashboard will start with the node. Example: `false` - `RATE_DENY_LIST`: Blocked list of IPs and peer IDs. Example: `"{ \"peers\": [\"16Uiu2HAkuYfgjXoGcSSLSpRPD6XtUgV71t5RqmTmcqdbmrWY9MJo\"], \"ips\": [\"127.0.0.1\"] }"` -- `MAX_REQ_PER_MINUTE`: Number of requests per minute allowed by the same client. Example: `30` +- `MAX_REQ_PER_MINUTE`: Number of requests per minute allowed by the same client (IP or Peer id). Example: `30` +- `MAX_CONNECTIONS_PER_MINUTE`: Max number of requests allowed per minute (all clients). Example: `120` - `MAX_CHECKSUM_LENGTH`: Define the maximum length for a file if checksum is required (Mb). Example: `10` - `IS_BOOTSTRAP`: Is this node to be used as bootstrap node or not. Default is `false`. From 05e2203157b3bdebf2b68d52b4ee6525434ba3ba Mon Sep 17 00:00:00 2001 From: paulo-ocean Date: Tue, 10 Dec 2024 17:36:46 +0000 Subject: [PATCH 06/11] refactor, move rate hanling to node instance, not handler --- src/OceanNode.ts | 21 +++++++++ src/components/P2P/handleProtocolCommands.ts | 21 +++++++-- src/components/core/handler/handler.ts | 39 +++++++---------- src/components/httpRoutes/requestValidator.ts | 2 +- src/test/unit/networking.test.ts | 43 ++++++++++++++++--- 5 files changed, 92 insertions(+), 34 deletions(-) diff --git a/src/OceanNode.ts b/src/OceanNode.ts index ba40feac9..e3dff43d0 100644 --- a/src/OceanNode.ts +++ b/src/OceanNode.ts @@ -11,6 +11,17 @@ import { pipe } from 'it-pipe' import { GENERIC_EMOJIS, LOG_LEVELS_STR } from './utils/logging/Logger.js' import { Handler } from './components/core/handler/handler.js' import { C2DEngines } from './components/c2d/compute_engines.js' + +export interface RequestLimiter { + requester: string | string[] // IP address or peer ID + lastRequestTime: number // time of the last request done (in miliseconds) + numRequests: number // number of requests done in the specific time period +} + +export interface RequestDataCheck { + valid: boolean + updatedRequestData: RequestLimiter +} export class OceanNode { // eslint-disable-next-line no-use-before-define private static instance: OceanNode @@ -20,6 +31,7 @@ export class OceanNode { private c2dEngines: C2DEngines // requester private remoteCaller: string | string[] + private requestMap: Map // eslint-disable-next-line no-useless-constructor private constructor( private db?: Database, @@ -28,6 +40,7 @@ export class OceanNode { private indexer?: OceanIndexer ) { this.coreHandlers = CoreHandlersRegistry.getInstance(this) + this.requestMap = new Map() if (node) { node.setCoreHandlers(this.coreHandlers) } @@ -95,6 +108,14 @@ export class OceanNode { return this.remoteCaller } + public getRequestMapSize(): number { + return this.requestMap.size + } + + public getRequestMap(): Map { + return this.requestMap + } + /** * Use this method to direct calls to the node as node cannot dial into itself * @param message command message diff --git a/src/components/P2P/handleProtocolCommands.ts b/src/components/P2P/handleProtocolCommands.ts index 1034dfb77..11e290c0e 100644 --- a/src/components/P2P/handleProtocolCommands.ts +++ b/src/components/P2P/handleProtocolCommands.ts @@ -7,10 +7,11 @@ import { Command } from '../../@types/commands.js' import { P2PCommandResponse } from '../../@types/OceanNode' import { GENERIC_EMOJIS, LOG_LEVELS_STR } from '../../utils/logging/Logger.js' import StreamConcat from 'stream-concat' -import { Handler, RequestLimiter } from '../core/handler/handler.js' +import { Handler } from '../core/handler/handler.js' import { getConfiguration } from '../../utils/index.js' import { checkConnectionsRateLimit } from '../httpRoutes/requestValidator.js' import { CONNECTIONS_RATE_INTERVAL } from '../../utils/constants.js' +import { RequestLimiter } from '../../OceanNode.js' // hold data about last request made const connectionsData: RequestLimiter = { @@ -74,7 +75,9 @@ export async function handleProtocolCommands(otherPeerConnection: any) { const { denyList } = configuration if (denyList.peers.length > 0) { if (denyList.peers.includes(remotePeer.toString())) { - P2P_LOGGER.error(`Incoming request denied to peer: ${remotePeer}`) + P2P_LOGGER.warn( + `Incoming request denied to peer: ${remotePeer} (peer its on deny list)` + ) if (connectionStatus === 'open') { statusStream = new ReadableString( @@ -105,10 +108,20 @@ export async function handleProtocolCommands(otherPeerConnection: any) { // check global rate limits (not ip related) const requestRateValidation = checkConnectionsRateLimit(configuration, connectionsData) if (!requestRateValidation.valid) { - await closeStreamConnection(otherPeerConnection.connection, remotePeer) P2P_LOGGER.warn( - `Incoming request denied to peer: ${remotePeer} due to rate limit exceeded!` + `Incoming request denied to peer: ${remotePeer} (rate limit exceeded)` ) + if (connectionStatus === 'open') { + statusStream = new ReadableString( + JSON.stringify(buildWrongCommandStatus(403, 'Rate limit exceeded')) + ) + try { + await pipe(statusStream, otherPeerConnection.stream.sink) + } catch (e) { + P2P_LOGGER.error(e) + } + } + await closeStreamConnection(otherPeerConnection.connection, remotePeer) return } diff --git a/src/components/core/handler/handler.ts b/src/components/core/handler/handler.ts index cbf0d5a8c..f66a74ef6 100644 --- a/src/components/core/handler/handler.ts +++ b/src/components/core/handler/handler.ts @@ -1,5 +1,5 @@ import { P2PCommandResponse } from '../../../@types/OceanNode.js' -import { OceanNode } from '../../../OceanNode.js' +import { OceanNode, RequestDataCheck, RequestLimiter } from '../../../OceanNode.js' import { Command, ICommandHandler } from '../../../@types/commands.js' import { ValidateParams, @@ -11,22 +11,10 @@ import { CORE_LOGGER } from '../../../utils/logging/common.js' import { ReadableString } from '../../P2P/handlers.js' import { CONNECTION_HISTORY_DELETE_THRESHOLD } from '../../../utils/constants.js' -export interface RequestLimiter { - requester: string | string[] // IP address or peer ID - lastRequestTime: number // time of the last request done (in miliseconds) - numRequests: number // number of requests done in the specific time period -} - -export interface RequestDataCheck { - valid: boolean - updatedRequestData: RequestLimiter -} export abstract class Handler implements ICommandHandler { - private nodeInstance?: OceanNode - private requestMap: Map + private nodeInstance: OceanNode public constructor(oceanNode: OceanNode) { this.nodeInstance = oceanNode - this.requestMap = new Map() } abstract validate(command: Command): ValidateParams @@ -39,14 +27,16 @@ export abstract class Handler implements ICommandHandler { // TODO LOG, implement all handlers async checkRateLimit(): Promise { + const requestMap = this.getOceanNode().getRequestMap() const ratePerMinute = (await getConfiguration()).rateLimit const caller: string | string[] = this.getOceanNode().getRemoteCaller() const requestTime = new Date().getTime() let isOK = true // we have to clear this from time to time, so it does not grow forever - if (this.requestMap.size > CONNECTION_HISTORY_DELETE_THRESHOLD) { - this.requestMap.clear() + if (requestMap.size > CONNECTION_HISTORY_DELETE_THRESHOLD) { + console.log('will clear the connection history') + requestMap.clear() } const self = this @@ -58,19 +48,19 @@ export abstract class Handler implements ICommandHandler { ratePerMinute ) isOK = updatedRequestData.valid - self.requestMap.set(remoteCaller, updatedRequestData.updatedRequestData) + requestMap.set(remoteCaller, updatedRequestData.updatedRequestData) } let data: RequestLimiter = null if (Array.isArray(caller)) { for (const remote of caller) { - if (!this.requestMap.has(remote)) { + if (!requestMap.has(remote)) { data = { requester: remote, lastRequestTime: requestTime, numRequests: 1 } - this.requestMap.set(remote, data) + requestMap.set(remote, data) } else { updateRequestData(remote) } @@ -78,20 +68,20 @@ export abstract class Handler implements ICommandHandler { if (!isOK) { CORE_LOGGER.warn( `Request denied (rate limit exceeded) for remote caller ${remote}. Current request map: ${JSON.stringify( - this.requestMap.get(remote) + requestMap.get(remote) )}` ) return false } } } else { - if (!this.requestMap.has(caller)) { + if (!requestMap.has(caller)) { data = { requester: caller, lastRequestTime: requestTime, numRequests: 1 } - this.requestMap.set(caller, data) + requestMap.set(caller, data) return true } else { updateRequestData(caller) @@ -99,7 +89,7 @@ export abstract class Handler implements ICommandHandler { if (!isOK) { CORE_LOGGER.warn( `Request denied (rate limit exceeded) for remote caller ${caller}. Current request map: ${JSON.stringify( - this.requestMap.get(caller) + requestMap.get(caller) )}` ) } @@ -119,7 +109,8 @@ export abstract class Handler implements ICommandHandler { currentTime: number, ratePerMinute: number ): RequestDataCheck { - const requestData: RequestLimiter = this.requestMap.get(remote) + const requestMap = this.getOceanNode().getRequestMap() + const requestData: RequestLimiter = requestMap.get(remote) const diffMinutes = ((currentTime - requestData.lastRequestTime) / 1000) * 60 // more than 1 minute difference means no problem if (diffMinutes >= 1) { diff --git a/src/components/httpRoutes/requestValidator.ts b/src/components/httpRoutes/requestValidator.ts index d46a81080..9b24ddbb0 100644 --- a/src/components/httpRoutes/requestValidator.ts +++ b/src/components/httpRoutes/requestValidator.ts @@ -2,7 +2,7 @@ import { Request, Response } from 'express' import { getConfiguration } from '../../utils/config.js' import { HTTP_LOGGER } from '../../utils/logging/common.js' import { OceanNodeConfig } from '../../@types/OceanNode.js' -import { RequestLimiter } from '../core/handler/handler.js' +import { RequestLimiter } from '../../OceanNode.js' import { CONNECTIONS_RATE_INTERVAL, DEFAULT_MAX_CONNECTIONS_PER_MINUTE diff --git a/src/test/unit/networking.test.ts b/src/test/unit/networking.test.ts index b2e6faac6..c030d1b58 100644 --- a/src/test/unit/networking.test.ts +++ b/src/test/unit/networking.test.ts @@ -1,11 +1,14 @@ import { DEFAULT_RATE_LIMIT_PER_MINUTE, + // DEFAULT_MAX_CONNECTIONS_PER_MINUTE, ENVIRONMENT_VARIABLES, PROTOCOL_COMMANDS, - getConfiguration + getConfiguration, + CONNECTION_HISTORY_DELETE_THRESHOLD } from '../../utils/index.js' import { expect } from 'chai' import { + DEFAULT_TEST_TIMEOUT, OverrideEnvConfig, buildEnvOverrideConfig, setupEnvironment, @@ -183,13 +186,12 @@ describe('Test rate limitations and deny list settings', () => { const ips = ['127.0.0.2', '127.0.0.3', '127.0.0.4', '127.0.0.5'] const rateLimitResponses = [] + const statusHandler: StatusHandler = CoreHandlersRegistry.getInstance( + node + ).getHandler(PROTOCOL_COMMANDS.STATUS) for (let i = 0; i < ips.length; i++) { node.setRemoteCaller(ips[i]) - const statusHandler: StatusHandler = CoreHandlersRegistry.getInstance( - node - ).getHandler(PROTOCOL_COMMANDS.STATUS) - const rateResp = await statusHandler.checkRateLimit() rateLimitResponses.push(rateResp) } @@ -197,6 +199,37 @@ describe('Test rate limitations and deny list settings', () => { // should have 4 valid responses expect(filtered.length).to.be.equal(ips.length) }) + + it('Test global rate limit, clear map after MAX (handler level) ', async function () { + // after more than CONNECTION_HISTORY_DELETE_THRESHOLD connections the map will be cleared + this.timeout(DEFAULT_TEST_TIMEOUT * 3) + let originalIPPiece = '127.0.' + + const rateLimitResponses = [] + + const statusHandler: StatusHandler = CoreHandlersRegistry.getInstance( + node + ).getHandler(PROTOCOL_COMMANDS.STATUS) + + const aboveLimit = 20 + for (let i = 0, x = 0; i < CONNECTION_HISTORY_DELETE_THRESHOLD + aboveLimit; i++) { + const ip = originalIPPiece + x // start at 127.0.0.2 + node.setRemoteCaller(ip) + const rateResp = await statusHandler.checkRateLimit() + rateLimitResponses.push(rateResp) + x++ + // start back + if (x > 254) { + x = 0 + originalIPPiece = '127.0.0.' // next piece + } + } + // it should clear the history after CONNECTION_HISTORY_DELETE_THRESHOLD + expect(statusHandler.getOceanNode().getRequestMapSize()).to.be.lessThanOrEqual( + aboveLimit + ) + }) + after(async () => { await tearDownEnvironment(envOverrides) }) From 105703be77cae86ab1e2d425cdf8ca04ff925a4c Mon Sep 17 00:00:00 2001 From: paulo-ocean Date: Wed, 11 Dec 2024 12:58:38 +0000 Subject: [PATCH 07/11] update script, + add clear cache log --- scripts/ocean-node-quickstart.sh | 1 + src/components/core/handler/handler.ts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/ocean-node-quickstart.sh b/scripts/ocean-node-quickstart.sh index e2fd601f0..7eb5145a5 100755 --- a/scripts/ocean-node-quickstart.sh +++ b/scripts/ocean-node-quickstart.sh @@ -156,6 +156,7 @@ services: DASHBOARD: 'true' # RATE_DENY_LIST: '' # MAX_REQ_PER_MINUTE: '' +# MAX_CONNECTIONS_PER_MINUTE: '' # MAX_CHECKSUM_LENGTH: '' # LOG_LEVEL: '' HTTP_API_PORT: '$HTTP_API_PORT' diff --git a/src/components/core/handler/handler.ts b/src/components/core/handler/handler.ts index f66a74ef6..49b3b3c75 100644 --- a/src/components/core/handler/handler.ts +++ b/src/components/core/handler/handler.ts @@ -35,7 +35,7 @@ export abstract class Handler implements ICommandHandler { // we have to clear this from time to time, so it does not grow forever if (requestMap.size > CONNECTION_HISTORY_DELETE_THRESHOLD) { - console.log('will clear the connection history') + CORE_LOGGER.info('Request history reached threeshold, cleaning cache...') requestMap.clear() } From 78cb82e13dc8cb3f65cd1dc7ba6753920c6d995e Mon Sep 17 00:00:00 2001 From: paulo-ocean Date: Fri, 13 Dec 2024 13:08:32 +0000 Subject: [PATCH 08/11] update seeting on CI for test --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b71cd3e0b..93c55f72f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -278,6 +278,7 @@ jobs: P2P_ENABLE_AUTONAT: 'false' ALLOWED_ADMINS: '["0xe2DD09d719Da89e5a3D0F2549c7E24566e947260"]' DB_TYPE: 'elasticsearch' + MAX_REQ_PER_MINUTE: 120 - name: Check Ocean Node is running run: | for i in $(seq 1 90); do From b9a400746f534853bb9c228b4699c3599196859b Mon Sep 17 00:00:00 2001 From: paulo-ocean Date: Fri, 13 Dec 2024 13:44:41 +0000 Subject: [PATCH 09/11] update setting on CI for system test --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 93c55f72f..7793e6f05 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -278,7 +278,7 @@ jobs: P2P_ENABLE_AUTONAT: 'false' ALLOWED_ADMINS: '["0xe2DD09d719Da89e5a3D0F2549c7E24566e947260"]' DB_TYPE: 'elasticsearch' - MAX_REQ_PER_MINUTE: 120 + MAX_REQ_PER_MINUTE: 320 - name: Check Ocean Node is running run: | for i in $(seq 1 90); do From f2cfda6e42b15a5183796806ac79841ca0f6bc12 Mon Sep 17 00:00:00 2001 From: paulo-ocean Date: Mon, 16 Dec 2024 09:29:44 +0000 Subject: [PATCH 10/11] just debug system test --- src/components/httpRoutes/requestValidator.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/components/httpRoutes/requestValidator.ts b/src/components/httpRoutes/requestValidator.ts index 9b24ddbb0..719757460 100644 --- a/src/components/httpRoutes/requestValidator.ts +++ b/src/components/httpRoutes/requestValidator.ts @@ -66,6 +66,11 @@ export function checkConnectionsRateLimit( const connectionLimits = configuration.maxConnections || DEFAULT_MAX_CONNECTIONS_PER_MINUTE const ok = connectionsData.numRequests <= connectionLimits + if (!ok) { + console.log( + `Connection limit exceeded ${connectionsData.numRequests} limit: ${connectionLimits}` + ) + } return { valid: ok, error: ok ? '' : 'Unauthorized request. Rate limit exceeded!' From f9a5c0384819608e292402a66ea9e96c4d894621 Mon Sep 17 00:00:00 2001 From: paulo-ocean Date: Mon, 16 Dec 2024 09:40:21 +0000 Subject: [PATCH 11/11] add setting for max connection on ci --- .github/workflows/ci.yml | 1 + src/components/httpRoutes/requestValidator.ts | 5 ----- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7793e6f05..14f83117f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -279,6 +279,7 @@ jobs: ALLOWED_ADMINS: '["0xe2DD09d719Da89e5a3D0F2549c7E24566e947260"]' DB_TYPE: 'elasticsearch' MAX_REQ_PER_MINUTE: 320 + MAX_CONNECTIONS_PER_MINUTE: 320 - name: Check Ocean Node is running run: | for i in $(seq 1 90); do diff --git a/src/components/httpRoutes/requestValidator.ts b/src/components/httpRoutes/requestValidator.ts index 719757460..9b24ddbb0 100644 --- a/src/components/httpRoutes/requestValidator.ts +++ b/src/components/httpRoutes/requestValidator.ts @@ -66,11 +66,6 @@ export function checkConnectionsRateLimit( const connectionLimits = configuration.maxConnections || DEFAULT_MAX_CONNECTIONS_PER_MINUTE const ok = connectionsData.numRequests <= connectionLimits - if (!ok) { - console.log( - `Connection limit exceeded ${connectionsData.numRequests} limit: ${connectionLimits}` - ) - } return { valid: ok, error: ok ? '' : 'Unauthorized request. Rate limit exceeded!'