From ab2283ef35f443aa2027c28ca4e4484611135cf8 Mon Sep 17 00:00:00 2001 From: paulo-ocean Date: Fri, 6 Dec 2024 09:42:52 +0000 Subject: [PATCH] refactor connections rate, per minute not second --- .env.example | 2 +- docs/dockerDeployment.md | 2 +- docs/env.md | 2 +- scripts/ocean-node-quickstart.sh | 2 +- src/components/core/handler/handler.ts | 16 ++++++++-------- src/test/unit/networking.test.ts | 8 ++++---- src/utils/config.ts | 18 +++++++++--------- src/utils/constants.ts | 14 ++++++++------ 8 files changed, 33 insertions(+), 31 deletions(-) diff --git a/.env.example b/.env.example index 264e0dfdc..67a9312fe 100644 --- a/.env.example +++ b/.env.example @@ -27,7 +27,7 @@ export INDEXER_INTERVAL= export ALLOWED_ADMINS= export DASHBOARD=true export RATE_DENY_LIST= -export MAX_REQ_PER_SECOND= +export MAX_REQ_PER_MINUTE= export MAX_CHECKSUM_LENGTH= export LOG_LEVEL= export HTTP_API_PORT= diff --git a/docs/dockerDeployment.md b/docs/dockerDeployment.md index ef1d675d9..dfa3d3393 100644 --- a/docs/dockerDeployment.md +++ b/docs/dockerDeployment.md @@ -99,7 +99,7 @@ services: # INDEXER_INTERVAL: '' DASHBOARD: 'true' # RATE_DENY_LIST: '' - # MAX_REQ_PER_SECOND: '' + # MAX_REQ_PER_MINUTE: '' # MAX_CHECKSUM_LENGTH: '' # LOG_LEVEL: '' HTTP_API_PORT: '8000' diff --git a/docs/env.md b/docs/env.md index 9c718177c..2ed6937a5 100644 --- a/docs/env.md +++ b/docs/env.md @@ -23,7 +23,7 @@ Environmental variables are also tracked in `ENVIRONMENT_VARIABLES` within `src/ - `ALLOWED_ADMINS`: Sets the public address of accounts which have access to admin endpoints e.g. shutting down the node. Example: `"[\"0x967da4048cD07aB37855c090aAF366e4ce1b9F48\",\"0x388C818CA8B9251b393131C08a736A67ccB19297\"]"` - `DASHBOARD`: If `false` the dashboard will not run. If not set or `true` the dashboard will start with the node. Example: `false` - `RATE_DENY_LIST`: Blocked list of IPs and peer IDs. Example: `"{ \"peers\": [\"16Uiu2HAkuYfgjXoGcSSLSpRPD6XtUgV71t5RqmTmcqdbmrWY9MJo\"], \"ips\": [\"127.0.0.1\"] }"` -- `MAX_REQ_PER_SECOND`: Number of requests per second allowed by the same client. Example: `3` +- `MAX_REQ_PER_MINUTE`: Number of requests per minute allowed by the same client. Example: `30` - `MAX_CHECKSUM_LENGTH`: Define the maximum length for a file if checksum is required (Mb). Example: `10` - `IS_BOOTSTRAP`: Is this node to be used as bootstrap node or not. Default is `false`. diff --git a/scripts/ocean-node-quickstart.sh b/scripts/ocean-node-quickstart.sh index 4c3085338..e2fd601f0 100755 --- a/scripts/ocean-node-quickstart.sh +++ b/scripts/ocean-node-quickstart.sh @@ -155,7 +155,7 @@ services: # INDEXER_INTERVAL: '' DASHBOARD: 'true' # RATE_DENY_LIST: '' -# MAX_REQ_PER_SECOND: '' +# MAX_REQ_PER_MINUTE: '' # MAX_CHECKSUM_LENGTH: '' # LOG_LEVEL: '' HTTP_API_PORT: '$HTTP_API_PORT' diff --git a/src/components/core/handler/handler.ts b/src/components/core/handler/handler.ts index 93aafdc94..179ab0d78 100644 --- a/src/components/core/handler/handler.ts +++ b/src/components/core/handler/handler.ts @@ -38,7 +38,7 @@ export abstract class Handler implements ICommandHandler { // TODO LOG, implement all handlers async checkRateLimit(): Promise { - const ratePerSecond = (await getConfiguration()).rateLimit + const ratePerMinute = (await getConfiguration()).rateLimit const caller: string | string[] = this.getOceanNode().getRemoteCaller() const requestTime = new Date().getTime() let isOK = true @@ -49,7 +49,7 @@ export abstract class Handler implements ICommandHandler { const updatedRequestData = self.checkRequestData( remoteCaller, requestTime, - ratePerSecond + ratePerMinute ) isOK = updatedRequestData.valid self.requestMap.set(remoteCaller, updatedRequestData.updatedRequestData) @@ -105,18 +105,18 @@ export abstract class Handler implements ICommandHandler { /** * Checks if the request is within the rate limit defined * @param remote remote endpoint (ip or peer identifier) - * @param ratePerSecond number of calls per second allowed + * @param ratePerMinute number of calls per minute allowed (per ip or peer identifier) * @returns updated request data */ checkRequestData( remote: string, currentTime: number, - ratePerSecond: number + ratePerMinute: number ): RequestDataCheck { const requestData: RequestLimiter = this.requestMap.get(remote) - const diffSeconds = (currentTime - requestData.lastRequestTime) / 1000 - // more than 1 sec difference means no problem - if (diffSeconds >= 1) { + const diffMinutes = ((currentTime - requestData.lastRequestTime) / 1000) * 60 + // more than 1 minute difference means no problem + if (diffMinutes >= 1) { // its fine requestData.lastRequestTime = currentTime requestData.numRequests = 1 @@ -128,7 +128,7 @@ export abstract class Handler implements ICommandHandler { // requests in the same interval of 1 second requestData.numRequests++ return { - valid: requestData.numRequests <= ratePerSecond, + valid: requestData.numRequests <= ratePerMinute, updatedRequestData: requestData } } diff --git a/src/test/unit/networking.test.ts b/src/test/unit/networking.test.ts index 36e4a2a19..b2e6faac6 100644 --- a/src/test/unit/networking.test.ts +++ b/src/test/unit/networking.test.ts @@ -1,5 +1,5 @@ import { - DEFAULT_RATE_LIMIT_PER_SECOND, + DEFAULT_RATE_LIMIT_PER_MINUTE, ENVIRONMENT_VARIABLES, PROTOCOL_COMMANDS, getConfiguration @@ -105,7 +105,7 @@ describe('Test rate limitations and deny list defaults', () => { // const node: OceanNode = OceanNode.getInstance() before(async () => { envOverrides = buildEnvOverrideConfig( - [ENVIRONMENT_VARIABLES.RATE_DENY_LIST, ENVIRONMENT_VARIABLES.MAX_REQ_PER_SECOND], + [ENVIRONMENT_VARIABLES.RATE_DENY_LIST, ENVIRONMENT_VARIABLES.MAX_REQ_PER_MINUTE], [undefined, undefined] ) await setupEnvironment(null, envOverrides) @@ -115,7 +115,7 @@ describe('Test rate limitations and deny list defaults', () => { const config = await getConfiguration(true) expect(config.denyList.ips).to.be.length(0) expect(config.denyList.peers).to.be.length(0) - expect(config.rateLimit).to.be.equal(DEFAULT_RATE_LIMIT_PER_SECOND) + expect(config.rateLimit).to.be.equal(DEFAULT_RATE_LIMIT_PER_MINUTE) }) // put it back @@ -132,7 +132,7 @@ describe('Test rate limitations and deny list settings', () => { [ ENVIRONMENT_VARIABLES.PRIVATE_KEY, ENVIRONMENT_VARIABLES.RATE_DENY_LIST, - ENVIRONMENT_VARIABLES.MAX_REQ_PER_SECOND + ENVIRONMENT_VARIABLES.MAX_REQ_PER_MINUTE ], [ '0xcb345bd2b11264d523ddaf383094e2675c420a17511c3102a53817f13474a7ff', diff --git a/src/utils/config.ts b/src/utils/config.ts index c96b4f3ff..4ce1f352c 100644 --- a/src/utils/config.ts +++ b/src/utils/config.ts @@ -9,7 +9,7 @@ import { C2DClusterType } from '../@types/C2D.js' import { createFromPrivKey } from '@libp2p/peer-id-factory' import { keys } from '@libp2p/crypto' import { - DEFAULT_RATE_LIMIT_PER_SECOND, + DEFAULT_RATE_LIMIT_PER_MINUTE, ENVIRONMENT_VARIABLES, EnvVariable, hexStringToByteArray @@ -428,19 +428,19 @@ function logMissingVariableWithDefault(envVariable: EnvVariable) { } // have a rate limit for handler calls function getRateLimit(isStartup: boolean = false) { - if (!existsEnvironmentVariable(ENVIRONMENT_VARIABLES.MAX_REQ_PER_SECOND)) { + if (!existsEnvironmentVariable(ENVIRONMENT_VARIABLES.MAX_REQ_PER_MINUTE)) { if (isStartup) { - logMissingVariableWithDefault(ENVIRONMENT_VARIABLES.MAX_REQ_PER_SECOND) + logMissingVariableWithDefault(ENVIRONMENT_VARIABLES.MAX_REQ_PER_MINUTE) } - return DEFAULT_RATE_LIMIT_PER_SECOND + return DEFAULT_RATE_LIMIT_PER_MINUTE } else { try { - return getIntEnvValue(process.env.MAX_REQ_PER_SECOND, DEFAULT_RATE_LIMIT_PER_SECOND) + return getIntEnvValue(process.env.MAX_REQ_PER_MINUTE, DEFAULT_RATE_LIMIT_PER_MINUTE) } catch (err) { CONFIG_LOGGER.error( - `Invalid "${ENVIRONMENT_VARIABLES.MAX_REQ_PER_SECOND.name}" env variable...` + `Invalid "${ENVIRONMENT_VARIABLES.MAX_REQ_PER_MINUTE.name}" env variable...` ) - return DEFAULT_RATE_LIMIT_PER_SECOND + return DEFAULT_RATE_LIMIT_PER_MINUTE } } } @@ -549,7 +549,7 @@ async function getEnvConfig(isStartup?: boolean): Promise { ), dhtMaxInboundStreams: getIntEnvValue(process.env.P2P_dhtMaxInboundStreams, 500), dhtMaxOutboundStreams: getIntEnvValue(process.env.P2P_dhtMaxOutboundStreams, 500), - enableDHTServer: getBoolEnvValue(process.env.P2P_ENABLE_DHT_SERVER, false), + enableDHTServer: getBoolEnvValue('P2P_ENABLE_DHT_SERVER', false), mDNSInterval: getIntEnvValue(process.env.P2P_mDNSInterval, 20e3), // 20 seconds connectionsMaxParallelDials: getIntEnvValue( process.env.P2P_connectionsMaxParallelDials, @@ -617,7 +617,7 @@ async function getEnvConfig(isStartup?: boolean): Promise { isStartup, knownUnsafeURLs ), - isBootstrap: getBoolEnvValue(process.env.IS_BOOTSTRAP, false) + isBootstrap: getBoolEnvValue('IS_BOOTSTRAP', false) } if (!previousConfiguration) { diff --git a/src/utils/constants.ts b/src/utils/constants.ts index 2bf91f761..c141ddd86 100644 --- a/src/utils/constants.ts +++ b/src/utils/constants.ts @@ -266,10 +266,10 @@ export const ENVIRONMENT_VARIABLES: Record = { value: process.env.DASHBOARD, required: false }, - MAX_REQ_PER_SECOND: { - // rate limit per second - name: 'MAX_REQ_PER_SECOND', - value: process.env.MAX_REQ_PER_SECOND, + MAX_REQ_PER_MINUTE: { + // rate limit per minute + name: 'MAX_REQ_PER_MINUTE', + value: process.env.MAX_REQ_PER_MINUTE, required: false }, RATE_DENY_LIST: { @@ -325,8 +325,10 @@ export const ENVIRONMENT_VARIABLES: Record = { } } -// default to 3 requests per second (configurable) -export const DEFAULT_RATE_LIMIT_PER_SECOND = 3 +// default to 30 requests per minute (configurable), per ip/peer +export const DEFAULT_RATE_LIMIT_PER_MINUTE = 30 +// max connections per minute (configurable), all connections +export const DEFAULT_INCOMING_CONNECTIONS_PER_MINUTE = 60 * 2 // 120 requests per minute // Typesense's maximum limit to send 250 hits at a time export const TYPESENSE_HITS_CAP = 250 export const DDO_IDENTIFIER_PREFIX = 'did:op:'