From cf7e3015a2d9a29539aa2298222f2cf92fe05d2b Mon Sep 17 00:00:00 2001 From: Lean Mendoza Date: Tue, 19 Nov 2024 07:20:31 -0300 Subject: [PATCH] other approach --- content/src/components.ts | 2 + .../handlers/active-entities-handler.ts | 17 +++ .../handlers/active-entities-ids-handler.ts | 41 ------- content/src/controller/routes.ts | 23 ++-- content/src/logic/deployments.ts | 44 ++++--- content/src/ports/activeEntities.ts | 108 +++++++++++++++--- 6 files changed, 153 insertions(+), 82 deletions(-) delete mode 100644 content/src/controller/handlers/active-entities-ids-handler.ts diff --git a/content/src/components.ts b/content/src/components.ts index bf04f8d55..dd354b925 100644 --- a/content/src/components.ts +++ b/content/src/components.ts @@ -186,6 +186,8 @@ export async function initComponentsWithEnv(env: Environment): Promise +): Promise<{ status: 200; body: Pick[] }> { + const { activeEntities, denylist } = context.components + const entities: Entity[] = activeEntities.getAllCachedScenes().filter((result) => !denylist.isDenylisted(result.id)) + const mapping = entities.map((entity) => ({ + id: entity.id, + pointers: entity.pointers, + timestamp: entity.timestamp + })) + return { + status: 200, + body: mapping + } +} diff --git a/content/src/controller/handlers/active-entities-ids-handler.ts b/content/src/controller/handlers/active-entities-ids-handler.ts deleted file mode 100644 index e9456c9ed..000000000 --- a/content/src/controller/handlers/active-entities-ids-handler.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { Entity } from '@dcl/catalyst-api-specs/lib/client' -import { HandlerContextWithPath, InvalidRequestError } from '../../types' -import Joi from 'joi' - -const schema = Joi.alternatives().try( - Joi.object({ - pointers: Joi.array().items(Joi.string()).min(1).required() - }) -) - -// Method: POST -// Body: { pointers: string[]} -export async function getActiveEntitiesIdsHandler( - context: HandlerContextWithPath<'database' | 'activeEntities' | 'denylist', '/entities/active'> -): Promise<{ status: 200; body: Pick[] }> { - const { database, activeEntities, denylist } = context.components - const { error, value: body } = schema.validate(await context.request.json()) - - if (error) { - throw new InvalidRequestError( - 'pointers must be present. They must be arrays and contain at least one element. None of the elements can be empty.' - ) - } - - const entities: Pick[] = ( - await activeEntities.withPointers(database, body.pointers) - ) - .filter((result) => !denylist.isDenylisted(result.id)) - .map((entity) => { - return { - id: entity.id, - pointers: entity.pointers, - timestamp: entity.timestamp - } - }) - - return { - status: 200, - body: entities - } -} diff --git a/content/src/controller/routes.ts b/content/src/controller/routes.ts index 9213328a4..02be443ab 100644 --- a/content/src/controller/routes.ts +++ b/content/src/controller/routes.ts @@ -2,25 +2,24 @@ import { Router } from '@well-known-components/http-server' import { multipartParserWrapper } from '@well-known-components/multipart-wrapper' import { EnvironmentConfig } from '../Environment' import { GlobalContext } from '../types' -import { getActiveEntitiesHandler } from './handlers/active-entities-handler' +import { getActiveEntitiesHandler, getActiveEntitiesScenesHandler } from './handlers/active-entities-handler' import { createEntity } from './handlers/create-entity-handler' -import { createErrorHandler, preventExecutionIfBoostrapping } from './middlewares' import { getFailedDeploymentsHandler } from './handlers/failed-deployments-handler' import { getEntitiesByPointerPrefixHandler } from './handlers/filter-by-urn-handler' +import { getActiveEntityIdsByDeploymentHashHandler } from './handlers/get-active-entities-by-deployment-hash-handler' import { getEntityAuditInformationHandler } from './handlers/get-audit-handler' import { getAvailableContentHandler } from './handlers/get-available-content-handler' -import { getPointerChangesHandler } from './handlers/pointer-changes-handler' -import { getStatusHandler } from './handlers/status-handler' -import { getSnapshotsHandler } from './handlers/get-snapshots-handler' -import { getEntitiesHandler } from './handlers/get-entities-handler' +import { getChallengeHandler } from './handlers/get-challenge-handler' import { getContentHandler } from './handlers/get-content-handler' -import { getEntityThumbnailHandler } from './handlers/get-entity-thumbnail-handler' +import { getDeploymentsHandler } from './handlers/get-deployments-handler' +import { getEntitiesHandler } from './handlers/get-entities-handler' import { getEntityImageHandler } from './handlers/get-entity-image-handler' +import { getEntityThumbnailHandler } from './handlers/get-entity-thumbnail-handler' import { getERC721EntityHandler } from './handlers/get-erc721-entity-handler' -import { getDeploymentsHandler } from './handlers/get-deployments-handler' -import { getChallengeHandler } from './handlers/get-challenge-handler' -import { getActiveEntityIdsByDeploymentHashHandler } from './handlers/get-active-entities-by-deployment-hash-handler' -import { getActiveEntitiesIdsHandler } from './handlers/active-entities-ids-handler' +import { getSnapshotsHandler } from './handlers/get-snapshots-handler' +import { getPointerChangesHandler } from './handlers/pointer-changes-handler' +import { getStatusHandler } from './handlers/status-handler' +import { createErrorHandler, preventExecutionIfBoostrapping } from './middlewares' // We return the entire router because it will be easier to test than a whole server export async function setupRouter({ components }: GlobalContext): Promise> { @@ -43,7 +42,7 @@ export async function setupRouter({ components }: GlobalContext): Promise { - // Generate the select according the info needed - const bothPresent = entityIds && entityIds.length > 0 && pointers && pointers.length > 0 - const nonePresent = !entityIds && !pointers - if (bothPresent || nonePresent) { - throw Error('in getDeploymentsForActiveEntities ids or pointers must be present, but not both') + // Validate that only one parameter is provided + const providedParams = [entityIds && entityIds.length > 0, pointers && pointers.length > 0, entityType].filter( + Boolean + ) + + if (providedParams.length !== 1) { + throw Error('getDeploymentsForActiveEntities requires exactly one of: entityIds, pointers, or entityType') } const query: SQLStatement = SQL` @@ -248,15 +251,23 @@ export async function getDeploymentsForActiveEntities( FROM deployments AS dep1 WHERE dep1.deleter_deployment IS NULL AND `.append( - entityIds + entityIds && entityIds.length > 0 ? SQL`dep1.entity_id = ANY (${entityIds})` - : SQL`dep1.entity_pointers && ${pointers!.map((p) => p.toLowerCase())}` + : pointers && pointers.length > 0 + ? SQL`dep1.entity_pointers && ${pointers.map((p) => p.toLowerCase())}` + : SQL`dep1.entity_type = ${entityType}` ) - const historicalDeploymentsResponse = await database.queryWithValues(query, 'get_active_entities') + const BATCH_SIZE = 1000 + const deploymentsResult: HistoricalDeployment[] = [] + const cursor = await database.streamQuery( + query, + { batchSize: BATCH_SIZE }, + 'get_active_entities' + ) - const deploymentsResult: HistoricalDeployment[] = historicalDeploymentsResponse.rows.map( - (row: HistoricalDeploymentsRow): HistoricalDeployment => ({ + for await (const row of cursor) { + deploymentsResult.push({ deploymentId: row.id, entityType: row.entity_type, entityId: row.entity_id, @@ -269,12 +280,15 @@ export async function getDeploymentsForActiveEntities( localTimestamp: row.local_timestamp, overwrittenBy: row.overwritten_by ?? undefined }) - ) - - const deploymentIds = deploymentsResult.map(({ deploymentId }) => deploymentId) + } - const content = await getContentFiles(database, deploymentIds) + // Batch fetch all content files at once + const content = await getContentFiles( + database, + deploymentsResult.map((d) => d.deploymentId) + ) + // Map results to final format return deploymentsResult.map((result) => ({ entityVersion: result.version as EntityVersion, entityType: result.entityType as EntityType, diff --git a/content/src/ports/activeEntities.ts b/content/src/ports/activeEntities.ts index 3297a6d0d..ee5a243d7 100644 --- a/content/src/ports/activeEntities.ts +++ b/content/src/ports/activeEntities.ts @@ -69,6 +69,16 @@ export type ActiveEntities = IBaseComponent & { * Note: only used in stale profiles GC */ clearPointers(pointers: string[]): Promise + + /** + * Initialize the cache with the active entities for the cached entity types + */ + initialize(database: DatabaseClient): Promise + + /** + * Get all cached scenes + */ + getAllCachedScenes(): Entity[] } /** @@ -81,9 +91,6 @@ export function createActiveEntitiesComponent( components: Pick ): ActiveEntities { const logger = components.logs.getLogger('ActiveEntities') - const cache = new LRU({ - max: components.env.getConfig(EnvironmentConfig.ENTITIES_CACHE_SIZE) - }) const collectionUrnsByPrefixCache = new LRU({ ttl: 1000 * 60 * 60 * 24, // 24 hours @@ -94,6 +101,47 @@ export function createActiveEntitiesComponent( const normalizePointerCacheKey = (pointer: string) => pointer.toLowerCase() + const cache = new LRU({ + max: components.env.getConfig(EnvironmentConfig.ENTITIES_CACHE_SIZE) + }) + const fixedCache = new Map() + + const createLRUandFixedCache = (maxLRU: number, fixedTypes: EntityType[]) => { + return { + get(entityId: string) { + return cache.get(entityId) || fixedCache.get(entityId) + }, + set(entityId: string, entity: Entity | NotActiveEntity) { + const isFixed = fixedCache.has(entityId) || (typeof entity !== 'string' && fixedTypes.includes(entity.type)) + if (isFixed) { + return fixedCache.set(entityId, entity) + } + return cache.set(entityId, entity) + }, + setFixed(entityId: string, entity: Entity | NotActiveEntity) { + return fixedCache.set(entityId, entity) + }, + get max() { + return cache.max + }, + clear() { + cache.clear() + fixedCache.clear() + }, + has(entityId: string) { + return cache.has(entityId) || fixedCache.has(entityId) + } + } + } + + const cachedEntityntityTypes = [EntityType.SCENE] + + // Entities cache by key=entityId + const entityCache = createLRUandFixedCache( + components.env.getConfig(EnvironmentConfig.ENTITIES_CACHE_SIZE), + cachedEntityntityTypes + ) + const createEntityByPointersCache = (): Map => { const entityIdByPointers = new Map() return { @@ -116,7 +164,7 @@ export function createActiveEntitiesComponent( const entityIdByPointers = createEntityByPointersCache() // init gauge metrics - components.metrics.observe('dcl_entities_cache_storage_max_size', {}, cache.max) + components.metrics.observe('dcl_entities_cache_storage_max_size', {}, entityCache.max) Object.values(EntityType).forEach((entityType) => { components.metrics.observe('dcl_entities_cache_storage_size', { entity_type: entityType }, 0) }) @@ -133,9 +181,9 @@ export function createActiveEntitiesComponent( // pointer now have a different active entity, let's update the old one const entityId = entityIdByPointers.get(pointer) if (isPointingToEntity(entityId)) { - const entity = cache.get(entityId) // it should be present + const entity = entityCache.get(entityId) // it should be present if (isEntityPresent(entity)) { - cache.set(entityId, 'NOT_ACTIVE_ENTITY') + entityCache.set(entityId, 'NOT_ACTIVE_ENTITY') for (const pointer of entity.pointers) { entityIdByPointers.set(pointer, 'NOT_ACTIVE_ENTITY') } @@ -158,7 +206,7 @@ export function createActiveEntitiesComponent( entityIdByPointers.set(pointer, isEntityPresent(entity) ? entity.id : entity) } if (isEntityPresent(entity)) { - cache.set(entity.id, entity) + entityCache.set(entity.id, entity) components.metrics.increment('dcl_entities_cache_storage_size', { entity_type: entity.type }) // Store in the db the new entity pointed by pointers await updateActiveDeployments(database, pointers, entity.id) @@ -196,7 +244,7 @@ export function createActiveEntitiesComponent( ) for (const entityId of entityIdsWithoutActiveEntity) { - cache.set(entityId, 'NOT_ACTIVE_ENTITY') + entityCache.set(entityId, 'NOT_ACTIVE_ENTITY') logger.debug('entityId has no active entity', { entityId }) } } @@ -227,6 +275,19 @@ export function createActiveEntitiesComponent( return entities } + async function populateEntityType(database: DatabaseClient, entityType: EntityType): Promise { + const deployments = await getDeploymentsForActiveEntities(database, undefined, undefined, entityType) + + logger.info('Populating cache for entity type', { entityType, deployments: deployments.length }) + + for (const deployment of deployments) { + reportCacheAccess(deployment.entityType, 'miss') + } + + const entities = mapDeploymentsToEntities(deployments) + await updateCache(database, entities, {}) + } + /** * Retrieve active entities by their ids */ @@ -236,7 +297,7 @@ export function createActiveEntitiesComponent( const onCache: (Entity | NotActiveEntity)[] = [] const remaining: string[] = [] for (const entityId of uniqueEntityIds) { - const entity = cache.get(entityId) + const entity = entityCache.get(entityId) if (entity) { onCache.push(entity) if (isEntityPresent(entity)) { @@ -278,6 +339,8 @@ export function createActiveEntitiesComponent( } } + logger.info('Retrieving entities by pointers', { pointers: remaining.length }) + // once we get the ids, retrieve from cache or find const entityIds = Array.from(uniqueEntityIds.values()) const entitiesById = await withIds(database, entityIds) @@ -313,7 +376,7 @@ export function createActiveEntitiesComponent( for (const pointer of pointers) { if (entityIdByPointers.has(pointer)) { const entityId = entityIdByPointers.get(pointer)! - cache.set(entityId, 'NOT_ACTIVE_ENTITY') + entityCache.set(entityId, 'NOT_ACTIVE_ENTITY') entityIdByPointers.set(pointer, 'NOT_ACTIVE_ENTITY') } } @@ -322,7 +385,23 @@ export function createActiveEntitiesComponent( function reset() { entityIdByPointers.clear() collectionUrnsByPrefixCache.clear() - cache.clear() + entityCache.clear() + } + + async function initialize(database: DatabaseClient) { + logger.info('Initializing active entities cache', { + entityTypes: cachedEntityntityTypes.map((t) => t.toString()).toString() + }) + for (const entityType of cachedEntityntityTypes) { + await populateEntityType(database, entityType) + } + logger.info('Active entities cache initialized') + } + + function getAllCachedScenes() { + return Array.from(fixedCache.values()).filter( + (entity): entity is Entity => typeof entity !== 'string' && entity.type === EntityType.SCENE + ) } return { @@ -333,10 +412,11 @@ export function createActiveEntitiesComponent( update, clear, clearPointers, - + initialize, + getAllCachedScenes, getCachedEntity(idOrPointer) { - if (cache.has(idOrPointer)) { - const cachedEntity = cache.get(idOrPointer) + if (entityCache.has(idOrPointer)) { + const cachedEntity = entityCache.get(idOrPointer) return isEntityPresent(cachedEntity) ? cachedEntity.id : cachedEntity } return entityIdByPointers.get(idOrPointer)