Merge pull request #209 from decentraland/feat/add-reader-endpoint-logic

feat: add reader endpoint for most of the queries
decentraland · Dec 11, 2024 · 8acb165 · 8acb165
2 parents 0f7db90 + ff249bd
commit 8acb165
Show file tree

Hide file tree

Showing 7 changed files with 84 additions and 44 deletions.
diff --git a/.env.default b/.env.default
@@ -39,6 +39,12 @@ DAPPS_PG_COMPONENT_PSQL_SCHEMA='marketplace'
 DAPPS_PG_COMPONENT_PSQL_HOST='localhost'
 DAPPS_PG_COMPONENT_PSQL_PORT=5432
 DAPPS_PG_COMPONENT_PSQL_DATABASE='dapps'
+DAPPS_READ_PG_COMPONENT_PSQL_USER='dapps_read_admin'
+DAPPS_READ_PG_COMPONENT_PSQL_PASSWORD='password'
+DAPPS_READ_PG_COMPONENT_PSQL_SCHEMA='marketplace'
+DAPPS_READ_PG_COMPONENT_PSQL_HOST='localhost'
+DAPPS_READ_PG_COMPONENT_PSQL_PORT=5432
+DAPPS_READ_PG_COMPONENT_PSQL_DATABASE='dapps'
 SNAPSHOT_URL=https://score.snapshot.org/
 SNAPSHOT_NETWORK=1
 SNAPSHOT_SPACE=snapshot.dcl.eth
@@ -48,3 +54,5 @@ AWS_SNS_ARN=
 TRANSAK_API_SECRET='secret'
 TRANSAK_API_KEY='key'
 TRANSAK_API_URL='https://api-stg.transak.com/partners/api'
+RENTALS_SUBGRAPH_URL=https://api.studio.thegraph.com/query/49472/rentals-ethereum-sepolia/version/latest
+SIGNATURES_SERVER_URL=https://signatures-api.zone
diff --git a/src/components.ts b/src/components.ts
@@ -62,13 +62,21 @@ export async function initComponents(): Promise<AppComponents> {
     }
   )
 
-  const dappsDatabase = await createPgComponent(
+  const dappsWriteDatabase = await createPgComponent(
     { config, logs, metrics },
     {
       dbPrefix: 'DAPPS'
     }
   )
 
+  const dappsReadDatabase = await createPgComponent(
+    { config, logs, metrics },
+    {
+      dbPrefix: 'DAPPS_READ',
+      migrations: false
+    }
+  )
+
   const SEGMENT_WRITE_KEY = await config.requireString('SEGMENT_WRITE_KEY')
   const COVALENT_API_KEY = await config.getString('COVALENT_API_KEY')
   const WERT_PRIVATE_KEY = await config.requireString('WERT_PRIVATE_KEY')
@@ -90,7 +98,7 @@ export async function initComponents(): Promise<AppComponents> {
   const schemaValidator = await createSchemaValidatorComponent()
 
   const snapshot = await createSnapshotComponent({ fetch, config })
-  const items = createItemsComponent({ logs, dappsDatabase })
+  const items = createItemsComponent({ logs, dappsDatabase: dappsReadDatabase })
   const lists = createListsComponent({
     favoritesDatabase,
     items,
@@ -101,17 +109,17 @@ export async function initComponents(): Promise<AppComponents> {
   const picks = createPicksComponent({ favoritesDatabase, items, snapshot, logs, lists })
 
   // catalog
-  const catalog = await createCatalogComponent({ dappsDatabase, picks }, SEGMENT_WRITE_KEY)
-  const trades = await createTradesComponent({ dappsDatabase, eventPublisher, logs })
-  const bids = await createBidsComponents({ dappsDatabase })
-  const nfts = await createNFTsComponent({ dappsDatabase, config, rentals })
-  const orders = await createOrdersComponent({ dappsDatabase })
-  const sales = await createSalesComponents({ dappsDatabase })
-  const prices = await createPricesComponents({ dappsDatabase })
-  const trendings = await createTrendingsComponent({ dappsDatabase, items, picks })
-  const stats = await createStatsComponent({ dappsDatabase })
-  const rankings = await createRankingsComponent({ dappsDatabase })
-  const analyticsData = await createAnalyticsDayDataComponent({ dappsDatabase })
+  const catalog = await createCatalogComponent({ dappsDatabase: dappsReadDatabase, dappsWriteDatabase, picks }, SEGMENT_WRITE_KEY)
+  const trades = await createTradesComponent({ dappsDatabase: dappsReadDatabase, eventPublisher, logs })
+  const bids = await createBidsComponents({ dappsDatabase: dappsReadDatabase })
+  const nfts = await createNFTsComponent({ dappsDatabase: dappsReadDatabase, config, rentals })
+  const orders = await createOrdersComponent({ dappsDatabase: dappsReadDatabase })
+  const sales = await createSalesComponents({ dappsDatabase: dappsReadDatabase })
+  const prices = await createPricesComponents({ dappsDatabase: dappsReadDatabase })
+  const trendings = await createTrendingsComponent({ dappsDatabase: dappsReadDatabase, items, picks })
+  const stats = await createStatsComponent({ dappsDatabase: dappsReadDatabase })
+  const rankings = await createRankingsComponent({ dappsDatabase: dappsReadDatabase })
+  const analyticsData = await createAnalyticsDayDataComponent({ dappsDatabase: dappsReadDatabase })
   const volumes = await createVolumeComponent({ analyticsData })
 
   const transak = await createTransakComponent(
@@ -134,7 +142,8 @@ export async function initComponents(): Promise<AppComponents> {
     fetch,
     metrics,
     favoritesDatabase,
-    dappsDatabase,
+    dappsDatabase: dappsReadDatabase,
+    dappsWriteDatabase,
     catalog,
     balances,
     wertSigner,

diff --git a/src/ports/catalog/component.ts b/src/ports/catalog/component.ts
@@ -13,10 +13,10 @@ import { fromCollectionsItemDbResultToCatalogItem } from './utils'
 const sortByWordSimilarity = (a: { word_similarity: number }, b: { word_similarity: number }) => b.word_similarity - a.word_similarity
 
 export async function createCatalogComponent(
-  components: Pick<AppComponents, 'dappsDatabase' | 'picks'>,
+  components: Pick<AppComponents, 'dappsDatabase' | 'dappsWriteDatabase' | 'picks'>,
   segmentWriteKey: string
 ): Promise<ICatalogComponent> {
-  const { dappsDatabase: database, picks } = components
+  const { dappsDatabase: dataReadbase, dappsWriteDatabase, picks } = components
 
   async function fetch(
     filters: CatalogOptions,
@@ -25,7 +25,7 @@ export async function createCatalogComponent(
     const { network } = filters
     let catalogItems: Item[] = []
     let total = 0
-    const client = await database.getPool().connect()
+    const client = await dataReadbase.getPool().connect()
     try {
       if (filters.search) {
         const analytics = new Analytics({ writeKey: segmentWriteKey })
@@ -73,7 +73,7 @@ export async function createCatalogComponent(
 
       catalogItems = enhanceItemsWithPicksStats(catalogItems, pickStats.map(fromDBPickStatsToPickStats))
     } catch (e) {
-      if (e instanceof Error && e.message === 'Query read timeout') {
+      if ((e as Error).message === 'Query read timeout') {
         console.error('Query timeout exceeded (2 minutes)', {
           filters
         })
@@ -87,7 +87,7 @@ export async function createCatalogComponent(
   }
 
   async function updateBuilderServerItemsView() {
-    const client = await database.getPool().connect()
+    const client = await dappsWriteDatabase.getPool().connect()
     try {
       const query = SQL`
         REFRESH MATERIALIZED VIEW `

diff --git a/src/ports/db/component.ts b/src/ports/db/component.ts
@@ -6,10 +6,10 @@ import { IPgComponent } from './types'
 
 export async function createPgComponent(
   components: createBasePgComponent.NeededComponents,
-  options: { dbPrefix: string } & Options
+  options: { dbPrefix: string; migrations?: boolean } & Options
 ): Promise<IPgComponent & IBaseComponent> {
   const { config, logs, metrics } = components
-  const { dbPrefix } = options
+  const { dbPrefix, migrations = true } = options
   let databaseUrl: string | undefined = await config.getString(`${dbPrefix}_PG_COMPONENT_PSQL_CONNECTION_STRING`)
   if (!databaseUrl) {
     const dbUser = await config.requireString(`${dbPrefix}_PG_COMPONENT_PSQL_USER`)
@@ -31,14 +31,18 @@ export async function createPgComponent(
         connectionString: databaseUrl,
         query_timeout: 120000
       },
-      migration: {
-        databaseUrl,
-        ...(schema ? { schema } : {}),
-        dir: path.resolve(__dirname, `../../migrations/${dbPrefix.toLowerCase()}`),
-        migrationsTable: 'pgmigrations',
-        ignorePattern: '.*\\.map',
-        direction: 'up'
-      }
+      ...(migrations
+        ? {
+            migration: {
+              databaseUrl,
+              ...(schema ? { schema } : {}),
+              dir: path.resolve(__dirname, `../../migrations/${dbPrefix.toLowerCase()}`),
+              migrationsTable: 'pgmigrations',
+              ignorePattern: '.*\\.map',
+              direction: 'up'
+            }
+          }
+        : {})
     }
   )
 

diff --git a/src/types.ts b/src/types.ts
@@ -48,6 +48,7 @@ export type BaseComponents = {
   metrics: IMetricsComponent<keyof typeof metricDeclarations>
   favoritesDatabase: IPgComponent
   dappsDatabase: IPgComponent
+  dappsWriteDatabase: IPgComponent
   catalog: ICatalogComponent
   balances: IBalanceComponent
   wertSigner: IWertSignerComponent

diff --git a/test/components.ts b/test/components.ts
@@ -72,13 +72,21 @@ async function initComponents(): Promise<TestComponents> {
     }
   )
 
-  const dappsDatabase = await createPgComponent(
+  const dappsWriteDatabase = await createPgComponent(
     { config, logs, metrics },
     {
       dbPrefix: 'DAPPS'
     }
   )
 
+  const dappsReadDatabase = await createPgComponent(
+    { config, logs, metrics },
+    {
+      dbPrefix: 'DAPPS_READ',
+      migrations: false
+    }
+  )
+
   const SEGMENT_WRITE_KEY = await config.requireString('SEGMENT_WRITE_KEY')
   const COVALENT_API_KEY = await config.getString('COVALENT_API_KEY')
   const WERT_PRIVATE_KEY = await config.requireString('WERT_PRIVATE_KEY')
@@ -88,7 +96,7 @@ async function initComponents(): Promise<TestComponents> {
 
   // favorites stuff
   const snapshot = await createSnapshotComponent({ fetch, config })
-  const items = createItemsComponent({ logs, dappsDatabase })
+  const items = createItemsComponent({ logs, dappsDatabase: dappsReadDatabase })
   const lists = createListsComponent({
     favoritesDatabase,
     items,
@@ -97,11 +105,11 @@ async function initComponents(): Promise<TestComponents> {
   })
   const access = createAccessComponent({ favoritesDatabase, logs, lists })
   const picks = createPicksComponent({ favoritesDatabase, items, snapshot, logs, lists })
-  const catalog = await createCatalogComponent({ dappsDatabase, picks }, SEGMENT_WRITE_KEY)
+  const catalog = await createCatalogComponent({ dappsDatabase: dappsReadDatabase, dappsWriteDatabase, picks }, SEGMENT_WRITE_KEY)
   const schemaValidator = await createSchemaValidatorComponent()
   const balances = createBalanceComponent({ apiKey: COVALENT_API_KEY ?? '' })
-  const trades = createTradesComponent({ dappsDatabase, eventPublisher, logs })
-  const bids = createBidsComponents({ dappsDatabase })
+  const trades = createTradesComponent({ dappsDatabase: dappsReadDatabase, eventPublisher, logs })
+  const bids = createBidsComponents({ dappsDatabase: dappsReadDatabase })
 
   const rentalsSubgraph = await createSubgraphComponent(
     { logs, config, fetch, metrics },
@@ -110,21 +118,21 @@ async function initComponents(): Promise<TestComponents> {
   const SIGNATURES_SERVER_URL = await config.requireString('SIGNATURES_SERVER_URL')
   const rentals = createRentalsComponent({ fetch }, SIGNATURES_SERVER_URL, rentalsSubgraph)
 
-  const nfts = createNFTsComponent({ dappsDatabase, config, rentals })
-  const orders = createOrdersComponent({ dappsDatabase })
-  const sales = createSalesComponents({ dappsDatabase })
-  const prices = createPricesComponents({ dappsDatabase })
+  const nfts = createNFTsComponent({ dappsDatabase: dappsReadDatabase, config, rentals })
+  const orders = createOrdersComponent({ dappsDatabase: dappsReadDatabase })
+  const sales = createSalesComponents({ dappsDatabase: dappsReadDatabase })
+  const prices = createPricesComponents({ dappsDatabase: dappsReadDatabase })
   // Mock the start function to avoid connecting to a local database
   jest.spyOn(catalog, 'updateBuilderServerItemsView').mockResolvedValue(undefined)
   const updateBuilderServerItemsViewJob = createJobComponent({ logs }, () => undefined, 5 * 60 * 1000, {
     startupDelay: 30
   })
 
   const transak = createTransakComponent({ fetch }, { apiURL: '', apiKey: '', apiSecret: '' })
-  const stats = await createStatsComponent({ dappsDatabase })
-  const trendings = await createTrendingsComponent({ dappsDatabase, items, picks })
-  const rankings = await createRankingsComponent({ dappsDatabase })
-  const analyticsData = await createAnalyticsDayDataComponent({ dappsDatabase })
+  const stats = await createStatsComponent({ dappsDatabase: dappsReadDatabase })
+  const trendings = await createTrendingsComponent({ dappsDatabase: dappsReadDatabase, items, picks })
+  const rankings = await createRankingsComponent({ dappsDatabase: dappsReadDatabase })
+  const analyticsData = await createAnalyticsDayDataComponent({ dappsDatabase: dappsReadDatabase })
   const volumes = await createVolumeComponent({ analyticsData })
 
   return {
@@ -134,7 +142,8 @@ async function initComponents(): Promise<TestComponents> {
     localFetch: await createLocalFetchCompoment(config),
     fetch,
     metrics,
-    dappsDatabase,
+    dappsDatabase: dappsReadDatabase,
+    dappsWriteDatabase,
     favoritesDatabase,
     catalog,
     balances,

diff --git a/test/unit/catalog-component.spec.ts b/test/unit/catalog-component.spec.ts
@@ -12,6 +12,7 @@ jest.mock('@segment/analytics-node')
 
 let catalogComponent: ICatalogComponent
 let dappsDatabase: AppComponents['dappsDatabase']
+let dappsWriteDatabase: AppComponents['dappsDatabase']
 let picks: IPicksComponent
 let dbClientQueryMock: jest.Mock
 let dbClientReleaseMock: jest.Mock
@@ -30,6 +31,14 @@ beforeEach(async () => {
       })
     })
   })
+  dappsWriteDatabase = createTestPgComponent({
+    getPool: jest.fn().mockReturnValue({
+      connect: () => ({
+        query: dbClientQueryMock,
+        release: dbClientReleaseMock
+      })
+    })
+  })
 
   picks = {
     getPicksStats: jest.fn(),
@@ -38,7 +47,7 @@ beforeEach(async () => {
   }
 
   segmentWriteKey = 'testSegmentWriteKey'
-  catalogComponent = await createCatalogComponent({ dappsDatabase, picks }, segmentWriteKey)
+  catalogComponent = await createCatalogComponent({ dappsDatabase, dappsWriteDatabase, picks }, segmentWriteKey)
 })
 
 describe('Catalog Component', () => {