refactor: fine tune projects list query, add cache to all vestings fr…

…om subgraph, cache project list

src/clients/VestingData.ts

@@ -246,6 +246,8 @@ export function getTokenSymbolFromAddress(tokenAddress: string) {
       return 'USDC'
     case '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2':
       return 'WETH'
+    default:
+      console.log(`Unable to parse token contract address: ${tokenAddress}`)
+      return 'ETH'
   }
-  throw new Error(`Unable to parse token contract address: ${tokenAddress}`)
 }

src/clients/VestingsSubgraph.ts

@@ -5,6 +5,8 @@ import { VESTINGS_QUERY_ENDPOINT } from '../entities/Snapshot/constants'
 import { SubgraphVesting } from './VestingSubgraphTypes'
 import { trimLastForwardSlash } from './utils'
 
+const OLDEST_INDEXED_BLOCK = 20463272
+
 export class VestingsSubgraph {
   static Cache = new Map<string, VestingsSubgraph>()
   private readonly queryEndpoint: string
@@ -86,10 +88,15 @@ export class VestingsSubgraph {
     return body?.data?.vestings[0] || {}
   }
 
-  async getVestings(addresses: string[]): Promise<SubgraphVesting[]> {
+  async getVestings(addresses?: string[]): Promise<SubgraphVesting[]> {
+    const queryAddresses = addresses && addresses.length > 0
+    const addressesQuery = queryAddresses
+      ? `where: { id_in: $addresses }`
+      : 'block: {number_gte: $blockNumber}, first: 1000'
+    const addressesParam = queryAddresses ? `$addresses: [String]!` : '$blockNumber: Int!'
     const query = `
-    query getVestings($addresses: [String]!) {
-      vestings(where: { id_in: $addresses }){
+    query getVestings(${addressesParam}) {
+      vestings(${addressesQuery}){
         id
         version
         duration
@@ -122,14 +129,13 @@ export class VestingsSubgraph {
       }
     }
     `
-
-    const variables = { addresses }
+    const variables = queryAddresses ? { addresses } : { blockNumber: OLDEST_INDEXED_BLOCK }
     const response = await fetch(this.queryEndpoint, {
       method: 'post',
       headers: { 'Content-Type': 'application/json' },
       body: JSON.stringify({
         query,
-        variables: variables,
+        variables,
       }),
     })
 

src/entities/Proposal/types.ts

@@ -823,11 +823,13 @@ export type ProposalProject = {
   funding?: ProjectFunding
 }
 
-export type ProposalProjectWithUpdate = ProposalProject & {
+export type LatestUpdate = {
   update?: IndexedUpdate | null
   update_timestamp?: number
 }
 
+export type ProposalProjectWithUpdate = ProposalProject & LatestUpdate
+
 export enum PriorityProposalType {
   ActiveGovernance = 'active_governance',
   OpenPitch = 'open_pitch',

src/entities/Updates/types.ts

@@ -34,7 +34,7 @@ export type UpdateAttributes = Partial<UpdateGeneralSection> &
     discourse_topic_slug?: string
   }
 
-export type IndexedUpdate = UpdateAttributes & {
+export type IndexedUpdate = Partial<UpdateAttributes> & {
   index: number
 }
 

src/models/Project.ts

@@ -1,13 +1,13 @@
 import { Model } from 'decentraland-gatsby/dist/entities/Database/model'
-import { SQL, table } from 'decentraland-gatsby/dist/entities/Database/utils'
+import { SQL, conditional, table } from 'decentraland-gatsby/dist/entities/Database/utils'
 import isEthereumAddress from 'validator/lib/isEthereumAddress'
 import isUUID from 'validator/lib/isUUID'
 
 import CoauthorModel from '../entities/Coauthor/model'
 import { CoauthorStatus } from '../entities/Coauthor/types'
 import { ProjectStatus } from '../entities/Grant/types'
 import ProposalModel from '../entities/Proposal/model'
-import { ProjectFunding, ProposalAttributes } from '../entities/Proposal/types'
+import { LatestUpdate, ProjectFunding, ProposalAttributes } from '../entities/Proposal/types'
 import UpdateModel from '../entities/Updates/model'
 import { UpdateAttributes } from '../entities/Updates/types'
 
@@ -35,13 +35,24 @@ export type Project = ProjectAttributes & {
   coauthors: string[] | null
   vesting_addresses: string[]
   funding?: ProjectFunding
-  updates?: UpdateAttributes
 }
 
-export type ProjectQueryResult = Project &
-  Pick<ProposalAttributes, 'enacting_tx' | 'enacted_description'> & {
+export type ProjectQueryResult = Pick<
+  ProjectAttributes,
+  'id' | 'proposal_id' | 'status' | 'title' | 'created_at' | 'updated_at'
+> &
+  Pick<
+    ProposalAttributes,
+    'type' | 'enacting_tx' | 'enacted_description' | 'configuration' | 'user' | 'vesting_addresses'
+  > & {
     proposal_updated_at: string
-  }
+  } & { updates?: UpdateAttributes[] }
+
+export type ProjectInList = Pick<
+  Project,
+  'id' | 'proposal_id' | 'status' | 'title' | 'author' | 'created_at' | 'updated_at' | 'funding'
+> &
+  Pick<ProposalAttributes, 'type' | 'configuration'> & { latest_update?: LatestUpdate }
 
 export default class ProjectModel extends Model<ProjectAttributes> {
   static tableName = 'projects'
@@ -105,33 +116,44 @@ export default class ProjectModel extends Model<ProjectAttributes> {
     return result[0]?.exists || false
   }
 
-  static async getProjectsWithUpdates(): Promise<ProjectQueryResult[]> {
+  static async getProjectsWithUpdates(from?: Date, to?: Date): Promise<ProjectQueryResult[]> {
     const query = SQL`
         SELECT
-            pr.*,
-            p.user as author,
-            p.vesting_addresses as vesting_addresses,
-            COALESCE(json_agg(DISTINCT to_jsonb(pe.*)) FILTER (WHERE pe.id IS NOT NULL), '[]') as personnel,
-            COALESCE(json_agg(DISTINCT to_jsonb(mi.*)) FILTER (WHERE mi.id IS NOT NULL), '[]') as milestones,
-            COALESCE(json_agg(DISTINCT to_jsonb(li.*)) FILTER (WHERE li.id IS NOT NULL), '[]') as links,
-            COALESCE(array_agg(DISTINCT co.address) FILTER (WHERE co.address IS NOT NULL), '{}') AS coauthors,
-            COALESCE(json_agg(DISTINCT to_jsonb(ordered_updates.*)) FILTER 
-                (WHERE ordered_updates.id IS NOT NULL), '[]')  as updates,
+            pr.id,
+            pr.proposal_id,
+            pr.status,
+            pr.title,
+            pr.created_at,
+            pr.updated_at,
+            p.type,
             p.enacting_tx,
             p.enacted_description,
-            p.updated_at as proposal_updated_at
+            p.configuration,
+            p.user,
+            p.vesting_addresses,
+            p.updated_at as proposal_updated_at,
+            COALESCE(json_agg(DISTINCT to_jsonb(ordered_updates.*)) FILTER (WHERE ordered_updates.id IS NOT NULL), '[]') as updates
         FROM ${table(ProjectModel)} pr
                  JOIN ${table(ProposalModel)} p ON pr.proposal_id = p.id
-                 LEFT JOIN ${table(PersonnelModel)} pe ON pr.id = pe.project_id AND pe.deleted = false
-                 LEFT JOIN ${table(ProjectMilestoneModel)} mi ON pr.id = mi.project_id
-                 LEFT JOIN ${table(ProjectLinkModel)} li ON pr.id = li.project_id
-                 LEFT JOIN ${table(CoauthorModel)} co ON pr.proposal_id = co.proposal_id 
-                                           AND co.status = ${CoauthorStatus.APPROVED}
-                 LEFT JOIN (SELECT * FROM ${table(
-                   UpdateModel
-                 )} up ORDER BY up.created_at DESC) ordered_updates ON pr.id = ordered_updates.project_id
-        GROUP BY pr.id, p.user, p.vesting_addresses, p.enacting_tx, p.enacted_description, proposal_updated_at;
-
+                 LEFT JOIN (SELECT * FROM ${table(UpdateModel)} up ORDER BY up.created_at DESC) ordered_updates
+                           ON pr.id = ordered_updates.project_id
+        WHERE 1=1
+            ${conditional(!!from, SQL`AND pr.created_at >= ${from}`)}
+            ${conditional(!!to, SQL`AND pr.created_at <= ${to}`)}
+        GROUP BY
+            pr.id,
+            pr.proposal_id,
+            pr.status,
+            pr.title,
+            pr.created_at,
+            pr.updated_at,
+            p.type,
+            p.enacting_tx,
+            p.enacted_description,
+            p.configuration,
+            p.user,
+            p.vesting_addresses,
+            p.updated_at;
     `
 
     const result = await this.namedQuery<ProjectQueryResult>(`get_projects`, query)

src/routes/project.ts

@@ -11,7 +11,7 @@ import {
 } from '../entities/Project/types'
 import { ProposalProjectWithUpdate } from '../entities/Proposal/types'
 import PersonnelModel, { PersonnelAttributes } from '../models/Personnel'
-import { Project } from '../models/Project'
+import { ProjectInList } from '../models/Project'
 import ProjectLinkModel, { ProjectLink } from '../models/ProjectLink'
 import ProjectMilestoneModel, { ProjectMilestone } from '../models/ProjectMilestone'
 import CacheService, { TTL_1_HS } from '../services/CacheService'
@@ -22,20 +22,20 @@ import { isValidDate, validateCanEditProject, validateId } from '../utils/valida
 
 export default routes((route) => {
   const withAuth = auth()
-  route.get('/projects', handleJSON(getProjects))
-  route.get('/projects/updated/', handleJSON(getUpdatedProjects))
+  route.get('/projects', handleJSON(getProposalProjects))
+  route.get('/projects/updated', handleJSON(getProjectsList))
+  route.get('/projects/pitches-total', handleJSON(getOpenPitchesTotal))
+  route.get('/projects/tenders-total', handleJSON(getOpenTendersTotal))
   route.post('/projects/personnel/', withAuth, handleAPI(addPersonnel))
-  route.delete('/projects/personnel/:personnel_id', withAuth, handleAPI(deletePersonnel))
   route.post('/projects/links/', withAuth, handleAPI(addLink))
-  route.delete('/projects/links/:link_id', withAuth, handleAPI(deleteLink))
   route.post('/projects/milestones/', withAuth, handleAPI(addMilestone))
-  route.delete('/projects/milestones/:milestone_id', withAuth, handleAPI(deleteMilestone))
   route.get('/projects/:project', handleAPI(getProject))
-  route.get('/projects/pitches-total', handleJSON(getOpenPitchesTotal))
-  route.get('/projects/tenders-total', handleJSON(getOpenTendersTotal))
+  route.delete('/projects/links/:link_id', withAuth, handleAPI(deleteLink))
+  route.delete('/projects/personnel/:personnel_id', withAuth, handleAPI(deletePersonnel))
+  route.delete('/projects/milestones/:milestone_id', withAuth, handleAPI(deleteMilestone))
 })
 
-function filterProjectsByDate(
+function filterProposalProjectsByDate(
   projects: ProposalProjectWithUpdate[],
   from?: Date,
   to?: Date
@@ -46,29 +46,50 @@ function filterProjectsByDate(
   })
 }
 
-async function getProjects(req: Request) {
+function filterProjectsByDate(projects: ProjectInList[], from?: Date, to?: Date): ProjectInList[] {
+  return projects.filter((project) => {
+    const createdAt = new Date(project.created_at)
+    return (!from || createdAt >= from) && (!to || createdAt < to)
+  })
+}
+
+async function getProposalProjects(req: Request) {
   const from = isValidDate(req.query.from as string) ? new Date(req.query.from as string) : undefined
   const to = isValidDate(req.query.to as string) ? new Date(req.query.to as string) : undefined
 
   if (from && to && from > to) {
     throw new RequestError('Invalid date range', RequestError.BadRequest)
   }
 
-  const cacheKey = `projects`
+  const cacheKey = `proposal-projects`
   const cachedProjects = CacheService.get<ProposalProjectWithUpdate[]>(cacheKey)
   if (cachedProjects) {
-    return { data: filterProjectsByDate(cachedProjects, from, to) }
+    return { data: filterProposalProjectsByDate(cachedProjects, from, to) }
   }
   const projects = await ProjectService.getProposalProjects()
   CacheService.set(cacheKey, projects, TTL_1_HS)
-  return { data: filterProjectsByDate(projects, from, to) }
+  return { data: filterProposalProjectsByDate(projects, from, to) }
 }
 
-async function getUpdatedProjects(): Promise<Project[]> {
+async function getProjectsList(req: Request) {
   try {
-    return await ProjectService.getUpdatedProjects()
+    const from = isValidDate(req.query.from as string) ? new Date(req.query.from as string) : undefined
+    const to = isValidDate(req.query.to as string) ? new Date(req.query.to as string) : undefined
+
+    if (from && to && from > to) {
+      throw new RequestError('Invalid date range', RequestError.BadRequest)
+    }
+
+    const cacheKey = `projects`
+    const cachedProjects = CacheService.get<ProjectInList[]>(cacheKey)
+    if (cachedProjects) {
+      return { data: filterProjectsByDate(cachedProjects, from, to) }
+    }
+    const projects = await ProjectService.getProjects()
+    CacheService.set(cacheKey, projects, TTL_1_HS)
+    return { data: filterProjectsByDate(projects, from, to) }
   } catch (error) {
-    ErrorService.report('Error fetching projets', { error, category: ErrorCategory.Project })
+    ErrorService.report('Error fetching projects', { error, category: ErrorCategory.Project })
     throw new RequestError(`Unable to load projects`, RequestError.InternalServerError)
   }
 }