diff --git a/src/clients/VestingData.ts b/src/clients/VestingData.ts index d68a2b06c..e6f4a86c3 100644 --- a/src/clients/VestingData.ts +++ b/src/clients/VestingData.ts @@ -246,6 +246,8 @@ export function getTokenSymbolFromAddress(tokenAddress: string) { return 'USDC' case '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2': return 'WETH' + default: + console.log(`Unable to parse token contract address: ${tokenAddress}`) + return 'ETH' } - throw new Error(`Unable to parse token contract address: ${tokenAddress}`) } diff --git a/src/clients/VestingsSubgraph.ts b/src/clients/VestingsSubgraph.ts index 7f8fa6d32..4f4a6bf7d 100644 --- a/src/clients/VestingsSubgraph.ts +++ b/src/clients/VestingsSubgraph.ts @@ -5,6 +5,8 @@ import { VESTINGS_QUERY_ENDPOINT } from '../entities/Snapshot/constants' import { SubgraphVesting } from './VestingSubgraphTypes' import { trimLastForwardSlash } from './utils' +const OLDEST_INDEXED_BLOCK = 20463272 + export class VestingsSubgraph { static Cache = new Map() private readonly queryEndpoint: string @@ -86,10 +88,15 @@ export class VestingsSubgraph { return body?.data?.vestings[0] || {} } - async getVestings(addresses: string[]): Promise { + async getVestings(addresses?: string[]): Promise { + const queryAddresses = addresses && addresses.length > 0 + const addressesQuery = queryAddresses + ? `where: { id_in: $addresses }` + : 'block: {number_gte: $blockNumber}, first: 1000' + const addressesParam = queryAddresses ? `$addresses: [String]!` : '$blockNumber: Int!' const query = ` - query getVestings($addresses: [String]!) { - vestings(where: { id_in: $addresses }){ + query getVestings(${addressesParam}) { + vestings(${addressesQuery}){ id version duration @@ -122,14 +129,13 @@ export class VestingsSubgraph { } } ` - - const variables = { addresses } + const variables = queryAddresses ? { addresses } : { blockNumber: OLDEST_INDEXED_BLOCK } const response = await fetch(this.queryEndpoint, { method: 'post', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ query, - variables: variables, + variables, }), }) diff --git a/src/entities/Proposal/types.ts b/src/entities/Proposal/types.ts index 773df6015..08b6e703f 100644 --- a/src/entities/Proposal/types.ts +++ b/src/entities/Proposal/types.ts @@ -823,11 +823,13 @@ export type ProposalProject = { funding?: ProjectFunding } -export type ProposalProjectWithUpdate = ProposalProject & { +export type LatestUpdate = { update?: IndexedUpdate | null update_timestamp?: number } +export type ProposalProjectWithUpdate = ProposalProject & LatestUpdate + export enum PriorityProposalType { ActiveGovernance = 'active_governance', OpenPitch = 'open_pitch', diff --git a/src/entities/Updates/types.ts b/src/entities/Updates/types.ts index c09d4f44c..5abc76070 100644 --- a/src/entities/Updates/types.ts +++ b/src/entities/Updates/types.ts @@ -34,7 +34,7 @@ export type UpdateAttributes = Partial & discourse_topic_slug?: string } -export type IndexedUpdate = UpdateAttributes & { +export type IndexedUpdate = Partial & { index: number } diff --git a/src/models/Project.ts b/src/models/Project.ts index 843f1cdf6..fe4962db7 100644 --- a/src/models/Project.ts +++ b/src/models/Project.ts @@ -1,5 +1,5 @@ import { Model } from 'decentraland-gatsby/dist/entities/Database/model' -import { SQL, table } from 'decentraland-gatsby/dist/entities/Database/utils' +import { SQL, conditional, table } from 'decentraland-gatsby/dist/entities/Database/utils' import isEthereumAddress from 'validator/lib/isEthereumAddress' import isUUID from 'validator/lib/isUUID' @@ -7,7 +7,7 @@ import CoauthorModel from '../entities/Coauthor/model' import { CoauthorStatus } from '../entities/Coauthor/types' import { ProjectStatus } from '../entities/Grant/types' import ProposalModel from '../entities/Proposal/model' -import { ProjectFunding, ProposalAttributes } from '../entities/Proposal/types' +import { LatestUpdate, ProjectFunding, ProposalAttributes } from '../entities/Proposal/types' import UpdateModel from '../entities/Updates/model' import { UpdateAttributes } from '../entities/Updates/types' @@ -35,13 +35,24 @@ export type Project = ProjectAttributes & { coauthors: string[] | null vesting_addresses: string[] funding?: ProjectFunding - updates?: UpdateAttributes } -export type ProjectQueryResult = Project & - Pick & { +export type ProjectQueryResult = Pick< + ProjectAttributes, + 'id' | 'proposal_id' | 'status' | 'title' | 'created_at' | 'updated_at' +> & + Pick< + ProposalAttributes, + 'type' | 'enacting_tx' | 'enacted_description' | 'configuration' | 'user' | 'vesting_addresses' + > & { proposal_updated_at: string - } + } & { updates?: UpdateAttributes[] } + +export type ProjectInList = Pick< + Project, + 'id' | 'proposal_id' | 'status' | 'title' | 'author' | 'created_at' | 'updated_at' | 'funding' +> & + Pick & { latest_update?: LatestUpdate } export default class ProjectModel extends Model { static tableName = 'projects' @@ -105,33 +116,44 @@ export default class ProjectModel extends Model { return result[0]?.exists || false } - static async getProjectsWithUpdates(): Promise { + static async getProjectsWithUpdates(from?: Date, to?: Date): Promise { const query = SQL` SELECT - pr.*, - p.user as author, - p.vesting_addresses as vesting_addresses, - COALESCE(json_agg(DISTINCT to_jsonb(pe.*)) FILTER (WHERE pe.id IS NOT NULL), '[]') as personnel, - COALESCE(json_agg(DISTINCT to_jsonb(mi.*)) FILTER (WHERE mi.id IS NOT NULL), '[]') as milestones, - COALESCE(json_agg(DISTINCT to_jsonb(li.*)) FILTER (WHERE li.id IS NOT NULL), '[]') as links, - COALESCE(array_agg(DISTINCT co.address) FILTER (WHERE co.address IS NOT NULL), '{}') AS coauthors, - COALESCE(json_agg(DISTINCT to_jsonb(ordered_updates.*)) FILTER - (WHERE ordered_updates.id IS NOT NULL), '[]') as updates, + pr.id, + pr.proposal_id, + pr.status, + pr.title, + pr.created_at, + pr.updated_at, + p.type, p.enacting_tx, p.enacted_description, - p.updated_at as proposal_updated_at + p.configuration, + p.user, + p.vesting_addresses, + p.updated_at as proposal_updated_at, + COALESCE(json_agg(DISTINCT to_jsonb(ordered_updates.*)) FILTER (WHERE ordered_updates.id IS NOT NULL), '[]') as updates FROM ${table(ProjectModel)} pr JOIN ${table(ProposalModel)} p ON pr.proposal_id = p.id - LEFT JOIN ${table(PersonnelModel)} pe ON pr.id = pe.project_id AND pe.deleted = false - LEFT JOIN ${table(ProjectMilestoneModel)} mi ON pr.id = mi.project_id - LEFT JOIN ${table(ProjectLinkModel)} li ON pr.id = li.project_id - LEFT JOIN ${table(CoauthorModel)} co ON pr.proposal_id = co.proposal_id - AND co.status = ${CoauthorStatus.APPROVED} - LEFT JOIN (SELECT * FROM ${table( - UpdateModel - )} up ORDER BY up.created_at DESC) ordered_updates ON pr.id = ordered_updates.project_id - GROUP BY pr.id, p.user, p.vesting_addresses, p.enacting_tx, p.enacted_description, proposal_updated_at; - + LEFT JOIN (SELECT * FROM ${table(UpdateModel)} up ORDER BY up.created_at DESC) ordered_updates + ON pr.id = ordered_updates.project_id + WHERE 1=1 + ${conditional(!!from, SQL`AND pr.created_at >= ${from}`)} + ${conditional(!!to, SQL`AND pr.created_at <= ${to}`)} + GROUP BY + pr.id, + pr.proposal_id, + pr.status, + pr.title, + pr.created_at, + pr.updated_at, + p.type, + p.enacting_tx, + p.enacted_description, + p.configuration, + p.user, + p.vesting_addresses, + p.updated_at; ` const result = await this.namedQuery(`get_projects`, query) diff --git a/src/routes/project.ts b/src/routes/project.ts index 7cd08b6b6..64024f0c1 100644 --- a/src/routes/project.ts +++ b/src/routes/project.ts @@ -11,7 +11,7 @@ import { } from '../entities/Project/types' import { ProposalProjectWithUpdate } from '../entities/Proposal/types' import PersonnelModel, { PersonnelAttributes } from '../models/Personnel' -import { Project } from '../models/Project' +import { ProjectInList } from '../models/Project' import ProjectLinkModel, { ProjectLink } from '../models/ProjectLink' import ProjectMilestoneModel, { ProjectMilestone } from '../models/ProjectMilestone' import CacheService, { TTL_1_HS } from '../services/CacheService' @@ -22,20 +22,20 @@ import { isValidDate, validateCanEditProject, validateId } from '../utils/valida export default routes((route) => { const withAuth = auth() - route.get('/projects', handleJSON(getProjects)) - route.get('/projects/updated/', handleJSON(getUpdatedProjects)) + route.get('/projects', handleJSON(getProposalProjects)) + route.get('/projects/updated', handleJSON(getProjectsList)) + route.get('/projects/pitches-total', handleJSON(getOpenPitchesTotal)) + route.get('/projects/tenders-total', handleJSON(getOpenTendersTotal)) route.post('/projects/personnel/', withAuth, handleAPI(addPersonnel)) - route.delete('/projects/personnel/:personnel_id', withAuth, handleAPI(deletePersonnel)) route.post('/projects/links/', withAuth, handleAPI(addLink)) - route.delete('/projects/links/:link_id', withAuth, handleAPI(deleteLink)) route.post('/projects/milestones/', withAuth, handleAPI(addMilestone)) - route.delete('/projects/milestones/:milestone_id', withAuth, handleAPI(deleteMilestone)) route.get('/projects/:project', handleAPI(getProject)) - route.get('/projects/pitches-total', handleJSON(getOpenPitchesTotal)) - route.get('/projects/tenders-total', handleJSON(getOpenTendersTotal)) + route.delete('/projects/links/:link_id', withAuth, handleAPI(deleteLink)) + route.delete('/projects/personnel/:personnel_id', withAuth, handleAPI(deletePersonnel)) + route.delete('/projects/milestones/:milestone_id', withAuth, handleAPI(deleteMilestone)) }) -function filterProjectsByDate( +function filterProposalProjectsByDate( projects: ProposalProjectWithUpdate[], from?: Date, to?: Date @@ -46,7 +46,14 @@ function filterProjectsByDate( }) } -async function getProjects(req: Request) { +function filterProjectsByDate(projects: ProjectInList[], from?: Date, to?: Date): ProjectInList[] { + return projects.filter((project) => { + const createdAt = new Date(project.created_at) + return (!from || createdAt >= from) && (!to || createdAt < to) + }) +} + +async function getProposalProjects(req: Request) { const from = isValidDate(req.query.from as string) ? new Date(req.query.from as string) : undefined const to = isValidDate(req.query.to as string) ? new Date(req.query.to as string) : undefined @@ -54,21 +61,35 @@ async function getProjects(req: Request) { throw new RequestError('Invalid date range', RequestError.BadRequest) } - const cacheKey = `projects` + const cacheKey = `proposal-projects` const cachedProjects = CacheService.get(cacheKey) if (cachedProjects) { - return { data: filterProjectsByDate(cachedProjects, from, to) } + return { data: filterProposalProjectsByDate(cachedProjects, from, to) } } const projects = await ProjectService.getProposalProjects() CacheService.set(cacheKey, projects, TTL_1_HS) - return { data: filterProjectsByDate(projects, from, to) } + return { data: filterProposalProjectsByDate(projects, from, to) } } -async function getUpdatedProjects(): Promise { +async function getProjectsList(req: Request) { try { - return await ProjectService.getUpdatedProjects() + const from = isValidDate(req.query.from as string) ? new Date(req.query.from as string) : undefined + const to = isValidDate(req.query.to as string) ? new Date(req.query.to as string) : undefined + + if (from && to && from > to) { + throw new RequestError('Invalid date range', RequestError.BadRequest) + } + + const cacheKey = `projects` + const cachedProjects = CacheService.get(cacheKey) + if (cachedProjects) { + return { data: filterProjectsByDate(cachedProjects, from, to) } + } + const projects = await ProjectService.getProjects() + CacheService.set(cacheKey, projects, TTL_1_HS) + return { data: filterProjectsByDate(projects, from, to) } } catch (error) { - ErrorService.report('Error fetching projets', { error, category: ErrorCategory.Project }) + ErrorService.report('Error fetching projects', { error, category: ErrorCategory.Project }) throw new RequestError(`Unable to load projects`, RequestError.InternalServerError) } } diff --git a/src/services/ProjectService.ts b/src/services/ProjectService.ts index ccc1f0d66..2f63858f9 100644 --- a/src/services/ProjectService.ts +++ b/src/services/ProjectService.ts @@ -12,18 +12,20 @@ import ProposalModel from '../entities/Proposal/model' import { ProposalWithOutcome } from '../entities/Proposal/outcome' import { GrantProposalConfiguration, + LatestUpdate, ProposalAttributes, ProposalProjectWithUpdate, ProposalStatus, ProposalType, } from '../entities/Proposal/types' import { DEFAULT_CHOICES, asNumber, getProposalEndDate, isProjectProposal } from '../entities/Proposal/utils' +import { isSameAddress } from '../entities/Snapshot/utils' import UpdateModel from '../entities/Updates/model' -import { IndexedUpdate, UpdateAttributes } from '../entities/Updates/types' +import { UpdateAttributes } from '../entities/Updates/types' import { getPublicUpdates } from '../entities/Updates/utils' import { formatError, inBackground } from '../helpers' import PersonnelModel, { PersonnelAttributes } from '../models/Personnel' -import ProjectModel, { Project, ProjectAttributes, ProjectQueryResult } from '../models/Project' +import ProjectModel, { Project, ProjectAttributes, ProjectInList, ProjectQueryResult } from '../models/Project' import ProjectLinkModel, { ProjectLink } from '../models/ProjectLink' import ProjectMilestoneModel, { ProjectMilestone, ProjectMilestoneStatus } from '../models/ProjectMilestone' import Time from '../utils/date/Time' @@ -70,7 +72,7 @@ export class ProjectService { const update = await this.getProjectLatestUpdate(project.id) const projectWithUpdate: ProposalProjectWithUpdate = { ...project, - ...this.getUpdateData(update), + ...update, } return proposalProjects.push(projectWithUpdate) @@ -87,57 +89,71 @@ export class ProjectService { return proposalProjects } - public static async getUpdatedProjects() { + public static async getProjects(): Promise { const projectsQueryResults = await ProjectModel.getProjectsWithUpdates() - const latestVestingAddresses = projectsQueryResults.map( - (project) => project.vesting_addresses[project.vesting_addresses.length - 1] - ) - const latestVestings = await VestingService.getVestings(latestVestingAddresses) - const updatedProjects = this.mergeProjectsWithVestings(projectsQueryResults, latestVestings) + const vestings = await VestingService.getAllVestings2() + const updatedProjects = this.getProjectInList(projectsQueryResults, vestings) return updatedProjects } - private static mergeProjectsWithVestings( - projects: ProjectQueryResult[], + private static getProjectInList( + projectQueryResult: ProjectQueryResult[], latestVestings: VestingWithLogs[] - ): Project[] { + ): ProjectInList[] { return ( - projects.map((project) => { - const latestVestingAddress = project.vesting_addresses[project.vesting_addresses.length - 1] - const vestingWithLogs = latestVestings.find((vesting) => vesting.address === latestVestingAddress) - const funding = getProjectFunding(project, vestingWithLogs) - const status = getProjectStatus(project, vestingWithLogs) + projectQueryResult.map((result) => { + const latestVestingAddress = result.vesting_addresses[result.vesting_addresses.length - 1] + const vestingWithLogs = latestVestings.find((vesting) => isSameAddress(vesting.address, latestVestingAddress)) + const funding = getProjectFunding(result, vestingWithLogs) + const status = getProjectStatus(result, vestingWithLogs) + const { size, tier, category } = result.configuration + const { updates, user, ...rest } = result return { - ...project, + ...rest, + author: user, + configuration: { size, tier, category: category || result.type }, status, funding, + latest_update: this.getProjectLatestUpdate2(updates ?? []), } }) || [] ) } - private static getUpdateData(update: (UpdateAttributes & { index: number }) | null) { + private static getProjectLatestUpdate2(updates: UpdateAttributes[]): LatestUpdate { + if (!updates || updates.length === 0) { + return { update_timestamp: 0 } + } + + const publicUpdates = getPublicUpdates(updates) + const currentUpdate = publicUpdates[0] + if (!currentUpdate) { + return { update_timestamp: 0 } + } + const { id, introduction, status, health, completion_date } = currentUpdate return { - update, - update_timestamp: update?.completion_date ? Time(update?.completion_date).unix() : 0, + update: { id, introduction, status, health, completion_date, index: publicUpdates.length }, + update_timestamp: currentUpdate?.completion_date ? Time(currentUpdate?.completion_date).unix() : 0, } } - private static async getProjectLatestUpdate(proposalId: string): Promise { + private static async getProjectLatestUpdate(proposalId: string): Promise { const updates = await UpdateModel.find({ proposal_id: proposalId }, { created_at: 'desc', } as never) if (!updates || updates.length === 0) { - return null + return { update_timestamp: 0 } } const publicUpdates = getPublicUpdates(updates) const currentUpdate = publicUpdates[0] if (!currentUpdate) { - return null + return { update_timestamp: 0 } + } + return { + update: { ...currentUpdate, index: publicUpdates.length }, + update_timestamp: currentUpdate?.completion_date ? Time(currentUpdate?.completion_date).unix() : 0, } - - return { ...currentUpdate, index: publicUpdates.length } } public static async getGrantInCreation(grantRequest: GrantRequest, user: string) { diff --git a/src/services/VestingService.ts b/src/services/VestingService.ts index d936872a4..f20fa2aa4 100644 --- a/src/services/VestingService.ts +++ b/src/services/VestingService.ts @@ -16,7 +16,7 @@ import { VestingStatus } from '../entities/Grant/types' import { ContractVersion, TopicsByVersion } from '../utils/contracts/vesting' import { ErrorCategory } from '../utils/errorCategories' -import CacheService, { TTL_24_HS } from './CacheService' +import CacheService, { TTL_1_HS, TTL_24_HS } from './CacheService' import { ErrorService } from './ErrorService' export class VestingService { @@ -33,9 +33,25 @@ export class VestingService { return transparencyVestings } + static async getAllVestings2(): Promise { + const cacheKey = `vesting-subgraph-data` + + const cachedData = CacheService.get(cacheKey) + if (cachedData) { + return cachedData + } + const vestingsData = await VestingsSubgraph.get().getVestings() + const sortedVestings = vestingsData + .map((data) => this.parseSubgraphVesting(data)) + .sort((a, b) => this.sortVestingsByDate(a, b)) + CacheService.set(cacheKey, sortedVestings, TTL_1_HS) + return sortedVestings + } + static async getVestings(addresses: string[]): Promise { const vestingsData = await VestingsSubgraph.get().getVestings(addresses) - return vestingsData.map(this.parseSubgraphVesting).sort(this.sortVestingsByDate) + const sortedVestings = vestingsData.map(this.parseSubgraphVesting).sort(this.sortVestingsByDate) + return sortedVestings } static async getVestingWithLogs( diff --git a/src/utils/projects.ts b/src/utils/projects.ts index a33566eaf..40ae21ded 100644 --- a/src/utils/projects.ts +++ b/src/utils/projects.ts @@ -47,7 +47,7 @@ function getFunding(proposal: ProposalAttributes, transparencyVesting?: Transpar } } -export function getProjectFunding(project: ProjectQueryResult, vesting: VestingWithLogs | undefined): ProjectFunding { +export function getProjectFunding(project: ProjectQueryResult, vesting: Vesting | undefined): ProjectFunding { if (project.enacting_tx) { // one time payment return { @@ -61,14 +61,13 @@ export function getProjectFunding(project: ProjectQueryResult, vesting: VestingW if (!vesting) { return {} } - return { enacted_at: vesting.start_at, vesting, } } -export function getProjectStatus(project: ProjectQueryResult, vesting: VestingWithLogs | undefined) { +export function getProjectStatus(project: ProjectQueryResult, vesting: VestingWithLogs | undefined): ProjectStatus { const legacyCondition = !vesting && project.enacted_description if (project.enacting_tx || legacyCondition) { return ProjectStatus.Finished @@ -78,9 +77,7 @@ export function getProjectStatus(project: ProjectQueryResult, vesting: VestingWi return ProjectStatus.Pending } - const { status } = vesting - - return toGovernanceProjectStatus(status) + return toGovernanceProjectStatus(vesting.status) } function getProposalProjectStatus(proposal: ProposalAttributes, vesting?: TransparencyVesting) {