From f6057f33b57d867e8a63a14890a6f33378dafa9a Mon Sep 17 00:00:00 2001 From: paulo-ocean Date: Thu, 31 Oct 2024 11:18:52 +0000 Subject: [PATCH 01/20] add datasets on ComputeAsset, new start compute fn --- src/@types/Compute.ts | 10 +++++ src/services/Provider.ts | 85 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 93 insertions(+), 2 deletions(-) diff --git a/src/@types/Compute.ts b/src/@types/Compute.ts index 81142c7cb..5c6638b97 100644 --- a/src/@types/Compute.ts +++ b/src/@types/Compute.ts @@ -59,7 +59,17 @@ export interface ComputeOutput { whitelist?: string[] } +export enum EncryptMethod { + AES = 'AES', + ECIES = 'ECIES' +} +export interface BaseFileObject { + type: string + encryptedBy?: string + encryptMethod?: EncryptMethod +} export interface ComputeAsset { + fileObject?: BaseFileObject // C2D v2 documentId: string serviceId: string transferTxId?: string diff --git a/src/services/Provider.ts b/src/services/Provider.ts index e5e497347..169f9eaf3 100644 --- a/src/services/Provider.ts +++ b/src/services/Provider.ts @@ -524,7 +524,7 @@ export class Provider { return consumeUrl } - /** Instruct the provider to start a compute job + /** Instruct the provider to start a compute job (Old C2D V1) Kept for now, for backwards compatibility * @param {string} providerUri The provider URI. * @param {Signer} signer The consumer signer object. * @param {string} computeEnv The compute environment. @@ -535,7 +535,7 @@ export class Provider { * @param {ComputeOutput} output The compute job output settings. * @return {Promise} The compute job or jobs. */ - public async computeStart( + public async computeStartV1( providerUri: string, consumer: Signer, computeEnv: string, @@ -607,6 +607,87 @@ export class Provider { return null } + /** Instruct the provider to start a compute job (new C2D V2) + * @param {string} providerUri The provider URI. + * @param {Signer} signer The consumer signer object. + * @param {string} computeEnv The compute environment. + * @param {ComputeAsset} datasets The dataset to start compute on + additionalDatasets (the additional datasets if that is the case) + * @param {ComputeAlgorithm} algorithm The algorithm to start compute with. + * @param {AbortSignal} signal abort signal + * @param {ComputeOutput} output The compute job output settings. + * @return {Promise} The compute job or jobs. + */ + public async computeStart( + providerUri: string, + consumer: Signer, + computeEnv: string, + datasets: ComputeAsset[], + algorithm: ComputeAlgorithm, + signal?: AbortSignal, + output?: ComputeOutput + ): Promise { + const providerEndpoints = await this.getEndpoints(providerUri) + const serviceEndpoints = await this.getServiceEndpoints( + providerUri, + providerEndpoints + ) + const computeStartUrl = this.getEndpointURL(serviceEndpoints, 'computeStart') + ? this.getEndpointURL(serviceEndpoints, 'computeStart').urlPath + : null + + const consumerAddress = await consumer.getAddress() + const nonce = ( + (await this.getNonce( + providerUri, + consumerAddress, + signal, + providerEndpoints, + serviceEndpoints + )) + 1 + ).toString() + + let signatureMessage = consumerAddress + signatureMessage += datasets[0].documentId + signatureMessage += nonce + const signature = await this.signProviderRequest(consumer, signatureMessage) + const payload = Object() + payload.consumerAddress = consumerAddress + payload.signature = signature + payload.nonce = nonce + payload.environment = computeEnv + payload.datasets = datasets + payload.algorithm = algorithm + + if (output) payload.output = output + if (!computeStartUrl) return null + let response + try { + response = await fetch(computeStartUrl, { + method: 'POST', + body: JSON.stringify(payload), + headers: { 'Content-Type': 'application/json' }, + signal + }) + } catch (e) { + LoggerInstance.error('Compute start failed:') + LoggerInstance.error(e) + LoggerInstance.error('Payload was:', payload) + throw new Error('HTTP request failed calling Provider') + } + if (response?.ok) { + const params = await response.json() + return params + } + LoggerInstance.error( + 'Compute start failed: ', + response.status, + response.statusText, + await response.json() + ) + LoggerInstance.error('Payload was:', payload) + return null + } + /** Instruct the provider to Stop the execution of a to stop a compute job. * @param {string} did the asset did * @param {string} consumerAddress The consumer address. From 9e6ec4d29680d19184587537fea015888b44633e Mon Sep 17 00:00:00 2001 From: paulo-ocean Date: Thu, 31 Oct 2024 18:01:50 +0000 Subject: [PATCH 02/20] fix type mismatch new c2d v2 --- test/integration/ComputeExamples.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/ComputeExamples.test.ts b/test/integration/ComputeExamples.test.ts index cbfb81b30..cb5970e6d 100644 --- a/test/integration/ComputeExamples.test.ts +++ b/test/integration/ComputeExamples.test.ts @@ -655,7 +655,7 @@ describe('Compute-to-data example tests', async () => { providerUrl, consumerAccount, computeEnv.id, - assets[0], + assets, algo ) From 50e3ed81c8a39ccda7b3a07d0639ffa7ca2619db Mon Sep 17 00:00:00 2001 From: paulo-ocean Date: Thu, 31 Oct 2024 18:26:43 +0000 Subject: [PATCH 03/20] fix tests --- ComputeExamples.md | 2 +- test/integration/ComputeFlow.test.ts | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/ComputeExamples.md b/ComputeExamples.md index 41f5c618b..bd9b7ee7d 100644 --- a/ComputeExamples.md +++ b/ComputeExamples.md @@ -655,7 +655,7 @@ Let's have 5 minute of compute access providerUrl, consumerAccount, computeEnv.id, - assets[0], + assets, algo ) diff --git a/test/integration/ComputeFlow.test.ts b/test/integration/ComputeFlow.test.ts index 1b5664bc0..cfd5ed7ec 100644 --- a/test/integration/ComputeFlow.test.ts +++ b/test/integration/ComputeFlow.test.ts @@ -461,7 +461,7 @@ describe('Compute flow tests', async () => { providerUrl, consumerAccount, computeEnv.id, - assets[0], + assets, algo ) freeEnvDatasetTxId = assets[0].transferTxId @@ -530,7 +530,7 @@ describe('Compute flow tests', async () => { providerUrl, consumerAccount, computeEnv.id, - assets[0], + assets, algo ) assert(computeJobs, 'Cannot start compute job') @@ -594,7 +594,7 @@ describe('Compute flow tests', async () => { providerUrl, consumerAccount, computeEnv.id, - assets[0], + assets, algo ) paidEnvDatasetTxId = assets[0].transferTxId @@ -670,7 +670,7 @@ describe('Compute flow tests', async () => { providerUrl, consumerAccount, computeEnv.id, - assets[0], + assets, algo ) assert(computeJobs, 'Cannot start compute job') @@ -762,7 +762,7 @@ describe('Compute flow tests', async () => { providerUrl, consumerAccount, computeEnv.id, - assets[0], + assets, algo ) // freeEnvDatasetTxId = assets[0].transferTxId @@ -846,7 +846,7 @@ describe('Compute flow tests', async () => { providerUrl, consumerAccount, computeEnv.id, - assets[0], + assets, algo ) // freeEnvDatasetTxId = assets[0].transferTxId From 0f0ad3e8ffd7fc65c3702f53af64ce12f421b3a1 Mon Sep 17 00:00:00 2001 From: paulo-ocean Date: Thu, 31 Oct 2024 19:33:54 +0000 Subject: [PATCH 04/20] fix tests, keep backwards compatibility due to tests running with existing provider --- src/services/Provider.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/services/Provider.ts b/src/services/Provider.ts index 169f9eaf3..3d7786908 100644 --- a/src/services/Provider.ts +++ b/src/services/Provider.ts @@ -626,6 +626,7 @@ export class Provider { signal?: AbortSignal, output?: ComputeOutput ): Promise { + console.log('called new compute start method...') const providerEndpoints = await this.getEndpoints(providerUri) const serviceEndpoints = await this.getServiceEndpoints( providerUri, @@ -655,9 +656,12 @@ export class Provider { payload.signature = signature payload.nonce = nonce payload.environment = computeEnv + // kept for backwards compatibility (tests running against existing provider) + payload.dataset = datasets[0] + // new field for C2D v2 payload.datasets = datasets payload.algorithm = algorithm - + // if (additionalDatasets) payload.additionalDatasets = additionalDatasets if (output) payload.output = output if (!computeStartUrl) return null let response From a9a099d0a01d2fb339d3f8ebe7cc4ad4b1dd422f Mon Sep 17 00:00:00 2001 From: paulo-ocean Date: Mon, 4 Nov 2024 17:44:45 +0000 Subject: [PATCH 05/20] support for starting a free env compute job --- src/@types/Compute.ts | 1 + src/services/Provider.ts | 18 ++++++++++++++---- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/@types/Compute.ts b/src/@types/Compute.ts index 5c6638b97..83df541bf 100644 --- a/src/@types/Compute.ts +++ b/src/@types/Compute.ts @@ -22,6 +22,7 @@ export interface ComputeEnvironment { storageExpiry: number maxJobDuration: number lastSeen: number + free: boolean } export interface ComputeResult { diff --git a/src/services/Provider.ts b/src/services/Provider.ts index 3d7786908..00b53c359 100644 --- a/src/services/Provider.ts +++ b/src/services/Provider.ts @@ -615,6 +615,7 @@ export class Provider { * @param {ComputeAlgorithm} algorithm The algorithm to start compute with. * @param {AbortSignal} signal abort signal * @param {ComputeOutput} output The compute job output settings. + * @param {boolean} freeEnvironment is it a free environment? uses different route * @return {Promise} The compute job or jobs. */ public async computeStart( @@ -624,7 +625,8 @@ export class Provider { datasets: ComputeAsset[], algorithm: ComputeAlgorithm, signal?: AbortSignal, - output?: ComputeOutput + output?: ComputeOutput, + freeEnvironment?: boolean ): Promise { console.log('called new compute start method...') const providerEndpoints = await this.getEndpoints(providerUri) @@ -632,9 +634,17 @@ export class Provider { providerUri, providerEndpoints ) - const computeStartUrl = this.getEndpointURL(serviceEndpoints, 'computeStart') - ? this.getEndpointURL(serviceEndpoints, 'computeStart').urlPath - : null + let computeStartUrl = null + + if (freeEnvironment) { + computeStartUrl = this.getEndpointURL(serviceEndpoints, 'freeCompute') + ? this.getEndpointURL(serviceEndpoints, 'freeCompute').urlPath + : null + } else { + computeStartUrl = this.getEndpointURL(serviceEndpoints, 'computeStart') + ? this.getEndpointURL(serviceEndpoints, 'computeStart').urlPath + : null + } const consumerAddress = await consumer.getAddress() const nonce = ( From 21071979101fbdc407efceefe6b7679894dba0db Mon Sep 17 00:00:00 2001 From: paulo-ocean Date: Tue, 5 Nov 2024 16:54:14 +0000 Subject: [PATCH 06/20] minor fix --- src/@types/Compute.ts | 7 +++++++ test/integration/helpers.ts | 3 ++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/@types/Compute.ts b/src/@types/Compute.ts index 83df541bf..26a6dbd07 100644 --- a/src/@types/Compute.ts +++ b/src/@types/Compute.ts @@ -60,6 +60,12 @@ export interface ComputeOutput { whitelist?: string[] } +export enum FileObjectType { + URL = 'url', + IPFS = 'ipfs', + ARWEAVE = 'arweave' +} + export enum EncryptMethod { AES = 'AES', ECIES = 'ECIES' @@ -78,6 +84,7 @@ export interface ComputeAsset { } export interface ComputeAlgorithm { + fileObject?: BaseFileObject // C2D v2 documentId?: string serviceId?: string meta?: MetadataAlgorithm diff --git a/test/integration/helpers.ts b/test/integration/helpers.ts index 6e3f7c75d..fac14b9b2 100644 --- a/test/integration/helpers.ts +++ b/test/integration/helpers.ts @@ -137,7 +137,8 @@ export async function handleComputeOrder( if (config.chainId !== chainID) { throw new Error('Chain ID from DDO is different than the configured network.') } - if (order.providerFee && order.providerFee.providerFeeAmount) { + const hasProviderFees = order.providerFee && order.providerFee.providerFeeAmount + if (hasProviderFees && Number(order.providerFee.providerFeeAmount) > 0) { await approveWei( payerAccount, config, From 583db8e65df39b91b198186c9df85fa585713740 Mon Sep 17 00:00:00 2001 From: paulo-ocean Date: Wed, 6 Nov 2024 09:44:30 +0000 Subject: [PATCH 07/20] add file object types --- src/@types/Compute.ts | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/@types/Compute.ts b/src/@types/Compute.ts index 26a6dbd07..85b5b123b 100644 --- a/src/@types/Compute.ts +++ b/src/@types/Compute.ts @@ -70,11 +70,30 @@ export enum EncryptMethod { AES = 'AES', ECIES = 'ECIES' } + +export interface HeadersObject { + [key: string]: string +} + export interface BaseFileObject { type: string encryptedBy?: string encryptMethod?: EncryptMethod } + +export interface UrlFileObject extends BaseFileObject { + url: string + method: string + headers?: [HeadersObject] +} + +export interface IpfsFileObject extends BaseFileObject { + hash: string +} + +export interface ArweaveFileObject extends BaseFileObject { + transactionId: string +} export interface ComputeAsset { fileObject?: BaseFileObject // C2D v2 documentId: string From 7df390a3bee7fc1650e9b14a8985102fe7ad7652 Mon Sep 17 00:00:00 2001 From: paulo-ocean Date: Thu, 7 Nov 2024 15:45:44 +0000 Subject: [PATCH 08/20] enforce asset files object type --- src/utils/Assets.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/utils/Assets.ts b/src/utils/Assets.ts index 0322314d9..513240ae6 100644 --- a/src/utils/Assets.ts +++ b/src/utils/Assets.ts @@ -17,6 +17,7 @@ import { ProviderInstance } from '../services/Provider' import AccessListFactory from '@oceanprotocol/contracts/artifacts/contracts/accesslists/AccessListFactory.sol/AccessListFactory.json' import ERC20Template4 from '@oceanprotocol/contracts/artifacts/contracts/templates/ERC20Template4.sol/ERC20Template4.json' import { calculateActiveTemplateIndex } from './Adresses' +import { FileObjectType } from '../@types' // import * as hre from 'hardhat' @@ -113,6 +114,15 @@ export async function createAsset( mpFeeAddress: ZERO_ADDRESS } + if ( + !assetUrl.type || + ![FileObjectType.ARWEAVE, FileObjectType.IPFS, FileObjectType.URL].includes( + assetUrl.type.toLowerCase() + ) + ) { + console.log('Missing or invalid files object type, defaulting to "url"') + assetUrl.type = FileObjectType.URL + } // include fileObject in the DT constructor if (config.sdk === 'oasis') { datatokenParams.filesObject = assetUrl From e312ec7119b63110b5b05ccaef2287bff94c6c90 Mon Sep 17 00:00:00 2001 From: paulo-ocean Date: Tue, 12 Nov 2024 12:18:32 +0000 Subject: [PATCH 09/20] support for consumer signature on initialize compute --- src/services/Provider.ts | 88 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 87 insertions(+), 1 deletion(-) diff --git a/src/services/Provider.ts b/src/services/Provider.ts index 00b53c359..3d02dcc1f 100644 --- a/src/services/Provider.ts +++ b/src/services/Provider.ts @@ -418,7 +418,7 @@ export class Provider { * @param {AbortSignal} signal abort signal * @return {Promise} ProviderComputeInitialize data */ - public async initializeCompute( + public async initializeComputeV1( assets: ComputeAsset[], algorithm: ComputeAlgorithm, computeEnv: string, @@ -470,6 +470,90 @@ export class Provider { throw new Error(JSON.stringify(resolvedResponse)) } + /** Initializes the provider for a compute request. + * @param {ComputeAsset[]} assets The datasets array to initialize compute request. + * @param {ComputeAlgorithmber} algorithm The algorithm to use. + * @param {string} computeEnv The compute environment. + * @param {number} validUntil The job expiration date. + * @param {string} providerUri The provider URI. + * @param {Signer} signer caller address + * @param {AbortSignal} signal abort signal + * @return {Promise} ProviderComputeInitialize data + */ + public async initializeCompute( + assets: ComputeAsset[], + algorithm: ComputeAlgorithm, + computeEnv: string, + validUntil: number, + providerUri: string, + signer: Signer, + signal?: AbortSignal + ): Promise { + const providerEndpoints = await this.getEndpoints(providerUri) + const serviceEndpoints = await this.getServiceEndpoints( + providerUri, + providerEndpoints + ) + + // Diff from V1. We might need a signature to get the files object, specially if dealing with confidential evm and template 4 + // otherwise it can be ignored + const consumerAddress = await signer.getAddress() + const nonce = ( + (await this.getNonce( + providerUri, + consumerAddress, + signal, + providerEndpoints, + serviceEndpoints + )) + 1 + ).toString() + + // same signed message as for start compute (consumer address + did[0] + nonce) + let signatureMessage = consumerAddress + signatureMessage += assets[0].documentId + signatureMessage += nonce + const signature = await this.signProviderRequest(signer, signatureMessage) + + const providerData = { + datasets: assets, + algorithm, + compute: { env: computeEnv, validUntil }, + consumerAddress, + signature + } + const initializeUrl = this.getEndpointURL(serviceEndpoints, 'initializeCompute') + ? this.getEndpointURL(serviceEndpoints, 'initializeCompute').urlPath + : null + if (!initializeUrl) return null + + let response + try { + response = await fetch(initializeUrl, { + method: 'POST', + body: JSON.stringify(providerData), + headers: { 'Content-Type': 'application/json' }, + signal + }) + } catch (e) { + LoggerInstance.error('Initialize compute failed: ') + LoggerInstance.error(e) + throw new Error('ComputeJob cannot be initialized') + } + if (response?.ok) { + const params = await response.json() + return params + } + const resolvedResponse = await response.json() + LoggerInstance.error( + 'Initialize compute failed: ', + response.status, + response.statusText, + resolvedResponse + ) + LoggerInstance.error('Payload was:', providerData) + throw new Error(JSON.stringify(resolvedResponse)) + } + /** * Gets the download URL. * @param {string} did - The DID. @@ -629,6 +713,8 @@ export class Provider { freeEnvironment?: boolean ): Promise { console.log('called new compute start method...') + console.log('datasets: ', datasets) + console.log('algorithm: ', algorithm) const providerEndpoints = await this.getEndpoints(providerUri) const serviceEndpoints = await this.getServiceEndpoints( providerUri, From 88ee8d3b94889dd247899de45cdc3b4411da4ed2 Mon Sep 17 00:00:00 2001 From: paulo-ocean Date: Tue, 12 Nov 2024 12:40:05 +0000 Subject: [PATCH 10/20] update docs and tests, pass signer account, not signer address --- ComputeExamples.md | 2 +- docs/classes/Provider.md | 2 +- test/integration/ComputeExamples.test.ts | 2 +- test/integration/ComputeFlow.test.ts | 12 ++++++------ 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ComputeExamples.md b/ComputeExamples.md index bd9b7ee7d..83dc646f9 100644 --- a/ComputeExamples.md +++ b/ComputeExamples.md @@ -627,7 +627,7 @@ Let's have 5 minute of compute access computeEnv.id, computeValidUntil, providerUrl, - await consumerAccount.getAddress() + consumerAccount ) ```