diff --git a/engines/config-query-sparql-incremental/config/query-operation/actors.json b/engines/config-query-sparql-incremental/config/query-operation/actors.json index c5dd7cf..67ebf57 100644 --- a/engines/config-query-sparql-incremental/config/query-operation/actors.json +++ b/engines/config-query-sparql-incremental/config/query-operation/actors.json @@ -12,6 +12,7 @@ "ccqs:config/query-operation/actors/query/minus.json", "ccqs:config/query-operation/actors/query/nop.json", "icqsi:config/query-operation/actors/query/slice.json", + "icqsi:config/query-operation/actors/query/reduced.json", "ccqs:config/query-operation/actors/query/leftjoin.json", "ccqs:config/query-operation/actors/query/values.json", "ccqs:config/query-operation/actors/query/bgp.json", diff --git a/engines/config-query-sparql-incremental/config/query-operation/actors/query/reduced.json b/engines/config-query-sparql-incremental/config/query-operation/actors/query/reduced.json new file mode 100644 index 0000000..23da923 --- /dev/null +++ b/engines/config-query-sparql-incremental/config/query-operation/actors/query/reduced.json @@ -0,0 +1,17 @@ +{ + "@context": [ + "https://linkedsoftwaredependencies.org/bundles/npm/@comunica/runner/^4.0.0/components/context.jsonld", + + "https://linkedsoftwaredependencies.org/bundles/npm/@incremunica/actor-query-operation-reduced-hash/^1.0.0/components/context.jsonld" + ], + "@id": "urn:comunica:default:Runner", + "@type": "Runner", + "actors": [ + { + "@id": "urn:comunica:default:query-operation/actors#reduced", + "@type": "ActorQueryOperationReducedHash", + "mediatorQueryOperation": { "@id": "urn:comunica:default:query-operation/mediators#main" }, + "mediatorHashBindings": { "@id": "urn:comunica:default:hash-bindings/mediators#main" } + } + ] +} diff --git a/engines/query-sparql-incremental/package.json b/engines/query-sparql-incremental/package.json index 3979460..8825ffd 100644 --- a/engines/query-sparql-incremental/package.json +++ b/engines/query-sparql-incremental/package.json @@ -269,6 +269,7 @@ "@incremunica/actor-merge-bindings-context-is-addition": "^1.3.0", "@incremunica/actor-query-operation-distinct-hash": "^1.3.0", "@incremunica/actor-query-operation-group": "^1.3.0", + "@incremunica/actor-query-operation-reduced-hash": "^1.3.0", "@incremunica/actor-query-operation-slice": "^1.3.0", "@incremunica/actor-query-source-identify-hypermedia-none": "^1.3.0", "@incremunica/actor-query-source-identify-stream": "^1.3.0", diff --git a/packages/actor-query-operation-reduced-hash/README.md b/packages/actor-query-operation-reduced-hash/README.md new file mode 100644 index 0000000..fc87218 --- /dev/null +++ b/packages/actor-query-operation-reduced-hash/README.md @@ -0,0 +1,38 @@ +# Incremunica Reduced Hash Query Operation Actor + +[![npm version](https://badge.fury.io/js/%40incremunica%2Factor-query-operation-reduced-hash.svg)](https://www.npmjs.com/package/@incremunica/actor-query-operation-reduced-hash) + +A [Query Operation](https://github.com/comunica/comunica/tree/master/packages/bus-query-operation) actor that handles [SPARQL `REDUCED`](https://www.w3.org/TR/sparql11-query/#sparqlReduced) operations +by maintaining a hash-based cache of a fixed size. + +## Install + +```bash +$ yarn add @incremunica/actor-query-operation-reduced-hash +``` + +## Configure + +After installing, this package can be added to your engine's configuration as follows: +```text +{ + "@context": [ + ... + "https://linkedsoftwaredependencies.org/bundles/npm/@incremunica/actor-query-operation-reduced-hash/^1.0.0/components/context.jsonld" + ], + "actors": [ + ... + { + "@id": "urn:comunica:default:query-operation/actors#reduced", + "@type": "ActorQueryOperationReducedHash", + "mediatorQueryOperation": { "@id": "urn:comunica:default:query-operation/mediators#main" }, + "mediatorHashBindings": { "@id": "urn:comunica:default:hash-bindings/mediators#main" } + } + ] +} +``` + +### Config Parameters + +* `mediatorQueryOperation`: A mediator over the [Query Operation bus](https://github.com/comunica/comunica/tree/master/packages/bus-query-operation). +* `mediatorHashBindings`: A mediator over the [Hash Bindings bus](https://github.com/comunica/comunica/tree/master/packages/bus-hash-bindings). diff --git a/packages/actor-query-operation-reduced-hash/lib/ActorQueryOperationReducedHash.ts b/packages/actor-query-operation-reduced-hash/lib/ActorQueryOperationReducedHash.ts new file mode 100644 index 0000000..08f6e07 --- /dev/null +++ b/packages/actor-query-operation-reduced-hash/lib/ActorQueryOperationReducedHash.ts @@ -0,0 +1,88 @@ +import type { MediatorHashBindings } from '@comunica/bus-hash-bindings'; +import type { IActorQueryOperationTypedMediatedArgs } from '@comunica/bus-query-operation'; +import { ActorQueryOperationTypedMediated } from '@comunica/bus-query-operation'; +import type { IActorTest, TestResult } from '@comunica/core'; +import { passTestVoid } from '@comunica/core'; +import type { + BindingsStream, + IActionContext, + IQueryOperationResult, + IQueryOperationResultBindings, +} from '@comunica/types'; +import type { Bindings } from '@comunica/utils-bindings-factory'; +import { getSafeBindings } from '@comunica/utils-query-operation'; +import { KeysBindings } from '@incremunica/context-entries'; +import type * as RDF from '@rdfjs/types'; +import type { AsyncIterator } from 'asynciterator'; +import type { Algebra } from 'sparqlalgebrajs'; + +/** + * An Incremunica Reduced Hash Query Operation Actor. + */ +export class ActorQueryOperationReducedHash extends ActorQueryOperationTypedMediated { + public readonly mediatorHashBindings: MediatorHashBindings; + + public constructor(args: IActorQueryOperationReducedHashArgs) { + super(args, 'reduced'); + } + + public async testOperation(_operation: Algebra.Reduced, _context: IActionContext): Promise> { + return passTestVoid(); + } + + public async runOperation(operation: Algebra.Reduced, context: IActionContext): Promise { + const output: IQueryOperationResultBindings = getSafeBindings( + await this.mediatorQueryOperation.mediate({ operation: operation.input, context }), + ); + const variables = (await output.metadata()).variables.map(v => v.variable); + const bindingsStream: BindingsStream = + (>output.bindingsStream) + .filter(await this.newHashFilter(context, variables)); + return { + type: 'bindings', + bindingsStream, + metadata: output.metadata, + }; + } + + /** + * Create a new distinct filter function. + * This will maintain an internal hash datastructure so that every bindings object only returns true once. + * @param context The action context. + * @param variables The variables to take into account while hashing. + * @return {(bindings: Bindings) => boolean} A distinct filter for bindings. + */ + public async newHashFilter( + context: IActionContext, + variables: RDF.Variable[], + ): Promise<(bindings: Bindings) => boolean> { + const { hashFunction } = await this.mediatorHashBindings.mediate({ context }); + const hashes: Map = new Map(); + return (bindings: Bindings) => { + const hash = hashFunction(bindings, variables); + const hasMapValue = hashes.get(hash); + const isAddition = bindings.getContextEntry(KeysBindings.isAddition) ?? true; + if (isAddition) { + if (hasMapValue) { + hashes.set(hash, hasMapValue + 1); + return false; + } + hashes.set(hash, 1); + return true; + } + if (!hasMapValue) { + return false; + } + if (hasMapValue === 1) { + hashes.delete(hash); + return true; + } + hashes.set(hash, hasMapValue - 1); + return false; + }; + } +} + +export interface IActorQueryOperationReducedHashArgs extends IActorQueryOperationTypedMediatedArgs { + mediatorHashBindings: MediatorHashBindings; +} diff --git a/packages/actor-query-operation-reduced-hash/lib/index.ts b/packages/actor-query-operation-reduced-hash/lib/index.ts new file mode 100644 index 0000000..41317a2 --- /dev/null +++ b/packages/actor-query-operation-reduced-hash/lib/index.ts @@ -0,0 +1 @@ +export * from './ActorQueryOperationReducedHash'; diff --git a/packages/actor-query-operation-reduced-hash/package.json b/packages/actor-query-operation-reduced-hash/package.json new file mode 100644 index 0000000..ee99688 --- /dev/null +++ b/packages/actor-query-operation-reduced-hash/package.json @@ -0,0 +1,50 @@ +{ + "name": "@incremunica/actor-query-operation-reduced-hash", + "version": "1.3.0", + "description": "A reduced-hash query-operation actor", + "lsd:module": true, + "license": "MIT", + "homepage": "https://maartyman.github.io/incremunica/", + "repository": { + "type": "git", + "url": "https://github.com/comunica/comunica.git", + "directory": "packages/actor-query-operation-reduced-hash" + }, + "bugs": { + "url": "https://github.com/comunica/comunica/issues" + }, + "keywords": [ + "incremunica", + "actor", + "query-operation", + "reduced-hash" + ], + "sideEffects": false, + "main": "lib/index.js", + "typings": "lib/index", + "publishConfig": { + "access": "public" + }, + "files": [ + "components", + "lib/**/*.d.ts", + "lib/**/*.js", + "lib/**/*.js.map" + ], + "scripts": { + "build": "yarn run build:ts && yarn run build:components", + "build:ts": "node \"../../node_modules/typescript/bin/tsc\"", + "build:components": "componentsjs-generator" + }, + "dependencies": { + "@comunica/bus-hash-bindings": "^4.0.2", + "@comunica/bus-query-operation": "^4.0.2", + "@comunica/core": "^4.0.2", + "@comunica/types": "^4.0.2", + "@comunica/utils-bindings-factory": "^4.0.2", + "@comunica/utils-query-operation": "^4.0.2", + "@incremunica/context-entries": "^1.3.0", + "@rdfjs/types": "*", + "sparqlalgebrajs": "^4.3.8" + } +} diff --git a/packages/actor-query-operation-reduced-hash/test/ActorQueryOperationReducedHash-test.ts b/packages/actor-query-operation-reduced-hash/test/ActorQueryOperationReducedHash-test.ts new file mode 100644 index 0000000..bc79bcf --- /dev/null +++ b/packages/actor-query-operation-reduced-hash/test/ActorQueryOperationReducedHash-test.ts @@ -0,0 +1,273 @@ +import type { MediatorHashBindings } from '@comunica/bus-hash-bindings'; +import { ActionContext, Bus } from '@comunica/core'; +import type { BindingsFactory } from '@comunica/utils-bindings-factory'; +import { getSafeBindings } from '@comunica/utils-query-operation'; +import { KeysBindings } from '@incremunica/context-entries'; +import { createTestMediatorHashBindings, createTestBindingsFactory } from '@incremunica/dev-tools'; +import { ArrayIterator } from 'asynciterator'; +import { DataFactory } from 'rdf-data-factory'; +import { ActorQueryOperationReducedHash } from '../lib'; +import '@comunica/utils-jest'; + +const DF = new DataFactory(); + +describe('ActorQueryOperationReducedHash', () => { + let bus: any; + let mediatorQueryOperation: any; + let BF: BindingsFactory; + + beforeEach(async() => { + BF = await createTestBindingsFactory(DF); + bus = new Bus({ name: 'bus' }); + mediatorQueryOperation = { + mediate: (arg: any) => Promise.resolve({ + bindingsStream: new ArrayIterator([ + BF.bindings([[ DF.variable('a'), DF.literal('1') ]]), + BF.bindings([[ DF.variable('a'), DF.literal('2') ]]), + BF.bindings([[ DF.variable('a'), DF.literal('1') ]]), + BF.bindings([[ DF.variable('a'), DF.literal('3') ]]), + BF.bindings([[ DF.variable('a'), DF.literal('2') ]]).setContextEntry(KeysBindings.isAddition, false), + ]), + metadata: () => Promise.resolve({ + cardinality: 5, + variables: [ + { + variable: DF.variable('a'), + isUndef: false, + }, + ], + }), + operated: arg, + type: 'bindings', + }), + }; + }); + + describe('newHashFilter', () => { + let actor: ActorQueryOperationReducedHash; + let mediatorHashBindings: MediatorHashBindings; + + beforeEach(() => { + mediatorHashBindings = createTestMediatorHashBindings(); + actor = new ActorQueryOperationReducedHash( + { name: 'actor', bus, mediatorQueryOperation, mediatorHashBindings }, + ); + }); + + it('should create a filter', async() => { + await expect(actor.newHashFilter({}, [])).resolves.toBeInstanceOf(Function); + }); + + it('should create a filter that is a predicate', async() => { + const filter = await actor.newHashFilter({}, []); + expect(filter(BF.bindings([ + [ DF.variable('a'), DF.literal('a') ], + ]).setContextEntry(KeysBindings.isAddition, true))).toBe(true); + }); + + it('should create a filter that only returns true once for equal objects', async() => { + const filter = await actor.newHashFilter({}, [ DF.variable('a') ]); + expect(filter(BF.bindings([ + [ DF.variable('a'), DF.literal('a') ], + ]).setContextEntry(KeysBindings.isAddition, true))).toBe(true); + expect(filter(BF.bindings([ + [ DF.variable('a'), DF.literal('a') ], + ]).setContextEntry(KeysBindings.isAddition, true))).toBe(false); + expect(filter(BF.bindings([ + [ DF.variable('a'), DF.literal('a') ], + ]).setContextEntry(KeysBindings.isAddition, true))).toBe(false); + expect(filter(BF.bindings([ + [ DF.variable('a'), DF.literal('a') ], + ]).setContextEntry(KeysBindings.isAddition, true))).toBe(false); + + expect(filter(BF.bindings([ + [ DF.variable('a'), DF.literal('b') ], + ]).setContextEntry(KeysBindings.isAddition, true))).toBe(true); + expect(filter(BF.bindings([ + [ DF.variable('a'), DF.literal('b') ], + ]).setContextEntry(KeysBindings.isAddition, true))).toBe(false); + expect(filter(BF.bindings([ + [ DF.variable('a'), DF.literal('b') ], + ]).setContextEntry(KeysBindings.isAddition, true))).toBe(false); + expect(filter(BF.bindings([ + [ DF.variable('a'), DF.literal('b') ], + ]).setContextEntry(KeysBindings.isAddition, true))).toBe(false); + }); + + it('should create a filters that are independent', async() => { + const filter1 = await actor.newHashFilter({}, []); + const filter2 = await actor.newHashFilter({}, []); + const filter3 = await actor.newHashFilter({}, []); + expect(filter1(BF.bindings([ + [ DF.variable('a'), DF.literal('b') ], + ]).setContextEntry(KeysBindings.isAddition, true))).toBe(true); + expect(filter1(BF.bindings([ + [ DF.variable('a'), DF.literal('b') ], + ]).setContextEntry(KeysBindings.isAddition, true))).toBe(false); + + expect(filter2(BF.bindings([ + [ DF.variable('a'), DF.literal('b') ], + ]).setContextEntry(KeysBindings.isAddition, true))).toBe(true); + expect(filter2(BF.bindings([ + [ DF.variable('a'), DF.literal('b') ], + ]).setContextEntry(KeysBindings.isAddition, true))).toBe(false); + + expect(filter3(BF.bindings([ + [ DF.variable('a'), DF.literal('b') ], + ]).setContextEntry(KeysBindings.isAddition, true))).toBe(true); + expect(filter3(BF.bindings([ + [ DF.variable('a'), DF.literal('b') ], + ]).setContextEntry(KeysBindings.isAddition, true))).toBe(false); + }); + + it('should create a filter that returns true if everything is deleted', async() => { + const filter = await actor.newHashFilter({}, [ DF.variable('a') ]); + expect(filter(BF.bindings([ + [ DF.variable('a'), DF.literal('a') ], + ]).setContextEntry(KeysBindings.isAddition, true))).toBe(true); + expect(filter(BF.bindings([ + [ DF.variable('a'), DF.literal('a') ], + ]).setContextEntry(KeysBindings.isAddition, true))).toBe(false); + expect(filter(BF.bindings([ + [ DF.variable('a'), DF.literal('a') ], + ]).setContextEntry(KeysBindings.isAddition, false))).toBe(false); + expect(filter(BF.bindings([ + [ DF.variable('a'), DF.literal('a') ], + ]).setContextEntry(KeysBindings.isAddition, false))).toBe(true); + expect(filter(BF.bindings([ + [ DF.variable('a'), DF.literal('a') ], + ]).setContextEntry(KeysBindings.isAddition, true))).toBe(true); + expect(filter(BF.bindings([ + [ DF.variable('a'), DF.literal('a') ], + ]).setContextEntry(KeysBindings.isAddition, true))).toBe(false); + + expect(filter(BF.bindings([ + [ DF.variable('a'), DF.literal('b') ], + ]).setContextEntry(KeysBindings.isAddition, true))).toBe(true); + expect(filter(BF.bindings([ + [ DF.variable('a'), DF.literal('b') ], + ]).setContextEntry(KeysBindings.isAddition, true))).toBe(false); + expect(filter(BF.bindings([ + [ DF.variable('a'), DF.literal('b') ], + ]).setContextEntry(KeysBindings.isAddition, true))).toBe(false); + expect(filter(BF.bindings([ + [ DF.variable('a'), DF.literal('b') ], + ]).setContextEntry(KeysBindings.isAddition, false))).toBe(false); + expect(filter(BF.bindings([ + [ DF.variable('a'), DF.literal('b') ], + ]).setContextEntry(KeysBindings.isAddition, false))).toBe(false); + expect(filter(BF.bindings([ + [ DF.variable('a'), DF.literal('b') ], + ]).setContextEntry(KeysBindings.isAddition, true))).toBe(false); + expect(filter(BF.bindings([ + [ DF.variable('a'), DF.literal('b') ], + ]).setContextEntry(KeysBindings.isAddition, false))).toBe(false); + expect(filter(BF.bindings([ + [ DF.variable('a'), DF.literal('b') ], + ]).setContextEntry(KeysBindings.isAddition, false))).toBe(true); + expect(filter(BF.bindings([ + [ DF.variable('a'), DF.literal('b') ], + ]).setContextEntry(KeysBindings.isAddition, true))).toBe(true); + }); + + it('should create a filter that returns false if too much is deleted', async() => { + const filter = await actor.newHashFilter({}, [ DF.variable('a') ]); + expect(filter(BF.bindings([ + [ DF.variable('a'), DF.literal('a') ], + ]).setContextEntry(KeysBindings.isAddition, true))).toBe(true); + expect(filter(BF.bindings([ + [ DF.variable('a'), DF.literal('a') ], + ]).setContextEntry(KeysBindings.isAddition, true))).toBe(false); + expect(filter(BF.bindings([ + [ DF.variable('a'), DF.literal('a') ], + ]).setContextEntry(KeysBindings.isAddition, false))).toBe(false); + expect(filter(BF.bindings([ + [ DF.variable('a'), DF.literal('a') ], + ]).setContextEntry(KeysBindings.isAddition, false))).toBe(true); + expect(filter(BF.bindings([ + [ DF.variable('a'), DF.literal('a') ], + ]).setContextEntry(KeysBindings.isAddition, false))).toBe(false); + expect(filter(BF.bindings([ + [ DF.variable('a'), DF.literal('a') ], + ]).setContextEntry(KeysBindings.isAddition, true))).toBe(true); + expect(filter(BF.bindings([ + [ DF.variable('a'), DF.literal('a') ], + ]).setContextEntry(KeysBindings.isAddition, true))).toBe(false); + + expect(filter(BF.bindings([ + [ DF.variable('a'), DF.literal('b') ], + ]).setContextEntry(KeysBindings.isAddition, true))).toBe(true); + expect(filter(BF.bindings([ + [ DF.variable('a'), DF.literal('b') ], + ]).setContextEntry(KeysBindings.isAddition, true))).toBe(false); + expect(filter(BF.bindings([ + [ DF.variable('a'), DF.literal('b') ], + ]).setContextEntry(KeysBindings.isAddition, true))).toBe(false); + expect(filter(BF.bindings([ + [ DF.variable('a'), DF.literal('b') ], + ]).setContextEntry(KeysBindings.isAddition, false))).toBe(false); + expect(filter(BF.bindings([ + [ DF.variable('a'), DF.literal('b') ], + ]).setContextEntry(KeysBindings.isAddition, false))).toBe(false); + expect(filter(BF.bindings([ + [ DF.variable('a'), DF.literal('b') ], + ]).setContextEntry(KeysBindings.isAddition, true))).toBe(false); + expect(filter(BF.bindings([ + [ DF.variable('a'), DF.literal('b') ], + ]).setContextEntry(KeysBindings.isAddition, false))).toBe(false); + expect(filter(BF.bindings([ + [ DF.variable('a'), DF.literal('b') ], + ]).setContextEntry(KeysBindings.isAddition, false))).toBe(true); + expect(filter(BF.bindings([ + [ DF.variable('a'), DF.literal('a') ], + ]).setContextEntry(KeysBindings.isAddition, false))).toBe(false); + expect(filter(BF.bindings([ + [ DF.variable('a'), DF.literal('b') ], + ]).setContextEntry(KeysBindings.isAddition, true))).toBe(true); + }); + }); + + describe('An ActorQueryOperationReducedHash instance', () => { + let actor: ActorQueryOperationReducedHash; + let mediatorHashBindings: MediatorHashBindings; + + beforeEach(() => { + mediatorHashBindings = createTestMediatorHashBindings(); + actor = new ActorQueryOperationReducedHash( + { name: 'actor', bus, mediatorQueryOperation, mediatorHashBindings }, + ); + }); + + it('should test on reduced', async() => { + const op: any = { operation: { type: 'reduced' }, context: new ActionContext() }; + await expect(actor.test(op)).resolves.toBeTruthy(); + }); + + it('should not test on non-reduced', async() => { + const op: any = { operation: { type: 'some-other-type' }, context: new ActionContext() }; + await expect(actor.test(op)).resolves + .toFailTest('Actor actor only supports reduced operations, but got some-other-type'); + }); + + it('should run with bindings', async() => { + const op: any = { operation: { type: 'reduced' }, context: new ActionContext() }; + const output = getSafeBindings(await actor.runOperation(op, undefined)); + await expect(output.metadata()).resolves.toEqual({ + cardinality: 5, + variables: [ + { + variable: DF.variable('a'), + isUndef: false, + }, + ], + }); + expect(output.type).toBe('bindings'); + await expect(output.bindingsStream).toEqualBindingsStream([ + BF.bindings([[ DF.variable('a'), DF.literal('1') ]]).setContextEntry(KeysBindings.isAddition, true), + BF.bindings([[ DF.variable('a'), DF.literal('2') ]]).setContextEntry(KeysBindings.isAddition, true), + BF.bindings([[ DF.variable('a'), DF.literal('3') ]]).setContextEntry(KeysBindings.isAddition, true), + BF.bindings([[ DF.variable('a'), DF.literal('2') ]]).setContextEntry(KeysBindings.isAddition, false), + ]); + }); + }); +});