added all the up-to-date stuff

maartyman · Jan 24, 2024 · a058852 · a058852
1 parent cf50cd0
commit a058852
Show file tree

Hide file tree

Showing 20 changed files with 192 additions and 39 deletions.
diff --git a/engines/query-sparql-incremental/package.json b/engines/query-sparql-incremental/package.json
@@ -215,6 +215,12 @@
     "@incremunica/incremental-rdf-streaming-store": "^1.1.0",
     "@incremunica/incremental-types": "^1.0.0"
   },
+  "overrides": {
+    "asynciterator": "git+https://github.com/maartyman/AsyncIterator.git#up-to-date-event"
+  },
+  "resolutions": {
+    "asynciterator": "git+https://github.com/maartyman/AsyncIterator.git#up-to-date-event"
+  },
   "scripts": {
     "build": "npm run build:ts",
     "build:ts": "node \"../../node_modules/typescript/bin/tsc\"",

diff --git a/package.json b/package.json
@@ -22,7 +22,7 @@
     "@typescript-eslint/parser": "^5.43.0",
     "abort-controller": "^3.0.0",
     "arrayify-stream": "^2.0.1",
-    "asynciterator": "^3.8.1",
+    "asynciterator": "git+https://github.com/maartyman/AsyncIterator.git#up-to-date-event",
     "babel-loader": "^9.1.0",
     "componentsjs-generator": "^3.1.0",
     "depcheck": "^1.4.3",
@@ -75,6 +75,12 @@
     "webpack-cli": "^5.0.0",
     "@comunica/jest": "^2.10.0"
   },
+  "overrides": {
+    "asynciterator": "git+https://github.com/maartyman/AsyncIterator.git#up-to-date-event"
+  },
+  "resolutions": {
+    "asynciterator": "git+https://github.com/maartyman/AsyncIterator.git#up-to-date-event"
+  },
   "pre-commit": [
     "build",
     "lint",

diff --git a/packages/actor-query-operation-incremental-filter/package.json b/packages/actor-query-operation-incremental-filter/package.json
@@ -38,9 +38,15 @@
     "@comunica/types": "^2.10.0",
     "@incremunica/hash-bindings": "^1.1.0",
     "@incremunica/incremental-bindings-factory": "^1.0.0",
-    "asynciterator": "^3.8.1",
+    "asynciterator": "git+https://github.com/maartyman/AsyncIterator.git#up-to-date-event",
     "sparqlalgebrajs": "^4.2.0"
   },
+  "overrides": {
+    "asynciterator": "git+https://github.com/maartyman/AsyncIterator.git#up-to-date-event"
+  },
+  "resolutions": {
+    "asynciterator": "git+https://github.com/maartyman/AsyncIterator.git#up-to-date-event"
+  },
   "scripts": {
     "build": "npm run build:ts && npm run build:components",
     "build:ts": "node \"../../node_modules/typescript/bin/tsc\"",

diff --git a/packages/actor-query-operation-incremental-quadpattern/package.json b/packages/actor-query-operation-incremental-quadpattern/package.json
@@ -39,11 +39,17 @@
     "@comunica/types": "^2.10.0",
     "@incremunica/incremental-bindings-factory": "^1.0.0",
     "@rdfjs/types": "*",
-    "asynciterator": "^3.8.1",
+    "asynciterator": "git+https://github.com/maartyman/AsyncIterator.git#up-to-date-event",
     "rdf-data-factory": "^1.1.1",
     "rdf-terms": "^1.9.1",
     "sparqlalgebrajs": "^4.0.5"
   },
+  "overrides": {
+    "asynciterator": "$asynciterator-local"
+  },
+  "resolutions": {
+    "asynciterator": "$asynciterator-local"
+  },
   "scripts": {
     "build": "npm run build:ts && npm run build:components",
     "build:ts": "node \"../../node_modules/typescript/bin/tsc\"",

diff --git a/packages/actor-rdf-join-inner-incremental-computational-multi-bind/package.json b/packages/actor-rdf-join-inner-incremental-computational-multi-bind/package.json
@@ -42,9 +42,15 @@
     "@incremunica/hash-bindings": "^1.1.0",
     "@incremunica/incremental-bindings-factory": "^1.0.0",
     "@incremunica/context-entries": "^1.0.0",
-    "asynciterator": "^3.8.1",
+    "asynciterator": "git+https://github.com/maartyman/AsyncIterator.git#up-to-date-event",
     "sparqlalgebrajs": "^4.0.5"
   },
+  "overrides": {
+    "asynciterator": "$asynciterator-local"
+  },
+  "resolutions": {
+    "**/asynciterator": "$asynciterator-local"
+  },
   "scripts": {
     "build": "npm run build:ts && npm run build:components",
     "build:ts": "node \"../../node_modules/typescript/bin/tsc\"",

diff --git a/packages/actor-rdf-join-inner-incremental-full-hash/lib/IncrementalFullHashJoin.ts b/packages/actor-rdf-join-inner-incremental-full-hash/lib/IncrementalFullHashJoin.ts
@@ -108,6 +108,9 @@ export class IncrementalFullHashJoin extends IncrementalInnerJoin {
       }
 
       this.readable = false;
+      if (this.leftIterator.upToDate && this.rightIterator.upToDate) {
+        this.upToDate = true;
+      }
       return null;
     }
   }

diff --git a/packages/actor-rdf-join-inner-incremental-memory-multi-bind/package.json b/packages/actor-rdf-join-inner-incremental-memory-multi-bind/package.json
@@ -40,9 +40,15 @@
     "@comunica/types": "^2.10.0",
     "@incremunica/hash-bindings": "^1.1.0",
     "@incremunica/incremental-bindings-factory": "^1.0.0",
-    "asynciterator": "^3.8.1",
+    "asynciterator": "git+https://github.com/maartyman/AsyncIterator.git#up-to-date-event",
     "sparqlalgebrajs": "^4.0.5"
   },
+  "overrides": {
+    "asynciterator": "$asynciterator-local"
+  },
+  "resolutions": {
+    "asynciterator": "$asynciterator-local"
+  },
   "scripts": {
     "build": "npm run build:ts && npm run build:components",
     "build:ts": "node \"../../node_modules/typescript/bin/tsc\"",

diff --git a/packages/actor-rdf-join-inner-incremental-multi-delta-query/lib/DeltaQueryIterator.ts b/packages/actor-rdf-join-inner-incremental-multi-delta-query/lib/DeltaQueryIterator.ts
@@ -33,25 +33,23 @@ export class DeltaQueryIterator extends AsyncIterator<Bindings> {
       .set(KeysDeltaQueryJoin.fromDeltaQuery, true);
     this.mediatorQueryOperation = mediatorQueryOperation;
 
-    this.readable = false;
-    this.setProperty('up-to-date', true);
-
     for (const entry of this.entries) {
       if (entry.output.bindingsStream.readable) {
         this.readable = true;
-        this.setProperty('up-to-date', false);
       }
 
       entry.output.bindingsStream.on('end', () => {
         this.readable = true;
-        this.setProperty('up-to-date', false);
+      });
+
+      entry.output.bindingsStream.on('up-to-date', () => {
+        this.readable = true;
       });
 
       entry.output.bindingsStream.on('error', error => this.destroy(error));
 
       entry.output.bindingsStream.on('readable', () => {
         this.readable = true;
-        this.setProperty('up-to-date', false);
       });
     }
   }
@@ -70,18 +68,19 @@ export class DeltaQueryIterator extends AsyncIterator<Bindings> {
     for (const entry of this.entries) {
       if (!entry.output.bindingsStream.done) {
         this.getNewBindingsStream();
+        for (const entry of this.entries) {
+          if (!entry.output.bindingsStream.upToDate)
+            return null;
+        }
         if (
           !this.pending &&
           (this.currentSource === undefined || this.currentSource.done)
         ) {
-          this.setProperty('up-to-date', true);
-          this.emit('up-to-date');
+          this.upToDate = true;
         }
         return null;
       }
     }
-    this.setProperty('up-to-date', true);
-    this.emit('up-to-date');
     this.close();
     return null;
   }
@@ -173,7 +172,6 @@ export class DeltaQueryIterator extends AsyncIterator<Bindings> {
 
         bindingsStream.on('readable', () => {
           this.readable = true;
-          this.setProperty('up-to-date', false);
         });
 
         this.currentSource = <BindingsStream><unknown>bindingsStream;

diff --git a/packages/actor-rdf-join-inner-incremental-multi-delta-query/package.json b/packages/actor-rdf-join-inner-incremental-multi-delta-query/package.json
@@ -41,10 +41,16 @@
     "@comunica/types": "^2.10.0",
     "@incremunica/incremental-types": "^1.0.0",
     "@incremunica/context-entries": "^1.0.0",
-    "asynciterator": "^3.8.1",
+    "asynciterator": "git+https://github.com/maartyman/AsyncIterator.git#up-to-date-event",
     "n3": "^1.16.4",
     "sparqlalgebrajs": "^4.1.0"
   },
+  "overrides": {
+    "asynciterator": "$asynciterator-local"
+  },
+  "resolutions": {
+    "asynciterator": "$asynciterator-local"
+  },
   "scripts": {
     "build": "npm run build:ts && npm run build:components",
     "build:ts": "node \"../../node_modules/typescript/bin/tsc\"",

diff --git a/packages/actor-rdf-join-inner-incremental-nestedloop/lib/IncrementalNestedLoopJoin.ts b/packages/actor-rdf-join-inner-incremental-nestedloop/lib/IncrementalNestedLoopJoin.ts
@@ -90,6 +90,9 @@ export class IncrementalNestedLoopJoin extends IncrementalInnerJoin {
       }
 
       this.readable = false;
+      if (this.leftIterator.upToDate && this.rightIterator.upToDate) {
+        this.upToDate = true;
+      }
       return null;
     }
   }

diff --git a/packages/actor-rdf-join-inner-incremental-partial-hash/lib/IncrementalPartialHashJoin.ts b/packages/actor-rdf-join-inner-incremental-partial-hash/lib/IncrementalPartialHashJoin.ts
@@ -116,6 +116,9 @@ export class IncrementalPartialHashJoin extends IncrementalInnerJoin {
       }
 
       this.readable = false;
+      if (this.leftIterator.upToDate && this.rightIterator.upToDate) {
+        this.upToDate = true;
+      }
       return null;
     }
   }

diff --git a/packages/actor-rdf-resolve-quad-pattern-incremental-federated/package.json b/packages/actor-rdf-resolve-quad-pattern-incremental-federated/package.json
@@ -40,11 +40,17 @@
     "@comunica/types": "^2.10.0",
     "@comunica/metadata": "^2.10.0",
     "@rdfjs/types": "*",
-    "asynciterator": "^3.8.1",
+    "asynciterator": "git+https://github.com/maartyman/AsyncIterator.git#up-to-date-event",
     "rdf-data-factory": "^1.1.1",
     "rdf-terms": "^1.9.1",
     "sparqlalgebrajs": "^4.0.5"
   },
+  "overrides": {
+    "asynciterator": "$asynciterator-local"
+  },
+  "resolutions": {
+    "asynciterator": "$asynciterator-local"
+  },
   "scripts": {
     "build": "npm run build:ts && npm run build:components",
     "build:ts": "node \"../../node_modules/typescript/bin/tsc\"",

diff --git a/...ges/actor-rdf-resolve-quad-pattern-rdfjs-streaming-source/lib/RdfJsQuadStreamingSource.ts b/...ges/actor-rdf-resolve-quad-pattern-rdfjs-streaming-source/lib/RdfJsQuadStreamingSource.ts
@@ -53,19 +53,32 @@ export class RdfJsQuadStreamingSource implements IQuadSource {
 
     const it = wrapAsyncIterator<Quad>(rawStream, { autoStart: false });
 
-    // Set up-to-date property
-    it.setProperty('up-to-date', true);
-    if (this.context) {
-      const guardEvents = this.context.get<IGuardEvents>(KeysGuard.events);
-      if (guardEvents) {
-        guardEvents.on('modified', () => {
-          it.setProperty('up-to-date', false);
-        });
-        guardEvents.on('up-to-date', () => {
-          it.setProperty('up-to-date', true);
-        });
-      }
+    let upToDate = this.store.isHalted();
+    // Set up-to-date event listener
+    const guardEvents = this.context?.get<IGuardEvents>(KeysGuard.events);
+    if (guardEvents) {
+      guardEvents.on('up-to-date', () => {
+        upToDate = true;
+      });
+      guardEvents.on('modified', () => {
+        upToDate = false;
+      });
     }
+    else {
+      this.store.on('resume', () => {
+        upToDate = false;
+      })
+      this.store.on('halt', () => {
+        upToDate = true;
+      });
+      this.store.on('flush', () => {
+        upToDate = true;
+      });
+    }
+    it.on("unreadable", () => {
+      if (upToDate)
+        (it as any).upToDate = true;
+    });
 
     // In case this setMetadata can cause errors, catch the error and emit it on the iterator (it). For now ignore it!
     /* eslint-disable-next-line @typescript-eslint/no-floating-promises */

diff --git a/packages/actor-rdf-resolve-quad-pattern-rdfjs-streaming-source/package.json b/packages/actor-rdf-resolve-quad-pattern-rdfjs-streaming-source/package.json
@@ -40,7 +40,13 @@
     "@incremunica/incremental-types": "^1.0.0",
     "@incremunica/context-entries": "^1.0.0",
     "@rdfjs/types": "*",
-    "asynciterator": "^3.8.1"
+    "asynciterator": "git+https://github.com/maartyman/AsyncIterator.git#up-to-date-event"
+  },
+  "overrides": {
+    "asynciterator": "git+https://github.com/maartyman/AsyncIterator.git#up-to-date-event"
+  },
+  "resolutions": {
+    "asynciterator": "git+https://github.com/maartyman/AsyncIterator.git#up-to-date-event"
   },
   "scripts": {
     "build": "npm run build:ts && npm run build:components",

diff --git a/packages/incremental-inner-join/lib/IncrementalInnerJoin.ts b/packages/incremental-inner-join/lib/IncrementalInnerJoin.ts
@@ -1,5 +1,5 @@
 import type { Bindings, BindingsStream } from '@incremunica/incremental-types';
-import { AsyncIterator } from 'asynciterator';
+import {AsyncIterator} from 'asynciterator';
 
 export abstract class IncrementalInnerJoin extends AsyncIterator<Bindings> {
   protected leftIterator: BindingsStream;
@@ -32,6 +32,15 @@ export abstract class IncrementalInnerJoin extends AsyncIterator<Bindings> {
       this.readable = true;
     });
 
+    this.leftIterator.on('up-to-date', () => {
+      if (this.rightIterator.upToDate)
+        this.readable = true;
+    });
+    this.rightIterator.on('up-to-date', () => {
+      if (this.leftIterator.upToDate)
+        this.readable = true;
+    });
+
     this.leftIterator.on('end', () => {
       if (!this.hasResults()) {
         this._end();

diff --git a/packages/incremental-inner-join/package.json b/packages/incremental-inner-join/package.json
@@ -33,7 +33,13 @@
   ],
   "dependencies": {
     "@incremunica/incremental-types": "^1.0.0",
-    "asynciterator": "^3.8.1"
+    "asynciterator": "git+https://github.com/maartyman/AsyncIterator.git#up-to-date-event"
+  },
+  "overrides": {
+    "asynciterator": "$asynciterator-local"
+  },
+  "resolutions": {
+    "asynciterator": "$asynciterator-local"
   },
   "scripts": {
     "build": "npm run build:ts && npm run build:components",

diff --git a/packages/incremental-rdf-streaming-store/lib/LinkedList.ts b/packages/incremental-rdf-streaming-store/lib/LinkedList.ts
@@ -0,0 +1,44 @@
+interface LinkedNode<V> {
+  value: V;
+  next: LinkedNode<V> | null;
+}
+
+/**
+ * A list with O(1) push and shift operations.
+ */
+export class LinkedList<V> {
+  private _length: number = 0;
+  private _head: LinkedNode<V> | null = null;
+  private _tail: LinkedNode<V> | null = null;
+
+  get length() { return this._length; }
+  get first()  { return this._head?.value; }
+  get last()   { return this._tail?.value; }
+  get empty()  { return this._head === null; }
+
+  push(value: V) {
+    const node = { value, next: null } as LinkedNode<V>;
+    if (this._tail === null)
+      this._head = this._tail = node;
+    else
+      this._tail.next = this._tail = node;
+    this._length++;
+  }
+
+  shift(): V | undefined {
+    if (this._head === null)
+      return undefined;
+
+    const { value, next } = this._head;
+    this._head = next;
+    if (next === null)
+      this._tail = null;
+    this._length--;
+    return value;
+  }
+
+  clear() {
+    this._length = 0;
+    this._head = this._tail = null;
+  }
+}