Get pytest working in CI; fix py tests

.github/workflows/rust-python.yml

@@ -20,9 +20,10 @@ jobs:
       run: cargo build --verbose
     - name: Run Rust tests
       run: cargo test --verbose
-    - name: Install Python tokengrams
-      run: |
-        python -m pip install --upgrade pip
-        pip install ./tokengrams[dev]
     - name: Run Python tests
-      run: pytest --verbose
+      working-directory: ./tokengrams
+      run: |   
+        python -m pip install --upgrade pip
+        pip install pytest hypothesis numpy maturin
+        maturin build
+        pytest --verbose

tokengrams/tests/test_gram_index.py

@@ -28,25 +28,25 @@ def check_gram_index(index: InMemoryIndex | MemmapIndex, tokens: list[int]):
 )
 def test_gram_index(tokens: list[int]):
     # Construct index
-    index = InMemoryIndex(tokens)
+    index = InMemoryIndex(tokens, False)
     check_gram_index(index, tokens)
 
     # Save to disk and check that we can load it back
     with NamedTemporaryFile() as f:
         memmap = np.memmap(f, dtype=np.uint16, mode="w+", shape=(len(tokens),))
         memmap[:] = tokens
 
-        index = InMemoryIndex.from_token_file(f.name, None)
+        index = InMemoryIndex.from_token_file(f.name, False, None)
         check_gram_index(index, tokens)
 
         with NamedTemporaryFile() as idx:
-            index = MemmapIndex.build(f.name, idx.name)
+            index = MemmapIndex.build(f.name, idx.name, False)
             check_gram_index(index, tokens)
 
             index = MemmapIndex(f.name, idx.name)
             check_gram_index(index, tokens)
 
         # Now check limited token loading
         for limit in range(1, len(tokens) + 1):
-            index = InMemoryIndex.from_token_file(f.name, limit)
+            index = InMemoryIndex.from_token_file(f.name, False, limit)
             check_gram_index(index, tokens[:limit])

tokengrams/tokengrams.pyi

@@ -1,11 +1,11 @@
 class InMemoryIndex:
     """An n-gram index."""
 
-    def __init__(self, tokens: list[int]) -> None:
+    def __init__(self, tokens: list[int], verbose: bool) -> None:
         ...
 
     @staticmethod
-    def from_token_file(path: str, token_limit: int | None, verbose: bool) -> "InMemoryIndex":
+    def from_token_file(path: str, verbose: bool, token_limit: int | None) -> "InMemoryIndex":
         """Construct a `InMemoryIndex` from a file containing raw little-endian tokens."""
 
     def contains(self, query: list[int]) -> bool: