Merge pull request #510 from mabel-dev/v0.4.0

V0.4.0

docs/Contributor Guide/01 Guide.md

@@ -2,7 +2,7 @@
 
 **Welcome to the Opteryx Contributor Guide.**
 
-In this section you will find information to help you to bring your unique skills and experience and join us in building 
+In this section you will find information to help you to bring your unique skills and experience and join us in building Opteryx.
 
 Opteryx is primarily written in Python, but you don't need to be a Python developer to contribute. All contributions, [bug reports](https://github.com/mabel-dev/opteryx/issues/new/choose), documentation improvements, and [ideas](https://github.com/mabel-dev/opteryx/discussions) are welcome.
 

docs/Contributor Guide/95 Project Structure.md

@@ -9,8 +9,9 @@ Opteryx's repository folder structure is described below:
  ├── connectors/         <- modules to connect to data sources
  ├── functions/          <- modules to execute functions within SQL statements
  ├── managers/           <- libraries responsible for key functional units
- │   ├── cache/          <- modules implementing the caching mechanism
+mechanism
  │   ├── expression/     <- modules implementing expression evaluation
+│   ├── kvstore/         <- modules implementing interfacing with KV Stores (internal usage) 
  │   ├── process/        <- modules implementing process management
  │   ├── query/           
  │   │   └── planner/    <- modules implementing query planning 

docs/Deployment/Internals/Query Engine.md

@@ -2,7 +2,6 @@
 
 > If you are interested in how databases work, I recommend the resources from [The CMU Database Group](https://db.cs.cmu.edu/) and the collection of resources at [Awesome Database Learning](https://github.com/pingcap/awesome-database-learning).
 
-
 The Opteryx query engine has the following key components and general process:
 
 1) The Parser & Lexer, which recieves the user SQL and builds an Abstract Syntax Tree (AST).

docs/Features/Command Line.md

@@ -0,0 +1,12 @@
+# Command Line Interface
+
+Opteryx supports being used on the command line to process files using SQL to filter and process files.
+
+The full SQL feature set (e.g. time travel, caching) is available using the CLI.
+
+The CLI can be used to be used to save the results to output files for onward processing:
+
+- direct to the console
+- CSV
+- PARQUET
+- JSONL

docs/Release Notes/Change Log.md

@@ -4,7 +4,7 @@ All notable changes to this project will be documented in this file, where appro
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [Unreleased]
+## [0.4.0] - 2022-09-12
 
 **Added**
 

opteryx/config.py

@@ -48,4 +48,8 @@
 MAX_SIZE_SINGLE_CACHE_ITEM: int = _config.get("MAX_SIZE_SINGLE_CACHE_ITEM", 1024 * 1024)
 # Approximate Page Size
 PAGE_SIZE: int = _config.get("PAGE_SIZE", 64 * 1024 * 1024)
+
+
+# The number of metadata cache records to hold
+LOCAL_METADATA_CACHE: int = int(_config.get("LOCAL_METADATA_CACHE", 512))
 # fmt:on

opteryx/connection.py

@@ -24,7 +24,7 @@
 from pyarrow import Table
 
 from opteryx.exceptions import CursorInvalidStateError, ProgrammingError, SqlError
-from opteryx.managers.cache import BaseBufferCache
+from opteryx.managers.kvstores import BaseKeyValueStore
 from opteryx.models import QueryStatistics
 from opteryx.utils import arrow
 
@@ -39,7 +39,7 @@ class Connection:
     def __init__(
         self,
         *,
-        cache: Optional[BaseBufferCache] = None,
+        cache: Optional[BaseKeyValueStore] = None,
         **kwargs,
     ):
         self._results = None

opteryx/managers/cache/__init__.py → opteryx/managers/kvstores/__init__.py

@@ -10,6 +10,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from .base_buffer_cache import BaseBufferCache
-from .memcached_cache import MemcachedCache
-from .memory_cache import InMemoryCache
+from .base_kv_store import BaseKeyValueStore
+
+from .kv_firestore import FireStoreKVStore
+from .kv_memory import InMemoryKVStore
+from .kv_local import LocalKVStore
+from .kv_memcached import MemcachedKVStore
+from .kv_mongodb import MongoDbKVStore

opteryx/managers/cache/base_buffer_cache.py → opteryx/managers/kvstores/base_kv_store.py

@@ -10,16 +10,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """
-This is a Base class for a caching layer.
+This is a Base class for KV Value Storage adapter.
 
-It's used by injecting an instantiated object into the Reader. If the object in None
-we skip any caching, if it's set up, we use it as an aside cache.
+This is used by the metadata store and in-memory buffer cache. 
 """
 import abc
 from typing import Optional
 
 
-class BaseBufferCache(abc.ABC):
+class BaseKeyValueStore(abc.ABC):
     """
     Base class for cache objects
     """

opteryx/managers/kvstores/kv_firestore.py

@@ -0,0 +1,7 @@
+# TODO: rocks db
+
+from opteryx.managers.kvstores import BaseKeyValueStore
+
+
+class FireStoreKVStore(BaseKeyValueStore):
+    pass

opteryx/managers/kvstores/kv_local.py

@@ -0,0 +1,7 @@
+# TODO: rocks db
+
+from opteryx.managers.kvstores import BaseKeyValueStore
+
+
+class LocalKVStore(BaseKeyValueStore):
+    pass

opteryx/managers/cache/memcached_cache.py → opteryx/managers/kvstores/kv_memcached.py

@@ -16,7 +16,7 @@
 import io
 import os
 from opteryx.exceptions import MissingDependencyError
-from opteryx.managers.cache import BaseBufferCache
+from opteryx.managers.kvstores import BaseKeyValueStore
 
 try:
     # added 3.9
@@ -65,7 +65,7 @@ def _memcached_server(**kwargs):
     )
 
 
-class MemcachedCache(BaseBufferCache):
+class MemcachedKVStore(BaseKeyValueStore):
     """
     Cache object
     """

opteryx/managers/cache/memory_cache.py → opteryx/managers/kvstores/kv_memory.py

@@ -21,10 +21,10 @@
 
 import io
 
-from opteryx.managers.cache import BaseBufferCache
+from opteryx.managers.kvstores import BaseKeyValueStore
 
 
-class InMemoryCache(BaseBufferCache):
+class InMemoryKVStore(BaseKeyValueStore):
     def __init__(self, **kwargs):
         """
         Parameters:

opteryx/managers/kvstores/kv_mongodb.py

@@ -0,0 +1,7 @@
+# TODO: rocks db
+
+from opteryx.managers.kvstores import BaseKeyValueStore
+
+
+class MongoDbKVStore(BaseKeyValueStore):
+    pass