Release 0.6.0

Co-authored-by: Michał Bartoszkiewicz <[email protected]>
Co-authored-by: Jan Chorowski <[email protected]>
Co-authored-by: Xavier Gendre <[email protected]>
Co-authored-by: Adrian Kosowski <[email protected]>
Co-authored-by: Jakub Kowalski <[email protected]>
Co-authored-by: Sergey Kulik <[email protected]>
Co-authored-by: Mateusz Lewandowski <[email protected]>
Co-authored-by: Mohamed Malhou <[email protected]>
Co-authored-by: Krzysztof Nowicki <[email protected]>
Co-authored-by: Richard Pelgrim <[email protected]>
Co-authored-by: Kamil Piechowiak <[email protected]>
Co-authored-by: Paweł Podhajski <[email protected]>
Co-authored-by: Olivier Ruas <[email protected]>
Co-authored-by: Przemysław Uznański <[email protected]>
Co-authored-by: Sebastian Włudzik <[email protected]>
GitOrigin-RevId: 26e4401be1779dd9af693e523da2768a5e190df2

.github/workflows/release.yml

@@ -49,7 +49,7 @@ jobs:
           - ec2-macOS
           - ${{ needs.start-runner.outputs.label }}
     runs-on: ${{ matrix.os }}
-    timeout-minutes: 30
+    timeout-minutes: 45
     steps:
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v4
@@ -147,7 +147,7 @@ jobs:
       matrix:
         python-version: ["3.10", "3.11"]
     runs-on: ubuntu-22.04
-    timeout-minutes: 15
+    timeout-minutes: 45
     steps:
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v4
@@ -188,7 +188,7 @@ jobs:
           - ec2-macOS
           - ${{ needs.start-runner.outputs.label }}
     runs-on: ${{ matrix.os }}
-    timeout-minutes: 15
+    timeout-minutes: 45
     steps:
       - name: Create dir for wheels
         run: |

CHANGELOG.md

@@ -6,6 +6,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 ## [Unreleased]
 
+## [0.6.0] - 2023-11-10
+
+### Added
+- A flag `with_metadata` for the filesystem-based connectors to attach the source file metadata to the table entries.
+- Methods `pw.debug.table_from_list_of_batches` and `pw.debug.table_from_list_of_batches_by_workers` for creating tables with defined data being inserted over time.
+
+### Changed
+- **BREAKING**: `pw.debug.table_from_pandas` and `pw.debug.table_from_markdown` now will create tables in the streaming mode, instead of static, if given table definition contains `_time` column.
+- **BREAKING**: Renamed the parameter `keep_queries` in `pw.io.http.rest_connector` to `delete_queries` with the opposite meaning. It changes the default behavior - it was `keep_queries=False`, now it is `delete_queries=False`.
+
 ## [0.5.3] - 2023-10-27
 
 ### Added

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "pathway"
-version = "0.5.3"
+version = "0.6.0"
 edition = "2021"
 publish = false
 rust-version = "1.72.0"
@@ -13,7 +13,7 @@ crate-type = ["cdylib", "lib"]
 [dev-dependencies]
 assert_matches = "1.5.0"
 eyre = "0.6.8"
-tempfile = "3.8.0"
+tempfile = "3.8.1"
 
 [dependencies]
 arc-swap = "1.6.0"
@@ -30,22 +30,22 @@ csv = "1.3.0"
 derivative = "2.2.0"
 differential-dataflow = { path = "./external/differential-dataflow" }
 elasticsearch = "8.5.0-alpha.1"
-futures = "0.3.28"
+futures = "0.3.29"
 glob = "0.3.1"
 hyper = { version = "0.14", features = ["server"] }
 id-arena = "2.2.1"
 itertools = "0.11.0"
 jemallocator = { version = "0.5.4", features = ["stats", "disable_initial_exec_tls"] }
 log = { version = "0.4.20", features = ["std"] }
 ndarray = { version = "0.15.6", features = ["serde"] }
-nix = { version = "0.27.1", features = ["fs"] }
+nix = { version = "0.27.1", features = ["fs", "user"] }
 num-integer = "0.1.45"
 numpy = "0.19.0"
 once_cell = "1.18.0"
 ordered-float = { version = "4.1.1", features = ["serde"] }
 pipe = "0.4.0"
 postgres = { version = "0.19.7", features = ["with-chrono-0_4", "with-serde_json-1"] }
-prometheus-client = "0.21.2"
+prometheus-client = "0.22.0"
 pyo3 = { version = "0.19.2", features = ["abi3-py310", "multiple-pymethods"] }
 pyo3-asyncio = "0.19.0"
 pyo3-log = "0.9.0"
@@ -54,7 +54,7 @@ rdkafka = { version = "0.34.0", features = ["ssl-vendored", "cmake-build", "zstd
 rust-s3 = { version = "0.33.0", features = ["sync-native-tls-vendored", "sync-native-tls", "fail-on-err"], default-features = false }
 scopeguard = "1.2.0"
 send_wrapper = "0.6.0"
-serde = { version = "1.0.189", features = ["derive", "rc"] }
+serde = { version = "1.0.190", features = ["derive", "rc"] }
 serde_json = "1.0"
 serde_with = "3.4.0"
 smallvec = { version = "1.11.1", features = ["union", "const_generics"] }

integration_tests/kafka/test_backfilling.py

@@ -138,9 +138,9 @@ def run_backfilling_program(
 def test_backfilling_fs_storage(
     tmp_path: pathlib.Path, kafka_context: KafkaTestContext
 ):
-    fs_persistence_config = pw.io.PersistenceConfig.single_backend(
-        pw.io.PersistentStorageBackend.filesystem(tmp_path / "PStorage"),
-        refresh_duration_ms=5000,
+    fs_persistence_config = pw.persistence.Config.simple_config(
+        pw.persistence.Backend.filesystem(tmp_path / "PStorage"),
+        snapshot_interval_ms=5000,
     )
     run_backfilling_program(fs_persistence_config, tmp_path, kafka_context)
 
@@ -154,11 +154,11 @@ def test_backfilling_s3_storage(
             time.time()
         )
     )
-    s3_persistence_config = pw.io.PersistenceConfig.single_backend(
-        pw.io.PersistentStorageBackend.s3(
+    s3_persistence_config = pw.persistence.Config.simple_config(
+        pw.persistence.Backend.s3(
             root_path=pstorage_s3_path,
             bucket_settings=get_aws_s3_settings(),
         ),
-        refresh_duration_ms=2000,
+        snapshot_interval_ms=2000,
     )
     run_backfilling_program(s3_persistence_config, tmp_path, kafka_context)