Release 0.3.3

Co-authored-by: Michał Bartoszkiewicz <[email protected]> Co-authored-by: Jan Chorowski <[email protected]> Co-authored-by: Xavier Gendre <[email protected]> Co-authored-by: Adrian Kosowski <[email protected]> Co-authored-by: Jakub Kowalski <[email protected]> Co-authored-by: Sergey Kulik <[email protected]> Co-authored-by: Mateusz Lewandowski <[email protected]> Co-authored-by: Mohamed Malhou <[email protected]> Co-authored-by: Krzysztof Nowicki <[email protected]> Co-authored-by: Richard Pelgrim <[email protected]> Co-authored-by: Kamil Piechowiak <[email protected]> Co-authored-by: Paweł Podhajski <[email protected]> Co-authored-by: Olivier Ruas <[email protected]> Co-authored-by: Przemysław Uznański <[email protected]> Co-authored-by: Sebastian Włudzik <[email protected]> GitOrigin-RevId: 9d79adf05969ee3ec12ad775dd2f1064be656688
pathwaycom · Sep 14, 2023 · 1fbd828 · 1fbd828
1 parent 8812bc5
commit 1fbd828
Show file tree

Hide file tree

Showing 56 changed files with 2,354 additions and 971 deletions.
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -6,7 +6,6 @@ on:
       - 'v*.*.*'
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
 jobs:
   start-runner:
     name: Start self-hosted EC2 runner
@@ -17,7 +16,7 @@ jobs:
       ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
     steps:
       - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@v2
+        uses: aws-actions/configure-aws-credentials@v3
         with:
           aws-access-key-id: ${{ secrets.EC2_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.EC2_ACCESS_KEY_SECRET }}
@@ -73,6 +72,7 @@ jobs:
           manylinux: auto
           rustup-components: rust-std
           working-directory: .
+          before-script-linux: yum install -y perl-core
 
       - name: Build package macOS Apple silicon
         if: ${{ matrix.os == 'ec2-macOS'}}
@@ -224,7 +224,7 @@ jobs:
     timeout-minutes: 15
     steps:
       - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v2
+        uses: aws-actions/configure-aws-credentials@v3
         with:
           aws-access-key-id: ${{ secrets.ARTIFACT_AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.ARTIFACT_AWS_SECRET_ACCESS_KEY }}
@@ -293,7 +293,7 @@ jobs:
     timeout-minutes: 15
     steps:
       - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v2
+        uses: aws-actions/configure-aws-credentials@v3
         with:
           aws-access-key-id: ${{ secrets.ARTIFACT_AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.ARTIFACT_AWS_SECRET_ACCESS_KEY }}
@@ -328,7 +328,7 @@ jobs:
     if: ${{ always() }}
     steps:
       - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@v2
+        uses: aws-actions/configure-aws-credentials@v3
         with:
           aws-access-key-id: ${{ secrets.EC2_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.EC2_ACCESS_KEY_SECRET }}

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,6 +6,21 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 ## [Unreleased]
 
+## [0.3.3] - 2023-09-14
+
+### Added
+- Module `pathway.dt` to construct and manipulate DTypes.
+- New argument `keep_queries` in `pw.io.http.rest_connector`.
+
+### Changed
+- Internal representation of DTypes. Inputting types is compatible backwards.
+- Temporal functions now accept arguments of mixed types (ints and floats). For example, `pw.temporal.interval` can use ints while columns it interacts with are floats.
+- Single-element arrays are now treated as arrays, not as scalars.
+
+### Fixed
+- `to_string()` method on datetimes always prints 9 fractional digits.
+- `%f` format code in `strptime()` parses fractional part of a second correctly regardless of the number of digits.
+
 ## [0.3.2] - 2023-09-07
 
 ### Added

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "pathway"
-version = "0.3.2"
+version = "0.3.3"
 edition = "2021"
 publish = false
 rust-version = "1.71.0"
@@ -22,7 +22,7 @@ arcstr = { version = "1.1.5", default-features = false, features = ["serde", "st
 base32 = "0.4.0"
 bincode = "1.3.3"
 cfg-if = "1.0.0"
-chrono = { version = "0.4.28", features = ["std", "clock"], default-features = false }
+chrono = { version = "0.4.30", features = ["std", "clock"], default-features = false }
 chrono-tz = "0.8.3"
 crossbeam-channel = "0.5.8"
 csv = "1.2.2"
@@ -60,7 +60,7 @@ thiserror = "1.0.48"
 timely = { path = "./external/timely-dataflow/timely", features = ["bincode"] }
 tokio = "1.32.0"
 typed-arena = "2.0.2"
-xxhash-rust = { version = "0.8.6", features = ["xxh3"] }
+xxhash-rust = { version = "0.8.7", features = ["xxh3"] }
 
 [target.'cfg(target_os = "linux")'.dependencies]
 inotify = "0.10.2"

diff --git a/integration_tests/kafka/test_kafka.py b/integration_tests/kafka/test_kafka.py
@@ -5,31 +5,15 @@
 import pathlib
 import random
 
-import pandas as pd
 from utils import KafkaTestContext
 
 import pathway as pw
 from pathway.internals.parse_graph import G
-from pathway.tests.utils import wait_result_with_checker, write_lines
-
-
-def expect_csv_checker(expected, output_path, usecols=("k", "v"), index_col=("k")):
-    expected = (
-        pw.debug._markdown_to_pandas(expected)
-        .set_index(index_col, drop=False)
-        .sort_index()
-    )
-
-    def checker():
-        result = (
-            pd.read_csv(output_path, usecols=[*usecols, *index_col])
-            .convert_dtypes()
-            .set_index(index_col, drop=False)
-            .sort_index()
-        )
-        return expected.equals(result)
-
-    return checker
+from pathway.tests.utils import (
+    expect_csv_checker,
+    wait_result_with_checker,
+    write_lines,
+)
 
 
 def test_kafka_raw(tmp_path: pathlib.Path, kafka_context: KafkaTestContext):

diff --git a/integration_tests/webserver/test_rest_connector.py b/integration_tests/webserver/test_rest_connector.py
@@ -9,6 +9,7 @@
 import pathway as pw
 from pathway.tests.utils import (
     CsvLinesNumberChecker,
+    expect_csv_checker,
     wait_result_with_checker,
     xfail_on_darwin,
 )
@@ -126,3 +127,54 @@ def target():
     t = threading.Thread(target=target, daemon=True)
     t.start()
     assert wait_result_with_checker(CsvLinesNumberChecker(output_path, 4), 30)
+
+
+@xfail_on_darwin(reason="running pw.run from separate process not supported")
+@pytest.mark.xdist_group(name="streaming_tests")
+def test_server_keep_queries(tmp_path: pathlib.Path):
+    port = int(os.environ.get("PATHWAY_MONITORING_HTTP_PORT", "20000")) + 10003
+    output_path = tmp_path / "output.csv"
+
+    class InputSchema(pw.Schema):
+        k: int
+        v: int
+
+    def target():
+        time.sleep(5)
+        requests.post(
+            f"http://127.0.0.1:{port}/",
+            json={"k": 1, "v": 1},
+        ).raise_for_status()
+        requests.post(
+            f"http://127.0.0.1:{port}/",
+            json={"k": 1, "v": 2},
+        ).raise_for_status()
+
+    queries, response_writer = pw.io.http.rest_connector(
+        host="127.0.0.1", port=port, schema=InputSchema, keep_queries=True
+    )
+    response_writer(queries.select(query_id=queries.id, result=pw.this.v))
+
+    sum = queries.groupby(pw.this.k).reduce(
+        key=pw.this.k, sum=pw.reducers.sum(pw.this.v)
+    )
+
+    pw.io.csv.write(sum, output_path)
+
+    t = threading.Thread(target=target, daemon=True)
+    t.start()
+
+    assert wait_result_with_checker(
+        expect_csv_checker(
+            """
+            key | sum    | diff
+            1   | 1      | 1
+            1   | 1      | -1
+            1   | 3      | 1
+            """,
+            output_path,
+            usecols=["sum", "diff"],
+            index_col=["key"],
+        ),
+        10,
+    )
diff --git a/python/pathway/__init__.py b/python/pathway/__init__.py
@@ -6,6 +6,8 @@
 
 import os
 
+from pathway.internals.dtype import DATE_TIME_NAIVE, DATE_TIME_UTC, DURATION
+
 # flake8: noqa: E402
 
 if "PYTEST_CURRENT_TEST" not in os.environ:
@@ -16,15 +18,13 @@
     filterwarnings("ignore", category=BeartypeDecorHintPep585DeprecationWarning)
 
 
+import pathway.dt as dt
 import pathway.reducers as reducers
 from pathway import debug, demo, io
 from pathway.internals import (
     ClassArg,
     ColumnExpression,
     ColumnReference,
-    DateTimeNaive,
-    DateTimeUtc,
-    Duration,
     FilteredJoinResult,
     GroupedJoinResult,
     GroupedTable,
@@ -143,9 +143,9 @@
     "column_definition",
     "TableSlice",
     "demo",
-    "DateTimeNaive",
-    "DateTimeUtc",
-    "Duration",
+    "DATE_TIME_NAIVE",
+    "DATE_TIME_UTC",
+    "DURATION",
     "unwrap",
     "SchemaProperties",
 ]