From 1e6c31bde8f9b752c7148ac89bbe09913a6cfb25 Mon Sep 17 00:00:00 2001 From: Benjamin Thomas Schwertfeger Date: Mon, 25 Nov 2024 21:22:29 +0100 Subject: [PATCH] revert stuff and fix the error --- .github/workflows/_test_futures_public.yaml | 4 +- .github/workflows/_test_spot_private.yaml | 18 +- .github/workflows/cicd.yaml | 338 ++++++++++---------- kraken/futures/websocket/__init__.py | 7 +- kraken/spot/websocket/connectors.py | 10 +- tests/spot/test_spot_orderbook.py | 6 +- 6 files changed, 195 insertions(+), 188 deletions(-) diff --git a/.github/workflows/_test_futures_public.yaml b/.github/workflows/_test_futures_public.yaml index c7956e5..3b83782 100644 --- a/.github/workflows/_test_futures_public.yaml +++ b/.github/workflows/_test_futures_public.yaml @@ -69,7 +69,7 @@ jobs: uv pip install ".[test]" - name: Testing Futures REST endpoints - run: pytest -vv -m " not futures_auth and not futures_websocket" tests + run: pytest -vv -m "futures and not futures_auth and not futures_websocket" tests - name: Testing Futures websocket endpoints - run: pytest -vv -m "not futures_auth and futures_websocket" tests + run: pytest -vv -m "futures and not futures_auth and futures_websocket" tests diff --git a/.github/workflows/_test_spot_private.yaml b/.github/workflows/_test_spot_private.yaml index 07de15c..0a81f5d 100644 --- a/.github/workflows/_test_spot_private.yaml +++ b/.github/workflows/_test_spot_private.yaml @@ -35,9 +35,9 @@ jobs: name: Test ${{ inputs.os }} ${{ inputs.python-version }} runs-on: ${{ inputs.os }} timeout-minutes: 10 - # concurrency: # FIXME - # group: test_spot_private - # cancel-in-progress: true + concurrency: + group: test_spot_private + cancel-in-progress: true steps: - name: Harden Runner uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2 @@ -80,14 +80,14 @@ jobs: echo "$env:GITHUB_WORKSPACE\.venv\Scripts" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append uv pip install ".[test]" - # - name: Testing Spot REST endpoints # FIXME - # env: - # SPOT_API_KEY: ${{ secrets.SPOT_API_KEY }} - # SPOT_SECRET_KEY: ${{ secrets.SPOT_SECRET_KEY }} - # run: pytest -vv -m "spot_auth and not spot_websocket" tests + - name: Testing Spot REST endpoints + env: + SPOT_API_KEY: ${{ secrets.SPOT_API_KEY }} + SPOT_SECRET_KEY: ${{ secrets.SPOT_SECRET_KEY }} + run: pytest -vv -m "spot and spot_auth and not spot_websocket" tests - name: Testing Spot websocket client env: SPOT_API_KEY: ${{ secrets.SPOT_API_KEY }} SPOT_SECRET_KEY: ${{ secrets.SPOT_SECRET_KEY }} - run: pytest -vv --log-cli-level=DEBUG -m wip tests # "spot_auth and spot_websocket" # FIXME + run: pytest -vv "spot and spot_auth and spot_websocket" tests diff --git a/.github/workflows/cicd.yaml b/.github/workflows/cicd.yaml index ae3ed64..d1ce026 100644 --- a/.github/workflows/cicd.yaml +++ b/.github/workflows/cicd.yaml @@ -61,67 +61,67 @@ concurrency: cancel-in-progress: true jobs: - # ## =========================================================================== - # ## Checks the code logic, style and more - # ## - # Pre-Commit: - # uses: ./.github/workflows/_pre_commit.yaml + ## =========================================================================== + ## Checks the code logic, style and more + ## + Pre-Commit: + uses: ./.github/workflows/_pre_commit.yaml - # ## =========================================================================== - # ## Discover vulnerabilities - # ## - # CodeQL: - # uses: ./.github/workflows/_codeql.yaml + ## =========================================================================== + ## Discover vulnerabilities + ## + CodeQL: + uses: ./.github/workflows/_codeql.yaml - # ## =========================================================================== - # ## Builds the package on multiple OS - # ## - # Build: - # needs: [Pre-Commit] - # uses: ./.github/workflows/_build.yaml - # strategy: - # fail-fast: true - # matrix: - # os: [ubuntu-latest, macos-latest, windows-latest] - # python-version: ["3.11", "3.12"] - # with: - # os: ${{ matrix.os }} - # python-version: ${{ matrix.python-version }} + ## =========================================================================== + ## Builds the package on multiple OS + ## + Build: + needs: [Pre-Commit] + uses: ./.github/workflows/_build.yaml + strategy: + fail-fast: true + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + python-version: ["3.11", "3.12"] + with: + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} - # ## =========================================================================== - # ## Build the documentation - # ## - # Build-Doc: - # needs: [Pre-Commit] - # uses: ./.github/workflows/_build_doc.yaml - # with: - # os: "ubuntu-latest" - # python-version: "3.11" + ## =========================================================================== + ## Build the documentation + ## + Build-Doc: + needs: [Pre-Commit] + uses: ./.github/workflows/_build_doc.yaml + with: + os: "ubuntu-latest" + python-version: "3.11" - # ## =========================================================================== - # ## Run the Spot tests - # ## - # ## (public endpoints) - # ## - # Test-Spot-Public: - # needs: [Pre-Commit] - # uses: ./.github/workflows/_test_spot_public.yaml - # strategy: - # fail-fast: false - # matrix: - # os: [ubuntu-latest, windows-latest] - # python-version: ["3.11", "3.12"] - # with: - # os: ${{ matrix.os }} - # python-version: ${{ matrix.python-version }} - # ## - # ## (private endpoints) + ## =========================================================================== + ## Run the Spot tests + ## + ## (public endpoints) + ## + Test-Spot-Public: + needs: [Pre-Commit] + uses: ./.github/workflows/_test_spot_public.yaml + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + python-version: ["3.11", "3.12"] + with: + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + ## + ## (private endpoints) Test-Spot-Private: if: success() && github.actor != 'dependabot[bot]' - # needs: [Pre-Commit] + needs: [Pre-Commit] uses: ./.github/workflows/_test_spot_private.yaml strategy: - max-parallel: 4 # to avoid failing tests because of API Rate limits + max-parallel: 1 # to avoid failing tests because of API Rate limits fail-fast: false matrix: os: [ubuntu-latest, windows-latest] @@ -131,123 +131,123 @@ jobs: python-version: ${{ matrix.python-version }} secrets: inherit - # ## =========================================================================== - # ## Run the NFT tests - # ## - # ## (public endpoints) - # ## - # Test-NFT-Public: - # needs: [Pre-Commit] - # uses: ./.github/workflows/_test_nft_public.yaml - # strategy: - # fail-fast: false - # matrix: - # os: [ubuntu-latest, windows-latest] - # python-version: ["3.11", "3.12"] - # with: - # os: ${{ matrix.os }} - # python-version: ${{ matrix.python-version }} - # ## - # ## (private endpoints) - # Test-NFT-Private: - # if: success() && github.actor != 'dependabot[bot]' - # needs: [Test-Spot-Private] - # uses: ./.github/workflows/_test_nft_private.yaml - # strategy: - # max-parallel: 1 # to avoid failing tests because of API Rate limits - # fail-fast: false - # matrix: - # os: [ubuntu-latest, windows-latest] - # python-version: ["3.11", "3.12"] - # with: - # os: ${{ matrix.os }} - # python-version: ${{ matrix.python-version }} - # secrets: inherit + ## =========================================================================== + ## Run the NFT tests + ## + ## (public endpoints) + ## + Test-NFT-Public: + needs: [Pre-Commit] + uses: ./.github/workflows/_test_nft_public.yaml + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + python-version: ["3.11", "3.12"] + with: + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + ## + ## (private endpoints) + Test-NFT-Private: + if: success() && github.actor != 'dependabot[bot]' + needs: [Test-Spot-Private] + uses: ./.github/workflows/_test_nft_private.yaml + strategy: + max-parallel: 1 # to avoid failing tests because of API Rate limits + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + python-version: ["3.11", "3.12"] + with: + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + secrets: inherit - # ## =========================================================================== - # ## Run the Futures tests - # ## - # ## (public endpoints) - # Test-Futures-Public: - # needs: [Pre-Commit] - # uses: ./.github/workflows/_test_futures_public.yaml - # strategy: - # fail-fast: false - # matrix: - # os: [ubuntu-latest, windows-latest] - # python-version: ["3.11", "3.12"] - # with: - # os: ${{ matrix.os }} - # python-version: ${{ matrix.python-version }} - # ## - # ## (private endpoints) - # Test-Futures-Private: - # if: success() && github.actor != 'dependabot[bot]' - # needs: [Pre-Commit] - # uses: ./.github/workflows/_test_futures_private.yaml - # strategy: - # max-parallel: 1 # to avoid failing tests because of API Rate limits - # fail-fast: false - # matrix: - # os: [ubuntu-latest, windows-latest] - # python-version: ["3.11", "3.12"] - # with: - # os: ${{ matrix.os }} - # python-version: ${{ matrix.python-version }} - # secrets: inherit + ## =========================================================================== + ## Run the Futures tests + ## + ## (public endpoints) + Test-Futures-Public: + needs: [Pre-Commit] + uses: ./.github/workflows/_test_futures_public.yaml + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + python-version: ["3.11", "3.12"] + with: + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + ## + ## (private endpoints) + Test-Futures-Private: + if: success() && github.actor != 'dependabot[bot]' + needs: [Pre-Commit] + uses: ./.github/workflows/_test_futures_private.yaml + strategy: + max-parallel: 1 # to avoid failing tests because of API Rate limits + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + python-version: ["3.11", "3.12"] + with: + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + secrets: inherit - # ## =========================================================================== - # ## Generates and uploads the coverage statistics to codecov - # ## - # CodeCov: - # if: | - # (success() && github.actor == 'btschwertfeger') - # && (github.event_name == 'push' || github.event_name == 'release') - # needs: - # - Test-Spot-Public - # - Test-Spot-Private - # - Test-Futures-Public - # - Test-Futures-Private - # - Test-NFT-Public - # - Test-NFT-Private - # uses: ./.github/workflows/_codecov.yaml - # with: - # os: "ubuntu-latest" - # python-version: "3.11" - # secrets: inherit + ## =========================================================================== + ## Generates and uploads the coverage statistics to codecov + ## + CodeCov: + if: | + (success() && github.actor == 'btschwertfeger') + && (github.event_name == 'push' || github.event_name == 'release') + needs: + - Test-Spot-Public + - Test-Spot-Private + - Test-Futures-Public + - Test-Futures-Private + - Test-NFT-Public + - Test-NFT-Private + uses: ./.github/workflows/_codecov.yaml + with: + os: "ubuntu-latest" + python-version: "3.11" + secrets: inherit - # ## =========================================================================== - # ## Uploads the package to test.pypi.org on master if triggered by - # ## a regular commit/push. - # ## - # UploadTestPyPI: - # if: | - # (success() && github.ref == 'refs/heads/master') - # && (github.event_name == 'push' || github.event_name == 'release') - # needs: - # - Build - # - Build-Doc - # - CodeQL - # - CodeCov - # name: Upload development version to Test PyPI - # uses: ./.github/workflows/_pypi_test_publish.yaml - # secrets: - # API_TOKEN: ${{ secrets.TEST_PYPI_API_TOKEN }} + ## =========================================================================== + ## Uploads the package to test.pypi.org on master if triggered by + ## a regular commit/push. + ## + UploadTestPyPI: + if: | + (success() && github.ref == 'refs/heads/master') + && (github.event_name == 'push' || github.event_name == 'release') + needs: + - Build + - Build-Doc + - CodeQL + - CodeCov + name: Upload development version to Test PyPI + uses: ./.github/workflows/_pypi_test_publish.yaml + secrets: + API_TOKEN: ${{ secrets.TEST_PYPI_API_TOKEN }} - # ## =========================================================================== - # ## Upload the python-kraken-sdk to Production PyPI - # ## - # UploadPyPI: - # if: | - # success() - # && github.actor == 'btschwertfeger' - # && github.event_name == 'release' - # needs: - # - Build - # - Build-Doc - # - CodeQL - # - CodeCov - # name: Upload release to PyPI - # uses: ./.github/workflows/_pypi_publish.yaml - # secrets: - # API_TOKEN: ${{ secrets.PYPI_API_TOKEN }} + ## =========================================================================== + ## Upload the python-kraken-sdk to Production PyPI + ## + UploadPyPI: + if: | + success() + && github.actor == 'btschwertfeger' + && github.event_name == 'release' + needs: + - Build + - Build-Doc + - CodeQL + - CodeCov + name: Upload release to PyPI + uses: ./.github/workflows/_pypi_publish.yaml + secrets: + API_TOKEN: ${{ secrets.PYPI_API_TOKEN }} diff --git a/kraken/futures/websocket/__init__.py b/kraken/futures/websocket/__init__.py index 75e8d8b..d20ed2b 100644 --- a/kraken/futures/websocket/__init__.py +++ b/kraken/futures/websocket/__init__.py @@ -16,7 +16,7 @@ from random import random from typing import TYPE_CHECKING, Any -import websockets +from websockets.asyncio.client import connect from kraken.exceptions import MaxReconnectError @@ -95,9 +95,11 @@ async def __run( # noqa: C901 self.__new_challenge = None self.__last_challenge = None - async with websockets.connect( # pylint: disable=no-member # noqa: PLR1702 + async with connect( # pylint: disable=no-member # noqa: PLR1702 f"wss://{self.__ws_endpoint}", + additional_headers={"User-Agent": "btschwertfeger/python-kraken-sdk"}, ping_interval=30, + max_queue=None, # FIXME: This is not recommended by the docs https://websockets.readthedocs.io/en/stable/reference/asyncio/server.html ) as socket: LOG.info("Websocket connected!") self.socket = socket @@ -118,7 +120,6 @@ async def __run( # noqa: C901 LOG.exception("asyncio.CancelledError") self.keep_alive = False await self.__callback({"error": "asyncio.CancelledError"}) - await self.socket.close() else: try: message: dict = json.loads(_message) diff --git a/kraken/spot/websocket/connectors.py b/kraken/spot/websocket/connectors.py index eb18175..45427dd 100644 --- a/kraken/spot/websocket/connectors.py +++ b/kraken/spot/websocket/connectors.py @@ -22,7 +22,7 @@ from time import time from typing import TYPE_CHECKING, Any, Final -import websockets +from websockets.asyncio.client import connect from kraken.exceptions import MaxReconnectError @@ -127,10 +127,11 @@ async def __run(self: ConnectSpotWebsocketBase, event: asyncio.Event) -> None: ) LOG.debug("Websocket token: %s", self.ws_conn_details) - async with websockets.connect( # pylint: disable=no-member + async with connect( # pylint: disable=no-member f"wss://{self.__ws_endpoint}", additional_headers={"User-Agent": "btschwertfeger/python-kraken-sdk"}, ping_interval=30, + max_queue=None, # FIXME: This is not recommended by the docs https://websockets.readthedocs.io/en/stable/reference/asyncio/server.html ) as socket: LOG.info("Websocket connected!") self.socket = socket @@ -151,7 +152,6 @@ async def __run(self: ConnectSpotWebsocketBase, event: asyncio.Event) -> None: LOG.exception("asyncio.CancelledError") self.keep_alive = False await self.__callback({"error": "asyncio.CancelledError"}) - await self.socket.close() else: try: message: dict = json.loads(_message) @@ -187,6 +187,7 @@ async def __run_forever(self: ConnectSpotWebsocketBase) -> None: async def close_connection(self: ConnectSpotWebsocketBase) -> None: """Closes the websocket connection and thus forces a reconnect""" await self.socket.close() + await self.socket.wait_closed() async def __reconnect(self: ConnectSpotWebsocketBase) -> None: """ @@ -199,6 +200,7 @@ async def __reconnect(self: ConnectSpotWebsocketBase) -> None: LOG.info("Websocket start connect/reconnect") self.__reconnect_num += 1 + if self.__reconnect_num >= self.MAX_RECONNECT_NUM: raise MaxReconnectError( "The Kraken Spot websocket client encountered to many reconnects!", @@ -223,6 +225,7 @@ async def __reconnect(self: ConnectSpotWebsocketBase) -> None: tasks, return_when=asyncio.FIRST_EXCEPTION, ) + exception_occur: bool = False for task in finished: if task.exception(): @@ -241,6 +244,7 @@ async def __reconnect(self: ConnectSpotWebsocketBase) -> None: await self.__callback({"error": message}) if exception_occur: break + LOG.warning("Connection closed") def __get_reconnect_wait( diff --git a/tests/spot/test_spot_orderbook.py b/tests/spot/test_spot_orderbook.py index 27a9d91..27a9183 100644 --- a/tests/spot/test_spot_orderbook.py +++ b/tests/spot/test_spot_orderbook.py @@ -121,7 +121,6 @@ async def assign() -> None: asyncio.run(assign()) -@pytest.mark.wip @pytest.mark.spot @pytest.mark.spot_websocket @pytest.mark.spot_orderbook @@ -134,7 +133,10 @@ def test_add_book(caplog: pytest.LogCaptureFixture) -> None: async def execute_add_book() -> None: async with SpotOrderBookClientWrapper() as orderbook: - await orderbook.add_book(pairs=["BTC/USD"]) + # having multiple pairs to test the cancellation queue error absence + await orderbook.add_book( + pairs=["BTC/USD", "DOT/USD", "ETH/USD", "MATIC/USD", "BTC/EUR"], + ) await async_sleep(4) book: dict | None = orderbook.get(pair="BTC/USD")