Merge branch 'main' into cbor-remove-py-impl

.builder/actions/aws_crt_python.py

@@ -1,67 +1,37 @@
 import Builder
 import argparse
-import os
+from pathlib import Path
 import sys
 
-# Fall back on using the "{python}" builder variable
-PYTHON_DEFAULT = '{python}'
-
 
 class AWSCrtPython(Builder.Action):
-    python = PYTHON_DEFAULT
-
-    # Some CI containers have pip installed via "rpm" or non-Python methods, and this causes issues when
-    # we try to update pip via "python -m pip install --upgrade" because there are no RECORD files present.
-    # Therefore, we have to seek alternative ways with a last resort of installing with "--ignore-installed"
-    # if nothing else works AND the builder is running in GitHub actions.
-    # As of writing, this is primarily an issue with the AL2-x64 image.
-    def try_to_upgrade_pip(self, env):
-        did_upgrade = False
-
-        if (self.python == '{python}'):
-            self.python = env.config["variables"]["python"]
-
-        pip_result = env.shell.exec(self.python, '-m', 'pip', 'install', '--upgrade', 'pip', check=False)
-        if pip_result.returncode == 0:
-            did_upgrade = True
-        else:
-            print("Could not update pip via normal pip upgrade. Next trying via package manager...")
-
-        if (did_upgrade == False):
-            try:
-                Builder.InstallPackages(['pip']).run(env)
-                did_upgrade = True
-            except Exception:
-                print("Could not update pip via package manager. Next resorting to forcing an ignore install...")
-
-        if (did_upgrade == False):
-            # Only run in GitHub actions by checking for specific environment variable
-            # Source: https://docs.github.com/en/actions/learn-github-actions/variables#default-environment-variables
-            if (os.getenv("GITHUB_ACTIONS") is not None):
-                pip_result = env.shell.exec(
-                    self.python, '-m', 'pip', 'install', '--upgrade',
-                    '--ignore-installed', 'pip', check=False)
-                if pip_result.returncode == 0:
-                    did_upgrade = True
-                else:
-                    print("Could not update pip via ignore install! Something is terribly wrong!")
-                    sys.exit(12)
-            else:
-                print("Not on GitHub actions - skipping reinstalling Pip. Update/Install pip manually and rerun the builder")
 
     def run(self, env):
         # allow custom python to be used
         parser = argparse.ArgumentParser()
         parser.add_argument('--python')
         args = parser.parse_known_args(env.args.args)[0]
-        self.python = args.python if args.python else PYTHON_DEFAULT
+        if args.python:
+            self.python = args.python
+        else:
+            # Fall back on using the "{python}" builder variable
+            self.python = env.config['variables']['python']
+
+        # Create a virtual environment and use that.
+        # Otherwise, in places like ubuntu 24.04, PEP 668 stops
+        # you from globally installing/upgrading packages
+        venv_dirpath = Path.cwd() / '.venv-builder'
+        env.shell.exec(self.python, '-m', 'venv', str(venv_dirpath), check=True)
+        if sys.platform == 'win32':
+            self.python = str(venv_dirpath / 'Scripts/python')
+        else:
+            self.python = str(venv_dirpath / 'bin/python')
 
         # Enable S3 tests
         env.shell.setenv('AWS_TEST_S3', '1')
 
         actions = [
-            # Upgrade Pip via a number of different methods
-            self.try_to_upgrade_pip,
+            [self.python, '-m', 'pip', 'install', '--upgrade', 'pip'],
             [self.python, '-m', 'pip', 'install', '--upgrade', '--requirement', 'requirements-dev.txt'],
             Builder.SetupCrossCICrtEnvironment(),
             [self.python, '-m', 'pip', 'install', '--verbose', '.'],

.github/workflows/ci.yml

@@ -7,7 +7,7 @@ on:
       - 'docs'
 
 env:
-  BUILDER_VERSION: v0.9.56
+  BUILDER_VERSION: v0.9.59
   BUILDER_SOURCE: releases
   BUILDER_HOST: https://d19elf31gohf1l.cloudfront.net
   PACKAGE_NAME: aws-crt-python
@@ -158,7 +158,7 @@ jobs:
       - name: Assert libcrypto.so used
         run: |
           # assert it's linked against the system's libcrypto.so
-          AWSCRT_PATH=`python3 -c "import _awscrt; print(_awscrt.__file__)"`
+          AWSCRT_PATH=`aws-crt-python/.venv-builder/bin/python3 -c "import _awscrt; print(_awscrt.__file__)"`
           printf "AWSCRT_PATH: $AWSCRT_PATH\n"
 
           LINKED_AGAINST=`ldd $AWSCRT_PATH`
@@ -192,16 +192,21 @@ jobs:
 
   openbsd:
     runs-on: ubuntu-22.04 # latest
+    strategy:
+      fail-fast: false
+      matrix:
+        # OpenBSD only supports the two most recent releases
+        version: ['7.4', '7.5']
     steps:
       # Cannot use builder to checkout as OpenBSD doesn't ship git in the base install
       - uses: actions/checkout@v3
         with:
           submodules: true
       - name: Build ${{ env.PACKAGE_NAME }} + consumers
-        uses: cross-platform-actions/action@v0.23.0
+        uses: cross-platform-actions/action@v0.24.0
         with:
           operating_system: openbsd
-          version: '7.4'
+          version: ${{ matrix.version }}
           cpu_count: 4
           shell: bash
           environment_variables: AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_DEFAULT_REGION AWS_REGION
@@ -272,7 +277,7 @@ jobs:
           submodules: true
       - name: Check docs
         run: |
-          python3 -m pip install sphinx
+          python3 -m pip install -r requirements-dev.txt
           python3 -m pip install --verbose .
           ./scripts/make-docs.py
 

.github/workflows/docs.yml

@@ -9,7 +9,7 @@ on:
 
 jobs:
   update-docs-branch:
-    runs-on: ubuntu-20.04 # latest
+    runs-on: ubuntu-22.04 # latest
     permissions:
       contents: write # allow push
     steps:
@@ -20,7 +20,7 @@ jobs:
 
       - name: Update docs branch
         run: |
-          python3 -m pip install sphinx
+          python3 -m pip install -r requirements-dev.txt
           python3 -m pip install --verbose .
           ./scripts/make-docs.py
 

.github/workflows/lint.yml

@@ -9,17 +9,15 @@ on:
 jobs:
   clang-format:
 
-    runs-on: ubuntu-20.04 # latest
+    runs-on: ubuntu-24.04 # latest
 
     steps:
     - name: Checkout Sources
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
 
     - name: clang-format lint
-      uses: DoozyX/[email protected]
-      with:
-        # List of extensions to check
-        extensions: c,h
+      run: |
+        ./format-check.py
 
   autopep8:
     runs-on: ubuntu-20.04 # latest

.gitignore

@@ -187,7 +187,7 @@ ipython_config.py
 # pyenv
 #   For a library or package, you might want to ignore these files since the code is
 #   intended to run in multiple environments; otherwise, check them in:
-# .python-version
+.python-version
 
 # pipenv
 #   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
@@ -261,6 +261,84 @@ cython_debug/
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
 
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider
+# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
+
+# User-specific stuff
+.idea/**/workspace.xml
+.idea/**/tasks.xml
+.idea/**/usage.statistics.xml
+.idea/**/dictionaries
+.idea/**/shelf
+
+# AWS User-specific
+.idea/**/aws.xml
+
+# Generated files
+.idea/**/contentModel.xml
+
+# Sensitive or high-churn files
+.idea/**/dataSources/
+.idea/**/dataSources.ids
+.idea/**/dataSources.local.xml
+.idea/**/sqlDataSources.xml
+.idea/**/dynamic.xml
+.idea/**/uiDesigner.xml
+.idea/**/dbnavigator.xml
+
+# Gradle
+.idea/**/gradle.xml
+.idea/**/libraries
+
+# Gradle and Maven with auto-import
+# When using Gradle or Maven with auto-import, you should exclude module files,
+# since they will be recreated, and may cause churn.  Uncomment if using
+# auto-import.
+# .idea/artifacts
+# .idea/compiler.xml
+# .idea/jarRepositories.xml
+# .idea/modules.xml
+# .idea/*.iml
+# .idea/modules
+# *.iml
+# *.ipr
+
+# CMake
+cmake-build-*/
+
+# Mongo Explorer plugin
+.idea/**/mongoSettings.xml
+
+# File-based project format
+*.iws
+
+# IntelliJ
+out/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Cursive Clojure plugin
+.idea/replstate.xml
+
+# SonarLint plugin
+.idea/sonarlint/
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+fabric.properties
+
+# Editor-based Rest Client
+.idea/httpRequests
+
+# Android studio 3.1+ serialized cache file
+.idea/caches/build_file_checksums.ser
+
 ### Python Patch ###
 # Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration
 poetry.toml

awscrt/mqtt.py

@@ -444,11 +444,15 @@ def _on_connection_resumed(self, return_code, session_present):
 
     def _ws_handshake_transform(self, http_request_binding, http_headers_binding, native_userdata):
         if self._ws_handshake_transform_cb is None:
-            _awscrt.mqtt_ws_handshake_transform_complete(None, native_userdata)
+            _awscrt.mqtt_ws_handshake_transform_complete(None, native_userdata, 0)
             return
 
         def _on_complete(f):
-            _awscrt.mqtt_ws_handshake_transform_complete(f.exception(), native_userdata)
+            error_code = 0
+            hs_exception = f.exception()
+            if isinstance(hs_exception, awscrt.exceptions.AwsCrtError):
+                error_code = hs_exception.code
+            _awscrt.mqtt_ws_handshake_transform_complete(f.exception(), native_userdata, error_code)
 
         future = Future()
         future.add_done_callback(_on_complete)