Merge branch 'main' into new-checksum-algo

awslabs · Dec 5, 2024 · 7752810 · 7752810
2 parents 619b17b + 91f9370
commit 7752810
Show file tree

Hide file tree

Showing 32 changed files with 154 additions and 93 deletions.
diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml
@@ -12,6 +12,14 @@ body:
       description: What is the problem? A clear and concise description of the bug.
     validations:
       required: true
+  - type: checkboxes
+    id: regression
+    attributes:
+      label: Regression Issue
+      description: What is a regression? If it worked in a previous version but doesn't in the latest version, it's considered a regression. In this case, please provide specific version number in the report.
+      options:
+        - label: Select this option if this issue appears to be a regression.
+          required: false
   - type: textarea
     id: expected
     attributes:

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -6,6 +6,11 @@ on:
       - 'main'
       - 'docs'
 
+# cancel in-progress builds after a new commit
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 env:
   BUILDER_VERSION: v0.9.67
   BUILDER_SOURCE: releases
@@ -72,6 +77,7 @@ jobs:
     name: ARM (${{ matrix.arch }})
     runs-on: ubuntu-22.04 # latest
     strategy:
+      fail-fast: false
       matrix:
         arch: [armv6, armv7, arm64]
     steps:
@@ -150,7 +156,6 @@ jobs:
   raspberry:
     runs-on: ubuntu-22.04 # latest
     strategy:
-      fail-fast: false
       matrix:
         image:
           - raspbian-bullseye
@@ -221,10 +226,9 @@ jobs:
           python3 codebuild/macos_compatibility_check.py
 
   android:
-      # ubuntu-20.04 comes with Android tooling, see:
-      # https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-README.md#android
+      # ubuntu-24.04 comes with Android tooling
     name: Android
-    runs-on: ubuntu-20.04 # latest
+    runs-on: ubuntu-24.04 # latest
     steps:
       - name: Checkout Sources
         uses: actions/checkout@v4
@@ -236,11 +240,14 @@ jobs:
         with:
           java-version: '11'
           distribution: 'temurin'
-          cache: 'gradle'
       # Build and publish locally for the test app to find the SNAPSHOT version
       - name: Build ${{ env.PACKAGE_NAME }}
         run: |
-          ./gradlew :android:crt:build
+          # Manually set -Xmx (max heap size) to something huge (tested 2g and that works, but why not go bigger).
+          # Only in CI, gradle daemon runs out of memory during "lintAnalyzeDebug" task, unless you specify it this way.
+          # You'd think Java's default of 25% RAM (ubuntu24 runner has 12g, so max 4g) would be sufficient, but no.
+          # You'd think setting -Xmx via gradle.properties would help, but no.
+          ./gradlew :android:crt:build -Dorg.gradle.jvmargs="-Xmx8g"
           ./gradlew -PnewVersion="1.0.0-SNAPSHOT" :android:crt:publishToMavenLocal
       # Setup files required by test app for Device Farm testing
       - name: Setup Android Test Files

diff --git a/.github/workflows/issue-regression-labeler.yml b/.github/workflows/issue-regression-labeler.yml
@@ -0,0 +1,32 @@
+# Apply potential regression label on issues
+name: issue-regression-label
+on:
+  issues:
+    types: [opened, edited]
+jobs:
+  add-regression-label:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+    steps:
+    - name: Fetch template body
+      id: check_regression
+      uses: actions/github-script@v7
+      env: 
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        TEMPLATE_BODY: ${{ github.event.issue.body }}
+      with:
+        script: |
+          const regressionPattern = /\[x\] Select this option if this issue appears to be a regression\./i;
+          const template = `${process.env.TEMPLATE_BODY}`
+          const match = regressionPattern.test(template);
+          core.setOutput('is_regression', match);
+    - name: Manage regression label
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: |
+        if [ "${{ steps.check_regression.outputs.is_regression }}" == "true" ]; then
+          gh issue edit ${{ github.event.issue.number }} --add-label "potential-regression" -R ${{ github.repository }}
+        else
+          gh issue edit ${{ github.event.issue.number }} --remove-label "potential-regression" -R ${{ github.repository }}
+        fi
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -1,13 +1,9 @@
-cmake_minimum_required(VERSION 3.1)
+cmake_minimum_required(VERSION 3.9)
 
 project(aws-crt-jni C)
 option(BUILD_DEPS "Builds aws common runtime dependencies as part of build" ON)
 option(CRT_FIPS "Whether to build aws-lc with FIPS compliance" OFF)
 
-if (POLICY CMP0069)
-    cmake_policy(SET CMP0069 NEW) # Enable LTO/IPO if available in the compiler, see AwsCFlags
-endif()
-
 if (POLICY CMP0077)
     cmake_policy(SET CMP0077 NEW) # Enable options to get their values from normal variables
 endif()
@@ -60,46 +56,47 @@ if (BUILD_DEPS)
     set(BUILD_TESTING OFF)
     add_subdirectory(crt/aws-c-common)
     if (UNIX AND NOT APPLE)
-        set(BUILD_LIBSSL OFF CACHE BOOL "Don't need libssl, only need libcrypto")
+        include(AwsPrebuildDependency)
+
+        set(AWSLC_CMAKE_ARGUMENTS
+            -DBUILD_LIBSSL=OFF
+            -DBUILD_TESTING=OFF
+        )
+
         message("Build with FIPS: " ${CRT_FIPS})
         if (CRT_FIPS)
-            set(FIPS ON CACHE BOOL "FIPS compliance")
-            set(PERL_EXECUTABLE "perl")
+            list(APPEND AWSLC_CMAKE_ARGUMENTS -DFIPS=ON)
+            list(APPEND AWSLC_CMAKE_ARGUMENTS -DDISABLE_PERL=OFF)
+
+            # Pick up GO_PATH env-var, set by aws-crt-builder when cross-compiling, see:
+            # https://github.com/awslabs/aws-crt-builder/blob/31307c808ed9f2ea1eb16503b25a9b582f886481/builder/imports/golang.py#L84
+            # https://github.com/awslabs/aws-crt-builder/blob/31307c808ed9f2ea1eb16503b25a9b582f886481/builder/actions/cmake.py#L110
             if (DEFINED ENV{GO_PATH})
-                set(GO_EXECUTABLE $ENV{GO_PATH}/go)
+                list(APPEND AWSLC_CMAKE_ARGUMENTS -DGO_EXECUTABLE=$ENV{GO_PATH}/go)
                 message(STATUS "Overriding GO_EXECUTABLE to ${GO_EXECUTABLE}")
             endif()
         else()
-            set(DISABLE_PERL ON CACHE BOOL "Disable codegen")
-            set(DISABLE_GO ON CACHE BOOL "Disable codegen")
+            list(APPEND AWSLC_CMAKE_ARGUMENTS -DDISABLE_PERL=ON)  # Disable codegen
+            list(APPEND AWSLC_CMAKE_ARGUMENTS -DDISABLE_GO=ON)  # Disable codegen
         endif()
 
         if(CMAKE_C_COMPILER_ID MATCHES "GNU" AND CMAKE_C_COMPILER_VERSION VERSION_LESS "5.0")
-            set(DISABLE_PERL OFF CACHE BOOL "Build with Perl to avoid using pre-compiled binary with AVX512")
-            set(PERL_EXECUTABLE "perl")
-            set(MY_ASSEMBLER_IS_TOO_OLD_FOR_512AVX ON CACHE BOOL "Disable AVX512 on old GCC that not supports it")
+            # Disable AVX512 on old GCC that not supports it
+            list(APPEND AWSLC_CMAKE_ARGUMENTS -DMY_ASSEMBLER_IS_TOO_OLD_FOR_512AVX=ON)
         endif()
 
-        # temporarily disable certain warnings as errors for the aws-lc build
-        set(OLD_CMAKE_C_FLAGS "${CMAKE_C_FLAGS}")
-        if (NOT MSVC)
-            check_c_compiler_flag(-Wno-stringop-overflow HAS_WNO_STRINGOP_OVERFLOW)
-            if (HAS_WNO_STRINGOP_OVERFLOW)
-                set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-stringop-overflow")
-            endif()
-
-            check_c_compiler_flag(-Wno-array-parameter HAS_WNO_ARRAY_PARAMETER)
-            if (HAS_WNO_ARRAY_PARAMETER)
-                set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-array-parameter")
-            endif()
+        if (ANDROID)
+            list(APPEND AWSLC_CMAKE_ARGUMENTS -DANDROID_DEPS_DIR=${ANDROID_DEPS_DIR})
+            list(APPEND AWSLC_CMAKE_ARGUMENTS -DAWS_LIBRARY_OUTPUT_DIR="${AWS_LIBRARY_OUTPUT_DIR}")
         endif()
 
-        add_subdirectory(crt/aws-lc)
-
-        # restore previous build flags
-        set(CMAKE_C_FLAGS "${OLD_CMAKE_C_FLAGS}")
+        # s2n-tls uses libcrypto during its configuration, so we need to prebuild aws-lc.
+        aws_prebuild_dependency(
+            DEPENDENCY_NAME AWSLC
+            SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/crt/aws-lc
+            CMAKE_ARGUMENTS ${AWSLC_CMAKE_ARGUMENTS}
+        )
 
-        set(SEARCH_LIBCRYPTO OFF CACHE BOOL "Let S2N use libcrypto from AWS-LC.")
         set(UNSAFE_TREAT_WARNINGS_AS_ERRORS OFF CACHE BOOL "Disable warnings-as-errors when building S2N")
         add_subdirectory(crt/s2n)
     endif()
@@ -124,7 +121,7 @@ include(AwsPlatformDetect)
 include(AwsSharedLibSetup)
 include(AwsCRuntime)
 
-if (CRT_FIPS AND NOT FIPS)
+if (CRT_FIPS AND NOT AWSLC_PREBUILT)
     message(FATAL_ERROR "CRT_FIPS can only be set when build with aws-lc.")
 endif()
 

diff --git a/README.md b/README.md
@@ -30,7 +30,7 @@ This library is licensed under the Apache 2.0 License.
 ### Linux/Unix
 Requirements:
 * Clang 3.9+ or GCC 4.4+
-* cmake 3.1+
+* cmake 3.9+
 * Java: Any JDK8 or above, ensure `JAVA_HOME` is set
 * Maven
 
@@ -43,7 +43,7 @@ Building:
 
 ### OSX
 Requirements:
-* cmake 3.1
+* cmake 3.9+
 * ninja
 * Java: Any JDK8 or above, ensure `JAVA_HOME` is set
 * Maven
@@ -58,7 +58,7 @@ Building:
 ### Windows
 Requirements:
 * Visual Studio 2015 or above
-* CMake 3.1
+* CMake 3.9+
 * Java: Any JDK8 or above, ensure `JAVA_HOME` is set
 * Maven
 
@@ -161,13 +161,15 @@ Platforms without FIPS compliance are also included in this jar, for compatibili
 > [!WARNING]
 > The classifier, and platforms with FIPS compliance are subject to change in the future.
 
+Platforms with FIPS compliance use [AWS-LC](https://github.com/aws/aws-lc) as their cryptographic module ([NIST Certificate #4816](https://csrc.nist.gov/projects/cryptographic-module-validation-program/certificate/4816)).
+
 ## GraalVM support
 
 Since version v0.29.20, GraalVM native image was supported. You can compile your application with AWS CRT in a GraalVM native image project without any additional configuration.
 
 Since version v0.31.1, GraalVM support was updated. Instead of packaging the JNI shared libraries with native image as resources, the corresponding shared lib will be written to the same directory as the native image.
 In this way, it reduces the native image size around 30% (142 MB to 101 MB for a sample application), and avoids the extra loading time needed for extracting the JNI lib to the temporary path for load. No additional configuration needed.
-> [!NOTE] 
+> [!NOTE]
 > The JNI shared lib must be in the same directory as the GraalVM native image. If you move the native image, you must move this file too. It is `aws-crt-jni.dll` on Windows, `libaws-crt-jni.dylib` on macOS, and `libaws-crt-jni.so` on Unix.
 
 ## System Properties

diff --git a/android/build.gradle b/android/build.gradle
@@ -4,7 +4,7 @@ buildscript {
         mavenCentral()
     }
     dependencies {
-        classpath 'com.android.tools.build:gradle:7.0.0'
+        classpath 'com.android.tools.build:gradle:7.4.2'
     }
 }
 

diff --git a/android/crt/build.gradle b/android/crt/build.gradle
@@ -43,8 +43,7 @@ ext {
 }
 
 android {
-    compileSdkVersion 30
-    buildToolsVersion "30.0.3"
+    compileSdk 33
     ndkVersion "21.4.7075529" // LTS version
 
     useLibrary 'android.test.runner'
@@ -53,7 +52,8 @@ android {
 
     defaultConfig {
         minSdkVersion 24  // TODO - dictated by CompletableFuture which is API 24+
-        targetSdkVersion 30
+        targetSdkVersion 33
+
         versionCode = gitVersionCode()
         versionName = gitVersionName()
 
@@ -103,7 +103,6 @@ android {
         cmake {
             path "../../CMakeLists.txt"
             buildStagingDirectory "../../target/cmake-build"
-            version "3.10.2"
         }
     }
 

diff --git a/android/gradle.properties b/android/gradle.properties
@@ -6,7 +6,7 @@
 # http://www.gradle.org/docs/current/userguide/build_environment.html
 # Specifies the JVM arguments used for the daemon process.
 # The setting is particularly useful for tweaking memory settings.
-org.gradle.jvmargs=-Xmx1536m
+# org.gradle.jvmargs=-Xmx2G
 # When configured, Gradle will run in incubating parallel mode.
 # This option should only be used with decoupled projects. More details, visit
 # http://www.gradle.org/docs/current/userguide/multi_project_builds.html#sec:decoupled_projects

diff --git a/codebuild/cd/deploy-snapshot.yml b/codebuild/cd/deploy-snapshot.yml
@@ -16,7 +16,7 @@ phases:
       # This weird path needed for cmd tool to work
       - mv $ANDROID_SDK_ROOT/cmdline-tools/cmdline-tools $ANDROID_SDK_ROOT/cmdline-tools/latest
       # install android build tools
-      - echo y | $ANDROID_SDK_ROOT/cmdline-tools/latest/bin/sdkmanager --install "build-tools;30.0.3" "platforms;android-30" "ndk;21.4.7075529"
+      - echo y | $ANDROID_SDK_ROOT/cmdline-tools/latest/bin/sdkmanager --install "build-tools;30.0.3" "platforms;android-33" "ndk;21.4.7075529"
 
   pre_build:
     commands:

diff --git a/codebuild/cd/generic-unix-build.sh b/codebuild/cd/generic-unix-build.sh
@@ -29,10 +29,20 @@ chmod a+x builder
 GIT_TAG=$(git describe --tags)
 
 ./builder build -p aws-crt-java --target=$AWS_CRT_TARGET run_tests=false
-# Builder corss-compiles the shared lib to `target/cmake-build/aws-crt-java/`, move it to the expected path for mvn to generate the jar.
-mv target/cmake-build/aws-crt-java/* target/cmake-build/
+
+# When cross-compiling with builder, the shared lib gets an extra "/aws-crt-java/" in its path.
+# Move it to expected location.
+if [ -d target/cmake-build/aws-crt-java/lib ]; then
+    mv target/cmake-build/aws-crt-java/lib target/cmake-build/lib
+fi
+
+# Double check that shared lib is where we expect
+if ! find target/cmake-build/lib -type f -name "*.so" | grep -q .; then
+  echo "No .so files found"
+  exit 1
+fi
 
 JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64 mvn -B package -DskipTests -Dshared-lib.skip=true -Dcrt.classifier=$CLASSIFIER
 
-aws s3 cp --recursive --include "*.so" target/cmake-build/lib s3://aws-crt-java-pipeline/${GIT_TAG}/lib
+aws s3 cp --recursive --exclude "*" --include "*.so" target/cmake-build/lib s3://aws-crt-java-pipeline/${GIT_TAG}/lib
 aws s3 cp target/ s3://aws-crt-java-pipeline/${GIT_TAG}/jar/ --recursive --exclude "*" --include "aws-crt*.jar"
diff --git a/codebuild/cd/linux-aarch64-fips-build.sh b/codebuild/cd/linux-aarch64-fips-build.sh
@@ -18,8 +18,19 @@ chmod a+x builder
 GIT_TAG=$(git describe --tags)
 
 ./builder build -p aws-crt-java run_tests=false --target=linux-arm64 --cmake-extra=-DCRT_FIPS=ON
-mv target/cmake-build/aws-crt-java/* target/cmake-build/
+
+# When cross-compiling with builder, the shared lib gets an extra "/aws-crt-java/" in its path.
+# Move it to expected location.
+if [ -d target/cmake-build/aws-crt-java/lib ]; then
+    mv target/cmake-build/aws-crt-java/lib target/cmake-build/lib
+fi
+
+# Double check that shared lib is where we expect
+if ! find target/cmake-build/lib -type f -name "*.so" | grep -q .; then
+  echo "No .so files found"
+  exit 1
+fi
 
 JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64 mvn -B package -DskipTests -Dshared-lib.skip=true -Dcrt.classifier=linux-aarch_64-fips
 
-aws s3 cp --recursive --include "*.so" target/cmake-build/lib s3://aws-crt-java-pipeline/${GIT_TAG}/fips_lib
+aws s3 cp --recursive --exclude "*" --include "*.so" target/cmake-build/lib s3://aws-crt-java-pipeline/${GIT_TAG}/fips_lib
diff --git a/codebuild/cd/manylinux-x64-build.yml b/codebuild/cd/manylinux-x64-build.yml
@@ -1,20 +1,15 @@
 version: 0.2
-#this build spec assumes the manylinux CentOS5 custom image
-#additional packages we installed: cmake 3.5, libcrypto 1.1.0j, gcc 4.8.4, openjdk8, maven 3.6.0, gnupg 2.0.10
 phases:
   install:
     commands:
   pre_build:
     commands:
       - export CC=gcc
-      - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.17.51.zip" -o "awscliv2.zip"
-      - unzip awscliv2.zip
-      - ./aws/install
   build:
     commands:
       - cd aws-crt-java
       - git submodule update --init
-      - JAVA_HOME=/opt/java-se-8u40-ri/ mvn -B package -DskipTests -Dcrt.classifier=linux-x86_64
+      - mvn -B package -DskipTests -Dcrt.classifier=linux-x86_64
 
   post_build:
     commands:

diff --git a/codebuild/cd/manylinux-x64-fips-build.yml b/codebuild/cd/manylinux-x64-fips-build.yml
@@ -1,25 +1,19 @@
 version: 0.2
-#this build spec assumes the manylinux CentOS5 custom image
-#additional packages we installed: cmake 3.5, libcrypto 1.1.0j, gcc 4.8.4, openjdk8, maven 3.6.0, gnupg 2.0.10
 phases:
   install:
     commands:
   pre_build:
     commands:
       - export CC=gcc
-      - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.17.51.zip" -o "awscliv2.zip"
-      - unzip awscliv2.zip
-      - ./aws/install
   build:
     commands:
       - cd aws-crt-java
       - git submodule update --init
       # double check aws-lc is the FIPS approved branch.
       - bash ./codebuild/cd/test-fips-branch.sh
-      - curl -OL https://go.dev/dl/go1.21.6.linux-amd64.tar.gz && mkdir ./go
-      - tar -C ./go -xvf go1.21.6.linux-amd64.tar.gz
-      - export PATH=$PATH:./go/go/bin
-      - JAVA_HOME=/opt/java-se-8u40-ri/ mvn -B package -DskipTests -Dcrt.classifier=linux-x86_64-fips -Dcmake.crt_fips=ON
+      # aws-lc FIPS build requires golang for codegen
+      - yum install -y golang
+      - mvn -B package -DskipTests -Dcrt.classifier=linux-x86_64-fips -Dcmake.crt_fips=ON
 
   post_build:
     commands: