Merge branch 'master' into v6_address_design

.bazelignore

@@ -1,4 +1,6 @@
 bin
+doc/_build
+doc/venv
 docker/_build
-rules_openapi/tools/node_modules
+private/mgmtapi/tools/node_modules
 tools/lint/logctxcheck/testdata/src

.bazelrc

@@ -1,11 +1,12 @@
 ### common options for all subcommands (help, query, build, ...)
-common --show_timestamps
+common --show_timestamps --enable_platform_specific_config
 
 # connect to buchgr/bazel-remote cache
 # These flags can unfortunately not be specified for `common`, as they are not accepted by all subcommands (help, version, dump)
-build --remote_cache=grpc://localhost:9092 --experimental_remote_downloader=grpc://localhost:9092
-query --remote_cache=grpc://localhost:9092 --experimental_remote_downloader=grpc://localhost:9092
-fetch --remote_cache=grpc://localhost:9092 --experimental_remote_downloader=grpc://localhost:9092
+# The --experimental_remote_downloader_local_fallback is used as workaround for issue with rules_oci (https://github.com/bazel-contrib/rules_oci/issues/275).
+build --remote_cache=grpc://localhost:9092 --experimental_remote_downloader=grpc://localhost:9092 --experimental_remote_downloader_local_fallback=true
+query --remote_cache=grpc://localhost:9092 --experimental_remote_downloader=grpc://localhost:9092 --experimental_remote_downloader_local_fallback=true
+fetch --remote_cache=grpc://localhost:9092 --experimental_remote_downloader=grpc://localhost:9092 --experimental_remote_downloader_local_fallback=true
 
 ### options for build, test, run, clean, etc.
 # expose git version (etc) to bazel
@@ -14,13 +15,18 @@ build --workspace_status_command=./tools/bazel-build-env
 build --java_runtime_version=remotejdk_11
 # disable legacy_create_init for py_binary, py_test etc. This may eventually become the default.
 build --incompatible_default_to_explicit_init_py
+# Enable resolution of cc toolchain by go toolchain
+build --incompatible_enable_cc_toolchain_resolution
+build --flag_alias=file_name_version=//:file_name_version
+build  --build_python_zip
 
 # include one of "--define gotags=sqlite_mattn" or "--define gotags=sqlite_modernc"
 # cannot be in common, because query chokes on it.
-build --define gotags=sqlite_modernc
+build --define gotags=sqlite_modernc,netgo
+build:osx --define gotags=sqlite_modernc
 
 ### options for test
-test --build_tests_only --print_relative_test_log_paths --test_output=errors
+test --build_tests_only --print_relative_test_log_paths --test_output=errors --nobuild_python_zip
 
 ### predefined configurations (use with --config, eg. `bazel test --config=unit_all`)
 test:race --@io_bazel_rules_go//go/config:race
@@ -30,7 +36,7 @@ test:unit_all --config=unit //...
 test:integration --test_tag_filters=integration,-lint
 test:integration_all --config=integration //...
 
-test:lint --test_tag_filters=lint --test_summary=terse --noshow_progress --experimental_convenience_symlinks=ignore //...
+test:lint --test_tag_filters=lint,write_src --test_summary=terse --noshow_progress --experimental_convenience_symlinks=ignore
 
 # run quietly, only display errors
 common:quiet --ui_event_filters=-warning,-info,-debug,-stdout,-stderr --noshow_progress

.bazelversion

@@ -1 +1 @@
-5.1.0
+6.4.0

.buildkite/cleanup-leftovers.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+echo "~~~ Cleaning up any leftovers"
+cntrs="$(docker ps -aq | grep -v -f <(docker ps -q --filter "name=go-module-proxy" --filter "name=bazel-remote-cache"))"
+[ -n "$cntrs" ] && { echo "Remove leftover containers..."; docker rm -f $cntrs; }
+echo "Remove leftover networks"
+docker network prune -f
+echo "Remove leftover volumes"
+docker volume prune -f
+
+rm -rf bazel-testlogs logs/* traces gen gen-cache /tmp/test-artifacts test-out.tar.gz

.buildkite/hooks/bazel-remote.yml

@@ -1,4 +1,5 @@
 version: "2.4"
+name: bazel_remote
 services:
   bazel-remote:
     container_name: bazel-remote-cache

.buildkite/hooks/go-module-proxy.yml

@@ -1,5 +1,6 @@
 ---
 version: "2.4"
+name: athens
 services:
   go-module-proxy:
     container_name: go-module-proxy

.buildkite/hooks/pre-command

@@ -11,16 +11,29 @@ printenv PATH
 # Install build tools on first job for this runner.
 .buildkite/provision-agent.sh
 
-# Clean up left-overs from previous run
-PRE_COMMAND_SETUP=true . .buildkite/hooks/pre-exit
-
 set -euo pipefail
 
-echo "--- Increase receive network buffer size"
+echo "~~~ Increase receive network buffer size"
 sudo sysctl -w net.core.rmem_max=1048576
 
-echo "--- Setting up bazel environment"
 
+# Export SCION_VERSION as environment variable, unless it's set from outside (e.g. as variable for this build)
+# Note that this precommand hook runs even in "step zero" of the pipeline, where we `buildkite-agent upload'.
+# With this, the SCION_VERSION can be interpolated by the agent throughout the pipeline yaml.
+if [ -z ${SCION_VERSION+x} ]; then
+    echo "~~~ Export SCION_VERSION"
+    if [ "$BUILDKITE_PIPELINE_SLUG" == "scion" ]; then
+        # Shorten the git version to omit commit information, improving cache reuse.
+        # The format of git-version is "<tag>-<number-of-commits-since-the-tag>-<commit-short-hash>"
+        # This will be shortened to "<tag>-modified-ci"
+        export SCION_VERSION=$(tools/git-version | sed 's/-.*/-modified-ci/')
+    else
+        export SCION_VERSION=$(tools/git-version)
+    fi
+    echo SCION_VERSION=${SCION_VERSION}
+fi
+
+echo "~~~ Setting up bazel environment"
 if [ -z ${BAZEL_REMOTE_S3_ACCESS_KEY_ID+x} ]; then
     echo "S3 env not set, not starting bazel remote proxy"
     exit 0
@@ -33,20 +46,17 @@ rm -f $HOME/.bazelrc
 # --nostamp is required for better caching (only on non-release jobs).
 if [ "$BUILDKITE_PIPELINE_SLUG" == "scion" ]; then
     echo "build --nostamp" > $HOME/.bazelrc
-    # Also set a fixed GIT_VERSION so that the workspace_status_command always
-    # returns the same value on CI to improve cache reuse.
-    export GIT_VERSION="ci-fixed"
 else
     echo "build --stamp" > $HOME/.bazelrc
 fi
 echo "test --test_env CI" >> $HOME/.bazelrc
 
-echo "--- Starting bazel remote cache proxy"
+echo "~~~ Starting bazel remote cache proxy"
 
 # Start bazel remote cache proxy for S3
 # Note that S3 keys are injected by buildkite, see
 # https://buildkite.com/docs/pipelines/secrets#storing-secrets-with-the-elastic-ci-stack-for-aws
-docker-compose -f .buildkite/hooks/bazel-remote.yml -p bazel_remote up -d
+docker compose -f .buildkite/hooks/bazel-remote.yml up -d
 
-echo "--- Starting go module proxy"
-docker-compose -f .buildkite/hooks/go-module-proxy.yml -p athens up -d
+echo "~~~ Starting go module proxy"
+docker compose -f .buildkite/hooks/go-module-proxy.yml up -d

.buildkite/hooks/pre-exit

@@ -1,8 +1,7 @@
 #!/bin/bash
 
-
-if [ -f ".buildkite/hooks/bazel-remote.yml" -a -z "$PRE_COMMAND_SETUP" ]; then
-    echo "--- Uploading bazel-remote and go-module-proxy logs/metrics"
+if [ -f ".buildkite/hooks/bazel-remote.yml" ]; then
+    echo "~~~ Uploading bazel-remote and go-module-proxy logs/metrics"
     curl http://localhost:8080/metrics > bazel-remote-cache.metrics
     docker logs bazel-remote-cache &> bazel-remote-cache.log
 
@@ -11,19 +10,3 @@ if [ -f ".buildkite/hooks/bazel-remote.yml" -a -z "$PRE_COMMAND_SETUP" ]; then
 
     buildkite-agent artifact upload "bazel-remote-cache.*;go-module-proxy.*"
 fi
-
-echo "--- Cleaning up the topology"
-
-./scion.sh topo_clean
-
-echo "--- Cleaning up docker containers/networks/volumes"
-cntrs="$(docker ps -aq | grep -v -f <(docker ps -q --filter "name=go-module-proxy" --filter "name=bazel-remote-cache"))"
-[ -n "$cntrs" ] && { echo "Remove leftover containers..."; docker rm -f $cntrs; }
-
-echo "Remove leftover networks"
-docker network prune -f
-echo "Remove leftover volumes"
-docker volume prune -f
-
-echo "--- Cleaning up logs and artifacts"
-rm -rf bazel-testlogs logs/* traces gen gen-cache /tmp/test-artifacts