Skip to content

Commit

Permalink
Merge pull request #1857 from hyperledger/develop
Browse files Browse the repository at this point in the history 
Iroha v1.4-rc.2
  • Loading branch information
@iceseer
iceseer authored Jan 31, 2022
2 parents 653c61f + 842491c commit 310d1cd
Show file tree
Hide file tree
Showing 45 changed files with 957 additions and 525 deletions.
112 changes: 24 additions & 88 deletions .github/build-iroha1.src.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,6 @@ jobs:
## anyway please read .github/README.md
check_workflow_yaml_coressponds_to_src_yaml:
runs-on: ubuntu-20.04 #ubuntu-latest
#container: ubuntu:latest ## This is required as barrier between AWS-hosted runners and GitHub-hosted runners - they have different set of software, so run in container
name: Check if github workflows were properly made from sources
steps:
- &step_detect_commented_pr
Expand Down Expand Up @@ -232,14 +231,6 @@ jobs:
matrix_dockerimage_release: ${{steps.matrixes.outputs.matrix_dockerimage_release}}
matrix_dockerimage_debug: ${{steps.matrixes.outputs.matrix_dockerimage_debug}}

buildspec_ubuntu: ${{steps.matrixes.outputs.buildspec_ubuntu}}
buildspec_ubuntu_debug: ${{steps.matrixes.outputs.buildspec_ubuntu_release}}
buildspec_ubuntu_release: ${{steps.matrixes.outputs.buildspec_ubuntu_debug}}
buildspec_macos: ${{steps.matrixes.outputs.buildspec_macos}}
buildspec_windows: ${{steps.matrixes.outputs.buildspec_windows}}
buildspec_dockerimage_release: ${{steps.matrixes.outputs.buildspec_dockerimage_release}}
buildspec_dockerimage_debug: ${{steps.matrixes.outputs.buildspec_dockerimage_debug}}

## Build docker image named 'hyperledger/iroha-builder' with all stuff to compile iroha and its dependancies
## The result docker image is pushed with tags :pr-NUMBER, :commit-HASH, :branch-name, :tag-name,
## and conditional tags :edge for development branch, and :latest for git-tags.
Expand Down Expand Up @@ -318,18 +309,11 @@ jobs:
type=ref,event=pr
type=ref,event=tag
type=schedule
type=edge,branch=develop
# type=semver,pattern={{version}}
# type=semver,pattern={{major}}.{{minor}}
# type=sha,prefix=commit-,format=short
# type=sha,prefix=commit-,format=long
## Docker image will be pushed with tags:
## - hash of file Dockerfile.builder
## - branchname, when branch is pushed
## - pr-NUMBER, when pushed to PR
## - git tag when tag is pushed
## - semver like 1.2.3 and 1.2 when tag vX.X.X is pushed
## - tag 'edge' when branch support/1.2.x is pushed
## - schedule, see the docs
- &step_docker_meta_ghcr
<<: *step_docker_meta
Expand Down Expand Up @@ -380,29 +364,16 @@ jobs:
name: Check if dockertaghash exists in remote registry
id: dockertag_already
run: |
## FIXME page size is 100 and cannot be more, this needs to be extended in loop asking tags till the end
exists=$( curl -fL https://hub.docker.com/v2/repositories/$DOCKERHUB_ORG/iroha-builder/tags |
jq 'any( .results[]|.name == "${{env.dockertag}}" ; .)' )
echo "::set-output name=exists::$exists"
if test $exists = true ;then
tag=$dockertag
else
tag=edge
fi
container="$DOCKERHUB_ORG/iroha-builder:$tag"
echo "::set-output name=container::$container"
echo "::set-output name=container_tag::$tag"
echo "::set-output name=container::$DOCKERHUB_ORG/iroha-builder:$dockertag"
-
if: ${{ steps.dockertag_already.outputs.container_tag != env.dockertag ||
steps.dockertag_already.outputs.container == '' }}
name: Possible ERROR, edited but not pushed
name: Possible ERROR, Dockerfile edited but image cannot be pushed
if: ${{ steps.docker_login.outcome != 'success' || steps.build_and_push.outcome != 'success' }}
env:
container: ${{steps.dockertag_already.outputs.container}}
dockertag: ${{env.dockertag}}
run: |
cat <<END
::error::Running on container '$container', YOUR CHANGES TO Dockerfile.builder WERE NOT APPLYED.
::error::CHANGES TO Dockerfile.builder WERE NOT APPLYED.
It seems container with tag '$dockertag' was not pushed to registry and does not exist remotely.
The most possible reason is GitHub secrets are inaccessable to PRs from public forks.
Expand All @@ -425,12 +396,9 @@ jobs:
## WARN secret dropped from output!, output may not contain secret,
## and secret cannot be used in job:container directly, and there is no github non-secret variables...
## if dockertag is already pushed then use it. But let it be empty when tag does not exist remotely.
dockertag: ${{steps.dockertag.outputs.dockertag}}
_dockertag: :${{steps.dockertag.outputs.dockertag}}
dockertag: ${{steps.dockertag.outputs.dockertag}}
container: ${{steps.dockertag_already.outputs.container}}
pushed: ${{ steps.docker_login.outcome == 'success' && steps.build_and_push.outcome == 'success' }}
dockertag_already_exists: ${{steps.dockertag_already.outputs.exists}}
container: ${{steps.dockertag_already.outputs.container}}
container_tag: ${{steps.dockertag_already.outputs.container_tag}}

## Build iroha in a container made of the image earlier prepared
## Result artifacts are
Expand All @@ -440,9 +408,8 @@ jobs:
needs:
- Docker-iroha-builder
- generate_matrixes
runs-on: [ self-hosted, Linux ] #ubuntu-latest
## Container is taken from previous job
container: #&container
runs-on: [ self-hosted, Linux ]
container: ## Container is taken from previous job
image: &container_image ${{needs.Docker-iroha-builder.outputs.container}}
options: --user root
strategy: &strategy_ubuntu_debug
Expand All @@ -461,36 +428,9 @@ jobs:
${{ toJson(needs) }}
END
-
run: test -n "$container"
env:
container: *container_image
-
if: ${{ needs.Docker-iroha-builder.outputs.container_tag != needs.Docker-iroha-builder.outputs.dockertag ||
needs.Docker-iroha-builder.outputs.container == '' }}
name: Possible WARNING
env:
container: *container_image
dockertag: ${{needs.Docker-iroha-builder.outputs.dockertag}}
run: |
cat <<END
::warning::Running on container '$container', YOUR CHANGES TO Dockerfile.builder WERE NOT APPLYED.
It seems container with tag '$dockertag' was not pushed to registry and does not exist remotely.
The most possible reason is GitHub secrets are inaccessable to PRs from public forks.
$(test ${{github.event.pull_request.head.repo.full_name}} != ${{github.event.pull_request.base.repo.full_name}} \
&& echo -n "SECRETS ARE NOT EXPOSED TO FORKS" || echo -n "SECRETS AVAILABLE")
**Consider to open PR from the same organization.**
What we know about this build:
- PR URL is ${{github.event.pull_request.html_url}}
- head repo is '${{github.event.pull_request.head.repo.full_name}}'
- base repo is '${{github.event.pull_request.base.repo.full_name}}'
- See more information in previous step 'Show context'
Please ask @kuvaldini on GitHub or in Telegram if you need help.
END
run: test -n "$container"
- *step_system_info
- *step_build_info
- &step_env_from_buildspec
Expand Down Expand Up @@ -541,7 +481,6 @@ jobs:
- &step_store_ccache_stats
run: ccache --show-stats | tee /tmp/ccache-stats
- &step_vcpkg_cache
#if: ${{false}} ## This works bad when something patched or something updated, seems they does not recalc hash of changed packages. See todos in the begining of file.
## Read the docs https://vcpkg.readthedocs.io/en/latest/users/binarycaching/ https://github.com/microsoft/vcpkg/blob/master/docs/users/binarycaching.md
name: Restore cache Vcpkg binarycache ## NOTE not useng NuGet because on ubuntu nuget needs mono of 433MB, unusable.
uses: actions/cache@v2
Expand Down Expand Up @@ -810,7 +749,6 @@ jobs:
- &step_brew_postgres
name: Install Postgres on MacOS
run: brew install postgresql
## ToDo may be optimize, i.e. cache package
- <<: *step_ctest
timeout-minutes: 70

Expand All @@ -832,10 +770,7 @@ jobs:
strategy:
fail-fast: false
matrix: ${{ fromJSON( needs.generate_matrixes.outputs.matrix_windows ) }}
#if: ${{ false }} ##FIXME Somehow cmake fails to find GTest and others
if: ${{ false && ( fromJSON( needs.generate_matrixes.outputs.matrix_windows ).include[0] ) }}
# matrix:
# BuildYype: [ Debug ] #,Release, RelWithDebInfo
defaults:
run:
shell: bash #pwsh
Expand Down Expand Up @@ -872,17 +807,16 @@ jobs:
- *step_cmake_configure
- *step_cmake_build
- *step_always_after_build
-
name: Install Postgres on Windows
- name: Install Postgres on Windows
run: choco install postgresql
# - *step_ctest

## Build and publish docker image named 'hyperledger/iroha' with irohad and iroha tools inside.
## The result docker image is pushed with tags :pr-NUMBER, :commit-HASH, :branch-name, :tag-name,
## and conditional tags :edge (for development branches) and :latest (for git-tags)
## The result docker image is pushed with tags :branch_name, :pr-NUMBER, :tag_name,
## and :latest (for git-tags).
## Those docker image tags could be extended with suffixes with compiler and build type like
## -gcc10, -clang, -debug, -gcc10-debug.
## Result image name could look like: hyperledger/iroha:pr-1117, hyperledger/iroha-burrow:commit-XXXXX-debug
## -gcc10, -clang, -debug, -gcc10-debug. Like :latest-debug, :main-debug, :1.3.0-gcc10-debug.
## Result image name could look like: hyperledger/iroha:pr-1117, hyperledger/iroha:nightly.
## Note: image is push only when DockerHub login-token pair available - not to PRs from forks
docker-R: &job_docker_image_release
needs:
Expand All @@ -906,26 +840,28 @@ jobs:
- *step_detect_commented_pr
- *step_checkout
- *step_artifact_suffix
- &step_download_artifact_iroha_deb
name: Download artifact
- name: Download artifact
uses: actions/download-artifact@v2
with:
name: iroha-deb${{env.ARTIFACT_SUFFIX}}
- &rename_artifact_deb
name: Rename artifact debs
- name: Rename artifact debs
run: |
mv *iroha_shepherd.deb docker/release/iroha_shepherd.deb
mv *irohad.deb docker/release/iroha.deb
- &step_dockertag_release
<<: *step_docker_tag
- <<: *step_docker_tag
env:
dockertag: ${{ hashFiles('docker/release/**') }}
- <<: *step_docker_meta
with: &step_docker_release_meta_with
<<: *step_docker_meta_with
images: |
${{ env.DOCKERHUB_ORG }}/${{ env.IMAGE_NAME }}${{ env._uses_suffix }}
flavor: suffix=${{env._compiler_suffix}}${{env._debug_suffix}}
tags: |
type=ref,event=branch
type=ref,event=pr
type=ref,event=tag
type=schedule
flavor: |
suffix=${{env._compiler_suffix}}${{env._debug_suffix}},onlatest=true
#maybetodo flavor: prefix=${{ env.USES_PREFIX }} ## In case creating repository hyperledger/iroha-burrow denied, Use tag prefix hyperledger/iroha:burrow-xxxx
- <<: *step_docker_meta_ghcr
with:
Expand Down
24 changes: 4 additions & 20 deletions .github/chatops-gen-matrix.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ handle_user_line(){
return
fi
if [[ "${1:-}" != '/build' ]] ;then
echowarn "Line skipped, should start with '/build'"
echowarn "Line skipped, should start with '/build' : '${1:-}'"
return
fi
shift
Expand Down Expand Up @@ -129,9 +129,9 @@ handle_user_line(){
done
done
done
if test "$used_compilers" = ''; then
echowarn "No available compilers for '$os' among '$compilers', available: '$AVAILABLE_compilers'"
fi
# if test "$used_compilers" = ''; then
# echowarn "No available compilers for '$os' among '$compilers', available: '$AVAILABLE_compilers'"
# fi
done
}

Expand Down Expand Up @@ -160,12 +160,6 @@ rm -f $ignored


to_json(){
# echo "{
# os:\"$1\",
# cc:\"$2\",
# BuildType:\"$3\",
# CMAKE_USE:\"$( [[ "$4" = normal ]] || echo "-DUSE_${4^^}=ON" )\"
# }"
echo "{buildspec:\"$@\"}"
}
to_json_multiline(){
Expand All @@ -185,16 +179,6 @@ json_include(){
MATRIX="$(echo "$MATRIX" | sed '/^$/d' | sort -uV)"
echo "$MATRIX"

echo "$MATRIX" >buildspec
echo "$MATRIX" | awk -v IGNORECASE=1 '/ubuntu/' >buildspec_ubuntu
echo "$MATRIX" | awk -v IGNORECASE=1 '/ubuntu/ && /release/' >buildspec_ubuntu_release
echo "$MATRIX" | awk -v IGNORECASE=1 '/ubuntu/ && /debug/' >buildspec_ubuntu_debug
echo "$MATRIX" | awk -v IGNORECASE=1 '/macos/' >buildspec_macos
echo "$MATRIX" | awk -v IGNORECASE=1 '/windows/' >buildspec_windows
## Build Docker images only with GCC-9 (mainstream compiler)
echo "$MATRIX" | awk -v IGNORECASE=1 '/ubuntu/ && /release/ && /gcc-9/' >buildspec_dockerimage_release
echo "$MATRIX" | awk -v IGNORECASE=1 '/ubuntu/ && /debug/ && /gcc-9/' >buildspec_dockerimage_debug

echo "$MATRIX" | json_include >matrix
echo "$MATRIX" | awk -v IGNORECASE=1 '/ubuntu/' | json_include >matrix_ubuntu
echo "$MATRIX" | awk -v IGNORECASE=1 '/ubuntu/ && /release/' | json_include >matrix_ubuntu_release
Expand Down
Loading

0 comments on commit 310d1cd

Please sign in to comment.