From 61cc75189b70c453a1e547804c6ab7d8e808815c Mon Sep 17 00:00:00 2001 From: CH Albach Date: Tue, 24 Apr 2018 15:18:05 -0700 Subject: [PATCH] Integrate jira-ruby into deploy tool (#739) --- api/libproject/devstart.rb | 4 +- ci/Dockerfile.circle_build | 1 + deploy/libproject/deploy.rb | 217 +++++++++++++++++++++---------- deploy/libproject/jirarelease.rb | 153 ++++++++++++++++++++++ 4 files changed, 305 insertions(+), 70 deletions(-) create mode 100644 deploy/libproject/jirarelease.rb diff --git a/api/libproject/devstart.rb b/api/libproject/devstart.rb index 1990e2e0ebb..c6896242b73 100644 --- a/api/libproject/devstart.rb +++ b/api/libproject/devstart.rb @@ -1128,7 +1128,7 @@ def deploy(cmd_name, args) common = Common.new common.status "Running database migrations..." - with_cloud_proxy_and_db(gcc, service_account=op.opts.account) do |ctx| + with_cloud_proxy_and_db(gcc, service_account=op.opts.account, key_file=op.opts.key_file) do |ctx| migrate_database load_config(ctx.project) @@ -1141,7 +1141,7 @@ def deploy(cmd_name, args) --version #{op.opts.version} #{op.opts.promote ? "--promote" : "--no-promote"} --quiet - } + (op.opts.key_file.nil? ? [] : %W{--key-file #{op.opts.key_file}}) + } deploy_api(cmd_name, deploy_args) deploy_public_api(cmd_name, deploy_args) end diff --git a/ci/Dockerfile.circle_build b/ci/Dockerfile.circle_build index 026b3724730..fa22077ad50 100644 --- a/ci/Dockerfile.circle_build +++ b/ci/Dockerfile.circle_build @@ -38,3 +38,4 @@ RUN curl https://dl.google.com/cloudsql/cloud_sql_proxy.linux.amd64 > /tmp/cloud && sudo mv /tmp/cloud_sql_proxy /usr/local/bin && sudo chmod +x /usr/local/bin/cloud_sql_proxy RUN sudo apt-get install gradle +RUN sudo gem install jira-ruby \ No newline at end of file diff --git a/deploy/libproject/deploy.rb b/deploy/libproject/deploy.rb index 2c05f053f98..6e99abb5190 100644 --- a/deploy/libproject/deploy.rb +++ b/deploy/libproject/deploy.rb @@ -9,7 +9,11 @@ DOCKER_KEY_FILE_PATH = "/creds/sa-key.json" + STAGING_PROJECT = "all-of-us-rw-staging" +STABLE_PROJECT = "all-of-us-rw-stable" +RELEASE_MANAGED_PROJECTS = [STAGING_PROJECT, STABLE_PROJECT] + VERSION_RE = /^v[[:digit:]]+-[[:digit:]]+-rc[[:digit:]]+$/ # TODO(calbach): Factor these utils down into common.rb @@ -65,6 +69,70 @@ def get_live_gae_version(project, validate_version=true) return v end +def setup_and_enter_docker(cmd_name, opts) + common = Common.new + if not opts.git_version or not opts.app_version + if opts.project == STAGING_PROJECT + common.error "--git-version and --app-version are required when " + + "using this script to deploy to staging. Note: this " + + "should be an uncommon use case, please see the release " + + "documentation for details" + exit 1 + end + live_staging_version = get_live_gae_version(STAGING_PROJECT) + if not live_staging_version + common.error "No default staging version could be determined for " + + "promotion; please investigate or else be explicit in " + + "the version to promote by specifying both " + + "--git-version and --app-version" + exit 1 + end + if live_staging_version and not opts.app_version + common.status "--app-version defaulting to '#{live_staging_version}'; " + + "found on project '#{STAGING_PROJECT}'" + opts.app_version = live_staging_version + end + if live_staging_version and not opts.git_version + common.status "--git-version defaulting to '#{live_staging_version}'; " + + "found on project '#{STAGING_PROJECT}'" + opts.git_version = live_staging_version + end + end + + # TODO: Might be nice to emit the last version creation time here as a + # sanity check (need to pick which service to do that for...). + live_version = get_live_gae_version(opts.project, validate_version=false) + common.status "Current live version is '#{live_version}' (project " + + "#{opts.project})" + puts "Will deploy git version '#{opts.git_version}' as App Engine " + + "version '#{opts.app_version}' in project '#{opts.project}'" + printf "Continue? (Y/n): " + got = STDIN.gets.chomp.strip.upcase + unless got == '' or got == 'Y' + exit 1 + end + + key_file = Tempfile.new(["#{opts.account}-key", ".json"]) + ServiceAccountContext.new( + opts.project, account=opts.account, path=key_file.path).run do + common.run_inline %W{docker-compose build deploy} + common.run_inline %W{ + docker-compose run --rm + -e WORKBENCH_VERSION=#{opts.git_version} + -v #{key_file.path}:#{DOCKER_KEY_FILE_PATH} + deploy deploy/project.rb #{cmd_name} + --account #{opts.account} + --project #{opts.project} + #{opts.promote ? "--promote" : "--no-promote"} + --app-version #{opts.app_version} + --git-version #{opts.git_version} + --key-file #{DOCKER_KEY_FILE_PATH} + } + + (opts.circle_url.nil? ? [] : %W{--circle-url #{opts.circle_url}}) + + (opts.update_jira ? [] : %W{"--no-update-jira"}) + end +end + def deploy(cmd_name, args) op = WbOptionsParser.new(cmd_name, args) op.add_option( @@ -79,23 +147,29 @@ def deploy(cmd_name, args) "default service account." ) op.add_option( - "--key_file [key file]", + "--key-file [key file]", lambda {|opts, v| opts.key_file = v}, "Path to a service account key file to be used for deployment" ) op.add_option( - "--git_version [git version]", + "--git-version [git version]", lambda {|opts, v| opts.git_version = v}, "GitHub tag or branch, e.g. 'v1-0-rc1', 'origin/master'. Branch names " + "must be prefixed with 'origin/'. By default, uses the current live " + "staging release tag (if staging is in a good state)" ) op.add_option( - "--app_version [app version]", + "--app-version [app version]", lambda {|opts, v| opts.app_version = v}, "App Engine version to deploy as. By default, uses the current live " + "staging release version (if staging is in a good state)" ) + op.add_option( + "--no-update-jira", + lambda {|opts, v| opts.update_jira = false}, + "Don't update or create a ticket in JIRA; by default will pick " + + "depending on the target project. On --no-promote JIRA is never updated" + ) op.add_option( "--promote", lambda {|opts, v| opts.promote = true}, @@ -104,7 +178,14 @@ def deploy(cmd_name, args) op.add_option( "--no-promote", lambda {|opts, v| opts.promote = false}, - "Deploy, but do not yet serve traffic from this version - DB migrations are still applied" + "Deploy, but do not yet serve traffic from this version - DB migrations " + + "are still applied" + ) + op.add_option( + "--circle-url", + lambda {|opts, v| opts.circle_url = v}, + "Circle test output URL to attach to the release tracker; only " + + "relevant for runs where a release ticket is created (staging)" ) op.add_validator lambda {|opts| raise ArgumentError if opts.project.nil?} op.add_validator lambda {|opts| raise ArgumentError if opts.account.nil?} @@ -112,82 +193,74 @@ def deploy(cmd_name, args) op.parse.validate - common = Common.new - unless Workbench::in_docker? - if not op.opts.git_version or not op.opts.app_version - if op.opts.project == STAGING_PROJECT - common.error "--git_version and --app_version are required when " + - "using this script to deploy to staging. Note: this " + - "should be an uncommon use case, please see the release " + - "documentation for details" - exit 1 - end - live_staging_version = get_live_gae_version(STAGING_PROJECT) - if not live_staging_version - common.error "No default staging version could be determined for " + - "promotion; please investigate or else be explicit in " + - "the version to promote by specifying both " + - "--git_version and --app_version" - exit 1 - end - if live_staging_version and not op.opts.app_version - common.status "--app_version defaulting to '#{live_staging_version}'; " + - "found on project '#{STAGING_PROJECT}'" - op.opts.app_version = live_staging_version - end - if live_staging_version and not op.opts.git_version - common.status "--git_version defaulting to '#{live_staging_version}'; " + - "found on project '#{STAGING_PROJECT}'" - op.opts.git_version = live_staging_version - end - end - - # TODO: Might be nice to emit the last version creation time here as a - # sanity check (need to pick which service to do that for...). - live_version = get_live_gae_version(op.opts.project, validate_version=false) - common.status "Current live version is '#{live_version}' (project " + - "#{op.opts.project})" - puts "Will deploy git version '#{op.opts.git_version}' as App Engine " + - "version '#{op.opts.app_version}' in project '#{op.opts.project}'" - printf "Continue? (Y/n): " - got = STDIN.gets.chomp.strip.upcase - unless got == '' or got == 'Y' - exit 1 - end + if op.opts.update_jira.nil? + op.opts.update_jira = RELEASE_MANAGED_PROJECTS.include? op.opts.project + end + op.opts.update_jira = op.opts.update_jira and op.opts.promote - key_file = Tempfile.new(["#{op.opts.account}-key", ".json"]) - ServiceAccountContext.new( - op.opts.project, account=op.opts.account, path=key_file.path).run do - common.run_inline %W{docker-compose build deploy} - common.run_inline %W{ - docker-compose run --rm - -e WORKBENCH_VERSION=#{op.opts.git_version} - -v #{key_file.path}:#{DOCKER_KEY_FILE_PATH} - deploy deploy/project.rb #{cmd_name} - --account #{op.opts.account} - --project #{op.opts.project} - #{op.opts.promote ? "--promote" : "--no-promote"} - --app_version #{op.opts.app_version} - --git_version #{op.opts.git_version} - --key_file #{DOCKER_KEY_FILE_PATH} - } - return - end + unless Workbench::in_docker? + return setup_and_enter_docker(cmd_name, op.opts) end # Everything following runs only within Docker. + # Only require Jira stuff within Docker to avoid burdening the user with local + # workstation Ruby gem setup. + require_relative 'jirarelease' + if op.opts.key_file.nil? - raise ArgumentError.new("--key_file is required when running within docker") + raise ArgumentError.new("--key-file is required when running within docker") end if op.opts.app_version.nil? - raise ArgumentError.new("--app_version is required when running within docker") + raise ArgumentError.new("--app-version is required when running within docker") end if op.opts.git_version.nil? - raise ArgumentError.new("--git_version is required when running within docker") + raise ArgumentError.new("--git-version is required when running within docker") end + common = Common.new common.run_inline %W{gcloud auth activate-service-account -q --key-file #{op.opts.key_file}} - # TODO: Create/update the Jira ticket. + jira_client = nil + create_ticket = false + from_version = nil + maybe_log_jira = lambda { |msg| common.status msg } + if op.opts.update_jira + if not VERSION_RE.match(op.opts.app_version) or + op.opts.app_version != op.opts.git_version + raise RuntimeError.new "for releases, the --git_version and " + + "--app_version should be equal and should be a " + + "release tag (e.g. v0-1-rc1); you shouldn't " + + "bypass this, but if you need to you can pass " + + "--no-update-jira" + end + + # We're either creating a new ticket (staging), or commenting on an existing + # release ticket (stable, prod). + jira_client = JiraReleaseClient.from_gcs_creds(op.opts.project) + if op.opts.update_jira and op.opts.project == STAGING_PROJECT + create_ticket = true + from_version = get_live_gae_version(STAGING_PROJECT) + if not from_version + # Alternatively, we could support a --from_version flag + raise RuntimeError "could not determine live staging version, and " + + "therefore could not generate a delta commit log; " + + "please manually deploy staging with the old " + + "version and supply --no-update-jira, then retry" + end + else + maybe_log_jira = lambda { |msg| + begin + jira_client.comment_ticket(op.opts.app_version, msg) + rescue StandardError => e + common.error "comment_ticket failed: #{e}" + end + } + end + end + + # TODO: Add more granular logging, e.g. call deploy natively and pass an + # optional log writer. Also rescue and log if deployment fails. + maybe_log_jira.call "'#{op.opts.project}': Beginning deploy of api and " + + "public-api services (including DB updates)" common.run_inline %W{ ../api/project.rb deploy --project #{op.opts.project} @@ -197,6 +270,8 @@ def deploy(cmd_name, args) #{op.opts.promote ? "--promote" : "--no-promote"} } + maybe_log_jira.call "'#{op.opts.project}': completed api and public-api " + + "service deployment; beginning deploy of UI service" common.run_inline %W{ ../ui/project.rb deploy-ui --project #{op.opts.project} @@ -206,6 +281,12 @@ def deploy(cmd_name, args) #{op.opts.promote ? "--promote" : "--no-promote"} --quiet } + maybe_log_jira.call "'#{op.opts.project}': completed UI service deployment" + + if create_ticket + jira_client.create_ticket(op.opts.project, live_staging_version, + op.opts.git_version, op.opts.circle_url) + end end Common.register_command({ diff --git a/deploy/libproject/jirarelease.rb b/deploy/libproject/jirarelease.rb new file mode 100644 index 00000000000..7d08fa380f4 --- /dev/null +++ b/deploy/libproject/jirarelease.rb @@ -0,0 +1,153 @@ +# Wraps the jira-ruby library to support creating and appending to All of Us +# release tickets. Requires basic auth credentials. The initial ticket is +# populated with a formatted git commit log since the last version. +# +# Note, this script is a Ruby analog to the RDR python release scripts: +# - https://github.com/all-of-us/raw-data-repository/blob/master/ci/release_notes.py +# - https://github.com/all-of-us/raw-data-repository/blob/master/rest-api/tools/update_release_tracker.py + +require "open3" +require "jira-ruby" + +require_relative "../../aou-utils/utils/common" + + +REPO_BASE_URL = "https://github.com/all-of-us/workbench" + +# Formatting for release notes in JIRA comments. +# Note that JIRA auto-linkifies JIRA IDs, so avoid using commit message text in a link. +LOG_LINE_FORMAT = "--format=* [%aN %ad|" + REPO_BASE_URL + "/commit/%h] %s" +# Overall release notes template. +RELEASE_NOTES_T = """h1. Release Notes for %{current} +h2. deployed to %{project}, listing changes since %{prev} +%{history} +""" + +JIRA_INSTANCE_URL = "https://precisionmedicineinitiative.atlassian.net/" +JIRA_PROJECT_NAME = "PD" + +# TODO(calbach): Factor these utils down into common.rb +def capture_stdout(cmd) + # common.capture_stdout suppresses stderr, which is not desired. + out, _ = Open3.capture2(*cmd) + return out +end + +def yellow_term_text(text) + "\033[0;33m#{text}\033[0m" +end + +def warning(text) + STDERR.puts yellow_term_text(text) +end + +def linkify_pull_request_ids(text) + # Converts all substrings like "(#123)" to links to pull requests. + return text.gsub( + /\(#([0-9]+)\)/, + '([#\1|' + REPO_BASE_URL + '/pull/\1])') +end + +def get_release_notes_between_tags(project, from_tag, to_tag) + """Formats release notes for JIRA from commit messages, between the two tags.""" + commit_messages = capture_stdout( + ['git', 'log', "#{from_tag}..#{to_tag}", LOG_LINE_FORMAT]) + if not commit_messages + raise RuntimeError.new "failed to retrieve commits" + end + + return RELEASE_NOTES_T % { + :current => to_tag, + :project => project, + :prev => from_tag, + :history => linkify_pull_request_ids(commit_messages), + } +end + +def format_jira_error(e) + return "JIRA request failed with #{e.response.code} #{e.response.message}: " + + e.response.body +end + +class JiraReleaseClient + def initialize(username, password) + # Set :http_debug => true to see outgoing JIRA requests + @client = JIRA::Client.new({ + :site => JIRA_INSTANCE_URL, + :context_path => "", + :username => username, + :password => password, + :auth_type => :basic, + }) + end + + def self.from_gcs_creds(project) + gcs_uri = "gs://#{project}-credentials/jira-login.json" + jira_creds = capture_stdout(['gsutil', 'cat', gcs_uri]) + if not jira_creds + raise RuntimeError.new "failed to read JIRA login from '#{gcs_uri}'" + end + jira_json = JSON.parse(jira_json) + return JiraReleaseClient.new(jira_json['username'], jira_json['password']) + end + + def ticket_summary(tag) + return "Release tracker: Workbench #{tag}" + end + + def create_ticket(project, from_tag, to_tag, circle_url = nil) + # Adds release notes to a new or existing JIRA issue. + summary = ticket_summary(to_tag) + description = get_release_notes_between_tags(project, from_tag, to_tag) + if circle_url + description += "\nCircle test output: #{circle_url}" + end + + jira_project = @client.Project.find(JIRA_PROJECT_NAME).id + issue = @client.Issue.build + begin + issue.save!({"fields" => { + "project" => {"id" => jira_project}, + "summary" => summary, + "description" => description, + "issuetype" => { + "name" => "Task" + } + } + }) + rescue JIRA::HTTPError => e + raise RuntimeError.new format_jira_error(e) + end + + Common.new.status "Created [#{issue.key}] with release notes for #{to_tag}" + end + + def comment_ticket(tag, msg) + common = Common.new + + summary = ticket_summary(tag) + begin + issues = @client.Issue.jql( + "project = \"#{JIRA_PROJECT_NAME}\" AND " + + "summary ~ \"#{summary}\" ORDER BY created DESC") + rescue JIRA::HTTPError => e + raise RuntimeError.new format_jira_error(e) + end + if issues.empty? + raise StandardError.new "no JIRA ticket found for summary \"#{summary}\"" + end + if issues.length > 1 + warning "Found multiple release tracker matches, using newest: " + + issues.map { |iss| "[#{iss.key}] #{iss.fields['summary']}" }.join(', ') + end + issue = issues.first + comment = issue.comments.build + begin + comment.save!(:body => msg) + rescue JIRA::HTTPError => e + raise RuntimeError.new format_jira_error(e) + end + + common.status "Added comment \"#{msg}\" to issue [#{issue.key}]" + end +end