Integrate jira-ruby into deploy tool (#739)

api/libproject/devstart.rb

@@ -1128,7 +1128,7 @@ def deploy(cmd_name, args)
 
   common = Common.new
   common.status "Running database migrations..."
-  with_cloud_proxy_and_db(gcc, service_account=op.opts.account) do |ctx|
+  with_cloud_proxy_and_db(gcc, service_account=op.opts.account, key_file=op.opts.key_file) do |ctx|
     migrate_database
     load_config(ctx.project)
 
@@ -1141,7 +1141,7 @@ def deploy(cmd_name, args)
       --version #{op.opts.version}
       #{op.opts.promote ? "--promote" : "--no-promote"}
       --quiet
-    } + (op.opts.key_file.nil? ? [] : %W{--key-file #{op.opts.key_file}})
+    }
     deploy_api(cmd_name, deploy_args)
     deploy_public_api(cmd_name, deploy_args)
   end

ci/Dockerfile.circle_build

@@ -38,3 +38,4 @@ RUN curl https://dl.google.com/cloudsql/cloud_sql_proxy.linux.amd64 > /tmp/cloud
   && sudo mv /tmp/cloud_sql_proxy /usr/local/bin && sudo chmod +x /usr/local/bin/cloud_sql_proxy
 
 RUN sudo apt-get install gradle
+RUN sudo gem install jira-ruby

deploy/libproject/deploy.rb

@@ -9,7 +9,11 @@
 
 
 DOCKER_KEY_FILE_PATH = "/creds/sa-key.json"
+
 STAGING_PROJECT = "all-of-us-rw-staging"
+STABLE_PROJECT = "all-of-us-rw-stable"
+RELEASE_MANAGED_PROJECTS = [STAGING_PROJECT, STABLE_PROJECT]
+
 VERSION_RE = /^v[[:digit:]]+-[[:digit:]]+-rc[[:digit:]]+$/
 
 # TODO(calbach): Factor these utils down into common.rb
@@ -65,6 +69,70 @@ def get_live_gae_version(project, validate_version=true)
   return v
 end
 
+def setup_and_enter_docker(cmd_name, opts)
+  common = Common.new
+  if not opts.git_version or not opts.app_version
+    if opts.project == STAGING_PROJECT
+      common.error "--git-version and --app-version are required when " +
+                   "using this script to deploy to staging. Note: this " +
+                   "should be an uncommon use case, please see the release " +
+                   "documentation for details"
+      exit 1
+    end
+    live_staging_version = get_live_gae_version(STAGING_PROJECT)
+    if not live_staging_version
+      common.error "No default staging version could be determined for " +
+                   "promotion; please investigate or else be explicit in " +
+                   "the version to promote by specifying both " +
+                   "--git-version and --app-version"
+      exit 1
+    end
+    if live_staging_version and not opts.app_version
+      common.status "--app-version defaulting to '#{live_staging_version}'; " +
+                    "found on project '#{STAGING_PROJECT}'"
+      opts.app_version = live_staging_version
+    end
+    if live_staging_version and not opts.git_version
+      common.status "--git-version defaulting to '#{live_staging_version}'; " +
+                    "found on project '#{STAGING_PROJECT}'"
+      opts.git_version = live_staging_version
+    end
+  end
+
+  # TODO: Might be nice to emit the last version creation time here as a
+  # sanity check (need to pick which service to do that for...).
+  live_version = get_live_gae_version(opts.project, validate_version=false)
+  common.status "Current live version is '#{live_version}' (project " +
+                "#{opts.project})"
+  puts "Will deploy git version '#{opts.git_version}' as App Engine " +
+       "version '#{opts.app_version}' in project '#{opts.project}'"
+  printf "Continue? (Y/n): "
+  got = STDIN.gets.chomp.strip.upcase
+  unless got == '' or got == 'Y'
+    exit 1
+  end
+
+  key_file = Tempfile.new(["#{opts.account}-key", ".json"])
+  ServiceAccountContext.new(
+    opts.project, account=opts.account, path=key_file.path).run do
+    common.run_inline %W{docker-compose build deploy}
+    common.run_inline %W{
+      docker-compose run --rm
+      -e WORKBENCH_VERSION=#{opts.git_version}
+      -v #{key_file.path}:#{DOCKER_KEY_FILE_PATH}
+      deploy deploy/project.rb #{cmd_name}
+      --account #{opts.account}
+      --project #{opts.project}
+      #{opts.promote ? "--promote" : "--no-promote"}
+      --app-version #{opts.app_version}
+      --git-version #{opts.git_version}
+      --key-file #{DOCKER_KEY_FILE_PATH}
+    } +
+      (opts.circle_url.nil? ? [] : %W{--circle-url #{opts.circle_url}}) +
+      (opts.update_jira ? [] : %W{"--no-update-jira"})
+  end
+end
+
 def deploy(cmd_name, args)
   op = WbOptionsParser.new(cmd_name, args)
   op.add_option(
@@ -79,23 +147,29 @@ def deploy(cmd_name, args)
     "default service account."
   )
   op.add_option(
-    "--key_file [key file]",
+    "--key-file [key file]",
     lambda {|opts, v| opts.key_file = v},
     "Path to a service account key file to be used for deployment"
   )
   op.add_option(
-    "--git_version [git version]",
+    "--git-version [git version]",
     lambda {|opts, v| opts.git_version = v},
     "GitHub tag or branch, e.g. 'v1-0-rc1', 'origin/master'. Branch names " +
     "must be prefixed with 'origin/'. By default, uses the current live " +
     "staging release tag (if staging is in a good state)"
   )
   op.add_option(
-    "--app_version [app version]",
+    "--app-version [app version]",
     lambda {|opts, v| opts.app_version = v},
     "App Engine version to deploy as. By default, uses the current live " +
     "staging release version (if staging is in a good state)"
   )
+  op.add_option(
+    "--no-update-jira",
+    lambda {|opts, v| opts.update_jira = false},
+    "Don't update or create a ticket in JIRA; by default will pick " +
+    "depending on the target project. On --no-promote JIRA is never updated"
+  )
   op.add_option(
     "--promote",
     lambda {|opts, v| opts.promote = true},
@@ -104,90 +178,89 @@ def deploy(cmd_name, args)
   op.add_option(
     "--no-promote",
     lambda {|opts, v| opts.promote = false},
-    "Deploy, but do not yet serve traffic from this version - DB migrations are still applied"
+    "Deploy, but do not yet serve traffic from this version - DB migrations " +
+    "are still applied"
+  )
+  op.add_option(
+    "--circle-url",
+    lambda {|opts, v| opts.circle_url = v},
+    "Circle test output URL to attach to the release tracker; only " +
+    "relevant for runs where a release ticket is created (staging)"
   )
   op.add_validator lambda {|opts| raise ArgumentError if opts.project.nil?}
   op.add_validator lambda {|opts| raise ArgumentError if opts.account.nil?}
   op.add_validator lambda {|opts| raise ArgumentError if opts.promote.nil?}
 
   op.parse.validate
 
-  common = Common.new
-  unless Workbench::in_docker?
-    if not op.opts.git_version or not op.opts.app_version
-      if op.opts.project == STAGING_PROJECT
-        common.error "--git_version and --app_version are required when " +
-                     "using this script to deploy to staging. Note: this " +
-                     "should be an uncommon use case, please see the release " +
-                     "documentation for details"
-        exit 1
-      end
-      live_staging_version = get_live_gae_version(STAGING_PROJECT)
-      if not live_staging_version
-        common.error "No default staging version could be determined for " +
-                     "promotion; please investigate or else be explicit in " +
-                     "the version to promote by specifying both " +
-                     "--git_version and --app_version"
-        exit 1
-      end
-      if live_staging_version and not op.opts.app_version
-        common.status "--app_version defaulting to '#{live_staging_version}'; " +
-                      "found on project '#{STAGING_PROJECT}'"
-        op.opts.app_version = live_staging_version
-      end
-      if live_staging_version and not op.opts.git_version
-        common.status "--git_version defaulting to '#{live_staging_version}'; " +
-                      "found on project '#{STAGING_PROJECT}'"
-        op.opts.git_version = live_staging_version
-      end
-    end
-
-    # TODO: Might be nice to emit the last version creation time here as a
-    # sanity check (need to pick which service to do that for...).
-    live_version = get_live_gae_version(op.opts.project, validate_version=false)
-    common.status "Current live version is '#{live_version}' (project " +
-                  "#{op.opts.project})"
-    puts "Will deploy git version '#{op.opts.git_version}' as App Engine " +
-         "version '#{op.opts.app_version}' in project '#{op.opts.project}'"
-    printf "Continue? (Y/n): "
-    got = STDIN.gets.chomp.strip.upcase
-    unless got == '' or got == 'Y'
-      exit 1
-    end
+  if op.opts.update_jira.nil?
+     op.opts.update_jira = RELEASE_MANAGED_PROJECTS.include? op.opts.project
+  end
+  op.opts.update_jira = op.opts.update_jira and op.opts.promote
 
-    key_file = Tempfile.new(["#{op.opts.account}-key", ".json"])
-    ServiceAccountContext.new(
-      op.opts.project, account=op.opts.account, path=key_file.path).run do
-      common.run_inline %W{docker-compose build deploy}
-      common.run_inline %W{
-        docker-compose run --rm
-        -e WORKBENCH_VERSION=#{op.opts.git_version}
-        -v #{key_file.path}:#{DOCKER_KEY_FILE_PATH}
-        deploy deploy/project.rb #{cmd_name}
-        --account #{op.opts.account}
-        --project #{op.opts.project}
-        #{op.opts.promote ? "--promote" : "--no-promote"}
-        --app_version #{op.opts.app_version}
-        --git_version #{op.opts.git_version}
-        --key_file #{DOCKER_KEY_FILE_PATH}
-      }
-      return
-    end
+  unless Workbench::in_docker?
+    return setup_and_enter_docker(cmd_name, op.opts)
   end
 
   # Everything following runs only within Docker.
+  # Only require Jira stuff within Docker to avoid burdening the user with local
+  # workstation Ruby gem setup.
+  require_relative 'jirarelease'
+
   if op.opts.key_file.nil?
-    raise ArgumentError.new("--key_file is required when running within docker")
+    raise ArgumentError.new("--key-file is required when running within docker")
   end
   if op.opts.app_version.nil?
-    raise ArgumentError.new("--app_version is required when running within docker")
+    raise ArgumentError.new("--app-version is required when running within docker")
   end
   if op.opts.git_version.nil?
-    raise ArgumentError.new("--git_version is required when running within docker")
+    raise ArgumentError.new("--git-version is required when running within docker")
   end
+  common = Common.new
   common.run_inline %W{gcloud auth activate-service-account -q --key-file #{op.opts.key_file}}
 
-  # TODO: Create/update the Jira ticket.
+  jira_client = nil
+  create_ticket = false
+  from_version = nil
+  maybe_log_jira = lambda { |msg| common.status msg }
+  if op.opts.update_jira
+    if not VERSION_RE.match(op.opts.app_version) or
+      op.opts.app_version != op.opts.git_version
+      raise RuntimeError.new "for releases, the --git_version and " +
+                             "--app_version should be equal and should be a " +
+                             "release tag (e.g. v0-1-rc1); you shouldn't " +
+                             "bypass this, but if you need to you can pass " +
+                             "--no-update-jira"
+    end
+
+    # We're either creating a new ticket (staging), or commenting on an existing
+    # release ticket (stable, prod).
+    jira_client = JiraReleaseClient.from_gcs_creds(op.opts.project)
+    if op.opts.update_jira and op.opts.project == STAGING_PROJECT
+      create_ticket = true
+      from_version = get_live_gae_version(STAGING_PROJECT)
+      if not from_version
+        # Alternatively, we could support a --from_version flag
+        raise RuntimeError "could not determine live staging version, and " +
+                           "therefore could not generate a delta commit log; " +
+                           "please manually deploy staging with the old " +
+                           "version and supply --no-update-jira, then retry"
+      end
+    else
+      maybe_log_jira = lambda { |msg|
+        begin
+          jira_client.comment_ticket(op.opts.app_version, msg)
+        rescue StandardError => e
+          common.error "comment_ticket failed: #{e}"
+        end
+      }
+    end
+  end
+
+  # TODO: Add more granular logging, e.g. call deploy natively and pass an
+  # optional log writer. Also rescue and log if deployment fails.
+  maybe_log_jira.call "'#{op.opts.project}': Beginning deploy of api and " +
+                      "public-api services (including DB updates)"
   common.run_inline %W{
     ../api/project.rb deploy
       --project #{op.opts.project}
@@ -197,6 +270,8 @@ def deploy(cmd_name, args)
       #{op.opts.promote ? "--promote" : "--no-promote"}
   }
 
+  maybe_log_jira.call "'#{op.opts.project}': completed api and public-api " +
+                      "service deployment; beginning deploy of UI service"
   common.run_inline %W{
     ../ui/project.rb deploy-ui
       --project #{op.opts.project}
@@ -206,6 +281,12 @@ def deploy(cmd_name, args)
       #{op.opts.promote ? "--promote" : "--no-promote"}
       --quiet
   }
+  maybe_log_jira.call "'#{op.opts.project}': completed UI service deployment"
+
+  if create_ticket
+    jira_client.create_ticket(op.opts.project, live_staging_version,
+                              op.opts.git_version, op.opts.circle_url)
+  end
 end
 
 Common.register_command({