diff --git a/.github/workflows/ship_it.yml b/.github/workflows/ship_it.yml new file mode 100644 index 0000000..5627e80 --- /dev/null +++ b/.github/workflows/ship_it.yml @@ -0,0 +1,33 @@ +name: "Ship It!" + +concurrency: + # There should only be able one running job per repository / branch combo. + # We do not want multiple deploys running in parallel. + group: ${{ github.repository }}-${{ github.ref_name }} + +on: + push: + branches: + - 'master' + - 'daggerize' + pull_request: + workflow_dispatch: + +jobs: + dagger: + runs-on: ubuntu-latest + steps: + - name: "Checkout code..." + uses: actions/checkout@v3 + + - name: "Setup Go..." + uses: actions/setup-go@v4 + with: + go-version: "1.20" + + - name: "Ship it!" + env: + FLY_API_TOKEN: "${{ secrets.FLY_API_TOKEN }}" + OP_SERVICE_ACCOUNT_TOKEN: "${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}" + run: | + go run . cicd --app "${{ vars.APP }}" diff --git a/.tool-versions b/.tool-versions index 69ed28b..e51da7d 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1 +1,2 @@ ruby 2.3.3 +flyctl 0.1.107 diff --git a/Gemfile b/Gemfile index 88d75cb..8e7558b 100644 --- a/Gemfile +++ b/Gemfile @@ -14,6 +14,7 @@ gem "sqlite3" gem "rest-client" gem "obscenity" gem "whatlanguage" +gem "foreman" group :test do gem "rspec" diff --git a/Gemfile.lock b/Gemfile.lock index a03151a..9746bd1 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -33,6 +33,7 @@ GEM extlib (0.9.16) faraday (0.10.1) multipart-post (>= 1.2, < 3) + foreman (0.87.2) google-api-client (0.8.6) activesupport (>= 3.2) addressable (~> 2.3) @@ -124,6 +125,7 @@ DEPENDENCIES bigquery createsend dotenv + foreman gemoji! hashie obscenity @@ -140,4 +142,4 @@ RUBY VERSION ruby 2.3.3p222 BUNDLED WITH - 1.13.6 + 1.14.6 diff --git a/Procfile b/Procfile new file mode 100644 index 0000000..4e171f7 --- /dev/null +++ b/Procfile @@ -0,0 +1,2 @@ +web: nginx +cron: supercronic -debug crontab diff --git a/Rakefile b/Rakefile index 543a516..ff3bdff 100644 --- a/Rakefile +++ b/Rakefile @@ -5,6 +5,7 @@ require "json" require "dotenv/tasks" require "createsend" require "pry" +require "uri" require_relative "lib/core_ext/date" require_relative "lib/core_ext/integer" @@ -15,11 +16,11 @@ require_relative "lib/repo" require_relative "lib/template" require_relative "lib/buffer" -DATE = Date.parse(ENV["DATE"]) rescue Date.today +DATE = Date.parse(ENV.fetch("DATE")) rescue Date.today DIST_DIR = "dist" -ISSUE_DIR = "#{DIST_DIR}/#{DATE.path}" -ISSUE_URL = "http://nightly.changelog.com/#{DATE.path}" -DATA_FILE = "#{ISSUE_DIR}/data.json" +ISSUE_DIR = File.join(DIST_DIR, DATE.path) +ISSUE_URL = URI.join(ENV.fetch("URL", "https://nightly.changelog.com/"), DATE.path) +DATA_FILE = File.join(ISSUE_DIR, "data.json") THEMES = %w(night day) MAX_REPOS = 15 @@ -108,8 +109,8 @@ namespace :issue do task buffer: [:data] do json = JSON.load File.read DATA_FILE issue = Issue.new DATE, json - gotime = Buffer.new ENV["BUFFER_GO_TIME"], %w(Go), "#golang" - jsparty = Buffer.new ENV["BUFFER_JS_PARTY"], %w(CSS JavaScript JSX PureScript TypeScript Vue) + gotime = Buffer.new ENV.fetch("BUFFER_GO_TIME"), %w(Go), "#golang" + jsparty = Buffer.new ENV.fetch("BUFFER_JS_PARTY"), %w(CSS JavaScript JSX PureScript TypeScript Vue) [gotime, jsparty].each do |buffer| buffer.injest issue.top_new @@ -155,9 +156,9 @@ namespace :issue do json = JSON.load File.read DATA_FILE next unless json["top_new"].any? || json["top_all"].any? - auth = {api_key: ENV["CAMPAIGN_MONITOR_KEY"]} + auth = {api_key: ENV.fetch("CAMPAIGN_MONITOR_KEY")} - CreateSend::List.new(auth, ENV["CAMPAIGN_MONITOR_LIST"]).segments.each do |segment| + CreateSend::List.new(auth, ENV.fetch("CAMPAIGN_MONITOR_LIST")).segments.each do |segment| theme_name = segment.Title.downcase theme_id = segment.SegmentID @@ -165,7 +166,7 @@ namespace :issue do campaign_id = CreateSend::Campaign.create( auth, - ENV["CAMPAIGN_MONITOR_ID"], # client id + ENV.fetch("CAMPAIGN_MONITOR_ID"), # client id "The Hottest Repos on GitHub - #{DATE.day_month_abbrev}", # subject "Nightly – #{DATE} (#{theme_name} theme)", # campaign name "Changelog Nightly", # from name diff --git a/crontab b/crontab new file mode 100644 index 0000000..03152fb --- /dev/null +++ b/crontab @@ -0,0 +1,2 @@ +# generate/deliver Nightly at 9:59pm CDT (2:59am UTC) +59 2 * * * rake generate deliver diff --git a/env.op b/env.op new file mode 100644 index 0000000..600e96b --- /dev/null +++ b/env.op @@ -0,0 +1,6 @@ +export GITHUB_TOKEN={{ op://nightly/app/GITHUB_TOKEN }} +export BQ_CLIENT_ID={{ op://nightly/app/BQ_CLIENT_ID }} +export BQ_SERVICE_EMAIL={{ op://nightly/app//BQ_SERVICE_EMAIL }} +export CAMPAIGN_MONITOR_ID={{ op://nightly/app/CAMPAIGN_MONITOR_ID }} +export CAMPAIGN_MONITOR_KEY={{ op://nightly/app/CAMPAIGN_MONITOR_KEY }} +export CAMPAIGN_MONITOR_LIST={{ op://nightly/app/CAMPAIGN_MONITOR_LIST_TEST }} diff --git a/fly.toml b/fly.toml new file mode 100644 index 0000000..46a8cc4 --- /dev/null +++ b/fly.toml @@ -0,0 +1,22 @@ +# https://fly.io/docs/reference/configuration/ +app = "changelog-nightly-2023-10-10" +primary_region = "ord" + +[env] + # used by supercronic - https://changelog-media.sentry.io/settings/projects/changelog-com/keys/ + SENTRY_DSN = "https://2b1aed8f16f5404cb2bc79b855f2f92d@o546963.ingest.sentry.io/5668962" + DB_DIR = "/app/dist" + +[mounts] + source = "changelog_nightly_2023_10_10" + destination = "/app/dist" + +[http_service] + internal_port = 80 + force_https = true + +[[http_service.checks]] + method = "GET" + path = "/health" + interval = "5s" + timeout = "4s" \ No newline at end of file diff --git a/flyio.go b/flyio.go new file mode 100644 index 0000000..5788d32 --- /dev/null +++ b/flyio.go @@ -0,0 +1,176 @@ +package main + +import ( + "fmt" + "os" + "strings" + "time" + + "dagger.io/dagger" +) + +type Flyio struct { + app string + deployWait string + publishedImageRef string + org string + pipeline *Pipeline + region string + registry string + token *dagger.Secret + version string + volume string + volumeSize string +} + +func newFlyio(p *Pipeline) *Flyio { + token := os.Getenv("FLY_API_TOKEN") + if token == "" { + panic("FLY_API_TOKEN env var must be set") + } + + f := &Flyio{ + app: p.app, + deployWait: "180", + org: "changelog", + pipeline: p, + region: "ord", + registry: "registry.fly.io", + token: p.dag.SetSecret("FLY_API_TOKEN", token), + version: p.tools.Flyctl(), + volumeSize: "2", + } + + f.volume = strings.ReplaceAll(f.app, "-", "_") + + return f +} + +func (f *Flyio) Cli() *dagger.Container { + flyctl := f.pipeline.Container().Pipeline("flyctl"). + From(fmt.Sprintf("flyio/flyctl:v%s", f.version)). + File("/flyctl") + + // we need Alpine so that we can run shell scripts that set secrets in secure way + container := f.pipeline.Container().Pipeline("fly.io"). + From(fmt.Sprintf("alpine:%s", f.pipeline.tools.Alpine())). + WithFile("/usr/local/bin/flyctl", flyctl, dagger.ContainerWithFileOpts{Permissions: 755}). + WithExec([]string{"flyctl", "version"}). + WithSecretVariable("FLY_API_TOKEN", f.token). + WithEnvVariable("RUN_AT", time.Now().String()). + WithNewFile("fly.toml", dagger.ContainerWithNewFileOpts{ + Contents: f.Config(), + }) + + _, err := container.File("fly.toml").Export(f.pipeline.ctx, "fly.toml") + if err != nil { + panic(err) + } + + return container +} + +func (f *Flyio) Config() string { + return fmt.Sprintf(`# https://fly.io/docs/reference/configuration/ +app = "%s" +primary_region = "%s" + +[env] + # used by supercronic - https://changelog-media.sentry.io/settings/projects/changelog-com/keys/ + SENTRY_DSN = "https://2b1aed8f16f5404cb2bc79b855f2f92d@o546963.ingest.sentry.io/5668962" + DB_DIR = "/app/dist" + +[mounts] + source = "%s" + destination = "/app/dist" + +[http_service] + internal_port = 80 + force_https = true + +[[http_service.checks]] + method = "GET" + path = "/health" + interval = "5s" + timeout = "4s"`, f.app, f.region, f.volume) +} + +func (f *Flyio) App() *Flyio { + cli := f.Cli() + + _, err := cli. + WithExec([]string{"flyctl", "status"}). + Sync(f.pipeline.ctx) + if err != nil { + _, err = cli. + WithExec([]string{"flyctl", "apps", "create", f.app, "--org", f.org}). + WithExec([]string{"flyctl", "volume", "create", f.volume, "--yes", "--region", f.region, "--size", f.volumeSize}). + Sync(f.pipeline.ctx) + if err != nil { + panic(err) + } + } + + return f +} + +func (f *Flyio) ImageRef() string { + gitSHA := os.Getenv("GITHUB_SHA") + if gitSHA == "" { + gitSHA = "dev" + } + + return fmt.Sprintf("%s/%s:%s", f.registry, f.app, gitSHA) +} + +func (f *Flyio) Publish() *Flyio { + var err error + + f.publishedImageRef, err = f.pipeline.workspace. + Pipeline("publish"). + WithRegistryAuth(f.registry, "x", f.token). + Publish(f.pipeline.ctx, f.ImageRef()) + if err != nil { + panic(err) + } + + return f +} + +func (f *Flyio) Secrets(secrets map[string]string) *Flyio { + cli := f.Cli().Pipeline("secrets") + var envs []string + for name, secret := range secrets { + cli = cli.WithSecretVariable(name, f.pipeline.dag.SetSecret(name, secret)) + envs = append(envs, fmt.Sprintf(`%s="$%s"`, name, name)) + } + + _, err := cli.WithNewFile("/flyctl-set-secrets-and-keep-hidden.sh", dagger.ContainerWithNewFileOpts{ + Contents: fmt.Sprintf(`#!/bin/sh +flyctl secrets set %s --app %s --stage`, strings.Join(envs, " "), f.app), + Permissions: 755, + }). + WithExec([]string{"/flyctl-set-secrets-and-keep-hidden.sh"}). + Sync(f.pipeline.ctx) + if err != nil { + panic(err) + } + + return f +} + +func (f *Flyio) Deploy() *Flyio { + _, err := f.Cli().Pipeline("deploy"). + WithExec([]string{ + "flyctl", "deploy", "--now", + "--app", f.app, + "--image", f.publishedImageRef, + "--wait-timeout", f.deployWait, + }). + Sync(f.pipeline.ctx) + if err != nil { + panic(err) + } + + return f +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..f5e992c --- /dev/null +++ b/go.mod @@ -0,0 +1,21 @@ +module github.com/thechangelog/nightly + +go 1.20 + +require ( + dagger.io/dagger v0.8.8 + github.com/urfave/cli/v2 v2.25.7 +) + +require ( + github.com/99designs/gqlgen v0.17.31 // indirect + github.com/Khan/genqlient v0.6.0 // indirect + github.com/adrg/xdg v0.4.0 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/vektah/gqlparser/v2 v2.5.6 // indirect + github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect + golang.org/x/sync v0.3.0 // indirect + golang.org/x/sys v0.12.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..b8d30fd --- /dev/null +++ b/go.sum @@ -0,0 +1,50 @@ +dagger.io/dagger v0.8.8 h1:vzQVbRJ2xg+uKmcg3Y/FHn1Ry513JVOACn0gDBFFEVA= +dagger.io/dagger v0.8.8/go.mod h1:d6yRd1k37fs5fCOlLKAkSB//qB26+z23UiikIb0z+uU= +github.com/99designs/gqlgen v0.17.31 h1:VncSQ82VxieHkea8tz11p7h/zSbvHSxSDZfywqWt158= +github.com/99designs/gqlgen v0.17.31/go.mod h1:i4rEatMrzzu6RXaHydq1nmEPZkb3bKQsnxNRHS4DQB4= +github.com/Khan/genqlient v0.6.0 h1:Bwb1170ekuNIVIwTJEqvO8y7RxBxXu639VJOkKSrwAk= +github.com/Khan/genqlient v0.6.0/go.mod h1:rvChwWVTqXhiapdhLDV4bp9tz/Xvtewwkon4DpWWCRM= +github.com/adrg/xdg v0.4.0 h1:RzRqFcjH4nE5C6oTAxhBtoE2IRyjBSa62SCbyPidvls= +github.com/adrg/xdg v0.4.0/go.mod h1:N6ag73EX4wyxeaoeHctc1mas01KZgsj5tYiAIwqJE/E= +github.com/agnivade/levenshtein v1.1.1/go.mod h1:veldBMzWxcCG2ZvUTKD2kJNRdCk5hVbJomOvKkmgYbo= +github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNgfBlViaCIJKLlCJ6/fmUseuG0wVQ= +github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= +github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE= +github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= +github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= +github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs= +github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= +github.com/vektah/gqlparser/v2 v2.5.6 h1:Ou14T0N1s191eRMZ1gARVqohcbe1e8FrcONScsq8cRU= +github.com/vektah/gqlparser/v2 v2.5.6/go.mod h1:z8xXUff237NntSuH8mLFijZ+1tjV1swDbpDqjJmk6ME= +github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= +github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/lib/bq_client.rb b/lib/bq_client.rb index 0721d2b..c4f74d1 100644 --- a/lib/bq_client.rb +++ b/lib/bq_client.rb @@ -6,10 +6,10 @@ class BqClient def initialize day @day = day @bq = BigQuery::Client.new({ - "client_id" => ENV["BQ_CLIENT_ID"], - "service_email" => ENV["BQ_SERVICE_EMAIL"], - "key" => ENV["BQ_KEY"], - "project_id" => ENV["BQ_PROJECT_ID"] + "client_id" => ENV.fetch("BQ_CLIENT_ID"), + "service_email" => ENV.fetch("BQ_SERVICE_EMAIL"), + "key" => ENV.fetch("BQ_KEY", "bq-key.p12"), + "project_id" => ENV.fetch("BQ_PROJECT_ID", "changelog-nightly") }) end diff --git a/lib/db.rb b/lib/db.rb index 870b213..a3d0ec2 100644 --- a/lib/db.rb +++ b/lib/db.rb @@ -2,7 +2,7 @@ require "sqlite3" module DB - @gh = Sequel.sqlite "github.db" + @gh = Sequel.sqlite(File.join(ENV.fetch("DB_DIR", "."), "github.db")) def self.create unless @gh.table_exists? :listings diff --git a/main.go b/main.go new file mode 100644 index 0000000..3b05bb4 --- /dev/null +++ b/main.go @@ -0,0 +1,109 @@ +package main + +import ( + "context" + "log" + "os" + "time" + + "dagger.io/dagger" + "github.com/urfave/cli/v2" +) + +func main() { + ctx := context.Background() + dag, err := dagger.Connect(ctx, dagger.WithLogOutput(os.Stderr)) + if err != nil { + panic(err) + } + defer dag.Close() + + app := &cli.App{ + Name: "nightly", + Usage: "Changelog Nightly CI/CD pipeline commands", + Version: "v2023.10.10", + Compiled: time.Now(), + Authors: []*cli.Author{ + { + Name: "Jerod Santo", + Email: "jerod@changelog.com", + }, + { + Name: "Gerhard Lazu", + Email: "gerhard@changelog.com", + }, + }, + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "nocache", + Aliases: []string{"n"}, + Usage: "Bust Dagger ops cache", + EnvVars: []string{"NOCACHE"}, + }, + &cli.BoolFlag{ + Name: "debug", + Aliases: []string{"d"}, + Usage: "Debug command", + EnvVars: []string{"DEBUG"}, + }, + &cli.StringFlag{ + Name: "platform", + Aliases: []string{"p"}, + Usage: "Runtime platform", + Value: "linux/amd64", + EnvVars: []string{"PLATFORM"}, + }, + }, + Commands: []*cli.Command{ + { + Name: "build", + Aliases: []string{"b"}, + Usage: "Builds container image", + Action: func(cCtx *cli.Context) error { + newPipeline(ctx, cCtx, dag). + Build() + + return nil + }, + }, + { + Name: "test", + Aliases: []string{"t"}, + Usage: "Runs tests", + Action: func(cCtx *cli.Context) error { + newPipeline(ctx, cCtx, dag). + Build(). + Test() + + return nil + }, + }, + { + Name: "cicd", + Usage: "Runs the entire CI/CD pipeline", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "app", + Aliases: []string{"a"}, + Usage: "Fly.io app name", + EnvVars: []string{"APP"}, + Required: true, + }, + }, + Action: func(cCtx *cli.Context) error { + newPipeline(ctx, cCtx, dag). + Build(). + Test(). + Prod(). + Deploy() + + return nil + }, + }, + }, + } + + if err := app.Run(os.Args); err != nil { + log.Fatal(err) + } +} diff --git a/nginx.conf b/nginx.conf new file mode 100644 index 0000000..8a3cea0 --- /dev/null +++ b/nginx.conf @@ -0,0 +1,37 @@ +daemon off; +user nginx; +worker_processes auto; + +error_log /dev/stderr warn; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; +} + +http { + include mime.types; + default_type text/plain; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" "$http_x_forwarded_for"'; + access_log /dev/stdout main; + + sendfile on; + + server_tokens off; + + server { + listen 80; + listen [::]:80; + server_name localhost; + + location / { + root /app/dist; + try_files $uri $uri/index.html $uri.html =404; + } + + location /health { + return 204; + } + } +} diff --git a/pipeline.go b/pipeline.go new file mode 100644 index 0000000..e386273 --- /dev/null +++ b/pipeline.go @@ -0,0 +1,217 @@ +package main + +import ( + "context" + "fmt" + "os" + "strings" + "time" + + "dagger.io/dagger" + "github.com/urfave/cli/v2" +) + +type Pipeline struct { + app string + ctx context.Context + dag *dagger.Client + debug bool + nocache bool + platform dagger.Platform + workspace *dagger.Container + tools *Versions +} + +func newPipeline(ctx context.Context, cCtx *cli.Context, dag *dagger.Client) *Pipeline { + p := &Pipeline{ + app: cCtx.String("app"), + ctx: ctx, + platform: dagger.Platform(cCtx.String("platform")), + debug: cCtx.Bool("debug"), + nocache: cCtx.Bool("nocache"), + dag: dag, + tools: currentToolVersions(), + } + + p.workspace = p.Container() + + return p +} + +func (p *Pipeline) OK() *Pipeline { + var err error + p.workspace, err = p.workspace.Sync(p.ctx) + if err != nil { + panic(err) + } + return p +} + +func (p *Pipeline) platformKebab() string { + return strings.ReplaceAll(string(p.platform), "/", "-") +} + +func (p *Pipeline) platformSnake() string { + return strings.ReplaceAll(string(p.platform), "/", "_") +} + +func (p *Pipeline) Container() *dagger.Container { + return p.dag.Container(dagger.ContainerOpts{ + Platform: p.platform, + }) +} + +func (p *Pipeline) Build() *Pipeline { + p.workspace = p.workspace.Pipeline("container image"). + From(fmt.Sprintf("ruby:%s-alpine", p.tools.Ruby())). + WithExec([]string{"ruby", "--version"}). + WithExec([]string{"apk", "update"}). + WithExec([]string{"apk", "add", "git", "build-base", "sqlite-dev", "bash"}) + + if p.nocache { + p.workspace = p.workspace.WithEnvVariable("DAGGER_CACHE_BUSTED_AT", time.Now().String()) + } + + app := p.dag.Host().Directory(".", dagger.HostDirectoryOpts{ + Include: []string{ + "images", + "lib", + "styles", + "views", + "Gemfile", + "Gemfile.lock", + "LICENSE", + "Procfile", + "Rakefile", + "env.op", + }}) + + pathWithBundleBin := "/usr/local/bundle/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + + p.workspace = p.workspace. + WithDirectory("/app", app). + WithWorkdir("/app"). + WithExec([]string{"bundle", "install", "--frozen", "--without=test"}). + WithEnvVariable("PATH", pathWithBundleBin). + WithNewFile("/etc/profile.d/append-bundle-bin-to-path.sh", dagger.ContainerWithNewFileOpts{ + Contents: fmt.Sprintf("export PATH=%s", pathWithBundleBin), + }). + WithExec([]string{"rake", "-T"}). + WithExec([]string{"foreman", "check"}). + WithEntrypoint(nil). + WithDefaultArgs() + + p.workspace = p.workspace. + WithExec([]string{"apk", "add", "nginx"}). + WithFile("/etc/nginx/nginx.conf", p.dag.Host().File("nginx.conf")). + WithExec([]string{"nginx", "-t"}) + + p.workspace = p.workspace. + WithFile("/usr/local/bin/supercronic", + p.supercronic(), dagger.ContainerWithFileOpts{Permissions: 555}). + WithFile("/app/crontab", + p.dag.Host().File("crontab")). + WithExec([]string{"supercronic", "-test", "crontab"}) + + p.workspace = p.workspace. + WithFile("/usr/local/bin/op", + p.op(), dagger.ContainerWithFileOpts{Permissions: 555}). + WithExec([]string{"op", "--version"}) + + if p.debug { + token := os.Getenv("OP_SERVICE_ACCOUNT_TOKEN") + if token == "" { + panic("OP_SERVICE_ACCOUNT_TOKEN env var must be set") + } + + p.workspace = p.workspace.Pipeline("generate with local config"). + WithSecretVariable("OP_SERVICE_ACCOUNT_TOKEN", p.dag.SetSecret("OP_SERVICE_ACCOUNT_TOKEN", token)). + WithExec([]string{"op", "inject", "--in-file", "env.op", "--out-file", ".env"}). + WithExec([]string{"op", "read", "--out-file", "bq-key.p12", "op://nightly/app/bq-key.p12"}). + WithExec([]string{"op", "read", "--out-file", "github.db", "--force", "op://nightly/app/github.db"}). + WithExec([]string{"apk", "add", "tmux", "vim", "htop", "strace"}). + WithExec([]string{"bash", "-c", `DATE=2023-10-10 rake generate`}). + WithEntrypoint([]string{"tmux"}) + + _, err := p.workspace.Pipeline("export tmp/image.tar"). + Export(p.ctx, "tmp/image.tar") + if err != nil { + panic(err) + } + } + + return p.OK() +} + +func (p *Pipeline) Test() *Pipeline { + if p.nocache { + p.workspace = p.workspace.WithEnvVariable("DAGGER_CACHE_BUSTED_AT", time.Now().String()) + } + + p.workspace = p.workspace. + WithExec([]string{"bundle", "install", "--frozen", "--with=test"}). + WithDirectory("/app/spec", p.dag.Host().Directory("spec")). + WithExec([]string{"rspec"}) + + return p.OK() +} + +func (p *Pipeline) Prod() *Pipeline { + if p.nocache { + p.workspace = p.workspace.WithEnvVariable("DAGGER_CACHE_BUSTED_AT", time.Now().String()) + } + + p.workspace = p.workspace.WithNewFile("/entrypoint.sh", dagger.ContainerWithNewFileOpts{ + Contents: `#!/bin/bash +set -ex +op inject --in-file env.op --out-file .env +op read --out-file bq-key.p12 --force op://nightly/app/bq-key.p12 +foreman start`, + Permissions: 555, + }). + WithEntrypoint([]string{"/entrypoint.sh"}) + + return p.OK() +} + +func (p *Pipeline) Deploy() *Pipeline { + token := os.Getenv("OP_SERVICE_ACCOUNT_TOKEN") + if token == "" { + panic("OP_SERVICE_ACCOUNT_TOKEN env var must be set") + } + + secretEnvs := map[string]string{ + "OP_SERVICE_ACCOUNT_TOKEN": token, + } + + newFlyio(p). + App(). + Publish(). + Secrets(secretEnvs). + Deploy() + + return p +} + +func (p *Pipeline) op() *dagger.File { + file := fmt.Sprintf("op_%s_v%s.zip", p.platformSnake(), p.tools._1Password()) + url := fmt.Sprintf("https://cache.agilebits.com/dist/1P/op2/pkg/v%s/%s", p.tools._1Password(), file) + + // https://hub.docker.com/layers/library/alpine/3.18.4/images/sha256-48d9183eb12a05c99bcc0bf44a003607b8e941e1d4f41f9ad12bdcc4b5672f86 + return p.Container().From("alpine@sha256:48d9183eb12a05c99bcc0bf44a003607b8e941e1d4f41f9ad12bdcc4b5672f86"). + WithFile(file, p.dag.HTTP(url)). + WithExec([]string{"unzip", file}). + WithExec([]string{"mv", "op", "/usr/local/bin/op"}). + WithExec([]string{"op", "--version"}). + File("/usr/local/bin/op") +} + +func (p *Pipeline) supercronic() *dagger.File { + return p.dag.HTTP( + fmt.Sprintf( + "https://github.com/aptible/supercronic/releases/download/v%s/supercronic-%s", + p.tools.Supercronic(), + p.platformKebab(), + ), + ) +} diff --git a/versions.go b/versions.go new file mode 100644 index 0000000..73724a1 --- /dev/null +++ b/versions.go @@ -0,0 +1,63 @@ +package main + +import ( + "bufio" + "os" + "path/filepath" + "strings" +) + +type Versions struct { + toolVersions map[string]string +} + +// https://www.ruby-lang.org/en/downloads/releases/ || asdf list all ruby +func (v *Versions) Ruby() string { + return v.toolVersions["ruby"] +} + +// https://hub.docker.com/r/flyio/flyctl/tags +func (v *Versions) Flyctl() string { + return v.toolVersions["flyctl"] +} + +// https://github.com/aptible/supercronic/releases +func (v *Versions) Supercronic() string { + return "0.2.26" +} + +// https://app-updates.agilebits.com/product_history/CLI2 +func (v *Versions) _1Password() string { + return "2.21.0" +} + +// https://hub.docker.com/_/alpine/tags +func (v *Versions) Alpine() string { + return "3.18.4@sha256:48d9183eb12a05c99bcc0bf44a003607b8e941e1d4f41f9ad12bdcc4b5672f86" +} + +func currentToolVersions() *Versions { + return &Versions{ + toolVersions: toolVersions(), + } +} + +func toolVersions() map[string]string { + wd, err := os.Getwd() + if err != nil { + panic(err) + } + versions, err := os.Open(filepath.Join(wd, ".tool-versions")) + if err != nil { + panic(err) + } + toolVersions := make(map[string]string) + scanner := bufio.NewScanner(versions) + for scanner.Scan() { + line := scanner.Text() + toolAndVersion := strings.Split(line, " ") + toolVersions[toolAndVersion[0]] = toolAndVersion[1] + } + + return toolVersions +}