Add setting to create machines using selected VPC (#45)

* api: adding support for listing digitalocean vpcs. also update godo package.

* web: add more digitalocean settings.

* api: add digitalocean vpc setting

* web: digitalocean vpc setting option

* sql: add digitalocean enabled, region and vpc settings.

* api: use configured region and vpc from settings when creating a machine
web: increase container width

api/machine/cloudinit.go

@@ -26,10 +26,6 @@ runcmd:
 
 // UserData defines the userdata used for cloud-init.
 type UserData struct {
-	AWSAccessKey string
-	AWSSecretKey string
-	SlackWebhook string
-
 	CloudinitRedisHost        string
 	CloudinitRedisPort        int
 	CloudinitDatabaseHost     string

api/machine/digitalocean.go

@@ -77,7 +77,7 @@ func (do *DigitalOcean) ListDropletByTag(ctx context.Context, tag string) ([]Mac
 }
 
 // CreateDroplets creates a new DigitalOcean droplet.
-func (do *DigitalOcean) CreateDroplets(ctx context.Context, region, size string, count int) ([]CreatedResponse, error) {
+func (do *DigitalOcean) CreateDroplets(ctx context.Context, region, size, vpc string, count int) ([]CreatedResponse, error) {
 
 	var (
 		ipv6              = true
@@ -106,6 +106,7 @@ func (do *DigitalOcean) CreateDroplets(ctx context.Context, region, size string,
 		Monitoring:        monitoring,
 		IPv6:              ipv6,
 		PrivateNetworking: privateNetworking,
+		VPCUUID:           vpc,
 	}
 
 	droplets, _, err := do.client.Droplets.CreateMultiple(ctx, createRequest)
@@ -194,6 +195,28 @@ func (do *DigitalOcean) ListSizes(ctx context.Context) ([]Size, error) {
 	return list, err
 }
 
+// ListVPCs gets a list of DigitalOcean VPCs.
+func (do *DigitalOcean) ListVPCs(ctx context.Context) ([]VPC, error) {
+	opt := &godo.ListOptions{
+		Page:    1,
+		PerPage: 200,
+	}
+
+	vpcs, _, err := do.client.VPCs.List(ctx, opt)
+	if err != nil {
+		return nil, err
+	}
+
+	list := []VPC{}
+	for _, d := range vpcs {
+		list = append(list, VPC{
+			ID:   d.ID,
+			Name: d.Name,
+		})
+	}
+	return list, err
+}
+
 // GetCurrentPricing gets the current pricing data of running machines.
 func (do *DigitalOcean) GetCurrentPricing(ctx context.Context, tag string) (*Pricing, error) {
 

api/machine/machine.go

@@ -62,3 +62,9 @@ type Pricing struct {
 	PriceHourly  float64 `json:"price_hourly"`
 	PriceMonthly float64 `json:"price_monthly"`
 }
+
+// VPC defines the response for listing VPCs.
+type VPC struct {
+	ID   string `json:"id"`
+	Name string `json:"name"`
+}

api/server/machines.go

@@ -7,12 +7,12 @@ import (
 
 	"github.com/alfg/openencoder/api/data"
 	"github.com/alfg/openencoder/api/machine"
+	"github.com/alfg/openencoder/api/types"
 	"github.com/gin-gonic/gin"
 )
 
 type machineRequest struct {
 	Provider string `json:"provider" binding:"required"`
-	Region   string `json:"region" binding:"required"`
 	Size     string `json:"size" binding:"required"`
 	Count    int    `json:"count" binding:"required,min=1,max=10"` // Max of 10 machines.
 }
@@ -27,7 +27,7 @@ func machinesHandler(c *gin.Context) {
 	}
 
 	d := data.New()
-	token := d.Settings.GetSetting(DigitalOceanAccessToken).Value
+	token := d.Settings.GetSetting(types.DigitalOceanAccessToken).Value
 	client, _ := machine.NewDigitalOceanClient(token)
 	ctx := context.TODO()
 
@@ -63,12 +63,14 @@ func createMachineHandler(c *gin.Context) {
 	}
 
 	d := data.New()
-	token := d.Settings.GetSetting(DigitalOceanAccessToken).Value
+	token := d.Settings.GetSetting(types.DigitalOceanAccessToken).Value
+	region := d.Settings.GetSetting(types.DigitalOceanRegion).Value
+	vpc := d.Settings.GetSetting(types.DigitalOceanVPC).Value
 	client, _ := machine.NewDigitalOceanClient(token)
 	ctx := context.TODO()
 
 	// Create machine.
-	machine, err := client.CreateDroplets(ctx, json.Region, json.Size, json.Count)
+	machine, err := client.CreateDroplets(ctx, region, json.Size, vpc, json.Count)
 	if err != nil {
 		log.Error(err)
 		c.JSON(http.StatusUnauthorized, gin.H{
@@ -97,7 +99,7 @@ func deleteMachineHandler(c *gin.Context) {
 	id, _ := strconv.Atoi(c.Param("id"))
 
 	d := data.New()
-	token := d.Settings.GetSetting(DigitalOceanAccessToken).Value
+	token := d.Settings.GetSetting(types.DigitalOceanAccessToken).Value
 	client, _ := machine.NewDigitalOceanClient(token)
 	ctx := context.TODO()
 
@@ -127,7 +129,7 @@ func deleteMachineByTagHandler(c *gin.Context) {
 	}
 
 	d := data.New()
-	token := d.Settings.GetSetting(DigitalOceanAccessToken).Value
+	token := d.Settings.GetSetting(types.DigitalOceanAccessToken).Value
 	client, _ := machine.NewDigitalOceanClient(token)
 	ctx := context.TODO()
 
@@ -157,7 +159,8 @@ func listMachineRegionsHandler(c *gin.Context) {
 	}
 
 	d := data.New()
-	token := d.Settings.GetSetting(DigitalOceanAccessToken).Value
+	token := d.Settings.GetSetting(types.DigitalOceanAccessToken).Value
+	region := d.Settings.GetSetting(types.DigitalOceanRegion).Value
 	client, _ := machine.NewDigitalOceanClient(token)
 	ctx := context.TODO()
 
@@ -171,8 +174,16 @@ func listMachineRegionsHandler(c *gin.Context) {
 		})
 	}
 
+	// Filter regions by configured region from settings.
+	var filteredRegions []machine.Region
+	for _, r := range regions {
+		if r.Slug == region {
+			filteredRegions = append(filteredRegions, r)
+		}
+	}
+
 	c.JSON(200, gin.H{
-		"regions": regions,
+		"regions": filteredRegions,
 	})
 }
 
@@ -186,7 +197,7 @@ func listMachineSizesHandler(c *gin.Context) {
 	}
 
 	d := data.New()
-	token := d.Settings.GetSetting(DigitalOceanAccessToken).Value
+	token := d.Settings.GetSetting(types.DigitalOceanAccessToken).Value
 	client, _ := machine.NewDigitalOceanClient(token)
 	ctx := context.TODO()
 
@@ -207,7 +218,7 @@ func listMachineSizesHandler(c *gin.Context) {
 
 func getCurrentMachinePricing(c *gin.Context) {
 	d := data.New()
-	token := d.Settings.GetSetting(DigitalOceanAccessToken).Value
+	token := d.Settings.GetSetting(types.DigitalOceanAccessToken).Value
 	client, _ := machine.NewDigitalOceanClient(token)
 	ctx := context.TODO()
 
@@ -225,3 +236,32 @@ func getCurrentMachinePricing(c *gin.Context) {
 		"pricing": pricing,
 	})
 }
+
+func listVPCsHandler(c *gin.Context) {
+	user, _ := c.Get(JwtIdentityKey)
+
+	// Role check.
+	if !isAdminOrOperator(user) {
+		c.JSON(http.StatusUnauthorized, gin.H{"message": "unauthorized"})
+		return
+	}
+
+	d := data.New()
+	token := d.Settings.GetSetting(types.DigitalOceanAccessToken).Value
+	client, _ := machine.NewDigitalOceanClient(token)
+	ctx := context.TODO()
+
+	// Get list of machine regions from DO client.
+	vpcs, err := client.ListVPCs(ctx)
+	if err != nil {
+		log.Error(err)
+		c.JSON(http.StatusUnauthorized, gin.H{
+			"status":  http.StatusUnauthorized,
+			"message": "machines not configured",
+		})
+	}
+
+	c.JSON(200, gin.H{
+		"vpcs": vpcs,
+	})
+}

api/server/routes.go

@@ -49,6 +49,7 @@ func registerRoutes(r *gin.Engine) {
 		api.GET("/machines/regions", listMachineRegionsHandler)
 		api.GET("/machines/sizes", listMachineSizesHandler)
 		api.GET("/machines/pricing", getCurrentMachinePricing)
+		api.GET("/machines/vpc", listVPCsHandler)
 
 		// Presets.
 		api.POST("/presets", createPresetHandler)

api/server/server.go

@@ -22,8 +22,7 @@ const (
 	NOK           = "NOK"
 
 	// Machines.
-	DigitalOceanAccessToken = "DIGITAL_OCEAN_ACCESS_TOKEN"
-	WorkerTag               = "openencoder-worker"
+	WorkerTag = "openencoder-worker"
 
 	// JWT settings.
 	JwtRealm       = "openencoder"

api/server/settings.go

@@ -25,7 +25,10 @@ type settingsUpdateRequest struct {
 	FTPUsername string `json:"FTP_USERNAME"`
 	FTPPassword string `json:"FTP_PASSWORD"`
 
+	DigitalOceanEnabled     string `json:"DIGITAL_OCEAN_ENABLED" binding:"eq=enabled|eq=disabled"`
 	DigitalOceanAccessToken string `json:"DIGITAL_OCEAN_ACCESS_TOKEN"`
+	DigitalOceanRegion      string `json:"DIGITAL_OCEAN_REGION"`
+	DigitalOceanVPC         string `json:"DIGITAL_OCEAN_VPC"`
 	SlackWebhook            string `json:"SLACK_WEBHOOK"`
 }
 
@@ -97,7 +100,10 @@ func updateSettingsHandler(c *gin.Context) {
 		types.FTPUsername: json.FTPUsername,
 		types.FTPPassword: json.FTPPassword,
 
+		types.DigitalOceanEnabled:     json.DigitalOceanEnabled,
 		types.DigitalOceanAccessToken: json.DigitalOceanAccessToken,
+		types.DigitalOceanRegion:      json.DigitalOceanRegion,
+		types.DigitalOceanVPC:         json.DigitalOceanVPC,
 		types.SlackWebhook:            json.SlackWebhook,
 	}
 

api/types/setting.go

@@ -18,7 +18,10 @@ const (
 	FTPUsername = "FTP_USERNAME"
 	FTPPassword = "FTP_PASSWORD"
 
+	DigitalOceanEnabled     = "DIGITAL_OCEAN_ENABLED"
 	DigitalOceanAccessToken = "DIGITAL_OCEAN_ACCESS_TOKEN"
+	DigitalOceanRegion      = "DIGITAL_OCEAN_REGION"
+	DigitalOceanVPC         = "DIGITAL_OCEAN_VPC"
 	SlackWebhook            = "SLACK_WEBHOOK"
 
 	DigitalOceanSpaces = "DIGITALOCEANSPACES"

api/worker/job.go

@@ -163,10 +163,14 @@ func sendAlert(job types.Job) error {
 
 	db := data.New()
 	webhook := db.Settings.GetSetting(types.SlackWebhook).Value
-	message := fmt.Sprintf(AlertMessageFormat, job.GUID, job.Preset, job.Source, job.Destination)
-	err := notify.SendSlackMessage(webhook, message)
-	if err != nil {
-		return err
+
+	// Only send Slack alert if configured.
+	if webhook != "" {
+		message := fmt.Sprintf(AlertMessageFormat, job.GUID, job.Preset, job.Source, job.Destination)
+		err := notify.SendSlackMessage(webhook, message)
+		if err != nil {
+			return err
+		}
 	}
 	return nil
 }

go.mod

@@ -5,7 +5,7 @@ go 1.14
 require (
 	github.com/appleboy/gin-jwt/v2 v2.6.2
 	github.com/aws/aws-sdk-go v1.20.15
-	github.com/digitalocean/godo v1.19.0
+	github.com/digitalocean/godo v1.42.0
 	github.com/gin-gonic/gin v1.6.3
 	github.com/gocraft/work v0.5.1
 	github.com/gomodule/redigo v2.0.0+incompatible
@@ -18,5 +18,5 @@ require (
 	github.com/spf13/cobra v0.0.5
 	github.com/spf13/viper v1.4.0
 	golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2
-	golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a
+	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
 )

go.sum

@@ -43,6 +43,8 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/digitalocean/godo v1.19.0 h1:9ApuchfzGD/XI8Zm0RRnZnytdfYHPjPTRKTnmzQNV7o=
 github.com/digitalocean/godo v1.19.0/go.mod h1:AAPQ+tiM4st79QHlEBTg8LM7JQNre4SAQCbn56wEyKY=
+github.com/digitalocean/godo v1.42.0 h1:xQlEFLhQ1zZUryJAfiWb8meLPPCWnLO901U5Imhh0Mc=
+github.com/digitalocean/godo v1.42.0/go.mod h1:p7dOjjtSBqCTUksqtA5Fd3uaKs9kyTq2xcz76ulEJRU=
 github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
 github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
@@ -84,6 +86,8 @@ github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I=
 github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.5 h1:F768QJ1E9tib+q5Sc8MkdJi1RxLTbRcTf8LJV56aRls=
+github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0=
 github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
@@ -251,9 +255,14 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190522155817-f3200d17e092 h1:4QSRKanuywn15aTZvI/mIDEgPQpswuFndXpOj3rKEco=
 golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k=
+golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a h1:tImsplftrFpALCYumobsd0K86vlAs/eXGFms2txfJfA=
 golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -267,6 +276,7 @@ golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223 h1:DH4skfRX4EBpamg7iV4ZlCpbl
 golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42 h1:vEOn+mP2zCOVzKckCZy6YsCtDblrpj/w7B9nxGNELpg=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
@@ -279,6 +289,8 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM=
+google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=

scripts/20_settings_options.sql

@@ -13,5 +13,8 @@ INSERT INTO public.settings_option (id, name, description, title, secure) VALUES
 INSERT INTO public.settings_option (id, name, description, title, secure) VALUES (13, 'FTP_PASSWORD', 'FTP Password', 'FTP Password', true);
 INSERT INTO public.settings_option (id, name, description, title, secure) VALUES (14, 'STORAGE_DRIVER', 'Storage Driver for input and output', 'Storage Driver', false);
 INSERT INTO public.settings_option (id, name, description, title, secure) VALUES (15, 'S3_ENDPOINT', 'Provide a custom endpoint if using a service provider that uses the S3 protocol.', 'S3 Custom Endpoint', false);
+INSERT INTO public.settings_option (id, name, description, title, secure) VALUES (16, 'DIGITAL_OCEAN_REGION', 'Digital Ocean Machines Region (Required for Machines)', 'Digital Ocean Region', false);
+INSERT INTO public.settings_option (id, name, description, title, secure) VALUES (17, 'DIGITAL_OCEAN_ENABLED', 'Enable Digital Ocean Machines', 'Digital Ocean Machines', false);
+INSERT INTO public.settings_option (id, name, description, title, secure) VALUES (18, 'DIGITAL_OCEAN_VPC', 'Enable Digital Ocean Machines VPC', 'Digital Ocean Machines VPC', false);
 
 SELECT setval('settings_option_id_seq', max(id)) FROM settings_option;

web/src/App.vue

@@ -112,6 +112,10 @@ export default {
   text-transform: uppercase;
 }
 
+.container {
+  max-width: 1280px;
+}
+
 footer ul {
   display: inline-block;
   padding-left: 0;

web/src/api.js

@@ -16,6 +16,7 @@ const Endpoints = {
   Machines: `${root}/machines`,
   MachinesRegions: `${root}/machines/regions`,
   MachinesSizes: `${root}/machines/sizes`,
+  MachinesVPCs: `${root}/machines/vpc`,
   MachinesId: id => `${root}/machines/${id}`,
 
   PresetsList: page => `${root}/presets?page=${page}`,
@@ -140,6 +141,10 @@ export default {
     return get(context, Endpoints.MachinesSizes, callback);
   },
 
+  getMachineVPCs(context, callback) {
+    return get(context, Endpoints.MachinesVPCs, callback);
+  },
+
   createMachine(context, data, callback) {
     return post(context, Endpoints.Machines, data, callback);
   },