WIP - adapt scheduler

jobs/metricsforwarder/spec

@@ -36,6 +36,13 @@ properties:
   autoscaler.metricsforwarder.server.port:
     description: "Port on which the metricsforwarder server will listen"
     default: 6201
+  autoscaler.metricsforwarder.server.ca_cert:
+    description: "PEM-encoded CA certificate for the metricsforwarder server"
+  autoscaler.metricsforwarder.server.server_cert:
+    description: "PEM-encoded server certificate for the metricsforwarder server"
+  autoscaler.metricsforwarder.server.server_key:
+    description: "PEM-encoded server key for the metricsforwarder server"
+
   autoscaler.metricsforwarder.loggregator.metron_address:
     description: "IP address and port where the metron agent is running"
     default: "127.0.0.1:3458"

jobs/metricsforwarder/templates/metricsforwarder.crt.erb

@@ -0,0 +1,3 @@
+<% if_p("autoscaler.metricsforwarder.health.server_cert") do |value| %>
+<%= value %>
+<% end %>

jobs/metricsforwarder/templates/metricsforwarder.key.erb

@@ -0,0 +1,3 @@
+<% if_p("autoscaler.metricsforwarder.health.server_key") do |value| %>
+<%= value %>
+<% end %>

jobs/metricsforwarder/templates/metricsforwarder.yml.erb

@@ -46,6 +46,13 @@ end
 
 server:
   port: <%= p("autoscaler.metricsforwarder.server.port") %>
+  <% if_p("autoscaler.metricsforwarder.server.ca_cert", "autoscaler.metricsforwarder.server.server_cert", "autoscaler.metricsforwarder.server.server_key") do %>
+  tls:
+    ca_file: /var/vcap/jobs/metricsforwarder/config/certs/metricsforwarder/ca.crt
+    cert_file: /var/vcap/jobs/metricsforwarder/config/certs/metricsforwarder/server.crt
+    key_file: /var/vcap/jobs/metricsforwarder/config/certs/metricsforwarder/server.key
+  <% end %>
+
 logging:
   level: <%= p("autoscaler.metricsforwarder.logging.level") %>
 loggregator:

jobs/metricsforwarder/templates/metricsforwarder_ca.crt.erb

@@ -0,0 +1,3 @@
+<% if_p("autoscaler.metricsforwarder.health.ca_cert") do |value| %>
+<%= value %>
+<% end %>

jobs/scheduler/spec

@@ -12,6 +12,10 @@ templates:
   scheduler_server.crt.erb: config/certs/server.crt
   scheduler_server.key.erb: config/certs/server.key
 
+  healthendpoint_ca.crt.erb: config/certs/healthendpoint/ca.crt
+  healthendpoint.crt.erb: config/certs/healthendpoint/server.crt
+  healthendpoint.key.erb: config/certs/healthendpoint/server.key
+
   scalingengine_ca.crt.erb: config/certs/scalingengine/ca.crt
   scalingengine_client.crt.erb: config/certs/scalingengine/client.crt
   scalingengine_client.key.erb: config/certs/scalingengine/client.key
@@ -119,6 +123,12 @@ properties:
   autoscaler.scheduler.health.port:
     description: "the listening port of health endpoint"
     default: 6204
+  autoscaler.scheduler.health.ca_cert:
+    description: "PEM-encoded CA certificate for the health endpoint"
+  autoscaler.scheduler.health.server_cert:
+    description: "PEM-encoded server certificate for the health endpoint"
+  autoscaler.scheduler.health.server_key:
+    description: "PEM-encoded server key for the health endpoint"
   autoscaler.scheduler.health.basicAuthEnabled:
     description: "if true, basic auth is enabled on the endpoint"
     default: false

jobs/scheduler/templates/healthendpoint.crt.erb

@@ -0,0 +1,3 @@
+<% if_p("autoscaler.scheduler.health.server_cert") do |value| %>
+<%= value %>
+<% end %>

jobs/scheduler/templates/healthendpoint.key.erb

@@ -0,0 +1,3 @@
+<% if_p("autoscaler.scheduler.health.server_key") do |value| %>
+<%= value %>
+<% end %>

jobs/scheduler/templates/healthendpoint_ca.crt.erb

@@ -0,0 +1,3 @@
+<% if_p("autoscaler.scheduler.health.ca_cert") do |value| %>
+<%= value %>
+<% end %>

jobs/scheduler/templates/scheduler.yml.erb

@@ -99,6 +99,31 @@ spring:
             instanceName: app-autoscaler
           threadPool:
             threadCount: 10
+    ############################################################
+    #    SSL Bundles
+    ############################################################
+    ssl:
+      bundle:
+        jks:
+          server:
+            key:
+              alias: "scheduler"
+            keystore:
+              location: "/var/vcap/jobs/scheduler/config/certs/server.p12"
+              password: "123456"
+            truststore:
+              location: "/var/vcap/jobs/scheduler/config/certs/cacerts"
+              password: "123456"
+        <% if_p("autoscaler.scheduler.health.ca_cert", "autoscaler.scheduler.health.server_cert", "autoscaler.scheduler.health.server_key") do %>
+        pem:
+          healthendpoint:
+            keystore:
+              certificate: "/var/vcap/jobs/scheduler/config/certs/healthendpoint/server.crt"
+              private-key: "/var/vcap/jobs/scheduler/config/certs/healthendpoint/server.key"
+            truststore:
+              certificate: "/var/vcap/jobs/scheduler/config/certs/healthendpoint/ca.crt"
+        <% end %>
+
 ############################################################
 #    Client SSL keys
 ############################################################
@@ -108,7 +133,7 @@ client:
     key-store: /var/vcap/jobs/scheduler/config/certs/scalingengine/client.p12
     key-store-password: 123456
     key-store-type: PKCS12
-    protocol: TLSv1.2
+    protocol: TLSv1.3
     trust-store: /var/vcap/jobs/scheduler/config/certs/scalingengine/cacerts
     trust-store-password: 123456
 ############################################################
@@ -142,15 +167,10 @@ scheduler:
 server:
   port: <%=p('autoscaler.scheduler.port') %>
   ssl:
-    ciphers: TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_CBC_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_128_CBC_SHA256,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_RC4_128_SHA,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,SSL_RSA_WITH_RC4_128_SHA
-    enabled-protocols: TLSv1.2
-    key-alias: scheduler
-    key-store: /var/vcap/jobs/scheduler/config/certs/server.p12
-    key-store-password: 123456
-    key-store-type: PKCS12
-    trust-store: /var/vcap/jobs/scheduler/config/certs/cacerts
-    trust-store-password: 123456
-
+    ciphers: TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256
+    enabled-protocols: TLSv1.3
+    bundle: "server"
+    client-auth: NEED
 
 #User added properties
 <%=p('autoscaler.scheduler.application.props')%>

spec/jobs/common/health_endpoint_spec.rb

@@ -21,13 +21,13 @@
           @properties = YAML.safe_load(fixture(properties_file).read)
           @template = release.job(release_job).template(config_file)
           @links = case service
-                  when "eventgenerator"
-                    [ Bosh::Template::Test::Link.new(name: "eventgenerator") ]
-                  when "metricsgateway", "metricsserver"
-                    [ Bosh::Template::Test::Link.new(name: "metricsserver") ]
-                  else
-                    []
-                  end
+          when "eventgenerator"
+            [Bosh::Template::Test::Link.new(name: "eventgenerator")]
+          when "metricsgateway", "metricsserver"
+            [Bosh::Template::Test::Link.new(name: "metricsserver")]
+          else
+            []
+          end
           @rendered_template = YAML.safe_load(@template.render(@properties, consumes: @links))
         end
         it "by default TLS is not configured" do
@@ -46,10 +46,10 @@
 
           expect(rendered_template["health"]["tls"]).not_to be_nil
           expect(rendered_template["health"]["tls"]).to include({
-                                                                  "key_file" => "/var/vcap/jobs/#{release_job}/config/certs/healthendpoint/server.key",
-                                                                  "ca_file" => "/var/vcap/jobs/#{release_job}/config/certs/healthendpoint/ca.crt",
-                                                                  "cert_file" => "/var/vcap/jobs/#{release_job}/config/certs/healthendpoint/server.crt"
-                                                                })
+            "key_file" => "/var/vcap/jobs/#{release_job}/config/certs/healthendpoint/server.key",
+            "ca_file" => "/var/vcap/jobs/#{release_job}/config/certs/healthendpoint/ca.crt",
+            "cert_file" => "/var/vcap/jobs/#{release_job}/config/certs/healthendpoint/server.crt"
+          })
         end
       end
     end

src/autoscaler/integration/components_test.go

@@ -1,6 +1,9 @@
 package integration_test
 
 import (
+	_ "embed"
+	"text/template"
+
 	apiConfig "code.cloudfoundry.org/app-autoscaler/src/autoscaler/api/config"
 	"code.cloudfoundry.org/app-autoscaler/src/autoscaler/cf"
 	"code.cloudfoundry.org/app-autoscaler/src/autoscaler/db"
@@ -44,6 +47,9 @@ var golangSchemaValidationPath = "../api/schemas/catalog.schema.json"
 var golangApiServerPolicySchemaPath = "../api/policyvalidator/policy_json.schema.json"
 var golangServiceCatalogPath = "../servicebroker/config/catalog.json"
 
+//go:embed scheduler_application.template.yml
+var schedulerApplicationConfigTemplate string
+
 type Executables map[string]string
 type Ports map[string]int
 
@@ -316,70 +322,36 @@ func (components *Components) PrepareSchedulerConfig(dbUri string, scalingEngine
 		jdbcDBUri = fmt.Sprintf("jdbc:%s://%s/%s", scheme, host, path)
 		driverClassName = "com.mysql.cj.jdbc.Driver"
 	}
-	settingStrTemplate := `
-#datasource for application and quartz
-spring.datasource.driverClassName=%s
-spring.datasource.url=%s
-spring.datasource.username=%s
-spring.datasource.password=%s
-#policy db
-spring.policy-db-datasource.driverClassName=%s
-spring.policy-db-datasource.url=%s
-spring.policy-db-datasource.username=%s
-spring.policy-db-datasource.password=%s
-#quartz job
-scalingenginejob.reschedule.interval.millisecond=10000
-scalingenginejob.reschedule.maxcount=3
-scalingengine.notification.reschedule.maxcount=3
-# scaling engine url
-autoscaler.scalingengine.url=%s
-#ssl
-server.ssl.key-store=%s/scheduler.p12
-server.ssl.key-alias=scheduler
-server.ssl.key-store-password=123456
-server.ssl.key-store-type=PKCS12
-server.ssl.trust-store=%s/autoscaler.truststore
-server.ssl.trust-store-password=123456
-client.ssl.key-store=%s/scheduler.p12
-client.ssl.key-store-password=123456
-client.ssl.key-store-type=PKCS12
-client.ssl.trust-store=%s/autoscaler.truststore
-client.ssl.trust-store-password=123456
-client.ssl.protocol=TLSv1.2
-server.ssl.enabled-protocols=TLSv1,TLSv1.1,TLSv1.2
-server.ssl.ciphers=TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_CBC_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_128_CBC_SHA256,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_RC4_128_SHA,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,SSL_RSA_WITH_RC4_128_SHA
-
-server.port=%d
-scheduler.healthserver.port=0
-client.httpClientTimeout=%d
-#Quartz
-org.quartz.scheduler.instanceName=app-autoscaler
-org.quartz.scheduler.instanceId=0
-spring.quartz.properties.org.quartz.scheduler.instanceName=app-autoscaler
-spring.quartz.properties.org.quartz.scheduler.instanceId=scheduler-12345
-#The the number of milliseconds the scheduler will ‘tolerate’ a trigger to pass its next-fire-time by,
-# before being considered “misfired”. The default value (if not specified in  configuration) is 60000 (60 seconds)
-spring.quartz.properties.org.quartz.jobStore.misfireThreshold=120000
-spring.quartz.properties.org.quartz.jobStore.driverDelegateClass=org.quartz.impl.jdbcjobstore.PostgreSQLDelegate
-spring.quartz.properties.org.quartz.jobStore.isClustered=true
-spring.quartz.properties.org.quartz.threadPool.threadCount=10
-spring.application.name=scheduler
-spring.mvc.servlet.load-on-startup=1
-spring.aop.auto=false
-endpoints.enabled=false
-spring.data.jpa.repositories.enabled=false
-spring.main.allow-bean-definition-overriding=true
-`
-	settingJsonStr := fmt.Sprintf(settingStrTemplate,
-		driverClassName, jdbcDBUri, userName, password,
-		driverClassName, jdbcDBUri, userName, password,
-		scalingEngineUri,
-		testCertDir, testCertDir, testCertDir, testCertDir,
-		components.Ports[Scheduler],
-		int(httpClientTimeout/time.Second))
-	cfgFile, err := os.Create(filepath.Join(tmpDir, "application.properties"))
+
+	type TemplateParameters struct {
+		ScalingEngineUri  string
+		HttpClientTimeout int
+		TestCertDir       string
+		Port              int
+		DriverClassName   string
+		DBUser            string
+		DBPassword        string
+		JDBCURI           string
+	}
+
+	templateParameters := TemplateParameters{
+		ScalingEngineUri:  scalingEngineUri,
+		HttpClientTimeout: int(httpClientTimeout / time.Second),
+		TestCertDir:       testCertDir,
+		Port:              components.Ports[Scheduler],
+		DriverClassName:   driverClassName,
+		DBUser:            userName,
+		DBPassword:        password,
+		JDBCURI:           jdbcDBUri,
+	}
+
+	ut, err := template.New("application.yaml").Parse(schedulerApplicationConfigTemplate)
 	Expect(err).NotTo(HaveOccurred())
-	err = os.WriteFile(cfgFile.Name(), []byte(settingJsonStr), 0600)
+
+	cfgFile, err := os.Create(filepath.Join(tmpDir, "application.yaml"))
+	Expect(err).NotTo(HaveOccurred())
+
+	err = ut.Execute(cfgFile, templateParameters)
 	Expect(err).NotTo(HaveOccurred())
 	cfgFile.Close()
 	return cfgFile.Name()

src/autoscaler/integration/scheduler_application.template.yml

@@ -0,0 +1,87 @@
+autoscaler:
+    scalingengine:
+        url: {{ .ScalingEngineUri }}
+client:
+    httpClientTimeout: {{ .HttpClientTimeout }}
+    ssl:
+        key-store: {{ .TestCertDir }}/scheduler.p12
+        key-store-password: 123456
+        key-store-type: PKCS12
+        protocol: TLSv1.3
+        trust-store: {{ .TestCertDir }}/autoscaler.truststore
+        trust-store-password: 123456
+endpoints:
+    enabled: false
+org:
+    quartz:
+        scheduler:
+            instanceId: 0
+            instanceName: app-autoscaler
+scalingengine:
+    notification:
+        reschedule:
+            maxcount: 3
+scalingenginejob:
+    reschedule:
+        interval:
+            millisecond: 10000
+        maxcount: 3
+scheduler:
+    healthserver:
+        port: 0
+server:
+    port: {{ .Port }}
+    ssl:
+        ciphers: TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256
+        enabled-protocols: TLSv1.3
+        bundle: "server"
+        client-auth: NEED
+spring:
+    aop:
+        auto: false
+    application:
+        name: scheduler
+    data:
+        jpa:
+            repositories:
+                enabled: false
+    datasource:
+        driverClassName: {{ .DriverClassName }}
+        password: {{ .DBPassword }}
+        url: {{ .JDBCURI }}
+        username: {{ .DBUser }}
+    main:
+        allow-bean-definition-overriding: true
+    mvc:
+        servlet:
+            load-on-startup: 1
+    policy-db-datasource:
+        driverClassName: {{ .DriverClassName }}
+        password: {{ .DBPassword }}
+        url: {{ .JDBCURI }}
+        username: {{ .DBUser }}
+    quartz:
+        properties:
+            org:
+                quartz:
+                    jobStore:
+                        driverDelegateClass: org.quartz.impl.jdbcjobstore.PostgreSQLDelegate
+                        isClustered: true
+                        misfireThreshold: 120000
+                    scheduler:
+                        instanceId: scheduler-12345
+                        instanceName: app-autoscaler
+                    threadPool:
+                        threadCount: 10
+    ssl:
+        bundle:
+            jks:
+                server:
+                    key:
+                        alias: scheduler
+                    keystore:
+                        location: {{ .TestCertDir }}/scheduler.p12
+                        password: '123456'
+                    truststore:
+                        location: {{ .TestCertDir }}/autoscaler.truststore
+                        password: '123456'