Pipeline config batch update (#1823)

* feat(pipeline executions/gate) : Added code to save multiple pipelines at once to sql db. This is part of: spinnaker/spinnaker#6147. Enhanced PipelineController.groovy to Added new rest api method bulksavePipeline(List<Map> pipelineList, Boolean staleCheck) This method accepts a list of pipelines json. This method returns a Map object having the below data: { “Successful”: <count>, “Failed”: <cound>, “Failed_list”: [<array of failed pipelines - (application, pipeline name, etc) and the error message] } Enhanced TaskService.java to Added code to poll the task until the SavePipelineTask and MonitorFront50Task status : succeeded, terminal. * feat(pipelines executions/gate): Support for bulk saving pipelines * fix(gate-web) Fix functional test * refactor(gate-web): enable pipeline controller config props in GateConfg instead of GateWebConfig GateWebConfig seems to exist more for spring specific configurations, whereas GateConfig is a general config for the application beans/logic. --------- Co-authored-by: sanopsmx <[email protected]> Co-authored-by: Arifullah Pattan <[email protected]> Co-authored-by: Richard Timpson <[email protected]>
spinnaker · Aug 23, 2024 · 7373cfe · 7373cfe
1 parent 47f666f
commit 7373cfe
Show file tree

Hide file tree

Showing 6 changed files with 287 additions and 10 deletions.
diff --git a/gate-core/src/main/java/com/netflix/spinnaker/gate/services/TaskService.java b/gate-core/src/main/java/com/netflix/spinnaker/gate/services/TaskService.java
@@ -108,7 +108,7 @@ public Map createAndWaitForCompletion(Map body, int maxPolls, int intervalMs) {
     String taskId = ((String) createResult.get("ref")).split("/")[2];
     log.info("Create succeeded; polling task for completion: " + taskId);
 
-    LinkedHashMap<String, String> map = new LinkedHashMap<String, String>(1);
+    LinkedHashMap<String, String> map = new LinkedHashMap<>(1);
     map.put("id", taskId);
     Map task = map;
     for (int i = 0; i < maxPolls; i++) {
@@ -122,7 +122,16 @@ public Map createAndWaitForCompletion(Map body, int maxPolls, int intervalMs) {
           .contains((String) task.get("status"))) {
         return task;
       }
+      log.debug(
+          "Task {} not completed after checking {} times. Waiting {}ms before checking again",
+          taskId,
+          i + 1,
+          intervalMs);
     }
+    log.error(
+        "Task {} still not complete after exhausting {} max polls. Not checking anymore.",
+        taskId,
+        maxPolls);
 
     return task;
   }

diff --git a/gate-web/src/main/groovy/com/netflix/spinnaker/gate/config/GateConfig.groovy b/gate-web/src/main/groovy/com/netflix/spinnaker/gate/config/GateConfig.groovy
@@ -28,6 +28,7 @@ import com.netflix.spinnaker.fiat.shared.FiatPermissionEvaluator
 import com.netflix.spinnaker.fiat.shared.FiatService
 import com.netflix.spinnaker.fiat.shared.FiatStatus
 import com.netflix.spinnaker.filters.AuthenticatedRequestFilter
+import com.netflix.spinnaker.gate.config.controllers.PipelineControllerConfigProperties
 import com.netflix.spinnaker.gate.converters.JsonHttpMessageConverter
 import com.netflix.spinnaker.gate.converters.YamlHttpMessageConverter
 import com.netflix.spinnaker.gate.filters.RequestLoggingFilter
@@ -51,6 +52,7 @@ import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.beans.factory.annotation.Value
 import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty
+import org.springframework.boot.context.properties.EnableConfigurationProperties
 import org.springframework.boot.web.servlet.FilterRegistrationBean
 import org.springframework.context.annotation.Bean
 import org.springframework.context.annotation.Configuration
@@ -71,6 +73,7 @@ import static retrofit.Endpoints.newFixedEndpoint
 @CompileStatic
 @Configuration
 @Slf4j
+@EnableConfigurationProperties([PipelineControllerConfigProperties.class])
 @Import([PluginsAutoConfiguration, DeckPluginConfiguration, PluginWebConfiguration])
 class GateConfig {
 

diff --git a/...ovy/com/netflix/spinnaker/gate/config/controllers/PipelineControllerConfigProperties.java b/...ovy/com/netflix/spinnaker/gate/config/controllers/PipelineControllerConfigProperties.java
@@ -0,0 +1,18 @@
+package com.netflix.spinnaker.gate.config.controllers;
+
+import lombok.Data;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+
+@Data
+@ConfigurationProperties(prefix = "controller.pipeline")
+public class PipelineControllerConfigProperties {
+
+  /** Holds the configurations to be used for bulk save controller mapping */
+  private BulkSaveConfigProperties bulksave = new BulkSaveConfigProperties();
+
+  @Data
+  public static class BulkSaveConfigProperties {
+    private int maxPollsForTaskCompletion = 300;
+    private int taskCompletionCheckIntervalMs = 2000;
+  }
+}
diff --git a/gate-web/src/main/groovy/com/netflix/spinnaker/gate/controllers/PipelineController.groovy b/gate-web/src/main/groovy/com/netflix/spinnaker/gate/controllers/PipelineController.groovy
@@ -18,6 +18,7 @@
 package com.netflix.spinnaker.gate.controllers
 
 import com.fasterxml.jackson.databind.ObjectMapper
+import com.netflix.spinnaker.gate.config.controllers.PipelineControllerConfigProperties
 import com.netflix.spinnaker.gate.services.PipelineService
 import com.netflix.spinnaker.gate.services.TaskService
 import com.netflix.spinnaker.gate.services.internal.Front50Service
@@ -41,6 +42,8 @@ import org.springframework.security.access.prepost.PreAuthorize
 import org.springframework.web.bind.annotation.*
 import retrofit.RetrofitError
 
+import java.nio.charset.StandardCharsets
+
 import static net.logstash.logback.argument.StructuredArguments.value
 
 @Slf4j
@@ -60,6 +63,9 @@ class PipelineController {
   @Autowired
   ObjectMapper objectMapper
 
+  @Autowired
+  PipelineControllerConfigProperties pipelineControllerConfigProperties
+
   @CompileDynamic
   @ApiOperation(value = "Delete a pipeline definition")
   @DeleteMapping("/{application}/{pipelineName:.+}")
@@ -125,6 +131,46 @@ class PipelineController {
     }
   }
 
+  @CompileDynamic
+  @ApiOperation(value = "Save a list of pipelines")
+  @PostMapping('/bulksave')
+  Map bulksavePipeline(
+    @RequestParam(defaultValue = "bulk_save_placeholder_app")
+    @ApiParam(value = "Application in which to run the bulk save task",
+      defaultValue = "bulk_save_placeholder_app",
+      required = false) String application,
+    @RequestBody List<Map> pipelines) {
+    def operation = [
+      description: "Bulk save pipelines",
+      application: application,
+      job        : [
+        [
+          type                      : "savePipeline",
+          pipelines                 : Base64.encoder
+            .encodeToString(objectMapper.writeValueAsString(pipelines).getBytes(StandardCharsets.UTF_8)),
+          user                      : AuthenticatedRequest.spinnakerUser.orElse("anonymous"),
+          isBulkSavingPipelines : true
+        ]
+      ]
+    ]
+
+    def result = taskService.createAndWaitForCompletion(operation,
+      pipelineControllerConfigProperties.getBulksave().getMaxPollsForTaskCompletion(),
+      pipelineControllerConfigProperties.getBulksave().getTaskCompletionCheckIntervalMs())
+    String resultStatus = result.get("status")
+
+    if (!"SUCCEEDED".equalsIgnoreCase(resultStatus)) {
+      String exception = result.variables.find { it.key == "exception" }?.value?.details?.errors?.getAt(0)
+      throw new PipelineException(
+        exception ?: "Pipeline bulk save operation did not succeed: ${result.get("id", "unknown task id")} " +
+          "(status: ${resultStatus})"
+      )
+    } else {
+      def retVal = result.variables.find { it.key == "bulksave"}?.value
+      return retVal
+    }
+  }
+
   @ApiOperation(value = "Rename a pipeline definition")
   @PostMapping('move')
   void renamePipeline(@RequestBody Map renameCommand) {

diff --git a/gate-web/src/test/groovy/com/netflix/spinnaker/gate/FunctionalSpec.groovy b/gate-web/src/test/groovy/com/netflix/spinnaker/gate/FunctionalSpec.groovy
@@ -22,6 +22,7 @@ import com.netflix.spinnaker.config.ErrorConfiguration
 import com.netflix.spinnaker.fiat.shared.FiatClientConfigurationProperties
 import com.netflix.spinnaker.fiat.shared.FiatStatus
 import com.netflix.spinnaker.gate.config.ServiceConfiguration
+import com.netflix.spinnaker.gate.config.controllers.PipelineControllerConfigProperties
 import com.netflix.spinnaker.gate.controllers.ApplicationController
 import com.netflix.spinnaker.gate.controllers.PipelineController
 import com.netflix.spinnaker.gate.services.*
@@ -39,7 +40,7 @@ import org.springframework.core.annotation.Order
 import org.springframework.security.config.annotation.web.builders.HttpSecurity
 import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter
 import retrofit.RetrofitError
-import retrofit.RestAdapter;
+import retrofit.RestAdapter
 import retrofit.client.OkClient
 import retrofit.converter.JacksonConverter
 import retrofit.mime.TypedInput
@@ -287,6 +288,11 @@ class FunctionalSpec extends Specification {
       )
     }
 
+    @Bean
+    PipelineControllerConfigProperties pipelineControllerConfigProperties() {
+      new PipelineControllerConfigProperties();
+    }
+
     @Override
     protected void configure(HttpSecurity http) throws Exception {
       http