Merge branch 'develop' into issue_2026_shortcuts

kestra-io · Jan 12, 2025 · e0a56dd · e0a56dd
2 parents 4a21163 + 6a5ec8d
commit e0a56dd
Show file tree

Hide file tree

Showing 362 changed files with 11,267 additions and 3,940 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -334,7 +334,10 @@ jobs:
       always() &&
       startsWith(github.ref, 'refs/tags/v') &&
       needs.check.result == 'success' &&
-      (needs.check-e2e.result == 'skipped' || needs.check-e2e.result == 'success')
+      (
+        github.event.inputs.skip-test == 'true' ||
+        (needs.check-e2e.result == 'skipped' || needs.check-e2e.result == 'success')
+      )
     steps:
       # Download Exec
       - name: Download executable
@@ -379,7 +382,10 @@ jobs:
       always() &&
       github.ref == 'refs/heads/develop' &&
       needs.check.result == 'success' &&
-      (needs.check-e2e.result == 'skipped' || needs.check-e2e.result == 'success')
+      (
+        github.event.inputs.skip-test == 'true' ||
+        (needs.check-e2e.result == 'skipped' || needs.check-e2e.result == 'success')
+      )
     strategy:
       matrix:
         image:
@@ -448,9 +454,12 @@ jobs:
     needs: [check, check-e2e]
     if: |
       always() &&
-      github.ref == 'refs/heads/develop' || startsWith(github.ref, 'refs/tags/v') &&
+      (github.ref == 'refs/heads/develop' || startsWith(github.ref, 'refs/tags/v')) &&
       needs.check.result == 'success' &&
-      (needs.check-e2e.result == 'skipped' || needs.check-e2e.result == 'success')
+      (
+        github.event.inputs.skip-test == 'true' ||
+        (needs.check-e2e.result == 'skipped' || needs.check-e2e.result == 'success')
+      )
     steps:
       - uses: actions/checkout@v4
 

diff --git a/build.gradle b/build.gradle
@@ -197,7 +197,8 @@ subprojects {
         test {
             useJUnitPlatform()
 
-            maxHeapSize = "4048m"
+            // set Xmx for test workers
+            maxHeapSize = '4g'
 
             // configure en_US default locale for tests
             systemProperty 'user.language', 'en'

diff --git a/core/build.gradle b/core/build.gradle
@@ -36,6 +36,7 @@ dependencies {
     implementation group: 'de.focus-shift', name: 'jollyday-jaxb'
     implementation 'nl.basjes.gitignore:gitignore-reader'
     implementation group: 'dev.failsafe', name: 'failsafe'
+    api 'org.apache.httpcomponents.client5:httpclient5'
 
     // scheduler
     implementation group: 'com.cronutils', name: 'cron-utils'
@@ -64,4 +65,8 @@ dependencies {
     testImplementation "io.micronaut:micronaut-http-client"
     testImplementation "io.micronaut:micronaut-http-server-netty"
     testImplementation "io.micronaut:micronaut-management"
+
+    testImplementation "org.testcontainers:testcontainers:1.20.3"
+    testImplementation "org.testcontainers:junit-jupiter:1.20.3"
+    testImplementation "org.bouncycastle:bcpkix-jdk18on:1.78.1"
 }
diff --git a/core/src/main/java/io/kestra/core/docs/DocumentationGenerator.java b/core/src/main/java/io/kestra/core/docs/DocumentationGenerator.java
@@ -4,6 +4,7 @@
 import com.google.common.collect.ImmutableMap;
 import io.kestra.core.models.annotations.PluginSubGroup;
 import io.kestra.core.models.conditions.Condition;
+import io.kestra.core.models.tasks.logs.LogExporter;
 import io.kestra.core.models.tasks.runners.TaskRunner;
 import io.kestra.core.models.tasks.Task;
 import io.kestra.core.models.triggers.AbstractTrigger;
@@ -69,7 +70,9 @@ public List<Document> generate(RegisteredPlugin registeredPlugin) throws Excepti
         result.addAll(this.generate(registeredPlugin, registeredPlugin.getTasks(), Task.class, "tasks"));
         result.addAll(this.generate(registeredPlugin, registeredPlugin.getTriggers(), AbstractTrigger.class, "triggers"));
         result.addAll(this.generate(registeredPlugin, registeredPlugin.getConditions(), Condition.class, "conditions"));
-        result.addAll(this.generate(registeredPlugin, registeredPlugin.getTaskRunners(), TaskRunner.class, "task-runners"));
+        //noinspection unchecked
+        result.addAll(this.generate(registeredPlugin, registeredPlugin.getTaskRunners(), (Class) TaskRunner.class, "task-runners"));
+        result.addAll(this.generate(registeredPlugin, registeredPlugin.getLogExporters(), LogExporter.class, "log-exporters"));
 
         result.addAll(guides(registeredPlugin));
 

diff --git a/core/src/main/java/io/kestra/core/docs/JsonSchemaGenerator.java b/core/src/main/java/io/kestra/core/docs/JsonSchemaGenerator.java
@@ -28,6 +28,7 @@
 import io.kestra.core.models.tasks.Output;
 import io.kestra.core.models.tasks.Task;
 import io.kestra.core.models.tasks.common.EncryptedString;
+import io.kestra.core.models.tasks.logs.LogExporter;
 import io.kestra.core.models.tasks.runners.TaskRunner;
 import io.kestra.core.models.triggers.AbstractTrigger;
 import io.kestra.core.plugins.PluginRegistry;
@@ -482,6 +483,13 @@ protected List<ResolvedType> subtypeResolver(ResolvedType declaredType, TypeCont
                 .filter(Predicate.not(io.kestra.core.models.Plugin::isInternal))
                 .flatMap(clz -> safelyResolveSubtype(declaredType, clz, typeContext).stream())
                 .toList();
+        } else if (declaredType.getErasedType() == LogExporter.class) {
+            return getRegisteredPlugins()
+                .stream()
+                .flatMap(registeredPlugin -> registeredPlugin.getLogExporters().stream())
+                .filter(Predicate.not(io.kestra.core.models.Plugin::isInternal))
+                .flatMap(clz -> safelyResolveSubtype(declaredType, clz, typeContext).stream())
+                .toList();
         } else if (declaredType.getErasedType() == Chart.class) {
             return getRegisteredPlugins()
                 .stream()

diff --git a/core/src/main/java/io/kestra/core/docs/Plugin.java b/core/src/main/java/io/kestra/core/docs/Plugin.java
@@ -33,6 +33,8 @@ public class Plugin {
     private List<String> aliases;
     private List<String> apps;
     private List<String> appBlocks;
+    private List<String> charts;
+    private List<String> dataFilters;
     private List<PluginSubGroup.PluginCategory> categories;
     private String subGroup;
 
@@ -85,6 +87,8 @@ public static Plugin of(RegisteredPlugin registeredPlugin, @Nullable String subg
         plugin.taskRunners = filterAndGetClassName(registeredPlugin.getTaskRunners()).stream().filter(c -> subgroup == null || c.startsWith(subgroup)).toList();
         plugin.apps = filterAndGetClassName(registeredPlugin.getApps()).stream().filter(c -> subgroup == null || c.startsWith(subgroup)).toList();
         plugin.appBlocks = filterAndGetClassName(registeredPlugin.getAppBlocks()).stream().filter(c -> subgroup == null || c.startsWith(subgroup)).toList();
+        plugin.charts = filterAndGetClassName(registeredPlugin.getCharts()).stream().filter(c -> subgroup == null || c.startsWith(subgroup)).toList();
+        plugin.dataFilters = filterAndGetClassName(registeredPlugin.getDataFilters()).stream().filter(c -> subgroup == null || c.startsWith(subgroup)).toList();
 
         return plugin;
     }