Merge pull request #29 from nextflow-io/ignore_blob_storage_validation

Do not validate Azure or GCP storage paths

CHANGELOG.md

@@ -19,6 +19,7 @@ To migrate from nf-validation please follow the [migration guide](https://nextfl
 - `samplesheetToList()` now does dynamic typecasting instead of using the `type` fields in the JSON schema. This is done due to the complexity of `draft-2020-12` JSON schemas. This should not have that much impact but keep in mind that some types can be different between this version and older versions in nf-validation ([#141](https://github.com/nextflow-io/nf-validation/pull/141))
 - `samplesheetToList()` will now set all missing values as `[]` instead of the type specific defaults (because of the changes in the previous point). This should not change that much as this will also result in `false` when used in conditions. ([#141](https://github.com/nextflow-io/nf-validation/pull/141))
 - Removed the configuration parameters and added configuration options instead. For a full list of these new options, please have a look at the [configuration docs](https://nextflow-io.github.io/nf-schema/latest/configuration/)
+- Ignore validation of Azure and GCP hosted blob storage files in addition to AWS S3 hosted files. This is because they are not true POSIX compliant files and would incorrectly fail validation ([#29](https://github.com/nextflow-io/nf-schema/pull/29))
 
 ## Improvements
 

plugins/nf-schema/src/main/nextflow/validation/CustomEvaluators/ExistsEvaluator.groovy

@@ -32,7 +32,7 @@ class ExistsEvaluator implements Evaluator {
         def String value = node.asString()
 
         // Skip validation of S3 paths for now
-        if (value.startsWith('s3://')) {
+        if (value.startsWith('s3://') || value.startsWith('az://') || value.startsWith('gs://')) {
             log.debug("S3 paths are not supported by 'ExistsEvaluator': '${value}'")
             return Evaluator.Result.success()
         }

plugins/nf-schema/src/main/nextflow/validation/CustomEvaluators/FormatDirectoryPathEvaluator.groovy

@@ -25,9 +25,9 @@ class FormatDirectoryPathEvaluator implements Evaluator {
 
         def String value = node.asString()
 
-        // Skip validation of S3 paths for now
-        if (value.startsWith('s3://')) {
-            log.debug("S3 paths are not supported by 'FormatDirectoryPathEvaluator': '${value}'")
+        // Skip validation of blob storage paths
+        if (value.startsWith('s3://') || value.startsWith('az://') || value.startsWith('gs://')) {
+            log.debug("Cloud blob storage paths are not supported by 'FormatDirectoryPathEvaluator': '${value}'")
             return Evaluator.Result.success()
         }
 
@@ -38,4 +38,4 @@ class FormatDirectoryPathEvaluator implements Evaluator {
         }
         return Evaluator.Result.success()
     }
-}
+}

plugins/nf-schema/src/main/nextflow/validation/CustomEvaluators/FormatFilePathEvaluator.groovy

@@ -26,7 +26,7 @@ class FormatFilePathEvaluator implements Evaluator {
         def String value = node.asString()
 
         // Skip validation of S3 paths for now
-        if (value.startsWith('s3://')) {
+        if (value.startsWith('s3://') || value.startsWith('az://') || value.startsWith('gs://')) {
             log.debug("S3 paths are not supported by 'FormatFilePathEvaluator': '${value}'")
             return Evaluator.Result.success()
         }

plugins/nf-schema/src/main/nextflow/validation/CustomEvaluators/FormatFilePathPatternEvaluator.groovy

@@ -26,7 +26,7 @@ class FormatFilePathPatternEvaluator implements Evaluator {
         def String value = node.asString()
 
         // Skip validation of S3 paths for now
-        if (value.startsWith('s3://')) {
+        if (value.startsWith('s3://') || value.startsWith('az://') || value.startsWith('gs://')) {
             log.debug("S3 paths are not supported by 'FormatFilePathPatternEvaluator': '${value}'")
             return Evaluator.Result.success()
         }

plugins/nf-schema/src/main/nextflow/validation/CustomEvaluators/FormatPathEvaluator.groovy

@@ -26,7 +26,7 @@ class FormatPathEvaluator implements Evaluator {
         def String value = node.asString()
 
         // Skip validation of S3 paths for now
-        if (value.startsWith('s3://')) {
+        if (value.startsWith('s3://') || value.startsWith('az://') || value.startsWith('gs://')) {
             log.debug("S3 paths are not supported by 'FormatPathEvaluator': '${value}'")
             return Evaluator.Result.success()
         }

plugins/nf-schema/src/test/nextflow/validation/ValidateParametersTest.groovy

@@ -674,6 +674,75 @@ class ValidateParametersTest extends Dsl2Spec{
         !stdout
     }
 
+    def 'ignore validation of aws s3' () {
+        given:
+        def schema = Path.of('src/testResources/nextflow_schema_file_path_pattern.json').toAbsolutePath().toString()
+        def SCRIPT = """
+            params.glob = 's3://src/testResources/correct.csv'
+            include { validateParameters } from 'plugin/nf-schema'
+            
+            validateParameters(parameters_schema: '$schema')
+        """
+
+        when:
+        def config = [:]
+        def result = new MockScriptRunner(config).setScript(SCRIPT).execute()
+        def stdout = capture
+                .toString()
+                .readLines()
+                .findResults {it.contains('WARN nextflow.validation.SchemaValidator') || it.startsWith('* --') ? it : null }
+
+        then:
+        noExceptionThrown()
+        !stdout
+    }
+
+    def 'ignore validation of azure blob storage' () {
+        given:
+        def schema = Path.of('src/testResources/nextflow_schema_file_path_pattern.json').toAbsolutePath().toString()
+        def SCRIPT = """
+            params.glob = 'az://src/testResources/correct.csv'
+            include { validateParameters } from 'plugin/nf-schema'
+            
+            validateParameters(parameters_schema: '$schema')
+        """
+
+        when:
+        def config = [:]
+        def result = new MockScriptRunner(config).setScript(SCRIPT).execute()
+        def stdout = capture
+                .toString()
+                .readLines()
+                .findResults {it.contains('WARN nextflow.validation.SchemaValidator') || it.startsWith('* --') ? it : null }
+
+        then:
+        noExceptionThrown()
+        !stdout
+    }
+
+    def 'ignore validation of gcp cloud storage' () {
+        given:
+        def schema = Path.of('src/testResources/nextflow_schema_file_path_pattern.json').toAbsolutePath().toString()
+        def SCRIPT = """
+            params.glob = 'gs://src/testResources/correct.csv'
+            include { validateParameters } from 'plugin/nf-schema'
+            
+            validateParameters(parameters_schema: '$schema')
+        """
+
+        when:
+        def config = [:]
+        def result = new MockScriptRunner(config).setScript(SCRIPT).execute()
+        def stdout = capture
+                .toString()
+                .readLines()
+                .findResults {it.contains('WARN nextflow.validation.SchemaValidator') || it.startsWith('* --') ? it : null }
+
+        then:
+        noExceptionThrown()
+        !stdout
+    }
+
     def 'correct validation of numbers with lenient mode' () {
         given:
         def schema = Path.of('src/testResources/nextflow_schema.json').toAbsolutePath().toString()