diff --git a/CHANGELOG.md b/CHANGELOG.md index ab8f6e5a..9914f789 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # nextflow-io/nf-schema: Changelog +# Version 2.3.0dev + +## Bug fixes + +1. The help message will now also be printed out when no functions of the plugin get included in the pipeline. +2. JSON and YAML files that are not a list of values should now also be validated correctly. (Mind that samplesheets always have to be a list of values to work with `samplesheetToList`) + # Version 2.2.1 ## Bug fixes diff --git a/plugins/nf-schema/src/main/nextflow/validation/CustomEvaluators/SchemaEvaluator.groovy b/plugins/nf-schema/src/main/nextflow/validation/CustomEvaluators/SchemaEvaluator.groovy index 25a4526b..db7ec927 100644 --- a/plugins/nf-schema/src/main/nextflow/validation/CustomEvaluators/SchemaEvaluator.groovy +++ b/plugins/nf-schema/src/main/nextflow/validation/CustomEvaluators/SchemaEvaluator.groovy @@ -50,11 +50,11 @@ class SchemaEvaluator implements Evaluator { log.debug("Started validating ${file.toString()}") def String schemaFull = Utils.getSchemaPath(this.baseDir, this.schema) - def JSONArray arrayJSON = Utils.fileToJsonArray(file, Path.of(schemaFull)) + def Object json = Utils.fileToJson(file, Path.of(schemaFull)) def String schemaContents = Files.readString( Path.of(schemaFull) ) def validator = new JsonSchemaValidator(config) - def List validationErrors = validator.validate(arrayJSON, schemaContents) + def List validationErrors = validator.validate(json, schemaContents) if (validationErrors) { def List errors = ["Validation of file failed:"] + validationErrors.collect { "\t${it}" as String} return Evaluator.Result.failure(errors.join("\n")) diff --git a/plugins/nf-schema/src/main/nextflow/validation/SamplesheetConverter.groovy b/plugins/nf-schema/src/main/nextflow/validation/SamplesheetConverter.groovy index 010a14d8..33d73a37 100644 --- a/plugins/nf-schema/src/main/nextflow/validation/SamplesheetConverter.groovy +++ b/plugins/nf-schema/src/main/nextflow/validation/SamplesheetConverter.groovy @@ -80,6 +80,13 @@ class SamplesheetConverter { throw new SchemaValidationException(msg) } + def LinkedHashMap schemaMap = new JsonSlurper().parseText(schemaFile.text) as LinkedHashMap + def List schemaKeys = schemaMap.keySet() as List + if(schemaKeys.contains("properties") || !schemaKeys.contains("items")) { + def msg = "${colors.red}The schema for '${samplesheetFile.toString()}' (${schemaFile.toString()}) is not valid. Please make sure that 'items' is the top level keyword and not 'properties'\n${colors.reset}\n" + throw new SchemaValidationException(msg) + } + if(!samplesheetFile.exists()) { def msg = "${colors.red}Samplesheet file ${samplesheetFile.toString()} does not exist\n${colors.reset}\n" throw new SchemaValidationException(msg) @@ -87,7 +94,7 @@ class SamplesheetConverter { // Validate final validator = new JsonSchemaValidator(config) - def JSONArray samplesheet = Utils.fileToJsonArray(samplesheetFile, schemaFile) + def JSONArray samplesheet = Utils.fileToJson(samplesheetFile, schemaFile) as JSONArray def List validationErrors = validator.validate(samplesheet, schemaFile.text) if (validationErrors) { def msg = "${colors.red}The following errors have been detected in ${samplesheetFile.toString()}:\n\n" + validationErrors.join('\n').trim() + "\n${colors.reset}\n" @@ -96,8 +103,7 @@ class SamplesheetConverter { } // Convert - def LinkedHashMap schemaMap = new JsonSlurper().parseText(schemaFile.text) as LinkedHashMap - def List samplesheetList = Utils.fileToList(samplesheetFile, schemaFile) + def List samplesheetList = Utils.fileToObject(samplesheetFile, schemaFile) as List this.rows = [] diff --git a/plugins/nf-schema/src/main/nextflow/validation/Utils.groovy b/plugins/nf-schema/src/main/nextflow/validation/Utils.groovy index c016b14a..57563645 100644 --- a/plugins/nf-schema/src/main/nextflow/validation/Utils.groovy +++ b/plugins/nf-schema/src/main/nextflow/validation/Utils.groovy @@ -52,13 +52,19 @@ public class Utils { } // Converts a given file to a List - public static List fileToList(Path file, Path schema) { + public static Object fileToObject(Path file, Path schema) { def String fileType = Utils.getFileType(file) def String delimiter = fileType == "csv" ? "," : fileType == "tsv" ? "\t" : null + def Map schemaMap = (Map) new JsonSlurper().parse( schema ) def Map types = variableTypes(schema) - if (types.find{ it.value == "array" } as Boolean && fileType in ["csv", "tsv"]){ - def msg = "Using \"type\": \"array\" in schema with a \".$fileType\" samplesheet is not supported\n" + if (schemaMap.type == "object" && fileType in ["csv", "tsv"]) { + def msg = "CSV or TSV files are not supported. Use a JSON or YAML file instead of ${file.toString()}. (Expected a non-list data structure, which is not supported in CSV or TSV)" + throw new SchemaValidationException(msg, []) + } + + if ((types.find{ it.value == "array" || it.value == "object" } as Boolean) && fileType in ["csv", "tsv"]){ + def msg = "Using \"type\": \"array\" or \"type\": \"object\" in schema with a \".$fileType\" samplesheet is not supported\n" log.error("ERROR: Validation of pipeline parameters failed!") throw new SchemaValidationException(msg, []) } @@ -67,7 +73,7 @@ public class Utils { return new Yaml().load((file.text)) } else if(fileType == "json"){ - return new JsonSlurper().parseText(file.text) as List + return new JsonSlurper().parseText(file.text) } else { def Boolean header = getValueFromJson("#/items/properties", new JSONObject(schema.text)) ? true : false @@ -82,13 +88,21 @@ public class Utils { } // Converts a given file to a JSONArray - public static JSONArray fileToJsonArray(Path file, Path schema) { + public static Object fileToJson(Path file, Path schema) { // Remove all null values from JSON object // and convert the groovy object to a JSONArray def jsonGenerator = new JsonGenerator.Options() .excludeNulls() .build() - return new JSONArray(jsonGenerator.toJson(fileToList(file, schema))) + def Object obj = fileToObject(file, schema) + if (obj instanceof List) { + return new JSONArray(jsonGenerator.toJson(obj)) + } else if (obj instanceof Map) { + return new JSONObject(jsonGenerator.toJson(obj)) + } else { + def msg = "Could not determine if the file is a list or map of values" + throw new SchemaValidationException(msg, []) + } } // @@ -141,7 +155,7 @@ public class Utils { def Map parsed = (Map) slurper.parse( schema ) // Obtain the type of each variable in the schema - def Map properties = (Map) parsed['items']['properties'] + def Map properties = (Map) parsed['items'] ? parsed['items']['properties'] : parsed["properties"] for (p in properties) { def String key = (String) p.key def Map property = properties[key] as Map diff --git a/plugins/nf-schema/src/test/nextflow/validation/ValidateParametersTest.groovy b/plugins/nf-schema/src/test/nextflow/validation/ValidateParametersTest.groovy index 4407b83e..b6c074f0 100644 --- a/plugins/nf-schema/src/test/nextflow/validation/ValidateParametersTest.groovy +++ b/plugins/nf-schema/src/test/nextflow/validation/ValidateParametersTest.groovy @@ -1224,4 +1224,102 @@ class ValidateParametersTest extends Dsl2Spec{ !stdout } + def 'should validate a map file - yaml' () { + given: + def schema = Path.of('src/testResources/nextflow_schema_with_map_file.json').toAbsolutePath().toString() + def SCRIPT = """ + params.map_file = 'src/testResources/map_file.yaml' + include { validateParameters } from 'plugin/nf-schema' + + validateParameters(parameters_schema: '$schema') + """ + + when: + def config = [:] + def result = new MockScriptRunner(config).setScript(SCRIPT).execute() + def stdout = capture + .toString() + .readLines() + .findResults {it.contains('WARN nextflow.validation.SchemaValidator') || it.startsWith('* --') ? it : null } + + then: + noExceptionThrown() + !stdout + } + + def 'should validate a map file - json' () { + given: + def schema = Path.of('src/testResources/nextflow_schema_with_map_file.json').toAbsolutePath().toString() + def SCRIPT = """ + params.map_file = 'src/testResources/map_file.json' + include { validateParameters } from 'plugin/nf-schema' + + validateParameters(parameters_schema: '$schema') + """ + + when: + def config = [:] + def result = new MockScriptRunner(config).setScript(SCRIPT).execute() + def stdout = capture + .toString() + .readLines() + .findResults {it.contains('WARN nextflow.validation.SchemaValidator') || it.startsWith('* --') ? it : null } + + then: + noExceptionThrown() + !stdout + } + + def 'should give an error when a map file is wrong - yaml' () { + given: + def schema = Path.of('src/testResources/nextflow_schema_with_map_file.json').toAbsolutePath().toString() + def SCRIPT = """ + params.map_file = 'src/testResources/map_file_wrong.yaml' + include { validateParameters } from 'plugin/nf-schema' + + validateParameters(parameters_schema: '$schema') + """ + + when: + def config = [:] + def result = new MockScriptRunner(config).setScript(SCRIPT).execute() + def stdout = capture + .toString() + .readLines() + .findResults {it.contains('WARN nextflow.validation.SchemaValidator') || it.startsWith('* --') ? it : null } + + + then: + def error = thrown(SchemaValidationException) + error.message.contains("* --map_file (src/testResources/map_file_wrong.yaml): Validation of file failed:") + error.message.contains(" * --this.is.deep (hello): Value is [string] but should be [integer]") + !stdout + } + + def 'should give an error when a map file is wrong - json' () { + given: + def schema = Path.of('src/testResources/nextflow_schema_with_map_file.json').toAbsolutePath().toString() + def SCRIPT = """ + params.map_file = 'src/testResources/map_file_wrong.json' + include { validateParameters } from 'plugin/nf-schema' + + validateParameters(parameters_schema: '$schema') + """ + + when: + def config = [:] + def result = new MockScriptRunner(config).setScript(SCRIPT).execute() + def stdout = capture + .toString() + .readLines() + .findResults {it.contains('WARN nextflow.validation.SchemaValidator') || it.startsWith('* --') ? it : null } + + + then: + def error = thrown(SchemaValidationException) + error.message.contains("* --map_file (src/testResources/map_file_wrong.json): Validation of file failed:") + error.message.contains(" * --this.is.deep (hello): Value is [string] but should be [integer]") + !stdout + } + } \ No newline at end of file diff --git a/plugins/nf-schema/src/testResources/map_file.json b/plugins/nf-schema/src/testResources/map_file.json new file mode 100644 index 00000000..0dda9a91 --- /dev/null +++ b/plugins/nf-schema/src/testResources/map_file.json @@ -0,0 +1,8 @@ +{ + "test": "hello", + "this": { + "is": { + "deep": 20 + } + } +} \ No newline at end of file diff --git a/plugins/nf-schema/src/testResources/map_file.yaml b/plugins/nf-schema/src/testResources/map_file.yaml new file mode 100644 index 00000000..aa724ba8 --- /dev/null +++ b/plugins/nf-schema/src/testResources/map_file.yaml @@ -0,0 +1,4 @@ +test: hello +this: + is: + deep: 20 \ No newline at end of file diff --git a/plugins/nf-schema/src/testResources/map_file_wrong.json b/plugins/nf-schema/src/testResources/map_file_wrong.json new file mode 100644 index 00000000..4a02e6a3 --- /dev/null +++ b/plugins/nf-schema/src/testResources/map_file_wrong.json @@ -0,0 +1,8 @@ +{ + "test": "hello", + "this": { + "is": { + "deep": "hello" + } + } +} \ No newline at end of file diff --git a/plugins/nf-schema/src/testResources/map_file_wrong.yaml b/plugins/nf-schema/src/testResources/map_file_wrong.yaml new file mode 100644 index 00000000..54a27e86 --- /dev/null +++ b/plugins/nf-schema/src/testResources/map_file_wrong.yaml @@ -0,0 +1,4 @@ +test: hello +this: + is: + deep: hello \ No newline at end of file diff --git a/plugins/nf-schema/src/testResources/nextflow_schema_with_map_file.json b/plugins/nf-schema/src/testResources/nextflow_schema_with_map_file.json new file mode 100644 index 00000000..24aa7875 --- /dev/null +++ b/plugins/nf-schema/src/testResources/nextflow_schema_with_map_file.json @@ -0,0 +1,14 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://raw.githubusercontent.com/nf-core/testpipeline/master/nextflow_schema.json", + "title": "nf-core/testpipeline pipeline parameters", + "description": "this is a test", + "type": "object", + "properties": { + "map_file": { + "type": "string", + "format": "file-path", + "schema": "src/testResources/schema_map_file.json" + } + } +} diff --git a/plugins/nf-schema/src/testResources/schema_map_file.json b/plugins/nf-schema/src/testResources/schema_map_file.json new file mode 100644 index 00000000..8829cb2a --- /dev/null +++ b/plugins/nf-schema/src/testResources/schema_map_file.json @@ -0,0 +1,25 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://raw.githubusercontent.com/test/test/master/assets/schema_input.json", + "title": "Test schema for samplesheets", + "description": "Schema for the file provided with params.input", + "type": "object", + "properties": { + "test": { + "type": "string" + }, + "this": { + "type": "object", + "properties": { + "is": { + "type": "object", + "properties": { + "deep": { + "type": "integer" + } + } + } + } + } + } +}