refactor: remove preprocessing_config and instead supply all argument…

…s directly in the call to `silo preprocessing`j

refactor: remove preprocessing_config and instead supply all arguments directly in the call to `silo preprocessing`

kubernetes/loculus/silo_import_job.sh

@@ -5,8 +5,6 @@ set -e
 root_dir=""
 last_etag=""
 lineage_definition_file=/preprocessing/input/lineage_definitions.yaml
-preprocessing_config_file=preprocessing_config.yaml
-preprocessing_config_file_merged=preprocessing_config_merged.yaml
 
 # Parse command-line arguments
 usage() {
@@ -151,14 +149,9 @@ download_data() {
   echo
 }
 
-# Generate the preprocessing config file with the lineage file for the current pipeline version.
-# the lineage definition file needs to be downloaded first.
-prepare_preprocessing_config() {
-  rm -f $lineage_definition_file $preprocessing_config_file_merged
-
+download_lineage_definitions() {
   if [[ -z "$LINEAGE_DEFINITIONS" ]]; then
     echo "No LINEAGE_DEFINITIONS given, nothing to configure;"
-    cp $preprocessing_config_file $preprocessing_config_file_merged
     return
   fi
 
@@ -186,11 +179,6 @@ prepare_preprocessing_config() {
     echo "Multiple pipeline versions in data to import: $pipelineVersion"
     exit 1
   fi
-
-  # the lineage definition filename needs to be set in the config
-  # Once https://github.com/GenSpectrum/LAPIS-SILO/pull/633 is merged, it can be done as a commandline arg
-  cp $preprocessing_config_file $preprocessing_config_file_merged
-  echo -e "lineageDefinitionsFilename: \"$lineage_definition_file\"\n" >> $preprocessing_config_file_merged
 }
 
 preprocessing() {
@@ -203,7 +191,10 @@ preprocessing() {
   cp "$new_input_data_path" "$silo_input_data_path"
 
   set +e
-  time /app/silo preprocessing --preprocessing-config=$preprocessing_config_file_merged
+  time /app/silo preprocessing \
+    --lineage-definitions-filename "$lineage_definition_file" \
+    --ndjson-input-filename "$silo_input_data_path" \
+    --reference-genome-filename reference_genomes.json
   exit_code=$?
   set -e
 
@@ -274,7 +265,7 @@ main() {
   # cleanup at start in case we fail later
   cleanup_output_data
   download_data
-  prepare_preprocessing_config
+  download_lineage_definitions
   preprocessing
 
   echo "done"

kubernetes/loculus/templates/lapis-silo-database-config.yaml

@@ -17,10 +17,6 @@ data:
     {{ include "loculus.siloDatabaseConfig" $args | nindent 4 }}
   {{- end }}
 
-  preprocessing_config.yaml: |
-    ndjsonInputFilename: data.ndjson.zst
-    referenceGenomeFilename: reference_genomes.json
-
   reference_genomes.json: |
     {{ $referenceGenomes | toJson }}
 

kubernetes/loculus/templates/lapis-silo-deployment.yaml

@@ -96,9 +96,6 @@ spec:
             - name: lapis-silo-database-config-processed
               mountPath: /preprocessing/input/database_config.yaml
               subPath: database_config.yaml
-            - name: lapis-silo-database-config-processed
-              mountPath: /app/preprocessing_config.yaml
-              subPath: preprocessing_config.yaml
             - name: lapis-silo-shared-data
               mountPath: /preprocessing/output
             - name: lapis-silo-input-data-cache