refactor: remove preprocessing_config_merged and instead insert the l…

…ineageDefinitionsFilename directly in the template (#3533)

kubernetes/loculus/silo_import_job.sh

@@ -5,8 +5,6 @@ set -e
 root_dir=""
 last_etag=""
 lineage_definition_file=/preprocessing/input/lineage_definitions.yaml
-preprocessing_config_file=preprocessing_config.yaml
-preprocessing_config_file_merged=preprocessing_config_merged.yaml
 
 # Parse command-line arguments
 usage() {
@@ -151,14 +149,9 @@ download_data() {
   echo
 }
 
-# Generate the preprocessing config file with the lineage file for the current pipeline version.
-# the lineage definition file needs to be downloaded first.
-prepare_preprocessing_config() {
-  rm -f $lineage_definition_file $preprocessing_config_file_merged
-
+download_lineage_definitions() {
   if [[ -z "$LINEAGE_DEFINITIONS" ]]; then
     echo "No LINEAGE_DEFINITIONS given, nothing to configure;"
-    cp $preprocessing_config_file $preprocessing_config_file_merged
     return
   fi
 
@@ -186,11 +179,6 @@ prepare_preprocessing_config() {
     echo "Multiple pipeline versions in data to import: $pipelineVersion"
     exit 1
   fi
-
-  # the lineage definition filename needs to be set in the config
-  # Once https://github.com/GenSpectrum/LAPIS-SILO/pull/633 is merged, it can be done as a commandline arg
-  cp $preprocessing_config_file $preprocessing_config_file_merged
-  echo -e "lineageDefinitionsFilename: \"$lineage_definition_file\"\n" >> $preprocessing_config_file_merged
 }
 
 preprocessing() {
@@ -203,7 +191,7 @@ preprocessing() {
   cp "$new_input_data_path" "$silo_input_data_path"
 
   set +e
-  time /app/silo preprocessing --preprocessing-config=$preprocessing_config_file_merged
+  time /app/silo preprocessing
   exit_code=$?
   set -e
 
@@ -274,7 +262,7 @@ main() {
   # cleanup at start in case we fail later
   cleanup_output_data
   download_data
-  prepare_preprocessing_config
+  download_lineage_definitions
   preprocessing
 
   echo "done"

kubernetes/loculus/templates/lapis-silo-database-config.yaml

@@ -4,7 +4,8 @@
 
 {{- range $key, $instance := (.Values.organisms | default .Values.defaultOrganisms) }}
 
-{{ $referenceGenomes:= include "loculus.generateReferenceGenome" $instance.referenceGenomes | fromYaml }}
+{{- $referenceGenomes:= include "loculus.generateReferenceGenome" $instance.referenceGenomes | fromYaml }}
+{{- $lineageSystem := $instance | include "loculus.lineageSystemForOrganism" }}
 ---
 apiVersion: v1
 kind: ConfigMap
@@ -20,6 +21,9 @@ data:
   preprocessing_config.yaml: |
     ndjsonInputFilename: data.ndjson.zst
     referenceGenomeFilename: reference_genomes.json
+    {{- if $lineageSystem }}
+    lineageDefinitionsFilename: lineage_definitions.yaml
+    {{- end }}
 
   reference_genomes.json: |
     {{ $referenceGenomes | toJson }}