add env vars

openclimatefix · Jan 10, 2025 · f4799a1 · f4799a1
1 parent 7307235
commit f4799a1
Showing 1 changed file with 5 additions and 2 deletions.
diff --git a/src/nwp_consumer/internal/services/consumer_service.py b/src/nwp_consumer/internal/services/consumer_service.py
@@ -114,8 +114,11 @@ def _parallelize_generator[T](
             n_jobs = 1
 
 
-        n_jobs=4
-        prefer="processes"
+        if os.getenv("USE_PROCESSES", "False").capitalize() == "True":
+            prefer="processes"
+
+        if os.getenv("N_JOBS", None) is not None:
+            n_jobs = int(os.getenv("N_JOBS"))
 
         log.debug(f"Using {n_jobs} concurrent {prefer}")