diff --git a/docs/Guides/Configuration.md b/docs/Guides/Configuration.md index 9f7c462c..53ed3015 100644 --- a/docs/Guides/Configuration.md +++ b/docs/Guides/Configuration.md @@ -62,6 +62,13 @@ python { } ``` +- conda (anaconda executable) +``` +conda { + executable='/some/path/to/conda/binary' +} +``` + - R ``` diff --git a/docs/Guides/Containers.md b/docs/Guides/Containers.md index cbba01e2..611f1d98 100644 --- a/docs/Guides/Containers.md +++ b/docs/Guides/Containers.md @@ -195,4 +195,37 @@ For example, to use the `delly` container image for amd64 on Apple Silicon: ``` Note that it is also required to specify the shell as `/bin/sh` as the container does not include -`bash`. \ No newline at end of file +`bash`. + + +## Anaconda (conda) + +Although not strictly a container platform, Anaconda is also often used to create isolated +environments for commands to run in. Bootstrapping an Anaconda enironment within a command +can be awkward without damaging the portability of the command, so Bpipe supports +basic activation of conda environments through command level configuration. + +To cause Bpipe to inject bootstrapping shell code to enable Anaconda to work as well +as activation of a specific Anaconda environment, add a `conda_env` attribute to the +command configuration. For example: + +```groovy +conda_env = 'gcp' +``` + +By default, the environment will be selected relative to the `conda` executable found in the PATH. +You may wish to specify the absolute path to the conda installation so that the user's +environment does not alter which environment is used. This can be done using the `conda` +configuration in Bpipe configuration: + +``` +conda { + executable = '/home/bob/anaconda/bin/conda' +} +``` + +Note that conda environment support is implemented by prefixing your command with +additional script contents to cause activation of the anaconda environment. In +some contexts this could interact with your actual command or alter things like +line numbers reported in error messages. + diff --git a/src/main/groovy/bpipe/executor/ThrottledDelegatingCommandExecutor.groovy b/src/main/groovy/bpipe/executor/ThrottledDelegatingCommandExecutor.groovy index 40dbcd6d..53ac6547 100755 --- a/src/main/groovy/bpipe/executor/ThrottledDelegatingCommandExecutor.groovy +++ b/src/main/groovy/bpipe/executor/ThrottledDelegatingCommandExecutor.groovy @@ -19,6 +19,7 @@ import bpipe.PipelineDevRetry import bpipe.ResourceUnit; import bpipe.Runner import bpipe.Utils +import bpipe.processors.CondaEnvContainerWrapper import bpipe.processors.DockerContainerWrapper import bpipe.processors.EnvironmentVariableSetter import bpipe.processors.MemoryLimitReplacer @@ -199,17 +200,18 @@ class ThrottledDelegatingCommandExecutor implements CommandExecutor { List processors = [ new EnvironmentVariableSetter(), + new CondaEnvContainerWrapper(), new MemoryLimitReplacer(), new ThreadAllocationReplacer(), new StorageResolver(commandExecutor), ] String containerType = ((Map)cmd.processedConfig.container)?.type - log.info "Container type for command $cmd.id is $containerType" + log.info "Container type for command $cmd.id in stage $cmd.name is $containerType based on config $cmd.configName" if(containerType) { if(containerType == "docker") { log.info "Configuring command with docker shell wrapper for command $cmd.id" - processors << new DockerContainerWrapper() + processors << new DockerContainerWrapper(commandExecutor) } else if(containerType == "singularity") { diff --git a/src/main/groovy/bpipe/processors/CondaEnvContainerWrapper.groovy b/src/main/groovy/bpipe/processors/CondaEnvContainerWrapper.groovy new file mode 100644 index 00000000..37ee97e9 --- /dev/null +++ b/src/main/groovy/bpipe/processors/CondaEnvContainerWrapper.groovy @@ -0,0 +1,43 @@ +package bpipe.processors + +import bpipe.Command +import bpipe.CommandProcessor +import bpipe.Config +import bpipe.ResourceUnit +import bpipe.Utils +import groovy.transform.CompileStatic +import groovy.util.logging.Log + +@Log +class CondaEnvContainerWrapper implements CommandProcessor { + + @CompileStatic + @Override + public void transform(Command command, List resources) { + if(!command.processedConfig.containsKey('conda_env')) + return + + String condaEnv = (String)command.processedConfig.conda_env + + Map config = (Map)command.processedConfig + + String shell = config.getOrDefault('shell', '/bin/bash') + + // Crude but works for most shell setups + String shellType = shell.tokenize("/")[-1] + + String conda = Utils.resolveExe("conda", "conda") + + String prefix = + """ + + $conda info --envs + + export SHELL="$shell"; eval "\$('$conda' 'shell.${shellType}' 'hook' 2> /dev/null)"; conda activate $condaEnv ; + """.stripIndent() + + log.info "Configuring conda environment using command prefix: $prefix" + + command.command = prefix + command.command + } +}