Preserve attributes of published files (take 2) (#5660)

Signed-off-by: Ben Sherman <[email protected]> Signed-off-by: Paolo Di Tommaso <[email protected]> Co-authored-by: Paolo Di Tommaso <[email protected]>
nextflow-io · Jan 24, 2025 · 422f289 · 422f289
1 parent f8c3b63
commit 422f289
Show file tree

Hide file tree

Showing 3 changed files with 41 additions and 3 deletions.
diff --git a/docs/reference/config.md b/docs/reference/config.md
@@ -1651,6 +1651,12 @@ The `workflow` scope provides workflow execution options.
 : *Currently only supported for S3.*
 : Specify the media type, also known as [MIME type](https://developer.mozilla.org/en-US/docs/Web/HTTP/MIME_types), of published files (default: `false`). Can be a string (e.g. `'text/html'`), or `true` to infer the content type from the file extension.
 
+`workflow.output.copyAttributes`
+: :::{versionadded} 25.01.0-edge
+  :::
+: *Currently only supported for local and shared filesystems.*
+: Copy file attributes (such as the last modified timestamp) to the published file (default: `false`).
+
 `workflow.output.enabled`
 : Enable or disable publishing (default: `true`).
 

diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy
@@ -18,6 +18,7 @@ package nextflow.processor
 
 import static nextflow.util.CacheHelper.*
 
+import java.nio.file.CopyOption
 import java.nio.file.FileAlreadyExistsException
 import java.nio.file.FileSystem
 import java.nio.file.FileSystems
@@ -26,6 +27,7 @@ import java.nio.file.LinkOption
 import java.nio.file.NoSuchFileException
 import java.nio.file.Path
 import java.nio.file.PathMatcher
+import java.nio.file.StandardCopyOption
 import java.time.temporal.ChronoUnit
 import java.util.concurrent.ExecutorService
 
@@ -36,6 +38,7 @@ import dev.failsafe.event.ExecutionAttemptedEvent
 import groovy.transform.CompileDynamic
 import groovy.transform.CompileStatic
 import groovy.transform.EqualsAndHashCode
+import groovy.transform.Memoized
 import groovy.transform.PackageScope
 import groovy.transform.ToString
 import groovy.util.logging.Slf4j
@@ -509,19 +512,27 @@ class PublishDir {
             FilesEx.mklink(source, [hard:true], destination)
         }
         else if( mode == Mode.MOVE ) {
-            FileHelper.movePath(source, destination)
+            FileHelper.movePath(source, destination, copyOpts())
         }
         else if( mode == Mode.COPY ) {
-            FileHelper.copyPath(source, destination)
+            FileHelper.copyPath(source, destination, copyOpts())
         }
         else if( mode == Mode.COPY_NO_FOLLOW ) {
-            FileHelper.copyPath(source, destination, LinkOption.NOFOLLOW_LINKS)
+            FileHelper.copyPath(source, destination, copyOpts(LinkOption.NOFOLLOW_LINKS))
         }
         else {
             throw new IllegalArgumentException("Unknown file publish mode: ${mode}")
         }
     }
 
+    @Memoized
+    protected CopyOption[] copyOpts(CopyOption... opts) {
+        final copyAttributes = session.config.navigate('workflow.output.copyAttributes', false)
+        return copyAttributes
+            ? opts + StandardCopyOption.COPY_ATTRIBUTES
+            : opts
+    }
+
     protected void createPublishDir() {
         try {
             makeDirs(path)

diff --git a/modules/nextflow/src/test/groovy/nextflow/processor/PublishDirTest.groovy b/modules/nextflow/src/test/groovy/nextflow/processor/PublishDirTest.groovy
@@ -16,9 +16,12 @@
 
 package nextflow.processor
 
+import java.nio.file.CopyOption
 import java.nio.file.FileSystems
 import java.nio.file.Files
+import java.nio.file.LinkOption
 import java.nio.file.Paths
+import java.nio.file.StandardCopyOption
 
 import nextflow.Global
 import nextflow.Session
@@ -417,4 +420,22 @@ class PublishDirTest extends Specification {
         [NXF_PUBLISH_FAIL_ON_ERROR: 'true']         | true
         [NXF_PUBLISH_FAIL_ON_ERROR: 'false']        | false
     }
+
+    def 'should return copy attributes' () {
+        expect:
+        new PublishDir().copyOpts() == [] as CopyOption[]
+        and:
+        new PublishDir().copyOpts(LinkOption.NOFOLLOW_LINKS) == [LinkOption.NOFOLLOW_LINKS] as CopyOption[]
+
+        when:
+        Global.session = Mock(Session) { getConfig()>>[workflow:[output:[copyAttributes: true]]] }
+        then:
+        new PublishDir().copyOpts() == [StandardCopyOption.COPY_ATTRIBUTES] as CopyOption[]
+        and:
+        new PublishDir().copyOpts(LinkOption.NOFOLLOW_LINKS) == [LinkOption.NOFOLLOW_LINKS,StandardCopyOption.COPY_ATTRIBUTES] as CopyOption[]
+
+        cleanup:
+        Global.session = null
+    }
+
 }