diff --git a/build.gradle.kts b/build.gradle.kts index 6462a18c..8fbc709c 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -15,7 +15,7 @@ plugins { allprojects { repositories { - mavenLocal() + //mavenLocal() // Allows you to specify your own repository manager instance. if (project.hasProperty("s3fs.proxy.url")) { maven { @@ -38,6 +38,50 @@ java { withJavadocJar() } +// Configure multiple test sources +testing { + suites { + // Just for self reference, technically this is already configured by default. + val test by getting(JvmTestSuite::class) { + useJUnitJupiter() // already the default. + testType.set(TestSuiteType.UNIT_TEST) // already the default. + } + + // testIntegration test sources + val testIntegration by registering(JvmTestSuite::class) { + val self = this + testType.set(TestSuiteType.INTEGRATION_TEST) + + // We need to manually add the "main" sources to the classpath. + sourceSets { + named(self.name) { + compileClasspath += sourceSets.main.get().output + sourceSets.test.get().output + runtimeClasspath += sourceSets.main.get().output + sourceSets.test.get().output + } + } + + // Inherit implementation, runtime and test dependencies (adds them to the compile classpath) + configurations.named("${self.name}Implementation") { + extendsFrom(configurations.testImplementation.get()) + extendsFrom(configurations.runtimeOnly.get()) + extendsFrom(configurations.implementation.get()) + } + + // Make sure the integration test is executed as part of the "check" task. + tasks.named("check") { + dependsOn(named(self.name)) + } + + tasks.named(self.name) { + mustRunAfter(test) + } + + } + } + + +} + dependencies { api(platform("software.amazon.awssdk:bom:2.29.9")) api("software.amazon.awssdk:s3") { @@ -49,6 +93,9 @@ dependencies { exclude("org.slf4j", "slf4j-api") } api("com.google.code.findbugs:jsr305:3.0.2") + api("com.github.ben-manes.caffeine:caffeine:2.9.3") { + because("Last version to support JDK 8.") + } testImplementation("ch.qos.logback:logback-classic:1.5.12") testImplementation("org.junit.jupiter:junit-jupiter:5.11.3") @@ -140,6 +187,10 @@ tasks { } } + named("check") { + dependsOn(named("testIntegration")) + } + named("jacocoTestReport") { group = "jacoco" dependsOn(named("test")) // tests are required to run before generating the report @@ -162,32 +213,10 @@ tasks { group = "sonar" } - named("test") { - description = "Run unit tests" - outputs.upToDateWhen { false } - useJUnitPlatform { - filter { - excludeTestsMatching("*IT") - } - } - } - withType { defaultCharacterEncoding = "UTF-8" } - create("it-s3") { - group = "verification" - description = "Run integration tests using S3" - useJUnitPlatform { - filter { - includeTestsMatching("*IT") - includeTags("it-s3") - } - } - mustRunAfter(named("test")) - } - // TODO: There are some problems with using minio that overcomplicate the setup. // For the time being we'll be disabling it until we figure out the best path forward. // create("it-minio") { @@ -201,10 +230,6 @@ tasks { // } // } - named("check") { - dependsOn(named("it-s3")) - } - withType { onlyIf { (project.hasProperty("withSignature") && project.findProperty("withSignature") == "true") || diff --git a/docs/content/contributing/developer-guide/index.md b/docs/content/contributing/developer-guide/index.md index da4537c2..6c6b6f7e 100644 --- a/docs/content/contributing/developer-guide/index.md +++ b/docs/content/contributing/developer-guide/index.md @@ -10,7 +10,7 @@ Before you start writing code, please read: ## System requirements 1. Gradle 8.1, or higher -2. `JDK8`, `JDK11` or `JDK17` +2. `JDK8`, `JDK11`, `JDK17` or `JDK21` ## Finding issues to work on @@ -85,7 +85,7 @@ s3fs.proxy.url=https://my.local.domain/path/to/repository ### Build Builds the entire code and runs unit and integration tests. -It is assumed you already have the `amazon-test.properties` configuration in place. +It is assumed you already have the `amazon-test.properties` configuration in place under the `src/test/resources` or `src/testIntegration/resources`. ``` ./gradlew build @@ -100,9 +100,11 @@ It is assumed you already have the `amazon-test.properties` configuration in pla ### Run only integration tests ``` -./gradlew it-s3 +./gradlew testIntegration ``` +You can also use `./gradlew build -x testIntegration` to skip the integration tests. + ### Run all tests ``` diff --git a/docs/content/reference/configuration-options.md b/docs/content/reference/configuration-options.md index b0283977..d7da85bb 100644 --- a/docs/content/reference/configuration-options.md +++ b/docs/content/reference/configuration-options.md @@ -4,28 +4,30 @@ A complete list of environment variables which can be set to configure the client. -| Key | Default | Description | -|-------------------------------------------|---------|-------------------------------------------------------------------------------------------------------------------------| -| s3fs.access.key | none | AWS access key, used to identify the user interacting with AWS | -| s3fs.secret.key | none | AWS secret access key, used to authenticate the user interacting with AWS | -| s3fs.request.metric.collector.class | TODO | Fully-qualified class name to instantiate an AWS SDK request/response metric collector | -| s3fs.connection.timeout | TODO | Timeout (in milliseconds) for establishing a connection to a remote service | -| s3fs.max.connections | TODO | Maximum number of connections allowed in a connection pool | -| s3fs.max.retry.error | TODO | Maximum number of times that a single request should be retried, assuming it fails for a retryable error | -| s3fs.protocol | TODO | Protocol (HTTP or HTTPS) to use when connecting to AWS | -| s3fs.proxy.domain | none | For NTLM proxies: The Windows domain name to use when authenticating with the proxy | -| s3fs.proxy.protocol | none | Proxy connection protocol. | -| s3fs.proxy.host | none | Proxy host name either from the configured endpoint or from the "http.proxyHost" system property | -| s3fs.proxy.password | none | The password to use when connecting through a proxy | -| s3fs.proxy.port | none | Proxy port either from the configured endpoint or from the "http.proxyPort" system property | -| s3fs.proxy.username | none | The username to use when connecting through a proxy | -| s3fs.proxy.workstation | none | For NTLM proxies: The Windows workstation name to use when authenticating with the proxy | -| s3fs.region | none | The AWS Region to configure the client | -| s3fs.socket.send.buffer.size.hint | TODO | The size hint (in bytes) for the low level TCP send buffer | -| s3fs.socket.receive.buffer.size.hint | TODO | The size hint (in bytes) for the low level TCP receive buffer | -| s3fs.socket.timeout | TODO | Timeout (in milliseconds) for each read to the underlying socket | -| s3fs.user.agent.prefix | TODO | Prefix of the user agent that is sent with each request to AWS | -| s3fs.amazon.s3.factory.class | TODO | Fully-qualified class name to instantiate a S3 factory base class which creates a S3 client instance | -| s3fs.signer.override | TODO | Fully-qualified class name to define the signer that should be used when authenticating with AWS | -| s3fs.path.style.access | TODO | Boolean that indicates whether the client uses path-style access for all requests | -| s3fs.request.header.cache-control | blank | Configures the `cacheControl` on request builders (i.e. `CopyObjectRequest`, `PutObjectRequest`, etc) | +| Key | Default | Description | +|-------------------------------------|---------|-------------------------------------------------------------------------------------------------------------------------| +| s3fs.access.key | none | AWS access key, used to identify the user interacting with AWS | +| s3fs.secret.key | none | AWS secret access key, used to authenticate the user interacting with AWS | +| s3fs.request.metric.collector.class | TODO | Fully-qualified class name to instantiate an AWS SDK request/response metric collector | +| s3fs.cache.attributes.ttl | `60000` | TTL for the cached file attributes (in millis) | +| s3fs.cache.attributes.size | `5000` | Total size of cached file attributes | +| s3fs.connection.timeout | TODO | Timeout (in milliseconds) for establishing a connection to a remote service | +| s3fs.max.connections | TODO | Maximum number of connections allowed in a connection pool | +| s3fs.max.retry.error | TODO | Maximum number of times that a single request should be retried, assuming it fails for a retryable error | +| s3fs.protocol | TODO | Protocol (HTTP or HTTPS) to use when connecting to AWS | +| s3fs.proxy.domain | none | For NTLM proxies: The Windows domain name to use when authenticating with the proxy | +| s3fs.proxy.protocol | none | Proxy connection protocol. | +| s3fs.proxy.host | none | Proxy host name either from the configured endpoint or from the "http.proxyHost" system property | +| s3fs.proxy.password | none | The password to use when connecting through a proxy | +| s3fs.proxy.port | none | Proxy port either from the configured endpoint or from the "http.proxyPort" system property | +| s3fs.proxy.username | none | The username to use when connecting through a proxy | +| s3fs.proxy.workstation | none | For NTLM proxies: The Windows workstation name to use when authenticating with the proxy | +| s3fs.region | none | The AWS Region to configure the client | +| s3fs.socket.send.buffer.size.hint | TODO | The size hint (in bytes) for the low level TCP send buffer | +| s3fs.socket.receive.buffer.size.hint | TODO | The size hint (in bytes) for the low level TCP receive buffer | +| s3fs.socket.timeout | TODO | Timeout (in milliseconds) for each read to the underlying socket | +| s3fs.user.agent.prefix | TODO | Prefix of the user agent that is sent with each request to AWS | +| s3fs.amazon.s3.factory.class | TODO | Fully-qualified class name to instantiate a S3 factory base class which creates a S3 client instance | +| s3fs.signer.override | TODO | Fully-qualified class name to define the signer that should be used when authenticating with AWS | +| s3fs.path.style.access | TODO | Boolean that indicates whether the client uses path-style access for all requests | +| s3fs.request.header.cache-control | blank | Configures the `cacheControl` on request builders (i.e. `CopyObjectRequest`, `PutObjectRequest`, etc) | diff --git a/src/main/java/org/carlspring/cloud/storage/s3fs/S3Factory.java b/src/main/java/org/carlspring/cloud/storage/s3fs/S3Factory.java index 00325699..7836e8c5 100644 --- a/src/main/java/org/carlspring/cloud/storage/s3fs/S3Factory.java +++ b/src/main/java/org/carlspring/cloud/storage/s3fs/S3Factory.java @@ -5,6 +5,8 @@ import java.time.Duration; import java.util.Properties; +import org.carlspring.cloud.storage.s3fs.attribute.S3BasicFileAttributes; +import org.carlspring.cloud.storage.s3fs.attribute.S3PosixFileAttributes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; @@ -41,6 +43,18 @@ public abstract class S3Factory public static final String SECRET_KEY = "s3fs.secret.key"; + /** + * Maximum TTL in millis to cache {@link S3BasicFileAttributes} and {@link S3PosixFileAttributes}. + */ + public static final String CACHE_ATTRIBUTES_TTL = "s3fs.cache.attributes.ttl"; + public static final int CACHE_ATTRIBUTES_TTL_DEFAULT = 60000; + + /** + * Total size of {@link S3BasicFileAttributes} and {@link S3PosixFileAttributes} cache. + */ + public static final String CACHE_ATTRIBUTES_SIZE = "s3fs.cache.attributes.size"; + public static final int CACHE_ATTRIBUTES_SIZE_DEFAULT = 30000; + public static final String REQUEST_METRIC_COLLECTOR_CLASS = "s3fs.request.metric.collector.class"; public static final String CONNECTION_TIMEOUT = "s3fs.connection.timeout"; diff --git a/src/main/java/org/carlspring/cloud/storage/s3fs/S3FileSystem.java b/src/main/java/org/carlspring/cloud/storage/s3fs/S3FileSystem.java index 03aed220..9b6ddbb9 100644 --- a/src/main/java/org/carlspring/cloud/storage/s3fs/S3FileSystem.java +++ b/src/main/java/org/carlspring/cloud/storage/s3fs/S3FileSystem.java @@ -1,5 +1,12 @@ package org.carlspring.cloud.storage.s3fs; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import org.carlspring.cloud.storage.s3fs.cache.S3FileAttributesCache; +import org.carlspring.cloud.storage.s3fs.util.S3Utils; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.Bucket; + import java.io.IOException; import java.nio.file.FileStore; import java.nio.file.FileSystem; @@ -10,10 +17,6 @@ import java.util.Properties; import java.util.Set; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import software.amazon.awssdk.services.s3.S3Client; -import software.amazon.awssdk.services.s3.model.Bucket; import static org.carlspring.cloud.storage.s3fs.S3Path.PATH_SEPARATOR; /** @@ -34,7 +37,7 @@ public class S3FileSystem private final String endpoint; - private final int cache; + private S3FileAttributesCache fileAttributesCache; private final Properties properties; @@ -48,8 +51,12 @@ public S3FileSystem(final S3FileSystemProvider provider, this.key = key; this.client = client; this.endpoint = endpoint; - this.cache = 60000; // 1 minute cache for the s3Path this.properties = properties; + + int cacheTTL = Integer.parseInt(String.valueOf(properties.getOrDefault(S3Factory.CACHE_ATTRIBUTES_TTL, S3Factory.CACHE_ATTRIBUTES_TTL_DEFAULT))); + int cacheSize = Integer.parseInt(String.valueOf(properties.getOrDefault(S3Factory.CACHE_ATTRIBUTES_SIZE, S3Factory.CACHE_ATTRIBUTES_SIZE_DEFAULT))); + + this.fileAttributesCache = new S3FileAttributesCache(cacheTTL, cacheSize); } public S3FileSystem(final S3FileSystemProvider provider, @@ -75,6 +82,7 @@ public String getKey() public void close() throws IOException { + this.fileAttributesCache.invalidateAll(); this.provider.close(this); } @@ -171,14 +179,22 @@ public String getEndpoint() return endpoint; } + /** + * @deprecated Use {@link org.carlspring.cloud.storage.s3fs.util.S3Utils#key2Parts(String)} instead. To be removed in one of next majors versions. + * @param keyParts + * @return String[] + */ public String[] key2Parts(String keyParts) { - return keyParts.split(PATH_SEPARATOR); + return S3Utils.key2Parts(keyParts); } - public int getCache() + /** + * @return The {@link S3FileAttributesCache} instance holding the path attributes cache for this file provider. + */ + public S3FileAttributesCache getFileAttributesCache() { - return cache; + return fileAttributesCache; } /** diff --git a/src/main/java/org/carlspring/cloud/storage/s3fs/S3FileSystemProvider.java b/src/main/java/org/carlspring/cloud/storage/s3fs/S3FileSystemProvider.java index 5bd5c485..bed9ddfb 100644 --- a/src/main/java/org/carlspring/cloud/storage/s3fs/S3FileSystemProvider.java +++ b/src/main/java/org/carlspring/cloud/storage/s3fs/S3FileSystemProvider.java @@ -1,13 +1,25 @@ package org.carlspring.cloud.storage.s3fs; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; import org.carlspring.cloud.storage.s3fs.attribute.S3BasicFileAttributeView; import org.carlspring.cloud.storage.s3fs.attribute.S3BasicFileAttributes; import org.carlspring.cloud.storage.s3fs.attribute.S3PosixFileAttributeView; import org.carlspring.cloud.storage.s3fs.attribute.S3PosixFileAttributes; import org.carlspring.cloud.storage.s3fs.util.AttributesUtils; -import org.carlspring.cloud.storage.s3fs.util.Cache; import org.carlspring.cloud.storage.s3fs.util.Constants; import org.carlspring.cloud.storage.s3fs.util.S3Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.core.ResponseInputStream; +import software.amazon.awssdk.core.exception.SdkException; +import software.amazon.awssdk.core.internal.util.Mimetype; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.*; +import software.amazon.awssdk.utils.StringUtils; import java.io.IOException; import java.io.InputStream; @@ -19,91 +31,18 @@ import java.nio.channels.AsynchronousFileChannel; import java.nio.channels.SeekableByteChannel; import java.nio.charset.StandardCharsets; -import java.nio.file.AccessMode; -import java.nio.file.AtomicMoveNotSupportedException; -import java.nio.file.CopyOption; -import java.nio.file.DirectoryStream; -import java.nio.file.FileAlreadyExistsException; -import java.nio.file.FileStore; -import java.nio.file.FileSystem; -import java.nio.file.FileSystemAlreadyExistsException; -import java.nio.file.FileSystemNotFoundException; -import java.nio.file.Files; -import java.nio.file.LinkOption; -import java.nio.file.NoSuchFileException; -import java.nio.file.OpenOption; -import java.nio.file.Path; -import java.nio.file.StandardCopyOption; -import java.nio.file.StandardOpenOption; -import java.nio.file.attribute.BasicFileAttributeView; -import java.nio.file.attribute.BasicFileAttributes; -import java.nio.file.attribute.FileAttribute; -import java.nio.file.attribute.FileAttributeView; -import java.nio.file.attribute.PosixFileAttributeView; -import java.nio.file.attribute.PosixFileAttributes; +import java.nio.file.*; +import java.nio.file.attribute.*; import java.nio.file.spi.FileSystemProvider; -import java.util.ArrayDeque; -import java.util.Arrays; -import java.util.Deque; -import java.util.EnumSet; -import java.util.HashMap; -import java.util.Iterator; -import java.util.LinkedHashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.Set; +import java.util.*; +import java.util.concurrent.CompletionException; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutorService; -import java.util.stream.Collectors; -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Sets; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import software.amazon.awssdk.core.ResponseInputStream; -import software.amazon.awssdk.core.exception.SdkException; -import software.amazon.awssdk.core.internal.util.Mimetype; -import software.amazon.awssdk.core.sync.RequestBody; -import software.amazon.awssdk.services.s3.S3Client; -import software.amazon.awssdk.services.s3.model.Bucket; -import software.amazon.awssdk.services.s3.model.CopyObjectRequest; -import software.amazon.awssdk.services.s3.model.CreateBucketRequest; -import software.amazon.awssdk.services.s3.model.Delete; -import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest; -import software.amazon.awssdk.services.s3.model.GetObjectRequest; -import software.amazon.awssdk.services.s3.model.GetObjectResponse; -import software.amazon.awssdk.services.s3.model.ObjectIdentifier; -import software.amazon.awssdk.services.s3.model.PutObjectRequest; -import software.amazon.awssdk.services.s3.model.S3Exception; -import software.amazon.awssdk.services.s3.model.S3Object; -import software.amazon.awssdk.utils.StringUtils; import static com.google.common.collect.Sets.difference; import static java.lang.String.format; -import static org.carlspring.cloud.storage.s3fs.S3Factory.ACCESS_KEY; -import static org.carlspring.cloud.storage.s3fs.S3Factory.CONNECTION_TIMEOUT; -import static org.carlspring.cloud.storage.s3fs.S3Factory.MAX_CONNECTIONS; -import static org.carlspring.cloud.storage.s3fs.S3Factory.MAX_ERROR_RETRY; -import static org.carlspring.cloud.storage.s3fs.S3Factory.PATH_STYLE_ACCESS; -import static org.carlspring.cloud.storage.s3fs.S3Factory.PROTOCOL; -import static org.carlspring.cloud.storage.s3fs.S3Factory.PROXY_DOMAIN; -import static org.carlspring.cloud.storage.s3fs.S3Factory.PROXY_HOST; -import static org.carlspring.cloud.storage.s3fs.S3Factory.PROXY_PASSWORD; -import static org.carlspring.cloud.storage.s3fs.S3Factory.PROXY_PORT; -import static org.carlspring.cloud.storage.s3fs.S3Factory.PROXY_USERNAME; -import static org.carlspring.cloud.storage.s3fs.S3Factory.PROXY_WORKSTATION; -import static org.carlspring.cloud.storage.s3fs.S3Factory.REGION; -import static org.carlspring.cloud.storage.s3fs.S3Factory.REQUEST_METRIC_COLLECTOR_CLASS; -import static org.carlspring.cloud.storage.s3fs.S3Factory.SECRET_KEY; -import static org.carlspring.cloud.storage.s3fs.S3Factory.SIGNER_OVERRIDE; -import static org.carlspring.cloud.storage.s3fs.S3Factory.SOCKET_RECEIVE_BUFFER_SIZE_HINT; -import static org.carlspring.cloud.storage.s3fs.S3Factory.SOCKET_SEND_BUFFER_SIZE_HINT; -import static org.carlspring.cloud.storage.s3fs.S3Factory.SOCKET_TIMEOUT; -import static org.carlspring.cloud.storage.s3fs.S3Factory.USER_AGENT; +import static org.carlspring.cloud.storage.s3fs.S3Factory.*; import static software.amazon.awssdk.http.Header.CONTENT_TYPE; import static software.amazon.awssdk.http.HttpStatusCode.NOT_FOUND; @@ -148,6 +87,8 @@ public class S3FileSystemProvider private static final List PROPS_TO_OVERLOAD = Arrays.asList(ACCESS_KEY, SECRET_KEY, + CACHE_ATTRIBUTES_TTL, + CACHE_ATTRIBUTES_SIZE, REQUEST_METRIC_COLLECTOR_CLASS, CONNECTION_TIMEOUT, MAX_CONNECTIONS, @@ -174,9 +115,6 @@ public class S3FileSystemProvider private final S3Utils s3Utils = new S3Utils(); - private Cache cache = new Cache(); - - @Override public String getScheme() { @@ -487,7 +425,7 @@ private S3Path toS3Path(Path path) @Override public Path getPath(URI uri) { - FileSystem fileSystem = getFileSystem(uri); + S3FileSystem fileSystem = getFileSystem(uri); /** * TODO: set as a list. one s3FileSystem by region @@ -605,7 +543,7 @@ public OutputStream newOutputStream(final Path path, S3FileSystem fileSystem = s3Path.getFileSystem(); - return new S3OutputStream(fileSystem.getClient(), s3Path.toS3ObjectId(), null, metadata, fileSystem.getRequestHeaderCacheControlProperty()); + return new S3OutputStream(fileSystem.getClient(), s3Path.toS3ObjectId(), null, metadata, fileSystem.getRequestHeaderCacheControlProperty(), s3Path.getFileAttributesCache()); } private void validateCreateAndTruncateOptions(final Path path, @@ -706,9 +644,12 @@ public void createDirectory(Path dir, .key(directoryKey) .cacheControl(s3Path.getFileSystem().getRequestHeaderCacheControlProperty()) .contentLength(0L) + .contentType("application/x-directory") .build(); - client.putObject(request, RequestBody.fromBytes(new byte[0])); + client.putObject(request, RequestBody.empty()); + + s3Path.getFileAttributesCache().invalidate(s3Path); } @Override @@ -733,45 +674,31 @@ private void deleteBatch(S3Client client, throws IOException { - List keys = batch.stream() - .map(s3Path -> ObjectIdentifier.builder() - .key(s3Path.getKey()) - .build()) - .collect(Collectors.toList()); + // Create a combined list of ObjectIdentifiers in one loop + List objectIdentifiers = new ArrayList<>(batch.size() * 2); + for (S3Path s3Path : batch) { + // The original key + objectIdentifiers.add(ObjectIdentifier.builder().key(s3Path.getKey()).build()); + // The key with '/' appended + objectIdentifiers.add(ObjectIdentifier.builder().key(s3Path.getKey() + '/').build()); + } + + // Create the multi-object delete request with both sets of keys DeleteObjectsRequest multiObjectDeleteRequest = DeleteObjectsRequest.builder() .bucket(bucketName) .delete(Delete.builder() - .objects(keys) + .objects(objectIdentifiers) .build()) .build(); - try - { - client.deleteObjects(multiObjectDeleteRequest); - } - catch (SdkException e) - { - throw new IOException(e); - } - - // we delete the two objects (sometimes exists the key '/' and sometimes not) - keys = batch.stream() - .map(s3Path -> ObjectIdentifier.builder().key(s3Path.getKey() + '/').build()) - .collect(Collectors.toList()); - multiObjectDeleteRequest = DeleteObjectsRequest.builder() - .bucket(bucketName) - .delete(Delete.builder() - .objects(keys) - .build()) - .build(); - - try - { + // Try to delete all objects at once + try { client.deleteObjects(multiObjectDeleteRequest); - } - catch (SdkException e) - { + for (S3Path path : batch) { + path.getFileAttributesCache().invalidate(path); + } + } catch (SdkException e) { throw new IOException(e); } } @@ -855,15 +782,16 @@ public void copy(Path source, S3Path s3Source = toS3Path(source); S3Path s3Target = toS3Path(target); - // TODO: implements support for copying directories + // TODO: implements support for copying directories Preconditions.checkArgument(!Files.isDirectory(source), "copying directories is not yet supported: %s", source); Preconditions.checkArgument(!Files.isDirectory(target), "copying directories is not yet supported: %s", target); ImmutableSet actualOptions = ImmutableSet.copyOf(options); verifySupportedOptions(EnumSet.of(StandardCopyOption.REPLACE_EXISTING), actualOptions); - if (exists(s3Target) && !actualOptions.contains(StandardCopyOption.REPLACE_EXISTING)) + // Better check before if we want to replace the file (default) or not to avoid a call to exists + if (!actualOptions.contains(StandardCopyOption.REPLACE_EXISTING) && exists(s3Target)) { throw new FileAlreadyExistsException(format("target already exists: %s", target)); } @@ -884,6 +812,9 @@ public void copy(Path source, .build(); client.copyObject(request); + + s3Source.getFileAttributesCache().invalidate(s3Source); + s3Source.getFileAttributesCache().invalidate(s3Target); } private String encodeUrl(final String bucketNameOrigin, @@ -993,39 +924,22 @@ public A readAttributes(Path path, LinkOption... options) throws IOException { - S3Path s3Path = toS3Path(path); - if (type == BasicFileAttributes.class) + try { - if (cache.isInTime(s3Path.getFileSystem().getCache(), s3Path.getFileAttributes())) - { - A result = type.cast(s3Path.getFileAttributes()); - s3Path.setFileAttributes(null); - - return result; - } - else + S3Path s3Path = toS3Path(path); + if (type == BasicFileAttributes.class || type == S3BasicFileAttributes.class || + type == PosixFileAttributes.class || type == S3PosixFileAttributes.class) { - S3BasicFileAttributes attrs = s3Utils.getS3FileAttributes(s3Path); - s3Path.setFileAttributes(attrs); - + S3BasicFileAttributes attrs = s3Path.getFileAttributesCache().get(s3Path, type); + if(attrs == null) { + throw new NoSuchFileException(path.toString()); + } return type.cast(attrs); } } - else if (type == PosixFileAttributes.class) + catch (CompletionException e) { - if (s3Path.getFileAttributes() instanceof PosixFileAttributes && - cache.isInTime(s3Path.getFileSystem().getCache(), s3Path.getFileAttributes())) - { - A result = type.cast(s3Path.getFileAttributes()); - s3Path.setFileAttributes(null); - - return result; - } - - S3PosixFileAttributes attrs = s3Utils.getS3PosixFileAttributes(s3Path); - s3Path.setFileAttributes(attrs); - - return type.cast(attrs); + throw new IOException(e); } throw new UnsupportedOperationException(format("only %s or %s supported", @@ -1192,36 +1106,17 @@ boolean exists(S3Path path) { S3Path s3Path = toS3Path(path); - if (isBucketRoot(s3Path)) - { - // check to see if bucket exists - try - { - s3Utils.listS3Objects(s3Path); - return true; - } - catch (SdkException e) - { - return false; - } - } + S3BasicFileAttributes attrs = s3Path.getFileAttributesCache() + .get(s3Path, BasicFileAttributes.class); - try - { - s3Utils.getS3Object(s3Path); - - return true; - } - catch (NoSuchFileException e) - { - return false; - } + return attrs != null; } public void close(S3FileSystem fileSystem) { if (fileSystem.getKey() != null && fileSystems.containsKey(fileSystem.getKey())) { + fileSystem.getFileAttributesCache().invalidateAll(); fileSystems.remove(fileSystem.getKey()); } } @@ -1240,14 +1135,4 @@ protected static ConcurrentMap getFilesystems() return fileSystems; } - public Cache getCache() - { - return cache; - } - - public void setCache(Cache cache) - { - this.cache = cache; - } - } diff --git a/src/main/java/org/carlspring/cloud/storage/s3fs/S3Iterator.java b/src/main/java/org/carlspring/cloud/storage/s3fs/S3Iterator.java index 60948ec7..5bdf3334 100644 --- a/src/main/java/org/carlspring/cloud/storage/s3fs/S3Iterator.java +++ b/src/main/java/org/carlspring/cloud/storage/s3fs/S3Iterator.java @@ -1,7 +1,9 @@ package org.carlspring.cloud.storage.s3fs; +import org.carlspring.cloud.storage.s3fs.cache.S3FileAttributesCache; import org.carlspring.cloud.storage.s3fs.util.S3Utils; +import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; @@ -32,6 +34,8 @@ public class S3Iterator private final S3FileSystem fileSystem; + private final S3FileAttributesCache fileAttributesCache; + private final S3FileStore fileStore; private final String key; @@ -50,6 +54,7 @@ public class S3Iterator private final S3Utils s3Utils = new S3Utils(); + private int totalProcessed = 0; public S3Iterator(S3Path path) { @@ -69,6 +74,7 @@ public S3Iterator(S3FileStore fileStore, String key, boolean incremental) this.fileStore = fileStore; this.fileSystem = fileStore.getFileSystem(); + this.fileAttributesCache = fileSystem.getFileAttributesCache(); this.key = key; this.current = fileSystem.getClient().listObjectsV2(listObjectsV2Request); this.incremental = incremental; @@ -90,11 +96,11 @@ public S3Path next() ListObjectsV2Request request = ListObjectsV2Request.builder() .bucket(fileStore.name()) .prefix(key) + .fetchOwner(true) .continuationToken(current.nextContinuationToken()) .build(); - final S3Client client = fileSystem.getClient(); - this.current = client.listObjectsV2(request); + this.current = fileSystem.getClient().listObjectsV2(request); loadObjects(); } @@ -104,6 +110,8 @@ public S3Path next() throw new NoSuchElementException(); } + ++totalProcessed; + return items.get(cursor++); } @@ -113,6 +121,11 @@ public void remove() throw new UnsupportedOperationException(); } + public int getTotalProcessed() + { + return totalProcessed; + } + private void loadObjects() { this.items.clear(); @@ -136,7 +149,7 @@ private void parseObjects() { final String objectKey = object.key(); - String[] keyParts = fileSystem.key2Parts(objectKey); + String[] keyParts = S3Utils.key2Parts(objectKey); addParentPaths(keyParts); @@ -202,8 +215,13 @@ private void parseObjectListing(String key, List listPath, ListObjectsV2 { if (!commonPrefix.prefix().equals("/")) { - listPath.add(new S3Path(fileSystem, "/" + fileStore.name(), - fileSystem.key2Parts(commonPrefix.prefix()))); + S3Path s3Path = new S3Path(fileSystem, "/" + fileStore.name(), S3Utils.key2Parts(commonPrefix.prefix())); + listPath.add(s3Path); + try { + fileAttributesCache.put(s3Path, s3Utils.getS3FileAttributes(s3Path)); + } catch (NoSuchFileException e) { //NOPMD + //NOPMD + } } } @@ -218,12 +236,12 @@ private void parseObjectListing(String key, List listPath, ListObjectsV2 { S3Path descendentPart = new S3Path(fileSystem, "/" + fileStore.name(), - fileSystem.key2Parts(immediateDescendantKey)); + S3Utils.key2Parts(immediateDescendantKey)); - descendentPart.setFileAttributes(s3Utils.toS3FileAttributes(object, descendentPart.getKey())); if (!listPath.contains(descendentPart)) { listPath.add(descendentPart); + fileAttributesCache.put(descendentPart, s3Utils.toS3FileAttributes(object, descendentPart.getKey())); } } } diff --git a/src/main/java/org/carlspring/cloud/storage/s3fs/S3ObjectId.java b/src/main/java/org/carlspring/cloud/storage/s3fs/S3ObjectId.java index 607c4832..877df8d9 100644 --- a/src/main/java/org/carlspring/cloud/storage/s3fs/S3ObjectId.java +++ b/src/main/java/org/carlspring/cloud/storage/s3fs/S3ObjectId.java @@ -68,7 +68,7 @@ public String toString() return "bucket: " + bucket + ", key: " + key; } - static final class Builder + public static final class Builder { private String bucket; diff --git a/src/main/java/org/carlspring/cloud/storage/s3fs/S3OutputStream.java b/src/main/java/org/carlspring/cloud/storage/s3fs/S3OutputStream.java index fef162c8..36291b4d 100644 --- a/src/main/java/org/carlspring/cloud/storage/s3fs/S3OutputStream.java +++ b/src/main/java/org/carlspring/cloud/storage/s3fs/S3OutputStream.java @@ -5,6 +5,8 @@ import java.io.InputStream; import java.io.OutputStream; import java.io.SequenceInputStream; +import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.attribute.PosixFileAttributes; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -14,6 +16,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; +import org.carlspring.cloud.storage.s3fs.cache.S3FileAttributesCache; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.core.exception.SdkException; @@ -75,6 +78,11 @@ public final class S3OutputStream */ private final Map metadata; + /** + * File attribute cache + */ + private final S3FileAttributesCache fileAttributesCache; + /** * Indicates if the stream has been closed. */ @@ -114,11 +122,7 @@ public final class S3OutputStream public S3OutputStream(final S3Client s3Client, final S3ObjectId objectId) { - this.s3Client = requireNonNull(s3Client); - this.objectId = requireNonNull(objectId); - this.metadata = new HashMap<>(); - this.storageClass = null; - this.requestCacheControlHeader = ""; + this(s3Client, objectId, null, new HashMap<>(), "", new S3FileAttributesCache(S3Factory.CACHE_ATTRIBUTES_TTL_DEFAULT, S3Factory.CACHE_ATTRIBUTES_SIZE_DEFAULT)); } /** @@ -134,11 +138,7 @@ public S3OutputStream(final S3Client s3Client, final S3ObjectId objectId, final StorageClass storageClass) { - this.s3Client = requireNonNull(s3Client); - this.objectId = requireNonNull(objectId); - this.metadata = new HashMap<>(); - this.storageClass = storageClass; - this.requestCacheControlHeader = ""; + this(s3Client, objectId, storageClass, new HashMap<>(), ""); } /** @@ -155,11 +155,7 @@ public S3OutputStream(final S3Client s3Client, final S3ObjectId objectId, final Map metadata) { - this.s3Client = requireNonNull(s3Client); - this.objectId = requireNonNull(objectId); - this.storageClass = null; - this.metadata = new HashMap<>(metadata); - this.requestCacheControlHeader = ""; + this(s3Client, objectId, null, metadata, "", new S3FileAttributesCache(S3Factory.CACHE_ATTRIBUTES_TTL_DEFAULT, S3Factory.CACHE_ATTRIBUTES_SIZE_DEFAULT)); } /** @@ -177,11 +173,7 @@ public S3OutputStream(final S3Client s3Client, final StorageClass storageClass, final Map metadata) { - this.s3Client = requireNonNull(s3Client); - this.objectId = requireNonNull(objectId); - this.storageClass = storageClass; - this.metadata = new HashMap<>(metadata); - this.requestCacheControlHeader = ""; + this(s3Client, objectId, storageClass, metadata, "", new S3FileAttributesCache(S3Factory.CACHE_ATTRIBUTES_TTL_DEFAULT, S3Factory.CACHE_ATTRIBUTES_SIZE_DEFAULT)); } /** @@ -200,12 +192,34 @@ public S3OutputStream(final S3Client s3Client, final StorageClass storageClass, final Map metadata, final String requestCacheControlHeader) + { + this(s3Client, objectId, storageClass, metadata, requestCacheControlHeader, new S3FileAttributesCache(S3Factory.CACHE_ATTRIBUTES_TTL_DEFAULT, S3Factory.CACHE_ATTRIBUTES_SIZE_DEFAULT)); + } + + /** + * Creates a new {@code S3OutputStream} that writes data directly into the S3 object with the given {@code objectId}. + * The given {@code metadata} will be attached to the written object. + * + * @param s3Client S3 ClientAPI to use + * @param objectId ID of the S3 object to store data into + * @param storageClass S3 Client storage class to apply to the newly created S3 object, if any + * @param metadata metadata to attach to the written object + * @param requestCacheControlHeader Controls + * @throws NullPointerException if at least one parameter except {@code storageClass} is {@code null} + */ + public S3OutputStream(final S3Client s3Client, + final S3ObjectId objectId, + final StorageClass storageClass, + final Map metadata, + final String requestCacheControlHeader, + final S3FileAttributesCache fileAttributesCache) { this.s3Client = requireNonNull(s3Client); this.objectId = requireNonNull(objectId); this.storageClass = storageClass; this.metadata = new HashMap<>(metadata); this.requestCacheControlHeader = requestCacheControlHeader; + this.fileAttributesCache = fileAttributesCache; } //protected for testing purposes @@ -308,10 +322,24 @@ public void close() completeMultipartUpload(); } + invalidateAttributeCache(); + closed.set(true); } } + @Override + public void flush() throws IOException + { + invalidateAttributeCache(); + } + + private void invalidateAttributeCache() + { + fileAttributesCache.invalidate(S3FileAttributesCache.generateCacheKey(objectId, BasicFileAttributes.class)); + fileAttributesCache.invalidate(S3FileAttributesCache.generateCacheKey(objectId, PosixFileAttributes.class)); + } + /** * Creates a multipart upload and gets the upload id. * diff --git a/src/main/java/org/carlspring/cloud/storage/s3fs/S3Path.java b/src/main/java/org/carlspring/cloud/storage/s3fs/S3Path.java index 8908e1cc..608af1d4 100644 --- a/src/main/java/org/carlspring/cloud/storage/s3fs/S3Path.java +++ b/src/main/java/org/carlspring/cloud/storage/s3fs/S3Path.java @@ -12,6 +12,7 @@ import java.nio.file.WatchEvent; import java.nio.file.WatchKey; import java.nio.file.WatchService; +import java.nio.file.attribute.BasicFileAttributes; import java.util.ArrayDeque; import java.util.Deque; import java.util.Iterator; @@ -22,6 +23,7 @@ import com.google.common.base.Splitter; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; +import org.carlspring.cloud.storage.s3fs.cache.S3FileAttributesCache; import software.amazon.awssdk.core.Protocol; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.S3Utilities; @@ -53,10 +55,9 @@ public class S3Path private final S3FileSystem fileSystem; /** - * S3BasicFileAttributes cache + * S3FileAttributesCache cache */ - private S3BasicFileAttributes fileAttributes; - + private S3FileAttributesCache fileAttributesCache; /** * Build an S3Path from path segments. '/' are stripped from each segment. @@ -121,6 +122,7 @@ public S3Path(S3FileSystem fileSystem, String first, String... more) } this.uri = localUri; this.fileSystem = fileSystem; + this.fileAttributesCache = fileSystem.getFileAttributesCache(); } /** @@ -829,14 +831,18 @@ private String decode(URI uri) } } - public S3BasicFileAttributes getFileAttributes() + public S3BasicFileAttributes getFileAttributes(Class type) { - return fileAttributes; + return fileAttributesCache.get(this, type); } - public void setFileAttributes(S3BasicFileAttributes fileAttributes) + /** + * Shortcut to getFileSystem().getFileAttributesCache() + * @return + */ + public S3FileAttributesCache getFileAttributesCache() { - this.fileAttributes = fileAttributes; + return fileAttributesCache; } } diff --git a/src/main/java/org/carlspring/cloud/storage/s3fs/S3SeekableByteChannel.java b/src/main/java/org/carlspring/cloud/storage/s3fs/S3SeekableByteChannel.java index e77f6735..5dba47d9 100644 --- a/src/main/java/org/carlspring/cloud/storage/s3fs/S3SeekableByteChannel.java +++ b/src/main/java/org/carlspring/cloud/storage/s3fs/S3SeekableByteChannel.java @@ -184,6 +184,9 @@ protected void sync() S3Client client = path.getFileSystem().getClient(); client.putObject(builder.build(), RequestBody.fromInputStream(stream, length)); + + // Invalidate cache as writing to the object would change some metadata. + path.getFileAttributesCache().invalidate(path); } } diff --git a/src/main/java/org/carlspring/cloud/storage/s3fs/attribute/S3PosixFileAttributeView.java b/src/main/java/org/carlspring/cloud/storage/s3fs/attribute/S3PosixFileAttributeView.java index 6834259d..3cb1e48d 100644 --- a/src/main/java/org/carlspring/cloud/storage/s3fs/attribute/S3PosixFileAttributeView.java +++ b/src/main/java/org/carlspring/cloud/storage/s3fs/attribute/S3PosixFileAttributeView.java @@ -16,9 +16,6 @@ public class S3PosixFileAttributeView private static final Logger log = LoggerFactory.getLogger(S3PosixFileAttributeView.class); private S3Path s3Path; - private PosixFileAttributes posixFileAttributes; - - public S3PosixFileAttributeView(S3Path s3Path) { this.s3Path = s3Path; @@ -75,12 +72,7 @@ public void setTimes(FileTime lastModifiedTime, FileTime lastAccessTime, FileTim public PosixFileAttributes read() throws IOException { - if (posixFileAttributes == null) - { - posixFileAttributes = s3Path.getFileSystem().provider().readAttributes(s3Path, PosixFileAttributes.class); - } - - return posixFileAttributes; + return s3Path.getFileSystem().provider().readAttributes(s3Path, PosixFileAttributes.class); } } diff --git a/src/main/java/org/carlspring/cloud/storage/s3fs/cache/S3FileAttributesCache.java b/src/main/java/org/carlspring/cloud/storage/s3fs/cache/S3FileAttributesCache.java new file mode 100644 index 00000000..058c6001 --- /dev/null +++ b/src/main/java/org/carlspring/cloud/storage/s3fs/cache/S3FileAttributesCache.java @@ -0,0 +1,285 @@ +package org.carlspring.cloud.storage.s3fs.cache; + +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; +import com.github.benmanes.caffeine.cache.RemovalCause; +import com.github.benmanes.caffeine.cache.stats.CacheStats; +import org.carlspring.cloud.storage.s3fs.S3ObjectId; +import org.carlspring.cloud.storage.s3fs.S3Path; +import org.carlspring.cloud.storage.s3fs.attribute.S3BasicFileAttributes; +import org.carlspring.cloud.storage.s3fs.attribute.S3PosixFileAttributes; +import org.carlspring.cloud.storage.s3fs.util.S3Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.file.NoSuchFileException; +import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.attribute.PosixFileAttributes; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CompletionException; + +public class S3FileAttributesCache +{ + + private static final Logger logger = LoggerFactory.getLogger(S3FileAttributesCache.class); + + private final S3Utils s3Utils = new S3Utils(); + + // This should be volatile, despite what IntelliJ / Sonar says. + // When using `Files.exists` / `Files.isDirectory` / etc -- different threads will be entering this class + // which could cause problems. + private volatile Cache> cache; + + /** + * @param cacheTTL TTL in milliseconds + * @param cacheSize Total cache size. + */ + public S3FileAttributesCache(int cacheTTL, int cacheSize) + { + this.cache = cacheBuilder(cacheTTL, cacheSize).build(); + } + + /** + * Generates a cache key based on S3Path and the attribute class type. + * The key is a combination of the S3Path's hashCode and the attribute class name. + * + * @param path The {@link S3Path}. + * @param attributeClass The class type of {@link BasicFileAttributes}. + * @return A unique string key. + */ + public static String generateCacheKey(S3Path path, Class attributeClass) + { + S3ObjectId s3ObjectId = path.toS3ObjectId(); + return generateCacheKey(s3ObjectId, attributeClass); + } + + /** + * Generates a cache key based on S3Path and the attribute class type. + * The key is a combination of the S3Path's hashCode and the attribute class name. + * + * @param s3ObjectId An {@link software.amazon.awssdk.services.s3.model.S3Object} instance. + * @param attributeClass The class type of {@link BasicFileAttributes}. + * @return A unique string key. + */ + public static String generateCacheKey(S3ObjectId s3ObjectId, Class attributeClass) + { + StringBuilder key = new StringBuilder(); + key.append(s3ObjectId.getBucket().replaceAll("/", "%2F")) + .append("_") + .append(s3ObjectId.getKey().replaceAll("/", "%2F")) + .append("_"); + + if (attributeClass == BasicFileAttributes.class) { + key.append(S3BasicFileAttributes.class.getSimpleName()); + } else if (attributeClass == PosixFileAttributes.class) { + key.append(S3PosixFileAttributes.class.getSimpleName()); + } else { + key.append(attributeClass.getSimpleName()); + } + + return key.toString(); + } + + + /** + * Retrieves the file attributes of the given S3Path (either BasicFileAttributes or PosixFileAttributes) + * + * @param path The {@link S3Path} + * + * @return The {@link S3BasicFileAttributes} or {@link S3PosixFileAttributes} for the given {@link S3Path}. Is null + * when `attrType` is not {@link BasicFileAttributes}, {@link PosixFileAttributes} or the path does not exist. + * + * @throws CompletionException if a checked exception was thrown while loading the value from AWS. + */ + public S3BasicFileAttributes get(final S3Path path, final Class attrType) + { + String key = generateCacheKey(path, attrType); + logger.trace("Get cache for key {}", key); + + Optional attrs = cache.getIfPresent(key); + + // Don't get confused - Caffeine returns `null` if the key does not exist. + if(attrs == null) + { + logger.trace("No cache found for key {}", key); + // We need a way to preserve non-existing files/paths. + // This is necessary, because the Files.exist() method is called multiple times from different threads + // during checks. As a result multiple requests for the same path are executed within milliseconds. + logger.trace("Fetch data for key {}", key); + attrs = Optional.ofNullable(fetchAttribute(path, key)); + put(path, attrs); + } + + return attrs.orElse(null); + } + + public boolean contains(final S3Path path, final Class attrType) + { + String key = generateCacheKey(path, attrType); + return contains(key); + } + + public boolean contains(final String key) + { + return cache.asMap().containsKey(key); + } + + + /** + * @param path The S3 path. + * @param attrs the file attributes to store in the cache. Can be the posix ones + */ + public void put(final S3Path path, final S3BasicFileAttributes attrs) + { + put(path, Optional.ofNullable(attrs)); + } + + /** + * @param path The S3 path. + * @param attrs the file attributes to store in the cache. Can be the posix ones + */ + public void put(final S3Path path, final Optional attrs) + { + // There is an off-chance we could have both BasicFileAttributes and PosixFileAttributes cached at different times. + // This could cause a temporary situation where the cache serves slightly outdated instance of BasicFileAttributes. + // To ensure this does not happen we always need to replace the BasicFileAttributes instances when + // the PosixFileAttributes type is cached/updated. + String basicKey = generateCacheKey(path, BasicFileAttributes.class); + logger.trace("Save response for key {}", basicKey); + cache.put(basicKey, attrs); + + if(attrs.isPresent() && attrs.get() instanceof PosixFileAttributes) + { + String posixKey = generateCacheKey(path, PosixFileAttributes.class); + logger.trace("Save response for key {}", posixKey); + cache.put(posixKey, attrs); + } + } + + /** + * Invalidates the file attributes in the cache for the given s3Path + * + * @param path The S3 path. + */ + public void invalidate(final S3Path path, final Class attrType) + { + String key = generateCacheKey(path, attrType); + logger.trace("Invalidate cache key {}", key); + cache.invalidate(key); + } + + /** + * Invalidates the file attributes in the cache for the given s3Path + * + * @param key The cache key + */ + public void invalidate(final String key) + { + cache.invalidate(key); + } + + public void invalidate(final S3Path path) + { + invalidate(path.toS3ObjectId()); + } + + public void invalidate(final S3ObjectId objectId) + { + List keys = new ArrayList<>(); + + keys.add(generateCacheKey(objectId, BasicFileAttributes.class)); + keys.add(generateCacheKey(objectId, PosixFileAttributes.class)); + + /** + * This handles an edge case - depending on where the code is triggered from, there might be a fallback check + * that attempts to resolve a file OR virtual directory. We need to invalidate the cache for both when + * this method is called. + */ + if (objectId.getKey().endsWith("/")) + { + S3ObjectId fileObjectId = S3ObjectId.builder() + .bucket(objectId.getBucket()) + .key(objectId.getKey().substring(0, objectId.getKey().length() - 1)) + .build(); + + keys.add(generateCacheKey(fileObjectId, BasicFileAttributes.class)); + keys.add(generateCacheKey(fileObjectId, PosixFileAttributes.class)); + } + else + { + S3ObjectId fileObjectId = S3ObjectId.builder() + .bucket(objectId.getBucket()) + .key(objectId.getKey() + S3Path.PATH_SEPARATOR) + .build(); + + keys.add(generateCacheKey(fileObjectId, BasicFileAttributes.class)); + keys.add(generateCacheKey(fileObjectId, PosixFileAttributes.class)); + } + + + for (String key : keys) + { + try + { + logger.trace("Invalidate cache key {}", key); + cache.invalidate(key); + } + catch (NullPointerException e) + { + // noop + } + } + + } + + + public void invalidateAll() + { + logger.trace("Invalidate all cache"); + cache.invalidateAll(); + } + + public CacheStats stats() + { + return cache.stats(); + } + + protected Caffeine> cacheBuilder(int cacheTTL, int cacheSize) + { + Caffeine> builder = Caffeine.newBuilder() + .expireAfter(new S3FileAttributesCachePolicy(cacheTTL)); + + builder.maximumSize(cacheSize); + builder.recordStats(); + builder.evictionListener((String key, Optional value, RemovalCause cause) -> + logger.trace("Key {} was evicted (reason: {})", key, cause)); + builder.removalListener((String key, Optional value, RemovalCause cause) -> + logger.trace("Key {} was removed (reason: {})", key, cause)); + + return builder; + } + + protected S3BasicFileAttributes fetchAttribute(S3Path path, String key) + { + try + { + if (key.contains(BasicFileAttributes.class.getSimpleName())) + { + return s3Utils.getS3FileAttributes(path); + } + else if (key.contains(PosixFileAttributes.class.getSimpleName())) + { + return s3Utils.getS3PosixFileAttributes(path); + } + return null; + } + catch (NoSuchFileException e) + { + return null; + } + } + + +} diff --git a/src/main/java/org/carlspring/cloud/storage/s3fs/cache/S3FileAttributesCachePolicy.java b/src/main/java/org/carlspring/cloud/storage/s3fs/cache/S3FileAttributesCachePolicy.java new file mode 100644 index 00000000..9302045e --- /dev/null +++ b/src/main/java/org/carlspring/cloud/storage/s3fs/cache/S3FileAttributesCachePolicy.java @@ -0,0 +1,49 @@ +package org.carlspring.cloud.storage.s3fs.cache; + +import com.github.benmanes.caffeine.cache.Expiry; +import org.carlspring.cloud.storage.s3fs.attribute.S3BasicFileAttributes; + +import java.util.Optional; +import java.util.concurrent.TimeUnit; + +public class S3FileAttributesCachePolicy implements Expiry> +{ + + private int cacheTTL; + + public S3FileAttributesCachePolicy(int cacheTTL) + { + this.cacheTTL = cacheTTL; + } + + public int getTTL() + { + return cacheTTL; + } + + public void setTTL(int cacheTTL) + { + this.cacheTTL = cacheTTL; + } + + @Override + public long expireAfterCreate(String key, Optional value, long currentTime) + { + // Set initial TTL upon creation + return TimeUnit.MILLISECONDS.toNanos(cacheTTL); + } + + @Override + public long expireAfterUpdate(String key, Optional value, long currentTime, long currentDuration) + { + // Reset TTL on update + return TimeUnit.MILLISECONDS.toNanos(cacheTTL); + } + + @Override + public long expireAfterRead(String key, Optional value, long currentTime, long currentDuration) + { + // Use already assigned TTL. + return currentDuration; + } +} diff --git a/src/main/java/org/carlspring/cloud/storage/s3fs/util/Cache.java b/src/main/java/org/carlspring/cloud/storage/s3fs/util/Cache.java deleted file mode 100644 index 5340bc2b..00000000 --- a/src/main/java/org/carlspring/cloud/storage/s3fs/util/Cache.java +++ /dev/null @@ -1,35 +0,0 @@ -package org.carlspring.cloud.storage.s3fs.util; - -import org.carlspring.cloud.storage.s3fs.attribute.S3BasicFileAttributes; - -public class Cache -{ - - /** - * check if the cache of the S3FileAttributes is still valid - * - * @param cache int cache time of the fileAttributes in milliseconds - * @param fileAttributes S3FileAttributes to check if is still valid, can be null - * @return true or false, if cache are -1 and fileAttributes are not null then always return true - */ - public boolean isInTime(int cache, S3BasicFileAttributes fileAttributes) - { - if (fileAttributes == null) - { - return false; - } - - if (cache == -1) - { - return true; - } - - return getCurrentTime() - cache <= fileAttributes.getCacheCreated(); - } - - public long getCurrentTime() - { - return System.currentTimeMillis(); - } - -} diff --git a/src/main/java/org/carlspring/cloud/storage/s3fs/util/S3Utils.java b/src/main/java/org/carlspring/cloud/storage/s3fs/util/S3Utils.java index f9113537..db1ee07b 100644 --- a/src/main/java/org/carlspring/cloud/storage/s3fs/util/S3Utils.java +++ b/src/main/java/org/carlspring/cloud/storage/s3fs/util/S3Utils.java @@ -31,6 +31,8 @@ import software.amazon.awssdk.services.s3.model.Permission; import software.amazon.awssdk.services.s3.model.S3Exception; import software.amazon.awssdk.services.s3.model.S3Object; + +import static org.carlspring.cloud.storage.s3fs.S3Path.PATH_SEPARATOR; import static software.amazon.awssdk.http.HttpStatusCode.NOT_FOUND; /** @@ -310,4 +312,10 @@ else if (!resolvedKey.equals(key) && resolvedKey.startsWith(key)) return new S3BasicFileAttributes(resolvedKey, lastModifiedTime, size, directory, regularFile); } + public static String[] key2Parts(String keyParts) + { + return keyParts.split(PATH_SEPARATOR); + } + + } diff --git a/src/test/java/org/carlspring/cloud/storage/s3fs/BaseTest.java b/src/test/java/org/carlspring/cloud/storage/s3fs/BaseTest.java index 6ba289ca..dcb09b35 100644 --- a/src/test/java/org/carlspring/cloud/storage/s3fs/BaseTest.java +++ b/src/test/java/org/carlspring/cloud/storage/s3fs/BaseTest.java @@ -29,7 +29,7 @@ public abstract class BaseTest * @return abbreviated fully-qualified class name and method (i.e. o.c.c.s.s.u.BaseIntegrationTest/methodName) or * null if it can't find . */ - protected String getTestBasePath() + protected static String getTestBasePath() { // Filter out noise and leave only `org.carlspring.cloud.storage.s3fs` calls in the stack trace. List elements = Arrays.stream(Thread.currentThread().getStackTrace()) @@ -102,7 +102,7 @@ protected String getTestBasePath() * * @return o.c.c.s.s.u.BaseIntegrationTest/methodName/subPath */ - protected String getTestBasePath(String subPath) + protected static String getTestBasePath(String subPath) { String basePath = getTestBasePath(); @@ -119,7 +119,7 @@ protected String getTestBasePath(String subPath) * * @return */ - protected String getTestBasePath(Class clazz, + protected static String getTestBasePath(Class clazz, String methodName) { return abbreviateClassName(clazz.getName()) + "/" + methodName; @@ -128,7 +128,7 @@ protected String getTestBasePath(Class clazz, /** * @return o.c.c.s.s.u.BaseIntegrationTest/methodName/UUID */ - protected String getTestBasePathWithUUID() + protected static String getTestBasePathWithUUID() { return getTestBasePath(randomUUID().toString()); } @@ -142,7 +142,7 @@ protected String getTestBasePathWithUUID() * * @return */ - protected String getTestBasePathWithUUID(Class clazz, + protected static String getTestBasePathWithUUID(Class clazz, String methodName) { return getTestBasePath(clazz, methodName) + "/" + UUID.randomUUID(); @@ -153,7 +153,7 @@ protected String getTestBasePathWithUUID(Class clazz, * * @return abbreviated version of the fully-qualified class name (i.e. o.c.c.s.s.u.BaseIntegrationTest) */ - private String abbreviateClassName(String fqClassName) + private static String abbreviateClassName(String fqClassName) { return new TargetLengthBasedClassNameAbbreviator(5).abbreviate(fqClassName); } diff --git a/src/test/java/org/carlspring/cloud/storage/s3fs/CacheTest.java b/src/test/java/org/carlspring/cloud/storage/s3fs/CacheTest.java deleted file mode 100644 index f2755b91..00000000 --- a/src/test/java/org/carlspring/cloud/storage/s3fs/CacheTest.java +++ /dev/null @@ -1,60 +0,0 @@ -package org.carlspring.cloud.storage.s3fs; - -import org.carlspring.cloud.storage.s3fs.attribute.S3BasicFileAttributes; -import org.carlspring.cloud.storage.s3fs.util.Cache; - -import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.spy; - -class CacheTest -{ - - @Test - void cacheIsInclusive() - { - Cache cache = spy(new Cache()); - - doReturn(300L).when(cache).getCurrentTime(); - - S3BasicFileAttributes attributes = new S3BasicFileAttributes("key", null, 0, false, true); - attributes.setCacheCreated(0); - - boolean result = cache.isInTime(300, attributes); - - assertTrue(result); - } - - @Test - void outOfTime() - { - Cache cache = spy(new Cache()); - - doReturn(200L).when(cache).getCurrentTime(); - - S3BasicFileAttributes attributes = new S3BasicFileAttributes("key", null, 0, false, true); - attributes.setCacheCreated(0); - - boolean result = cache.isInTime(100, attributes); - - assertFalse(result); - } - - @Test - void infinite() - { - Cache cache = spy(new Cache()); - - doReturn(200L).when(cache).getCurrentTime(); - - S3BasicFileAttributes attributes = new S3BasicFileAttributes("key", null, 0, false, true); - attributes.setCacheCreated(100); - - boolean result = cache.isInTime(-1, attributes); - - assertTrue(result); - } - -} diff --git a/src/test/java/org/carlspring/cloud/storage/s3fs/fileSystemProvider/ReadAttributesTest.java b/src/test/java/org/carlspring/cloud/storage/s3fs/fileSystemProvider/ReadAttributesTest.java index b83f3787..9523a117 100644 --- a/src/test/java/org/carlspring/cloud/storage/s3fs/fileSystemProvider/ReadAttributesTest.java +++ b/src/test/java/org/carlspring/cloud/storage/s3fs/fileSystemProvider/ReadAttributesTest.java @@ -1,20 +1,20 @@ package org.carlspring.cloud.storage.s3fs.fileSystemProvider; +import com.github.benmanes.caffeine.cache.stats.CacheStats; +import com.google.common.collect.Sets; import org.carlspring.cloud.storage.s3fs.S3FileSystem; import org.carlspring.cloud.storage.s3fs.S3Path; import org.carlspring.cloud.storage.s3fs.S3UnitTestBase; +import org.carlspring.cloud.storage.s3fs.cache.S3FileAttributesCache; import org.carlspring.cloud.storage.s3fs.util.MockBucket; import org.carlspring.cloud.storage.s3fs.util.S3ClientMock; import org.carlspring.cloud.storage.s3fs.util.S3EndpointConstant; import org.carlspring.cloud.storage.s3fs.util.S3MockFactory; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import java.io.IOException; -import java.nio.file.FileSystem; -import java.nio.file.FileSystemNotFoundException; -import java.nio.file.FileSystems; -import java.nio.file.Files; -import java.nio.file.NoSuchFileException; -import java.nio.file.Path; +import java.nio.file.*; import java.nio.file.attribute.BasicFileAttributes; import java.nio.file.attribute.DosFileAttributes; import java.nio.file.attribute.PosixFileAttributes; @@ -22,17 +22,9 @@ import java.util.Map; import java.util.concurrent.TimeUnit; -import com.google.common.collect.Sets; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; import static org.carlspring.cloud.storage.s3fs.util.FileAttributeBuilder.build; -import static org.junit.jupiter.api.Assertions.assertArrayEquals; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.*; class ReadAttributesTest extends S3UnitTestBase @@ -173,51 +165,190 @@ void readAttributesDirectoryNotExistsAtAmazon() } @Test - void readAttributesRegenerateCacheWhenNotExists() + void readAttributesRegenerateCacheWhenNotExistsBasic() throws IOException { // fixtures S3ClientMock client = S3MockFactory.getS3ClientMock(); - client.bucket("bucketA").dir("dir").file("dir/file1", "".getBytes()); - - S3Path file1 = createNewS3FileSystem().getPath("/bucketA/dir/file1"); - - // create the cache - s3fsProvider.readAttributes(file1, BasicFileAttributes.class); - - assertNotNull(file1.getFileAttributes()); - - s3fsProvider.readAttributes(file1, BasicFileAttributes.class); - - assertNull(file1.getFileAttributes()); - - s3fsProvider.readAttributes(file1, BasicFileAttributes.class); - - assertNotNull(file1.getFileAttributes()); + client.bucket("bucketA").dir("dir").file("dir/file-basic", "".getBytes()); + + S3FileSystem fs = createNewS3FileSystem(); + + // No cache assertion + S3FileAttributesCache cache = fs.getFileAttributesCache(); + CacheStats stats = cache.stats(); // temporary snapshot + assertThat(stats.hitCount()).isEqualTo(0); + assertThat(stats.missCount()).isEqualTo(0); + + // Pre-requisites (cache entry key should not exist) + S3Path file1 = fs.getPath("/bucketA/dir/file-basic"); + String fileAttrCacheKey = cache.generateCacheKey(file1, BasicFileAttributes.class); + assertThat(cache.contains(fileAttrCacheKey)).isFalse(); + stats = cache.stats(); + assertThat(stats.hitCount()).isEqualTo(0); + assertThat(stats.missCount()).isEqualTo(0); + + // Reading the attributes should create the cache entry. + BasicFileAttributes attrs = s3fsProvider.readAttributes(file1, BasicFileAttributes.class); + assertThat(attrs).isNotNull(); + assertThat(attrs).isEqualTo(file1.getFileAttributes(BasicFileAttributes.class)); + stats = cache.stats(); + assertThat(stats.hitCount()).isEqualTo(1); + assertThat(stats.missCount()).isEqualTo(1); + + // Should hit the cache. + attrs = s3fsProvider.readAttributes(file1, BasicFileAttributes.class); + assertThat(attrs).isNotNull(); + assertThat(attrs).isEqualTo(file1.getFileAttributes(BasicFileAttributes.class)); + stats = cache.stats(); + assertThat(stats.hitCount()).isEqualTo(3); + assertThat(stats.missCount()).isEqualTo(1); + + // Should hit the cache. + attrs = s3fsProvider.readAttributes(file1, BasicFileAttributes.class); + assertThat(attrs).isNotNull(); + assertThat(attrs).isEqualTo(file1.getFileAttributes(BasicFileAttributes.class)); + stats = cache.stats(); + assertThat(stats.hitCount()).isEqualTo(5); + assertThat(stats.missCount()).isEqualTo(1); + + // Invalidate cache manually. + cache.invalidate(fileAttrCacheKey); + assertThat(cache.contains(fileAttrCacheKey)).isFalse(); + + // Should populate the cache again. + attrs = s3fsProvider.readAttributes(file1, BasicFileAttributes.class); + assertThat(attrs).isNotNull(); + assertNotNull(file1.getFileAttributes(BasicFileAttributes.class)); + stats = cache.stats(); + assertThat(stats.hitCount()).isEqualTo(6); + assertThat(stats.missCount()).isEqualTo(2); } @Test - void readAttributesPosixRegenerateCacheWhenNotExists() + void readAttributesRegenerateCacheWhenNotExistsPosix() throws IOException { // fixtures S3ClientMock client = S3MockFactory.getS3ClientMock(); - client.bucket("bucketA").dir("dir").file("dir/file1", "".getBytes()); - - S3Path file1 = createNewS3FileSystem().getPath("/bucketA/dir/file1"); - - // create the cache - s3fsProvider.readAttributes(file1, PosixFileAttributes.class); - - assertNotNull(file1.getFileAttributes()); - - s3fsProvider.readAttributes(file1, PosixFileAttributes.class); - - assertNull(file1.getFileAttributes()); + client.bucket("bucketA").dir("dir").file("dir/file-posix", "".getBytes()); + + S3FileSystem fs = createNewS3FileSystem(); + + // No cache assertion + S3FileAttributesCache cache = fs.getFileAttributesCache(); + CacheStats stats = cache.stats(); // temporary snapshot + assertThat(stats.hitCount()).isEqualTo(0); + assertThat(stats.missCount()).isEqualTo(0); + + // Pre-requisites (cache entry key should not exist) + S3Path file1 = fs.getPath("/bucketA/dir/file-posix"); + String fileAttrCacheKey = cache.generateCacheKey(file1, PosixFileAttributes.class); + assertThat(cache.contains(fileAttrCacheKey)).isFalse(); + stats = cache.stats(); + assertThat(stats.hitCount()).isEqualTo(0); + assertThat(stats.missCount()).isEqualTo(0); + + // Reading the attributes should create the cache entry. + PosixFileAttributes attrs = s3fsProvider.readAttributes(file1, PosixFileAttributes.class); + assertThat(attrs).isNotNull(); + assertThat(attrs).isEqualTo(file1.getFileAttributes(PosixFileAttributes.class)); + stats = cache.stats(); + assertThat(stats.hitCount()).isEqualTo(1); + assertThat(stats.missCount()).isEqualTo(1); + + // Should hit the cache. + attrs = s3fsProvider.readAttributes(file1, PosixFileAttributes.class); + assertThat(attrs).isNotNull(); + assertThat(attrs).isEqualTo(file1.getFileAttributes(PosixFileAttributes.class)); + stats = cache.stats(); + assertThat(stats.hitCount()).isEqualTo(3); + assertThat(stats.missCount()).isEqualTo(1); + + // Should hit the cache. + attrs = s3fsProvider.readAttributes(file1, PosixFileAttributes.class); + assertThat(attrs).isNotNull(); + assertThat(attrs).isEqualTo(file1.getFileAttributes(PosixFileAttributes.class)); + stats = cache.stats(); + assertThat(stats.hitCount()).isEqualTo(5); + assertThat(stats.missCount()).isEqualTo(1); + + // Invalidate cache manually. + cache.invalidate(fileAttrCacheKey); + assertThat(cache.contains(fileAttrCacheKey)).isFalse(); + + // Should populate the cache again. + attrs = s3fsProvider.readAttributes(file1, PosixFileAttributes.class); + assertThat(attrs).isNotNull(); + assertNotNull(file1.getFileAttributes(PosixFileAttributes.class)); + stats = cache.stats(); + assertThat(stats.hitCount()).isEqualTo(6); + assertThat(stats.missCount()).isEqualTo(2); - s3fsProvider.readAttributes(file1, PosixFileAttributes.class); + } - assertNotNull(file1.getFileAttributes()); + @Test + void readAttributesCastDownFromPosixToBasic() + throws IOException + { + // fixtures + S3ClientMock client = S3MockFactory.getS3ClientMock(); + client.bucket("bucketA").dir("dir").file("dir/file-posix2", "".getBytes()); + + S3FileSystem fs = createNewS3FileSystem(); + + // No cache assertion + S3FileAttributesCache cache = fs.getFileAttributesCache(); + CacheStats stats = cache.stats(); // temporary snapshot + assertThat(stats.hitCount()).isEqualTo(0); + assertThat(stats.missCount()).isEqualTo(0); + + // Pre-requisites (cache entry key should not exist) + S3Path file = fs.getPath("/bucketA/dir/file-posix2"); + String basicFileAttrCacheKey = cache.generateCacheKey(file, BasicFileAttributes.class); + String posixFileAttrCacheKey = cache.generateCacheKey(file, PosixFileAttributes.class); + assertThat(cache.contains(basicFileAttrCacheKey)).isFalse(); + assertThat(cache.contains(posixFileAttrCacheKey)).isFalse(); + + stats = cache.stats(); + assertThat(stats.hitCount()).isEqualTo(0); + assertThat(stats.missCount()).isEqualTo(0); + + // Reading the attributes should create the cache entry. + BasicFileAttributes attrs = s3fsProvider.readAttributes(file, PosixFileAttributes.class); + assertThat(attrs).isNotNull(); + assertThat(attrs).isEqualTo(file.getFileAttributes(BasicFileAttributes.class)); + stats = cache.stats(); + assertThat(stats.hitCount()).isEqualTo(1); + assertThat(stats.missCount()).isEqualTo(1); + + // Should hit the cache. + attrs = s3fsProvider.readAttributes(file, BasicFileAttributes.class); + assertThat(attrs).isNotNull(); + assertThat(attrs).isEqualTo(file.getFileAttributes(BasicFileAttributes.class)); + stats = cache.stats(); + assertThat(stats.hitCount()).isEqualTo(3); + assertThat(stats.missCount()).isEqualTo(1); + + // Should hit the cache. + attrs = s3fsProvider.readAttributes(file, BasicFileAttributes.class); + assertThat(attrs).isNotNull(); + assertThat(attrs).isEqualTo(file.getFileAttributes(BasicFileAttributes.class)); + stats = cache.stats(); + assertThat(stats.hitCount()).isEqualTo(5); + assertThat(stats.missCount()).isEqualTo(1); + + // Invalidate cache manually. + cache.invalidate(basicFileAttrCacheKey); + assertThat(cache.contains(basicFileAttrCacheKey)).isFalse(); + + // Should populate the cache again. + attrs = s3fsProvider.readAttributes(file, PosixFileAttributes.class); + assertThat(attrs).isNotNull(); + assertNotNull(file.getFileAttributes(BasicFileAttributes.class)); + stats = cache.stats(); + assertThat(stats.hitCount()).isEqualTo(6); + assertThat(stats.missCount()).isEqualTo(2); } @Test diff --git a/src/test/java/org/carlspring/cloud/storage/s3fs/path/S3PathTest.java b/src/test/java/org/carlspring/cloud/storage/s3fs/path/S3PathTest.java index 5bef07ea..4d76f9fb 100644 --- a/src/test/java/org/carlspring/cloud/storage/s3fs/path/S3PathTest.java +++ b/src/test/java/org/carlspring/cloud/storage/s3fs/path/S3PathTest.java @@ -12,6 +12,8 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; import static org.carlspring.cloud.storage.s3fs.util.S3EndpointConstant.S3_GLOBAL_URI_TEST; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -329,4 +331,17 @@ void registerWatchService() assertNotNull(exception); } + @Test + void sameObjectsMustHaveSameHashCode() + { + S3Path first = forPath("/buck/same"); + S3Path second = forPath("/buck/same"); + S3Path third = forPath("/buck/other"); + + assertThat(first).isEqualTo(second); + assertThat(first.hashCode()).isEqualTo(second.hashCode()); + assertThat(first).isNotEqualTo(third); + assertThat(first.hashCode()).isNotEqualTo(third.hashCode()); + } + } diff --git a/src/test/resources/logback-test.xml b/src/test/resources/logback-test.xml index b94ddca1..13c23c4d 100644 --- a/src/test/resources/logback-test.xml +++ b/src/test/resources/logback-test.xml @@ -17,5 +17,7 @@ + + diff --git a/src/test/java/org/carlspring/cloud/storage/s3fs/BaseIntegrationTest.java b/src/testIntegration/java/org/carlspring/cloud/storage/s3fs/BaseIntegrationTest.java similarity index 100% rename from src/test/java/org/carlspring/cloud/storage/s3fs/BaseIntegrationTest.java rename to src/testIntegration/java/org/carlspring/cloud/storage/s3fs/BaseIntegrationTest.java diff --git a/src/test/java/org/carlspring/cloud/storage/s3fs/FileSystemsIT.java b/src/testIntegration/java/org/carlspring/cloud/storage/s3fs/FileSystemsIT.java similarity index 100% rename from src/test/java/org/carlspring/cloud/storage/s3fs/FileSystemsIT.java rename to src/testIntegration/java/org/carlspring/cloud/storage/s3fs/FileSystemsIT.java diff --git a/src/test/java/org/carlspring/cloud/storage/s3fs/FilesIT.java b/src/testIntegration/java/org/carlspring/cloud/storage/s3fs/FilesIT.java similarity index 100% rename from src/test/java/org/carlspring/cloud/storage/s3fs/FilesIT.java rename to src/testIntegration/java/org/carlspring/cloud/storage/s3fs/FilesIT.java diff --git a/src/test/java/org/carlspring/cloud/storage/s3fs/S3ClientIT.java b/src/testIntegration/java/org/carlspring/cloud/storage/s3fs/S3ClientIT.java similarity index 98% rename from src/test/java/org/carlspring/cloud/storage/s3fs/S3ClientIT.java rename to src/testIntegration/java/org/carlspring/cloud/storage/s3fs/S3ClientIT.java index c60c5ce3..b897bac5 100644 --- a/src/test/java/org/carlspring/cloud/storage/s3fs/S3ClientIT.java +++ b/src/testIntegration/java/org/carlspring/cloud/storage/s3fs/S3ClientIT.java @@ -40,7 +40,7 @@ class S3ClientIT extends BaseIntegrationTest public void setup() { // s3client - final Map credentials = getRealEnv(); + final Map credentials = EnvironmentBuilder.getRealEnv(); final AwsCredentials credentialsS3 = AwsBasicCredentials.create(credentials.get(ACCESS_KEY).toString(), credentials.get(SECRET_KEY).toString()); diff --git a/src/test/java/org/carlspring/cloud/storage/s3fs/S3UtilsIT.java b/src/testIntegration/java/org/carlspring/cloud/storage/s3fs/S3UtilsIT.java similarity index 100% rename from src/test/java/org/carlspring/cloud/storage/s3fs/S3UtilsIT.java rename to src/testIntegration/java/org/carlspring/cloud/storage/s3fs/S3UtilsIT.java diff --git a/src/testIntegration/java/org/carlspring/cloud/storage/s3fs/fileSystemProvider/CacheTestIT.java b/src/testIntegration/java/org/carlspring/cloud/storage/s3fs/fileSystemProvider/CacheTestIT.java new file mode 100644 index 00000000..4fde71a5 --- /dev/null +++ b/src/testIntegration/java/org/carlspring/cloud/storage/s3fs/fileSystemProvider/CacheTestIT.java @@ -0,0 +1,252 @@ +package org.carlspring.cloud.storage.s3fs.fileSystemProvider; + +import org.carlspring.cloud.storage.s3fs.BaseIntegrationTest; +import org.carlspring.cloud.storage.s3fs.S3Factory; +import org.carlspring.cloud.storage.s3fs.S3FileSystem; +import org.carlspring.cloud.storage.s3fs.S3FileSystemProvider; +import org.carlspring.cloud.storage.s3fs.S3Path; +import org.carlspring.cloud.storage.s3fs.util.EnvironmentBuilder; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.DisabledIfEnvironmentVariable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.URI; +import java.nio.file.FileSystem; +import java.nio.file.FileSystemNotFoundException; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.carlspring.cloud.storage.s3fs.util.S3EndpointConstant.S3_GLOBAL_URI_IT; + +@DisabledIfEnvironmentVariable(named = "CI", matches = ".*", disabledReason = "CI runs are slow and unreliable; Run locally.") +public class CacheTestIT + extends BaseIntegrationTest + +{ + + private static final Logger logger = LoggerFactory.getLogger(CacheTestIT.class); + + private static final String bucket = EnvironmentBuilder.getBucket(); + + private static final URI uriGlobal = EnvironmentBuilder.getS3URI(S3_GLOBAL_URI_IT); + + private static S3FileSystem fs; + + private static S3FileSystemProvider provider; + + private static final List files = Collections.synchronizedList(new ArrayList<>()); + private static final int multiplier = 1; + private static final int totalFilesToCreate = 1000 * multiplier; + private static final int createResourceWaitTime = Math.max(0, totalFilesToCreate / 100); + private static final int maxCacheSize = (int) Math.round(totalFilesToCreate * 1.1); // caching 110% of the total files to prevent "cache trashing" + private static final AtomicInteger totalFilesAdded = new AtomicInteger(0); + // Assert the loop executed within ~100 seconds (the normal execution time is around 90 for 1000 files with good connection) + private static final int expectedTime = 105 * multiplier; + + private static String testBasePathString = getTestBasePath(CacheTestIT.class, "common_cache_test_artifacts"); + + @BeforeAll + public static void prepareS3Resources() throws IOException, InterruptedException + { + fs = (S3FileSystem) build(); + provider = fs.provider(); + + S3Path rootPath = fs.getPath(bucket, testBasePathString); + boolean createResources = Files.notExists(rootPath); + + // Create a thread pool with a number of threads equal to the number of available processors + ExecutorService executorService = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); + + // Settings + int batchSize = 20; + + logger.info("Preparing assets."); + + // Create files + for (int i = 0; i < totalFilesToCreate; i += batchSize) { + final int start = i; // Capture the starting index for this batch + final int end = Math.min(start + batchSize, totalFilesToCreate); // Calculate end index + + executorService.submit(() -> { + try { + for (int j = start; j < end; j++) { + int index = totalFilesAdded.incrementAndGet(); + S3Path file = (S3Path) rootPath.resolve("file-" + index); + if(createResources) { + Files.write(file, String.valueOf(index).getBytes(), StandardOpenOption.CREATE); + } + files.add(file); + } + } catch (Exception e) { + totalFilesAdded.decrementAndGet(); + logger.error("Filed to write file.", e); + } + }); + } + + // Shutdown the executor service and await termination + executorService.shutdown(); + executorService.awaitTermination(1, TimeUnit.MINUTES); + + // Assert the remote file list count is correct. + // NB: The more S3 objects you put, the longer sleep is necessary, because of eventually consistent replication. + // Using no thread.sleep causes the `Files.list(rootPath)` to have slightly inaccurate total count. + // + if(createResources) { + logger.info("Waiting for {} seconds for S3 to replication to finish (eventual consistency)", createResourceWaitTime); + Thread.sleep(Duration.ofSeconds(createResourceWaitTime).toMillis()); + assertThat(Files.list(rootPath).count()).isEqualTo(totalFilesToCreate); + } + + fs.getFileAttributesCache().invalidateAll(); + fs.close(); + } + + @BeforeEach + public void setup() + throws IOException + { + fs = (S3FileSystem) build(); + provider = fs.provider(); + } + + private static FileSystem build() + throws IOException + { + try + { + FileSystems.getFileSystem(uriGlobal).close(); + + return createNewFileSystem(); + } + catch (FileSystemNotFoundException e) + { + return createNewFileSystem(); + } + } + + private static FileSystem createNewFileSystem() + throws IOException + { + Map env = new HashMap<>(); + env.putAll(EnvironmentBuilder.getRealEnv()); + env.put(S3Factory.REQUEST_HEADER_CACHE_CONTROL, "max-age=60"); + env.put(S3Factory.CACHE_ATTRIBUTES_TTL, "120000"); + env.put(S3Factory.CACHE_ATTRIBUTES_SIZE, String.valueOf(maxCacheSize)); + env.put(S3Factory.MAX_CONNECTIONS, "200"); + return FileSystems.newFileSystem(uriGlobal, env); + } + + @Test + void testScenario001() throws IOException + { + /** + * Create many files in a folder and attempt operations that should use the cache such as `Files.exists`, + * `Files.isDirectory()`, etc. + */ + + fs.getFileAttributesCache().invalidateAll(); + + // Measure start time + long startTime = System.currentTimeMillis(); + int processedFileCount = 0; + for (int i = 0; i < totalFilesToCreate; i++) + { + // Calling multiple times to ensure we are hitting the cache. + S3Path path = files.get(i); + assertThat(Files.exists(path)).withFailMessage(path + " does not exist.").isTrue(); + assertThat(Files.exists(path)).withFailMessage(path + " does not exist.").isTrue(); + assertThat(Files.exists(path)).withFailMessage(path + " does not exist.").isTrue(); + + assertThat(Files.isDirectory(path)).withFailMessage(path + " should not be a directory.").isFalse(); + assertThat(Files.isDirectory(path)).withFailMessage(path + " should not be a directory.").isFalse(); + assertThat(Files.isDirectory(path)).withFailMessage(path + " should not be a directory.").isFalse(); + + assertThat(Files.isRegularFile(path)).withFailMessage(path + " should be a regular file.").isTrue(); + assertThat(Files.isRegularFile(path)).withFailMessage(path + " should be a regular file.").isTrue(); + assertThat(Files.isRegularFile(path)).withFailMessage(path + " should be a regular file.").isTrue(); + + assertThat(Files.notExists(path)).withFailMessage(path + " should exist.").isFalse(); + assertThat(Files.notExists(path)).withFailMessage(path + " should exist.").isFalse(); + assertThat(Files.notExists(path)).withFailMessage(path + " should exist.").isFalse(); + + assertThat(Files.getLastModifiedTime(path)).withFailMessage(path + " should not have null lastModifiedTime") + .isNotNull(); + assertThat(Files.getLastModifiedTime(path)).withFailMessage(path + " should not have null lastModifiedTime") + .isNotNull(); + assertThat(Files.getLastModifiedTime(path)).withFailMessage(path + " should not have null lastModifiedTime") + .isNotNull(); + + assertThat(Files.size(path)).isGreaterThan(0); + assertThat(Files.size(path)).isGreaterThan(0); + assertThat(Files.size(path)).isGreaterThan(0); + + ++processedFileCount; + } + + // Measure end time + long endTime = System.currentTimeMillis(); + long elapsedTime = (endTime - startTime) / 1000; + + // Assert the loop executed within ~100 seconds (the normal execution time is around 90 for 1000 files with good connection) + logger.info("Cache test ended"); + logger.info("Inserted files {} / Processed files: {} / Test case {}", totalFilesAdded.get(), processedFileCount, testBasePathString); + logger.info("Start time {} / End time {} / Elapsed time {}s", startTime, endTime, elapsedTime); + assertThat(elapsedTime).isLessThanOrEqualTo(expectedTime); // seconds + + } + + @Test + void testScenario002() throws IOException + { + S3Path rootPath = fs.getPath(bucket, testBasePathString); + fs.getFileAttributesCache().invalidateAll(); + + // Measure start time + long startTime = System.currentTimeMillis(); + AtomicInteger processedFileCount = new AtomicInteger(0); + + // Reminder: The `.limit()` will not limit the amount of requests sent to AWS. + // If the rootPath contains 100k files and the `.limit(1000)` -- it will still fetch the 100k files first. + try (Stream stream = Files.list(rootPath).limit(totalFilesAdded.get())) + { + stream.filter(path -> path != null && + Files.exists(path) && + !Files.isDirectory(path) && + Files.isRegularFile(path)) + .forEach(path -> { + processedFileCount.incrementAndGet(); + logger.info("Found file {}", path); + }); + } + + // Measure end time + long endTime = System.currentTimeMillis(); + long elapsedTime = (endTime - startTime) / 1000; + + logger.info("Cache test ended"); + logger.info("Inserted files {} / Processed files: {} / Test case {}", totalFilesAdded.get(), processedFileCount.get(), testBasePathString); + logger.info("Start time {} / End time {} / Elapsed time {}s", startTime, endTime, elapsedTime); + assertThat(elapsedTime).isLessThanOrEqualTo(expectedTime); + assertThat(processedFileCount.get()).isEqualTo(totalFilesToCreate); + } + +} diff --git a/src/test/java/org/carlspring/cloud/storage/s3fs/fileSystemProvider/GetFileSystemIT.java b/src/testIntegration/java/org/carlspring/cloud/storage/s3fs/fileSystemProvider/GetFileSystemIT.java similarity index 100% rename from src/test/java/org/carlspring/cloud/storage/s3fs/fileSystemProvider/GetFileSystemIT.java rename to src/testIntegration/java/org/carlspring/cloud/storage/s3fs/fileSystemProvider/GetFileSystemIT.java diff --git a/src/test/java/org/carlspring/cloud/storage/s3fs/fileSystemProvider/NewAsynchronousFileChannelTestIT.java b/src/testIntegration/java/org/carlspring/cloud/storage/s3fs/fileSystemProvider/NewAsynchronousFileChannelTestIT.java similarity index 100% rename from src/test/java/org/carlspring/cloud/storage/s3fs/fileSystemProvider/NewAsynchronousFileChannelTestIT.java rename to src/testIntegration/java/org/carlspring/cloud/storage/s3fs/fileSystemProvider/NewAsynchronousFileChannelTestIT.java diff --git a/src/test/java/org/carlspring/cloud/storage/s3fs/fileSystemProvider/NewByteChannelIT.java b/src/testIntegration/java/org/carlspring/cloud/storage/s3fs/fileSystemProvider/NewByteChannelIT.java similarity index 100% rename from src/test/java/org/carlspring/cloud/storage/s3fs/fileSystemProvider/NewByteChannelIT.java rename to src/testIntegration/java/org/carlspring/cloud/storage/s3fs/fileSystemProvider/NewByteChannelIT.java diff --git a/src/test/java/org/carlspring/cloud/storage/s3fs/fileSystemProvider/NewByteChannelTest.java b/src/testIntegration/java/org/carlspring/cloud/storage/s3fs/fileSystemProvider/NewByteChannelTest.java similarity index 100% rename from src/test/java/org/carlspring/cloud/storage/s3fs/fileSystemProvider/NewByteChannelTest.java rename to src/testIntegration/java/org/carlspring/cloud/storage/s3fs/fileSystemProvider/NewByteChannelTest.java diff --git a/src/test/java/org/carlspring/cloud/storage/s3fs/fileSystemProvider/NewFileSystemIT.java b/src/testIntegration/java/org/carlspring/cloud/storage/s3fs/fileSystemProvider/NewFileSystemIT.java similarity index 100% rename from src/test/java/org/carlspring/cloud/storage/s3fs/fileSystemProvider/NewFileSystemIT.java rename to src/testIntegration/java/org/carlspring/cloud/storage/s3fs/fileSystemProvider/NewFileSystemIT.java diff --git a/src/testIntegration/java/org/carlspring/cloud/storage/s3fs/fileSystemProvider/ReadAttributesIT.java b/src/testIntegration/java/org/carlspring/cloud/storage/s3fs/fileSystemProvider/ReadAttributesIT.java new file mode 100644 index 00000000..2a0549d2 --- /dev/null +++ b/src/testIntegration/java/org/carlspring/cloud/storage/s3fs/fileSystemProvider/ReadAttributesIT.java @@ -0,0 +1,271 @@ +package org.carlspring.cloud.storage.s3fs.fileSystemProvider; + +import com.github.benmanes.caffeine.cache.stats.CacheStats; +import com.github.marschall.memoryfilesystem.MemoryFileSystemBuilder; +import org.carlspring.cloud.storage.s3fs.BaseIntegrationTest; +import org.carlspring.cloud.storage.s3fs.S3FileSystem; +import org.carlspring.cloud.storage.s3fs.S3FileSystemProvider; +import org.carlspring.cloud.storage.s3fs.S3Path; +import org.carlspring.cloud.storage.s3fs.cache.S3FileAttributesCache; +import org.carlspring.cloud.storage.s3fs.util.EnvironmentBuilder; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.net.URI; +import java.nio.file.FileSystem; +import java.nio.file.FileSystemNotFoundException; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.attribute.PosixFileAttributes; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.carlspring.cloud.storage.s3fs.util.S3EndpointConstant.S3_GLOBAL_URI_IT; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +class ReadAttributesIT + extends BaseIntegrationTest +{ + + private static final String bucket = EnvironmentBuilder.getBucket(); + + private static final URI uriGlobal = EnvironmentBuilder.getS3URI(S3_GLOBAL_URI_IT); + + private S3FileSystem fileSystemAmazon; + + private S3FileSystemProvider provider; + + @BeforeEach + public void setup() + throws IOException + { + System.clearProperty(S3FileSystemProvider.S3_FACTORY_CLASS); + + fileSystemAmazon = (S3FileSystem) build(); + provider = fileSystemAmazon.provider(); + } + + private static FileSystem build() + throws IOException + { + try + { + FileSystems.getFileSystem(uriGlobal).close(); + + return createNewFileSystem(); + } + catch (FileSystemNotFoundException e) + { + return createNewFileSystem(); + } + } + + private static FileSystem createNewFileSystem() + throws IOException + { + return FileSystems.newFileSystem(uriGlobal, EnvironmentBuilder.getRealEnv()); + } + + private Path uploadSingleFile(String content) + throws IOException + { + try (FileSystem linux = MemoryFileSystemBuilder.newLinux().build("linux")) + { + if (content != null) + { + Files.write(linux.getPath("/index.html"), content.getBytes()); + } + else + { + Files.createFile(linux.getPath("/index.html")); + } + + Path result = fileSystemAmazon.getPath(bucket, getTestBasePathWithUUID()); + + Files.copy(linux.getPath("/index.html"), result); + + return result; + } + } + + @Test + void readAttributesRegenerateCacheWhenNotExistsBasic() + throws IOException + { + S3FileSystem fs = fileSystemAmazon; + S3Path file = fileSystemAmazon.getPath(bucket, getTestBasePathWithUUID(), "1234"); + Files.write(file, "1234".getBytes()); + + // No cache assertion + S3FileAttributesCache cache = fs.getFileAttributesCache(); + CacheStats stats = cache.stats(); // temporary snapshot + assertThat(stats.hitCount()).isEqualTo(0); + assertThat(stats.missCount()).isEqualTo(0); + + // Pre-requisites (cache entry key should not exist) + String fileAttrCacheKey = cache.generateCacheKey(file, BasicFileAttributes.class); + assertThat(cache.contains(fileAttrCacheKey)).isFalse(); + stats = cache.stats(); + assertThat(stats.hitCount()).isEqualTo(0); + assertThat(stats.missCount()).isEqualTo(0); + + // Reading the attributes should create the cache entry. + BasicFileAttributes attrs = provider.readAttributes(file, BasicFileAttributes.class); + assertThat(attrs).isNotNull(); + assertThat(attrs).isEqualTo(file.getFileAttributes(BasicFileAttributes.class)); + stats = cache.stats(); + assertThat(stats.hitCount()).isEqualTo(1); + assertThat(stats.missCount()).isEqualTo(1); + + // Should hit the cache. + attrs = provider.readAttributes(file, BasicFileAttributes.class); + assertThat(attrs).isNotNull(); + assertThat(attrs).isEqualTo(file.getFileAttributes(BasicFileAttributes.class)); + stats = cache.stats(); + assertThat(stats.hitCount()).isEqualTo(3); + assertThat(stats.missCount()).isEqualTo(1); + + // Should hit the cache. + attrs = provider.readAttributes(file, BasicFileAttributes.class); + assertThat(attrs).isNotNull(); + assertThat(attrs).isEqualTo(file.getFileAttributes(BasicFileAttributes.class)); + stats = cache.stats(); + assertThat(stats.hitCount()).isEqualTo(5); + assertThat(stats.missCount()).isEqualTo(1); + + // Invalidate cache manually. + cache.invalidate(fileAttrCacheKey); + assertThat(cache.contains(fileAttrCacheKey)).isFalse(); + + // Should populate the cache again. + attrs = provider.readAttributes(file, BasicFileAttributes.class); + assertThat(attrs).isNotNull(); + assertNotNull(file.getFileAttributes(BasicFileAttributes.class)); + stats = cache.stats(); + assertThat(stats.hitCount()).isEqualTo(6); + assertThat(stats.missCount()).isEqualTo(2); + } + + @Test + void readAttributesRegenerateCacheWhenNotExistsPosix() + throws IOException + { + S3FileSystem fs = fileSystemAmazon; + S3Path file = fileSystemAmazon.getPath(bucket, getTestBasePathWithUUID(), "1234"); + Files.write(file, "1234".getBytes()); + + // No cache assertion + S3FileAttributesCache cache = fs.getFileAttributesCache(); + CacheStats stats = cache.stats(); // temporary snapshot + assertThat(stats.hitCount()).isEqualTo(0); + assertThat(stats.missCount()).isEqualTo(0); + + // Pre-requisites (cache entry key should not exist) + String fileAttrCacheKey = cache.generateCacheKey(file, PosixFileAttributes.class); + assertThat(cache.contains(fileAttrCacheKey)).isFalse(); + stats = cache.stats(); + assertThat(stats.hitCount()).isEqualTo(0); + assertThat(stats.missCount()).isEqualTo(0); + + // Reading the attributes should create the cache entry. + PosixFileAttributes attrs = provider.readAttributes(file, PosixFileAttributes.class); + assertThat(attrs).isNotNull(); + assertThat(attrs).isEqualTo(file.getFileAttributes(PosixFileAttributes.class)); + stats = cache.stats(); + assertThat(stats.hitCount()).isEqualTo(1); + assertThat(stats.missCount()).isEqualTo(1); + + // Should hit the cache. + attrs = provider.readAttributes(file, PosixFileAttributes.class); + assertThat(attrs).isNotNull(); + assertThat(attrs).isEqualTo(file.getFileAttributes(PosixFileAttributes.class)); + stats = cache.stats(); + assertThat(stats.hitCount()).isEqualTo(3); + assertThat(stats.missCount()).isEqualTo(1); + + // Should hit the cache. + attrs = provider.readAttributes(file, PosixFileAttributes.class); + assertThat(attrs).isNotNull(); + assertThat(attrs).isEqualTo(file.getFileAttributes(PosixFileAttributes.class)); + stats = cache.stats(); + assertThat(stats.hitCount()).isEqualTo(5); + assertThat(stats.missCount()).isEqualTo(1); + + // Invalidate cache manually. + cache.invalidate(fileAttrCacheKey); + assertThat(cache.contains(fileAttrCacheKey)).isFalse(); + + // Should populate the cache again. + attrs = provider.readAttributes(file, PosixFileAttributes.class); + assertThat(attrs).isNotNull(); + assertNotNull(file.getFileAttributes(PosixFileAttributes.class)); + stats = cache.stats(); + assertThat(stats.hitCount()).isEqualTo(6); + assertThat(stats.missCount()).isEqualTo(2); + + } + + @Test + void readAttributesCastDownFromPosixToBasic() + throws IOException + { + S3FileSystem fs = fileSystemAmazon; + S3Path file = fileSystemAmazon.getPath(bucket, getTestBasePathWithUUID(), "1234"); + Files.write(file, "1234".getBytes()); + + // No cache assertion + S3FileAttributesCache cache = fs.getFileAttributesCache(); + CacheStats stats = cache.stats(); // temporary snapshot + assertThat(stats.hitCount()).isEqualTo(0); + assertThat(stats.missCount()).isEqualTo(0); + + // Pre-requisites (cache entry key should not exist) + String basicFileAttrCacheKey = cache.generateCacheKey(file, BasicFileAttributes.class); + String posixFileAttrCacheKey = cache.generateCacheKey(file, PosixFileAttributes.class); + assertThat(cache.contains(basicFileAttrCacheKey)).isFalse(); + assertThat(cache.contains(posixFileAttrCacheKey)).isFalse(); + + stats = cache.stats(); + assertThat(stats.hitCount()).isEqualTo(0); + assertThat(stats.missCount()).isEqualTo(0); + + // Reading the attributes should create the cache entry. + BasicFileAttributes attrs = provider.readAttributes(file, PosixFileAttributes.class); + assertThat(attrs).isNotNull(); + assertThat(attrs).isEqualTo(file.getFileAttributes(BasicFileAttributes.class)); + stats = cache.stats(); + assertThat(stats.hitCount()).isEqualTo(1); + assertThat(stats.missCount()).isEqualTo(1); + + // Should hit the cache. + attrs = provider.readAttributes(file, BasicFileAttributes.class); + assertThat(attrs).isNotNull(); + assertThat(attrs).isEqualTo(file.getFileAttributes(BasicFileAttributes.class)); + stats = cache.stats(); + assertThat(stats.hitCount()).isEqualTo(3); + assertThat(stats.missCount()).isEqualTo(1); + + // Should hit the cache. + attrs = provider.readAttributes(file, BasicFileAttributes.class); + assertThat(attrs).isNotNull(); + assertThat(attrs).isEqualTo(file.getFileAttributes(BasicFileAttributes.class)); + stats = cache.stats(); + assertThat(stats.hitCount()).isEqualTo(5); + assertThat(stats.missCount()).isEqualTo(1); + + // Invalidate cache manually. + cache.invalidate(basicFileAttrCacheKey); + assertThat(cache.contains(basicFileAttrCacheKey)).isFalse(); + + // Should populate the cache again. + attrs = provider.readAttributes(file, PosixFileAttributes.class); + assertThat(attrs).isNotNull(); + assertNotNull(file.getFileAttributes(BasicFileAttributes.class)); + stats = cache.stats(); + assertThat(stats.hitCount()).isEqualTo(6); + assertThat(stats.missCount()).isEqualTo(2); + } + +} diff --git a/src/test/java/org/carlspring/cloud/storage/s3fs/junit/annotations/BaseAnnotationTest.java b/src/testIntegration/java/org/carlspring/cloud/storage/s3fs/junit/annotations/BaseAnnotationTest.java similarity index 100% rename from src/test/java/org/carlspring/cloud/storage/s3fs/junit/annotations/BaseAnnotationTest.java rename to src/testIntegration/java/org/carlspring/cloud/storage/s3fs/junit/annotations/BaseAnnotationTest.java diff --git a/src/test/java/org/carlspring/cloud/storage/s3fs/junit/annotations/MinioIntegrationTest.java b/src/testIntegration/java/org/carlspring/cloud/storage/s3fs/junit/annotations/MinioIntegrationTest.java similarity index 100% rename from src/test/java/org/carlspring/cloud/storage/s3fs/junit/annotations/MinioIntegrationTest.java rename to src/testIntegration/java/org/carlspring/cloud/storage/s3fs/junit/annotations/MinioIntegrationTest.java diff --git a/src/test/java/org/carlspring/cloud/storage/s3fs/junit/annotations/MinioIntegrationTestAnnotationTest.java b/src/testIntegration/java/org/carlspring/cloud/storage/s3fs/junit/annotations/MinioIntegrationTestAnnotationTest.java similarity index 100% rename from src/test/java/org/carlspring/cloud/storage/s3fs/junit/annotations/MinioIntegrationTestAnnotationTest.java rename to src/testIntegration/java/org/carlspring/cloud/storage/s3fs/junit/annotations/MinioIntegrationTestAnnotationTest.java diff --git a/src/test/java/org/carlspring/cloud/storage/s3fs/junit/annotations/S3IntegrationTest.java b/src/testIntegration/java/org/carlspring/cloud/storage/s3fs/junit/annotations/S3IntegrationTest.java similarity index 100% rename from src/test/java/org/carlspring/cloud/storage/s3fs/junit/annotations/S3IntegrationTest.java rename to src/testIntegration/java/org/carlspring/cloud/storage/s3fs/junit/annotations/S3IntegrationTest.java diff --git a/src/test/java/org/carlspring/cloud/storage/s3fs/junit/annotations/S3IntegrationTestAnnotationTest.java b/src/testIntegration/java/org/carlspring/cloud/storage/s3fs/junit/annotations/S3IntegrationTestAnnotationTest.java similarity index 100% rename from src/test/java/org/carlspring/cloud/storage/s3fs/junit/annotations/S3IntegrationTestAnnotationTest.java rename to src/testIntegration/java/org/carlspring/cloud/storage/s3fs/junit/annotations/S3IntegrationTestAnnotationTest.java diff --git a/src/test/java/org/carlspring/cloud/storage/s3fs/junit/examples/CombinedIT.java b/src/testIntegration/java/org/carlspring/cloud/storage/s3fs/junit/examples/CombinedIT.java similarity index 100% rename from src/test/java/org/carlspring/cloud/storage/s3fs/junit/examples/CombinedIT.java rename to src/testIntegration/java/org/carlspring/cloud/storage/s3fs/junit/examples/CombinedIT.java diff --git a/src/test/java/org/carlspring/cloud/storage/s3fs/junit/examples/CombinedMinioS3IT.java b/src/testIntegration/java/org/carlspring/cloud/storage/s3fs/junit/examples/CombinedMinioS3IT.java similarity index 100% rename from src/test/java/org/carlspring/cloud/storage/s3fs/junit/examples/CombinedMinioS3IT.java rename to src/testIntegration/java/org/carlspring/cloud/storage/s3fs/junit/examples/CombinedMinioS3IT.java diff --git a/src/test/java/org/carlspring/cloud/storage/s3fs/junit/examples/CombinedS3MinioIT.java b/src/testIntegration/java/org/carlspring/cloud/storage/s3fs/junit/examples/CombinedS3MinioIT.java similarity index 100% rename from src/test/java/org/carlspring/cloud/storage/s3fs/junit/examples/CombinedS3MinioIT.java rename to src/testIntegration/java/org/carlspring/cloud/storage/s3fs/junit/examples/CombinedS3MinioIT.java diff --git a/src/test/java/org/carlspring/cloud/storage/s3fs/junit/examples/MinioClassAnnotationIT.java b/src/testIntegration/java/org/carlspring/cloud/storage/s3fs/junit/examples/MinioClassAnnotationIT.java similarity index 100% rename from src/test/java/org/carlspring/cloud/storage/s3fs/junit/examples/MinioClassAnnotationIT.java rename to src/testIntegration/java/org/carlspring/cloud/storage/s3fs/junit/examples/MinioClassAnnotationIT.java diff --git a/src/test/java/org/carlspring/cloud/storage/s3fs/junit/examples/MinioMethodAnnotationIT.java b/src/testIntegration/java/org/carlspring/cloud/storage/s3fs/junit/examples/MinioMethodAnnotationIT.java similarity index 100% rename from src/test/java/org/carlspring/cloud/storage/s3fs/junit/examples/MinioMethodAnnotationIT.java rename to src/testIntegration/java/org/carlspring/cloud/storage/s3fs/junit/examples/MinioMethodAnnotationIT.java diff --git a/src/test/java/org/carlspring/cloud/storage/s3fs/junit/examples/S3ClassAnnotationIT.java b/src/testIntegration/java/org/carlspring/cloud/storage/s3fs/junit/examples/S3ClassAnnotationIT.java similarity index 100% rename from src/test/java/org/carlspring/cloud/storage/s3fs/junit/examples/S3ClassAnnotationIT.java rename to src/testIntegration/java/org/carlspring/cloud/storage/s3fs/junit/examples/S3ClassAnnotationIT.java diff --git a/src/test/java/org/carlspring/cloud/storage/s3fs/junit/examples/S3MethodAnnotationIT.java b/src/testIntegration/java/org/carlspring/cloud/storage/s3fs/junit/examples/S3MethodAnnotationIT.java similarity index 100% rename from src/test/java/org/carlspring/cloud/storage/s3fs/junit/examples/S3MethodAnnotationIT.java rename to src/testIntegration/java/org/carlspring/cloud/storage/s3fs/junit/examples/S3MethodAnnotationIT.java diff --git a/src/test/java/org/carlspring/cloud/storage/s3fs/path/ToURLIT.java b/src/testIntegration/java/org/carlspring/cloud/storage/s3fs/path/ToURLIT.java similarity index 100% rename from src/test/java/org/carlspring/cloud/storage/s3fs/path/ToURLIT.java rename to src/testIntegration/java/org/carlspring/cloud/storage/s3fs/path/ToURLIT.java diff --git a/src/test/java/org/carlspring/cloud/storage/s3fs/spike/EnvironmentIT.java b/src/testIntegration/java/org/carlspring/cloud/storage/s3fs/spike/EnvironmentIT.java similarity index 100% rename from src/test/java/org/carlspring/cloud/storage/s3fs/spike/EnvironmentIT.java rename to src/testIntegration/java/org/carlspring/cloud/storage/s3fs/spike/EnvironmentIT.java diff --git a/src/test/java/org/carlspring/cloud/storage/s3fs/util/EnvironmentBuilder.java b/src/testIntegration/java/org/carlspring/cloud/storage/s3fs/util/EnvironmentBuilder.java similarity index 100% rename from src/test/java/org/carlspring/cloud/storage/s3fs/util/EnvironmentBuilder.java rename to src/testIntegration/java/org/carlspring/cloud/storage/s3fs/util/EnvironmentBuilder.java