From 67e13a21ab44f09863169cc60e9270edfca172c8 Mon Sep 17 00:00:00 2001 From: brharrington Date: Thu, 17 Aug 2023 16:12:57 -0500 Subject: [PATCH] aws: add option to enable dualstack (#584) Adds a new config option to enable dualstack on the AWS client. For services we commonly use that support dualstack, it is enabled by default. --- .../netflix/iep/aws2/AwsClientFactory.java | 46 +++++++++------- .../src/main/resources/reference.conf | 11 ++++ .../iep/aws2/AwsClientFactoryTest.java | 53 ++++++++++++++----- 3 files changed, 76 insertions(+), 34 deletions(-) diff --git a/iep-spring-aws2/src/main/java/com/netflix/iep/aws2/AwsClientFactory.java b/iep-spring-aws2/src/main/java/com/netflix/iep/aws2/AwsClientFactory.java index 868587c7..39e7e2ff 100644 --- a/iep-spring-aws2/src/main/java/com/netflix/iep/aws2/AwsClientFactory.java +++ b/iep-spring-aws2/src/main/java/com/netflix/iep/aws2/AwsClientFactory.java @@ -104,8 +104,7 @@ private void setRetriesIfPresent(Config cfg, ClientOverrideConfiguration.Builder } } - ClientOverrideConfiguration createClientConfig(String name) { - final Config cfg = getConfig(name, "client"); + ClientOverrideConfiguration createClientConfig(Config cfg) { ClientOverrideConfiguration.Builder builder = ClientOverrideConfiguration.builder(); setIfPresent(cfg, "api-call-timeout", builder::apiCallTimeout); @@ -130,11 +129,18 @@ ClientOverrideConfiguration createClientConfig(String name) { return builder.build(); } - private Config getConfig(String name, String suffix) { + Config getConfig(String name, Class cls) { final String cfgPrefix = "netflix.iep.aws"; - return (name != null && config.hasPath(cfgPrefix + "." + name + "." + suffix)) - ? config.getConfig(cfgPrefix + "." + name + "." + suffix) - : config.getConfig(cfgPrefix + ".default." + suffix); + Config cfg = config.getConfig(cfgPrefix + ".default"); + + final String service = getDefaultName(cls); + if (config.hasPath(cfgPrefix + "." + service)) { + cfg = config.getConfig(cfgPrefix + "." + service).withFallback(cfg); + } + + return (name != null && config.hasPath(cfgPrefix + "." + name)) + ? config.getConfig(cfgPrefix + "." + name).withFallback(cfg) + : cfg; } private String createRoleArn(String arnPattern, String accountId) { @@ -172,11 +178,10 @@ private AwsCredentialsProvider createAssumeRoleProvider( } AwsCredentialsProvider createCredentialsProvider( - String name, String accountId, SdkHttpService service) { + Config cfg, String accountId, SdkHttpService service) { final AwsCredentialsProvider dflt = DefaultCredentialsProvider.builder() .asyncCredentialUpdateEnabled(true) .build(); - final Config cfg = getConfig(name, "credentials"); if (cfg.hasPath("role-arn")) { return createAssumeRoleProvider(cfg, accountId, dflt, service); } else { @@ -187,9 +192,8 @@ AwsCredentialsProvider createCredentialsProvider( } } - AttributeMap getSdkHttpConfigurationOptions(String name) { + AttributeMap getSdkHttpConfigurationOptions(Config clientConfig) { Map, Object> configuration = new HashMap<>(); - Config clientConfig = getConfig(name, "client"); if(clientConfig.hasPath("http-configuration")) { Config httpConfig = clientConfig.getConfig("http-configuration"); @@ -233,8 +237,7 @@ private Region chooseRegion(String name, Class cls) { return Region.of(endpointRegion); } - private SdkHttpService createSyncHttpService(String name) { - Config clientConfig = getConfig(name, "client"); + private SdkHttpService createSyncHttpService(Config clientConfig) { if (clientConfig.hasPath("sync-http-impl")) { String clsName = clientConfig.getString("sync-http-impl"); try { @@ -256,8 +259,7 @@ private SdkHttpService createSyncHttpService(String name) { } } - private SdkAsyncHttpService createAsyncHttpService(String name) { - Config clientConfig = getConfig(name, "client"); + private SdkAsyncHttpService createAsyncHttpService(Config clientConfig) { if (clientConfig.hasPath("async-http-impl")) { String clsName = clientConfig.getString("async-http-impl"); try { @@ -366,21 +368,25 @@ public T newInstance(String name, Class cls, String accountId) { * @return * AWS client instance. */ + @SuppressWarnings("unchecked") public T newInstance(String name, Class cls, String accountId, Optional region) { try { - SdkHttpService service = createSyncHttpService(name); + Config cfg = getConfig(name, cls); + Config clientConfig = cfg.getConfig("client"); + SdkHttpService service = createSyncHttpService(clientConfig); Method builderMethod = cls.getMethod("builder"); AwsClientBuilder builder = ((AwsClientBuilder) builderMethod.invoke(null)) - .credentialsProvider(createCredentialsProvider(name, accountId, service)) + .credentialsProvider(createCredentialsProvider(cfg.getConfig("credentials"), accountId, service)) .region(region.orElseGet(() -> chooseRegion(name, cls))) - .overrideConfiguration(createClientConfig(name)); - AttributeMap attributeMap = getSdkHttpConfigurationOptions(name); + .dualstackEnabled(cfg.getBoolean("dualstack")) + .overrideConfiguration(createClientConfig(clientConfig)); + AttributeMap attributeMap = getSdkHttpConfigurationOptions(clientConfig); if (builder instanceof AwsSyncClientBuilder) { ((AwsSyncClientBuilder) builder) .httpClient(service.createHttpClientBuilder().buildWithDefaults(attributeMap)); } else if (builder instanceof AwsAsyncClientBuilder) { - SdkAsyncHttpService asyncService = createAsyncHttpService(name); + SdkAsyncHttpService asyncService = createAsyncHttpService(clientConfig); ((AwsAsyncClientBuilder) builder) .httpClient(asyncService.createAsyncHttpClientFactory().buildWithDefaults(attributeMap)); } @@ -453,7 +459,6 @@ public T getInstance(Class cls, String accountId) { * @return * AWS client instance. */ - @SuppressWarnings("unchecked") public T getInstance(String name, Class cls, String accountId) { return getInstance(name, cls, accountId, Optional.empty()); } @@ -474,6 +479,7 @@ public T getInstance(String name, Class cls, String accountId) { * @return * AWS client instance. */ + @SuppressWarnings("unchecked") public T getInstance(String name, Class cls, String accountId, Optional region) { try { final String key = name + ":" + cls.getName() + ":" + accountId + ":" + region.orElseGet(() -> chooseRegion(name, cls)); diff --git a/iep-spring-aws2/src/main/resources/reference.conf b/iep-spring-aws2/src/main/resources/reference.conf index e76ec605..2e0694af 100644 --- a/iep-spring-aws2/src/main/resources/reference.conf +++ b/iep-spring-aws2/src/main/resources/reference.conf @@ -39,8 +39,19 @@ netflix.iep.aws { //user-agent-prefix //user-agent-suffix } + + // Should dualstack be enabled for the client? + // https://docs.aws.amazon.com/vpc/latest/userguide/aws-ipv6-support.html + dualstack = false } + // Overrides for services that support IPv6 to use dualstack + // https://docs.aws.amazon.com/vpc/latest/userguide/aws-ipv6-support.html + ec2.dualstack = true + elasticloadbalancingv2.dualstack = true + route53.dualstack = true + s3.dualstack = true + // http://docs.aws.amazon.com/general/latest/gr/rande.html // This is used to set overrides when a service is not available locally in the // region we are running in. diff --git a/iep-spring-aws2/src/test/java/com/netflix/iep/aws2/AwsClientFactoryTest.java b/iep-spring-aws2/src/test/java/com/netflix/iep/aws2/AwsClientFactoryTest.java index a2a05e06..99d55e2c 100644 --- a/iep-spring-aws2/src/test/java/com/netflix/iep/aws2/AwsClientFactoryTest.java +++ b/iep-spring-aws2/src/test/java/com/netflix/iep/aws2/AwsClientFactoryTest.java @@ -32,6 +32,7 @@ import software.amazon.awssdk.services.ec2.Ec2AsyncClient; import software.amazon.awssdk.services.ec2.Ec2Client; import software.amazon.awssdk.services.ec2.model.DescribeAddressesRequest; +import software.amazon.awssdk.services.sts.StsClient; import software.amazon.awssdk.services.sts.auth.StsAssumeRoleCredentialsProvider; import software.amazon.awssdk.services.sts.model.AssumeRoleRequest; import software.amazon.awssdk.utils.AttributeMap; @@ -74,21 +75,24 @@ private String getUserAgentSuffix(ClientOverrideConfiguration settings) { @Test public void createClientConfig() { AwsClientFactory factory = new AwsClientFactory(config); - ClientOverrideConfiguration settings = factory.createClientConfig(null); + Config cfg = factory.getConfig(null, Ec2Client.class).getConfig("client"); + ClientOverrideConfiguration settings = factory.createClientConfig(cfg); Assert.assertEquals("default", getUserAgentPrefix(settings)); } @Test public void createClientConfigOverride() { AwsClientFactory factory = new AwsClientFactory(config); - ClientOverrideConfiguration settings = factory.createClientConfig("ec2-test"); + Config cfg = factory.getConfig("ec2-test", Ec2Client.class).getConfig("client"); + ClientOverrideConfiguration settings = factory.createClientConfig(cfg); Assert.assertEquals("ignored-defaults", getUserAgentPrefix(settings)); } @Test public void createClientConfigOverrideWithDefaults() { AwsClientFactory factory = new AwsClientFactory(config); - ClientOverrideConfiguration settings = factory.createClientConfig("ec2-test-default"); + Config cfg = factory.getConfig("ec2-test-default", Ec2Client.class).getConfig("client"); + ClientOverrideConfiguration settings = factory.createClientConfig(cfg); Assert.assertEquals("override-defaults", getUserAgentPrefix(settings)); } @@ -99,7 +103,8 @@ private SdkHttpService httpService() { @Test public void createCredentialsProvider() { AwsClientFactory factory = new AwsClientFactory(config); - AwsCredentialsProvider creds = factory.createCredentialsProvider(null, null, httpService()); + Config cfg = factory.getConfig(null, Ec2Client.class).getConfig("credentials"); + AwsCredentialsProvider creds = factory.createCredentialsProvider(cfg, null, httpService()); Assert.assertTrue(creds instanceof DefaultCredentialsProvider); } @@ -116,7 +121,8 @@ private AssumeRoleRequest getRequest(AwsCredentialsProvider creds) throws Except @Test public void createCredentialsProviderOverride() throws Exception { AwsClientFactory factory = new AwsClientFactory(config); - AwsCredentialsProvider creds = factory.createCredentialsProvider("ec2-test", null, httpService()); + Config cfg = factory.getConfig("ec2-test", Ec2Client.class).getConfig("credentials"); + AwsCredentialsProvider creds = factory.createCredentialsProvider(cfg, null, httpService()); Assert.assertTrue(creds instanceof StsAssumeRoleCredentialsProvider); Assert.assertEquals("arn:aws:iam::1234567890:role/IepTest", getRequest(creds).roleArn()); Assert.assertEquals("iep", getRequest(creds).roleSessionName()); @@ -125,7 +131,8 @@ public void createCredentialsProviderOverride() throws Exception { @Test public void createCredentialsProviderForAccount() throws Exception { AwsClientFactory factory = new AwsClientFactory(config); - AwsCredentialsProvider creds = factory.createCredentialsProvider("ec2-account", "123", httpService()); + Config cfg = factory.getConfig("ec2-account", Ec2Client.class).getConfig("credentials"); + AwsCredentialsProvider creds = factory.createCredentialsProvider(cfg, "123", httpService()); Assert.assertTrue(creds instanceof StsAssumeRoleCredentialsProvider); Assert.assertEquals("arn:aws:iam::123:role/IepTest", getRequest(creds).roleArn()); Assert.assertEquals("ieptest", getRequest(creds).roleSessionName()); @@ -134,13 +141,15 @@ public void createCredentialsProviderForAccount() throws Exception { @Test(expected = IllegalArgumentException.class) public void createCredentialsProviderForAccountNull() throws Exception { AwsClientFactory factory = new AwsClientFactory(config); - factory.createCredentialsProvider("ec2-account", null, httpService()); + Config cfg = factory.getConfig("ec2-account", Ec2Client.class).getConfig("credentials"); + factory.createCredentialsProvider(cfg, null, httpService()); } @Test public void createCredentialsProviderForAccountIgnored() throws Exception { AwsClientFactory factory = new AwsClientFactory(config); - AwsCredentialsProvider creds = factory.createCredentialsProvider("ec2-test", "123", httpService()); + Config cfg = factory.getConfig("ec2-test", Ec2Client.class).getConfig("credentials"); + AwsCredentialsProvider creds = factory.createCredentialsProvider(cfg, "123", httpService()); Assert.assertTrue(creds instanceof StsAssumeRoleCredentialsProvider); Assert.assertEquals("arn:aws:iam::1234567890:role/IepTest", getRequest(creds).roleArn()); Assert.assertEquals("iep", getRequest(creds).roleSessionName()); @@ -239,21 +248,24 @@ public void closeClients() throws Exception { @Test public void settingsUserAgentPrefix() { AwsClientFactory factory = new AwsClientFactory(config); - ClientOverrideConfiguration settings = factory.createClientConfig(null); + Config cfg = factory.getConfig(null, Ec2Client.class).getConfig("client"); + ClientOverrideConfiguration settings = factory.createClientConfig(cfg); Assert.assertEquals("default", getUserAgentPrefix(settings)); } @Test public void settingsUserAgentSuffix() { AwsClientFactory factory = new AwsClientFactory(config); - ClientOverrideConfiguration settings = factory.createClientConfig(null); + Config cfg = factory.getConfig(null, Ec2Client.class).getConfig("client"); + ClientOverrideConfiguration settings = factory.createClientConfig(cfg); Assert.assertEquals("suffix", getUserAgentSuffix(settings)); } @Test public void settingsHeaders() { AwsClientFactory factory = new AwsClientFactory(config); - ClientOverrideConfiguration settings = factory.createClientConfig("headers"); + Config cfg = factory.getConfig("headers", Ec2Client.class).getConfig("client"); + ClientOverrideConfiguration settings = factory.createClientConfig(cfg); Map> headers = settings.headers(); Assert.assertEquals(1, headers.size()); Assert.assertEquals(Collections.singletonList("gzip"), headers.get("Accept-Encoding")); @@ -262,7 +274,8 @@ public void settingsHeaders() { @Test public void settingsHeadersInvalid() { AwsClientFactory factory = new AwsClientFactory(config); - ClientOverrideConfiguration settings = factory.createClientConfig("headers-invalid"); + Config cfg = factory.getConfig("headers-invalid", Ec2Client.class).getConfig("client"); + ClientOverrideConfiguration settings = factory.createClientConfig(cfg); Map> headers = settings.headers(); Assert.assertTrue(headers.isEmpty()); } @@ -270,7 +283,8 @@ public void settingsHeadersInvalid() { @Test public void settingsTimeout() { AwsClientFactory factory = new AwsClientFactory(config); - ClientOverrideConfiguration settings = factory.createClientConfig("timeouts"); + Config cfg = factory.getConfig("timeouts", Ec2Client.class).getConfig("client"); + ClientOverrideConfiguration settings = factory.createClientConfig(cfg); Assert.assertEquals(Duration.ofMillis(42000), settings.apiCallAttemptTimeout().get()); Assert.assertEquals(Duration.ofMillis(13000), settings.apiCallTimeout().get()); Assert.assertEquals(Integer.valueOf(5), settings.retryPolicy().get().numRetries()); @@ -279,11 +293,22 @@ public void settingsTimeout() { @Test public void settingsSdkHttpClient() { AwsClientFactory factory = new AwsClientFactory(config); - AttributeMap settings = factory.getSdkHttpConfigurationOptions("sdk-http-client"); + Config cfg = factory.getConfig("sdk-http-client", Ec2Client.class).getConfig("client"); + AttributeMap settings = factory.getSdkHttpConfigurationOptions(cfg); Assert.assertEquals(Duration.ofMillis(60000), settings.get(SdkHttpConfigurationOption.CONNECTION_TIMEOUT)); Assert.assertEquals(Duration.ofMillis(120000), settings.get(SdkHttpConfigurationOption.CONNECTION_MAX_IDLE_TIMEOUT)); Assert.assertEquals(500, settings.get(SdkHttpConfigurationOption.MAX_CONNECTIONS).intValue()); Assert.assertEquals(true, settings.get(SdkHttpConfigurationOption.REAP_IDLE_CONNECTIONS)); + } + + private boolean isDualstackEnabled(AwsClientFactory factory, Class cls) { + return factory.getConfig(null, cls).getBoolean("dualstack"); + } + @Test + public void dualstack() { + AwsClientFactory factory = new AwsClientFactory(config); + Assert.assertTrue(isDualstackEnabled(factory, Ec2Client.class)); + Assert.assertFalse(isDualstackEnabled(factory, StsClient.class)); } }