feat: GCS provisioner using ADC or existing service account for acces…

…s tokens (#111) * feat: IAM service account impersonation and ADC access token * refactor: use type inference * refactor: variable name * refactor: single-field builder for IAM Service - IamServiceImpl.Builder now single-field - Added AccessTokenProvider interface for adding app default credential provider and related test * chore: update dependencies * refactor: added @OverRide annotation * refactor: IAM service unit test * refactor: IAM service impl. ADC token provider with record * feat: GCS provisioner using existing credentials GCS provisioner uses either application default credentials or existing service account to generate the access token for the sink * feat: GCS provisioner uses connector configuration to fetch user account - GCS provisioner finds service account name for access token from (first valid in sequence): - transfer request configuration - connector configuration from GcpConfiguration - application default credentials - refactor serviceAccount to serviceAccountName for GcsResourceDefinition * chore: updated dependencies google-api-services-iam v2-rev20240108-2.0.0 license is apache 2.0 https://mvnrepository.com/artifact/com.google.apis/google-api-services-iam/v2-rev20240108-2.0.0 * refactor: GCP config returns null if no parameter, GCS provisioner removed / inlined functions
eclipse-edc · Jan 29, 2024 · f469278 · f469278
1 parent b31f543
commit f469278
Show file tree

Hide file tree

Showing 13 changed files with 244 additions and 221 deletions.
diff --git a/DEPENDENCIES b/DEPENDENCIES
@@ -28,6 +28,7 @@ maven/mavencentral/com.google.api/api-common/2.21.0, BSD-3-Clause, approved, cle
 maven/mavencentral/com.google.api/gax-grpc/2.38.0, BSD-3-Clause, approved, clearlydefined
 maven/mavencentral/com.google.api/gax-httpjson/2.38.0, BSD-3-Clause, approved, clearlydefined
 maven/mavencentral/com.google.api/gax/2.38.0, BSD-3-Clause, approved, #12035
+maven/mavencentral/com.google.apis/google-api-services-iam/v2-rev20240108-2.0.0, , restricted, clearlydefined
 maven/mavencentral/com.google.apis/google-api-services-storage/v1-rev20231117-2.0.0, Apache-2.0, approved, clearlydefined
 maven/mavencentral/com.google.auth/google-auth-library-credentials/1.20.0, BSD-3-Clause, approved, clearlydefined
 maven/mavencentral/com.google.auth/google-auth-library-oauth2-http/1.20.0, BSD-3-Clause, approved, clearlydefined
@@ -52,14 +53,21 @@ maven/mavencentral/com.google.guava/failureaccess/1.0.1, Apache-2.0, approved, C
 maven/mavencentral/com.google.guava/guava/29.0-android, Apache-2.0, approved, clearlydefined
 maven/mavencentral/com.google.guava/guava/30.1.1-android, Apache-2.0 AND CC0-1.0 AND LicenseRef-Public-Domain, approved, CQ23244
 maven/mavencentral/com.google.guava/guava/31.0.1-jre, Apache-2.0, approved, clearlydefined
+maven/mavencentral/com.google.guava/guava/31.1-android, Apache-2.0, approved, clearlydefined
+maven/mavencentral/com.google.guava/guava/31.1-jre, Apache-2.0, approved, clearlydefined
 maven/mavencentral/com.google.guava/guava/32.0.0-android, Apache-2.0 AND CC0-1.0 AND CC-PDDC, approved, #8772
 maven/mavencentral/com.google.guava/guava/32.0.1-jre, Apache-2.0 AND CC0-1.0 AND CC-PDDC, approved, #8772
 maven/mavencentral/com.google.guava/guava/32.1.3-jre, Apache-2.0 AND CC0-1.0 AND LicenseRef-Public-Domain, approved, #9229
 maven/mavencentral/com.google.guava/listenablefuture/9999.0-empty-to-avoid-conflict-with-guava, Apache-2.0, approved, CQ22657
+maven/mavencentral/com.google.http-client/google-http-client-apache-v2/1.42.3, Apache-2.0, approved, clearlydefined
 maven/mavencentral/com.google.http-client/google-http-client-apache-v2/1.43.3, Apache-2.0, approved, clearlydefined
 maven/mavencentral/com.google.http-client/google-http-client-appengine/1.43.3, Apache-2.0, approved, clearlydefined
+maven/mavencentral/com.google.http-client/google-http-client-gson/1.42.0, Apache-2.0, approved, clearlydefined
+maven/mavencentral/com.google.http-client/google-http-client-gson/1.42.3, Apache-2.0, approved, clearlydefined
 maven/mavencentral/com.google.http-client/google-http-client-gson/1.43.3, Apache-2.0, approved, clearlydefined
 maven/mavencentral/com.google.http-client/google-http-client-jackson2/1.43.3, Apache-2.0, approved, clearlydefined
+maven/mavencentral/com.google.http-client/google-http-client/1.42.0, Apache-2.0, approved, clearlydefined
+maven/mavencentral/com.google.http-client/google-http-client/1.42.3, Apache-2.0, approved, clearlydefined
 maven/mavencentral/com.google.http-client/google-http-client/1.43.3, Apache-2.0, approved, clearlydefined
 maven/mavencentral/com.google.j2objc/j2objc-annotations/1.3, Apache-2.0, approved, CQ21195
 maven/mavencentral/com.google.j2objc/j2objc-annotations/2.8, Apache-2.0, approved, clearlydefined
@@ -77,6 +85,7 @@ maven/mavencentral/com.squareup.okio/okio-jvm/3.6.0, Apache-2.0, approved, #1115
 maven/mavencentral/com.squareup.okio/okio/3.6.0, Apache-2.0, approved, #11155
 maven/mavencentral/commons-beanutils/commons-beanutils/1.9.4, Apache-2.0, approved, CQ12654
 maven/mavencentral/commons-codec/commons-codec/1.11, Apache-2.0 AND BSD-3-Clause, approved, CQ15971
+maven/mavencentral/commons-codec/commons-codec/1.15, Apache-2.0 AND BSD-3-Clause AND LicenseRef-Public-Domain, approved, CQ22641
 maven/mavencentral/commons-codec/commons-codec/1.16.0, Apache-2.0 AND (Apache-2.0 AND BSD-3-Clause), approved, #9157
 maven/mavencentral/commons-collections/commons-collections/3.2.2, Apache-2.0, approved, CQ10385
 maven/mavencentral/commons-logging/commons-logging/1.2, Apache-2.0, approved, CQ10162
@@ -117,9 +126,7 @@ maven/mavencentral/jakarta.ws.rs/jakarta.ws.rs-api/3.1.0, EPL-2.0 OR GPL-2.0-onl
 maven/mavencentral/jakarta.xml.bind/jakarta.xml.bind-api/4.0.0, BSD-3-Clause, approved, ee4j.jaxb
 maven/mavencentral/javax.annotation/javax.annotation-api/1.3.2, CDDL-1.1 OR GPL-2.0-only WITH Classpath-exception-2.0, approved, CQ16910
 maven/mavencentral/junit/junit/4.13.2, EPL-2.0, approved, CQ23636
-maven/mavencentral/net.bytebuddy/byte-buddy-agent/1.14.10, Apache-2.0, approved, #7164
-maven/mavencentral/net.bytebuddy/byte-buddy/1.12.21, Apache-2.0 AND BSD-3-Clause, approved, #1811
-maven/mavencentral/net.bytebuddy/byte-buddy/1.14.10, Apache-2.0 AND BSD-3-Clause, approved, #7163
+maven/mavencentral/net.bytebuddy/byte-buddy-agent/1.14.11, Apache-2.0, approved, #7164
 maven/mavencentral/net.bytebuddy/byte-buddy/1.14.11, Apache-2.0 AND BSD-3-Clause, approved, #7163
 maven/mavencentral/net.java.dev.jna/jna/5.13.0, Apache-2.0 AND LGPL-2.1-or-later, approved, #6709
 maven/mavencentral/net.sf.saxon/Saxon-HE/10.6, MPL-2.0 AND W3C, approved, #7945
@@ -128,7 +135,6 @@ maven/mavencentral/org.apache.commons/commons-compress/1.24.0, Apache-2.0 AND BS
 maven/mavencentral/org.apache.httpcomponents/httpclient/4.5.14, Apache-2.0 AND LicenseRef-Public-Domain, approved, CQ23527
 maven/mavencentral/org.apache.httpcomponents/httpcore/4.4.16, Apache-2.0, approved, CQ23528
 maven/mavencentral/org.apiguardian/apiguardian-api/1.1.2, Apache-2.0, approved, clearlydefined
-maven/mavencentral/org.assertj/assertj-core/3.24.2, Apache-2.0, approved, #6161
 maven/mavencentral/org.assertj/assertj-core/3.25.1, Apache-2.0, approved, #12585
 maven/mavencentral/org.bouncycastle/bcpkix-jdk18on/1.77, MIT, approved, #11593
 maven/mavencentral/org.bouncycastle/bcprov-jdk18on/1.77, MIT AND CC0-1.0, approved, #11595
@@ -232,7 +238,7 @@ maven/mavencentral/org.junit/junit-bom/5.10.1, EPL-2.0, approved, #9844
 maven/mavencentral/org.junit/junit-bom/5.9.2, EPL-2.0, approved, #4711
 maven/mavencentral/org.jvnet.mimepull/mimepull/1.9.15, CDDL-1.1 OR GPL-2.0-only WITH Classpath-exception-2.0, approved, CQ21484
 maven/mavencentral/org.mockito/mockito-core/5.2.0, MIT AND (Apache-2.0 AND MIT) AND Apache-2.0, approved, #7401
-maven/mavencentral/org.mockito/mockito-core/5.8.0, MIT AND (Apache-2.0 AND MIT) AND Apache-2.0, approved, #11787
+maven/mavencentral/org.mockito/mockito-core/5.9.0, MIT AND (Apache-2.0 AND MIT) AND Apache-2.0, approved, #12774
 maven/mavencentral/org.objenesis/objenesis/3.3, Apache-2.0, approved, clearlydefined
 maven/mavencentral/org.opentest4j/opentest4j/1.3.0, Apache-2.0, approved, #9713
 maven/mavencentral/org.ow2.asm/asm-commons/9.5, BSD-3-Clause, approved, #7553

diff --git a/extensions/common/gcp/gcp-core/build.gradle.kts b/extensions/common/gcp/gcp-core/build.gradle.kts
@@ -24,6 +24,7 @@ dependencies {
     implementation(libs.googlecloud.iam.admin)
     implementation(libs.googlecloud.storage)
     implementation(libs.googlecloud.iam.credentials)
+    implementation(libs.googleapis.iam)
     testImplementation(libs.edc.junit)
 }
 

diff --git a/...nsions/common/gcp/gcp-core/src/main/java/org/eclipse/edc/gcp/common/GcpConfiguration.java b/...nsions/common/gcp/gcp-core/src/main/java/org/eclipse/edc/gcp/common/GcpConfiguration.java
@@ -41,9 +41,9 @@ public class GcpConfiguration {
 
     public GcpConfiguration(ServiceExtensionContext context) {
         projectId = context.getSetting(PROJECT_ID, ServiceOptions.getDefaultProjectId());
-        serviceAccountName = context.getSetting(SACCOUNT_NAME, "");
-        serviceAccountFile = context.getSetting(SACCOUNT_FILE, "");
-        universeDomain = context.getSetting(UNIVERSE_DOMAIN, "");
+        serviceAccountName = context.getSetting(SACCOUNT_NAME, null);
+        serviceAccountFile = context.getSetting(SACCOUNT_FILE, null);
+        universeDomain = context.getSetting(UNIVERSE_DOMAIN, null);
     }
 
     /**

diff --git a/...nsions/common/gcp/gcp-core/src/main/java/org/eclipse/edc/gcp/iam/AccessTokenProvider.java b/...nsions/common/gcp/gcp-core/src/main/java/org/eclipse/edc/gcp/iam/AccessTokenProvider.java
@@ -0,0 +1,29 @@
+/*
+ *  Copyright (c) 2024 Google LLC
+ *
+ *  This program and the accompanying materials are made available under the
+ *  terms of the Apache License, Version 2.0 which is available at
+ *  https://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  SPDX-License-Identifier: Apache-2.0
+ *
+ *  Contributors:
+ *       Google LCC - Initial implementation
+ *
+ */
+
+package org.eclipse.edc.gcp.iam;
+
+import org.eclipse.edc.gcp.common.GcpAccessToken;
+
+/**
+ * Interface for credentials providing access tokens.
+ */
+public interface AccessTokenProvider {
+    /**
+     * Returns the access token generated for the credentials.
+     *
+     * @return the {@link GcpAccessToken} for the credentials, null if error occurs.
+     */
+    GcpAccessToken getAccessToken();
+}
diff --git a/extensions/common/gcp/gcp-core/src/main/java/org/eclipse/edc/gcp/iam/IamService.java b/extensions/common/gcp/gcp-core/src/main/java/org/eclipse/edc/gcp/iam/IamService.java
@@ -23,13 +23,12 @@
  */
 public interface IamService {
     /**
-     * Creates or returns the service account with the matching name and description.
+     * Returns the existing service account with the matching name.
      *
      * @param serviceAccountName        the name for the service account. Limited to 30 chars
-     * @param serviceAccountDescription the unique description for the service account that is used to avoid reuse of service accounts
      * @return the {@link GcpServiceAccount} describing the service account
      */
-    GcpServiceAccount getOrCreateServiceAccount(String serviceAccountName, String serviceAccountDescription);
+    GcpServiceAccount getServiceAccount(String serviceAccountName);
 
     /**
      * Creates a temporary valid OAunth2.0 access token for the service account
@@ -40,10 +39,9 @@ public interface IamService {
     GcpAccessToken createAccessToken(GcpServiceAccount serviceAccount);
 
     /**
-     * Delete the specified service account if it exists.
-     * Do nothing in case it doesn't exist (anymore)
+     * Creates a temporary valid OAunth2.0 access token using the application default account credentials.
      *
-     * @param serviceAccount The service account that should be deleted
+     * @return {@link GcpAccessToken}
      */
-    void deleteServiceAccountIfExists(GcpServiceAccount serviceAccount);
+    GcpAccessToken createDefaultAccessToken();
 }
diff --git a/extensions/common/gcp/gcp-core/src/main/java/org/eclipse/edc/gcp/iam/IamServiceImpl.java b/extensions/common/gcp/gcp-core/src/main/java/org/eclipse/edc/gcp/iam/IamServiceImpl.java
@@ -16,81 +16,52 @@
 
 import com.google.api.gax.rpc.ApiException;
 import com.google.api.gax.rpc.StatusCode;
+import com.google.api.services.iam.v2.IamScopes;
+import com.google.auth.oauth2.GoogleCredentials;
 import com.google.cloud.iam.admin.v1.IAMClient;
 import com.google.cloud.iam.credentials.v1.GenerateAccessTokenRequest;
 import com.google.cloud.iam.credentials.v1.IamCredentialsClient;
 import com.google.cloud.iam.credentials.v1.ServiceAccountName;
-import com.google.common.collect.ImmutableList;
-import com.google.iam.admin.v1.CreateServiceAccountRequest;
-import com.google.iam.admin.v1.ProjectName;
-import com.google.iam.admin.v1.ServiceAccount;
 import com.google.protobuf.Duration;
 import org.eclipse.edc.gcp.common.GcpAccessToken;
 import org.eclipse.edc.gcp.common.GcpException;
 import org.eclipse.edc.gcp.common.GcpServiceAccount;
 import org.eclipse.edc.spi.monitor.Monitor;
 
 import java.io.IOException;
+import java.util.Collections;
 import java.util.Objects;
 import java.util.concurrent.TimeUnit;
 import java.util.function.Supplier;
 
 public class IamServiceImpl implements IamService {
-
-    private static final ImmutableList<String> OAUTH_SCOPE = ImmutableList.of("https://www.googleapis.com/auth/cloud-platform");
     private static final long ONE_HOUR_IN_S = TimeUnit.HOURS.toSeconds(1);
-    private final String gcpProjectId;
-    private final Supplier<IAMClient> iamClientSupplier;
-    private final Supplier<IamCredentialsClient> iamCredentialsClientSupplier;
     private final Monitor monitor;
+    private final String gcpProjectId;
+    private Supplier<IAMClient> iamClientSupplier;
+    private Supplier<IamCredentialsClient> iamCredentialsClientSupplier;
+    private AccessTokenProvider applicationDefaultCredentials;
 
-    private IamServiceImpl(Monitor monitor,
-                           String gcpProjectId,
-                           Supplier<IAMClient> iamClientSupplier,
-                           Supplier<IamCredentialsClient> iamCredentialsClientSupplier
-    ) {
+    private IamServiceImpl(Monitor monitor, String gcpProjectId) {
         this.monitor = monitor;
         this.gcpProjectId = gcpProjectId;
-        this.iamClientSupplier = iamClientSupplier;
-        this.iamCredentialsClientSupplier = iamCredentialsClientSupplier;
     }
 
     @Override
-    public GcpServiceAccount getOrCreateServiceAccount(String serviceAccountName, String serviceAccountDescription) {
-        var requestedServiceAccount = ServiceAccount.newBuilder()
-                .setDisplayName(serviceAccountName)
-                .setDescription(serviceAccountDescription)
-                .build();
-        var request = CreateServiceAccountRequest.newBuilder()
-                .setName(ProjectName.of(gcpProjectId).toString())
-                .setAccountId(serviceAccountName)
-                .setServiceAccount(requestedServiceAccount)
-                .build();
-
-        try (var client = iamClientSupplier.get()) {
-            var serviceAccount = client.createServiceAccount(request);
-            monitor.debug("Created service account: " + serviceAccount.getEmail());
-            return new GcpServiceAccount(serviceAccount.getEmail(), serviceAccount.getName(), serviceAccountDescription);
-        } catch (ApiException e) {
-            if (e.getStatusCode().getCode() == StatusCode.Code.ALREADY_EXISTS) {
-                return getServiceAccount(serviceAccountName, serviceAccountDescription);
-            }
-            monitor.severe("Unable to create service account", e);
-            throw new GcpException("Unable to create service account", e);
-        }
-    }
-
-    private GcpServiceAccount getServiceAccount(String serviceAccountName, String serviceAccountDescription) {
+    public GcpServiceAccount getServiceAccount(String serviceAccountName) {
         try (var client = iamClientSupplier.get()) {
             var serviceAccountEmail = getServiceAccountEmail(serviceAccountName, gcpProjectId);
             var name = ServiceAccountName.of(gcpProjectId, serviceAccountEmail).toString();
             var response = client.getServiceAccount(name);
-            if (!response.getDescription().equals(serviceAccountDescription)) {
-                String errorMessage = "A service account with the same name but different description existed already. Please ensure a unique name is used for every transfer process";
-                monitor.severe(errorMessage);
-                throw new GcpException(errorMessage);
-            }
+
             return new GcpServiceAccount(response.getEmail(), response.getName(), response.getDescription());
+        } catch (ApiException e) {
+            if (e.getStatusCode().getCode() == StatusCode.Code.NOT_FOUND) {
+                monitor.severe("Service account '" + serviceAccountName + "'not found", e);
+                throw new GcpException("Service account '" + serviceAccountName + "'not found", e);
+            }
+            monitor.severe("Unable to get service account '" + serviceAccountName + "'", e);
+            throw new GcpException("Unable to get service account '" + serviceAccountName + "'", e);
         }
     }
 
@@ -101,7 +72,7 @@ public GcpAccessToken createAccessToken(GcpServiceAccount serviceAccount) {
             var lifetime = Duration.newBuilder().setSeconds(ONE_HOUR_IN_S).build();
             var request = GenerateAccessTokenRequest.newBuilder()
                     .setName(name.toString())
-                    .addAllScope(OAUTH_SCOPE)
+                    .addAllScope(Collections.singleton(IamScopes.CLOUD_PLATFORM))
                     .setLifetime(lifetime)
                     .build();
             var response = iamCredentialsClient.generateAccessToken(request);
@@ -114,60 +85,56 @@ public GcpAccessToken createAccessToken(GcpServiceAccount serviceAccount) {
     }
 
     @Override
-    public void deleteServiceAccountIfExists(GcpServiceAccount serviceAccount) {
-        try (var client = iamClientSupplier.get()) {
-            var serviceAccountName = ServiceAccountName.of(gcpProjectId, serviceAccount.getEmail());
-            client.deleteServiceAccount(serviceAccountName.toString());
-            monitor.debug("Deleted service account: " + serviceAccount.getEmail());
-        } catch (ApiException e) {
-            if (e.getStatusCode().getCode() == StatusCode.Code.NOT_FOUND) {
-                monitor.severe("Service account not found", e);
-                return;
-            }
-            monitor.severe("Unable to delete service account", e);
-            throw new GcpException(e);
-        }
+    public GcpAccessToken createDefaultAccessToken() {
+        return applicationDefaultCredentials.getAccessToken();
     }
 
     private String getServiceAccountEmail(String name, String project) {
         return String.format("%s@%s.iam.gserviceaccount.com", name, project);
     }
 
     public static class Builder {
-        private final String gcpProjectId;
-        private final Monitor monitor;
-        private Supplier<IAMClient> iamClientSupplier;
-        private Supplier<IamCredentialsClient> iamCredentialsClientSupplier;
+        private IamServiceImpl iamServiceImpl;
 
         private Builder(Monitor monitor, String gcpProjectId) {
-            this.gcpProjectId = gcpProjectId;
-            this.monitor = monitor;
+            iamServiceImpl = new IamServiceImpl(monitor, gcpProjectId);
         }
 
         public static IamServiceImpl.Builder newInstance(Monitor monitor, String gcpProjectId) {
             return new Builder(monitor, gcpProjectId);
         }
 
         public Builder iamClientSupplier(Supplier<IAMClient> iamClientSupplier) {
-            this.iamClientSupplier = iamClientSupplier;
+            iamServiceImpl.iamClientSupplier = iamClientSupplier;
             return this;
         }
 
         public Builder iamCredentialsClientSupplier(Supplier<IamCredentialsClient> iamCredentialsClientSupplier) {
-            this.iamCredentialsClientSupplier = iamCredentialsClientSupplier;
+            iamServiceImpl.iamCredentialsClientSupplier = iamCredentialsClientSupplier;
+            return this;
+        }
+
+        public Builder applicationDefaultCredentials(AccessTokenProvider applicationDefaultCredentials) {
+            iamServiceImpl.applicationDefaultCredentials = applicationDefaultCredentials;
             return this;
         }
 
         public IamServiceImpl build() {
-            Objects.requireNonNull(gcpProjectId, "gcpProjectId");
-            Objects.requireNonNull(monitor, "monitor");
-            if (iamClientSupplier == null) {
-                iamClientSupplier = defaultIamClientSupplier();
+            Objects.requireNonNull(iamServiceImpl.gcpProjectId, "gcpProjectId");
+            Objects.requireNonNull(iamServiceImpl.monitor, "monitor");
+
+            if (iamServiceImpl.iamClientSupplier == null) {
+                iamServiceImpl.iamClientSupplier = defaultIamClientSupplier();
             }
-            if (iamCredentialsClientSupplier == null) {
-                iamCredentialsClientSupplier = defaultIamCredentialsClientSupplier();
+            if (iamServiceImpl.iamCredentialsClientSupplier == null) {
+                iamServiceImpl.iamCredentialsClientSupplier = defaultIamCredentialsClientSupplier();
             }
-            return new IamServiceImpl(monitor, gcpProjectId, iamClientSupplier, iamCredentialsClientSupplier);
+
+            if (iamServiceImpl.applicationDefaultCredentials == null) {
+                iamServiceImpl.applicationDefaultCredentials = new ApplicationDefaultCredentials(iamServiceImpl.monitor);
+            }
+
+            return iamServiceImpl;
         }
 
         /**
@@ -196,4 +163,19 @@ private Supplier<IamCredentialsClient> defaultIamCredentialsClientSupplier() {
             };
         }
     }
+
+    private record ApplicationDefaultCredentials(Monitor monitor) implements AccessTokenProvider {
+        @Override
+        public GcpAccessToken getAccessToken() {
+            try {
+                var credentials = GoogleCredentials.getApplicationDefault().createScoped(IamScopes.CLOUD_PLATFORM);
+                credentials.refreshIfExpired();
+                var token = credentials.getAccessToken();
+                return new GcpAccessToken(token.getTokenValue(), token.getExpirationTime().getTime());
+            } catch (IOException ioException) {
+                monitor.severe("Cannot get application default access token", ioException);
+                return null;
+            }
+        }
+    }
 }