move aws stuff to s3_ingest

auth.py

@@ -35,31 +35,3 @@ def get_refresh_token(token):
         client_secret = client_secret,
         refresh_token=token
         )
-
-#####
-# AWS stuff
-#####
-
-def get_minio_client(token, s3_endpoint, bucket, access_key=None, secret_key=None, region=None, secure=True):
-    return authx.auth.get_minio_client(token=token, s3_endpoint=s3_endpoint, bucket=bucket, access_key=access_key, secret_key=secret_key, region=region, secure=secure)
-
-
-def parse_s3_credential(awsfile):
-    # parse the awsfile:
-    access = None
-    secret = None
-    with open(awsfile) as f:
-        lines = f.readlines()
-        while len(lines) > 0 and (access is None or secret is None):
-            line = lines.pop(0)
-            parse_access = re.match(r"(aws_access_key_id|AWSAccessKeyId)\s*=\s*(.+)$", line)
-            if parse_access is not None:
-                access = parse_access.group(2)
-            parse_secret = re.match(r"(aws_secret_access_key|AWSSecretKey)\s*=\s*(.+)$", line)
-            if parse_secret is not None:
-                secret = parse_secret.group(2)
-    if access is None:
-        return {"error": "awsfile did not contain access ID"}
-    if secret is None:
-        return {"error": "awsfile did not contain secret key"}
-    return {"access": access, "secret": secret}

s3_ingest.py

@@ -7,6 +7,27 @@
 from pathlib import Path
 
 
+def parse_s3_credential(awsfile):
+    # parse the awsfile:
+    access = None
+    secret = None
+    with open(awsfile) as f:
+        lines = f.readlines()
+        while len(lines) > 0 and (access is None or secret is None):
+            line = lines.pop(0)
+            parse_access = re.match(r"(aws_access_key_id|AWSAccessKeyId)\s*=\s*(.+)$", line)
+            if parse_access is not None:
+                access = parse_access.group(2)
+            parse_secret = re.match(r"(aws_secret_access_key|AWSSecretKey)\s*=\s*(.+)$", line)
+            if parse_secret is not None:
+                secret = parse_secret.group(2)
+    if access is None:
+        return {"error": "awsfile did not contain access ID"}
+    if secret is None:
+        return {"error": "awsfile did not contain secret key"}
+    return {"access": access, "secret": secret}
+
+
 def main():
     parser = argparse.ArgumentParser(description="Script to ingest files into an S3-compatible bucket.")
 
@@ -33,7 +54,7 @@ def main():
 
     if args.awsfile:
         # parse the awsfile:
-        result = auth.parse_s3_credential(args.awsfile)
+        result = parse_s3_credential(args.awsfile)
         access_key = result["access"]
         secret_key = result["secret"]
         if "error" in result:
@@ -44,7 +65,7 @@ def main():
     else:
         raise Exception("Either awsfile or access/secret need to be provided.")
 
-    client = auth.get_minio_client(auth.get_site_admin_token(), args.endpoint, args.bucket, access_key=access_key, secret_key=secret_key)
+    client = authx.auth.get_minio_client(token=auth.get_site_admin_token(), s3_endpoint=args.endpoint, bucket=args.bucket, access_key=access_key, secret_key=secret_key)
 
     for sample in samples:
         file = Path(sample)