Pre-work for S3 access logs hive table feature (#152)

* Prework for S3 access logs hive table feature * fix bugs * manage s3 logs bucket * fix iam role * fix iam role Co-authored-by: Scott Barnhart <[email protected]>
ExpediaGroup · Apr 21, 2020 · 8ab5540 · 8ab5540
1 parent a1547af
commit 8ab5540
Show file tree

Hide file tree

Showing 8 changed files with 107 additions and 7 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -3,6 +3,11 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [6.1.0] - 2020-04-21
+### Added
+- If Apiary's default S3 access log management is enabled (i.e., `var.apiary_log_bucket` is not set by the user), signal the Hive metastore to create the Hive database `s3_logs_hive` on startup. This is pre-work to prepare for S3 access-log Hive tables in a future version of Apiary. Requires `apiary-metastore-docker` version `1.13.0` or above.
+
+
 ## [6.0.0] - 2020-04-08
 ### Added
 - Per-schema option to send S3 data notifications to an SQS queue.  See `enable_data_events_sqs` in the [apiary_managed_schemas](VARIABLES.md#apiary_managed_schemas) section of [VARIABLES.md](VARIABLES.md)

diff --git a/common.tf b/common.tf
@@ -29,6 +29,9 @@ locals {
   s3_inventory_prefix                  = "EntireBucketDaily"
   s3_inventory_bucket                  = var.s3_enable_inventory ? "${local.apiary_bucket_prefix}-s3-inventory" : ""
   create_sqs_data_event_queue          = contains([for schema in local.schemas_info: lookup(schema, "enable_data_events_sqs", "0")], "1") ? true : false
+  enable_apiary_s3_log_management      = var.apiary_log_bucket == "" ? true : false
+  apiary_s3_logs_bucket                = local.enable_apiary_s3_log_management ? "${local.apiary_bucket_prefix}-s3-logs" : ""
+  apiary_s3_hive_logs_bucket           = local.enable_apiary_s3_log_management ? "${local.apiary_s3_logs_bucket}-hive" : ""
 }
 
 data "aws_iam_account_alias" "current" {}

diff --git a/iam-policy-s3-buckets.tf b/iam-policy-s3-buckets.tf
@@ -173,3 +173,53 @@ resource "aws_iam_role_policy" "s3_inventory_for_hms_readonly" {
 }
 EOF
 }
+
+resource "aws_iam_role_policy" "s3_access_logs_for_hms_readwrite" {
+  count = local.enable_apiary_s3_log_management ? 1 : 0
+  name  = "s3-access-logs"
+  role  = "${aws_iam_role.apiary_hms_readwrite.id}"
+
+  policy = <<EOF
+{
+ "Version": "2012-10-17",
+ "Statement": [
+                {
+                  "Effect": "Allow",
+                  "Action": [
+                              "s3:Get*",
+                              "s3:List*"
+                            ],
+                  "Resource": [
+                                "${format("arn:aws:s3:::%s", local.apiary_s3_hive_logs_bucket)}",
+                                "${format("arn:aws:s3:::%s/*", local.apiary_s3_hive_logs_bucket)}"
+                              ]
+                }
+              ]
+}
+EOF
+}
+
+resource "aws_iam_role_policy" "s3_access_logs_for_hms_readonly" {
+  count = local.enable_apiary_s3_log_management ? 1 : 0
+  name  = "s3-access-logs"
+  role  = "${aws_iam_role.apiary_hms_readonly.id}"
+
+  policy = <<EOF
+{
+ "Version": "2012-10-17",
+ "Statement": [
+                {
+                  "Effect": "Allow",
+                  "Action": [
+                              "s3:Get*",
+                              "s3:List*"
+                            ],
+                  "Resource": [
+                                "${format("arn:aws:s3:::%s", local.apiary_s3_hive_logs_bucket)}",
+                                "${format("arn:aws:s3:::%s/*", local.apiary_s3_hive_logs_bucket)}"
+                              ]
+                }
+              ]
+}
+EOF
+}
diff --git a/k8s-readwrite.tf b/k8s-readwrite.tf
@@ -145,7 +145,12 @@ resource "kubernetes_deployment" "apiary_hms_readwrite" {
           }
           env {
             name  = "ENABLE_S3_INVENTORY"
-            value = var.s3_enable_inventory
+            value = var.s3_enable_inventory ? "1" : ""
+          }
+          env {
+            # If user sets "apiary_log_bucket", then they are doing their own access logs mgmt, and not using Apiary's log mgmt.
+            name  = "ENABLE_S3_LOGS"
+            value = local.enable_apiary_s3_log_management ? "1" : ""
           }
 
           resources {

diff --git a/s3-other.tf b/s3-other.tf
@@ -44,10 +44,10 @@ resource "aws_s3_bucket_public_access_block" "apiary_inventory_bucket" {
 }
 
 resource "aws_s3_bucket" "apiary_managed_logs_bucket" {
-  count  = var.apiary_log_bucket == "" ? 1 : 0
-  bucket = "${local.apiary_bucket_prefix}-s3-logs"
+  count  = local.enable_apiary_s3_log_management ? 1 : 0
+  bucket = local.apiary_s3_logs_bucket
   acl    = "log-delivery-write"
-  tags   = "${merge(map("Name", "${local.apiary_bucket_prefix}-s3-logs"), "${var.apiary_tags}")}"
+  tags   = merge(map("Name", local.apiary_s3_logs_bucket), var.apiary_tags)
 
   server_side_encryption_configuration {
     rule {
@@ -70,3 +70,35 @@ resource "aws_s3_bucket" "apiary_managed_logs_bucket" {
     }
   }
 }
+
+resource "aws_s3_bucket_public_access_block" "apiary_managed_logs_bucket" {
+  count  = local.enable_apiary_s3_log_management ? 1 : 0
+  bucket = local.apiary_s3_logs_bucket
+
+  block_public_acls   = true
+  block_public_policy = true
+  ignore_public_acls  = true
+}
+
+resource "aws_s3_bucket" "apiary_access_logs_hive" {
+  count  = local.enable_apiary_s3_log_management ? 1 : 0
+  bucket = local.apiary_s3_hive_logs_bucket
+  tags   = merge(map("Name", local.apiary_s3_hive_logs_bucket), var.apiary_tags)
+  acl    = "private"
+  server_side_encryption_configuration {
+    rule {
+      apply_server_side_encryption_by_default {
+        sse_algorithm = "AES256"
+      }
+    }
+  }
+}
+
+resource "aws_s3_bucket_public_access_block" "apiary_access_logs_hive" {
+  count  = local.enable_apiary_s3_log_management ? 1 : 0
+  bucket = local.apiary_s3_hive_logs_bucket
+
+  block_public_acls   = true
+  block_public_policy = true
+  ignore_public_acls  = true
+}
diff --git a/s3.tf b/s3.tf
@@ -41,7 +41,7 @@ resource "aws_s3_bucket" "apiary_data_bucket" {
                         jsondecode(lookup(each.value, "tags", "{}")))
 
   logging {
-    target_bucket = var.apiary_log_bucket == "" ? aws_s3_bucket.apiary_managed_logs_bucket[0].id : var.apiary_log_bucket
+    target_bucket = local.enable_apiary_s3_log_management ? aws_s3_bucket.apiary_managed_logs_bucket[0].id : var.apiary_log_bucket
     target_prefix = "${var.apiary_log_prefix}${each.value["data_bucket"]}/"
   }
 

diff --git a/templates.tf b/templates.tf
@@ -47,8 +47,9 @@ data "template_file" "hms_readwrite" {
     #to instruct ECS to use repositoryCredentials for private docker registry
     docker_auth = "${var.docker_registry_auth_secret_name == "" ? "" : format("\"repositoryCredentials\" :{\n \"credentialsParameter\":\"%s\"\n},", join("", data.aws_secretsmanager_secret.docker_registry.*.arn))}"
 
-    # S3 inventory
-    s3_enable_inventory = var.s3_enable_inventory
+    s3_enable_inventory = var.s3_enable_inventory ? "1" : ""
+    # If user sets "apiary_log_bucket", then they are doing their own access logs mgmt, and not using Apiary's log mgmt.
+    s3_enable_logs      = local.enable_apiary_s3_log_management ? "1" : ""
   }
 }
 

diff --git a/templates/apiary-hms-readwrite.json b/templates/apiary-hms-readwrite.json
@@ -141,6 +141,10 @@
      {
       "name": "ENABLE_S3_INVENTORY",
       "value": "${s3_enable_inventory}"
+     },
+     {
+       "name": "ENABLE_S3_LOGS",
+       "value": "${s3_enable_logs}"
      }
     ]
   }