support for SSE-KMS encryption in Apiary managed S3 buckets (#161)

* rename apiary-bucket-policy.json * add encryption option * kms keys * test for_each condition * kms policy * kms client policy * fix * restrict writes to managed kms key * fix admin kms policy * restrict s3 bucket access to client roles * update readme * fix client roles * fix client_roles in template * kms alias * kms alias fix * remove client roles in bucket policy * update kms key admin policy * update changelog * cleanup * Update kms.tf Co-authored-by: Scott Barnhart <[email protected]> * update readme * update VARIABLES.md * Update VARIABLES.md Co-authored-by: Scott Barnhart <[email protected]> Co-authored-by: Raj Poluri <[email protected]> Co-authored-by: Scott Barnhart <[email protected]>
ExpediaGroup · Jun 8, 2020 · 5ddc1d6 · 5ddc1d6
1 parent 3f0400e
commit 5ddc1d6
Show file tree

Hide file tree

Showing 8 changed files with 162 additions and 30 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [6.3.0] - 2020-06-08
+### Added
+- Added support for SSE-KMS encryption in Apiary managed S3 bucket.
+
 ## [6.2.1] - 2020-05-27
 ### Changed
 - Optional `customer_principal` and `producer_iamroles` in Apiary managed bucket policies.

diff --git a/README.md b/README.md
@@ -49,6 +49,12 @@ module "apiary" {
     {
         schema_name = "db_2",
         s3_storage_class = "INTELLIGENT_TIERING"
+    },
+    {
+        schema_name = "secure_db",
+        encryption   = "aws:kms" //supported values for encryption are AES256,aws:kms
+        admin_roles = "role1_arn,role2_arn" //kms key management will be restricted to these roles.
+        client_roles = "role3_arn,role4_arn" //s3 bucket read/write and kms key usage will be restricted to these roles.
     }
   ]
   apiary_customer_accounts = ["aws_account_no_1", "aws_account_no_2"]

diff --git a/VARIABLES.md b/VARIABLES.md
@@ -115,6 +115,9 @@ apiary_managed_schemas = [
    s3_object_expiration_days = 60
    tags=jsonencode({ Domain = "search", ComponentInfo = "1234" })
    enable_data_events_sqs = "1"
+   encryption   = "aws:kms" //supported values for encryption are AES256,aws:kms
+   admin_roles = "role1_arn,role2_arn" //kms key management will be restricted to these roles.
+   client_roles = "role3_arn,role4_arn" //s3 bucket read/write and kms key usage will be restricted to these roles.
   }
 ]
 ```
@@ -128,3 +131,6 @@ Name | Description | Type | Default | Required |
 | s3_storage_class | Destination S3 storage class for transition in the lifecycle policy. For valid values for S3 Storage classes, reference: https://www.terraform.io/docs/providers/aws/r/s3_bucket.html#storage_class | string | "INTELLIGENT_TIERING" | No |
 | s3_object_expiration_days | Number of days after which objects in Apiary managed schema buckets expire. | number | null | No |
 | tags | Additional tags added to the S3 data bucket. The map of tags must be encoded as a string using `jsonencode` (see sample above). If the `var.apiary_tags` collection and the tags passed to `apiary_managed_schemas` both contain the same tag name, the tag value passed to `apiary_managed_schemas` will be used. | string | null | no |
+| encryption | S3 objects encryption type, supported values are AES256,aws:kms. | string | null | no |
+| admin_roles | IAM roles configured with admin access on corresponding KMS keys,required when encryption type is `aws:kms`. | string | null | no |
+| client_roles | IAM roles configured with usage access on corresponding KMS keys,required when encryption type is `aws:kms`. | string | null | no |
diff --git a/common.tf b/common.tf
@@ -5,43 +5,44 @@
  */
 
 locals {
-  instance_alias                       = "${var.instance_name == "" ? "apiary" : format("apiary-%s", var.instance_name)}"
-  apiary_bucket_prefix                 = "${local.instance_alias}-${data.aws_caller_identity.current.account_id}-${data.aws_region.current.name}"
-  apiary_assume_role_bucket_prefix     = [for assumerole in var.apiary_assume_roles : "${local.instance_alias}-${data.aws_caller_identity.current.account_id}-${lookup(assumerole, "allow_cross_region_access", false) ? "*" : data.aws_region.current.name}"]
-  enable_route53_records               = var.apiary_domain_name == "" ? false : true
+  instance_alias                   = "${var.instance_name == "" ? "apiary" : format("apiary-%s", var.instance_name)}"
+  apiary_bucket_prefix             = "${local.instance_alias}-${data.aws_caller_identity.current.account_id}-${data.aws_region.current.name}"
+  apiary_assume_role_bucket_prefix = [for assumerole in var.apiary_assume_roles : "${local.instance_alias}-${data.aws_caller_identity.current.account_id}-${lookup(assumerole, "allow_cross_region_access", false) ? "*" : data.aws_region.current.name}"]
+  enable_route53_records           = var.apiary_domain_name == "" ? false : true
   #
   # Create a new list of maps with some extra attributes needed later
   #
   schemas_info = [
-    for schema in var.apiary_managed_schemas: merge(
+    for schema in var.apiary_managed_schemas : merge(
       {
+        encryption : lookup(schema, "encryption", "AES256"),
         resource_suffix : replace(schema["schema_name"], "_", "-"),
-        data_bucket     : "${local.apiary_bucket_prefix}-${replace(schema["schema_name"], "_", "-")}"
+        data_bucket : "${local.apiary_bucket_prefix}-${replace(schema["schema_name"], "_", "-")}"
       },
-      schema)
+    schema)
   ]
 
-  gluedb_prefix                        = "${var.instance_name == "" ? "" : "${var.instance_name}_"}"
-  cw_arn                               = "arn:aws:swf:${var.aws_region}:${data.aws_caller_identity.current.account_id}:action/actions/AWS_EC2.InstanceId.Reboot/1.0"
-  assume_allowed_principals            = split(",", join(",", [for role in var.apiary_assume_roles : join(",", [for principal in role.principals : replace(principal, "/:role.*/", ":root")])]))
-  producer_allowed_principals          = split(",", join(",", values(var.apiary_producer_iamroles)))
-  final_atlas_cluster_name             = "${var.atlas_cluster_name == "" ? local.instance_alias : var.atlas_cluster_name}"
-  s3_inventory_prefix                  = "EntireBucketDaily"
-  s3_inventory_bucket                  = var.s3_enable_inventory ? "${local.apiary_bucket_prefix}-s3-inventory" : ""
-  create_sqs_data_event_queue          = contains([for schema in local.schemas_info: lookup(schema, "enable_data_events_sqs", "0")], "1") ? true : false
-  enable_apiary_s3_log_management      = var.apiary_log_bucket == "" ? true : false
-  apiary_s3_logs_bucket                = local.enable_apiary_s3_log_management ? "${local.apiary_bucket_prefix}-s3-logs" : ""
-  apiary_s3_hive_logs_bucket           = local.enable_apiary_s3_log_management ? "${local.apiary_s3_logs_bucket}-hive" : ""
-
-  hms_ro_heapsize                      = ceil((var.hms_ro_heapsize * 85) / 100)
-  hms_ro_minthreads                    = max(25,  ceil((var.hms_ro_heapsize * 12.5) / 100))
-  hms_ro_maxthreads                    = max(100, ceil((var.hms_ro_heapsize * 50)   / 100))
-
-  hms_rw_heapsize                      = ceil((var.hms_rw_heapsize * 85) / 100)
-  hms_rw_minthreads                    = max(25,  ceil((var.hms_rw_heapsize * 12.5) / 100))
-  hms_rw_maxthreads                    = max(100, ceil((var.hms_rw_heapsize * 50)   / 100))
-
-  hms_alias                            = var.instance_name == "" ? "hms" : "hms-${var.instance_name}"
+  gluedb_prefix                   = "${var.instance_name == "" ? "" : "${var.instance_name}_"}"
+  cw_arn                          = "arn:aws:swf:${var.aws_region}:${data.aws_caller_identity.current.account_id}:action/actions/AWS_EC2.InstanceId.Reboot/1.0"
+  assume_allowed_principals       = split(",", join(",", [for role in var.apiary_assume_roles : join(",", [for principal in role.principals : replace(principal, "/:role.*/", ":root")])]))
+  producer_allowed_principals     = split(",", join(",", values(var.apiary_producer_iamroles)))
+  final_atlas_cluster_name        = "${var.atlas_cluster_name == "" ? local.instance_alias : var.atlas_cluster_name}"
+  s3_inventory_prefix             = "EntireBucketDaily"
+  s3_inventory_bucket             = var.s3_enable_inventory ? "${local.apiary_bucket_prefix}-s3-inventory" : ""
+  create_sqs_data_event_queue     = contains([for schema in local.schemas_info : lookup(schema, "enable_data_events_sqs", "0")], "1") ? true : false
+  enable_apiary_s3_log_management = var.apiary_log_bucket == "" ? true : false
+  apiary_s3_logs_bucket           = local.enable_apiary_s3_log_management ? "${local.apiary_bucket_prefix}-s3-logs" : ""
+  apiary_s3_hive_logs_bucket      = local.enable_apiary_s3_log_management ? "${local.apiary_s3_logs_bucket}-hive" : ""
+
+  hms_ro_heapsize   = ceil((var.hms_ro_heapsize * 85) / 100)
+  hms_ro_minthreads = max(25, ceil((var.hms_ro_heapsize * 12.5) / 100))
+  hms_ro_maxthreads = max(100, ceil((var.hms_ro_heapsize * 50) / 100))
+
+  hms_rw_heapsize   = ceil((var.hms_rw_heapsize * 85) / 100)
+  hms_rw_minthreads = max(25, ceil((var.hms_rw_heapsize * 12.5) / 100))
+  hms_rw_maxthreads = max(100, ceil((var.hms_rw_heapsize * 50) / 100))
+
+  hms_alias = var.instance_name == "" ? "hms" : "hms-${var.instance_name}"
 }
 
 data "aws_iam_account_alias" "current" {}

diff --git a/kms.tf b/kms.tf
@@ -0,0 +1,34 @@
+resource "aws_kms_key" "apiary_kms" {
+  for_each = {
+    for schema in local.schemas_info : "${schema["schema_name"]}" => schema if schema["encryption"] == "aws:kms"
+  }
+
+  description = "Apiary ${each.key} KMS key"
+  policy      = data.template_file.apiary_kms_key_policy[each.key].rendered
+
+  lifecycle {
+    prevent_destroy = true
+  }
+}
+
+data "template_file" "apiary_kms_key_policy" {
+  for_each = {
+    for schema in local.schemas_info : "${schema["schema_name"]}" => schema if schema["encryption"] == "aws:kms"
+  }
+
+  template = file("${path.module}/templates/apiary-kms-key-policy.json")
+
+  vars = {
+    admin_roles  = replace(each.value["admin_roles"], ",", "\",\"")
+    client_roles = replace(lookup(each.value, "client_roles", ""), ",", "\",\"")
+  }
+}
+
+resource "aws_kms_alias" "apiary_alias" {
+  for_each = {
+    for schema in local.schemas_info : "${schema["schema_name"]}" => schema if schema["encryption"] == "aws:kms"
+  }
+
+  name          = "alias/${local.instance_alias}/${each.value["schema_name"]}"
+  target_key_id = aws_kms_key.apiary_kms[each.key].key_id
+}
diff --git a/s3.tf b/s3.tf
@@ -12,16 +12,19 @@ data "template_file" "bucket_policy" {
   for_each = {
     for schema in local.schemas_info : "${schema["schema_name"]}" => schema
   }
-  template = "${file("${path.module}/templates/apiary_bucket_policy.json")}"
+  template = "${file("${path.module}/templates/apiary-bucket-policy.json")}"
 
   vars = {
     #if apiary_shared_schemas is empty or contains current schema, allow customer accounts to access this bucket.
     customer_principal = "${length(var.apiary_shared_schemas) == 0 || contains(var.apiary_shared_schemas, each.key) ?
     join("\",\"", formatlist("arn:aws:iam::%s:root", var.apiary_customer_accounts)) : ""}"
 
     bucket_name       = each.value["data_bucket"]
+    encryption        = each.value["encryption"]
+    kms_key_arn       = each.value["encryption"] == "aws:kms" ? aws_kms_key.apiary_kms[each.key].arn : ""
     producer_iamroles = replace(lookup(var.apiary_producer_iamroles, each.key, ""), ",", "\",\"")
     deny_iamroles     = join("\",\"", var.apiary_deny_iamroles)
+    client_roles      = replace(lookup(each.value, "client_roles", ""), ",", "\",\"")
   }
 }
 

diff --git a/templates/apiary_bucket_policy.json → templates/apiary-bucket-policy.json b/templates/apiary_bucket_policy.json → templates/apiary-bucket-policy.json
@@ -94,10 +94,24 @@
             "Resource": "arn:aws:s3:::${bucket_name}/*",
             "Condition": {
                 "StringNotEquals": {
-                    "s3:x-amz-server-side-encryption": "AES256"
+                    "s3:x-amz-server-side-encryption": "${encryption}"
                 }
             }
         },
+%{if kms_key_arn != ""}
+        {
+          "Sid": "DenyWrongKMSKey",
+          "Effect": "Deny",
+          "Principal": "*",
+          "Action": "s3:PutObject",
+          "Resource": "arn:aws:s3:::${bucket_name}/*",
+          "Condition": {
+            "StringNotEquals": {
+              "s3:x-amz-server-side-encryption-aws-kms-key-id": "${kms_key_arn}"
+            }
+          }
+        },
+%{endif}
         {
             "Sid": "DenyUnEncryptedObjectUploads",
             "Effect": "Deny",

diff --git a/templates/apiary-kms-key-policy.json b/templates/apiary-kms-key-policy.json
@@ -0,0 +1,64 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+%{if client_roles != ""}
+    {
+      "Sid": "Allow use of the key",
+      "Effect": "Allow",
+      "Principal": {
+        "AWS": ["${client_roles}"]
+      },
+      "Action": [
+        "kms:Encrypt",
+        "kms:Decrypt",
+        "kms:ReEncrypt*",
+        "kms:GenerateDataKey*",
+        "kms:DescribeKey"
+      ],
+      "Resource": "*"
+    },
+    {
+      "Sid": "Allow attachment of persistent resources",
+      "Effect": "Allow",
+      "Principal": {
+        "AWS": ["${client_roles}"]
+      },
+      "Action": [
+        "kms:CreateGrant",
+        "kms:ListGrants",
+        "kms:RevokeGrant"
+      ],
+      "Resource": "*",
+      "Condition": {
+        "Bool": {
+          "kms:GrantIsForAWSResource": "true"
+        }
+      }
+    },
+%{endif}
+    {
+      "Sid": "Allow access for Key Administrators",
+      "Effect": "Allow",
+      "Principal": {
+        "AWS": [ "${admin_roles}" ]
+      },
+      "Action": [
+        "kms:Create*",
+        "kms:Describe*",
+        "kms:Enable*",
+        "kms:List*",
+        "kms:Put*",
+        "kms:Update*",
+        "kms:Revoke*",
+        "kms:Disable*",
+        "kms:Get*",
+        "kms:Delete*",
+        "kms:TagResource",
+        "kms:UntagResource",
+        "kms:ScheduleKeyDeletion",
+        "kms:CancelKeyDeletion"
+      ],
+      "Resource": "*"
+    }
+  ]
+}