Add S3 versioning to managed schemas (#291)

* Adding versioning * add s3 * fix status * test 3 * fix tf bug * rename variables * add changelog * Update VARIABLES.md * Remove max versions * fix
ExpediaGroup · Jan 7, 2025 · e07d06c · e07d06c
1 parent f73c42a
commit e07d06c
Show file tree

Hide file tree

Showing 5 changed files with 44 additions and 3 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [7.8.0] - 2024-12-12
+### Added
+- Hive databases backed by S3 can now have versioning enabled.
+
 ## [7.7.0] - 2024-11-19
 ### Changed
 - Updated the hms namespaces for metrics for both readwrite and readonly.

diff --git a/README.md b/README.md
@@ -57,7 +57,12 @@ module "apiary" {
         admin_roles = "role1_arn,role2_arn" //kms key management will be restricted to these roles.
         client_roles = "role3_arn,role4_arn" //s3 bucket read/write and kms key usage will be restricted to these roles.
         customer_accounts = "account_id1,account_id2" //this will override module level apiary_customer_accounts
-    }
+    },
+    {
+        schema_name = "db_s3_versioning_enabled",
+        s3_versioning_enabled = "Enabled", // Enabled/Disabled/Suspended. Once enabled it can only be suspended
+        s3_versioning_expiration_days = 2  // If Enabled, default 7
+    },
   ]
   apiary_customer_accounts = ["aws_account_no_1", "aws_account_no_2"]
   # single policy with multiple conditions will use AND operator

diff --git a/VARIABLES.md b/VARIABLES.md
@@ -141,7 +141,7 @@
 | hms\_ecs\_metrics\_readonly\_namespace                                  | ECS readwrite metrics namespace                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            | `string`                 | `hmsreadonlylegacy`                                                                                                                                                                                                                                                                                 |    no    |
 | hms\_ecs\_metrics\_readwrite\_namespace                                 | ECS readonly metrics namespace                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            | `string`                 | `hmsreadwritelegacy`                                                                                                                                                                                                                                                                                |    no    |
 | hms\_k8s\_metrics\_readonly\_namespace                                  | K8s readwrite metrics namespace                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            | `string`                 | `hms_readonly`                                                                                                                                                                                                                                                                                 |    no    |
-| hms\_k8s\_metrics\_readwrite\_namespace                                 | K8s readonly metrics namespace                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            | `string`                 | `hms_readwrite`                                                                                                                                                                                                                                                                                |    no    |
+| s3\_versioning\_expiration\_days                                 | Number of days (TTL) before objects are expired. Bucket need to have versioning enabled.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            | `number`                 | `7`                                                                                                                                                                                                                                                                                |    no    |
 
 ### apiary_assume_roles
 
@@ -367,4 +367,4 @@ apiary_managed_schemas = [
    producer_roles = "arn:aws:iam::000000000:role/role-1,arn:aws:iam::000000000:role/role-2"
   }
 ]
-```
+```
diff --git a/s3.tf b/s3.tf
@@ -74,6 +74,32 @@ resource "aws_s3_bucket" "apiary_data_bucket" {
   }
 }
 
+resource "aws_s3_bucket_versioning" "apiary_data_bucket_versioning" {
+  for_each = {
+    for schema in local.schemas_info : "${schema["schema_name"]}" => schema
+  }
+  bucket = each.value["data_bucket"]
+  versioning_configuration {
+    status = lookup(each.value, "s3_versioning_enabled", "Disabled")
+  }
+}
+
+resource "aws_s3_bucket_lifecycle_configuration" "apiary_data_bucket_versioning_lifecycle" {
+  for_each = {
+    for schema in local.schemas_info : "${schema["schema_name"]}" => schema
+  }
+  bucket = each.value["data_bucket"]
+  # Rule enabled when expiration max days is set
+  rule {
+    id     = "expire-noncurrent-versions-days"
+    status = lookup(each.value, "s3_versioning_enabled", "") != "" ? "Enabled" : "Disabled"
+
+    noncurrent_version_expiration {
+      noncurrent_days = tonumber(lookup(each.value, "s3_versioning_expiration_days", var.s3_versioning_expiration_days))
+    }
+  }
+} 
+
 resource "aws_s3_bucket_inventory" "apiary_bucket" {
   for_each = var.s3_enable_inventory == true ? {
     for schema in local.schemas_info : "${schema["schema_name"]}" => schema

diff --git a/variables.tf b/variables.tf
@@ -1095,6 +1095,12 @@ variable "ecs_requires_compatibilities" {
   default     = ["EC2", "FARGATE"]
 }
 
+variable "s3_versioning_expiration_days" {
+  description = "Number of days (TTL) before objects are expired. Bucket need to have versioning enabled."
+  type        = number
+  default     = 7
+}
+
 variable "hms_ro_tolerations" {
   description = <<EOF
   Adds a list of tolerations for the HMS readonly pods. For example if you