Use nlb_logs_prefixes instead of nlb_logs_prefix (trussworks#45)

alexandervasylev · Feb 11, 2020 · f290672 · f290672
1 parent a0498ef
commit f290672
Show file tree

Hide file tree

Showing 6 changed files with 172 additions and 81 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -2,7 +2,7 @@ version: 2
 jobs:
   validate:
     docker:
-      - image: trussworks/circleci-docker-primary:40076395a6e6a349f92caa92c4de614e105fe672
+      - image: trussworks/circleci-docker-primary:ba270cf66c7a9ae6dbc8e6190b74163ad71729d0
     steps:
       - checkout
       - restore_cache:

diff --git a/.gitignore b/.gitignore
@@ -3,3 +3,4 @@
 terraform.tfstate
 terraform.tfstate.backup
 terraform.tfstate.*.backup
+.envrc.local
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,6 +1,6 @@
 repos:
   - repo: git://github.com/pre-commit/pre-commit-hooks
-    rev: v2.4.0
+    rev: v2.5.0
     hooks:
       - id: check-json
       - id: check-merge-conflict
@@ -12,18 +12,17 @@ repos:
       - id: trailing-whitespace
 
   - repo: git://github.com/igorshubovych/markdownlint-cli
-    rev: v0.21.0
+    rev: v0.22.0
     hooks:
       - id: markdownlint
 
   - repo: git://github.com/antonbabenko/pre-commit-terraform
-    rev: v1.24.0
+    rev: v1.25.0
     hooks:
       - id: terraform_docs
       - id: terraform_fmt
 
   - repo: git://github.com/golangci/golangci-lint
-    rev: v1.23.1
+    rev: v1.23.6
     hooks:
       - id: golangci-lint
-
diff --git a/README.md b/README.md
@@ -1,7 +1,8 @@
+<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 Supports two main uses cases:
 
-1. Creates and configures a single private S3 bucket for storing logs from various AWS services, which are nested as bucket prefixes. Logs will expire after a default of 90 days, with option to configure retention value.
-1. Creates and configures a single private S3 bucket for a single AWS service. Logs will expire after a default of 90 days, with option to configure retention value.
+* Creates and configures a single private S3 bucket for storing logs from various AWS services, which are nested as bucket prefixes. Logs will expire after a default of 90 days, with option to configure retention value.
+* Creates and configures a single private S3 bucket for a single AWS service. Logs will expire after a default of 90 days, with option to configure retention value.
 
 Logging from the following services is supported for both cases:
 
@@ -11,82 +12,87 @@ Logging from the following services is supported for both cases:
 * [RedShift](https://aws.amazon.com/redshift/)
 * [S3](https://aws.amazon.com/s3/)
 
+## Terraform Versions
+
+Terraform 0.12. Pin module version to ~> 5.1.0. Submit pull-requests to master branch.
+
+Terraform 0.11. Pin module version to ~> 3.5.0. Submit pull-requests to terraform011 branch.
+
 ## Usage for a single log bucket storing logs from all services
 
     # Allows all services to log to bucket
-    module "aws_logs" {
+    module "aws\_logs" {
       source         = "trussworks/logs/aws"
-      s3_bucket_name = "my-company-aws-logs"
+      s3\_bucket\_name = "my-company-aws-logs"
       region         = "us-west-2"
     }
 
 ## Usage for a single log bucket storing logs from a single service
 
     #  Allows only the service specified (elb in this case) to log to the bucket
-    module "aws_logs" {
+    module "aws\_logs" {
       source         = "trussworks/logs/aws"
-      s3_bucket_name = "my-company-aws-logs-elb"
+      s3\_bucket\_name = "my-company-aws-logs-elb"
       region         = "us-west-2"
-      default_allow  = false
-      allow_elb      = true
+      default\_allow  = false
+      allow\_elb      = true
     }
 
 ## Usage for a single log bucket storing logs from multiple specified services
 
     #  Allows only the services specified (alb and elb in this case) to log to the bucket
-    module "aws_logs" {
+    module "aws\_logs" {
       source         = "trussworks/logs/aws"
-      s3_bucket_name = "my-company-aws-logs-elb"
+      s3\_bucket\_name = "my-company-aws-logs-elb"
       region         = "us-west-2"
-      default_allow  = false
-      allow_alb      = true
-      allow_elb      = true
+      default\_allow  = false
+      allow\_alb      = true
+      allow\_elb      = true
     }
 
 ## Usage for a private bucket with no policies
 
     #  Allows no services to log to the bucket
-    module "aws_logs" {
+    module "aws\_logs" {
       source         = "trussworks/logs/aws"
-      s3_bucket_name = "my-company-aws-logs-elb"
-      s3_bucket_acl  = "private"
+      s3\_bucket\_name = "my-company-aws-logs-elb"
+      s3\_bucket\_acl  = "private"
       region         = "us-west-2"
-      default_allow  = false
+      default\_allow  = false
     }
 
 ## Usage for a single log bucket storing logs from multiple accounts
 
-    module "aws_logs" {
+    module "aws\_logs" {
       source         = "trussworks/logs/aws"
-      s3_bucket_name = "my-company-aws-logs-elb"
+      s3\_bucket\_name = "my-company-aws-logs-elb"
       region         = "us-west-2"
-      default_allow  = false
-      allow_cloudtrail      = true
-      cloudtrail_accounts = ["${data.aws_caller_identity.current.account_id}", "${aws_organizations_account.example.id}"]
+      default\_allow  = false
+      allow\_cloudtrail      = true
+      cloudtrail\_accounts = ["${data.aws\_caller\_identity.current.account\_id}", "${aws\_organizations\_account.example.id}"]
     }
 
-## Usage for a single log bucket storing logs from multiple application load balancers
-
-    module "aws_logs" {
-      source         = "trussworks/logs/aws"
-      s3_bucket_name = "my-company-aws-logs-alb"
-      region         = "us-west-2"
-      default_allow  = false
-      allow_alb      = true
-      alb_logs_prefixes = formatlist(format("alb/%%s/AWSLogs/%s", data.aws_caller_identity.current.account_id), [
-       "hello-world-prod",
-       "hello-world-staging",
-       "hello-world-experimental",
+## Usage for a single log bucket storing logs from multiple application load balancers and network load balancers
+
+    module "aws\_logs" {
+      source            = "trussworks/logs/aws"
+      s3\_bucket\_name    = "my-company-aws-logs-alb"
+      region            = "us-west-2"
+      default\_allow     = false
+      allow\_alb         = true
+      allow\_nlb         = true
+      alb\_logs\_prefixes = formatlist(format("alb/%%s/AWSLogs/%s", data.aws\_caller\_identity.current.account\_id), [
+       "alb-hello-world-prod",
+       "alb-hello-world-staging",
+       "alb-hello-world-experimental",
+      ])
+     nlb\_logs\_prefixes = formatlist(format("nlb/%%s/AWSLogs/%s", data.aws\_caller\_identity.current.account\_id), [
+       "nlb-hello-world-prod",
+       "nlb-hello-world-staging",
+       "nlb-hello-world-experimental",
       ])
     }
 
-## Terraform Versions
-
-Terraform 0.12. Pin module version to ~> 4.x Submit pull-requests to master branch.
-
-Terraform 0.11. Pin module version to ~> 3.5.0. Submit pull-requests to terraform011 branch.
-
-<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 ## Providers
 
 | Name | Version |
@@ -98,7 +104,7 @@ Terraform 0.11. Pin module version to ~> 3.5.0. Submit pull-requests to terrafor
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:-----:|
-| alb\_logs\_prefixes | S3 key prefixes for ALB logs. | `list(string)` | <pre>[<br>  "alb"<br>]<br></pre> | no |
+| alb\_logs\_prefixes | S3 key prefixes for ALB logs. | `list(string)` | <pre>[<br>  "alb"<br>]</pre> | no |
 | allow\_alb | Allow ALB service to log to bucket. | `string` | `false` | no |
 | allow\_cloudtrail | Allow Cloudtrail service to log to bucket. | `string` | `false` | no |
 | allow\_cloudwatch | Allow Cloudwatch service to export logs to bucket. | `string` | `false` | no |
@@ -116,8 +122,7 @@ Terraform 0.11. Pin module version to ~> 3.5.0. Submit pull-requests to terrafor
 | elb\_accounts | List of accounts for ELB logs.  By default limits to the current account. | `list(string)` | `[]` | no |
 | elb\_logs\_prefix | S3 prefix for ELB logs. | `string` | `"elb"` | no |
 | force\_destroy | A bool that indicates all objects (including any locked objects) should be deleted from the bucket so the bucket can be destroyed without error. | `bool` | `false` | no |
-| nlb\_accounts | List of accounts for NLB logs.  By default limits to the current account. | `list(string)` | `[]` | no |
-| nlb\_logs\_prefix | S3 prefix for NLB logs. | `string` | `"nlb"` | no |
+| nlb\_logs\_prefixes | S3 key prefixes for NLB logs. | `list(string)` | <pre>[<br>  "nlb"<br>]</pre> | no |
 | redshift\_logs\_prefix | S3 prefix for RedShift logs. | `string` | `"redshift"` | no |
 | region | Region where the AWS S3 bucket will be created. | `string` | n/a | yes |
 | s3\_bucket\_acl | Set bucket ACL per [AWS S3 Canned ACL](<https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl>) list. | `string` | `"log-delivery-write"` | no |
@@ -137,6 +142,18 @@ Terraform 0.11. Pin module version to ~> 3.5.0. Submit pull-requests to terrafor
 
 ## Upgrade Paths
 
+### Upgrading from 5.0.0 to 5.1.x
+
+Version 5.1.0 removed the `nlb_logs_prefix` and `nlb_accounts` variables and now uses one `nlb_logs_prefixes` list as input.  If you had not set the `nlb_logs_prefix` or `nlb_accounts` variables, then the default behavior does not change.  If you had set `nlb_logs_prefix`, then simply pass the original value as a 1 item list to `nlb_logs_prefixes` (while watching that path separators are not duplicated).  For example, `nlb_logs_prefixes = ["logs/nlb"]`.
+
+Use the `format` and `formatlist` functions in the caller module to support more complex logging that does limit by account id.  For example:
+
+    nlb_logs_prefixes = formatlist(format("nlb/%%s/AWSLogs/%s", data.aws_caller_identity.current.account_id), [
+      "hello-world-prod",
+      "hello-world-staging",
+      "hello-world-experimental",
+    ])
+
 ### Upgrading from 4.0.0 to 4.1.x
 
 Version 4.1.0 removed the `aws_s3_bucket_policy` resource and now applies the bucket policy directly to the
@@ -193,4 +210,3 @@ locally by running the following command:
 Or with aws-vault:
 
   AWS_VAULT_KEYCHAIN_NAME=YOUR-KEYCHAIN-NAME aws-vault exec YOUR-AWS-PROFILE -- make test
-
diff --git a/main.tf b/main.tf
@@ -1,3 +1,99 @@
+/**
+ * Supports two main uses cases:
+ *
+ * * Creates and configures a single private S3 bucket for storing logs from various AWS services, which are nested as bucket prefixes. Logs will expire after a default of 90 days, with option to configure retention value.
+ * * Creates and configures a single private S3 bucket for a single AWS service. Logs will expire after a default of 90 days, with option to configure retention value.
+ *
+ * Logging from the following services is supported for both cases:
+ *
+ * * [CloudTrail](https://aws.amazon.com/cloudtrail/)
+ * * [Config](https://aws.amazon.com/config/)
+ * * [Classic Load Balancer (ELB) and Application Load Balancer (ALB)](https://aws.amazon.com/elasticloadbalancing/)
+ * * [RedShift](https://aws.amazon.com/redshift/)
+ * * [S3](https://aws.amazon.com/s3/)
+ *
+ * ## Terraform Versions
+ *
+ * Terraform 0.12. Pin module version to ~> 5.1.0. Submit pull-requests to master branch.
+ *
+ * Terraform 0.11. Pin module version to ~> 3.5.0. Submit pull-requests to terraform011 branch.
+ *
+ * ## Usage for a single log bucket storing logs from all services
+ *
+ *     # Allows all services to log to bucket
+ *     module "aws_logs" {
+ *       source         = "trussworks/logs/aws"
+ *       s3_bucket_name = "my-company-aws-logs"
+ *       region         = "us-west-2"
+ *     }
+ *
+ * ## Usage for a single log bucket storing logs from a single service
+ *
+ *     #  Allows only the service specified (elb in this case) to log to the bucket
+ *     module "aws_logs" {
+ *       source         = "trussworks/logs/aws"
+ *       s3_bucket_name = "my-company-aws-logs-elb"
+ *       region         = "us-west-2"
+ *       default_allow  = false
+ *       allow_elb      = true
+ *     }
+ *
+ * ## Usage for a single log bucket storing logs from multiple specified services
+ *
+ *     #  Allows only the services specified (alb and elb in this case) to log to the bucket
+ *     module "aws_logs" {
+ *       source         = "trussworks/logs/aws"
+ *       s3_bucket_name = "my-company-aws-logs-elb"
+ *       region         = "us-west-2"
+ *       default_allow  = false
+ *       allow_alb      = true
+ *       allow_elb      = true
+ *     }
+ *
+ * ## Usage for a private bucket with no policies
+ *
+ *     #  Allows no services to log to the bucket
+ *     module "aws_logs" {
+ *       source         = "trussworks/logs/aws"
+ *       s3_bucket_name = "my-company-aws-logs-elb"
+ *       s3_bucket_acl  = "private"
+ *       region         = "us-west-2"
+ *       default_allow  = false
+ *     }
+ *
+ * ## Usage for a single log bucket storing logs from multiple accounts
+ *
+ *     module "aws_logs" {
+ *       source         = "trussworks/logs/aws"
+ *       s3_bucket_name = "my-company-aws-logs-elb"
+ *       region         = "us-west-2"
+ *       default_allow  = false
+ *       allow_cloudtrail      = true
+ *       cloudtrail_accounts = ["${data.aws_caller_identity.current.account_id}", "${aws_organizations_account.example.id}"]
+ *     }
+ *
+ * ## Usage for a single log bucket storing logs from multiple application load balancers and network load balancers
+ *
+ *     module "aws_logs" {
+ *       source            = "trussworks/logs/aws"
+ *       s3_bucket_name    = "my-company-aws-logs-alb"
+ *       region            = "us-west-2"
+ *       default_allow     = false
+ *       allow_alb         = true
+ *       allow_nlb         = true
+ *       alb_logs_prefixes = formatlist(format("alb/%%s/AWSLogs/%s", data.aws_caller_identity.current.account_id), [
+ *        "alb-hello-world-prod",
+ *        "alb-hello-world-staging",
+ *        "alb-hello-world-experimental",
+ *       ])
+  *      nlb_logs_prefixes = formatlist(format("nlb/%%s/AWSLogs/%s", data.aws_caller_identity.current.account_id), [
+ *        "nlb-hello-world-prod",
+ *        "nlb-hello-world-staging",
+ *        "nlb-hello-world-experimental",
+ *       ])
+ *     }
+ */
+
 # Get the account id of the AWS ELB service account in a given region for the
 # purpose of whitelisting in a S3 bucket policy.
 data "aws_elb_service_account" "main" {
@@ -239,23 +335,10 @@ JSON
       ),
     )
     nlb_effect = var.default_allow || var.allow_nlb ? "Allow" : "Deny"
-    nlb_resources = length(var.nlb_accounts) > 0 ? jsonencode(
-      sort(
-        formatlist(
-          format(
-            "arn:aws:s3:::%s/%s/AWSLogs/%%s/*",
-            var.s3_bucket_name,
-            var.nlb_logs_prefix,
-          ),
-          var.nlb_accounts,
-        ),
-      ),
-      ) : jsonencode(
-      format(
-        "arn:aws:s3:::%s/%s/AWSLogs/%s/*",
-        var.s3_bucket_name,
-        var.nlb_logs_prefix,
-        data.aws_caller_identity.current.account_id,
+    nlb_resources = jsonencode(
+      formatlist(
+        format("arn:aws:s3:::%s/%%s/*", var.s3_bucket_name),
+        var.nlb_logs_prefixes,
       ),
     )
     redshift_effect = var.default_allow || var.allow_redshift ? "Allow" : "Deny"
@@ -321,4 +404,3 @@ resource "aws_s3_bucket_public_access_block" "public_access_block" {
   # Retroactivley block public and cross-account access if bucket has public policies
   restrict_public_buckets = true
 }
-
diff --git a/variables.tf b/variables.tf
@@ -32,12 +32,6 @@ variable "alb_logs_prefixes" {
   type        = list(string)
 }
 
-variable "nlb_logs_prefix" {
-  description = "S3 prefix for NLB logs."
-  default     = "nlb"
-  type        = string
-}
-
 variable "cloudwatch_logs_prefix" {
   description = "S3 prefix for CloudWatch log exports."
   default     = "cloudwatch"
@@ -135,15 +129,14 @@ variable "elb_accounts" {
   type        = list(string)
 }
 
-variable "nlb_accounts" {
-  description = "List of accounts for NLB logs.  By default limits to the current account."
-  default     = []
-  type        = list(string)
-}
-
 variable "force_destroy" {
   description = "A bool that indicates all objects (including any locked objects) should be deleted from the bucket so the bucket can be destroyed without error."
   default     = false
   type        = bool
 }
 
+variable "nlb_logs_prefixes" {
+  description = "S3 key prefixes for NLB logs."
+  default     = ["nlb"]
+  type        = list(string)
+}