From f290672be0230b0ad81a8cd04b8e927724eaaff7 Mon Sep 17 00:00:00 2001 From: pjdufour-truss Date: Tue, 11 Feb 2020 14:59:55 -0500 Subject: [PATCH] Use nlb_logs_prefixes instead of nlb_logs_prefix (#45) --- .circleci/config.yml | 2 +- .gitignore | 1 + .pre-commit-config.yaml | 9 ++- README.md | 106 +++++++++++++++++++++--------------- main.tf | 118 ++++++++++++++++++++++++++++++++++------ variables.tf | 17 ++---- 6 files changed, 172 insertions(+), 81 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 50d2808..0ba6598 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,7 +2,7 @@ version: 2 jobs: validate: docker: - - image: trussworks/circleci-docker-primary:40076395a6e6a349f92caa92c4de614e105fe672 + - image: trussworks/circleci-docker-primary:ba270cf66c7a9ae6dbc8e6190b74163ad71729d0 steps: - checkout - restore_cache: diff --git a/.gitignore b/.gitignore index c871834..9dd74fc 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ terraform.tfstate terraform.tfstate.backup terraform.tfstate.*.backup +.envrc.local diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 615967f..e881236 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: git://github.com/pre-commit/pre-commit-hooks - rev: v2.4.0 + rev: v2.5.0 hooks: - id: check-json - id: check-merge-conflict @@ -12,18 +12,17 @@ repos: - id: trailing-whitespace - repo: git://github.com/igorshubovych/markdownlint-cli - rev: v0.21.0 + rev: v0.22.0 hooks: - id: markdownlint - repo: git://github.com/antonbabenko/pre-commit-terraform - rev: v1.24.0 + rev: v1.25.0 hooks: - id: terraform_docs - id: terraform_fmt - repo: git://github.com/golangci/golangci-lint - rev: v1.23.1 + rev: v1.23.6 hooks: - id: golangci-lint - diff --git a/README.md b/README.md index b3b2e69..57c5ef2 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,8 @@ + Supports two main uses cases: -1. Creates and configures a single private S3 bucket for storing logs from various AWS services, which are nested as bucket prefixes. Logs will expire after a default of 90 days, with option to configure retention value. -1. Creates and configures a single private S3 bucket for a single AWS service. Logs will expire after a default of 90 days, with option to configure retention value. +* Creates and configures a single private S3 bucket for storing logs from various AWS services, which are nested as bucket prefixes. Logs will expire after a default of 90 days, with option to configure retention value. +* Creates and configures a single private S3 bucket for a single AWS service. Logs will expire after a default of 90 days, with option to configure retention value. Logging from the following services is supported for both cases: @@ -11,82 +12,87 @@ Logging from the following services is supported for both cases: * [RedShift](https://aws.amazon.com/redshift/) * [S3](https://aws.amazon.com/s3/) +## Terraform Versions + +Terraform 0.12. Pin module version to ~> 5.1.0. Submit pull-requests to master branch. + +Terraform 0.11. Pin module version to ~> 3.5.0. Submit pull-requests to terraform011 branch. + ## Usage for a single log bucket storing logs from all services # Allows all services to log to bucket - module "aws_logs" { + module "aws\_logs" { source = "trussworks/logs/aws" - s3_bucket_name = "my-company-aws-logs" + s3\_bucket\_name = "my-company-aws-logs" region = "us-west-2" } ## Usage for a single log bucket storing logs from a single service # Allows only the service specified (elb in this case) to log to the bucket - module "aws_logs" { + module "aws\_logs" { source = "trussworks/logs/aws" - s3_bucket_name = "my-company-aws-logs-elb" + s3\_bucket\_name = "my-company-aws-logs-elb" region = "us-west-2" - default_allow = false - allow_elb = true + default\_allow = false + allow\_elb = true } ## Usage for a single log bucket storing logs from multiple specified services # Allows only the services specified (alb and elb in this case) to log to the bucket - module "aws_logs" { + module "aws\_logs" { source = "trussworks/logs/aws" - s3_bucket_name = "my-company-aws-logs-elb" + s3\_bucket\_name = "my-company-aws-logs-elb" region = "us-west-2" - default_allow = false - allow_alb = true - allow_elb = true + default\_allow = false + allow\_alb = true + allow\_elb = true } ## Usage for a private bucket with no policies # Allows no services to log to the bucket - module "aws_logs" { + module "aws\_logs" { source = "trussworks/logs/aws" - s3_bucket_name = "my-company-aws-logs-elb" - s3_bucket_acl = "private" + s3\_bucket\_name = "my-company-aws-logs-elb" + s3\_bucket\_acl = "private" region = "us-west-2" - default_allow = false + default\_allow = false } ## Usage for a single log bucket storing logs from multiple accounts - module "aws_logs" { + module "aws\_logs" { source = "trussworks/logs/aws" - s3_bucket_name = "my-company-aws-logs-elb" + s3\_bucket\_name = "my-company-aws-logs-elb" region = "us-west-2" - default_allow = false - allow_cloudtrail = true - cloudtrail_accounts = ["${data.aws_caller_identity.current.account_id}", "${aws_organizations_account.example.id}"] + default\_allow = false + allow\_cloudtrail = true + cloudtrail\_accounts = ["${data.aws\_caller\_identity.current.account\_id}", "${aws\_organizations\_account.example.id}"] } -## Usage for a single log bucket storing logs from multiple application load balancers - - module "aws_logs" { - source = "trussworks/logs/aws" - s3_bucket_name = "my-company-aws-logs-alb" - region = "us-west-2" - default_allow = false - allow_alb = true - alb_logs_prefixes = formatlist(format("alb/%%s/AWSLogs/%s", data.aws_caller_identity.current.account_id), [ - "hello-world-prod", - "hello-world-staging", - "hello-world-experimental", +## Usage for a single log bucket storing logs from multiple application load balancers and network load balancers + + module "aws\_logs" { + source = "trussworks/logs/aws" + s3\_bucket\_name = "my-company-aws-logs-alb" + region = "us-west-2" + default\_allow = false + allow\_alb = true + allow\_nlb = true + alb\_logs\_prefixes = formatlist(format("alb/%%s/AWSLogs/%s", data.aws\_caller\_identity.current.account\_id), [ + "alb-hello-world-prod", + "alb-hello-world-staging", + "alb-hello-world-experimental", + ]) + nlb\_logs\_prefixes = formatlist(format("nlb/%%s/AWSLogs/%s", data.aws\_caller\_identity.current.account\_id), [ + "nlb-hello-world-prod", + "nlb-hello-world-staging", + "nlb-hello-world-experimental", ]) } -## Terraform Versions - -Terraform 0.12. Pin module version to ~> 4.x Submit pull-requests to master branch. - -Terraform 0.11. Pin module version to ~> 3.5.0. Submit pull-requests to terraform011 branch. - - ## Providers | Name | Version | @@ -98,7 +104,7 @@ Terraform 0.11. Pin module version to ~> 3.5.0. Submit pull-requests to terrafor | Name | Description | Type | Default | Required | |------|-------------|------|---------|:-----:| -| alb\_logs\_prefixes | S3 key prefixes for ALB logs. | `list(string)` | 
[
  "alb"
]
| no | +| alb\_logs\_prefixes | S3 key prefixes for ALB logs. | `list(string)` | 
[
  "alb"
]
| no | | allow\_alb | Allow ALB service to log to bucket. | `string` | `false` | no | | allow\_cloudtrail | Allow Cloudtrail service to log to bucket. | `string` | `false` | no | | allow\_cloudwatch | Allow Cloudwatch service to export logs to bucket. | `string` | `false` | no | @@ -116,8 +122,7 @@ Terraform 0.11. Pin module version to ~> 3.5.0. Submit pull-requests to terrafor | elb\_accounts | List of accounts for ELB logs. By default limits to the current account. | `list(string)` | `[]` | no | | elb\_logs\_prefix | S3 prefix for ELB logs. | `string` | `"elb"` | no | | force\_destroy | A bool that indicates all objects (including any locked objects) should be deleted from the bucket so the bucket can be destroyed without error. | `bool` | `false` | no | -| nlb\_accounts | List of accounts for NLB logs. By default limits to the current account. | `list(string)` | `[]` | no | -| nlb\_logs\_prefix | S3 prefix for NLB logs. | `string` | `"nlb"` | no | +| nlb\_logs\_prefixes | S3 key prefixes for NLB logs. | `list(string)` | 
[
  "nlb"
]
| no | | redshift\_logs\_prefix | S3 prefix for RedShift logs. | `string` | `"redshift"` | no | | region | Region where the AWS S3 bucket will be created. | `string` | n/a | yes | | s3\_bucket\_acl | Set bucket ACL per [AWS S3 Canned ACL]() list. | `string` | `"log-delivery-write"` | no | @@ -137,6 +142,18 @@ Terraform 0.11. Pin module version to ~> 3.5.0. Submit pull-requests to terrafor ## Upgrade Paths +### Upgrading from 5.0.0 to 5.1.x + +Version 5.1.0 removed the `nlb_logs_prefix` and `nlb_accounts` variables and now uses one `nlb_logs_prefixes` list as input. If you had not set the `nlb_logs_prefix` or `nlb_accounts` variables, then the default behavior does not change. If you had set `nlb_logs_prefix`, then simply pass the original value as a 1 item list to `nlb_logs_prefixes` (while watching that path separators are not duplicated). For example, `nlb_logs_prefixes = ["logs/nlb"]`. + +Use the `format` and `formatlist` functions in the caller module to support more complex logging that does limit by account id. For example: + + nlb_logs_prefixes = formatlist(format("nlb/%%s/AWSLogs/%s", data.aws_caller_identity.current.account_id), [ + "hello-world-prod", + "hello-world-staging", + "hello-world-experimental", + ]) + ### Upgrading from 4.0.0 to 4.1.x Version 4.1.0 removed the `aws_s3_bucket_policy` resource and now applies the bucket policy directly to the @@ -193,4 +210,3 @@ locally by running the following command: Or with aws-vault: AWS_VAULT_KEYCHAIN_NAME=YOUR-KEYCHAIN-NAME aws-vault exec YOUR-AWS-PROFILE -- make test - diff --git a/main.tf b/main.tf index 871191d..bc2dd3b 100644 --- a/main.tf +++ b/main.tf @@ -1,3 +1,99 @@ +/** + * Supports two main uses cases: + * + * * Creates and configures a single private S3 bucket for storing logs from various AWS services, which are nested as bucket prefixes. Logs will expire after a default of 90 days, with option to configure retention value. + * * Creates and configures a single private S3 bucket for a single AWS service. Logs will expire after a default of 90 days, with option to configure retention value. + * + * Logging from the following services is supported for both cases: + * + * * [CloudTrail](https://aws.amazon.com/cloudtrail/) + * * [Config](https://aws.amazon.com/config/) + * * [Classic Load Balancer (ELB) and Application Load Balancer (ALB)](https://aws.amazon.com/elasticloadbalancing/) + * * [RedShift](https://aws.amazon.com/redshift/) + * * [S3](https://aws.amazon.com/s3/) + * + * ## Terraform Versions + * + * Terraform 0.12. Pin module version to ~> 5.1.0. Submit pull-requests to master branch. + * + * Terraform 0.11. Pin module version to ~> 3.5.0. Submit pull-requests to terraform011 branch. + * + * ## Usage for a single log bucket storing logs from all services + * + * # Allows all services to log to bucket + * module "aws_logs" { + * source = "trussworks/logs/aws" + * s3_bucket_name = "my-company-aws-logs" + * region = "us-west-2" + * } + * + * ## Usage for a single log bucket storing logs from a single service + * + * # Allows only the service specified (elb in this case) to log to the bucket + * module "aws_logs" { + * source = "trussworks/logs/aws" + * s3_bucket_name = "my-company-aws-logs-elb" + * region = "us-west-2" + * default_allow = false + * allow_elb = true + * } + * + * ## Usage for a single log bucket storing logs from multiple specified services + * + * # Allows only the services specified (alb and elb in this case) to log to the bucket + * module "aws_logs" { + * source = "trussworks/logs/aws" + * s3_bucket_name = "my-company-aws-logs-elb" + * region = "us-west-2" + * default_allow = false + * allow_alb = true + * allow_elb = true + * } + * + * ## Usage for a private bucket with no policies + * + * # Allows no services to log to the bucket + * module "aws_logs" { + * source = "trussworks/logs/aws" + * s3_bucket_name = "my-company-aws-logs-elb" + * s3_bucket_acl = "private" + * region = "us-west-2" + * default_allow = false + * } + * + * ## Usage for a single log bucket storing logs from multiple accounts + * + * module "aws_logs" { + * source = "trussworks/logs/aws" + * s3_bucket_name = "my-company-aws-logs-elb" + * region = "us-west-2" + * default_allow = false + * allow_cloudtrail = true + * cloudtrail_accounts = ["${data.aws_caller_identity.current.account_id}", "${aws_organizations_account.example.id}"] + * } + * + * ## Usage for a single log bucket storing logs from multiple application load balancers and network load balancers + * + * module "aws_logs" { + * source = "trussworks/logs/aws" + * s3_bucket_name = "my-company-aws-logs-alb" + * region = "us-west-2" + * default_allow = false + * allow_alb = true + * allow_nlb = true + * alb_logs_prefixes = formatlist(format("alb/%%s/AWSLogs/%s", data.aws_caller_identity.current.account_id), [ + * "alb-hello-world-prod", + * "alb-hello-world-staging", + * "alb-hello-world-experimental", + * ]) + * nlb_logs_prefixes = formatlist(format("nlb/%%s/AWSLogs/%s", data.aws_caller_identity.current.account_id), [ + * "nlb-hello-world-prod", + * "nlb-hello-world-staging", + * "nlb-hello-world-experimental", + * ]) + * } + */ + # Get the account id of the AWS ELB service account in a given region for the # purpose of whitelisting in a S3 bucket policy. data "aws_elb_service_account" "main" { @@ -239,23 +335,10 @@ JSON ), ) nlb_effect = var.default_allow || var.allow_nlb ? "Allow" : "Deny" - nlb_resources = length(var.nlb_accounts) > 0 ? jsonencode( - sort( - formatlist( - format( - "arn:aws:s3:::%s/%s/AWSLogs/%%s/*", - var.s3_bucket_name, - var.nlb_logs_prefix, - ), - var.nlb_accounts, - ), - ), - ) : jsonencode( - format( - "arn:aws:s3:::%s/%s/AWSLogs/%s/*", - var.s3_bucket_name, - var.nlb_logs_prefix, - data.aws_caller_identity.current.account_id, + nlb_resources = jsonencode( + formatlist( + format("arn:aws:s3:::%s/%%s/*", var.s3_bucket_name), + var.nlb_logs_prefixes, ), ) redshift_effect = var.default_allow || var.allow_redshift ? "Allow" : "Deny" @@ -321,4 +404,3 @@ resource "aws_s3_bucket_public_access_block" "public_access_block" { # Retroactivley block public and cross-account access if bucket has public policies restrict_public_buckets = true } - diff --git a/variables.tf b/variables.tf index 9d46a77..f27e379 100644 --- a/variables.tf +++ b/variables.tf @@ -32,12 +32,6 @@ variable "alb_logs_prefixes" { type = list(string) } -variable "nlb_logs_prefix" { - description = "S3 prefix for NLB logs." - default = "nlb" - type = string -} - variable "cloudwatch_logs_prefix" { description = "S3 prefix for CloudWatch log exports." default = "cloudwatch" @@ -135,15 +129,14 @@ variable "elb_accounts" { type = list(string) } -variable "nlb_accounts" { - description = "List of accounts for NLB logs. By default limits to the current account." - default = [] - type = list(string) -} - variable "force_destroy" { description = "A bool that indicates all objects (including any locked objects) should be deleted from the bucket so the bucket can be destroyed without error." default = false type = bool } +variable "nlb_logs_prefixes" { + description = "S3 key prefixes for NLB logs." + default = ["nlb"] + type = list(string) +}