Set size of HMS thread pool based on instance memory. (#157)

* Set size of hms thread pool * cleanup locals * fix copy/paste error Co-authored-by: Scott Barnhart <[email protected]>
ExpediaGroup · May 11, 2020 · 8e67c7a · 8e67c7a
1 parent 4a0511c
commit 8e67c7a
Show file tree

Hide file tree

Showing 7 changed files with 51 additions and 9 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -3,6 +3,11 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [6.1.3] - 2020-05-11
+### Changed
+- Set min/max size of HMS thread pool based on memory.  Max will be set to 1 connection for every 2MB RAM.  Min will be 0.25% of max.  This will prevent large HMS instances from not having enough threads/connections available.
+
+
 ## [6.1.2] - 2020-05-07
 ### Changed
 - Change type of `apiary_managed_schemas` from `list(any)` to `list(map(string))` to support dynamically-generated schema lists. This type is backward-compatible with previous schema lists.  Schema lists were already lists of maps of strings, but this change makes TF 0.12 work in certain circumstances that were causing a fatal TF error.

diff --git a/common.tf b/common.tf
@@ -32,6 +32,16 @@ locals {
   enable_apiary_s3_log_management      = var.apiary_log_bucket == "" ? true : false
   apiary_s3_logs_bucket                = local.enable_apiary_s3_log_management ? "${local.apiary_bucket_prefix}-s3-logs" : ""
   apiary_s3_hive_logs_bucket           = local.enable_apiary_s3_log_management ? "${local.apiary_s3_logs_bucket}-hive" : ""
+
+  hms_ro_heapsize                      = ceil((var.hms_ro_heapsize * 85) / 100)
+  hms_ro_minthreads                    = max(25,  ceil((var.hms_ro_heapsize * 12.5) / 100))
+  hms_ro_maxthreads                    = max(100, ceil((var.hms_ro_heapsize * 50)   / 100))
+
+  hms_rw_heapsize                      = ceil((var.hms_rw_heapsize * 85) / 100)
+  hms_rw_minthreads                    = max(25,  ceil((var.hms_rw_heapsize * 12.5) / 100))
+  hms_rw_maxthreads                    = max(100, ceil((var.hms_rw_heapsize * 50)   / 100))
+
+  hms_alias                            = var.instance_name == "" ? "hms" : "hms-${var.instance_name}"
 }
 
 data "aws_iam_account_alias" "current" {}

diff --git a/k8s-readonly.tf b/k8s-readonly.tf
@@ -4,11 +4,6 @@
  * Licensed under the Apache License, Version 2.0 (the "License");
  */
 
-locals {
-  hms_ro_heapsize = ceil((var.hms_ro_heapsize * 85) / 100)
-  hms_alias       = var.instance_name == "" ? "hms" : "hms-${var.instance_name}"
-}
-
 resource "kubernetes_deployment" "apiary_hms_readonly" {
   count = "${var.hms_instance_type == "k8s" ? 1 : 0}"
   metadata {
@@ -120,6 +115,14 @@ resource "kubernetes_deployment" "apiary_hms_readonly" {
             name  = "LDAP_SECRET_ARN"
             value = "${var.ldap_url == "" ? "" : join("", data.aws_secretsmanager_secret.ldap_user.*.arn)}"
           }
+          env {
+            name  = "HMS_MIN_THREADS"
+            value = local.hms_ro_minthreads
+          }
+          env {
+            name  = "HMS_MAX_THREADS"
+            value = local.hms_ro_maxthreads
+          }
 
           resources {
             limits {

diff --git a/k8s-readwrite.tf b/k8s-readwrite.tf
@@ -4,10 +4,6 @@
  * Licensed under the Apache License, Version 2.0 (the "License");
  */
 
-locals {
-  hms_rw_heapsize = ceil((var.hms_rw_heapsize * 85) / 100)
-}
-
 resource "kubernetes_deployment" "apiary_hms_readwrite" {
   count = "${var.hms_instance_type == "k8s" ? 1 : 0}"
   metadata {
@@ -152,6 +148,14 @@ resource "kubernetes_deployment" "apiary_hms_readwrite" {
             name  = "ENABLE_S3_LOGS"
             value = local.enable_apiary_s3_log_management ? "1" : ""
           }
+          env {
+            name  = "HMS_MIN_THREADS"
+            value = local.hms_rw_minthreads
+          }
+          env {
+            name  = "HMS_MAX_THREADS"
+            value = local.hms_rw_maxthreads
+          }
 
           resources {
             limits {

diff --git a/templates.tf b/templates.tf
@@ -13,6 +13,8 @@ data "template_file" "hms_readwrite" {
     mysql_secret_arn           = "${data.aws_secretsmanager_secret.db_rw_user.arn}"
     hive_metastore_access_mode = "readwrite"
     hms_heapsize               = "${var.hms_rw_heapsize}"
+    hms_minthreads             = local.hms_ro_minthreads
+    hms_maxthreads             = local.hms_ro_maxthreads
     hms_docker_image           = "${var.hms_docker_image}"
     hms_docker_version         = "${var.hms_docker_version}"
     region                     = "${var.aws_region}"
@@ -62,6 +64,8 @@ data "template_file" "hms_readonly" {
     mysql_secret_arn           = "${data.aws_secretsmanager_secret.db_ro_user.arn}"
     hive_metastore_access_mode = "readonly"
     hms_heapsize               = "${var.hms_ro_heapsize}"
+    hms_minthreads             = local.hms_rw_minthreads
+    hms_maxthreads             = local.hms_rw_maxthreads
     hms_docker_image           = "${var.hms_docker_image}"
     hms_docker_version         = "${var.hms_docker_version}"
     region                     = "${var.aws_region}"

diff --git a/templates/apiary-hms-readonly.json b/templates/apiary-hms-readonly.json
@@ -101,6 +101,14 @@
       {
         "name": "ENABLE_METRICS",
         "value": "${enable_metrics}"
+      },
+      {
+        "name": "HMS_MIN_THREADS",
+        "value": "${hms_minthreads}"
+      },
+      {
+        "name": "HMS_MAX_THREADS",
+        "value": "${hms_maxthreads}"
       }
     ]
   }

diff --git a/templates/apiary-hms-readwrite.json b/templates/apiary-hms-readwrite.json
@@ -145,6 +145,14 @@
      {
        "name": "ENABLE_S3_LOGS",
        "value": "${s3_enable_logs}"
+     },
+     {
+       "name": "HMS_MIN_THREADS",
+       "value": "${hms_minthreads}"
+     },
+     {
+       "name": "HMS_MAX_THREADS",
+       "value": "${hms_maxthreads}"
      }
     ]
   }