From 07883e42d8aa31214601f69c8aa62132b7875e7b Mon Sep 17 00:00:00 2001 From: Cristhian Garcia Date: Wed, 11 Sep 2024 09:50:12 -0500 Subject: [PATCH] fix: add readiness probe for lms/cms (#133) (#136) * fix: add readiness probe for lms/cms * fix: add readiness probe for lms/cms * fix: add affinity to spread lms/cms to multiple nodes * chore: remove readiness probe * fix: reduce startup probe period seconds * fix: gracefully kill uwsgi workers * fix: disable local file loggers * fix: disable logging * fix: reduce max unavailable to 0 * fix: add liveness probe for cms and lms * fix: fail early on tracking logger removal * chore: remove rolling update options * fix: restore preStop hook * fix: use right host for cms livenessProbe * fix: use lms/cms host only * chore: restore prestopHook (cherry picked from commit f970e0102f36c3064201ff7be317d1a908557a87) --- drydock/patches/openedx-common-settings | 3 ++ drydock/patches/uwsgi-config | 2 ++ .../templates/drydock/k8s/lifecycle/cms.yml | 31 ++++++++++++++++--- .../templates/drydock/k8s/lifecycle/lms.yml | 31 ++++++++++++++++--- 4 files changed, 57 insertions(+), 10 deletions(-) diff --git a/drydock/patches/openedx-common-settings b/drydock/patches/openedx-common-settings index 4a5d7462..480e11fd 100644 --- a/drydock/patches/openedx-common-settings +++ b/drydock/patches/openedx-common-settings @@ -18,3 +18,6 @@ XBLOCK_SETTINGS["ScormXBlock"] = { "STORAGE_FUNC": scorm_xblock_storage, } {% endif %} + +LOGGING["loggers"].pop("tracking") +LOGGING["loggers"][""]["handlers"] = ["console"] diff --git a/drydock/patches/uwsgi-config b/drydock/patches/uwsgi-config index 673d0acb..a449ca2f 100644 --- a/drydock/patches/uwsgi-config +++ b/drydock/patches/uwsgi-config @@ -12,3 +12,5 @@ no-defer-accept = true master = true py-call-osafterfork = true vacuum = true +hook-master-start = unix_signal:1 gracefully_kill_them_all +disable-logging = true diff --git a/drydock/templates/drydock/k8s/lifecycle/cms.yml b/drydock/templates/drydock/k8s/lifecycle/cms.yml index ebd6ac75..ee85f192 100644 --- a/drydock/templates/drydock/k8s/lifecycle/cms.yml +++ b/drydock/templates/drydock/k8s/lifecycle/cms.yml @@ -12,16 +12,37 @@ spec: exec: command: - "sleep" - - "30" + - "15" startupProbe: httpGet: httpHeaders: - name: Host - value: {{ CMS_HOST }} + value: cms path: /heartbeat port: 8000 - initialDelaySeconds: 5 + initialDelaySeconds: 1 timeoutSeconds: 3 - periodSeconds: 5 - failureThreshold: 5 + periodSeconds: 1 + failureThreshold: 30 + livenessProbe: + httpGet: + httpHeaders: + - name: Host + value: lms + path: /heartbeat + port: 8000 + initialDelaySeconds: 3 + timeoutSeconds: 30 + periodSeconds: 60 + failureThreshold: 2 + successThreshold: 1 terminationGracePeriodSeconds: 60 + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchLabels: + app.kubernetes.io/name: cms + topologyKey: kubernetes.io/hostname diff --git a/drydock/templates/drydock/k8s/lifecycle/lms.yml b/drydock/templates/drydock/k8s/lifecycle/lms.yml index fc12f805..85895e7c 100644 --- a/drydock/templates/drydock/k8s/lifecycle/lms.yml +++ b/drydock/templates/drydock/k8s/lifecycle/lms.yml @@ -12,16 +12,37 @@ spec: exec: command: - "sleep" - - "30" + - "15" startupProbe: httpGet: httpHeaders: - name: Host - value: {{ LMS_HOST }} + value: lms path: /heartbeat port: 8000 - initialDelaySeconds: 5 + initialDelaySeconds: 1 timeoutSeconds: 3 - periodSeconds: 5 - failureThreshold: 5 + periodSeconds: 1 + failureThreshold: 30 + livenessProbe: + httpGet: + httpHeaders: + - name: Host + value: lms + path: /heartbeat + port: 8000 + initialDelaySeconds: 3 + timeoutSeconds: 30 + periodSeconds: 60 + failureThreshold: 2 + successThreshold: 1 terminationGracePeriodSeconds: 60 + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchLabels: + app.kubernetes.io/name: lms + topologyKey: kubernetes.io/hostname