From 45a67279e12565bc37b233c0a6f9d4fceffc923f Mon Sep 17 00:00:00 2001 From: Alex Lee <8878798+alee-x@users.noreply.github.com> Date: Mon, 18 Mar 2024 11:31:22 +0000 Subject: [PATCH] feat: allow backing DB choice of mysql or postgres, remove hardcoding of db settings (#55) --- charts/hive/Chart.yaml | 1 + charts/hive/templates/configmap.yaml | 18 +++++++++++++++++- charts/hive/templates/deployment.yaml | 13 ++++++++++++- charts/hive/values.yaml | 9 ++++++++- containers/hive/Dockerfile | 6 +++++- containers/hive/entrypoint.sh | 14 ++++++++------ 6 files changed, 51 insertions(+), 10 deletions(-) diff --git a/charts/hive/Chart.yaml b/charts/hive/Chart.yaml index 441da54..4f7d78a 100644 --- a/charts/hive/Chart.yaml +++ b/charts/hive/Chart.yaml @@ -2,6 +2,7 @@ apiVersion: v2 appVersion: 1.2.0 dependencies: - name: mysql + condition: mysql.enabled repository: https://charts.bitnami.com/bitnami version: 9.16.2 description: The Apache Hive ™ is a data warehouse software project built on top of diff --git a/charts/hive/templates/configmap.yaml b/charts/hive/templates/configmap.yaml index 0706edc..6909e11 100644 --- a/charts/hive/templates/configmap.yaml +++ b/charts/hive/templates/configmap.yaml @@ -25,19 +25,35 @@ data: javax.jdo.option.ConnectionDriverName + {{- if .Values.mysql.enabled }} com.mysql.cj.jdbc.Driver + {{- else }} + org.postgresql.Driver + {{- end }} javax.jdo.option.ConnectionURL - jdbc:mysql://{{ .Release.Name }}-{{ .Values.metastoreHostname }}:3306/{{ .Values.mysql.auth.database }} + {{- if .Values.mysql.enabled }} + jdbc:mysql://{{ .Release.Name }}-{{ .Values.metastoreDb.hostname }}:{{ .Values.metastoreDb.port }}/{{ .Values.mysql.auth.database }} + {{- else }} + jdbc:postgresql://{{ .Values.metastoreDb.hostname }}:{{ .Values.metastoreDb.port }}/{{ .Values.metastoreDb.database }} + {{- end }} javax.jdo.option.ConnectionUserName + {{- if .Values.mysql.enabled }} {{ .Values.mysql.auth.username }} + {{- else }} + {{ .Values.metastoreDb.auth.username }} + {{- end }} javax.jdo.option.ConnectionPassword + {{- if .Values.mysql.enabled }} {{ .Values.mysql.auth.password }} + {{- else }} + {{ .Values.metastoreDb.auth.password }} + {{- end }} fs.s3a.access.key diff --git a/charts/hive/templates/deployment.yaml b/charts/hive/templates/deployment.yaml index ecd9fc7..001d284 100644 --- a/charts/hive/templates/deployment.yaml +++ b/charts/hive/templates/deployment.yaml @@ -43,8 +43,19 @@ spec: {{- end }} imagePullPolicy: {{ .Values.image.pullPolicy }} env: + {{- if .Values.mysql.enabled }} - name: METASTORE_DB_HOSTNAME - value: {{ .Release.Name }}-{{ .Values.metastoreHostname }} + value: {{ .Release.Name }}-{{ .Values.metastoreDb.hostname }} + - name: METASTORE_DB_TYPE + value: "mysql" + {{- else }} + - name: METASTORE_DB_HOSTNAME + value: {{ .Values.metastoreDb.hostname }} + - name: METASTORE_DB_TYPE + value: "postgres" + {{- end }} + - name: METASTORE_DB_PORT + value: {{ .Values.metastoreDb.port }} {{- with .Values.env }} {{- . | toYaml | nindent 12 }} {{- end }} diff --git a/charts/hive/values.yaml b/charts/hive/values.yaml index 587b575..de5fe38 100644 --- a/charts/hive/values.yaml +++ b/charts/hive/values.yaml @@ -36,7 +36,13 @@ service: port: 9083 # the release name will be appended to the start of this -metastoreHostname: "mysql-primary-headless" +metastoreDb: + hostname: "mysql-primary-headless" + port: 3306 + database: metastore_db + auth: + username: "" + password: "" ## Auto-scaling (hpa) configuration # TODO: not implemented @@ -52,6 +58,7 @@ affinity: {} topologySpreadConstraints: [] mysql: + enabled: true architecture: replication auth: rootPassword: 'P4ssW0rd!1' diff --git a/containers/hive/Dockerfile b/containers/hive/Dockerfile index 4eb3c6f..8c9c5b0 100644 --- a/containers/hive/Dockerfile +++ b/containers/hive/Dockerfile @@ -4,7 +4,7 @@ WORKDIR /opt ENV HADOOP_VERSION=3.3.6 ENV METASTORE_VERSION=3.1.3 -ENV AWS_JAVA_BUNDLE_VERSION=1.12.262 +ENV AWS_JAVA_BUNDLE_VERSION=1.12.367 ENV HADOOP_HOME=/opt/hadoop-${HADOOP_VERSION} ENV HIVE_HOME=/opt/apache-hive-metastore-${METASTORE_VERSION}-bin @@ -17,6 +17,10 @@ RUN curl -s -L https://repo1.maven.org/maven2/org/apache/hive/hive-standalone-me curl -L https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-j-8.1.0.tar.gz | tar zxf - && \ mv mysql-connector-j-8.1.0/mysql-connector-j-8.1.0.jar ${HIVE_HOME}/lib/mysql-connector-java.jar +# Postgres driver +RUN wget -q https://jdbc.postgresql.org/download/postgresql-42.5.4.jar && \ + mv postgresql-42.5.4.jar ${HIVE_HOME}/lib/ + COPY ./containers/hive/entrypoint.sh /entrypoint.sh RUN addgroup --system hive --gid=1000 && \ diff --git a/containers/hive/entrypoint.sh b/containers/hive/entrypoint.sh index 1842753..b3d859c 100644 --- a/containers/hive/entrypoint.sh +++ b/containers/hive/entrypoint.sh @@ -1,15 +1,17 @@ #!/bin/sh export METASTORE_DB_HOSTNAME=${METASTORE_DB_HOSTNAME:-localhost} +export METASTORE_DB_PORT=${METASTORE_DB_PORT:-3306} +export METASTORE_DB_TYPE=${METASTORE_DB_TYPE:-mysql} -echo "Waiting for database on ${METASTORE_DB_HOSTNAME} to launch on 3306 ..." +echo "Waiting for database on ${METASTORE_DB_HOSTNAME} to launch on ${METASTORE_DB_PORT} ..." -while ! nc -z ${METASTORE_DB_HOSTNAME} 3306; do +while ! nc -z ${METASTORE_DB_HOSTNAME} ${METASTORE_DB_PORT}; do sleep 1 done -echo "Database on ${METASTORE_DB_HOSTNAME}:3306 started" -echo "Init apache hive metastore on ${METASTORE_DB_HOSTNAME}:3306" +echo "Database on ${METASTORE_DB_HOSTNAME}:${METASTORE_DB_PORT} started" +echo "Init apache hive metastore on ${METASTORE_DB_HOSTNAME}:${METASTORE_DB_PORT}" -/opt/apache-hive-metastore-3.1.3-bin/bin/schematool -initSchema -dbType mysql -/opt/apache-hive-metastore-3.1.3-bin/bin/start-metastore +/opt/apache-hive-metastore-3.1.3-bin/bin/schematool -initSchema -dbType ${METASTORE_DB_TYPE} +/opt/apache-hive-metastore-3.1.3-bin/bin/start-metastore \ No newline at end of file