feat: allow backing DB choice of mysql or postgres, remove hardcoding…

… of db settings (#55)
SwanseaUniversityMedical · Mar 18, 2024 · 45a6727 · 45a6727
1 parent 457675f
commit 45a6727
Show file tree

Hide file tree

Showing 6 changed files with 51 additions and 10 deletions.
diff --git a/charts/hive/Chart.yaml b/charts/hive/Chart.yaml
@@ -2,6 +2,7 @@ apiVersion: v2
 appVersion: 1.2.0
 dependencies:
 - name: mysql
+  condition: mysql.enabled
   repository: https://charts.bitnami.com/bitnami
   version: 9.16.2
 description: The Apache Hive ™ is a data warehouse software project built on top of

diff --git a/charts/hive/templates/configmap.yaml b/charts/hive/templates/configmap.yaml
@@ -25,19 +25,35 @@ data:
         </property>
         <property>
             <name>javax.jdo.option.ConnectionDriverName</name>
+            {{- if .Values.mysql.enabled }}
             <value>com.mysql.cj.jdbc.Driver</value>
+            {{- else }}
+            <value>org.postgresql.Driver</value>
+            {{- end }}
         </property>
         <property>
             <name>javax.jdo.option.ConnectionURL</name>
-            <value>jdbc:mysql://{{ .Release.Name }}-{{ .Values.metastoreHostname }}:3306/{{ .Values.mysql.auth.database }}</value>
+            {{- if .Values.mysql.enabled }}
+            <value>jdbc:mysql://{{ .Release.Name }}-{{ .Values.metastoreDb.hostname }}:{{ .Values.metastoreDb.port }}/{{ .Values.mysql.auth.database }}</value>
+            {{- else }}
+            <value>jdbc:postgresql://{{ .Values.metastoreDb.hostname }}:{{ .Values.metastoreDb.port }}/{{ .Values.metastoreDb.database }}</value>
+            {{- end }}       
         </property>
         <property>
             <name>javax.jdo.option.ConnectionUserName</name>
+            {{- if .Values.mysql.enabled }}
             <value>{{ .Values.mysql.auth.username }}</value>
+            {{- else }}
+            <value>{{ .Values.metastoreDb.auth.username }}</value>
+            {{- end }}
         </property>
         <property>
             <name>javax.jdo.option.ConnectionPassword</name>
+            {{- if .Values.mysql.enabled }}
             <value>{{ .Values.mysql.auth.password }}</value>
+            {{- else }}
+            <value>{{ .Values.metastoreDb.auth.password }}</value>
+            {{- end }}
         </property>
         <property>
             <name>fs.s3a.access.key</name>

diff --git a/charts/hive/templates/deployment.yaml b/charts/hive/templates/deployment.yaml
@@ -43,8 +43,19 @@ spec:
           {{- end }}
           imagePullPolicy: {{ .Values.image.pullPolicy }}
           env:
+            {{- if .Values.mysql.enabled }}
             - name: METASTORE_DB_HOSTNAME
-              value: {{ .Release.Name }}-{{ .Values.metastoreHostname }}
+              value: {{ .Release.Name }}-{{ .Values.metastoreDb.hostname }}
+            - name: METASTORE_DB_TYPE
+              value: "mysql"
+            {{- else }}
+            - name: METASTORE_DB_HOSTNAME
+              value: {{ .Values.metastoreDb.hostname }}
+            - name: METASTORE_DB_TYPE
+              value: "postgres"
+            {{- end }}
+            - name: METASTORE_DB_PORT
+              value: {{ .Values.metastoreDb.port }}
             {{- with .Values.env }}
             {{- . | toYaml | nindent 12 }}
             {{- end }}

diff --git a/charts/hive/values.yaml b/charts/hive/values.yaml
@@ -36,7 +36,13 @@ service:
   port: 9083
 
 # the release name will be appended to the start of this
-metastoreHostname: "mysql-primary-headless"
+metastoreDb:
+  hostname: "mysql-primary-headless"
+  port: 3306
+  database: metastore_db
+  auth:
+    username: ""
+    password: ""
 
 ## Auto-scaling (hpa) configuration
 # TODO: not implemented
@@ -52,6 +58,7 @@ affinity: {}
 topologySpreadConstraints: []
 
 mysql:
+  enabled: true
   architecture: replication
   auth:
     rootPassword: 'P4ssW0rd!1'

diff --git a/containers/hive/Dockerfile b/containers/hive/Dockerfile
@@ -4,7 +4,7 @@ WORKDIR /opt
 
 ENV HADOOP_VERSION=3.3.6
 ENV METASTORE_VERSION=3.1.3
-ENV AWS_JAVA_BUNDLE_VERSION=1.12.262
+ENV AWS_JAVA_BUNDLE_VERSION=1.12.367
 
 ENV HADOOP_HOME=/opt/hadoop-${HADOOP_VERSION}
 ENV HIVE_HOME=/opt/apache-hive-metastore-${METASTORE_VERSION}-bin
@@ -17,6 +17,10 @@ RUN curl -s -L https://repo1.maven.org/maven2/org/apache/hive/hive-standalone-me
     curl -L https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-j-8.1.0.tar.gz | tar zxf - && \
     mv mysql-connector-j-8.1.0/mysql-connector-j-8.1.0.jar ${HIVE_HOME}/lib/mysql-connector-java.jar
 
+# Postgres driver
+RUN wget -q https://jdbc.postgresql.org/download/postgresql-42.5.4.jar && \
+    mv postgresql-42.5.4.jar ${HIVE_HOME}/lib/
+
 COPY ./containers/hive/entrypoint.sh /entrypoint.sh
 
 RUN addgroup --system hive --gid=1000 && \

diff --git a/containers/hive/entrypoint.sh b/containers/hive/entrypoint.sh
@@ -1,15 +1,17 @@
 #!/bin/sh
 
 export METASTORE_DB_HOSTNAME=${METASTORE_DB_HOSTNAME:-localhost}
+export METASTORE_DB_PORT=${METASTORE_DB_PORT:-3306}
+export METASTORE_DB_TYPE=${METASTORE_DB_TYPE:-mysql}
 
-echo "Waiting for database on ${METASTORE_DB_HOSTNAME} to launch on 3306 ..."
+echo "Waiting for database on ${METASTORE_DB_HOSTNAME} to launch on ${METASTORE_DB_PORT} ..."
 
-while ! nc -z ${METASTORE_DB_HOSTNAME} 3306; do
+while ! nc -z ${METASTORE_DB_HOSTNAME} ${METASTORE_DB_PORT}; do
   sleep 1
 done
 
-echo "Database on ${METASTORE_DB_HOSTNAME}:3306 started"
-echo "Init apache hive metastore on ${METASTORE_DB_HOSTNAME}:3306"
+echo "Database on ${METASTORE_DB_HOSTNAME}:${METASTORE_DB_PORT} started"
+echo "Init apache hive metastore on ${METASTORE_DB_HOSTNAME}:${METASTORE_DB_PORT}"
 
-/opt/apache-hive-metastore-3.1.3-bin/bin/schematool -initSchema -dbType mysql
-/opt/apache-hive-metastore-3.1.3-bin/bin/start-metastore
+/opt/apache-hive-metastore-3.1.3-bin/bin/schematool -initSchema -dbType ${METASTORE_DB_TYPE}
+/opt/apache-hive-metastore-3.1.3-bin/bin/start-metastore