diff --git a/Makefile b/Makefile index a5b6e2bc5..12a9f3710 100644 --- a/Makefile +++ b/Makefile @@ -237,7 +237,9 @@ versioned-images := php-8.1-fpm \ ruby-3.1 \ ruby-3.2 \ ruby-3.3 \ - opensearch-2 + opensearch-2 \ + mysql-8.0 \ + mysql-8.4 # default-versioned-images are images that formerly had no versioning, and are made backwards-compatible. # the below versions are the ones that map to the unversioned namespace @@ -319,6 +321,7 @@ build/mariadb-10.11-drupal: build/mariadb-10.11 build/ruby-3.1 build/ruby-3.2 build/ruby-3.3: build/commons build/opensearch-2: build/commons build/mongo-4: build/commons +build/mysql-8.0 build/mysql-8.4: build/commons ####### ####### Building Images diff --git a/helpers/TESTING_service_images_dockercompose.md b/helpers/TESTING_service_images_dockercompose.md index c0fd6d1bc..d579d44b0 100644 --- a/helpers/TESTING_service_images_dockercompose.md +++ b/helpers/TESTING_service_images_dockercompose.md @@ -26,6 +26,8 @@ docker run --rm --net all-images_default jwilder/dockerize dockerize -wait tcp:/ docker run --rm --net all-images_default jwilder/dockerize dockerize -wait tcp://mariadb-10-5:3306 -timeout 1m docker run --rm --net all-images_default jwilder/dockerize dockerize -wait tcp://mariadb-10-6:3306 -timeout 1m docker run --rm --net all-images_default jwilder/dockerize dockerize -wait tcp://mariadb-10-11:3306 -timeout 1m +docker run --rm --net all-images_default jwilder/dockerize dockerize -wait tcp://mysql-8-0:3306 -timeout 1m +docker run --rm --net all-images_default jwilder/dockerize dockerize -wait tcp://mysql-8-4:3306 -timeout 1m docker run --rm --net all-images_default jwilder/dockerize dockerize -wait tcp://postgres-11:5432 -timeout 1m docker run --rm --net all-images_default jwilder/dockerize dockerize -wait tcp://postgres-12:5432 -timeout 1m docker run --rm --net all-images_default jwilder/dockerize dockerize -wait tcp://postgres-13:5432 -timeout 1m @@ -53,6 +55,8 @@ docker ps --filter label=com.docker.compose.project=all-images | grep Up | grep docker ps --filter label=com.docker.compose.project=all-images | grep Up | grep mariadb-10-5 docker ps --filter label=com.docker.compose.project=all-images | grep Up | grep mariadb-10-6 docker ps --filter label=com.docker.compose.project=all-images | grep Up | grep mariadb-10-11 +docker ps --filter label=com.docker.compose.project=all-images | grep Up | grep mysql-8-0 +docker ps --filter label=com.docker.compose.project=all-images | grep Up | grep mysql-8-4 docker ps --filter label=com.docker.compose.project=all-images | grep Up | grep mongo-4 docker ps --filter label=com.docker.compose.project=all-images | grep Up | grep postgres-11 docker ps --filter label=com.docker.compose.project=all-images | grep Up | grep postgres-12 @@ -222,6 +226,32 @@ docker compose exec -T mongo-4 sh -c "mongo < /tmp/dbstats.sql" | grep "db" | gr docker compose exec -T commons sh -c "curl -kL http://internal-services-test:3000/mongo?service=mongo-4" | grep "SERVICE_HOST=" docker compose exec -T commons sh -c "curl -kL http://internal-services-test:3000/mongo?service=mongo-4" | grep "LAGOON_TEST_VAR=all" +# mysql-8-0 should be version 8.0 client +docker compose exec -T mysql-8-0 sh -c "mysql -V" | grep "8.0" + +# mysql-8-0 should be version 8.0 server +docker compose exec -T mysql-8-0 sh -c "mysql -e 'SHOW variables;'" | grep "version" | grep "8.0" + +# mysql-8-0 should use default credentials +docker compose exec -T mysql-8-0 sh -c "mysql -D lagoon -u lagoon --password=lagoon -e 'SHOW databases;'" | grep lagoon + +# mysql-8-0 should be able to read/write data +docker compose exec -T commons sh -c "curl -kL http://internal-services-test:3000/mariadb?service=mysql-8-0" | grep "SERVICE_HOST=8.0" +docker compose exec -T commons sh -c "curl -kL http://internal-services-test:3000/mariadb?service=mysql-8-0" | grep "LAGOON_TEST_VAR=all-images" + +# mysql-8-4 should be version 8.4 client +docker compose exec -T mysql-8-4 sh -c "mysql -V" | grep "8.4" + +# mysql-8-4 should be version 8.4 server +docker compose exec -T mysql-8-4 sh -c "mysql -e 'SHOW variables;'" | grep "version" | grep "8.4" + +# mysql-8-4 should use default credentials +docker compose exec -T mysql-8-4 sh -c "mysql -D lagoon -u lagoon --password=lagoon -e 'SHOW databases;'" | grep lagoon + +# mysql-8-4 should be able to read/write data +docker compose exec -T commons sh -c "curl -kL http://internal-services-test:3000/mariadb?service=mysql-8-4" | grep "SERVICE_HOST=8.4" +docker compose exec -T commons sh -c "curl -kL http://internal-services-test:3000/mariadb?service=mysql-8-4" | grep "LAGOON_TEST_VAR=all-images" + # postgres-11 should be version 11 client docker compose exec -T postgres-11 bash -c "psql --version" | grep "psql" | grep "11." diff --git a/helpers/services-docker-compose.yml b/helpers/services-docker-compose.yml index 8b7c1bf7c..7e0ecb564 100644 --- a/helpers/services-docker-compose.yml +++ b/helpers/services-docker-compose.yml @@ -46,6 +46,18 @@ services: - "28017" << : *default-user # uses the defined user from top + mysql-8-0: + image: uselagoon/mysql-8.0:latest + ports: + - "3306" + << : *default-user # uses the defined user from top + + mysql-8-4: + image: uselagoon/mysql-8.4:latest + ports: + - "3306" + << : *default-user # uses the defined user from top + postgres-11: image: uselagoon/postgres-11:latest ports: diff --git a/images/mysql/8.0.Dockerfile b/images/mysql/8.0.Dockerfile new file mode 100644 index 000000000..b488499eb --- /dev/null +++ b/images/mysql/8.0.Dockerfile @@ -0,0 +1,70 @@ +ARG IMAGE_REPO +FROM ${IMAGE_REPO:-lagoon}/commons as commons +FROM mysql:8.0.37-oracle + +LABEL org.opencontainers.image.authors="The Lagoon Authors" maintainer="The Lagoon Authors" +LABEL org.opencontainers.image.source="https://github.com/uselagoon/lagoon-images" repository="https://github.com/uselagoon/lagoon-images" + +ARG LAGOON_VERSION +ENV LAGOON_VERSION=$LAGOON_VERSION + +# Copy commons files +COPY --from=commons /lagoon /lagoon +COPY --from=commons /bin/fix-permissions /bin/ep /bin/docker-sleep /bin/wait-for /bin/ +COPY --from=commons /home /home + +RUN fix-permissions /etc/passwd \ + && mkdir -p /home + +ENV TMPDIR=/tmp \ + TMP=/tmp \ + HOME=/home \ + # When Bash is invoked via `sh` it behaves like the old Bourne Shell and sources a file that is given in `ENV` + ENV=/home/.bashrc \ + # When Bash is invoked as non-interactive (like `bash -c command`) it sources a file that is given in `BASH_ENV` + BASH_ENV=/home/.bashrc + +ENV BACKUPS_DIR="/var/lib/mysql/backup" + +ENV MYSQL_DATABASE=lagoon \ + MYSQL_USER=lagoon \ + MYSQL_PASSWORD=lagoon \ + MYSQL_ROOT_PASSWORD=Lag00n + +RUN microdnf install -y epel-release \ + && microdnf update -y \ + && microdnf install -y \ + gettext \ + net-tools \ + pwgen \ + tini \ + wget; \ + rm -rf /var/lib/mysql/* /etc/mysql/ /etc/my.cnf*; \ + curl -sSL https://raw.githubusercontent.com/major/MySQLTuner-perl/master/mysqltuner.pl -o mysqltuner.pl + +COPY entrypoints/ /lagoon/entrypoints/ +COPY mysql-backup.sh /lagoon/ +COPY my.cnf /etc/mysql/my.cnf + +RUN for i in /var/run/mysqld /run/mysqld /var/lib/mysql /etc/mysql/conf.d /docker-entrypoint-initdb.d /home; \ + do mkdir -p $i; chown mysql $i; /bin/fix-permissions $i; \ + done + +COPY root/usr/share/container-scripts/mysql/readiness-probe.sh /usr/share/container-scripts/mysql/readiness-probe.sh +RUN /bin/fix-permissions /usr/share/container-scripts/mysql/ \ + && /bin/fix-permissions /etc/mysql + +RUN touch /var/log/mariadb-slow.log && /bin/fix-permissions /var/log/mariadb-slow.log \ + && touch /var/log/mariadb-queries.log && /bin/fix-permissions /var/log/mariadb-queries.log + +# We cannot start mysql as root, we add the user mysql to the group root and +# change the user of the Docker Image to this user. +RUN usermod -a -G root mysql +USER mysql +ENV USER_NAME mysql + +WORKDIR /var/lib/mysql +EXPOSE 3306 + +ENTRYPOINT ["/usr/bin/tini", "--", "/lagoon/entrypoints.bash"] +CMD ["mysqld"] diff --git a/images/mysql/8.4.Dockerfile b/images/mysql/8.4.Dockerfile new file mode 100644 index 000000000..6c44eb92e --- /dev/null +++ b/images/mysql/8.4.Dockerfile @@ -0,0 +1,70 @@ +ARG IMAGE_REPO +FROM ${IMAGE_REPO:-lagoon}/commons as commons +FROM mysql:8.4.0-oracle + +LABEL org.opencontainers.image.authors="The Lagoon Authors" maintainer="The Lagoon Authors" +LABEL org.opencontainers.image.source="https://github.com/uselagoon/lagoon-images" repository="https://github.com/uselagoon/lagoon-images" + +ARG LAGOON_VERSION +ENV LAGOON_VERSION=$LAGOON_VERSION + +# Copy commons files +COPY --from=commons /lagoon /lagoon +COPY --from=commons /bin/fix-permissions /bin/ep /bin/docker-sleep /bin/wait-for /bin/ +COPY --from=commons /home /home + +RUN fix-permissions /etc/passwd \ + && mkdir -p /home + +ENV TMPDIR=/tmp \ + TMP=/tmp \ + HOME=/home \ + # When Bash is invoked via `sh` it behaves like the old Bourne Shell and sources a file that is given in `ENV` + ENV=/home/.bashrc \ + # When Bash is invoked as non-interactive (like `bash -c command`) it sources a file that is given in `BASH_ENV` + BASH_ENV=/home/.bashrc + +ENV BACKUPS_DIR="/var/lib/mysql/backup" + +ENV MYSQL_DATABASE=lagoon \ + MYSQL_USER=lagoon \ + MYSQL_PASSWORD=lagoon \ + MYSQL_ROOT_PASSWORD=Lag00n + +RUN microdnf install -y epel-release \ + && microdnf update -y \ + && microdnf install -y \ + gettext \ + net-tools \ + pwgen \ + tini \ + wget; \ + rm -rf /var/lib/mysql/* /etc/mysql/ /etc/my.cnf*; \ + curl -sSL https://raw.githubusercontent.com/major/MySQLTuner-perl/master/mysqltuner.pl -o mysqltuner.pl + +COPY entrypoints/ /lagoon/entrypoints/ +COPY mysql-backup.sh /lagoon/ +COPY my.cnf /etc/mysql/my.cnf + +RUN for i in /var/run/mysqld /run/mysqld /var/lib/mysql /etc/mysql/conf.d /docker-entrypoint-initdb.d /home; \ + do mkdir -p $i; chown mysql $i; /bin/fix-permissions $i; \ + done + +COPY root/usr/share/container-scripts/mysql/readiness-probe.sh /usr/share/container-scripts/mysql/readiness-probe.sh +RUN /bin/fix-permissions /usr/share/container-scripts/mysql/ \ + && /bin/fix-permissions /etc/mysql + +RUN touch /var/log/mariadb-slow.log && /bin/fix-permissions /var/log/mariadb-slow.log \ + && touch /var/log/mariadb-queries.log && /bin/fix-permissions /var/log/mariadb-queries.log + +# We cannot start mysql as root, we add the user mysql to the group root and +# change the user of the Docker Image to this user. +RUN usermod -a -G root mysql +USER mysql +ENV USER_NAME mysql + +WORKDIR /var/lib/mysql +EXPOSE 3306 + +ENTRYPOINT ["/usr/bin/tini", "--", "/lagoon/entrypoints.bash"] +CMD ["mysqld"] diff --git a/images/mysql/entrypoints/100-myql-logging.bash b/images/mysql/entrypoints/100-myql-logging.bash new file mode 100755 index 000000000..52ecdbae7 --- /dev/null +++ b/images/mysql/entrypoints/100-myql-logging.bash @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +set -eo pipefail + +if [ -n "$MYSQL_LOG_SLOW" ]; then + echo "MYSQL_LOG_SLOW set, logging to /var/log/mysql-slow.log" + cat < /etc/mysql/conf.d/log-slow.cnf +[mysqld] +log_output=file +slow_query_log = 1 +slow_query_log_file = /var/log/mysql-slow.log +long_query_time = ${MYSQL_LONG_QUERY_TIME:-10} +log_slow_rate_limit = ${MYSQL_LOG_SLOW_RATE_LIMIT:-1} +EOF +fi + + +if [ -n "$MYSQL_LOG_QUERIES" ]; then + echo "MYSQL_LOG_QUERIES set, logging to /var/log/mysql-queries.log" + cat < /etc/mysql/conf.d/log-queries.cnf + +[mysqld] +general-log +log-output=file +general-log-file=/var/log/mysql-queries.log +EOF +fi diff --git a/images/mysql/entrypoints/150-mysql-performance.bash b/images/mysql/entrypoints/150-mysql-performance.bash new file mode 100755 index 000000000..44b549210 --- /dev/null +++ b/images/mysql/entrypoints/150-mysql-performance.bash @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +set -eo pipefail + +if [ "$LAGOON_ENVIRONMENT_TYPE" == "production" ]; then + # only set if not already defined + if [ -z ${MYSQL_INNODB_BUFFER_POOL_SIZE+x} ]; then + export MYSQL_INNODB_BUFFER_POOL_SIZE=1024M + fi + if [ -z ${MYSQL_INNODB_LOG_FILE_SIZE+x} ]; then + export MYSQL_INNODB_LOG_FILE_SIZE=256M + fi +fi + +if [ -n "$MYSQL_PERFORMANCE_SCHEMA" ]; then + echo "Enabling performance schema" + cat < /etc/mysql/conf.d/performance-schema.cnf +[mysqld] +performance_schema=ON +performance-schema-instrument='stage/%=ON' +performance-schema-consumer-events-stages-current=ON +performance-schema-consumer-events-stages-history=ON +performance-schema-consumer-events-stages-history-long=ON +EOF + +fi diff --git a/images/mysql/entrypoints/200-mysql-envplate.bash b/images/mysql/entrypoints/200-mysql-envplate.bash new file mode 100755 index 000000000..a4aa676f5 --- /dev/null +++ b/images/mysql/entrypoints/200-mysql-envplate.bash @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +set -eo pipefail + +/bin/ep /etc/mysql/* diff --git a/images/mysql/entrypoints/9999-mysql-init.bash b/images/mysql/entrypoints/9999-mysql-init.bash new file mode 100755 index 000000000..e4b8b14e0 --- /dev/null +++ b/images/mysql/entrypoints/9999-mysql-init.bash @@ -0,0 +1,152 @@ +#!/usr/bin/env bash + +set -eo pipefail + +if [ "$(ls -A /etc/mysql/conf.d/)" ]; then + ep /etc/mysql/conf.d/* +fi + +if [ "${1:0:1}" = '-' ]; then + set -- mysqld "$@" +fi + +wantHelp= +for arg; do + case "$arg" in + -'?'|--help|--print-defaults|-V|--version) + wantHelp=1 + break + ;; + esac +done + +# check if MYSQL_COPY_DATA_DIR_SOURCE is set, if yes we're coping the contents of the given folder into the data dir folder +# this allows to prefill the datadir with a provided datadir (either added in a Dockerfile build, or mounted into the running container). +# This is different than just setting $MYSQL_DATA_DIR to the source folder, as only /var/lib/mysql is a persistent folder, so setting +# $MYSQL_DATA_DIR to another folder will make mysql to not store the datadir across container restarts, while with this copy system +# the data will be prefilled and persistent across container restarts. +if [ -n "$MYSQL_COPY_DATA_DIR_SOURCE" ]; then + if [ -d ${MYSQL_DATA_DIR:-/var/lib/mysql}/mysql ]; then + echo "MYSQL_COPY_DATA_DIR_SOURCE is set, but MySQL directory already present in '${MYSQL_DATA_DIR:-/var/lib/mysql}/mysql' skipping copying" + else + echo "MYSQL_COPY_DATA_DIR_SOURCE is set, copying datadir contents from '$MYSQL_COPY_DATA_DIR_SOURCE' to '${MYSQL_DATA_DIR:-/var/lib/mysql}'" + CUR_DIR=${PWD} + cd ${MYSQL_COPY_DATA_DIR_SOURCE}/; tar cf - . | (cd ${MYSQL_DATA_DIR:-/var/lib/mysql}; tar xvf -) + cd $CUR_DIR + fi +fi + +ln -sf ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf /home/.my.cnf + +if [ "$1" = 'mysqld' -a -z "$wantHelp" ]; then + if [ ! -d "/run/mysqld" ]; then + mkdir -p /var/run/mysqld + chown -R mysql:mysql /var/run/mysqld + fi + + MYSQL_INIT_WAIT_SECONDS=${MYSQL_INIT_WAIT_SECONDS:-30} + MYSQL_INIT_PERIOD_SECONDS=${MYSQL_INIT_PERIOD_SECONDS:-1} + + if [ -d ${MYSQL_DATA_DIR:-/var/lib/mysql}/mysql ]; then + echo "MySQL directory already present, skipping creation" + + echo "starting mysql for mysql upgrade." + /usr/sbin/mysqld --skip-networking & + pid="$!" + echo "pid is $pid" + + for i in $(seq 0 $MYSQL_INIT_WAIT_SECONDS); do + if echo 'SELECT 1' | mysql -u root; then + break + fi + echo 'MySQL init process in progress...' + sleep $MYSQL_INIT_PERIOD_SECONDS + done + + if ! kill -s TERM "$pid" || ! wait "$pid"; then + echo >&2 'MySQL init process failed.' + exit 1 + fi + else + echo "MySQL data directory not found, creating initial DBs" + + /usr/sbin/mysqld --initialize-insecure --skip-name-resolve --datadir=${MYSQL_DATA_DIR:-/var/lib/mysql} --basedir=/usr + + echo "starting mysql for initdb.d import." + /usr/sbin/mysqld --skip-networking & + pid="$!" + echo "pid is $pid" + + for i in $(seq 0 $MYSQL_INIT_WAIT_SECONDS); do + if echo 'SELECT 1' | mysql -u root; then + break + fi + echo 'MySQL init process in progress...' + sleep $MYSQL_INIT_PERIOD_SECONDS + done + + if [ "$MYSQL_ROOT_PASSWORD" = "" ]; then + MYSQL_ROOT_PASSWORD=`pwgen 16 1` + echo "[i] MySQL root Password: $MYSQL_ROOT_PASSWORD" + fi + + MYSQL_DATABASE=${MYSQL_DATABASE:-""} + MYSQL_USER=${MYSQL_USER:-""} + MYSQL_PASSWORD=${MYSQL_PASSWORD:-""} + + tfile=`mktemp` + if [ ! -f "$tfile" ]; then + return 1 + fi + + cat << EOF > $tfile +DROP DATABASE IF EXISTS test; +USE mysql; +ALTER USER 'root'@'localhost' IDENTIFIED BY '$MYSQL_ROOT_PASSWORD'; +DELETE FROM proxies_priv WHERE User='root' AND Host NOT IN ('localhost', '127.0.0.1', '::1'); +FLUSH PRIVILEGES; + +EOF + + if [ "$MYSQL_DATABASE" != "" ]; then + echo "[i] Creating database: $MYSQL_DATABASE" + echo "CREATE DATABASE IF NOT EXISTS \`$MYSQL_DATABASE\` ;" >> $tfile + if [ "$MYSQL_USER" != "" ]; then + echo "[i] Creating user: $MYSQL_USER with password $MYSQL_PASSWORD" + echo "CREATE USER '$MYSQL_USER'@'%' IDENTIFIED BY '$MYSQL_PASSWORD';" >> $tfile + echo "GRANT ALL ON \`$MYSQL_DATABASE\`.* to '$MYSQL_USER'@'%';" >> $tfile + fi + fi + + + cat $tfile + cat $tfile | mysql -v -u root + rm -v -f $tfile + + echo "[client]" >> ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf + echo "user=root" >> ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf + echo "password=${MYSQL_ROOT_PASSWORD}" >> ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf + echo "[mysql]" >> ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf + echo "database=${MYSQL_DATABASE}" >> ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf + + for f in /docker-entrypoint-initdb.d/*; do + if [ -e "$f" ]; then + case "$f" in + *.sh) echo "$0: running $f"; . "$f" ;; + *.sql) echo "$0: running $f"; cat $f| envsubst | tee | mysql -u root -p${MYSQL_ROOT_PASSWORD}; echo ;; + *) echo "$0: ignoring $f" ;; + esac + fi + done + + if ! kill -s TERM "$pid" || ! wait "$pid"; then + echo >&2 'MySQL init process failed.' + exit 1 + fi + + fi + + echo "done, now starting daemon" + touch /tmp/mysql-init-complete + +fi diff --git a/images/mysql/my.cnf b/images/mysql/my.cnf new file mode 100644 index 000000000..ad6e18b0b --- /dev/null +++ b/images/mysql/my.cnf @@ -0,0 +1,48 @@ +# The following options will be passed to all mysql clients +[client] +port = 3306 +socket = /var/run/mysqld/mysqld.sock + +# Here follows entries for some specific programs + +# The mysql server +[mysqld] +authentication_policy = "${MYSQL_AUTHENTICATION_PLUGIN:-mysql_native_password},," +binlog_expire_logs_seconds = 864000 +character_set_server = ${MYSQL_CHARSET:-utf8mb4} +collation_server = ${MYSQL_COLLATION:-utf8mb4_bin} +datadir = ${MYSQL_DATA_DIR:-/var/lib/mysql} +innodb_buffer_pool_size = ${MYSQL_INNODB_BUFFER_POOL_SIZE:-256M} +innodb_force_recovery = ${{MYSQL_INNODB_FORCE_RECOVER:-0} +innodb_flush_log_at_trx_commit = 0 +innodb_log_buffer_size = 32M +innodb_redo_log_capacity = 134217728 +join_buffer_size = 2M +key_buffer_size = 16M +max_allowed_packet = ${MYSQL_MAX_ALLOWED_PACKET:-64M} +max_connections = 400 +max_heap_table_size = 512M +myisam_recover_options = BACKUP +optimizer_search_depth = 0 +port = 3306 +secure_file_priv = "" +skip_external_locking = 1 +skip_name_resolve = 1 +socket = /var/run/mysqld/mysqld.sock +# This setting has impacts on the number of open file descriptors. The mysqld +# `open_files_limit` should not exceed the OS `max_open_files` of 1,048,576. The +# formula for `open_files_limit` is: +# (table_open_cache * 2) * table_open_cache_instances + (max_connections + ?) +# ? is a "head room" number. A fresh db requires 51, below will allow for 208. +table_open_cache = 65498 +thread_cache_size = 8 +thread_stack = 256K +tmp_table_size = 512M +tmpdir = /tmp +transaction_isolation = READ-COMMITTED +wait_timeout = ${MYSQL_WAIT_TIMEOUT:-28800} + +[mysqld-8.4] +mysql_native_password = ON + +!includedir /etc/mysql/conf.d diff --git a/images/mysql/mysql-backup.sh b/images/mysql/mysql-backup.sh new file mode 100755 index 000000000..4a81e742b --- /dev/null +++ b/images/mysql/mysql-backup.sh @@ -0,0 +1,109 @@ +#!/bin/sh +#============================================================================== +#TITLE: mysql_backup.sh +#DESCRIPTION: script for automating the daily mysql backups on development computer +#AUTHOR: tleish +#DATE: 2013-12-20 +#VERSION: 0.4 +#USAGE: ./mysql_backup.sh +#CRON: + # example cron for daily db backup @ 9:15 am + # min hr mday month wday command + # 15 9 * * * /Users/[your user name]/scripts/mysql_backup.sh + +#RESTORE FROM BACKUP + #$ gunzip < [backupfile.sql.gz] | mysql -u [uname] -p[pass] [dbname] + +#============================================================================== +# CUSTOM SETTINGS +#============================================================================== + +set -eu -o pipefail + +# directory to put the backup files +BACKUP_DIR=${MYSQL_DATA_DIR:-/var/lib/mysql}/backup + +# MYSQL Parameters +MYSQL_USER=${MYSQL_USER:-lagoon} +MYSQL_PASSWORD=${MYSQL_PASSWORD:-lagoon} + +MYSQL_HOST=$1 + +# Don't backup databases with these names +# Example: starts with mysql (^mysql) or ends with _schema (_schema$) +IGNORE_DB="(^mysql|_schema$)" + +# Number of days to keep backups +KEEP_BACKUPS_FOR=4 #days + +#============================================================================== +# METHODS +#============================================================================== + +# YYYY-MM-DD_HHMMSS +TIMESTAMP=$(date +%F_%H%M%S) + +function prepare() +{ + mkdir -p $BACKUP_DIR +} + +function delete_old_backups() +{ + echo "Deleting $BACKUP_DIR/*.sql.gz older than $KEEP_BACKUPS_FOR days" + find $BACKUP_DIR -type f -name "*.sql.gz" -mtime +$KEEP_BACKUPS_FOR -exec rm {} \; +} + +function mysql_login() { + cmd="-u $MYSQL_USER -h $MYSQL_HOST" + if [ -n "$MYSQL_PASSWORD" ]; then + cmd="$cmd -p$MYSQL_PASSWORD" + fi + echo $cmd +} + +function database_list() { + local show_databases_sql="SHOW DATABASES WHERE \`Database\` NOT REGEXP '$IGNORE_DB'" + echo $(mysql $(mysql_login) -e "$show_databases_sql"|awk -F " " '{if (NR!=1) print $1}') +} + +function echo_status(){ + printf '\r'; + printf ' %0.s' {0..100} + printf '\r'; + printf "$1"'\r' +} + +function backup_database(){ + backup_file="$BACKUP_DIR/$TIMESTAMP.$database.sql.gz" + output="${output}${database} => $backup_file\n" + echo_status "...backing up $count of $total databases: $database" + $(mysqldump --max-allowed-packet=500M --events --routines --quick --add-locks --no-autocommit --single-transaction $(mysql_login) $database | gzip -9 > $backup_file) +} + +function backup_databases(){ + local databases=$(database_list) + local total=$(echo $databases | wc -w | xargs) + local output="" + local count=1 + for database in $databases; do + backup_database + local count=$((count+1)) + done + echo -ne $output +} + +function hr(){ + printf '=%.0s' {1..100} + printf "\n" +} + +#============================================================================== +# RUN SCRIPT +#============================================================================== +prepare +delete_old_backups +hr +backup_databases +hr +printf "All backed up!\n\n" diff --git a/images/mysql/root/usr/share/container-scripts/mysql/readiness-probe.sh b/images/mysql/root/usr/share/container-scripts/mysql/readiness-probe.sh new file mode 100755 index 000000000..ea6747a94 --- /dev/null +++ b/images/mysql/root/usr/share/container-scripts/mysql/readiness-probe.sh @@ -0,0 +1,12 @@ +#!/bin/bash +# +# mysqld readinessProbe for use in kubernetes +# + +mysql --defaults-file=${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf -e"SHOW DATABASES;" + +if [ $? -ne 0 ]; then + exit 1 +else + exit 0 +fi