diff --git a/docker/Dockerfile.streamline-emr.amd64 b/docker/Dockerfile.streamline-emr.amd64 index 90e6e1d8..b67a1a45 100644 --- a/docker/Dockerfile.streamline-emr.amd64 +++ b/docker/Dockerfile.streamline-emr.amd64 @@ -1 +1,4 @@ -FROM conrad96/streamline:2.0 \ No newline at end of file +FROM streamlinehealth/streamline:signalytic + +COPY my.cnf /etc/mysql/my.cnf +COPY mysql-init.sh /var/www/html/.docker/mysql/mysql-init.sh diff --git a/docker/Dockerfile.streamline-emr.arm64 b/docker/Dockerfile.streamline-emr.arm64 index 90e6e1d8..b67a1a45 100644 --- a/docker/Dockerfile.streamline-emr.arm64 +++ b/docker/Dockerfile.streamline-emr.arm64 @@ -1 +1,4 @@ -FROM conrad96/streamline:2.0 \ No newline at end of file +FROM streamlinehealth/streamline:signalytic + +COPY my.cnf /etc/mysql/my.cnf +COPY mysql-init.sh /var/www/html/.docker/mysql/mysql-init.sh diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 00000000..1fea814b --- /dev/null +++ b/docker/README.md @@ -0,0 +1,16 @@ +# Image Notes + +The base images used have a few issues that should be addressed, specifically: +- quite a few files seem to be unused, taking up space unnecessarily and making it difficult to follow +- services should ideally be run in separate docker containers - this looks to have been at least partially implemented, but is not currently used +- mysql configuration is managed in several places, in particular with respect to binary logging +- binary logging needs to be configured slightly differently to work reliably with Signalytic tools (in their current state) + +A minimal set of changes has been made to the images in order to implement an initial prototype system, this includes: +- modify `/etc/mysql/my.cnf` + - limit binary logging to the `streamline` database only (`binlog-do-db=streamline`) + - remove binary log size limit (`max_binlog_size`) +- modify `/var/www/html/.docker/mysql/mysql-init.sh` + - remove binary log configuration from `mysqld` commands, using `my.cnf` as the sole configuration source (`--log-basename=bin --log-bin=/var/lib/mysql/logs/bin`) + +The reason for these changes is that the Signalytic database sync tools currently work with only a single database at a time. When using MariaDB, several system databases are generated automatically, generating additional binary logs that we are not interested in. When attempting to later apply the binary logs to a database server, conflicts may arise. An alternative approach would be to drop all databases instead, this is worth investigating in the future. \ No newline at end of file diff --git a/docker/my.cnf b/docker/my.cnf new file mode 100644 index 00000000..a5d9fab3 --- /dev/null +++ b/docker/my.cnf @@ -0,0 +1,4 @@ +[mysqld] +log-basename=bin +log-bin=/var/lib/mysql/logs/bin +binlog-do-db=streamline \ No newline at end of file diff --git a/docker/mysql-init.sh b/docker/mysql-init.sh new file mode 100755 index 00000000..cdc794f7 --- /dev/null +++ b/docker/mysql-init.sh @@ -0,0 +1,135 @@ +#!/bin/sh + +# This script helps to initialise MariaDB and setup fresh database instance + +# execute any pre-init scripts +for i in /scripts/pre-init.d/*sh +do + if [ -e "${i}" ]; then + echo "[i] pre-init.d - processing $i" + . "${i}" + fi +done + +# create bin-logs folder +if [ ! -d "/var/lib/mysql/logs" ]; then + echo "[i] created logs folder." + mkdir -p /var/lib/mysql/logs/ +fi + +if [ -d "/run/mysqld" ]; then + echo "[i] mysqld already present, skipping creation" + chown -R mysql:mysql /run/mysqld +else + echo "[i] mysqld not found, creating...." + mkdir -p /run/mysqld + chown -R mysql:mysql /run/mysqld +fi + +if [ -d /var/lib/mysql/mysql ]; then + echo "[i] MySQL directory already present, skipping creation" + chown -R mysql:mysql /var/lib/mysql +else + echo "[i] MySQL data directory not found, creating initial DBs" + + chown -R mysql:mysql /var/lib/mysql + + mysql_install_db --user=mysql --data=/var/lib/mysql > /dev/null + + tfile=`mktemp` + if [ ! -f "$tfile" ]; then + return 1 + fi + + cat << EOF > $tfile + USE mysql; + FLUSH PRIVILEGES ; + GRANT ALL ON *.* TO 'root'@'%' identified by '$MYSQL_ROOT_PASSWORD' WITH GRANT OPTION ; + GRANT ALL ON *.* TO 'root'@'localhost' identified by '$MYSQL_ROOT_PASSWORD' WITH GRANT OPTION ; + SET PASSWORD FOR 'root'@'localhost'=PASSWORD('${MYSQL_ROOT_PASSWORD}') ; + DROP DATABASE IF EXISTS test ; + FLUSH PRIVILEGES ; +EOF + + if [[ "$MYSQL_DATABASE" != "" ]]; then + echo "[i] Creating database: $MYSQL_DATABASE" + if [ "$MYSQL_CHARSET" != "" ] && [ "$MYSQL_COLLATION" != "" ]; then + echo "[i] with character set [$MYSQL_CHARSET] and collation [$MYSQL_COLLATION]" + echo "CREATE DATABASE IF NOT EXISTS \`$MYSQL_DATABASE\` CHARACTER SET $MYSQL_CHARSET COLLATE $MYSQL_COLLATION;" >> $tfile + else + echo "[i] with character set: 'utf8' and collation: 'utf8_general_ci'" + echo "CREATE DATABASE IF NOT EXISTS \`$MYSQL_DATABASE\` CHARACTER SET utf8 COLLATE utf8_general_ci;" >> $tfile + fi + + if [ "$MYSQL_USER" != "" ]; then + echo "[i] Creating user: $MYSQL_USER with password $MYSQL_PASSWORD" + + echo "GRANT ALL ON `$MYSQL_DATABASE`.* TO '$MYSQL_USER'@'127.0.0.1' IDENTIFIED BY '$MYSQL_PASSWORD';" >> $tfile; + echo "GRANT ALL ON \`$MYSQL_DATABASE\`.* to '$MYSQL_USER'@'%' IDENTIFIED BY '$MYSQL_PASSWORD';" >> $tfile + fi + fi + + /usr/bin/mysqld --user=mysql --bootstrap --verbose=0 --skip-name-resolve --skip-networking=0 < $tfile + # /usr/bin/mysqld --user=mysql --bootstrap --verbose=0 --skip-name-resolve --skip-networking=0 --log-basename=bin --log-bin=/var/lib/mysql/logs/bin < $tfile + rm -f $tfile + + # only run if we have a starting MYSQL_DATABASE env variable AND + # the /docker-entrypoint-initdb.d/ file is not empty + if [ "$MYSQL_DATABASE" != "" ] && [ "$(ls -A /docker-entrypoint-initdb.d 2>/dev/null)" ]; then + + # start the server temporarily so that we can import seed files + echo + echo "Preparing to process the contents of /docker-entrypoint-initdb.d/" + echo + TEMP_OUTPUT_LOG=/tmp/mysqld_output + /usr/bin/mysqld --user=mysql --skip-name-resolve --skip-networking=0 --silent-startup > "${TEMP_OUTPUT_LOG}" 2>&1 & + # /usr/bin/mysqld --user=mysql --skip-name-resolve --skip-networking=0 --log-basename=bin --log-bin=/var/lib/mysql/logs/bin --silent-startup > "${TEMP_OUTPUT_LOG}" 2>&1 & + PID="$!" + + # watch the output log until the server is running + until tail "${TEMP_OUTPUT_LOG}" | grep -q "Version:"; do + sleep 0.2 + done + + # use mysql client to import seed files while temp db is running + # use the starting MYSQL_DATABASE so mysql knows where to import + MYSQL_CLIENT="/usr/bin/mysql -u root -p$MYSQL_ROOT_PASSWORD" + + # loop through all the files in the seed directory + # redirect input (<) from .sql files into the mysql client command line + # pipe (|) the output of using `gunzip -c` on .sql.gz files + for f in /docker-entrypoint-initdb.d/*; do + case "$f" in + *.sql) echo " $0: running $f"; eval "${MYSQL_CLIENT} ${MYSQL_DATABASE} < $f"; echo ;; + *.sql.gz) echo " $0: running $f"; gunzip -c "$f" | eval "${MYSQL_CLIENT} ${MYSQL_DATABASE}"; echo ;; + esac + done + + # send the temporary mysqld server a shutdown signal + # and wait till it's done before completeing the init process + kill -s TERM "${PID}" + wait "${PID}" + rm -f TEMP_OUTPUT_LOG + echo "Completed processing seed files." + fi; + + echo + echo 'MySQL init process done. Ready for start up.' + echo + + echo "exec /usr/bin/mysqld --user=mysql --console --skip-name-resolve --skip-networking=0" "$@" + # echo "exec /usr/bin/mysqld --user=mysql --console --skip-name-resolve --skip-networking=0 --log-basename=bin --log-bin=/var/lib/mysql/logs/bin " "$@" +fi + +# execute any pre-exec scripts +for i in /scripts/pre-exec.d/*sh +do + if [ -e "${i}" ]; then + echo "[i] pre-exec.d - processing $i" + . ${i} + fi +done + +exec /usr/bin/mysqld --user=mysql --console --skip-name-resolve --skip-networking=0 & + +return $? \ No newline at end of file diff --git a/root/var/signalytic/clientapps/streamline/streamline-emr/docker-compose.yml b/root/var/signalytic/clientapps/streamline/streamline-emr/docker-compose.yml index d6153312..1135e50a 100644 --- a/root/var/signalytic/clientapps/streamline/streamline-emr/docker-compose.yml +++ b/root/var/signalytic/clientapps/streamline/streamline-emr/docker-compose.yml @@ -1,7 +1,7 @@ version: '3.9' services: streamline-emr: - image: ${IMAGE:-streamline-emr:2.0} + image: ${IMAGE:-streamline-emr:latest} container_name: ${CONTAINER_NAME} volumes: - "${DATA_DIR}:/var/lib/mysql" diff --git a/root/var/signalytic/clientapps/streamline/streamline-emr/template.env b/root/var/signalytic/clientapps/streamline/streamline-emr/template.env index 00bac1d8..b4114fff 100644 --- a/root/var/signalytic/clientapps/streamline/streamline-emr/template.env +++ b/root/var/signalytic/clientapps/streamline/streamline-emr/template.env @@ -2,4 +2,4 @@ COMPOSE_PROJECT_NAME=streamline-emr-{{INSTANCE_NAME}} CONTAINER_NAME=streamline-emr-{{INSTANCE_NAME}} DATA_DIR=/var/signalytic/clientapps/streamline/streamline-emr/instances/{{INSTANCE_NAME}}/data APP_PORT={{INSTANCE_PORT}} -#IMAGE=streamline-emr:2.0 \ No newline at end of file +#IMAGE=streamline-emr:latest \ No newline at end of file