Simplify application management and align with other docker applications

The notion of application "instances" is dropped in favor of a simpler,
single-instance approach. Default values are provided for all configuration
variables so allow the app to run without additional manual configuration.

A health check and auto-heal script are added to automatically restart the
application in case it becomes unavailable during operation.
This commit is contained in:
2024-03-25 09:34:34 -07:00
parent 6c59b1ab76
commit 9cbefc841f
14 changed files with 189 additions and 211 deletions
@@ -1,14 +1,16 @@
[Unit]
Description=Run Streamline EMR
After=network.target
StartLimitIntervalSec=0
Requires=docker.service
After=docker.service
[Service]
Type=simple
Restart=always
RestartSec=10
Type=oneshot
RemainAfterExit=yes
User=root
ExecStart=/var/signalytic/clientapps/streamline/streamline-emr/entry
WorkingDirectory=/var/signalytic/clientapps/streamline/streamline-emr
ExecStart=/var/signalytic/clientapps/streamline/streamline-emr/start
ExecStop=/var/signalytic/clientapps/streamline/streamline-emr/stop
TimeoutStartSec=0
[Install]
WantedBy=multi-user.target
@@ -0,0 +1,45 @@
#!/bin/bash
LOCATION="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
INTERVAL=${INTERVAL:-60}
# track pid with a file `autoheal.pid`. If a previous instance is still running, exit early.
function cleanup() {
rm -f ${LOCATION}/autoheal.pid
}
trap cleanup EXIT
kill -0 $(cat ${LOCATION}/autoheal.pid 2> /dev/null) 2> /dev/null && echo "autoheal script already running" && exit 1
echo $$ > ${LOCATION}/autoheal.pid
# exit on SIGINT
trap exit INT
# run at fixed interval, checking for and restarting any service marked as 'unhealthy'
pushd $LOCATION > /dev/null
while [ 1 ]; do
# if setup is not running, make sure containers are active and running
if ! kill -0 $(cat ${LOCATION}/setup.pid 2> /dev/null) 2> /dev/null; then
# get current state (running or not) and status (healthy, unhealthy, etc)
app_state=$(docker compose ps --format '{{.State}}' streamline-emr)
app_status=$(docker compose ps --format '{{.Status}}' streamline-emr)
# check if app is running and healhty
if [ "${app_state}" = "running" ]; then
if [[ "${app_status}" = *\(unhealthy\) ]]; then
echo "Unhealthy service detected. Restarting 'streamline-emr'"
docker compose restart streamline-emr
fi
# app should be running - restart
else
echo "Container not running. Starting 'streamline-emr'"
docker compose up -d streamline-emr
fi
fi
# wait a little while before checking again
sleep $INTERVAL
done
popd > /dev/null
@@ -1,49 +0,0 @@
#!/bin/bash
LOCATION="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
CONF_FILE=${LOCATION}/instances.conf
INSTANCES_DIR=${LOCATION}/instances
COMPOSE_FILE=${LOCATION}/docker-compose.yml
SETUP_FILE=${LOCATION}/setup
ENV_TEMPLATE=${LOCATION}/template.env
VERSION_FILE=${LOCATION}/version
version=$(head -n 1 ${VERSION_FILE} 2>/dev/null)
if [ $? -ne 0 ];then
version="unknown version"
fi
echo "Configuring Kenya EMR: $version"
if [ ! -f "${CONF_FILE}" ]; then
echo "missing conf file: ${CONF_FILE}"
exit 1
fi
echo "processing config file: ${CONF_FILE}"
# load instance names and port numbers from conf file
instances=($(sed -n "s/INSTANCES=//p" ${CONF_FILE}))
db_ports=($(sed -n "s/DB_PORTS=//p" ${CONF_FILE}))
http_ports=($(sed -n "s/HTTP_PORTS=//p" ${CONF_FILE}))
https_ports=($(sed -n "s/HTTPS_PORTS=//p" ${CONF_FILE}))
for ((i=0;i<${#instances[@]};i++)); do
# prepare directory and config files for instance
instance=$(echo ${instances[$i]} | tr -d '"')
db_port=$(echo ${db_ports[$i]} | tr -d '"')
http_port=$(echo ${http_ports[$i]} | tr -d '"')
https_port=$(echo ${https_ports[$i]} | tr -d '"')
echo
echo "configuring instance $((i+1)) / ${#instances[@]}"
echo " name: ${instance}"
echo " db_port: ${db_port}"
echo " http_port: ${http_port}"
echo " https_port: ${https_port}"
mkdir -p ${INSTANCES_DIR}/${instance}/data
sed -e "s/{{INSTANCE_NAME}}/${instance}/g" -e "s/{{INSTANCE_DB_PORT}}/${db_port}/g" -e "s/{{INSTANCE_HTTP_PORT}}/${http_port}/g" -e "s/{{INSTANCE_HTTPS_PORT}}/${https_port}/g" ${ENV_TEMPLATE} > ${INSTANCES_DIR}/${instance}/.env
cp ${COMPOSE_FILE} ${INSTANCES_DIR}/${instance}/
cp ${SETUP_FILE} ${INSTANCES_DIR}/${instance}/
done
echo "done."
+14 -5
View File
@@ -1,9 +1,18 @@
version: '3.9'
version: "3.8"
services:
streamline-emr:
image: ${IMAGE:-streamline-emr:latest}
container_name: ${CONTAINER_NAME}
image: ${APP_IMAGE:-streamline-emr:latest}
container_name: ${APP_CONTAINER:-streamline-emr}
volumes:
- "${DATA_DIR}:/var/lib/mysql"
- "${DATA_DIR:-/var/signalytic/clientapps/streamline/streamline-emr/data}:/var/lib/mysql"
ports:
- "${APP_PORT}:80"
- "${APP_PORT:-3000}:80"
dns: 0.0.0.0
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:80"]
timeout: 30s
start_period: 3600s
start_interval: 10s
interval: 30s
retries: 5
@@ -1,27 +0,0 @@
#!/bin/bash
LOCATION="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
function cleanup {
${LOCATION}/stop
echo "done."
}
trap cleanup EXIT
# configure instances
${LOCATION}/configure
# start all instances
${LOCATION}/start
# wait for any service to stop
pids=()
for f in $(find ${LOCATION} -mindepth 2 -maxdepth 3 -name docker-compose.yml); do
pushd $(dirname $f) > /dev/null
docker compose wait streamline-emr &
pids+=($!)
popd > /dev/null
done
echo "waiting on processes: ${pids[@]}"
wait -n ${pids[@]} > /dev/null
@@ -2,20 +2,42 @@
LOCATION="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
for f in $(find ${LOCATION} -mindepth 2 -maxdepth 3 -name docker-compose.yml); do
pushd $(dirname $f) > /dev/null
for service in $(docker compose ps --services); do
status=$(docker compose ps --format '{"name":"{{.Names}}","created":"{{.CreatedAt}}","elapsed":"{{.RunningFor}}","state":"{{.State}}","status":"{{.Status}}"}' ${service})
if [ "${status}" ]; then
echo $status
if command -v bgsystemlog &> /dev/null
then
bgsystemlog -j streamline_emr -m ${status}
fi
else
echo "no services found"
SYSTEMLOG=${SYSTEMLOG:-1}
if ! command -v bgsystemlog &> /dev/null; then
SYSTEMLOG=0
fi
# get container info
APP_CONTAINER=${APP_CONTAINER:-streamline-emr}
if [ -f ${LOCATION}/.env ]; then
APP_CONTAINER=$(sed -n 's/APP_CONTAINER=//p' ${LOCATION}/.env)
fi
pushd $LOCATION > /dev/null
# get process status for each service
echo "checking streamline-emr service status"
for service in $(docker compose ps --services); do
status=$(docker compose ps --format '{"name":"{{.Names}}","created":"{{.CreatedAt}}","elapsed":"{{.RunningFor}}","state":"{{.State}}","status":"{{.Status}}"}' ${service})
if [ "${status}" ]; then
echo $status
if [ "$SYSTEMLOG" -eq 1 ]; then
bgsystemlog -j streamline_emr_status -m ${status}
fi
done
popd > /dev/null
done
else
echo "no status results found for service ${service}"
fi
done
# get resource usage
echo "checking streamline-emr resource usage"
status=$(docker stats --no-stream --format '{"name":"{{.Name}}","cpu_p":"{{.CPUPerc}}","mem_p":"{{.MemPerc}}","mem_u":"{{.MemUsage}}","net_io":"{{.NetIO}}"},"block_io":"{{.BlockIO}}"}' ${APP_CONTAINER})
if [ "${status}" ]; then
echo $status
if [ "$SYSTEMLOG" -eq 1 ]; then
bgsystemlog -j streamline_emr_usage -m ${status}
fi
else
echo "no usage results found for container ${APP_CONTAINER}"
fi
popd > /dev/null
echo "done."
@@ -2,40 +2,24 @@
LOCATION="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
CONF_FILE=${LOCATION}/instances.conf
INSTANCES_DIR=${LOCATION}/instances
VERSION_FILE=${LOCATION}/version
version=$(head -n 1 ${VERSION_FILE} 2>/dev/null)
if [ $? -ne 0 ];then
version="unknown version"
fi
echo "Starting Streamline EMR: $version"
if [ ! -f "${CONF_FILE}" ]; then
echo "missing conf file: ${CONF_FILE}"
exit 1
fi
# work from the install root
pushd $LOCATION > /dev/null
# make sure we're always working from a clean state
./stop
echo "processing config file: ${CONF_FILE}"
# start the application
echo "starting services"
docker compose up -d
# load instance names and port numbers from conf file
instances=($(sed -n "s/INSTANCES=//p" ${CONF_FILE}))
for ((i=0;i<${#instances[@]};i++)); do
# prepare directory and config files for instance
instance=$(echo ${instances[$i]} | tr -d '"')
echo
echo "starting instance $((i+1)) / ${#instances[@]}"
pushd ${INSTANCES_DIR}/${instance} > /dev/null
# setup instance
${INSTANCES_DIR}/${instance}/setup
# start app
docker compose up -d --force-recreate
popd > /dev/null
done
# start autoheal to restart unhealthy containers
./autoheal &
popd > /dev/null
echo "done."
@@ -2,10 +2,11 @@
LOCATION="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
echo "stopping all instances"
for f in $(find ${LOCATION} -mindepth 2 -maxdepth 3 -name docker-compose.yml); do
pushd $(dirname $f) > /dev/null
echo "stopping streamline emr"
pushd $LOCATION > /dev/null
if [ -f autoheal.pid ];then
kill $(cat autoheal.pid)
fi
docker compose down
popd > /dev/null
done
echo "done."
popd > /dev/null
echo "done."
@@ -17,6 +17,12 @@
"username": "",
"password": ""
},
"instances": {}
"instances": {
"streamline-emr": {
"source": "streamline-emr",
"node": "",
"location": "_default"
}
}
}
}
}
@@ -1,5 +1,5 @@
COMPOSE_PROJECT_NAME=streamline-emr-{{INSTANCE_NAME}}
CONTAINER_NAME=streamline-emr-{{INSTANCE_NAME}}
DATA_DIR=/var/signalytic/clientapps/streamline/streamline-emr/instances/{{INSTANCE_NAME}}/data
APP_PORT={{INSTANCE_PORT}}
#IMAGE=streamline-emr:latest
COMPOSE_PROJECT_NAME=streamline-emr
DATA_DIR=/var/signalytic/clientapps/streamline/streamline-emr/data
APP_IMAGE=streamline-emr:latest
APP_CONTAINER=streamline-emr
APP_PORT=3000
@@ -1,2 +0,0 @@
INSTANCES=
APP_PORTS=