Add image utils and bugfixes from kenya-emr

docker/Dockerfile.streamline-emr.amd64

@@ -0,0 +1 @@
+FROM conrad96/streamline:2.0

docker/Dockerfile.streamline-emr.arm64

@@ -0,0 +1 @@
+FROM conrad96/streamline:2.0

docker/build-image

@@ -0,0 +1,104 @@
+#!/bin/bash
+
+LOCATION="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+# default arguments
+TAG_BASENAME=registry.gitlab.com/signalytic/client-external/streamline/streamline-emr
+
+function print_usage {
+  echo "Streamline EMR Docker Image Utils"
+  echo
+  echo "Usage: build-image [ OPTIONS ]"
+  echo
+  echo "Options:"
+  echo "  -h                 Print this help prompt and exit"
+  echo "  -p                 Push images to a remote registry"
+  echo "  -f DOCKERFILE      Specify a docker file to build, may be passed multiple"
+  echo "                     times. If none specified, all targets built"
+  echo "  -b TAG_BASENAME    Images will be tagged with '<TAG_BASENAME>/<name>/<arch>:<version>'."
+  echo "                     Default: '${TAG_BASENAME}'"
+  echo "  -v VERSION         Set image tag. It not specified, version will be taken from git"
+  echo
+}
+
+# collect user args
+DOCKERFILES=()
+while getopts ":f:b:v:ph" options; do
+  case "${options}" in
+    p)
+      PUSH_IMAGES=1
+      ;;
+    f)
+      DOCKERFILES+=(${OPTARG})
+      ;;
+    b)
+      TAG_BASENAME=${OPTARG}
+      ;;
+    v)
+      VERSION_NAME=${OPTARG}
+      ;;
+    h)
+      print_usage
+      exit 0
+      ;;
+    *)
+      print_usage
+      exit 1
+      ;;
+  esac
+done
+shift $((OPTIND -1))
+set -e
+
+# if not set, determine version name from git
+if [ -z $VERSION_NAME ];then
+  # use ci tag name if available (e.g. "v1.2")
+  if [ $CI_COMMIT_TAG ]; then
+    VERSION_NAME="$CI_COMMIT_TAG"
+  # otherwise if ci, use BRANCH-COMMIT
+  elif [ $CI_COMMIT_BRANCH ]; then
+    VERSION_NAME="$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA"
+  # if not ci, try to use git
+  elif command -v git &> /dev/null; then
+    VERSION_NAME=$(git describe --always --tags --dirty)
+  # fallback
+  else
+    VERSION_NAME="latest"
+  fi
+fi
+
+# if not specified, process all docker files
+if [ ${#DOCKERFILES[@]} -eq 0 ]; then
+  DOCKERFILES=($(find ${LOCATION} -maxdepth 1 -name "Dockerfile.*" -exec basename {} \;))
+fi
+
+# process dockerfiles one at a time
+for f in ${DOCKERFILES[@]}; do
+  echo
+  echo "Processing file '$f'"
+
+  # extract remaining info from filename
+  basename=${f%.*}
+  name=${basename:11}
+  arch=${f##*.}
+  platform="linux/${arch}"
+  tag=${TAG_BASENAME}/${name}/${arch}:${VERSION_NAME}
+  echo "  name: '${name}'"
+  echo "  platform: '${platform}'"
+  echo "  tag: '${tag}'"
+
+  # build the image
+  echo "docker build --platform ${platform} -f ${LOCATION}/${f} -t ${tag} ${LOCATION}"
+  docker build --platform ${platform} -f ${LOCATION}/${f} -t ${tag} ${LOCATION}
+
+  # push image if requested build was successful
+  if [ $? -eq 0 ] && [ "${PUSH_IMAGES}" = "1" ];then
+    echo "docker push ${tag}"
+    docker push ${tag}
+
+    latest_tag="${TAG_BASENAME}/${name}/${arch}:latest"
+    docker tag ${tag} ${latest_tag}
+    echo "docker push ${latest_tag}"
+    docker push ${latest_tag}
+  fi
+done

images-aarch64

@@ -1 +0,0 @@
-streamline-emr:2.0=registry.gitlab.com/signalytic/client-external/streamline/streamline-emr/arm64:2.0

images-x86_64

@@ -1 +0,0 @@
-streamline-emr:2.0=registry.gitlab.com/signalytic/client-external/streamline/streamline-emr/amd64:2.0

images.csv

@@ -0,0 +1,2 @@
+linux/arm64,streamline-emr,registry.gitlab.com/signalytic/client-external/streamline/streamline-emr/streamline-emr/arm64
+linux/amd64,streamline-emr,registry.gitlab.com/signalytic/client-external/streamline/streamline-emr/streamline-emr/amd64

package.sh

@@ -12,14 +12,32 @@
 LOCATION="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 PROJECT_ROOT="$(realpath $LOCATION)"
 
+# determine version name from git
+# use ci tag name if available (e.g. "v1.2")
+if [ $CI_COMMIT_TAG ]; then
+  VERSION_NAME="$CI_COMMIT_TAG"
+# otherwise if ci, use BRANCH-COMMIT
+elif [ $CI_COMMIT_BRANCH ]; then
+  VERSION_NAME="$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA"
+# if not ci, try to use git
+elif command -v git &> /dev/null; then
+  VERSION_NAME=$(git describe --always --tags --dirty)
+# fallback
+else
+  VERSION_NAME="latest"
+fi
+
 # configurable options (override with env vars as needed)
 PACKAGE_NAME=${PACKAGE_NAME-"streamline-emr"}
 OUTPUT_DIR=${OUTPUT_DIR-"$PROJECT_ROOT/build-packages"}
 PACKAGE_IMAGES=${PACKAGE_IMAGES:-1}
+IMAGE_TAG=${IMAGE_TAG:-${VERSION_NAME}}
+IMAGES_LIST=${IMAGES_LIST:-${PROJECT_ROOT}/images.csv}
 
 echo "packaging $PACKAGE_NAME"
 echo "  output dir: $OUTPUT_DIR"
 echo "  package images: $PACKAGE_IMAGES"
+echo "  image tag: $IMAGE_TAG"
 
 # work from the project root
 pushd $PROJECT_ROOT > /dev/null
@@ -88,17 +106,20 @@ for platform in "linux/amd64" "linux/arm64"; do
     i=0
     # read image list from file (e.g. 'images-aarch64')
     # images are listed as SAVED_TAG=SOURCE_IMAGE
-    for image in $(cat ${PROJECT_ROOT}/images-${pkg_arch});do
-      i=$((i+1))
-      name="image${i}.img"
-      path="${IMAGES_DIR}/${name}"
-      IFS=\= read -r dest source <<< "${image}"
-      echo "pulling image '${source}' (${platform})"
-      docker pull --platform ${platform} ${source}
-      echo "tagging image as '${dest}'"
-      docker tag ${source} ${dest} 
-      echo "saving to '${path}'"
-      docker save -o ${path} $dest
+    for image in $(cat ${IMAGES_LIST});do
+      IFS=, read -r img_platform dest source <<< "${image}"
+      # package only if the platforms match
+      if [ "${img_platform}" = "${platform}" ]; then
+        echo "pulling image '${source}' (${platform})"
+        docker pull --platform ${platform} ${source}:${IMAGE_TAG}
+        echo "tagging image as '${dest}:latest'"
+        docker tag ${source}:${IMAGE_TAG} ${dest}:latest
+        i=$((i+1))
+        name="image${i}.img"
+        path="${IMAGES_DIR}/${name}"
+        echo "saving to '${path}'"
+        docker save -o ${path} ${dest}:latest
+      fi
     done
   else
     echo "skipping image packaging as requested"

root/var/signalytic/clientapps/streamline/streamline-emr/monitor

@@ -4,16 +4,18 @@ LOCATION="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 
 for f in $(find ${LOCATION} -mindepth 2 -maxdepth 3 -name docker-compose.yml); do
   pushd $(dirname $f) > /dev/null
-  status=$(docker compose ps --format '{"name":"{{.Names}}","created":"{{.CreatedAt}}","elapsed":"{{.RunningFor}}","state":"{{.State}}","status":"{{.Status}}"}')
-  popd > /dev/null
-  if [ "${status}" ]; then
-    echo $status
-    if command -v bgsystemlog &> /dev/null
-    then
-      bgsystemlog -j streamline_emr -m "'${status}'"
+    for service in $(docker compose ps --services); do
+      status=$(docker compose ps --format '{"name":"{{.Names}}","created":"{{.CreatedAt}}","elapsed":"{{.RunningFor}}","state":"{{.State}}","status":"{{.Status}}"}' ${service})
+      if [ "${status}" ]; then
+        echo $status
+        if command -v bgsystemlog &> /dev/null
+        then
+          bgsystemlog -j streamline_emr -m ${status}
+        fi
+      else
+        echo "no services found"
+      fi
     fi
-  else
-    echo "no services found"
-  fi
+  popd > /dev/null
 done
 echo "done."

root/var/signalytic/clientapps/streamline/streamline-emr/template.env

@@ -1,4 +1,4 @@
-COMPOSE_PROJECT_NAME={{INSTANCE_NAME}}
+COMPOSE_PROJECT_NAME=streamline-emr-{{INSTANCE_NAME}}
 CONTAINER_NAME=streamline-emr-{{INSTANCE_NAME}}
 DATA_DIR=/var/signalytic/clientapps/streamline/streamline-emr/instances/{{INSTANCE_NAME}}/data
 APP_PORT={{INSTANCE_PORT}}