From e88aef18f420da581794f7659b6b604d63a3e286 Mon Sep 17 00:00:00 2001 From: Alec Turner Date: Sat, 9 Mar 2024 14:02:30 -0800 Subject: [PATCH] Add image utils and bugfixes from kenya-emr --- docker/Dockerfile.streamline-emr.amd64 | 1 + docker/Dockerfile.streamline-emr.arm64 | 1 + docker/build-image | 104 ++++++++++++++++++ images-aarch64 | 1 - images-x86_64 | 1 - images.csv | 2 + package.sh | 43 ++++++-- .../streamline/streamline-emr/monitor | 22 ++-- .../streamline/streamline-emr/template.env | 2 +- 9 files changed, 153 insertions(+), 24 deletions(-) create mode 100644 docker/Dockerfile.streamline-emr.amd64 create mode 100644 docker/Dockerfile.streamline-emr.arm64 create mode 100755 docker/build-image delete mode 100644 images-aarch64 delete mode 100644 images-x86_64 create mode 100644 images.csv diff --git a/docker/Dockerfile.streamline-emr.amd64 b/docker/Dockerfile.streamline-emr.amd64 new file mode 100644 index 00000000..90e6e1d8 --- /dev/null +++ b/docker/Dockerfile.streamline-emr.amd64 @@ -0,0 +1 @@ +FROM conrad96/streamline:2.0 \ No newline at end of file diff --git a/docker/Dockerfile.streamline-emr.arm64 b/docker/Dockerfile.streamline-emr.arm64 new file mode 100644 index 00000000..90e6e1d8 --- /dev/null +++ b/docker/Dockerfile.streamline-emr.arm64 @@ -0,0 +1 @@ +FROM conrad96/streamline:2.0 \ No newline at end of file diff --git a/docker/build-image b/docker/build-image new file mode 100755 index 00000000..8e6952e3 --- /dev/null +++ b/docker/build-image @@ -0,0 +1,104 @@ +#!/bin/bash + +LOCATION="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +# default arguments +TAG_BASENAME=registry.gitlab.com/signalytic/client-external/streamline/streamline-emr + +function print_usage { + echo "Streamline EMR Docker Image Utils" + echo + echo "Usage: build-image [ OPTIONS ]" + echo + echo "Options:" + echo " -h Print this help prompt and exit" + echo " -p Push images to a remote registry" + echo " -f DOCKERFILE Specify a docker file to build, may be passed multiple" + echo " times. If none specified, all targets built" + echo " -b TAG_BASENAME Images will be tagged with '//:'." + echo " Default: '${TAG_BASENAME}'" + echo " -v VERSION Set image tag. It not specified, version will be taken from git" + echo +} + +# collect user args +DOCKERFILES=() +while getopts ":f:b:v:ph" options; do + case "${options}" in + p) + PUSH_IMAGES=1 + ;; + f) + DOCKERFILES+=(${OPTARG}) + ;; + b) + TAG_BASENAME=${OPTARG} + ;; + v) + VERSION_NAME=${OPTARG} + ;; + h) + print_usage + exit 0 + ;; + *) + print_usage + exit 1 + ;; + esac +done +shift $((OPTIND -1)) +set -e + +# if not set, determine version name from git +if [ -z $VERSION_NAME ];then + # use ci tag name if available (e.g. "v1.2") + if [ $CI_COMMIT_TAG ]; then + VERSION_NAME="$CI_COMMIT_TAG" + # otherwise if ci, use BRANCH-COMMIT + elif [ $CI_COMMIT_BRANCH ]; then + VERSION_NAME="$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA" + # if not ci, try to use git + elif command -v git &> /dev/null; then + VERSION_NAME=$(git describe --always --tags --dirty) + # fallback + else + VERSION_NAME="latest" + fi +fi + +# if not specified, process all docker files +if [ ${#DOCKERFILES[@]} -eq 0 ]; then + DOCKERFILES=($(find ${LOCATION} -maxdepth 1 -name "Dockerfile.*" -exec basename {} \;)) +fi + +# process dockerfiles one at a time +for f in ${DOCKERFILES[@]}; do + echo + echo "Processing file '$f'" + + # extract remaining info from filename + basename=${f%.*} + name=${basename:11} + arch=${f##*.} + platform="linux/${arch}" + tag=${TAG_BASENAME}/${name}/${arch}:${VERSION_NAME} + echo " name: '${name}'" + echo " platform: '${platform}'" + echo " tag: '${tag}'" + + # build the image + echo "docker build --platform ${platform} -f ${LOCATION}/${f} -t ${tag} ${LOCATION}" + docker build --platform ${platform} -f ${LOCATION}/${f} -t ${tag} ${LOCATION} + + # push image if requested build was successful + if [ $? -eq 0 ] && [ "${PUSH_IMAGES}" = "1" ];then + echo "docker push ${tag}" + docker push ${tag} + + latest_tag="${TAG_BASENAME}/${name}/${arch}:latest" + docker tag ${tag} ${latest_tag} + echo "docker push ${latest_tag}" + docker push ${latest_tag} + fi +done diff --git a/images-aarch64 b/images-aarch64 deleted file mode 100644 index 85c8f99f..00000000 --- a/images-aarch64 +++ /dev/null @@ -1 +0,0 @@ -streamline-emr:2.0=registry.gitlab.com/signalytic/client-external/streamline/streamline-emr/arm64:2.0 \ No newline at end of file diff --git a/images-x86_64 b/images-x86_64 deleted file mode 100644 index 700ea70f..00000000 --- a/images-x86_64 +++ /dev/null @@ -1 +0,0 @@ -streamline-emr:2.0=registry.gitlab.com/signalytic/client-external/streamline/streamline-emr/amd64:2.0 \ No newline at end of file diff --git a/images.csv b/images.csv new file mode 100644 index 00000000..c47ec349 --- /dev/null +++ b/images.csv @@ -0,0 +1,2 @@ +linux/arm64,streamline-emr,registry.gitlab.com/signalytic/client-external/streamline/streamline-emr/streamline-emr/arm64 +linux/amd64,streamline-emr,registry.gitlab.com/signalytic/client-external/streamline/streamline-emr/streamline-emr/amd64 \ No newline at end of file diff --git a/package.sh b/package.sh index b89d8024..16070624 100755 --- a/package.sh +++ b/package.sh @@ -12,14 +12,32 @@ LOCATION="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" PROJECT_ROOT="$(realpath $LOCATION)" +# determine version name from git +# use ci tag name if available (e.g. "v1.2") +if [ $CI_COMMIT_TAG ]; then + VERSION_NAME="$CI_COMMIT_TAG" +# otherwise if ci, use BRANCH-COMMIT +elif [ $CI_COMMIT_BRANCH ]; then + VERSION_NAME="$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA" +# if not ci, try to use git +elif command -v git &> /dev/null; then + VERSION_NAME=$(git describe --always --tags --dirty) +# fallback +else + VERSION_NAME="latest" +fi + # configurable options (override with env vars as needed) PACKAGE_NAME=${PACKAGE_NAME-"streamline-emr"} OUTPUT_DIR=${OUTPUT_DIR-"$PROJECT_ROOT/build-packages"} PACKAGE_IMAGES=${PACKAGE_IMAGES:-1} +IMAGE_TAG=${IMAGE_TAG:-${VERSION_NAME}} +IMAGES_LIST=${IMAGES_LIST:-${PROJECT_ROOT}/images.csv} echo "packaging $PACKAGE_NAME" echo " output dir: $OUTPUT_DIR" echo " package images: $PACKAGE_IMAGES" +echo " image tag: $IMAGE_TAG" # work from the project root pushd $PROJECT_ROOT > /dev/null @@ -88,17 +106,20 @@ for platform in "linux/amd64" "linux/arm64"; do i=0 # read image list from file (e.g. 'images-aarch64') # images are listed as SAVED_TAG=SOURCE_IMAGE - for image in $(cat ${PROJECT_ROOT}/images-${pkg_arch});do - i=$((i+1)) - name="image${i}.img" - path="${IMAGES_DIR}/${name}" - IFS=\= read -r dest source <<< "${image}" - echo "pulling image '${source}' (${platform})" - docker pull --platform ${platform} ${source} - echo "tagging image as '${dest}'" - docker tag ${source} ${dest} - echo "saving to '${path}'" - docker save -o ${path} $dest + for image in $(cat ${IMAGES_LIST});do + IFS=, read -r img_platform dest source <<< "${image}" + # package only if the platforms match + if [ "${img_platform}" = "${platform}" ]; then + echo "pulling image '${source}' (${platform})" + docker pull --platform ${platform} ${source}:${IMAGE_TAG} + echo "tagging image as '${dest}:latest'" + docker tag ${source}:${IMAGE_TAG} ${dest}:latest + i=$((i+1)) + name="image${i}.img" + path="${IMAGES_DIR}/${name}" + echo "saving to '${path}'" + docker save -o ${path} ${dest}:latest + fi done else echo "skipping image packaging as requested" diff --git a/root/var/signalytic/clientapps/streamline/streamline-emr/monitor b/root/var/signalytic/clientapps/streamline/streamline-emr/monitor index e9a6f6cd..c061e371 100755 --- a/root/var/signalytic/clientapps/streamline/streamline-emr/monitor +++ b/root/var/signalytic/clientapps/streamline/streamline-emr/monitor @@ -4,16 +4,18 @@ LOCATION="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" for f in $(find ${LOCATION} -mindepth 2 -maxdepth 3 -name docker-compose.yml); do pushd $(dirname $f) > /dev/null - status=$(docker compose ps --format '{"name":"{{.Names}}","created":"{{.CreatedAt}}","elapsed":"{{.RunningFor}}","state":"{{.State}}","status":"{{.Status}}"}') - popd > /dev/null - if [ "${status}" ]; then - echo $status - if command -v bgsystemlog &> /dev/null - then - bgsystemlog -j streamline_emr -m "'${status}'" + for service in $(docker compose ps --services); do + status=$(docker compose ps --format '{"name":"{{.Names}}","created":"{{.CreatedAt}}","elapsed":"{{.RunningFor}}","state":"{{.State}}","status":"{{.Status}}"}' ${service}) + if [ "${status}" ]; then + echo $status + if command -v bgsystemlog &> /dev/null + then + bgsystemlog -j streamline_emr -m ${status} + fi + else + echo "no services found" + fi fi - else - echo "no services found" - fi + popd > /dev/null done echo "done." \ No newline at end of file diff --git a/root/var/signalytic/clientapps/streamline/streamline-emr/template.env b/root/var/signalytic/clientapps/streamline/streamline-emr/template.env index cb2ad4be..00bac1d8 100644 --- a/root/var/signalytic/clientapps/streamline/streamline-emr/template.env +++ b/root/var/signalytic/clientapps/streamline/streamline-emr/template.env @@ -1,4 +1,4 @@ -COMPOSE_PROJECT_NAME={{INSTANCE_NAME}} +COMPOSE_PROJECT_NAME=streamline-emr-{{INSTANCE_NAME}} CONTAINER_NAME=streamline-emr-{{INSTANCE_NAME}} DATA_DIR=/var/signalytic/clientapps/streamline/streamline-emr/instances/{{INSTANCE_NAME}}/data APP_PORT={{INSTANCE_PORT}}