[lib] improve logging. fix test checks using env variables in subproc…

…ess.

src/entrypoint.sh

@@ -77,7 +77,9 @@ _read_docker_hub_rate() {
 gantry() {
   local PRE_RUN_CMD="${GANTRY_PRE_RUN_CMD:-""}"
   local POST_RUN_CMD="${GANTRY_POST_RUN_CMD:-""}"
-  local STACK="${1:-gantry}"
+  local STACK="${1}"
+  [ -z "${STACK}" ] && STACK=$(gantry_current_service_name)
+  [ -z "${STACK}" ] && STACK="gantry"
   export LOG_SCOPE="${STACK}"
   local START_TIME=
   START_TIME=$(date +%s)
@@ -131,7 +133,7 @@ gantry() {
 
   local TIME_ELAPSED=
   TIME_ELAPSED=$(time_elapsed_since "${START_TIME}")
-  local MESSAGE="Done. Use ${TIME_ELAPSED}. ${ACCUMULATED_ERRORS} errors."
+  local MESSAGE="Done. Use ${TIME_ELAPSED}. ${ACCUMULATED_ERRORS} error(s)."
   local RETURN_VALUE=0
   if [ "${ACCUMULATED_ERRORS}" -gt 0 ]; then
     log ERROR "${MESSAGE}"

src/lib-common.sh

@@ -376,10 +376,13 @@ docker_service_remove() {
     return 0
   fi
   log INFO "Removing service ${SERVICE_NAME}."
-  docker service rm "${SERVICE_NAME}" >/dev/null
-  local RETURN_VALUE=$?
+  local LOG=
+  if ! LOG=$(docker service rm "${SERVICE_NAME}" 2>&1); then
+    log ERROR "Failed to remove docker service ${SERVICE_NAME}: ${LOG}"
+    return 1
+  fi
   log INFO "Removed service ${SERVICE_NAME}."
-  return ${RETURN_VALUE}
+  return 0
 }
 
 # We do not expect failures when using docker_global_job.
@@ -389,10 +392,13 @@ docker_service_remove() {
 docker_global_job() {
   local SERVICE_NAME=
   SERVICE_NAME=$(_get_docker_command_name_arg "${@}")
-  log INFO "Starting service ${SERVICE_NAME}."
-  docker service create \
-    --mode global-job \
-    "${@}" >/dev/null
+  log INFO "Starting global-job ${SERVICE_NAME}."
+  local LOG=
+  if ! LOG=$(docker service create --mode global-job "${@}"  2>&1); then
+    log ERROR "Failed to create global-job ${SERVICE_NAME}: ${LOG}"
+    return 1
+  fi
+  return 0
 }
 
 # A job could fail when using docker_replicated_job.
@@ -403,16 +409,17 @@ docker_replicated_job() {
   IS_DETACH=$(_get_docker_command_detach "${@}")
   # Add "--detach" to work around https://github.com/docker/cli/issues/2979
   # The Docker CLI does not exit on failures.
-  log INFO "Starting service ${SERVICE_NAME}."
-  docker service create \
-    --mode replicated-job --detach \
-    "${@}" >/dev/null
-  local RETURN_VALUE=$?
+  log INFO "Starting replicated-job ${SERVICE_NAME}."
+  local LOG=
+  if ! LOG=$(docker service create --mode replicated-job --detach "${@}" 2>&1); then
+    log ERROR "Failed to create replicated-job ${SERVICE_NAME}: ${LOG}"
+    return 1
+  fi
   # If the command line does not contain '--detach', the function returns til the replicated job is complete.
   if ! "${IS_DETACH}"; then
     wait_service_state "${SERVICE_NAME}" --complete || return $?
   fi
-  return ${RETURN_VALUE}
+  return 0
 }
 
 _container_status() {
@@ -438,9 +445,7 @@ docker_run() {
   local RETRIES=0
   local MAX_RETRIES=5
   local SLEEP_SECONDS=10
-  while ! docker run \
-    "${@}" >/dev/null;
-  do
+  while ! docker run "${@}" >/dev/null; do
     if [ ${RETRIES} -ge ${MAX_RETRIES} ]; then
       echo "Failed to run docker. Reached the max retries ${MAX_RETRIES}." >&2
       return 1

src/lib-gantry.sh

@@ -224,24 +224,21 @@ _remove_images() {
     log INFO "- ${I}"
   done
   local IMAGES_REMOVER=
-  IMAGES_REMOVER=$(_get_service_image "$(_current_service_name)")
+  IMAGES_REMOVER=$(_get_service_image "$(gantry_current_service_name)")
   [ -z "${IMAGES_REMOVER}" ] && IMAGES_REMOVER="${DEFAULT_IMAGES_REMOVER}"
   log DEBUG "Set IMAGES_REMOVER=${IMAGES_REMOVER}"
   local IMAGES_TO_REMOVE_LIST=
   IMAGES_TO_REMOVE_LIST=$(echo "${IMAGES_TO_REMOVE}" | tr '\n' ' ')
   [ -n "${CLEANUP_IMAGES_OPTIONS}" ] && log DEBUG "Adding options \"${CLEANUP_IMAGES_OPTIONS}\" to the global job ${SERVICE_NAME}."
-  local RMI_MSG=
   # SC2086: Double quote to prevent globbing and word splitting.
   # shellcheck disable=SC2086
-  if ! RMI_MSG=$(docker_global_job --name "${SERVICE_NAME}" \
+  docker_global_job --name "${SERVICE_NAME}" \
     --restart-condition on-failure \
     --restart-max-attempts 1 \
     --mount type=bind,source=/var/run/docker.sock,destination=/var/run/docker.sock \
     --env "GANTRY_IMAGES_TO_REMOVE=${IMAGES_TO_REMOVE_LIST}" \
     ${CLEANUP_IMAGES_OPTIONS} \
-    "${IMAGES_REMOVER}"); then
-    log ERROR "Failed to remove images."
-  fi
+    "${IMAGES_REMOVER}";
   wait_service_state "${SERVICE_NAME}"
   docker_service_logs "${SERVICE_NAME}"
   docker_service_remove "${SERVICE_NAME}"
@@ -341,19 +338,26 @@ _current_container_name() {
   local CURRENT_CONTAINER_NAME=
   CURRENT_CONTAINER_NAME=$(_static_variable_read_list STATIC_VAR_CURRENT_CONTAINER_NAME)
   [ -n "${CURRENT_CONTAINER_NAME}" ] && echo "${CURRENT_CONTAINER_NAME}" && return 0
-  local ALL_NETWORKS GWBRIDGE_NETWORK IPS;
+  local ALL_NETWORKS=
   ALL_NETWORKS=$(docker network ls --format '{{.ID}}') || return 1;
   [ -z "${ALL_NETWORKS}" ] && return 0;
-  GWBRIDGE_NETWORK=$(docker network ls --format '{{.ID}}' --filter 'name=docker_gwbridge') || return 1;
+  local IPS=;
   IPS=$(ip route | grep src | sed -n "s/.* src \(\S*\).*$/\1/p");
   [ -z "${IPS}" ] && return 0;
+  local GWBRIDGE_NETWORK HOST_NETWORK;
+  GWBRIDGE_NETWORK=$(docker network ls --format '{{.ID}}' --filter 'name=^docker_gwbridge$') || return 1;
+  HOST_NETWORK=$(docker network ls --format '{{.ID}}' --filter 'name=^host$') || return 1;
   local NID=;
   for NID in ${ALL_NETWORKS}; do
+    # The output of gwbridge does not contain the container name. It looks like gateway_8f55496ce4f1/172.18.0.5/16.
     [ "${NID}" = "${GWBRIDGE_NETWORK}" ] && continue;
+    # The output of host does not contain an IP.
+    [ "${NID}" = "${HOST_NETWORK}" ] && continue;
     local ALL_LOCAL_NAME_AND_IP=;
     ALL_LOCAL_NAME_AND_IP=$(docker network inspect "${NID}" --format "{{range .Containers}}{{.Name}}/{{println .IPv4Address}}{{end}}") || return 1;
     for NAME_AND_IP in ${ALL_LOCAL_NAME_AND_IP}; do
       [ -z "${NAME_AND_IP}" ] && continue;
+      # NAME_AND_IP will be in one of the following formats:
       # '<container name>/<ip>/<mask>'
       # '<container name>/' (when network mode is host)
       local CNAME CIP
@@ -372,7 +376,7 @@ _current_container_name() {
   return 0;
 }
 
-_current_service_name() {
+gantry_current_service_name() {
   local CURRENT_SERVICE_NAME=
   CURRENT_SERVICE_NAME=$(_static_variable_read_list STATIC_VAR_CURRENT_SERVICE_NAME)
   [ -n "${CURRENT_SERVICE_NAME}" ] && echo "${CURRENT_SERVICE_NAME}" && return 0
@@ -389,10 +393,10 @@ _service_is_self() {
   if [ -z "${GANTRY_SERVICES_SELF}" ]; then
     # If _service_is_self is called inside a subprocess, export won't affect the parent process.
     # We only want to log it once, thus try to read the value from a static variable firstly.
-    # The static variable should be set inside the function _current_service_name. If it is set, skip logging.
+    # The static variable should be set inside the function gantry_current_service_name. If it is set, skip logging.
     GANTRY_SERVICES_SELF=$(_static_variable_read_list STATIC_VAR_CURRENT_SERVICE_NAME)
     if [ -z "${GANTRY_SERVICES_SELF}" ]; then
-      GANTRY_SERVICES_SELF=$(_current_service_name)
+      GANTRY_SERVICES_SELF=$(gantry_current_service_name)
       export GANTRY_SERVICES_SELF
       [ -n "${GANTRY_SERVICES_SELF}" ] && log INFO "Set GANTRY_SERVICES_SELF to ${GANTRY_SERVICES_SELF}."
     fi
@@ -830,6 +834,9 @@ gantry_finalize() {
   if ! _report_services; then
     RETURN_VALUE=1
   fi
-  [ -n "${STATIC_VARIABLES_FOLDER}" ] && log DEBUG "Removing STATIC_VARIABLES_FOLDER ${STATIC_VARIABLES_FOLDER}" && rm -r "${STATIC_VARIABLES_FOLDER}"
+  if [ -n "${STATIC_VARIABLES_FOLDER}" ]; then
+    log DEBUG "Removing STATIC_VARIABLES_FOLDER ${STATIC_VARIABLES_FOLDER}"
+    rm -r "${STATIC_VARIABLES_FOLDER}"
+  fi
   return "${RETURN_VALUE}"
 }

tests/gantry_cleanup_images_spec.sh

@@ -98,8 +98,8 @@ Describe 'Cleanup_images'
       The stderr should satisfy spec_expect_no_message "${NO_IMAGES_TO_REMOVE}"
       The stderr should satisfy spec_expect_message    "${REMOVING_NUM_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
-      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*${GANTRY_CLEANUP_IMAGES_OPTIONS}"
-      The stderr should satisfy spec_expect_message    "${FAILED_TO_REMOVE_IMAGE}.*${GANTRY_CLEANUP_IMAGES_OPTIONS}"
+      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--incorrect-option"
+      The stderr should satisfy spec_expect_message    "Failed.*--incorrect-option"
       The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
     End
@@ -141,8 +141,8 @@ Describe 'Cleanup_images'
       The stderr should satisfy spec_expect_no_message "${NO_IMAGES_TO_REMOVE}"
       The stderr should satisfy spec_expect_message    "${REMOVING_NUM_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
-      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*${GANTRY_CLEANUP_IMAGES_OPTIONS}"
-      The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${GANTRY_CLEANUP_IMAGES_OPTIONS}"
+      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--container-label=test"
+      The stderr should satisfy spec_expect_no_message "Failed.*--container-label=test"
       The stderr should satisfy spec_expect_message    "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
     End

tests/gantry_job_spec.sh

@@ -85,7 +85,7 @@ Describe 'Job'
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}"
       The stderr should satisfy spec_expect_message    "${NUM_SERVICES_UPDATING}"
-      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*${GANTRY_UPDATE_OPTIONS}"
+      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--detach=true"
       The stderr should satisfy spec_expect_message    "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"

tests/gantry_manifest_spec.sh

@@ -50,7 +50,7 @@ Describe 'Manifest_command'
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}"
       The stderr should satisfy spec_expect_message    "${NUM_SERVICES_UPDATING}"
-      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*${GANTRY_UPDATE_OPTIONS}"
+      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--force"
       The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${NO_UPDATES}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
@@ -130,7 +130,7 @@ Describe 'Manifest_command'
       The status should be success
       The stdout should satisfy display_output
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*${GANTRY_MANIFEST_OPTIONS}"
+      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--insecure"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}"
@@ -173,8 +173,8 @@ Describe 'Manifest_command'
       The stdout should satisfy display_output
       The stderr should satisfy display_output
       # No options are added to the unknwon command.
-      The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}.*${GANTRY_MANIFEST_OPTIONS}"
-      The stderr should satisfy spec_expect_message    "Unknown MANIFEST_CMD.*${GANTRY_MANIFEST_CMD}"
+      The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}.*--insecure"
+      The stderr should satisfy spec_expect_message    "Unknown MANIFEST_CMD.*unsupported_cmd"
       The stderr should satisfy spec_expect_message    "${SKIP_UPDATING}.*${SERVICE_NAME}.*${SKIP_REASON_MANIFEST_FAILURE}"
       The stderr should satisfy spec_expect_no_message "${PERFORM_UPDATING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}"

tests/gantry_options_spec.sh

@@ -83,7 +83,7 @@ Describe 'Options'
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}"
       The stderr should satisfy spec_expect_message    "${NUM_SERVICES_UPDATING}"
-      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*${GANTRY_UPDATE_OPTIONS}"
+      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--label-add=gantry.test=${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"

tests/gantry_rollback_spec.sh

@@ -93,7 +93,7 @@ Describe 'Rollback'
       The stderr should satisfy spec_expect_message    "${NUM_SERVICES_UPDATING}"
       The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
-      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*${GANTRY_ROLLBACK_OPTIONS}"
+      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--incorrect-option"
       The stderr should satisfy spec_expect_message    "${ROLLING_BACK}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${FAILED_TO_ROLLBACK}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${ROLLED_BACK}.*${SERVICE_NAME}"