Skip to content

Commit

Permalink
[gantry] inspect and update services in two loops
Browse files Browse the repository at this point in the history
  • Loading branch information
@shizunge
shizunge committed Feb 6, 2024
1 parent 07035c5 commit b384d60
Show file tree
Hide file tree
Showing 2 changed files with 68 additions and 60 deletions.
104 changes: 56 additions & 48 deletions src/lib-gantry.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,11 +126,6 @@ _static_variable_add_unique_to_list() {
echo "${NEW_LIST}" > "${FILE_NAME}"
}

_add_image_to_remove() {
local IMAGE="${1}"
_static_variable_add_unique_to_list STATIC_VAR_IMAGES_TO_REMOVE "${IMAGE}"
}

_remove_container() {
local IMAGE="${1}";
local STATUS="${2}";
Expand Down Expand Up @@ -219,11 +214,6 @@ _remove_images() {
docker_service_remove "${SERVICE_NAME}"
}

_add_service_updated() {
local SERVICE_NAME="${1}"
_static_variable_add_unique_to_list STATIC_VAR_SERVICES_UPDATED "${SERVICE_NAME}"
}

_report_services_updated() {
local SERVICES_UPDATED
SERVICES_UPDATED=$(_static_variable_read_list STATIC_VAR_SERVICES_UPDATED)
Expand Down Expand Up @@ -413,6 +403,20 @@ _get_config_from_service() {
echo "--config ${AUTH_CONFIG}"
}

_skip_jobs() {
local UPDATE_JOBS="${GANTRY_UPDATE_JOBS:-"false"}"
local SERVICE_NAME="${1}"
if is_true "${UPDATE_JOBS}"; then
return 1
fi
local MODE=
if MODE=$(_service_is_job "${SERVICE_NAME}"); then
log DEBUG "Skip updating ${SERVICE_NAME} because it is in ${MODE} mode."
return 0
fi
return 1
}

_get_image_info() {
local MANIFEST_OPTIONS="${GANTRY_MANIFEST_OPTIONS:-""}"
local MANIFEST_CMD="${1}"
Expand Down Expand Up @@ -448,27 +452,12 @@ _get_image_info() {
return 0
}

_skip_jobs() {
local UPDATE_JOBS="${GANTRY_UPDATE_JOBS:-"false"}"
local SERVICE_NAME="${1}"
if is_true "${UPDATE_JOBS}"; then
return 1
fi
local MODE=
if MODE=$(_service_is_job "${SERVICE_NAME}"); then
log DEBUG "Skip updating ${SERVICE_NAME} because it is in ${MODE} mode."
return 0
fi
return 1
}

# echo nothing if we found no new images.
# echo the image if we found a new image.
# return the number of errors.
_inspect_image() {
local MANIFEST_CMD="${GANTRY_MANIFEST_CMD:-"buildx"}"
local SERVICE_NAME="${1}"
local DOCKER_CONFIG="${2}"
local IMAGE_WITH_DIGEST=
if ! IMAGE_WITH_DIGEST=$(_get_service_image "${SERVICE_NAME}" 2>&1); then
log ERROR "Failed to obtain image from service ${SERVICE_NAME}. ${IMAGE_WITH_DIGEST}"
Expand Down Expand Up @@ -501,6 +490,9 @@ _inspect_image() {
echo "${IMAGE}"
return 0
fi
local DOCKER_CONFIG=
DOCKER_CONFIG=$(_get_config_from_service "${SERVICE}")
[ -n "${DOCKER_CONFIG}" ] && log DEBUG "Adding options \"${DOCKER_CONFIG}\" to docker commands."
local IMAGE_INFO=
if ! IMAGE_INFO=$(_get_image_info "${MANIFEST_CMD}" "${IMAGE}" "${DOCKER_CONFIG}"); then
log DEBUG "Skip updating ${SERVICE_NAME} because there is a failure to obtain the manifest from the registry of image ${IMAGE}."
Expand All @@ -518,6 +510,26 @@ _inspect_image() {
return 0
}

# return 0 if need to update the service
# return 1 if no need to update the service
_inspect_service() {
local SERVICE_NAME="${1}"
if _skip_jobs "${SERVICE_NAME}"; then
return 1
fi
local IMAGE=
if ! IMAGE=$(_inspect_image "${SERVICE_NAME}"); then
_add_service_update_failed "${SERVICE_NAME}"
return 1
fi
if [ -z "${IMAGE}" ]; then
log INFO "No new images for ${SERVICE_NAME}."
return 1
fi
_static_variable_add_unique_to_list STATIC_VAR_SERVICES_TO_UPDATE "${SERVICE_NAME} ${IMAGE}"
return 0
}

_get_number_of_running_tasks() {
local SERVICE_NAME="${1}"
local REPLICAS=
Expand Down Expand Up @@ -584,15 +596,19 @@ _update_single_service() {
local UPDATE_OPTIONS="${GANTRY_UPDATE_OPTIONS:-""}"
if ! is_number "${UPDATE_TIMEOUT_SECONDS}"; then
log ERROR "GANTRY_UPDATE_TIMEOUT_SECONDS must be a number. Got \"${GANTRY_UPDATE_TIMEOUT_SECONDS}\"."
_add_service_update_failed "${SERVICE}"
return 1;
fi
local SERVICE_NAME="${1}"
local IMAGE="${2}"
local DOCKER_CONFIG="${3}"
[ -z "${SERVICE_NAME}" ] && log ERROR "Updating service: SERVICE_NAME must not be empty." && return 1
[ -z "${IMAGE}" ] && log ERROR "Updating ${SERVICE_NAME}: IMAGE must not be empty." && return 1
log INFO "Updating ${SERVICE_NAME} with image ${IMAGE}"
local DOCKER_CONFIG=
local ADDITIONAL_OPTIONS=
DOCKER_CONFIG=$(_get_config_from_service "${SERVICE}")
ADDITIONAL_OPTIONS=$(_get_service_update_additional_options "${SERVICE_NAME}")
[ -n "${DOCKER_CONFIG}" ] && log DEBUG "Adding options \"${DOCKER_CONFIG}\" to docker commands."
[ -n "${ADDITIONAL_OPTIONS}" ] && log DEBUG "Adding options \"${ADDITIONAL_OPTIONS}\" to the command \"docker service update\"."
[ -n "${UPDATE_OPTIONS}" ] && log DEBUG "Adding options \"${UPDATE_OPTIONS}\" to the command \"docker service update\"."
local UPDATE_MSG=
Expand All @@ -602,6 +618,7 @@ _update_single_service() {
if ! UPDATE_MSG=$(timeout "${UPDATE_TIMEOUT_SECONDS}" docker ${DOCKER_CONFIG} service update --quiet ${ADDITIONAL_OPTIONS} ${UPDATE_OPTIONS} --image="${IMAGE}" "${SERVICE_NAME}" 2>&1); then
log ERROR "docker service update failed or timeout. ${UPDATE_MSG}"
_rollback_service "${SERVICE_NAME}" "${DOCKER_CONFIG}"
_add_service_update_failed "${SERVICE}"
return 1
fi
local PREVIOUS_IMAGE=
Expand All @@ -612,8 +629,8 @@ _update_single_service() {
log INFO "No updates for ${SERVICE_NAME}."
return 0
fi
_add_service_updated "${SERVICE_NAME}"
_add_image_to_remove "${PREVIOUS_IMAGE}"
_static_variable_add_unique_to_list STATIC_VAR_SERVICES_UPDATED "${SERVICE_NAME}"
_static_variable_add_unique_to_list STATIC_VAR_IMAGES_TO_REMOVE "${PREVIOUS_IMAGE}"
log INFO "UPDATED ${SERVICE_NAME}."
return 0
}
Expand Down Expand Up @@ -688,27 +705,18 @@ gantry_update_services_list() {
local LIST="${*}"
local LOG_SCOPE_SAVED="${LOG_SCOPE}"
for SERVICE in ${LIST}; do
export LOG_SCOPE="Updating ${SERVICE}"
if _skip_jobs "${SERVICE}"; then
continue;
fi
local DOCKER_CONFIG=
DOCKER_CONFIG=$(_get_config_from_service "${SERVICE}")
[ -n "${DOCKER_CONFIG}" ] && log DEBUG "Adding options \"${DOCKER_CONFIG}\" to docker commands."
local IMAGE=
if ! IMAGE=$(_inspect_image "${SERVICE}" "${DOCKER_CONFIG}"); then
_add_service_update_failed "${SERVICE}"
continue;
fi
if [ -z "${IMAGE}" ]; then
log INFO "No new images for ${SERVICE}."
continue;
fi
if ! _update_single_service "${SERVICE}" "${IMAGE}" "${DOCKER_CONFIG}"; then
_add_service_update_failed "${SERVICE}"
continue;
fi
export LOG_SCOPE="Inspecting ${SERVICE}"
_inspect_service "${SERVICE}"
done
local SERVICES_TO_UPDATE=
SERVICES_TO_UPDATE=$(_static_variable_read_list STATIC_VAR_SERVICES_TO_UPDATE)
while read -r SERVICE_AND_IMAGE; do
export LOG_SCOPE="Updating ${SERVICE}"
local SERVICE IMAGE
SERVICE=$(echo "${SERVICE_AND_IMAGE}" | cut -d ' ' -f 1)
IMAGE=$(echo "${SERVICE_AND_IMAGE}" | cut -d ' ' -f 2)
_update_single_service "${SERVICE}" "${IMAGE}"
done < <(echo "${SERVICES_TO_UPDATE}")
export LOG_SCOPE="${LOG_SCOPE_SAVED}"
local SERVICES_UPDATE_FAILED FAILED_NUM
SERVICES_UPDATE_FAILED=$(_static_variable_read_list STATIC_VAR_SERVICES_UPDATE_FAILED)
Expand Down
24 changes: 12 additions & 12 deletions tests/gantry_multiple_services_spec.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,18 +97,18 @@ Describe 'Multiple_services'
The stderr should satisfy spec_expect_no_message "${PERFORM_UPDATING}.*${SERVICE_NAME4}"
The stderr should satisfy spec_expect_message "${SKIP_UPDATING}.*${SERVICE_NAME5}.*${SKIP_REASON_NO_KNOWN_NEWER_IMAGE}"
The stderr should satisfy spec_expect_no_message "${PERFORM_UPDATING}.*${SERVICE_NAME5}"
The stderr should satisfy spec_expect_no_message "${SERVICE_NAME0}.*${NO_NEW_IMAGE}"
The stderr should satisfy spec_expect_no_message "${SERVICE_NAME1}.*${NO_NEW_IMAGE}"
The stderr should satisfy spec_expect_no_message "${SERVICE_NAME2}.*${NO_NEW_IMAGE}"
The stderr should satisfy spec_expect_no_message "${SERVICE_NAME3}.*${NO_NEW_IMAGE}"
The stderr should satisfy spec_expect_message "${SERVICE_NAME4}.*${NO_NEW_IMAGE}"
The stderr should satisfy spec_expect_message "${SERVICE_NAME5}.*${NO_NEW_IMAGE}"
The stderr should satisfy spec_expect_message "${SERVICE_NAME0}.*${UPDATED}"
The stderr should satisfy spec_expect_no_message "${SERVICE_NAME1}.*${UPDATED}"
The stderr should satisfy spec_expect_no_message "${SERVICE_NAME2}.*${UPDATED}"
The stderr should satisfy spec_expect_message "${SERVICE_NAME3}.*${UPDATED}"
The stderr should satisfy spec_expect_no_message "${SERVICE_NAME4}.*${UPDATED}"
The stderr should satisfy spec_expect_no_message "${SERVICE_NAME5}.*${UPDATED}"
The stderr should satisfy spec_expect_no_message "${NO_NEW_IMAGE}.*${SERVICE_NAME0}"
The stderr should satisfy spec_expect_no_message "${NO_NEW_IMAGE}.*${SERVICE_NAME1}"
The stderr should satisfy spec_expect_no_message "${NO_NEW_IMAGE}.*${SERVICE_NAME2}"
The stderr should satisfy spec_expect_no_message "${NO_NEW_IMAGE}.*${SERVICE_NAME3}"
The stderr should satisfy spec_expect_message "${NO_NEW_IMAGE}.*${SERVICE_NAME4}"
The stderr should satisfy spec_expect_message "${NO_NEW_IMAGE}.*${SERVICE_NAME5}"
The stderr should satisfy spec_expect_message "${UPDATED}.*${SERVICE_NAME0}"
The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME1}"
The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME2}"
The stderr should satisfy spec_expect_message "${UPDATED}.*${SERVICE_NAME3}"
The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME4}"
The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME5}"
The stderr should satisfy spec_expect_no_message "${NO_SERVICES_UPDATED}"
The stderr should satisfy spec_expect_message "${NUM_SERVICES_UPDATED}"
The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATE_FAILED}"
Expand Down

0 comments on commit b384d60

Please sign in to comment.