#!/usr/bin/env bash set -u set -o pipefail WORK_DIR=${PWD} EXCLUDED_FILE="${WORK_DIR}/excluded_vm.txt" STATUS_FILE="${WORK_DIR}/status.before" CHECK_STATUS_FILE="${WORK_DIR}/status.after" LOG_DIR="${WORK_DIR}/log" WAIT_FOR_START=${WAIT_FOR_START:-1} EMAIL_RECIPIENT=${EMAIL_RECIPIENT:-vplesnila@gmail.com} TIMESTAMP=$(date '+%Y-%m-%d_%H_%M') SUMMARY_FILE="${LOG_DIR}/summary_${TIMESTAMP}.log" SSH_OPTIONS=( -o BatchMode=yes -o ConnectTimeout=10 -o ConnectionAttempts=1 -o StrictHostKeyChecking=accept-new ) if [[ ! "$WAIT_FOR_START" =~ ^[0-9]+$ ]]; then printf 'ERROR: WAIT_FOR_START must be a non-negative integer number of minutes.\n' >&2 exit 2 fi if ! command -v python3 >/dev/null 2>&1; then printf 'ERROR: python3 is required to read and write JSON status files.\n' >&2 exit 1 fi if ! command -v msmtp >/dev/null 2>&1; then printf 'ERROR: msmtp is required to send the summary email.\n' >&2 exit 1 fi if [[ -z "$EMAIL_RECIPIENT" ]]; then printf 'ERROR: EMAIL_RECIPIENT must not be empty.\n' >&2 exit 1 fi if ! DOM0_HOSTNAME=$(hostnamectl --static); then printf 'ERROR: unable to determine Dom0 hostname for email subject.\n' >&2 exit 1 fi if ! mkdir -p -- "$LOG_DIR"; then printf 'ERROR: cannot create log directory: %s\n' "$LOG_DIR" >&2 exit 1 fi if ! find "$LOG_DIR" -maxdepth 1 -type f \ -name 'summary_*.log' -mmin +86400 -delete; then printf 'ERROR: failed to delete summary logs older than 60 days.\n' >&2 exit 1 fi if [[ ! -f "$STATUS_FILE" ]] || ! python3 - "$STATUS_FILE" <<'PY' import json import sys try: with open(sys.argv[1], encoding="utf-8") as status_file: data = json.load(status_file) if not isinstance(data.get("systems"), list): raise ValueError('missing "systems" array') except (OSError, ValueError, json.JSONDecodeError) as error: print(f"ERROR: invalid or unavailable status file: {error}", file=sys.stderr) sys.exit(1) PY then exit 1 fi is_excluded() { local vm=$1 [[ -f "$EXCLUDED_FILE" ]] || return 1 awk -v vm="$vm" ' /^[[:space:]]*#/ || /^[[:space:]]*$/ { next } { line = $0 sub(/[[:space:]]*#.*/, "", line) gsub(/^[[:space:]]+|[[:space:]]+$/, "", line) if (line == vm) found = 1 } END { exit(found ? 0 : 1) } ' "$EXCLUDED_FILE" } wait_for_ssh() { local vm=$1 local attempt for attempt in {1..6}; do if ssh "${SSH_OPTIONS[@]}" "root@${vm}" 'true' >/dev/null 2>&1; then return 0 fi [[ "$attempt" -lt 6 ]] && sleep 10 done return 1 } extract_hostname_details() { local hostname_output=$1 local os kernel os=$(awk ' /^[[:space:]]*Operating System:/ { sub(/^[^:]*:[[:space:]]*/, "") print exit } ' <<< "$hostname_output") kernel=$(awk ' /^[[:space:]]*Kernel:/ { sub(/^[^:]*:[[:space:]]*/, "") print exit } ' <<< "$hostname_output") [[ -n "$os" && -n "$kernel" ]] || return 1 printf '%s\t%s\n' "$os" "$kernel" } record_after_values() { local name=$1 local hostname_output details os kernel if [[ "$name" == "Dom0" ]]; then hostname_output=$(LC_ALL=C hostnamectl 2>&1) || return 1 else hostname_output=$(ssh "${SSH_OPTIONS[@]}" "root@${name}" \ 'LC_ALL=C hostnamectl' 2>&1) || return 1 fi details=$(extract_hostname_details "$hostname_output") || return 1 IFS=$'\t' read -r os kernel <<< "$details" if (( check_status_count > 0 )); then printf ',\n' >> "$CHECK_STATUS_FILE" || return 1 fi python3 -c ' import json import sys json.dump( { "vm_name": sys.argv[1], "after": { "operating_system": sys.argv[2], "kernel": sys.argv[3], }, }, sys.stdout, ensure_ascii=False, ) ' "$name" "$os" "$kernel" >> "$CHECK_STATUS_FILE" || return 1 check_status_count=$((check_status_count + 1)) printf '%s: after values recorded in %s.\n' "$name" "$CHECK_STATUS_FILE" } shutdown_vm() { local vm=$1 if virsh shutdown "$vm" >/dev/null 2>&1; then printf '%s: shutdown requested.\n' "$vm" else printf '%s: ERROR: failed to request shutdown.\n' "$vm" >&2 fi } if ! printf '{\n "systems": [\n' > "$CHECK_STATUS_FILE"; then printf 'ERROR: cannot create status file: %s\n' "$CHECK_STATUS_FILE" >&2 exit 1 fi check_status_count=0 if [[ ! -f "$EXCLUDED_FILE" ]]; then printf 'WARNING: %s does not exist; no VMs are excluded.\n' "$EXCLUDED_FILE" >&2 fi mapfile -t vms < <(virsh list --all --name) for vm in "${vms[@]}"; do [[ -n "$vm" ]] || continue if is_excluded "$vm"; then printf '%s: excluded.\n' "$vm" continue fi printf '%s: processing started.\n' "$vm" state=$(virsh domstate "$vm" 2>/dev/null | tr -d '\r' | head -n 1) if [[ "$state" != "running" ]]; then if ! virsh start "$vm"; then printf '%s: ERROR: failed to start VM.\n' "$vm" >&2 continue fi printf '%s: started.\n' "$vm" else printf '%s: already running.\n' "$vm" fi printf '%s: waiting %s minute(s).\n' "$vm" "$WAIT_FOR_START" sleep "$((WAIT_FOR_START * 60))" if ! wait_for_ssh "$vm"; then printf '%s: ERROR: SSH is not reachable.\n' "$vm" >&2 shutdown_vm "$vm" continue fi if ! record_after_values "$vm"; then printf '%s: ERROR: failed to collect hostnamectl after values.\n' "$vm" >&2 fi shutdown_vm "$vm" printf '%s: processing finished.\n' "$vm" done printf 'Dom0: collecting after values.\n' if ! record_after_values "Dom0"; then printf 'Dom0: ERROR: failed to collect hostnamectl after values.\n' >&2 fi if ! printf '\n ]\n}\n' >> "$CHECK_STATUS_FILE"; then printf 'ERROR: failed to finish JSON status file: %s\n' "$CHECK_STATUS_FILE" >&2 exit 1 fi python3 - "$STATUS_FILE" "$CHECK_STATUS_FILE" <<'PY' | tee "$SUMMARY_FILE" import json import sys before_path, after_path = sys.argv[1:] with open(before_path, encoding="utf-8") as file: before_data = json.load(file) with open(after_path, encoding="utf-8") as file: after_data = json.load(file) before = { item["vm_name"]: item.get("before", {}) for item in before_data["systems"] if isinstance(item, dict) and isinstance(item.get("vm_name"), str) } after = { item["vm_name"]: item.get("after", {}) for item in after_data["systems"] if isinstance(item, dict) and isinstance(item.get("vm_name"), str) } names = list(dict.fromkeys( [item["vm_name"] for item in before_data["systems"] if isinstance(item, dict) and isinstance(item.get("vm_name"), str)] + [item["vm_name"] for item in after_data["systems"] if isinstance(item, dict) and isinstance(item.get("vm_name"), str)] )) missing = "not available" def render(title, key): headers = ["vm_name", f"before {key}", f"after {key}", "Changed"] rows = [] for name in names: old_value = before.get(name, {}).get(key, missing) new_value = after.get(name, {}).get(key, missing) rows.append([ name, old_value, new_value, "NO" if old_value == new_value else "YES", ]) widths = [ max(len(headers[i]), *(len(str(row[i])) for row in rows)) if rows else len(headers[i]) for i in range(len(headers)) ] print(f"\n{title}") print(" | ".join(headers[i].ljust(widths[i]) for i in range(4))) print("-+-".join("-" * width for width in widths)) for row in rows: print(" | ".join(str(row[i]).ljust(widths[i]) for i in range(4))) render("operating_system", "operating_system") render("kernel", "kernel") PY if ! { printf 'To: %s\n' "$EMAIL_RECIPIENT" printf 'Subject: Update system on %s\n' "$DOM0_HOSTNAME" printf 'MIME-Version: 1.0\n' printf 'Content-Type: text/html; charset=UTF-8\n' printf '\n' printf '
'
python3 -c 'import html, pathlib, sys; print(html.escape(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8")), end="")' \
"$SUMMARY_FILE"
printf '\n'
} | msmtp -t; then
printf 'ERROR: failed to send summary email to %s.\n' "$EMAIL_RECIPIENT" >&2
exit 1
fi
printf 'Summary email accepted by msmtp for submission to %s using %s; final delivery is not confirmed.\n' \
"$EMAIL_RECIPIENT" "$SUMMARY_FILE"