2026-09-30 15:33:30
This commit is contained in:
support committed
1 parent
5ea2bdbed7
commit
d1dc3376bd
3 files changed
+695
No files matched your search
@@ -0,0 +1,302 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -u
|
||||
set -o pipefail
|
||||
|
||||
WORK_DIR=${PWD}
|
||||
EXCLUDED_FILE="${WORK_DIR}/excluded_vm.txt"
|
||||
STATUS_FILE="${WORK_DIR}/status.before"
|
||||
CHECK_STATUS_FILE="${WORK_DIR}/status.after"
|
||||
LOG_DIR="${WORK_DIR}/log"
|
||||
WAIT_FOR_START=${WAIT_FOR_START:-1}
|
||||
EMAIL_RECIPIENT=${EMAIL_RECIPIENT:-vplesnila@gmail.com}
|
||||
TIMESTAMP=$(date '+%Y-%m-%d_%H_%M')
|
||||
SUMMARY_FILE="${LOG_DIR}/summary_${TIMESTAMP}.log"
|
||||
SSH_OPTIONS=(
|
||||
-o BatchMode=yes
|
||||
-o ConnectTimeout=10
|
||||
-o ConnectionAttempts=1
|
||||
-o StrictHostKeyChecking=accept-new
|
||||
)
|
||||
|
||||
if [[ ! "$WAIT_FOR_START" =~ ^[0-9]+$ ]]; then
|
||||
printf 'ERROR: WAIT_FOR_START must be a non-negative integer number of minutes.\n' >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
if ! command -v python3 >/dev/null 2>&1; then
|
||||
printf 'ERROR: python3 is required to read and write JSON status files.\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v msmtp >/dev/null 2>&1; then
|
||||
printf 'ERROR: msmtp is required to send the summary email.\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ -z "$EMAIL_RECIPIENT" ]]; then
|
||||
printf 'ERROR: EMAIL_RECIPIENT must not be empty.\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! DOM0_HOSTNAME=$(hostnamectl --static); then
|
||||
printf 'ERROR: unable to determine Dom0 hostname for email subject.\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! mkdir -p -- "$LOG_DIR"; then
|
||||
printf 'ERROR: cannot create log directory: %s\n' "$LOG_DIR" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! find "$LOG_DIR" -maxdepth 1 -type f \
|
||||
-name 'summary_*.log' -mmin +86400 -delete; then
|
||||
printf 'ERROR: failed to delete summary logs older than 60 days.\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -f "$STATUS_FILE" ]] || ! python3 - "$STATUS_FILE" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
|
||||
try:
|
||||
with open(sys.argv[1], encoding="utf-8") as status_file:
|
||||
data = json.load(status_file)
|
||||
if not isinstance(data.get("systems"), list):
|
||||
raise ValueError('missing "systems" array')
|
||||
except (OSError, ValueError, json.JSONDecodeError) as error:
|
||||
print(f"ERROR: invalid or unavailable status file: {error}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
PY
|
||||
then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
is_excluded() {
|
||||
local vm=$1
|
||||
|
||||
[[ -f "$EXCLUDED_FILE" ]] || return 1
|
||||
awk -v vm="$vm" '
|
||||
/^[[:space:]]*#/ || /^[[:space:]]*$/ { next }
|
||||
{
|
||||
line = $0
|
||||
sub(/[[:space:]]*#.*/, "", line)
|
||||
gsub(/^[[:space:]]+|[[:space:]]+$/, "", line)
|
||||
if (line == vm) found = 1
|
||||
}
|
||||
END { exit(found ? 0 : 1) }
|
||||
' "$EXCLUDED_FILE"
|
||||
}
|
||||
|
||||
wait_for_ssh() {
|
||||
local vm=$1
|
||||
local attempt
|
||||
|
||||
for attempt in {1..6}; do
|
||||
if ssh "${SSH_OPTIONS[@]}" "root@${vm}" 'true' >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
[[ "$attempt" -lt 6 ]] && sleep 10
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
extract_hostname_details() {
|
||||
local hostname_output=$1
|
||||
local os kernel
|
||||
|
||||
os=$(awk '
|
||||
/^[[:space:]]*Operating System:/ {
|
||||
sub(/^[^:]*:[[:space:]]*/, "")
|
||||
print
|
||||
exit
|
||||
}
|
||||
' <<< "$hostname_output")
|
||||
kernel=$(awk '
|
||||
/^[[:space:]]*Kernel:/ {
|
||||
sub(/^[^:]*:[[:space:]]*/, "")
|
||||
print
|
||||
exit
|
||||
}
|
||||
' <<< "$hostname_output")
|
||||
|
||||
[[ -n "$os" && -n "$kernel" ]] || return 1
|
||||
printf '%s\t%s\n' "$os" "$kernel"
|
||||
}
|
||||
|
||||
record_after_values() {
|
||||
local name=$1
|
||||
local hostname_output details os kernel
|
||||
|
||||
if [[ "$name" == "Dom0" ]]; then
|
||||
hostname_output=$(LC_ALL=C hostnamectl 2>&1) || return 1
|
||||
else
|
||||
hostname_output=$(ssh "${SSH_OPTIONS[@]}" "root@${name}" \
|
||||
'LC_ALL=C hostnamectl' 2>&1) || return 1
|
||||
fi
|
||||
|
||||
details=$(extract_hostname_details "$hostname_output") || return 1
|
||||
IFS=$'\t' read -r os kernel <<< "$details"
|
||||
if (( check_status_count > 0 )); then
|
||||
printf ',\n' >> "$CHECK_STATUS_FILE" || return 1
|
||||
fi
|
||||
python3 -c '
|
||||
import json
|
||||
import sys
|
||||
|
||||
json.dump(
|
||||
{
|
||||
"vm_name": sys.argv[1],
|
||||
"after": {
|
||||
"operating_system": sys.argv[2],
|
||||
"kernel": sys.argv[3],
|
||||
},
|
||||
},
|
||||
sys.stdout,
|
||||
ensure_ascii=False,
|
||||
)
|
||||
' "$name" "$os" "$kernel" >> "$CHECK_STATUS_FILE" || return 1
|
||||
check_status_count=$((check_status_count + 1))
|
||||
printf '%s: after values recorded in %s.\n' "$name" "$CHECK_STATUS_FILE"
|
||||
}
|
||||
|
||||
shutdown_vm() {
|
||||
local vm=$1
|
||||
|
||||
if virsh shutdown "$vm" >/dev/null 2>&1; then
|
||||
printf '%s: shutdown requested.\n' "$vm"
|
||||
else
|
||||
printf '%s: ERROR: failed to request shutdown.\n' "$vm" >&2
|
||||
fi
|
||||
}
|
||||
|
||||
if ! printf '{\n "systems": [\n' > "$CHECK_STATUS_FILE"; then
|
||||
printf 'ERROR: cannot create status file: %s\n' "$CHECK_STATUS_FILE" >&2
|
||||
exit 1
|
||||
fi
|
||||
check_status_count=0
|
||||
|
||||
if [[ ! -f "$EXCLUDED_FILE" ]]; then
|
||||
printf 'WARNING: %s does not exist; no VMs are excluded.\n' "$EXCLUDED_FILE" >&2
|
||||
fi
|
||||
|
||||
mapfile -t vms < <(virsh list --all --name)
|
||||
for vm in "${vms[@]}"; do
|
||||
[[ -n "$vm" ]] || continue
|
||||
|
||||
if is_excluded "$vm"; then
|
||||
printf '%s: excluded.\n' "$vm"
|
||||
continue
|
||||
fi
|
||||
|
||||
printf '%s: processing started.\n' "$vm"
|
||||
state=$(virsh domstate "$vm" 2>/dev/null | tr -d '\r' | head -n 1)
|
||||
if [[ "$state" != "running" ]]; then
|
||||
if ! virsh start "$vm"; then
|
||||
printf '%s: ERROR: failed to start VM.\n' "$vm" >&2
|
||||
continue
|
||||
fi
|
||||
printf '%s: started.\n' "$vm"
|
||||
else
|
||||
printf '%s: already running.\n' "$vm"
|
||||
fi
|
||||
|
||||
printf '%s: waiting %s minute(s).\n' "$vm" "$WAIT_FOR_START"
|
||||
sleep "$((WAIT_FOR_START * 60))"
|
||||
|
||||
if ! wait_for_ssh "$vm"; then
|
||||
printf '%s: ERROR: SSH is not reachable.\n' "$vm" >&2
|
||||
shutdown_vm "$vm"
|
||||
continue
|
||||
fi
|
||||
|
||||
if ! record_after_values "$vm"; then
|
||||
printf '%s: ERROR: failed to collect hostnamectl after values.\n' "$vm" >&2
|
||||
fi
|
||||
shutdown_vm "$vm"
|
||||
printf '%s: processing finished.\n' "$vm"
|
||||
done
|
||||
|
||||
printf 'Dom0: collecting after values.\n'
|
||||
if ! record_after_values "Dom0"; then
|
||||
printf 'Dom0: ERROR: failed to collect hostnamectl after values.\n' >&2
|
||||
fi
|
||||
|
||||
if ! printf '\n ]\n}\n' >> "$CHECK_STATUS_FILE"; then
|
||||
printf 'ERROR: failed to finish JSON status file: %s\n' "$CHECK_STATUS_FILE" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
python3 - "$STATUS_FILE" "$CHECK_STATUS_FILE" <<'PY' | tee "$SUMMARY_FILE"
|
||||
import json
|
||||
import sys
|
||||
|
||||
before_path, after_path = sys.argv[1:]
|
||||
with open(before_path, encoding="utf-8") as file:
|
||||
before_data = json.load(file)
|
||||
with open(after_path, encoding="utf-8") as file:
|
||||
after_data = json.load(file)
|
||||
|
||||
before = {
|
||||
item["vm_name"]: item.get("before", {})
|
||||
for item in before_data["systems"]
|
||||
if isinstance(item, dict) and isinstance(item.get("vm_name"), str)
|
||||
}
|
||||
after = {
|
||||
item["vm_name"]: item.get("after", {})
|
||||
for item in after_data["systems"]
|
||||
if isinstance(item, dict) and isinstance(item.get("vm_name"), str)
|
||||
}
|
||||
|
||||
names = list(dict.fromkeys(
|
||||
[item["vm_name"] for item in before_data["systems"]
|
||||
if isinstance(item, dict) and isinstance(item.get("vm_name"), str)]
|
||||
+ [item["vm_name"] for item in after_data["systems"]
|
||||
if isinstance(item, dict) and isinstance(item.get("vm_name"), str)]
|
||||
))
|
||||
missing = "not available"
|
||||
|
||||
def render(title, key):
|
||||
headers = ["vm_name", f"before {key}", f"after {key}", "Changed"]
|
||||
rows = []
|
||||
for name in names:
|
||||
old_value = before.get(name, {}).get(key, missing)
|
||||
new_value = after.get(name, {}).get(key, missing)
|
||||
rows.append([
|
||||
name,
|
||||
old_value,
|
||||
new_value,
|
||||
"NO" if old_value == new_value else "YES",
|
||||
])
|
||||
widths = [
|
||||
max(len(headers[i]), *(len(str(row[i])) for row in rows))
|
||||
if rows else len(headers[i])
|
||||
for i in range(len(headers))
|
||||
]
|
||||
print(f"\n{title}")
|
||||
print(" | ".join(headers[i].ljust(widths[i]) for i in range(4)))
|
||||
print("-+-".join("-" * width for width in widths))
|
||||
for row in rows:
|
||||
print(" | ".join(str(row[i]).ljust(widths[i]) for i in range(4)))
|
||||
|
||||
render("operating_system", "operating_system")
|
||||
render("kernel", "kernel")
|
||||
PY
|
||||
|
||||
if ! {
|
||||
printf 'To: %s\n' "$EMAIL_RECIPIENT"
|
||||
printf 'Subject: Update system on %s\n' "$DOM0_HOSTNAME"
|
||||
printf 'MIME-Version: 1.0\n'
|
||||
printf 'Content-Type: text/html; charset=UTF-8\n'
|
||||
printf '\n'
|
||||
printf '<!doctype html><html><body><pre style="font-family: monospace; white-space: pre;">'
|
||||
python3 -c 'import html, pathlib, sys; print(html.escape(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8")), end="")' \
|
||||
"$SUMMARY_FILE"
|
||||
printf '</pre></body></html>\n'
|
||||
} | msmtp -t; then
|
||||
printf 'ERROR: failed to send summary email to %s.\n' "$EMAIL_RECIPIENT" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
printf 'Summary email accepted by msmtp for submission to %s using %s; final delivery is not confirmed.\n' \
|
||||
"$EMAIL_RECIPIENT" "$SUMMARY_FILE"
|
||||
Reference in new issue
Block a user