#!/usr/bin/env bash set -u set -o pipefail WORK_DIR=${PWD} EXCLUDED_FILE="${WORK_DIR}/excluded_vm.txt" STATUS_FILE="${WORK_DIR}/status.before" LOG_DIR="${WORK_DIR}/log" WAIT_FOR_START=${WAIT_FOR_START:-1} TIMESTAMP=$(date '+%Y-%m-%d_%H_%M') LOG_FILE="${LOG_DIR}/system_update_${TIMESTAMP}.log" SSH_OPTIONS=( -o BatchMode=yes -o ConnectTimeout=10 -o ConnectionAttempts=1 -o StrictHostKeyChecking=accept-new ) if [[ ! "$WAIT_FOR_START" =~ ^[0-9]+$ ]]; then printf 'ERROR: WAIT_FOR_START must be a non-negative integer number of minutes.\n' >&2 exit 2 fi mkdir -p -- "$LOG_DIR" || { printf 'ERROR: cannot create log directory: %s\n' "$LOG_DIR" >&2 exit 1 } exec > >(tee -a "$LOG_FILE") 2>&1 log() { printf '[%s] %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$*" } cleanup_logs() { log "Deleting compressed logs older than 6 months." if ! find "$LOG_DIR" -maxdepth 1 -type f \ -name 'dnf_update_all_*.log.gz' -mtime +180 -delete; then log "ERROR: failed to delete expired compressed logs." fi log "Compressing logs older than 30 days." if ! find "$LOG_DIR" -maxdepth 1 -type f \ -name 'dnf_update_all_*.log' -mtime +30 -exec gzip -- {} +; then log "ERROR: failed to compress old logs." fi } is_excluded() { local vm=$1 [[ -f "$EXCLUDED_FILE" ]] || return 1 awk -v vm="$vm" ' /^[[:space:]]*#/ || /^[[:space:]]*$/ { next } { line = $0 sub(/[[:space:]]*#.*/, "", line) gsub(/^[[:space:]]+|[[:space:]]+$/, "", line) if (line == vm) found = 1 } END { exit(found ? 0 : 1) } ' "$EXCLUDED_FILE" } wait_for_ssh() { local vm=$1 local attempt for attempt in {1..6}; do if ssh "${SSH_OPTIONS[@]}" "root@${vm}" 'true' >/dev/null 2>&1; then return 0 fi [[ "$attempt" -lt 6 ]] && sleep 10 done return 1 } extract_hostname_details() { local hostname_output=$1 local os kernel os=$(awk ' /^[[:space:]]*Operating System:/ { sub(/^[^:]*:[[:space:]]*/, "") print exit } ' <<< "$hostname_output") kernel=$(awk ' /^[[:space:]]*Kernel:/ { sub(/^[^:]*:[[:space:]]*/, "") print exit } ' <<< "$hostname_output") if [[ -z "$os" || -z "$kernel" ]]; then return 1 fi printf '%s\t%s\n' "$os" "$kernel" } record_before_values() { local name=$1 local hostname_output details os kernel if [[ "$name" == "Dom0" ]]; then hostname_output=$(LC_ALL=C hostnamectl 2>&1) || return 1 else hostname_output=$(ssh "${SSH_OPTIONS[@]}" "root@${name}" \ 'LC_ALL=C hostnamectl' 2>&1) || return 1 fi details=$(extract_hostname_details "$hostname_output") || return 1 IFS=$'\t' read -r os kernel <<< "$details" if (( status_count > 0 )); then printf ',\n' >> "$STATUS_FILE" fi if ! python3 -c ' import json import sys json.dump( { "vm_name": sys.argv[1], "before": { "operating_system": sys.argv[2], "kernel": sys.argv[3], }, }, sys.stdout, ensure_ascii=False, ) ' "$name" "$os" "$kernel" >> "$STATUS_FILE"; then return 1 fi status_count=$((status_count + 1)) log "$name: before values recorded in $STATUS_FILE." } shutdown_vm() { local vm=$1 if virsh shutdown "$vm" >/dev/null 2>&1; then log "$vm: shutdown requested." else log "$vm: ERROR: failed to request shutdown." fi } if ! command -v python3 >/dev/null 2>&1; then log "ERROR: python3 is required to write valid JSON to $STATUS_FILE." exit 1 fi cleanup_logs if ! printf '{\n "systems": [\n' > "$STATUS_FILE"; then log "ERROR: cannot create status file: $STATUS_FILE" exit 1 fi status_count=0 if [[ ! -f "$EXCLUDED_FILE" ]]; then log "WARNING: $EXCLUDED_FILE does not exist; no VMs are excluded." fi mapfile -t vms < <(virsh list --all --name) if [[ "${#vms[@]}" -eq 0 ]]; then log "No VMs found." else for vm in "${vms[@]}"; do [[ -n "$vm" ]] || continue if is_excluded "$vm"; then log "$vm: excluded." continue fi log "$vm: processing started." state=$(virsh domstate "$vm" 2>/dev/null | tr -d '\r' | head -n 1) if [[ "$state" != "running" ]]; then if ! virsh start "$vm"; then log "$vm: ERROR: failed to start VM." continue fi log "$vm: started." else log "$vm: already running." fi log "$vm: waiting ${WAIT_FOR_START} minute(s)." sleep "$((WAIT_FOR_START * 60))" if ! wait_for_ssh "$vm"; then log "$vm: ERROR: SSH is not reachable." shutdown_vm "$vm" continue fi if ! record_before_values "$vm"; then log "$vm: ERROR: failed to collect hostnamectl before values." fi log "$vm: running dnf update -y." if ! ssh "${SSH_OPTIONS[@]}" "root@${vm}" \ 'export LC_ALL=C; dnf update -y'; then log "$vm: ERROR: dnf update failed." fi shutdown_vm "$vm" log "$vm: processing finished." done fi log "Dom0: collecting before values." if ! record_before_values "Dom0"; then log "Dom0: ERROR: failed to collect hostnamectl before values." fi log "Dom0: running dnf update -y." if ! LC_ALL=C dnf update -y; then log "Dom0: ERROR: dnf update failed." fi if ! printf '\n ]\n}\n' >> "$STATUS_FILE"; then log "ERROR: failed to finish JSON status file: $STATUS_FILE" exit 1 fi log "System update processing finished. Status saved to $STATUS_FILE."