programming/KVM_system_update/system_update.sh

#!/usr/bin/env bash

set -u
set -o pipefail

WORK_DIR=${PWD}
EXCLUDED_FILE="${WORK_DIR}/excluded_vm.txt"
STATUS_FILE="${WORK_DIR}/status.before"
LOG_DIR="${WORK_DIR}/log"
WAIT_FOR_START=${WAIT_FOR_START:-1}
TIMESTAMP=$(date '+%Y-%m-%d_%H_%M')
LOG_FILE="${LOG_DIR}/system_update_${TIMESTAMP}.log"
SSH_OPTIONS=(
    -o BatchMode=yes
    -o ConnectTimeout=10
    -o ConnectionAttempts=1
    -o StrictHostKeyChecking=accept-new
)

if [[ ! "$WAIT_FOR_START" =~ ^[0-9]+$ ]]; then
    printf 'ERROR: WAIT_FOR_START must be a non-negative integer number of minutes.\n' >&2
    exit 2
fi

mkdir -p -- "$LOG_DIR" || {
    printf 'ERROR: cannot create log directory: %s\n' "$LOG_DIR" >&2
    exit 1
}

exec > >(tee -a "$LOG_FILE") 2>&1

log() {
    printf '[%s] %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$*"
}

cleanup_logs() {
    log "Deleting compressed logs older than 6 months."
    if ! find "$LOG_DIR" -maxdepth 1 -type f \
        -name 'dnf_update_all_*.log.gz' -mtime +180 -delete; then
        log "ERROR: failed to delete expired compressed logs."
    fi

    log "Compressing logs older than 30 days."
    if ! find "$LOG_DIR" -maxdepth 1 -type f \
        -name 'dnf_update_all_*.log' -mtime +30 -exec gzip -- {} +; then
        log "ERROR: failed to compress old logs."
    fi
}

is_excluded() {
    local vm=$1

    [[ -f "$EXCLUDED_FILE" ]] || return 1
    awk -v vm="$vm" '
        /^[[:space:]]*#/ || /^[[:space:]]*$/ { next }
        {
            line = $0
            sub(/[[:space:]]*#.*/, "", line)
            gsub(/^[[:space:]]+|[[:space:]]+$/, "", line)
            if (line == vm) found = 1
        }
        END { exit(found ? 0 : 1) }
    ' "$EXCLUDED_FILE"
}

wait_for_ssh() {
    local vm=$1
    local attempt

    for attempt in {1..6}; do
        if ssh "${SSH_OPTIONS[@]}" "root@${vm}" 'true' >/dev/null 2>&1; then
            return 0
        fi
        [[ "$attempt" -lt 6 ]] && sleep 10
    done
    return 1
}

extract_hostname_details() {
    local hostname_output=$1
    local os kernel

    os=$(awk '
        /^[[:space:]]*Operating System:/ {
            sub(/^[^:]*:[[:space:]]*/, "")
            print
            exit
        }
    ' <<< "$hostname_output")
    kernel=$(awk '
        /^[[:space:]]*Kernel:/ {
            sub(/^[^:]*:[[:space:]]*/, "")
            print
            exit
        }
    ' <<< "$hostname_output")

    if [[ -z "$os" || -z "$kernel" ]]; then
        return 1
    fi
    printf '%s\t%s\n' "$os" "$kernel"
}

record_before_values() {
    local name=$1
    local hostname_output details os kernel

    if [[ "$name" == "Dom0" ]]; then
        hostname_output=$(LC_ALL=C hostnamectl 2>&1) || return 1
    else
        hostname_output=$(ssh "${SSH_OPTIONS[@]}" "root@${name}" \
            'LC_ALL=C hostnamectl' 2>&1) || return 1
    fi

    details=$(extract_hostname_details "$hostname_output") || return 1
    IFS=$'\t' read -r os kernel <<< "$details"
    if (( status_count > 0 )); then
        printf ',\n' >> "$STATUS_FILE"
    fi
    if ! python3 -c '
import json
import sys

json.dump(
    {
        "vm_name": sys.argv[1],
        "before": {
            "operating_system": sys.argv[2],
            "kernel": sys.argv[3],
        },
    },
    sys.stdout,
    ensure_ascii=False,
)
' "$name" "$os" "$kernel" >> "$STATUS_FILE"; then
        return 1
    fi
    status_count=$((status_count + 1))
    log "$name: before values recorded in $STATUS_FILE."
}

shutdown_vm() {
    local vm=$1

    if virsh shutdown "$vm" >/dev/null 2>&1; then
        log "$vm: shutdown requested."
    else
        log "$vm: ERROR: failed to request shutdown."
    fi
}

if ! command -v python3 >/dev/null 2>&1; then
    log "ERROR: python3 is required to write valid JSON to $STATUS_FILE."
    exit 1
fi

cleanup_logs

if ! printf '{\n  "systems": [\n' > "$STATUS_FILE"; then
    log "ERROR: cannot create status file: $STATUS_FILE"
    exit 1
fi
status_count=0

if [[ ! -f "$EXCLUDED_FILE" ]]; then
    log "WARNING: $EXCLUDED_FILE does not exist; no VMs are excluded."
fi

mapfile -t vms < <(virsh list --all --name)
if [[ "${#vms[@]}" -eq 0 ]]; then
    log "No VMs found."
else
    for vm in "${vms[@]}"; do
        [[ -n "$vm" ]] || continue

        if is_excluded "$vm"; then
            log "$vm: excluded."
            continue
        fi

        log "$vm: processing started."
        state=$(virsh domstate "$vm" 2>/dev/null | tr -d '\r' | head -n 1)
        if [[ "$state" != "running" ]]; then
            if ! virsh start "$vm"; then
                log "$vm: ERROR: failed to start VM."
                continue
            fi
            log "$vm: started."
        else
            log "$vm: already running."
        fi

        log "$vm: waiting ${WAIT_FOR_START} minute(s)."
        sleep "$((WAIT_FOR_START * 60))"

        if ! wait_for_ssh "$vm"; then
            log "$vm: ERROR: SSH is not reachable."
            shutdown_vm "$vm"
            continue
        fi

        if ! record_before_values "$vm"; then
            log "$vm: ERROR: failed to collect hostnamectl before values."
        fi

        log "$vm: running dnf update -y."
        if ! ssh "${SSH_OPTIONS[@]}" "root@${vm}" \
            'export LC_ALL=C; dnf update -y'; then
            log "$vm: ERROR: dnf update failed."
        fi

        shutdown_vm "$vm"
        log "$vm: processing finished."
    done
fi

log "Dom0: collecting before values."
if ! record_before_values "Dom0"; then
    log "Dom0: ERROR: failed to collect hostnamectl before values."
fi

log "Dom0: running dnf update -y."
if ! LC_ALL=C dnf update -y; then
    log "Dom0: ERROR: dnf update failed."
fi

if ! printf '\n  ]\n}\n' >> "$STATUS_FILE"; then
    log "ERROR: failed to finish JSON status file: $STATUS_FILE"
    exit 1
fi

log "System update processing finished. Status saved to $STATUS_FILE."