From d1dc3376bd0e06577668eb421d94b8ce441015ac Mon Sep 17 00:00:00 2001 From: support Date: Wed, 30 Sep 2026 17:33:31 +0200 Subject: [PATCH] 2026-09-30 15:33:30 --- KVM_system_update/Readme.md | 161 +++++++++++++++ KVM_system_update/check_update.sh | 302 +++++++++++++++++++++++++++++ KVM_system_update/system_update.sh | 232 ++++++++++++++++++++++ 3 files changed, 695 insertions(+) create mode 100644 KVM_system_update/Readme.md create mode 100644 KVM_system_update/check_update.sh create mode 100644 KVM_system_update/system_update.sh diff --git a/KVM_system_update/Readme.md b/KVM_system_update/Readme.md new file mode 100644 index 0000000..78743da --- /dev/null +++ b/KVM_system_update/Readme.md @@ -0,0 +1,161 @@ +# Dom0 System Update Scripts + +These scripts update Linux guests managed by KVM on Dom0, collect OS and +kernel information before and after the updates, compare the results, and +email a summary. + +## Requirements + +Run the scripts as `root` on Dom0, from the directory containing the scripts +and `excluded_vm.txt`. The scripts use the current working directory for their +input and output files. + +Required tools: + +- Bash +- `virsh` (libvirt/KVM) +- `ssh` and working root SSH access from Dom0 to each VM +- `hostnamectl` +- `dnf` +- `python3` +- `tee`, `find`, and `gzip` +- `msmtp` (for `check_update.sh`) + +The scripts expect eligible guests to be Linux systems that support +`hostnamectl` and `dnf`. Exclude Windows guests. + +## Configuration and input + +### `excluded_vm.txt` + +Place one VM name per line. Lines beginning with `#`, blank lines, and inline +comments are ignored. VM names must match the names reported by `virsh` and +must also resolve as SSH hostnames, for example `linux-vm.example.com`. + +If the file is missing, the scripts warn and process all discovered VMs. + +### `WAIT_FOR_START` + +Both scripts wait after each VM is confirmed started, even if it was already +running. Set `WAIT_FOR_START` to a non-negative integer number of minutes; the +default is `1`. + +```bash +WAIT_FOR_START=5 ./system_update.sh +WAIT_FOR_START=5 ./check_update.sh +``` + +After this wait, the scripts check SSH connectivity with up to six attempts, +spaced 10 seconds apart. + +### `EMAIL_RECIPIENT` + +`check_update.sh` sends the summary to `vplesnila@gmail.com` by default. Override +the address with an environment variable: + +```bash +EMAIL_RECIPIENT=admin@example.com ./check_update.sh +``` + +Email is sent by `msmtp -t`; configure msmtp on Dom0 separately. The message +subject is `Update system on `, and its HTML body displays +the summary tables in a monospace `
` block.
+
+## Scripts and recommended order
+
+### `system_update.sh`
+
+Run this script first:
+
+```bash
+./system_update.sh
+```
+
+For every non-excluded VM returned by `virsh list --all --name`, it starts the
+VM if necessary, waits `WAIT_FOR_START` minutes, checks SSH, and records the
+VM's `Operating System` and `Kernel` values from `hostnamectl` **before** the
+update. It then runs `dnf update -y` over SSH and requests a graceful VM
+shutdown. After all VMs, it records Dom0's before values and runs `dnf update
+-y` on Dom0. This script does not reboot Dom0.
+
+The script overwrites `status.before` once at the beginning of each execution.
+It contains a JSON object with a `systems` array. Each collected system has
+this shape:
+
+```json
+{
+  "vm_name": "linux-vm.example.com",
+  "before": {
+    "operating_system": "Example Linux 9",
+    "kernel": "5.14.0-example"
+  }
+}
+```
+
+Dom0 is recorded with `"vm_name": "Dom0"`. A VM whose startup, SSH connection,
+or hostname query fails may not have a record in the file. Update failures are
+logged; the script continues processing.
+
+The script writes operational output to `log/system_update_YYYY-MM-DD_HH_MM.log`.
+
+### `check_update.sh`
+
+Run this after `system_update.sh` has successfully created `status.before`:
+
+```bash
+./check_update.sh
+```
+
+For each non-excluded VM, it starts the VM if needed, waits
+`WAIT_FOR_START` minutes, checks SSH, records its `hostnamectl` values as
+**after** values, and requests a graceful shutdown. It then records Dom0's
+after values. It does not run updates.
+
+Each run overwrites `status.after`, which uses the same `systems` array format
+as `status.before`, with an `after` object instead of a `before` object:
+
+```json
+{
+  "vm_name": "linux-vm.example.com",
+  "after": {
+    "operating_system": "Example Linux 9",
+    "kernel": "5.14.0-example"
+  }
+}
+```
+
+The script joins records from `status.before` and `status.after` by `vm_name`
+and prints two comparison tables:
+
+- `operating_system`: VM name, before OS, after OS, and `Changed`
+- `kernel`: VM name, before kernel, after kernel, and `Changed`
+
+`Changed` is `NO` for equal values and `YES` otherwise. Missing values are shown
+as `not available` and therefore compare as changed.
+
+The tables are saved to `log/summary_YYYY-MM-DD_HH_MM.log` and included in the
+summary email. Summary files older than 60 days are purged at script startup.
+
+## Logs and retention
+
+Both scripts create the `log/` directory when it does not exist. Filenames
+include local Dom0 date and time to the minute; two executions of the same
+script within one minute can use the same filename.
+
+**Current retention caveat:** `system_update.sh` writes files named
+`system_update_*.log`, but its cleanup commands currently match
+`dnf_update_all_*.log` and `.log.gz`. Consequently, those commands do not
+compress or delete the `system_update_*.log` files. The 60-day summary purge
+in `check_update.sh` applies to `summary_*.log` files.
+
+## Operational notes
+
+- These scripts use the directory from which they are launched, not the
+  directory containing the script, for `excluded_vm.txt`, status files, and
+  `log/`.
+- VM shutdowns are requested through `virsh shutdown`; the scripts do not wait
+  for a VM to finish shutting down.
+- `check_update.sh` requires a valid `status.before` JSON file. Run
+  `system_update.sh` first so both files describe the same update cycle.
+- Review the generated logs and comparison tables for per-VM errors before
+  relying on the update results.
diff --git a/KVM_system_update/check_update.sh b/KVM_system_update/check_update.sh
new file mode 100644
index 0000000..7360d4c
--- /dev/null
+++ b/KVM_system_update/check_update.sh
@@ -0,0 +1,302 @@
+#!/usr/bin/env bash
+
+set -u
+set -o pipefail
+
+WORK_DIR=${PWD}
+EXCLUDED_FILE="${WORK_DIR}/excluded_vm.txt"
+STATUS_FILE="${WORK_DIR}/status.before"
+CHECK_STATUS_FILE="${WORK_DIR}/status.after"
+LOG_DIR="${WORK_DIR}/log"
+WAIT_FOR_START=${WAIT_FOR_START:-1}
+EMAIL_RECIPIENT=${EMAIL_RECIPIENT:-vplesnila@gmail.com}
+TIMESTAMP=$(date '+%Y-%m-%d_%H_%M')
+SUMMARY_FILE="${LOG_DIR}/summary_${TIMESTAMP}.log"
+SSH_OPTIONS=(
+    -o BatchMode=yes
+    -o ConnectTimeout=10
+    -o ConnectionAttempts=1
+    -o StrictHostKeyChecking=accept-new
+)
+
+if [[ ! "$WAIT_FOR_START" =~ ^[0-9]+$ ]]; then
+    printf 'ERROR: WAIT_FOR_START must be a non-negative integer number of minutes.\n' >&2
+    exit 2
+fi
+
+if ! command -v python3 >/dev/null 2>&1; then
+    printf 'ERROR: python3 is required to read and write JSON status files.\n' >&2
+    exit 1
+fi
+
+if ! command -v msmtp >/dev/null 2>&1; then
+    printf 'ERROR: msmtp is required to send the summary email.\n' >&2
+    exit 1
+fi
+
+if [[ -z "$EMAIL_RECIPIENT" ]]; then
+    printf 'ERROR: EMAIL_RECIPIENT must not be empty.\n' >&2
+    exit 1
+fi
+
+if ! DOM0_HOSTNAME=$(hostnamectl --static); then
+    printf 'ERROR: unable to determine Dom0 hostname for email subject.\n' >&2
+    exit 1
+fi
+
+if ! mkdir -p -- "$LOG_DIR"; then
+    printf 'ERROR: cannot create log directory: %s\n' "$LOG_DIR" >&2
+    exit 1
+fi
+
+if ! find "$LOG_DIR" -maxdepth 1 -type f \
+    -name 'summary_*.log' -mmin +86400 -delete; then
+    printf 'ERROR: failed to delete summary logs older than 60 days.\n' >&2
+    exit 1
+fi
+
+if [[ ! -f "$STATUS_FILE" ]] || ! python3 - "$STATUS_FILE" <<'PY'
+import json
+import sys
+
+try:
+    with open(sys.argv[1], encoding="utf-8") as status_file:
+        data = json.load(status_file)
+    if not isinstance(data.get("systems"), list):
+        raise ValueError('missing "systems" array')
+except (OSError, ValueError, json.JSONDecodeError) as error:
+    print(f"ERROR: invalid or unavailable status file: {error}", file=sys.stderr)
+    sys.exit(1)
+PY
+then
+    exit 1
+fi
+
+is_excluded() {
+    local vm=$1
+
+    [[ -f "$EXCLUDED_FILE" ]] || return 1
+    awk -v vm="$vm" '
+        /^[[:space:]]*#/ || /^[[:space:]]*$/ { next }
+        {
+            line = $0
+            sub(/[[:space:]]*#.*/, "", line)
+            gsub(/^[[:space:]]+|[[:space:]]+$/, "", line)
+            if (line == vm) found = 1
+        }
+        END { exit(found ? 0 : 1) }
+    ' "$EXCLUDED_FILE"
+}
+
+wait_for_ssh() {
+    local vm=$1
+    local attempt
+
+    for attempt in {1..6}; do
+        if ssh "${SSH_OPTIONS[@]}" "root@${vm}" 'true' >/dev/null 2>&1; then
+            return 0
+        fi
+        [[ "$attempt" -lt 6 ]] && sleep 10
+    done
+    return 1
+}
+
+extract_hostname_details() {
+    local hostname_output=$1
+    local os kernel
+
+    os=$(awk '
+        /^[[:space:]]*Operating System:/ {
+            sub(/^[^:]*:[[:space:]]*/, "")
+            print
+            exit
+        }
+    ' <<< "$hostname_output")
+    kernel=$(awk '
+        /^[[:space:]]*Kernel:/ {
+            sub(/^[^:]*:[[:space:]]*/, "")
+            print
+            exit
+        }
+    ' <<< "$hostname_output")
+
+    [[ -n "$os" && -n "$kernel" ]] || return 1
+    printf '%s\t%s\n' "$os" "$kernel"
+}
+
+record_after_values() {
+    local name=$1
+    local hostname_output details os kernel
+
+    if [[ "$name" == "Dom0" ]]; then
+        hostname_output=$(LC_ALL=C hostnamectl 2>&1) || return 1
+    else
+        hostname_output=$(ssh "${SSH_OPTIONS[@]}" "root@${name}" \
+            'LC_ALL=C hostnamectl' 2>&1) || return 1
+    fi
+
+    details=$(extract_hostname_details "$hostname_output") || return 1
+    IFS=$'\t' read -r os kernel <<< "$details"
+    if (( check_status_count > 0 )); then
+        printf ',\n' >> "$CHECK_STATUS_FILE" || return 1
+    fi
+    python3 -c '
+import json
+import sys
+
+json.dump(
+    {
+        "vm_name": sys.argv[1],
+        "after": {
+            "operating_system": sys.argv[2],
+            "kernel": sys.argv[3],
+        },
+    },
+    sys.stdout,
+    ensure_ascii=False,
+)
+' "$name" "$os" "$kernel" >> "$CHECK_STATUS_FILE" || return 1
+    check_status_count=$((check_status_count + 1))
+    printf '%s: after values recorded in %s.\n' "$name" "$CHECK_STATUS_FILE"
+}
+
+shutdown_vm() {
+    local vm=$1
+
+    if virsh shutdown "$vm" >/dev/null 2>&1; then
+        printf '%s: shutdown requested.\n' "$vm"
+    else
+        printf '%s: ERROR: failed to request shutdown.\n' "$vm" >&2
+    fi
+}
+
+if ! printf '{\n  "systems": [\n' > "$CHECK_STATUS_FILE"; then
+    printf 'ERROR: cannot create status file: %s\n' "$CHECK_STATUS_FILE" >&2
+    exit 1
+fi
+check_status_count=0
+
+if [[ ! -f "$EXCLUDED_FILE" ]]; then
+    printf 'WARNING: %s does not exist; no VMs are excluded.\n' "$EXCLUDED_FILE" >&2
+fi
+
+mapfile -t vms < <(virsh list --all --name)
+for vm in "${vms[@]}"; do
+    [[ -n "$vm" ]] || continue
+
+    if is_excluded "$vm"; then
+        printf '%s: excluded.\n' "$vm"
+        continue
+    fi
+
+    printf '%s: processing started.\n' "$vm"
+    state=$(virsh domstate "$vm" 2>/dev/null | tr -d '\r' | head -n 1)
+    if [[ "$state" != "running" ]]; then
+        if ! virsh start "$vm"; then
+            printf '%s: ERROR: failed to start VM.\n' "$vm" >&2
+            continue
+        fi
+        printf '%s: started.\n' "$vm"
+    else
+        printf '%s: already running.\n' "$vm"
+    fi
+
+    printf '%s: waiting %s minute(s).\n' "$vm" "$WAIT_FOR_START"
+    sleep "$((WAIT_FOR_START * 60))"
+
+    if ! wait_for_ssh "$vm"; then
+        printf '%s: ERROR: SSH is not reachable.\n' "$vm" >&2
+        shutdown_vm "$vm"
+        continue
+    fi
+
+    if ! record_after_values "$vm"; then
+        printf '%s: ERROR: failed to collect hostnamectl after values.\n' "$vm" >&2
+    fi
+    shutdown_vm "$vm"
+    printf '%s: processing finished.\n' "$vm"
+done
+
+printf 'Dom0: collecting after values.\n'
+if ! record_after_values "Dom0"; then
+    printf 'Dom0: ERROR: failed to collect hostnamectl after values.\n' >&2
+fi
+
+if ! printf '\n  ]\n}\n' >> "$CHECK_STATUS_FILE"; then
+    printf 'ERROR: failed to finish JSON status file: %s\n' "$CHECK_STATUS_FILE" >&2
+    exit 1
+fi
+
+python3 - "$STATUS_FILE" "$CHECK_STATUS_FILE" <<'PY' | tee "$SUMMARY_FILE"
+import json
+import sys
+
+before_path, after_path = sys.argv[1:]
+with open(before_path, encoding="utf-8") as file:
+    before_data = json.load(file)
+with open(after_path, encoding="utf-8") as file:
+    after_data = json.load(file)
+
+before = {
+    item["vm_name"]: item.get("before", {})
+    for item in before_data["systems"]
+    if isinstance(item, dict) and isinstance(item.get("vm_name"), str)
+}
+after = {
+    item["vm_name"]: item.get("after", {})
+    for item in after_data["systems"]
+    if isinstance(item, dict) and isinstance(item.get("vm_name"), str)
+}
+
+names = list(dict.fromkeys(
+    [item["vm_name"] for item in before_data["systems"]
+     if isinstance(item, dict) and isinstance(item.get("vm_name"), str)]
+    + [item["vm_name"] for item in after_data["systems"]
+       if isinstance(item, dict) and isinstance(item.get("vm_name"), str)]
+))
+missing = "not available"
+
+def render(title, key):
+    headers = ["vm_name", f"before {key}", f"after {key}", "Changed"]
+    rows = []
+    for name in names:
+        old_value = before.get(name, {}).get(key, missing)
+        new_value = after.get(name, {}).get(key, missing)
+        rows.append([
+            name,
+            old_value,
+            new_value,
+            "NO" if old_value == new_value else "YES",
+        ])
+    widths = [
+        max(len(headers[i]), *(len(str(row[i])) for row in rows))
+        if rows else len(headers[i])
+        for i in range(len(headers))
+    ]
+    print(f"\n{title}")
+    print(" | ".join(headers[i].ljust(widths[i]) for i in range(4)))
+    print("-+-".join("-" * width for width in widths))
+    for row in rows:
+        print(" | ".join(str(row[i]).ljust(widths[i]) for i in range(4)))
+
+render("operating_system", "operating_system")
+render("kernel", "kernel")
+PY
+
+if ! {
+    printf 'To: %s\n' "$EMAIL_RECIPIENT"
+    printf 'Subject: Update system on %s\n' "$DOM0_HOSTNAME"
+    printf 'MIME-Version: 1.0\n'
+    printf 'Content-Type: text/html; charset=UTF-8\n'
+    printf '\n'
+    printf '
'
+    python3 -c 'import html, pathlib, sys; print(html.escape(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8")), end="")' \
+        "$SUMMARY_FILE"
+    printf '
\n' +} | msmtp -t; then + printf 'ERROR: failed to send summary email to %s.\n' "$EMAIL_RECIPIENT" >&2 + exit 1 +fi + +printf 'Summary email accepted by msmtp for submission to %s using %s; final delivery is not confirmed.\n' \ + "$EMAIL_RECIPIENT" "$SUMMARY_FILE" diff --git a/KVM_system_update/system_update.sh b/KVM_system_update/system_update.sh new file mode 100644 index 0000000..163a6e3 --- /dev/null +++ b/KVM_system_update/system_update.sh @@ -0,0 +1,232 @@ +#!/usr/bin/env bash + +set -u +set -o pipefail + +WORK_DIR=${PWD} +EXCLUDED_FILE="${WORK_DIR}/excluded_vm.txt" +STATUS_FILE="${WORK_DIR}/status.before" +LOG_DIR="${WORK_DIR}/log" +WAIT_FOR_START=${WAIT_FOR_START:-1} +TIMESTAMP=$(date '+%Y-%m-%d_%H_%M') +LOG_FILE="${LOG_DIR}/system_update_${TIMESTAMP}.log" +SSH_OPTIONS=( + -o BatchMode=yes + -o ConnectTimeout=10 + -o ConnectionAttempts=1 + -o StrictHostKeyChecking=accept-new +) + +if [[ ! "$WAIT_FOR_START" =~ ^[0-9]+$ ]]; then + printf 'ERROR: WAIT_FOR_START must be a non-negative integer number of minutes.\n' >&2 + exit 2 +fi + +mkdir -p -- "$LOG_DIR" || { + printf 'ERROR: cannot create log directory: %s\n' "$LOG_DIR" >&2 + exit 1 +} + +exec > >(tee -a "$LOG_FILE") 2>&1 + +log() { + printf '[%s] %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$*" +} + +cleanup_logs() { + log "Deleting compressed logs older than 6 months." + if ! find "$LOG_DIR" -maxdepth 1 -type f \ + -name 'dnf_update_all_*.log.gz' -mtime +180 -delete; then + log "ERROR: failed to delete expired compressed logs." + fi + + log "Compressing logs older than 30 days." + if ! find "$LOG_DIR" -maxdepth 1 -type f \ + -name 'dnf_update_all_*.log' -mtime +30 -exec gzip -- {} +; then + log "ERROR: failed to compress old logs." + fi +} + +is_excluded() { + local vm=$1 + + [[ -f "$EXCLUDED_FILE" ]] || return 1 + awk -v vm="$vm" ' + /^[[:space:]]*#/ || /^[[:space:]]*$/ { next } + { + line = $0 + sub(/[[:space:]]*#.*/, "", line) + gsub(/^[[:space:]]+|[[:space:]]+$/, "", line) + if (line == vm) found = 1 + } + END { exit(found ? 0 : 1) } + ' "$EXCLUDED_FILE" +} + +wait_for_ssh() { + local vm=$1 + local attempt + + for attempt in {1..6}; do + if ssh "${SSH_OPTIONS[@]}" "root@${vm}" 'true' >/dev/null 2>&1; then + return 0 + fi + [[ "$attempt" -lt 6 ]] && sleep 10 + done + return 1 +} + +extract_hostname_details() { + local hostname_output=$1 + local os kernel + + os=$(awk ' + /^[[:space:]]*Operating System:/ { + sub(/^[^:]*:[[:space:]]*/, "") + print + exit + } + ' <<< "$hostname_output") + kernel=$(awk ' + /^[[:space:]]*Kernel:/ { + sub(/^[^:]*:[[:space:]]*/, "") + print + exit + } + ' <<< "$hostname_output") + + if [[ -z "$os" || -z "$kernel" ]]; then + return 1 + fi + printf '%s\t%s\n' "$os" "$kernel" +} + +record_before_values() { + local name=$1 + local hostname_output details os kernel + + if [[ "$name" == "Dom0" ]]; then + hostname_output=$(LC_ALL=C hostnamectl 2>&1) || return 1 + else + hostname_output=$(ssh "${SSH_OPTIONS[@]}" "root@${name}" \ + 'LC_ALL=C hostnamectl' 2>&1) || return 1 + fi + + details=$(extract_hostname_details "$hostname_output") || return 1 + IFS=$'\t' read -r os kernel <<< "$details" + if (( status_count > 0 )); then + printf ',\n' >> "$STATUS_FILE" + fi + if ! python3 -c ' +import json +import sys + +json.dump( + { + "vm_name": sys.argv[1], + "before": { + "operating_system": sys.argv[2], + "kernel": sys.argv[3], + }, + }, + sys.stdout, + ensure_ascii=False, +) +' "$name" "$os" "$kernel" >> "$STATUS_FILE"; then + return 1 + fi + status_count=$((status_count + 1)) + log "$name: before values recorded in $STATUS_FILE." +} + +shutdown_vm() { + local vm=$1 + + if virsh shutdown "$vm" >/dev/null 2>&1; then + log "$vm: shutdown requested." + else + log "$vm: ERROR: failed to request shutdown." + fi +} + +if ! command -v python3 >/dev/null 2>&1; then + log "ERROR: python3 is required to write valid JSON to $STATUS_FILE." + exit 1 +fi + +cleanup_logs + +if ! printf '{\n "systems": [\n' > "$STATUS_FILE"; then + log "ERROR: cannot create status file: $STATUS_FILE" + exit 1 +fi +status_count=0 + +if [[ ! -f "$EXCLUDED_FILE" ]]; then + log "WARNING: $EXCLUDED_FILE does not exist; no VMs are excluded." +fi + +mapfile -t vms < <(virsh list --all --name) +if [[ "${#vms[@]}" -eq 0 ]]; then + log "No VMs found." +else + for vm in "${vms[@]}"; do + [[ -n "$vm" ]] || continue + + if is_excluded "$vm"; then + log "$vm: excluded." + continue + fi + + log "$vm: processing started." + state=$(virsh domstate "$vm" 2>/dev/null | tr -d '\r' | head -n 1) + if [[ "$state" != "running" ]]; then + if ! virsh start "$vm"; then + log "$vm: ERROR: failed to start VM." + continue + fi + log "$vm: started." + else + log "$vm: already running." + fi + + log "$vm: waiting ${WAIT_FOR_START} minute(s)." + sleep "$((WAIT_FOR_START * 60))" + + if ! wait_for_ssh "$vm"; then + log "$vm: ERROR: SSH is not reachable." + shutdown_vm "$vm" + continue + fi + + if ! record_before_values "$vm"; then + log "$vm: ERROR: failed to collect hostnamectl before values." + fi + + log "$vm: running dnf update -y." + if ! ssh "${SSH_OPTIONS[@]}" "root@${vm}" \ + 'export LC_ALL=C; dnf update -y'; then + log "$vm: ERROR: dnf update failed." + fi + + shutdown_vm "$vm" + log "$vm: processing finished." + done +fi + +log "Dom0: collecting before values." +if ! record_before_values "Dom0"; then + log "Dom0: ERROR: failed to collect hostnamectl before values." +fi + +log "Dom0: running dnf update -y." +if ! LC_ALL=C dnf update -y; then + log "Dom0: ERROR: dnf update failed." +fi + +if ! printf '\n ]\n}\n' >> "$STATUS_FILE"; then + log "ERROR: failed to finish JSON status file: $STATUS_FILE" + exit 1 +fi + +log "System update processing finished. Status saved to $STATUS_FILE."