#!/bin/bash
################################################################################
# Script Name: version/3.9.5/UPDATE.sh
# Description: Apply version 3.9.5:
#              - every account importer the admin offers is installed. The
#                runtime bundle carried only the cPanel and CyberPanel
#                importers, so on a server built from the mirror the
#                DirectAdmin, Plesk and aaPanel ones were never on disk:
#                INSTALL.sh copies them from the bundle it was unpacked from,
#                and no update installed any importer. The admin then fell
#                back to downloading one from the mirror on every import.
#                3.9.5's bundle carries all five; this puts them in
#                /opt/hitechcloud/integrations/ on a server that is already
#                running (step 1).
#              - the Enhance and ispmanager importers are part of the CLI
#                (`hitechcloudcli migrate import enhance|ispmanager`) and need
#                nothing here.
#
#              The update to 3.9.4 was run by 3.9.3's update.sh (the script that
#              performs an update is the one it replaces) and three of its
#              faults reached every server. Measured on 3.9.3 -> 3.9.4 updates
#              of Ubuntu 24.04 and AlmaLinux 9 in systemd containers:
#
#              - the engines were installed as hitechcloud-rustpanel and
#                hitechcloud-rustadmin, where no panel runs them. Both Go
#                servers run hitechcloudpanel-engine / hitechcloudadmin-engine,
#                which kept the 3.9.3 build: a 3.9.4 panel in front of a 3.9.3
#                engine, on every server installed since the rename (step 3).
#              - vendor.json was never refreshed, so 3.9.4's country database
#                was "no vendor.json entry named 'dbip-country-lite'" (step 2).
#              - the customer panel can put a Rust front door on its port
#                (front_door in hitechcloud.config, [PANEL]). Off by default:
#                this writes front_door=go where the key is missing and never
#                changes a value an administrator chose - unless the update is
#                asked to, with HITECHCLOUD_FRONT_DOOR=rust|go (step 9).
#              - 3.9.4's own migration failed at its step 6 on every server
#                (the engine above, run without the admin's database login),
#                and on the RHEL 9/10 family at its step 1 too: rspamd's
#                repository key is SHA-1 signed and EL9 refuses it, so no spam
#                filter was ever installed there. The CLI in this release
#                carries the key the rspamd project publishes (step 4).
#
#              3.9.5's update.sh fixes the first two for good, and hands every
#              later update to the release's own update.sh, so a fix to the
#              update reaches the update that ships it. But THIS update is
#              still run by 3.9.4's update.sh, so this script does the repairs
#              itself:
#
#                2. this release's vendor.json where the CLI reads it;
#                3. the engines under the names the panels run, checked against
#                   the release's SHA256SUMS;
#                4. every step of 3.9.4's migration again - each is idempotent,
#                   and a server where one failed is healed by this;
#                5. the health monitor, which no mirror-built server ever got
#                   (the installer skipped it and said it was installed);
#                6. the meetings service's binary, which no update refreshed.
#
#              Nothing here fails the update without saying what to run.
# Company/Brand: HiTechCloud by Pho Tue SoftWare Solutions JSC
# Copyright (c) HiTechCloud by Pho Tue SoftWare Solutions JSC
################################################################################

set -uo pipefail
GREEN='\033[0;32m'; YELLOW='\033[1;33m'; RED='\033[0;31m'; NC='\033[0m'
ok()   { echo -e "[3.9.5] ${GREEN}✓${NC} $*"; }
warn() { echo -e "[3.9.5] ${YELLOW}!${NC} $*" >&2; }
FAILURES=0
bad()  { echo -e "[3.9.5] ${RED}✗${NC} $*" >&2; FAILURES=$((FAILURES + 1)); }
[[ $EUID -eq 0 ]] || { echo "[3.9.5] This update must run as root." >&2; exit 1; }

################################################################################
# Step 1's work, defined here so it can also be run on its own (see below).
#
# The importer scripts the admin's catalogue names (hitechcloudadmin
# internal/handlers/importer.go: Dir and Script), as <directory>:<script>.
# .github/scripts/importers_shipped_test.sh fails when this list, the runtime
# bundle in release.yml or INSTALL.sh's loop misses one the catalogue has.
IMPORTERS=(
    "cpanel-import:cp-import.sh"
    "cyberpanel-import:cp-import.sh"
    "directadmin-import:da-import.sh"
    "plesk-import:plesk-import.sh"
    "aapanel-import:aapanel-import.sh"
)
#
# Where the new ones come from: the release this script belongs to. The
# update unpacks the runtime bundle over the install directory and runs this
# script from there (version/3.9.5/ of it), so the bundle's integrations/ is
# two levels up - and on a mirror-built server that directory already is
# /opt/hitechcloud/integrations, and there is nothing to copy. When this
# script was fetched from the mirror on its own, the install directories are
# looked in instead.
#
# This step never fails the update. An importer that is not installed is
# fetched from the mirror by the admin when it is used, so a server that
# cannot get one here is no worse off than before; but it says which one, and
# how to put it there.
#
# HITECHCLOUD_IMPORTERS_SRC and HITECHCLOUD_IMPORTERS_DEST point it at other
# trees, which is how .github/scripts/update_3_9_5_importers_test.sh runs it.
################################################################################
importers_step() {
    local dest="${HITECHCLOUD_IMPORTERS_DEST:-/opt/hitechcloud/integrations}"
    local here; here="$(cd "$(dirname "${BASH_SOURCE[0]}")" 2>/dev/null && pwd)"
    local -a sources=()
    if [[ -n "${HITECHCLOUD_IMPORTERS_SRC:-}" ]]; then
        sources=("$HITECHCLOUD_IMPORTERS_SRC")
    else
        sources=("${here}/../../integrations" /opt/hitechcloud/integrations /usr/local/hitechcloud/integrations /home/HiTechCloud-Panel/integrations)
    fi

    local entry dir script src s target installed=0 missing=()
    for entry in "${IMPORTERS[@]}"; do
        dir="${entry%%:*}"; script="${entry#*:}"
        target="${dest}/${dir}"
        src=""
        for s in "${sources[@]}"; do
            [[ -s "${s}/${dir}/${script}" ]] && { src="$(cd "${s}/${dir}" && pwd)"; break; }
        done

        if [[ -z "$src" ]]; then
            if [[ -s "${target}/${script}" ]]; then
                ok "${dir} is installed (no newer copy in this release to put over it)."
            else
                missing+=("$dir")
            fi
            continue
        fi
        if [[ -d "$target" && "$(cd "$target" && pwd)" == "$src" ]]; then
            ok "${dir} is installed."
            continue
        fi
        if [[ "${DRY_RUN:-0}" == 1 ]]; then
            echo "[3.9.5] would: install ${dir} from ${src} into ${target}"
            continue
        fi
        if mkdir -p "$target" 2>/dev/null && {
               if command -v rsync >/dev/null 2>&1; then rsync -a --delete "${src}/" "${target}/"
               else rm -rf "${target:?}/"* && cp -a "${src}/." "${target}/"; fi
           } >/dev/null 2>&1 && [[ -s "${target}/${script}" ]]; then
            ok "${dir} installed into ${target}."
            installed=$((installed + 1))
        else
            missing+=("$dir")
        fi
    done

    if (( ${#missing[@]} )); then
        warn "these importers are not installed: ${missing[*]}. The admin will download each from the mirror when it is used. To install them now, copy integrations/<name> from the 3.9.5 release into ${dest}/."
    fi
    return 0
}

VERSION="3.9.5"
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" 2>/dev/null && pwd)"
# Every path and command below can be pointed elsewhere, which is how
# .github/scripts/update_3_9_5_heal_test.sh runs each step against a throwaway
# tree and stand-in programs.
BIN_DIR="${HITECHCLOUD_BIN_DIR:-/usr/local/bin}"
MIRROR="${HITECHCLOUD_MIRROR:-https://mirrors.hitechcloud.org}"
SYSTEMCTL="${HITECHCLOUD_SYSTEMCTL:-systemctl}"
RELEASE_ROOT="${HITECHCLOUD_RELEASE_ROOT:-$(cd "${HERE}/../.." 2>/dev/null && pwd)}"

arch() {
    case "$(uname -m)" in
        x86_64|amd64)  echo amd64 ;;
        aarch64|arm64) echo arm64 ;;
        *) return 1 ;;
    esac
}

# The release's SHA256SUMS, once.
SUMS=""
SUMS_READ=0
published_sum() { # asset-name
    if [[ "$SUMS_READ" == 0 ]]; then
        SUMS=$(curl -4fsSL --retry 2 --connect-timeout 20 --max-time 120 "${MIRROR}/releases/${VERSION}/SHA256SUMS" 2>/dev/null) || SUMS=""
        SUMS_READ=1
    fi
    awk -v a="$1" '$2 == a || $2 == "*"a { print $1; exit }' <<<"$SUMS"
}
sha_of() { [[ -s "$1" ]] && sha256sum "$1" 2>/dev/null | awk '{print $1}'; }

# fetch_verified ASSET DEST - the release's ASSET at DEST, only if its checksum
# is the published one.
fetch_verified() {
    local asset="$1" dest="$2" want tmp
    want=$(published_sum "$asset")
    [[ -n "$want" ]] || return 1
    tmp=$(mktemp "${dest}.XXXXXX") || return 1
    if curl -4fsSL --retry 3 --connect-timeout 20 --max-time 900 "${MIRROR}/releases/${VERSION}/${asset}" -o "$tmp" \
       && [[ "$(sha_of "$tmp")" == "$want" ]]; then
        chmod 755 "$tmp" && mv -f "$tmp" "$dest" && return 0
    fi
    rm -f "$tmp"
    return 1
}

################################################################################
# 2. vendor.json: the release's list of pinned third-party downloads.
#
# native/vendor.rs reads /etc/hitechcloud/hitechadmin/config/vendor.json before
# the release tree, INSTALL.sh writes it once, and no update refreshed it until
# this release's update.sh. So 3.9.4's migration asked for dbip-country-lite
# and the CLI did not know the name. The previous file is kept as .prev.
################################################################################
vendor_step() {
    local src="${HITECHCLOUD_VENDOR_SRC:-${RELEASE_ROOT}/configuration/hitechadmin/config/vendor.json}"
    local dest="${HITECHCLOUD_VENDOR_DEST:-/etc/hitechcloud/hitechadmin/config/vendor.json}"
    if [[ ! -s "$src" ]]; then
        warn "this release's vendor.json is not at ${src}; third-party downloads keep the list this server has. Re-run the update: hitechcloudcli update --force"
        return 0
    fi
    if cmp -s "$src" "$dest"; then
        ok "vendor.json is this release's."
        return 0
    fi
    if command -v jq >/dev/null 2>&1 && ! jq empty "$src" >/dev/null 2>&1; then
        warn "${src} is not valid JSON, so vendor.json was left as it is. Re-run the update: hitechcloudcli update --force"
        return 0
    fi
    if [[ "${DRY_RUN:-0}" == 1 ]]; then
        echo "[3.9.5] would: install ${src} as ${dest}"
        return 0
    fi
    mkdir -p "$(dirname "$dest")"
    [[ -f "$dest" ]] && cp -a "$dest" "${dest}.prev"
    if install -m 0644 "$src" "$dest"; then
        ok "vendor.json is this release's (the previous one is ${dest}.prev)."
    else
        warn "vendor.json could not be written. Run: install -m 0644 ${src} ${dest}"
    fi
    return 0
}

################################################################################
# 3. The engines, under the names the panels run.
#
# For each engine: if /usr/local/bin/<engine> is not this release's build, the
# pre-rename file 3.9.4's updater just installed is used when it is (no
# download), and the mirror otherwise - checked against SHA256SUMS either way.
# The pre-rename name then becomes a link to it, so both names run one engine.
# The panels are restarted only when an engine changed.
#
# Fails the update when an engine cannot be put right, because a panel in front
# of an engine from another release is the fault being repaired - with the
# command that does it.
################################################################################
engines_step() {
    local a changed=0 pair engine legacy want cur leg
    a=$(arch) || { warn "no published engines for $(uname -m); the engines were left as they are"; return 0; }
    for pair in hitechcloudpanel-engine:hitechcloud-rustpanel hitechcloudadmin-engine:hitechcloud-rustadmin; do
        engine="${pair%%:*}" legacy="${pair#*:}"
        want=$(published_sum "${engine}-linux-${a}")
        [[ -n "$want" ]] || want=$(published_sum "${legacy}-linux-${a}")
        if [[ -z "$want" ]]; then
            warn "the mirror's SHA256SUMS for ${VERSION} could not be read, so ${engine} could not be checked. Run: hitechcloudcli update --force"
            continue
        fi
        cur=$(sha_of "${BIN_DIR}/${engine}")
        leg=$(sha_of "${BIN_DIR}/${legacy}")
        if [[ "$cur" == "$want" ]]; then
            ok "${engine} is this release's build."
        elif [[ "${DRY_RUN:-0}" == 1 ]]; then
            echo "[3.9.5] would: put this release's ${engine} at ${BIN_DIR}/${engine}"
            continue
        elif [[ "$leg" == "$want" && ! -L "${BIN_DIR}/${legacy}" ]] \
             && { [[ -f "${BIN_DIR}/${engine}" ]] && cp -a "${BIN_DIR}/${engine}" "${BIN_DIR}/${engine}.prev"; true; } \
             && install -m 755 "${BIN_DIR}/${legacy}" "${BIN_DIR}/${engine}"; then
            ok "${engine} is this release's build; the update had put it at ${legacy}, where no panel runs it."
            changed=1
        elif { [[ -f "${BIN_DIR}/${engine}" ]] && cp -a "${BIN_DIR}/${engine}" "${BIN_DIR}/${engine}.prev"; true; } \
             && { fetch_verified "${engine}-linux-${a}" "${BIN_DIR}/${engine}" || fetch_verified "${legacy}-linux-${a}" "${BIN_DIR}/${engine}"; }; then
            ok "${engine} is this release's build, downloaded and checked against SHA256SUMS."
            changed=1
        else
            bad "${engine} is not this release's build and could not be replaced, so the panel runs an engine from another release. Run: hitechcloudcli update --force"
            continue
        fi
        if [[ "${DRY_RUN:-0}" != 1 ]] && { [[ -e "${BIN_DIR}/${legacy}" ]] || [[ -L "${BIN_DIR}/${legacy}" ]]; } \
           && [[ "$(readlink "${BIN_DIR}/${legacy}" 2>/dev/null)" != "${BIN_DIR}/${engine}" ]]; then
            ln -sfn "${BIN_DIR}/${engine}" "${BIN_DIR}/${legacy}" && ok "${BIN_DIR}/${legacy} now points at ${engine}."
        fi
    done
    if [[ "$changed" == 1 ]]; then
        "$SYSTEMCTL" try-restart hitechcloud hitechadmin >/dev/null 2>&1 \
            && ok "both panels restarted onto this release's engines." \
            || warn "the panels could not be restarted; they use the new engines from their next start. Run: systemctl restart hitechcloud hitechadmin"
    fi
    return 0
}

################################################################################
# 4. Every step of 3.9.4's migration, again.
#
# 3.9.4's script is idempotent step by step - each checks before it changes,
# and the two one-time steps keep markers - so running it a second time repairs
# what failed and leaves alone what worked. HITECHCLOUD_UPDATE_REAPPLY=1 tells
# it not to record 3.9.4 as the version and not to re-render the vhosts, which
# the update has just done. The copy in this release is used: it carries the
# fixed engine lookup that 3.9.4's own copy did not have.
################################################################################
reapply_394_step() {
    local script="${HITECHCLOUD_394_SCRIPT:-${HERE}/../3.9.4/UPDATE.sh}"
    local by_hand="HITECHCLOUD_UPDATE_REAPPLY=1 bash /opt/hitechcloud/version/3.9.4/UPDATE.sh"
    if [[ ! -s "$script" ]]; then
        warn "3.9.4's migration is not in this release at ${script}, so it was not run again. Re-run the update: hitechcloudcli update --force"
        return 0
    fi
    echo "[3.9.5] Running 3.9.4's migration again, to finish what it could not do the first time:"
    local rc
    HITECHCLOUD_UPDATE_REAPPLY=1 HITECHCLOUD_UPDATE_DRY_RUN="${DRY_RUN:-0}" bash "$script" 2>&1 | sed 's/^/    /'
    rc=${PIPESTATUS[0]}
    if [[ $rc -eq 0 ]]; then
        ok "3.9.4's migration is complete on this server."
    else
        bad "3.9.4's migration still left something undone (the lines marked ✗ above say what and how). Once that is fixed, run it again with: ${by_hand}"
    fi
    return 0
}

################################################################################
# 5. The health monitor behind the Uptime, Log warnings and Alerts pages.
#
# deploy/install-services.sh returned "skipping" when the release tree had no
# integrations/sentinel - every server built from the mirror - and the installer
# printed "Sentinel health monitor installed" anyway. This release's copy
# fetches it from the mirror. Only where it is missing; never fails the update.
################################################################################
sentinel_step() {
    local installer="${HITECHCLOUD_SERVICES_SH:-${RELEASE_ROOT}/deploy/install-services.sh}"
    local by_hand="bash /opt/hitechcloud/deploy/install-services.sh sentinel"
    if [[ ! -f "${HITECHCLOUD_NATIVE_CONF:-/etc/hitechcloud/native/native.conf}" ]]; then
        ok "not a host-services install; sentinel is not this server's concern."
        return 0
    fi
    if [[ -x "${BIN_DIR}/sentinel" ]]; then
        ok "the sentinel health monitor is installed."
        return 0
    fi
    if [[ "${DRY_RUN:-0}" == 1 ]]; then
        echo "[3.9.5] would: ${by_hand}"
        return 0
    fi
    if [[ -f "$installer" ]] && HITECHCLOUD_VERSION="$VERSION" HITECHCLOUD_MIRROR="$MIRROR" \
         REPO_DIR="$RELEASE_ROOT" bash "$installer" sentinel >/dev/null 2>&1 && [[ -x "${BIN_DIR}/sentinel" ]]; then
        ok "the sentinel health monitor is installed and runs every five minutes; the Uptime pages fill from now on."
    else
        warn "the sentinel health monitor could not be installed, so the Uptime pages stay empty. Run it by hand and read the output: ${by_hand}"
    fi
    return 0
}

################################################################################
# 6. The meetings service, where it is installed.
#
# `meet install` downloads it once and no update refreshed it, so a server
# kept the build it was installed with. Checked against SHA256SUMS; never fails
# the update.
################################################################################
meet_step() {
    local bin="${BIN_DIR}/hitechcloud-meet" a want
    if [[ ! -e "$bin" ]]; then
        ok "the meetings service is not installed here."
        return 0
    fi
    a=$(arch) || return 0
    want=$(published_sum "hitechcloud-meet-linux-${a}")
    if [[ -z "$want" ]]; then
        warn "the mirror's SHA256SUMS for ${VERSION} could not be read, so hitechcloud-meet was not checked. Run: hitechcloudcli update --force"
    elif [[ "$(sha_of "$bin")" == "$want" ]]; then
        ok "hitechcloud-meet is this release's build."
    elif [[ "${DRY_RUN:-0}" == 1 ]]; then
        echo "[3.9.5] would: replace ${bin} with this release's hitechcloud-meet"
    elif cp -a "$bin" "${bin}.prev" && fetch_verified "hitechcloud-meet-linux-${a}" "$bin"; then
        "$SYSTEMCTL" try-restart hitechcloud-meet >/dev/null 2>&1
        ok "hitechcloud-meet is this release's build (the previous one is ${bin}.prev)."
    else
        warn "hitechcloud-meet could not be refreshed; the one installed keeps running. Run: hitechcloudcli meet install"
    fi
    return 0
}

# Step 1 alone, for its test. Records no version and touches nothing else.
################################################################################
# 7. FTP - no anonymous logins, and FTPS.
#
# On the RHEL family pure-ftpd accepted anonymous logins (the installer's
# hardening wrote Debian's conf/ layout and gave up where it is absent), and
# on every server FTP passwords crossed the network in clear text (TLS was
# never set). Both measured on the 3.9.4 demo. This release's CLI configures
# both layouts and turns TLS on with the host's certificate; re-running the
# ftp component (--extras-only: nothing else is reinstalled) brings a running
# server to that. 3.9.4.1 did the anonymous half for servers that took it.
#
# Anonymous logins still accepted afterwards fails the update. TLS missing
# afterwards only warns: it needs a certificate for the host, which a server
# may not have yet, and plain FTP keeps working as before.
#
# HITECHCLOUD_PURE_FTPD_CONF, HITECHCLOUD_PURE_FTPD_CONFDIR and HITECHCLOUD_CLI
# point it at a throwaway tree and a stand-in CLI, which is how
# .github/scripts/update_3_9_5_repairs_test.sh runs it.
################################################################################
ftp_setting() {
    # ftp_setting NAME -> the live value in either layout, or nothing.
    local conf="${HITECHCLOUD_PURE_FTPD_CONF:-/etc/pure-ftpd/pure-ftpd.conf}"
    local dir="${HITECHCLOUD_PURE_FTPD_CONFDIR:-/etc/pure-ftpd/conf}"
    if [[ -d "$dir" ]]; then
        [[ -f "$dir/$1" ]] && tr -d '[:space:]' < "$dir/$1"
    elif [[ -f "$conf" ]]; then
        awk -v k="$1" '$1 == k { v = $2 } END { if (v != "") print v }' "$conf"
    fi
}

ftp_step() {
    local conf="${HITECHCLOUD_PURE_FTPD_CONF:-/etc/pure-ftpd/pure-ftpd.conf}"
    local dir="${HITECHCLOUD_PURE_FTPD_CONFDIR:-/etc/pure-ftpd/conf}"
    local cli="${HITECHCLOUD_CLI:-hitechcloudcli}"
    local by_hand="hitechcloudcli native-install --extras-only --with ftp"
    if [[ ! -f "$conf" && ! -d "$dir" ]]; then
        ok "pure-ftpd is not installed here; no FTP to harden."
        return 0
    fi
    if [[ "$(ftp_setting NoAnonymous)" == yes && "$(ftp_setting TLS)" =~ ^[12]$ ]]; then
        ok "FTP refuses anonymous logins and offers TLS."
        return 0
    fi
    if [[ "${DRY_RUN:-0}" == 1 ]]; then
        echo "[3.9.5] would: ${by_hand}, to refuse anonymous FTP logins and offer FTPS"
        return 0
    fi
    if ! command -v "$cli" >/dev/null 2>&1; then
        bad "hitechcloudcli is not on PATH, so FTP was not hardened. Run: ${by_hand}"
        return 0
    fi
    timeout 600 "$cli" native-install --extras-only --with ftp >/dev/null 2>&1
    if [[ "$(ftp_setting NoAnonymous)" != yes ]]; then
        bad "pure-ftpd still accepts anonymous logins. Run it by hand and read the output: ${by_hand}"
    elif [[ ! "$(ftp_setting TLS)" =~ ^[12]$ ]]; then
        ok "pure-ftpd refuses anonymous logins."
        warn "FTP is still unencrypted: this host has no certificate for FTPS to present. Issue one (hitechcloudcli ssl panel), then run: ${by_hand}"
    else
        ok "pure-ftpd refuses anonymous logins and offers FTPS with this host's certificate."
    fi
    return 0
}

################################################################################
# 8. What deleted accounts and domains left behind, listed - not removed.
#
# Until 3.9.5, deleting an account left every database it created, their
# logins (with their passwords, and '<account>'@'localhost' still granted
# `<account>\_%`), its PostgreSQL role, and every deleted domain's DKIM
# signing entry. Measured on the 3.9.4 demo after deleting one test account.
# 3.9.5 removes them on delete. What earlier deletions left is data, and whose
# data it is only the administrator can say, so it is listed in a file and in
# the update log for them to review; nothing is dropped here.
#
# An account is recognised by its name (users.username, the part after
# SUSPENDED_<stamp>_ for a suspended one, and users.server). A database or
# login is the account's when it is named <account> or <account>_*, which is
# how the panel names every one it creates. System databases and logins are
# never listed.
#
# HITECHCLOUD_MYSQL, HITECHCLOUD_PSQL, HITECHCLOUD_DKIM_KEYTABLE and
# HITECHCLOUD_ORPHANS_FILE point it at stand-ins, which is how
# .github/scripts/update_3_9_5_orphans_test.sh runs it.
################################################################################
orphans_step() {
    local mysql_cmd="${HITECHCLOUD_MYSQL:-mysql}"
    local psql_cmd="${HITECHCLOUD_PSQL:-}"
    local keytable="${HITECHCLOUD_DKIM_KEYTABLE:-/etc/opendkim/KeyTable}"
    local out="${HITECHCLOUD_ORPHANS_FILE:-/var/lib/hitechcloud/orphans-3.9.5.txt}"
    local system_dbs=" information_schema mysql performance_schema sys panel "
    local system_logins=" root mysql mariadb.sys PUBLIC healthcheck "
    # Prefixes the stack's own services use - the panel's own login is
    # hitechcloud_panel - never an account's. Found by running this on the
    # demo before it shipped.
    local system_prefixes=" hitechcloud roundcube phpmyadmin pma mailflow pgadmin postfix dovecot pdns powerdns "
    local vdomains="${HITECHCLOUD_MAIL_VDOMAINS:-/etc/hitechcloud/mail/vdomains}"
    local zones="${HITECHCLOUD_DNS_ZONES:-}"
    if [[ -z "$zones" ]]; then
        for zones in /etc/bind/hitechcloud/zones /var/named/hitechcloud/zones /etc/named/hitechcloud/zones; do [[ -d "$zones" ]] && break; done
    fi

    local accounts_raw
    if ! command -v "${mysql_cmd%% *}" >/dev/null 2>&1 || ! accounts_raw=$($mysql_cmd -N -B -e "SELECT username, IFNULL(server,'') FROM panel.users" 2>/dev/null); then
        warn "the panel database could not be read, so leftovers of deleted accounts were not looked for."
        return 0
    fi
    local -A account=()
    local u s
    while IFS=$'\t' read -r u s; do
        [[ "$u" =~ ^SUSPENDED_[0-9]+_(.+)$ ]] && u="${BASH_REMATCH[1]}"
        [[ -n "$u" ]] && account["$u"]=1
        [[ -n "$s" ]] && account["$s"]=1
    done <<< "$accounts_raw"
    owner_gone() {
        # owner_gone NAME: NAME is <x> or <x>_..., x looks like an account
        # name, and no account x exists.
        local p="${1%%_*}"
        [[ "$p" =~ ^[a-z0-9]+$ ]] && [[ -z "${account[$p]:-}" ]] && [[ "$system_prefixes" != *" $p "* ]]
    }

    local report="" n=0 db login host role d
    local -A gone_prefix=()
    while IFS= read -r db; do
        [[ -z "$db" || "$system_dbs" == *" $db "* || "$db" != *_* ]] && continue
        if owner_gone "$db"; then report+="database        ${db}"$'\n'; n=$((n + 1)); gone_prefix["${db%%_*}"]=1; fi
    done < <($mysql_cmd -N -B -e "SHOW DATABASES" 2>/dev/null)
    while IFS=$'\t' read -r login host; do
        [[ -z "$login" || "$system_logins" == *" $login "* ]] && continue
        # A login with no underscore is listed only when it is the name of an
        # account whose databases are listed too - a bare service login such
        # as a webmail's must not be taken for an account's.
        if [[ "$login" == *_* ]]; then owner_gone "$login" || continue
        else [[ -n "${gone_prefix[$login]:-}" ]] || continue; fi
        report+="database login  '${login}'@'${host}'"$'\n'; n=$((n + 1))
    done < <($mysql_cmd -N -B -e "SELECT User, Host FROM mysql.user" 2>/dev/null)
    if [[ -z "$psql_cmd" ]] && command -v psql >/dev/null 2>&1 && id postgres >/dev/null 2>&1; then
        psql_cmd="su - postgres -c"
    fi
    if [[ -n "$psql_cmd" ]]; then
        # Only the roles the panel makes for an account: login, may create
        # databases, not a superuser.
        while IFS= read -r role; do
            [[ -z "$role" ]] && continue
            owner_gone "$role" && [[ "$role" != *_* ]] && { report+="postgres role   ${role}"$'\n'; n=$((n + 1)); }
        done < <($psql_cmd "psql -tAc \"SELECT rolname FROM pg_roles WHERE rolcanlogin AND rolcreatedb AND NOT rolsuper\"" 2>/dev/null)
    fi
    if [[ -f "$keytable" ]]; then
        local -A domain=()
        while IFS= read -r d; do [[ -n "$d" ]] && domain["${d,,}"]=1; done \
            < <($mysql_cmd -N -B -e "SELECT domain_url FROM panel.domains" 2>/dev/null)
        local host_name; host_name="$(hostname -f 2>/dev/null)"
        # A mail domain (a mailbox or alias of it) or a DNS zone this server
        # carries is not a leftover even without a website.
        [[ -f "$vdomains" ]] && while read -r d _; do [[ -n "$d" ]] && domain["${d,,}"]=1; done < "$vdomains"
        if [[ -d "$zones" ]]; then
            for d in "$zones"/db.*; do [[ -e "$d" ]] || continue; d="$(basename "$d")"; domain["${d#db.}"]=1; done
        fi
        local line spec
        while IFS= read -r line; do
            # Entries only: "<selector>._domainkey.<domain> <domain>:<selector>:<keyfile>".
            [[ "$line" =~ ^[[:space:]]*# ]] && continue
            read -r _ spec _ <<< "$line"
            [[ "$spec" =~ ^([A-Za-z0-9.-]+):[^:]+:.+$ ]] || continue
            d="${BASH_REMATCH[1],,}"
            [[ -n "${domain[$d]:-}" || "$d" == "${host_name,,}" ]] && continue
            report+="DKIM entry      ${d} (${keytable})"$'\n'; n=$((n + 1))
        done < "$keytable"
    fi

    if [[ $n -eq 0 ]]; then
        ok "nothing that a deleted account or domain left behind was found."
        return 0
    fi
    if [[ "${DRY_RUN:-0}" != 1 ]]; then
        mkdir -p "$(dirname "$out")" 2>/dev/null
        {
            echo "# Left behind by accounts and domains deleted before 3.9.5 - for review, nothing was removed."
            echo "# Drop what is not wanted: DROP DATABASE / DROP USER in MariaDB, DROP ROLE in PostgreSQL,"
            echo "# and remove a DKIM entry's lines from KeyTable and SigningTable, then reload opendkim."
            printf '%s' "$report"
        } > "$out"
        chmod 600 "$out" 2>/dev/null
    fi
    warn "${n} item(s) left behind by deleted accounts or domains - databases, their logins, PostgreSQL roles or DKIM entries. Nothing was removed; review the list in ${out}:"
    printf '%s' "$report" | sed 's/^/        /' >&2
    return 0
}

################################################################################
# Step 9's work: the front_door setting.
#
# The customer panel reads hitechcloud.config flat and starts its Rust front
# door only on front_door=rust (hitechcloudpanel internal/core/frontdoor); any
# other value, or none, is the Go server on the port as before. 2087's panel
# settings page shows and saves the key under [PANEL].
#
#   HITECHCLOUD_FRONT_DOOR unset   the key is added as front_door=go when the
#                                  file lacks it; an existing value is kept
#   HITECHCLOUD_FRONT_DOOR=rust|go that value is written, whatever was there
#   anything else                  refused, nothing written, counted a failure
#
# The panel reads the file when it starts, and the update restarts it.
# HITECHCLOUD_PANEL_CONFIG points it at another file, which is how
# .github/scripts/update_3_9_5_front_door_test.sh runs it.
################################################################################
front_door_step() {
    local cfg="${HITECHCLOUD_PANEL_CONFIG:-/etc/hitechcloud/hitechcloud/conf/hitechcloud.config}"
    local want="${HITECHCLOUD_FRONT_DOOR:-}"
    want="${want,,}"
    if [[ -n "$want" && "$want" != "go" && "$want" != "rust" ]]; then
        echo -e "[3.9.5] ${RED}✗${NC} HITECHCLOUD_FRONT_DOOR=${HITECHCLOUD_FRONT_DOOR} is not go or rust; front_door left as it is." >&2
        FAILURES=$((FAILURES + 1))
        return 0
    fi
    if [[ ! -f "$cfg" ]]; then
        warn "${cfg} does not exist; front_door not written (the panel serves its port from Go)."
        return 0
    fi
    local current
    current="$(grep -E '^front_door=' "$cfg" | tail -n 1 | cut -d= -f2-)"
    if [[ -z "$want" ]]; then
        if grep -qE '^front_door=' "$cfg"; then
            ok "front_door=${current} kept."
            return 0
        fi
        want="go"
    elif [[ "$current" == "$want" ]] && [[ "$(grep -cE '^front_door=' "$cfg")" == 1 ]]; then
        ok "front_door=${want} already."
        return 0
    fi
    if [[ "${DRY_RUN:-0}" == 1 ]]; then
        echo "[3.9.5] would: set front_door=${want} under [PANEL] in ${cfg}"
        return 0
    fi
    local tmp="${cfg}.front_door.$$"
    # Every existing front_door line goes, and one is written under [PANEL]
    # (a [PANEL] section is added at the end if the file has none).
    awk -v line="front_door=${want}" '
        /^front_door=/ { next }
        { print }
        /^\[PANEL\][[:space:]]*$/ && !done { print line; done = 1 }
        END { if (!done) { print ""; print "[PANEL]"; print line } }
    ' "$cfg" > "$tmp" && chmod --reference="$cfg" "$tmp" 2>/dev/null; chown --reference="$cfg" "$tmp" 2>/dev/null
    if [[ -s "$tmp" ]] && mv -f "$tmp" "$cfg"; then
        if [[ "$(grep -cE "^front_door=${want}\$" "$cfg")" == 1 ]]; then
            ok "front_door=${want} in ${cfg}."
            return 0
        fi
    fi
    rm -f "$tmp"
    echo -e "[3.9.5] ${RED}✗${NC} could not write front_door=${want} into ${cfg}." >&2
    FAILURES=$((FAILURES + 1))
    return 0
}

# Step 9 alone, for its test and for an administrator:
#   bash UPDATE.sh --only-front-door [rust|go]
if [[ "${1:-}" == "--only-front-door" ]]; then
    DRY_RUN="${HITECHCLOUD_UPDATE_DRY_RUN:-0}"
    [[ -n "${2:-}" ]] && HITECHCLOUD_FRONT_DOOR="$2"
    front_door_step
    [[ "$FAILURES" -eq 0 ]]; exit $?
fi

if [[ "${1:-}" == "--only-importers" ]]; then
    DRY_RUN="${HITECHCLOUD_UPDATE_DRY_RUN:-0}"
    importers_step
    [[ "$FAILURES" -eq 0 ]]; exit $?
fi

case "${1:-}" in
    --only-vendor)      DRY_RUN="${HITECHCLOUD_UPDATE_DRY_RUN:-0}"; vendor_step;      [[ "$FAILURES" -eq 0 ]]; exit $? ;;
    --only-engines)     DRY_RUN="${HITECHCLOUD_UPDATE_DRY_RUN:-0}"; engines_step;     [[ "$FAILURES" -eq 0 ]]; exit $? ;;
    --only-reapply-394) DRY_RUN="${HITECHCLOUD_UPDATE_DRY_RUN:-0}"; reapply_394_step; [[ "$FAILURES" -eq 0 ]]; exit $? ;;
    --only-sentinel)    DRY_RUN="${HITECHCLOUD_UPDATE_DRY_RUN:-0}"; sentinel_step;    [[ "$FAILURES" -eq 0 ]]; exit $? ;;
    --only-meet)        DRY_RUN="${HITECHCLOUD_UPDATE_DRY_RUN:-0}"; meet_step;        [[ "$FAILURES" -eq 0 ]]; exit $? ;;
    --only-ftp)         DRY_RUN="${HITECHCLOUD_UPDATE_DRY_RUN:-0}"; ftp_step;         [[ "$FAILURES" -eq 0 ]]; exit $? ;;
    --only-orphans)     DRY_RUN="${HITECHCLOUD_UPDATE_DRY_RUN:-0}"; orphans_step;     [[ "$FAILURES" -eq 0 ]]; exit $? ;;
esac

DRY_RUN="${HITECHCLOUD_UPDATE_DRY_RUN:-0}"

if [[ "$DRY_RUN" == 1 ]]; then
    echo "[3.9.5] would: record version ${VERSION} in /usr/local/hitechcloud/version and /etc/hitechcloud/version"
else
    mkdir -p /usr/local/hitechcloud /etc/hitechcloud 2>/dev/null || true
    echo "$VERSION" > /usr/local/hitechcloud/version 2>/dev/null || true
    echo "$VERSION" > /etc/hitechcloud/version 2>/dev/null || true
fi

################################################################################
# 1. Every importer the admin offers is on disk (see importers_step above).
################################################################################
importers_step

################################################################################
# 2-6. What the 3.9.4 update left undone (see the steps above). The order
# matters: 3.9.4's step 11 needs vendor.json's dbip entry, and its step 6 needs
# this release's admin engine.
################################################################################
vendor_step
engines_step
reapply_394_step
sentinel_step
meet_step

################################################################################
# 7. FTP refuses anonymous logins and offers TLS (see ftp_step above).
# 8. What deleted accounts and domains left behind, listed for review.
#
# The IPv6 ports of a firewall where IPv6 was already on are 3.9.4's step 9,
# fixed in this release's copy and run again by step 4 above.
################################################################################
ftp_step
orphans_step

################################################################################
# 9. The front_door setting (see front_door_step above): off unless asked.
################################################################################
front_door_step

################################################################################
if [[ "$FAILURES" -gt 0 ]]; then
    echo -e "[3.9.5] ${RED}$FAILURES check(s) left something undone - see above.${NC}" >&2
    exit 1
fi
ok "version ${VERSION} applied."
exit 0