#!/bin/bash ################################################################################ # Script Name: version/3.9.4/UPDATE.sh # Description: Apply version 3.9.4: # - nothing on any server has ever scored a message for spam. # rspamd is a component of the mail module, and every install # configures mail BEFORE rspamd exists, so the step that puts # rspamd's milter into Postfix ran while rspamd was not there # and quietly did nothing. Nothing ran it again. # - on the RHEL 10 family, Dovecot's Sieve plugin was reported as # missing while it was installed and working, because the probe # only looked in a modules/ subdirectory el10 does not have. # That branch also returned before it compiled the server-wide # spam rule and before it handed the sieve directory to vmail. # # The first two need REPAIR, not just new binaries. A fresh install now # does the right thing; a server that is already running has a # main.cf with no 11332 in it and a sieve tree nothing owns, and # a new binary that reads those finds them exactly as before. # This is the 3.7.2 lesson, and then the 3.9.3 lesson: a fix # whose effect only arrives two releases later, because nothing # repaired what was already written. # - server backup credentials in files accounts can read: the # destination an administrator set on 2087 was copied, keys # and all, into every new account's own backup.env. It now # lives in a root-only file and accounts carry a reference; # this moves what is already out there (step 4). # - the webmail calls `hitechcloudcli mailbox` now, not # `native-mailbox`, and reaches it through one sudoers rule # that named only the old word. sudo matches words, not # aliases, so the rule is rewritten to name both (step 5). # - AI becomes a plan feature (ai_assistant); on a server where AI # is on, every plan that exists keeps it (step 6), and the # provider key becomes root's alone (step 7). # - plans gain mail_storage_gb, the account-wide mailbox storage # a plan sells; the Email Pro tiers take the storage figure # they were sold with, every other plan keeps no limit (step 8). # - the firewall filters IPv6: CSF's default IPV6 = "0" had left # every port open to IPv6 clients on servers with the firewall # on. Switched on here where the firewall is on, and rolled # back if SSH over IPv6 would not be accepted (step 9). # - fail2ban jails mail and FTP logins (it ran sshd and the panel # only), and the optional HAProxy edge router binds IPv6 as # well as IPv4 (step 10). # - the monthly country-database job is scheduled, the database # fetched once, and every site's vhost re-rendered so an empty # site shows the panel's page on every WebEngine (step 11). # Company/Brand: HiTechCloud by Pho Tue SoftWare Solutions JSC # Copyright (c) HiTechCloud by Pho Tue SoftWare Solutions JSC ################################################################################ set -uo pipefail GREEN='\033[0;32m'; YELLOW='\033[1;33m'; RED='\033[0;31m'; NC='\033[0m' ok() { echo -e "[3.9.4] ${GREEN}✓${NC} $*"; } warn() { echo -e "[3.9.4] ${YELLOW}!${NC} $*" >&2; } FAILURES=0 bad() { echo -e "[3.9.4] ${RED}✗${NC} $*" >&2; FAILURES=$((FAILURES + 1)); } [[ $EUID -eq 0 ]] || { echo "[3.9.4] This update must run as root." >&2; exit 1; } ################################################################################ # Step 4's work, defined here so it can also be run on its own (see below). # # The server's backup destination - bucket, host, keys - used to be copied # into every new account's backup.env by user-add, a file the account owns and # that its own Backups > Settings page prints. `hitechcloudcli snapshots # server-default migrate` moves the destination into the root-only # /etc/hitechcloud/backups/server-destination.env, takes it out of the # template new accounts are given, and replaces every account's copy with # DESTINATION=server-default. It changes nothing on a server where no # destination was ever entered. # # This step never fails the update. A server that cannot run it is no worse # off than before the update, and a failed update would stop the version from # being recorded over something the administrator can run by hand. What it # must never do is stay quiet: when credentials had been readable by accounts # it says so, loudly, because moving the keys does not un-read them. # # HITECHCLOUD_CLI, HITECHCLOUD_BACKUP_TEMPLATE, HITECHCLOUD_BACKUP_SERVER_FILE # and HITECHCLOUD_BACKUP_HOMES point it at another CLI and another tree, which # is how .github/scripts/update_3_9_4_backup_credentials_test.sh runs it. ################################################################################ backup_credentials_step() { local cli="${HITECHCLOUD_CLI:-hitechcloudcli}" local args=(snapshots server-default migrate --json) [[ -n "${HITECHCLOUD_BACKUP_TEMPLATE:-}" ]] && args+=(--template "$HITECHCLOUD_BACKUP_TEMPLATE") [[ -n "${HITECHCLOUD_BACKUP_SERVER_FILE:-}" ]] && args+=(--server-file "$HITECHCLOUD_BACKUP_SERVER_FILE") [[ -n "${HITECHCLOUD_BACKUP_HOMES:-}" ]] && args+=(--homes "$HITECHCLOUD_BACKUP_HOMES") [[ "${DRY_RUN:-0}" == 1 ]] && args+=(--dry-run) local by_hand="hitechcloudcli snapshots server-default migrate" if ! command -v "$cli" >/dev/null 2>&1; then warn "hitechcloudcli is not on PATH, so server backup credentials were not taken out of account files. Run: ${by_hand}" return 0 fi local out rc out=$(timeout 300 "$cli" "${args[@]}" 2>/dev/null); rc=$? # --json prints its report even when a file could not be changed (and # then exits 1); no report at all means the command did not run. if [[ "$out" != *'"rotate_keys":'* ]]; then warn "could not check account files for server backup credentials (exit ${rc}). Run it by hand and read the output: ${by_hand}" return 0 fi local accounts failed accounts=$(sed -n 's/.*"accounts":\[\([^]]*\)\].*/\1/p' <<<"$out" | tr -d '"' | sed 's/,/, /g') failed=$(grep -o '"failed":\[[^]]*\]' <<<"$out" | grep -c '\["' || true) if [[ "$out" == *'"server_file_written":true'* ]]; then ok "the server's backup destination was moved into the root-only /etc/hitechcloud/backups/server-destination.env." fi if [[ "$out" == *'"template_changed":true'* ]]; then ok "the backup.env new accounts are given no longer holds any destination setting; it names the server's destination by reference." fi if [[ -n "$accounts" ]]; then ok "server backup credentials replaced with DESTINATION=server-default in: ${accounts}" fi if [[ "$out" == *'"rotate_keys":true'* ]]; then warn "ROTATE YOUR BACKUP STORAGE KEYS. The server's backup credentials were readable by hosting accounts on this server (in their own backup.env, or the world-readable template). They have been moved, but anyone who read them still has them: create new keys at the storage provider, delete the old ones, and enter the new ones on 2087 (Backups > User Backups > Configuration)." fi if [[ "$failed" != 0 ]]; then warn "some backup.env files could not be changed (a symlink or unreadable file is left alone). Run it by hand and read the output: hitechcloudcli snapshots server-default migrate" fi if [[ "$out" == *'"rotate_keys":false'* && -z "$accounts" && "$out" != *'"template_changed":true'* ]]; then ok "no server backup credential in any file an account can read." fi return 0 } # Step 4 alone, for its test. Records no version and touches nothing else. It # exits the way the whole update would: 1 only if the step counted a failure, # which it must never do. ################################################################################ # Step 5's work, defined here so it can also be run on its own (see below). # # The webmail's service account reaches `hitechcloudcli mailbox` - a mailbox's # own password and Sieve script - through one sudoers rule, which `mail # install` wrote as # hitechwebmail ALL=(root) NOPASSWD: /usr/local/bin/hitechcloudcli native-mailbox * # This release's webmail calls `mailbox`, and sudo compares the command word # for word: under that rule every password change and filter save from the # webmail would be refused. The CLI in this release writes the rule with both # words; `hitechcloudcli mailbox sudoers` rewrites it on its own. It stages # the file under a name sudo ignores, checks it with `visudo -cf`, and only # then renames it over the live one, so a rule that does not validate never # replaces the one in place and sudo keeps working either way. # # Only where the rule already exists: a server without the webmail has none, # and `mail install` writes the new one when the webmail arrives. # # This one does fail the update. Without it the webmail shipped in this same # release cannot change a password, and "the update succeeded" would hide it. # # HITECHCLOUD_CLI and HITECHCLOUD_MAILBOX_SUDOERS point it at another CLI and # another file, which is how .github/scripts/update_3_9_4_mailbox_sudoers_test.sh # runs it. ################################################################################ mailbox_sudoers_step() { local cli="${HITECHCLOUD_CLI:-hitechcloudcli}" local file="${HITECHCLOUD_MAILBOX_SUDOERS:-/etc/sudoers.d/hitechcloud-webmail-mailbox}" local by_hand="hitechcloudcli mailbox sudoers" if [[ ! -e "$file" ]]; then ok "No webmail sudoers rule on this server; the webmail's install writes it with both command names." return 0 fi if grep -qF 'hitechcloudcli mailbox *' "$file" 2>/dev/null; then ok "The webmail's sudoers rule already names \`mailbox\`." return 0 fi if [[ "${DRY_RUN:-0}" == 1 ]]; then echo "[3.9.4] would: ${by_hand}, so the webmail may call \`mailbox\` as well as \`native-mailbox\`" return 0 fi if ! command -v "$cli" >/dev/null 2>&1; then bad "hitechcloudcli is not on PATH, so the webmail's sudoers rule still names only native-mailbox and its password and filter pages will be refused. Run: ${by_hand}" return 0 fi "$cli" mailbox sudoers >/dev/null 2>&1 if grep -qF 'hitechcloudcli mailbox *' "$file" 2>/dev/null && grep -qF 'hitechcloudcli native-mailbox *' "$file" 2>/dev/null; then ok "The webmail's sudoers rule names \`mailbox\` and \`native-mailbox\`, checked by visudo before it replaced the old one." else bad "The webmail's sudoers rule could not be rewritten; the old one is still in place and sudo works, but the webmail's password and filter pages will be refused. Run it by hand and read the output: ${by_hand}" fi return 0 } ################################################################################ # Step 9's work: the firewall's IPv6 half, on servers where the firewall is on. # # CSF ships with IPV6 = "0" (csf.generic.conf line 490 of the csf.tgz the # installer downloads) and `hitechcloudcli firewall on` never changed it. With # that, CSF does not touch ip6tables: the IPv6 INPUT policy stays ACCEPT, so on # a server with "the firewall on" every port was open to IPv6 clients, and no # IPv6 address could be banned (`csf -d` answers "IPV6 is not enabled in # csf.conf") - lfd's bans and the panels' login bans did nothing to an IPv6 # attacker. This release's `firewall on` switches IPv6 on; a server where the # firewall is already on never runs `firewall on` again, so this does it once. # # What it does, only where CSF is installed AND enabled (no csf.disable): # - TCP6_IN / UDP6_IN get the ports TCP_IN / UDP_IN already open, keeping # what they had. CSF's own TCP6_IN names none of 2083, 2087 or 2096, so # switching IPv6 on without this would shut IPv6 clients out of the panel. # - IPV6 = "1", then `csf -r`. # - Then it reads what ip6tables actually loaded. If a port sshd listens on # is not accepted there, csf.conf is put back as it was and CSF reloaded, # so an administrator who reaches this server over IPv6 is never locked # out by an update; it says exactly why and what to run by hand. # Where the firewall is off it changes nothing and says one line. Without # ip6tables CSF would refuse to start with IPv6 on, so it changes nothing. # # This step never fails the update: a server left as it was is no worse off # than before, and the reason is printed. # # HITECHCLOUD_CSF_DIR, HITECHCLOUD_CSF_BIN, HITECHCLOUD_IP6TABLES and # HITECHCLOUD_SSHD_CONFIG point it at another tree and stand-in binaries, # which is how .github/scripts/update_3_9_4_csf_ipv6_test.sh runs it without # CSF or ip6tables ever being run. ################################################################################ csf_list() { sed -n "s/^$1 = \"\\([^\"]*\\)\".*/\\1/p" "$2" | head -n1; } # Port lists merged the way `firewall ports` does: numbers and ranges only, # sorted on the leading number, the first of an equal key kept. csf_merge_ports() { printf '%s\n' "$1" "$2" | tr ',' '\n' | grep -E '^[0-9]+(:[0-9]+)?$' \ | awk -F: '!seen[$1]++' | sort -t: -k1,1n | paste -sd, - } csf_ssh_ports() { local conf="$1" ports ports=$(awk '/^[[:space:]]*Port[[:space:]]+[0-9]+/ {print $2}' "$conf" "${conf}.d"/*.conf 2>/dev/null | sort -un | paste -sd' ' -) echo "${ports:-22}" } # Does `ip6tables -S` text accept a new connection to port $2? OUTPUT rules # do not count, and 2222 is not 222. csf_rules_accept() { grep -v '^-A OUTPUT' <<<"$1" | grep -E -- "--dport $2([ ,]|$)" | grep -qE -- '-j (ACCEPT|LOGACCEPT)' } csf_ipv6_step() { local dir="${HITECHCLOUD_CSF_DIR:-/etc/csf}" local csf="${HITECHCLOUD_CSF_BIN:-csf}" local ip6t="${HITECHCLOUD_IP6TABLES:-ip6tables}" local sshd="${HITECHCLOUD_SSHD_CONFIG:-/etc/ssh/sshd_config}" local conf="$dir/csf.conf" local by_hand="hitechcloudcli firewall on" if [[ ! -f "$conf" ]] || ! command -v "$csf" >/dev/null 2>&1; then ok "CSF is not installed here; there is no firewall to extend to IPv6." return 0 fi if [[ -e "$dir/csf.disable" ]]; then ok "The firewall is off; IPv6 will be filtered too when it is turned on (${by_hand})." return 0 fi if grep -qx 'IPV6 = "1"' "$conf"; then ok "The firewall already filters IPv6." return 0 fi if ! command -v "$ip6t" >/dev/null 2>&1; then warn "The firewall is on but IPv6 is NOT filtered, and ip6tables is not installed, so CSF cannot filter it. Install the iptables package, then run: ${by_hand}" return 0 fi local tcp4 udp4 tcp6 udp6 ssh p tcp4=$(csf_list TCP_IN "$conf"); udp4=$(csf_list UDP_IN "$conf") tcp6=$(csf_merge_ports "$tcp4" "$(csf_list TCP6_IN "$conf")") udp6=$(csf_merge_ports "$udp4" "$(csf_list UDP6_IN "$conf")") ssh=$(csf_ssh_ports "$sshd") for p in $ssh; do if ! tr ',' '\n' <<<"$tcp6" | awk -F: -v p="$p" '($2=="" && $1==p) || ($2!="" && p>=$1 && p<=$2) {f=1} END {exit !f}'; then warn "IPv6 was NOT switched on: sshd listens on port ${p}, which the firewall's TCP_IN does not open, so switching IPv6 on could lock IPv6 SSH out. Add ${p} to TCP_IN in ${conf}, then run: ${by_hand}" return 0 fi done if [[ "${DRY_RUN:-0}" == 1 ]]; then echo "[3.9.4] would: set IPV6 = \"1\", TCP6_IN = \"${tcp6}\", UDP6_IN = \"${udp6}\" in ${conf} and reload CSF" return 0 fi local backup="${conf}.pre-3.9.4-ipv6" if ! cp -p "$conf" "$backup"; then warn "IPv6 was NOT switched on: ${conf} could not be backed up first. Run: ${by_hand}" return 0 fi sed -i -e "s/^IPV6 = .*/IPV6 = \"1\"/" \ -e "s/^TCP6_IN = .*/TCP6_IN = \"${tcp6}\"/" \ -e "s/^UDP6_IN = .*/UDP6_IN = \"${udp6}\"/" "$conf" if ! grep -qx 'IPV6 = "1"' "$conf" || ! grep -qx "TCP6_IN = \"${tcp6}\"" "$conf"; then cp -p "$backup" "$conf" warn "IPV6 was NOT switched on: ${conf} has no IPV6 or TCP6_IN line to set (an old or hand-edited csf.conf). It is unchanged. Run: ${by_hand}" return 0 fi local reloaded=1 rules missing="" timeout 300 "$csf" -r >/dev/null 2>&1 || reloaded=0 rules=$("$ip6t" -S 2>/dev/null) for p in $ssh; do csf_rules_accept "$rules" "$p" || missing="${missing:+$missing,}$p" done if [[ $reloaded -eq 1 && -z "$missing" ]]; then ok "The firewall now filters IPv6 too (IPV6 = \"1\"; TCP6_IN = \"${tcp6}\"), and ip6tables accepts SSH on ${ssh// /,}. The previous csf.conf is ${backup}." return 0 fi cp -p "$backup" "$conf" timeout 300 "$csf" -r >/dev/null 2>&1 if [[ $reloaded -eq 0 ]]; then warn "IPv6 was NOT switched on: CSF failed to reload with IPV6 = \"1\" (\`csf -r\` exited non-zero), so csf.conf was put back and CSF reloaded as it was. Run by hand and read the output: ${by_hand}" else warn "IPv6 was NOT switched on: with IPV6 = \"1\" the loaded ip6tables rules did not accept SSH on port(s) ${missing}, which would lock out anyone reaching this server's SSH over IPv6. csf.conf was put back and CSF reloaded as it was. Check \`ip6tables -S\` and csf.allow, then run: ${by_hand}" fi return 0 } ################################################################################ # Step 10's work: what a fresh install now writes, brought to a server that # already has the old version of it. Both are the CLI's own code, not this # script reimplementing it: # - fail2ban's jail.d/hitechcloud.conf gained [postfix] (SASL failures # counted, both postfix unit names), [dovecot] and [pure-ftpd]. Only a # server whose jail file is ours and lacks them is touched, by re-running # the fail2ban component, which rewrites the file and reloads fail2ban. # - a managed haproxy.cfg with `bind *:` binds IPv4 only; # `enterprise haproxy repair` rewrites those binds to `:::PORT v4v6`, # checks the file with haproxy -c and restarts it. # Neither is installed where it is absent. Never fails the update: the server # is left as it was and the command to run is printed. # # HITECHCLOUD_CLI, HITECHCLOUD_FAIL2BAN_JAILS and HITECHCLOUD_HAPROXY_CFG # point it at a stand-in CLI and throwaway files, which is how # .github/scripts/update_3_9_4_jails_haproxy_test.sh runs it. ################################################################################ jails_haproxy_step() { local cli="${HITECHCLOUD_CLI:-hitechcloudcli}" local jails="${HITECHCLOUD_FAIL2BAN_JAILS:-/etc/fail2ban/jail.d/hitechcloud.conf}" local hap="${HITECHCLOUD_HAPROXY_CFG:-/etc/haproxy/haproxy.cfg}" if [[ ! -f "$jails" ]] || ! grep -q "Managed by HiTechCloud" "$jails" 2>/dev/null; then ok "fail2ban is not managed by this panel here; no jails to add." elif grep -q '^\[postfix\]' "$jails" && grep -q '^\[dovecot\]' "$jails" && grep -q '^\[pure-ftpd\]' "$jails"; then ok "fail2ban already jails mail and FTP logins." elif [[ "${DRY_RUN:-0}" == 1 ]]; then echo "[3.9.4] would: ${cli} native-install --extras-only --with fail2ban, to add the mail and FTP jails" elif ! command -v "$cli" >/dev/null 2>&1; then warn "hitechcloudcli is not on PATH, so fail2ban still does not jail mail and FTP logins. Run: hitechcloudcli native-install --extras-only --with fail2ban" elif timeout 600 "$cli" native-install --extras-only --with fail2ban >/dev/null 2>&1 && grep -q '^\[postfix\]' "$jails"; then ok "fail2ban now jails SMTP AUTH, IMAP/POP3 and FTP logins, over IPv6 as over IPv4." else warn "fail2ban's mail and FTP jails could not be added. Run it by hand and read the output: hitechcloudcli native-install --extras-only --with fail2ban" fi if [[ ! -f "$hap" ]] || ! grep -q "HITECHCLOUD-MANAGED" "$hap" 2>/dev/null; then ok "No managed HAProxy edge router here." elif ! sed -n '/HITECHCLOUD-MANAGED ABOVE/q;p' "$hap" | grep -q '^[[:space:]]*bind \*:'; then ok "The HAProxy edge router already binds IPv6 as well as IPv4." elif [[ "${DRY_RUN:-0}" == 1 ]]; then echo "[3.9.4] would: ${cli} enterprise haproxy repair, to bind the edge router on IPv6 as well" elif ! command -v "$cli" >/dev/null 2>&1; then warn "hitechcloudcli is not on PATH, so the HAProxy edge router still binds IPv4 only. Run: hitechcloudcli enterprise haproxy repair" elif timeout 300 "$cli" enterprise haproxy repair >/dev/null 2>&1 && ! sed -n '/HITECHCLOUD-MANAGED ABOVE/q;p' "$hap" | grep -q '^[[:space:]]*bind \*:'; then ok "The HAProxy edge router binds IPv6 as well as IPv4 (:::PORT v4v6), checked by haproxy -c and restarted." else warn "The HAProxy edge router could not be moved to IPv6 binds. Run it by hand and read the output: hitechcloudcli enterprise haproxy repair" fi return 0 } ################################################################################ # Step 11's work: three things 3.9.4 ships that an existing server only gets # if something runs them, because otherwise they arrive with a schedule or a # re-render that may be weeks away (the 3.7.2 lesson again): # - the scheduled jobs: 3.9.4's cron gains the monthly country-database # refresh. `native-install --scheduled-jobs-only` adds the lines this # server lacks and never touches one that is there. # - the country database itself: the login history looks a login's country # up in DB-IP's IP to Country Lite database first. Its job runs on the 5th # of the month; until then a server would have none and would use the # external lookup (or none, if the administrator turned that off). It is # fetched once here, through the pinned vendor.json entry (mirror first, # SHA-256 checked either way), unless a copy is already there. # - the WebEngine vhosts: a new site's empty document root now shows the # panel's default page on every engine, and a suspended site the page for # its reason. Both are in the rendered configs, so the sites that exist # are re-rendered with `vhost rebuild --all`. # All three are the CLI's own code. None fails the update: each says what to # run by hand when it could not be done. # # HITECHCLOUD_CLI, HITECHCLOUD_NATIVE_CONF and HITECHCLOUD_GEOIP_DIR point it # at a stand-in CLI and a throwaway tree, which is how # .github/scripts/update_3_9_4_schedule_geoip_vhosts_test.sh runs it. ################################################################################ schedule_geoip_vhosts_step() { local cli="${HITECHCLOUD_CLI:-hitechcloudcli}" local conf="${HITECHCLOUD_NATIVE_CONF:-/etc/hitechcloud/native/native.conf}" local dir="${HITECHCLOUD_GEOIP_DIR:-/var/lib/hitechcloud/geoip}" local db="${dir}/dbip-country-lite.mmdb" if [[ ! -f "$conf" ]]; then ok "Not a host-services install: no scheduled jobs, country database or vhosts to bring over." return 0 fi if [[ "${DRY_RUN:-0}" == 1 ]]; then echo "[3.9.4] would: ${cli} native-install --scheduled-jobs-only" [[ -s "$db" ]] || echo "[3.9.4] would: ${cli} vendor fetch dbip-country-lite ${db}.gz, and unpack it to ${db}" echo "[3.9.4] would: ${cli} vhost rebuild --all" return 0 fi if ! command -v "$cli" >/dev/null 2>&1; then warn "hitechcloudcli is not on PATH. Run by hand: hitechcloudcli native-install --scheduled-jobs-only; hitechcloudcli vendor fetch dbip-country-lite ${db}.gz && gunzip -c ${db}.gz > ${db}; hitechcloudcli vhost rebuild --all" return 0 fi if timeout 300 "$cli" native-install --scheduled-jobs-only >/dev/null 2>&1; then ok "Scheduled jobs are all in place, including the monthly country-database refresh." else warn "The scheduled jobs could not be checked. Run it by hand and read the output: hitechcloudcli native-install --scheduled-jobs-only" fi if [[ -s "$db" ]]; then ok "The country database for the login history is already here." elif mkdir -p "$dir" \ && timeout 300 "$cli" vendor fetch dbip-country-lite "${db}.gz" >/dev/null 2>&1 \ && gunzip -c "${db}.gz" > "${db}.part" 2>/dev/null \ && [[ -s "${db}.part" ]] && mv -f "${db}.part" "$db"; then ok "The country database for the login history is in place (DB-IP IP to Country Lite, checksum verified)." else rm -f "${db}.part" warn "The country database could not be fetched; logins use the external lookup until the monthly job gets it. Run by hand: hitechcloudcli vendor fetch dbip-country-lite ${db}.gz && gunzip -c ${db}.gz > ${db}" fi if timeout 1800 "$cli" vhost rebuild --all >/dev/null 2>&1; then ok "Every site's vhost re-rendered: an empty site shows the panel's page on every WebEngine." else warn "Some vhosts could not be re-rendered. Run it by hand and read the output: hitechcloudcli vhost rebuild --all" fi return 0 } if [[ "${1:-}" == "--only-schedule-geoip-vhosts" ]]; then DRY_RUN="${HITECHCLOUD_UPDATE_DRY_RUN:-0}" schedule_geoip_vhosts_step [[ "$FAILURES" -eq 0 ]]; exit $? fi if [[ "${1:-}" == "--only-jails-haproxy" ]]; then DRY_RUN="${HITECHCLOUD_UPDATE_DRY_RUN:-0}" jails_haproxy_step [[ "$FAILURES" -eq 0 ]]; exit $? fi if [[ "${1:-}" == "--only-csf-ipv6" ]]; then DRY_RUN="${HITECHCLOUD_UPDATE_DRY_RUN:-0}" csf_ipv6_step [[ "$FAILURES" -eq 0 ]]; exit $? fi # Step 5 alone, for its test. Records no version and touches nothing else. if [[ "${1:-}" == "--only-mailbox-sudoers" ]]; then DRY_RUN="${HITECHCLOUD_UPDATE_DRY_RUN:-0}" mailbox_sudoers_step [[ "$FAILURES" -eq 0 ]]; exit $? fi if [[ "${1:-}" == "--only-backup-credentials" ]]; then DRY_RUN="${HITECHCLOUD_UPDATE_DRY_RUN:-0}" backup_credentials_step [[ "$FAILURES" -eq 0 ]]; exit $? fi VERSION="3.9.4" DRY_RUN="${HITECHCLOUD_UPDATE_DRY_RUN:-0}" if [[ "$DRY_RUN" == 1 ]]; then echo "[3.9.4] would: record version ${VERSION} in /usr/local/hitechcloud/version and /etc/hitechcloud/version" else mkdir -p /usr/local/hitechcloud /etc/hitechcloud 2>/dev/null || true echo "$VERSION" > /usr/local/hitechcloud/version 2>/dev/null || true echo "$VERSION" > /etc/hitechcloud/version 2>/dev/null || true fi native() { [[ -f /etc/hitechcloud/native/native.conf ]]; } # The mail stack has actually been configured here, not merely packaged. Before # this file exists there is nothing to repair and nothing this script should # touch: a server that never asked for mail hosting does not get its Postfix # edited because an update ran. mail_configured() { [[ -f /etc/hitechcloud/mail/postfix-main.cf ]]; } MAIL_DIR=/etc/hitechcloud/mail SIEVE_DIR="${MAIL_DIR}/sieve" SPAM_RULE="${SIEVE_DIR}/before.d/10-spam-to-junk.sieve" RSPAMD_MILTER_PORT=11332 ################################################################################ # 1. Is anything on this server scoring mail for spam? # # Measured on the 3.9.3 demo, an AlmaLinux 10.2 native install: # # # postconf -n | grep smtpd_milters # smtpd_milters = inet:127.0.0.1:8891 # # ls /etc/rspamd/local.d # ls: cannot access '/etc/rspamd/local.d': No such file or directory # # 8891 is opendkim. 11332 is rspamd, and it is not there. # # The cause is an ordering nobody could see from one file: INSTALL.sh asks for # "...,mail,rspamd,..." and the module registry's mail module declares its # components as ["mail", "rspamd"], and native-install walks that list in # order. So `native-mail install` -> mail_configure_milters() always ran while # rspamd was not yet on the machine, and that function opens with # which("rspamd"). It wrote a milter list with no rspamd in it, rspamd then # installed and came up active, and nothing re-ran the step. # # What a customer saw: the spam quarantine page empty, every message in INBOX, # no rspamd verdict on the mail-trust page, and every spam-filter setting they # saved stored and applied to nothing. Nothing looked wrong - the component was # installed, the unit was active, and Postfix had no reason to log anything. # # The repair is one idempotent command in each of the two cases, and both are # the product's own code rather than this script reimplementing it: the CLI has # already been replaced by the time this runs (step 2 of the update, before # this directory at step 5), so `native-install --extras-only --with rspamd` # now ends by wiring it in, and `native-mail install` now re-runs the milter # step. # # --extras-only is not optional here. A plain `native-install --with X` re-runs # the WHOLE stack install - web engines, PHP, every component - which is right # for a fresh machine and wrong for a running one, and install.rs says exactly # that where the flag is parsed. Installing a spam filter must not reinstall a # customer's web server. ################################################################################ if ! native; then ok "Not a native install: the spam filter's wiring is not this server's concern." elif ! mail_configured; then ok "This server does not carry mail, so there is no spam filter to wire in." elif ! command -v postconf >/dev/null 2>&1; then warn "postconf is not on PATH, so the spam filter's wiring could not be checked." else milters=$(postconf -h smtpd_milters 2>/dev/null) if [[ "$milters" == *":${RSPAMD_MILTER_PORT}"* ]]; then ok "Postfix hands every incoming message to rspamd (${milters})." elif ! command -v rspamd >/dev/null 2>&1; then warn "no spam filter is installed on a server that carries mail, so nothing scores a single incoming message." if [[ "$DRY_RUN" == 1 ]]; then echo "[3.9.4] would: hitechcloudcli native-install --extras-only --with rspamd" elif ! command -v hitechcloudcli >/dev/null 2>&1; then bad "hitechcloudcli is not on PATH, so rspamd could not be installed. Run: hitechcloudcli native-install --extras-only --with rspamd" elif hitechcloudcli native-install --extras-only --with rspamd >/dev/null 2>&1 \ && [[ "$(postconf -h smtpd_milters 2>/dev/null)" == *":${RSPAMD_MILTER_PORT}"* ]]; then ok "rspamd installed and wired into Postfix; incoming mail is scored from now on." else bad "rspamd is still not scoring anything. Run it by hand and read the output: hitechcloudcli native-install --extras-only --with rspamd" fi else warn "rspamd is installed and Postfix never asks it anything, so no message has ever been scored on this server." if [[ "$DRY_RUN" == 1 ]]; then echo "[3.9.4] would: hitechcloudcli native-mail install, to write rspamd's rules and put its milter into Postfix" elif ! command -v hitechcloudcli >/dev/null 2>&1; then bad "hitechcloudcli is not on PATH, so the milter could not be added. Run: hitechcloudcli native-mail install" elif hitechcloudcli native-mail install >/dev/null 2>&1 \ && [[ "$(postconf -h smtpd_milters 2>/dev/null)" == *":${RSPAMD_MILTER_PORT}"* ]]; then ok "rspamd wired into Postfix; incoming mail is scored from now on." else bad "rspamd is still not in smtpd_milters, so nothing is scored. Run it by hand and read the output: hitechcloudcli native-mail install" fi fi fi ################################################################################ # 2. The two things the false "Sieve is not installed" left undone. # # `sieve_available` searched /usr/lib/dovecot/modules and # /usr/lib64/dovecot/modules only. The RHEL 10 family has neither: on # AlmaLinux 10.2, dovecot-pigeonhole-2.3.21-19.el10_2.1 installs # lib90_sieve_plugin.so straight into /usr/lib64/dovecot/. # # So on every el10 server the panel said "every filter and autoresponder you # save is written to disk and never run" while Sieve worked - and # sieve_configure took the same branch and returned early, before it # precompiled the server-wide spam rule with sievec and before it handed the # sieve tree to vmail. The warning broke the one thing it claimed was broken. # # The probe is fixed in the binary. These two are what it left on disk, and # each is one idempotent command. ################################################################################ sieve_plugin_present() { local d for d in /usr/lib/dovecot/modules /usr/lib64/dovecot/modules \ /usr/lib/dovecot /usr/lib64/dovecot; do [[ -d "$d" ]] || continue compgen -G "${d}/lib*sieve_plugin.so" >/dev/null 2>&1 && return 0 done return 1 } if ! native || ! mail_configured; then : # Said above. elif ! sieve_plugin_present; then warn "Dovecot's Sieve plugin really is absent here, so filters and autoresponders are saved and never run. Install it: hitechcloudcli native-install --extras-only --with mail" else # The compiled form Dovecot loads. Missing it is not fatal - Dovecot # compiles at delivery - but it is the step that reports a broken rule at # update time rather than silently per message. if [[ -f "$SPAM_RULE" && ! -f "${SPAM_RULE%.sieve}.svbin" ]]; then if [[ "$DRY_RUN" == 1 ]]; then echo "[3.9.4] would: sievec ${SPAM_RULE}" elif ! command -v sievec >/dev/null 2>&1; then warn "sievec is not on PATH, so the server-wide spam rule stays uncompiled; Dovecot will compile it at delivery." elif sievec "$SPAM_RULE" >/dev/null 2>&1; then ok "the server-wide spam rule is compiled, so spam is filed into Junk where the quarantine page reads it." else bad "the server-wide spam rule does not compile, so scored spam is delivered to INBOX: sievec ${SPAM_RULE}" fi elif [[ -f "$SPAM_RULE" ]]; then ok "the server-wide spam rule is compiled." fi # Ownership. Dovecot runs the script as vmail; a tree root owns is a tree # it cannot write its own compiled output into. if [[ -d "$SIEVE_DIR" ]] && id vmail >/dev/null 2>&1; then owner=$(stat -c '%U' "$SIEVE_DIR" 2>/dev/null) if [[ "$owner" == "vmail" ]]; then ok "the sieve tree belongs to vmail, which is the account Dovecot runs a script as." elif [[ "$DRY_RUN" == 1 ]]; then echo "[3.9.4] would: chown -R vmail:vmail ${SIEVE_DIR} (it is owned by ${owner:-unknown})" elif chown -R vmail:vmail "$SIEVE_DIR" >/dev/null 2>&1; then ok "the sieve tree handed to vmail (it was owned by ${owner:-unknown}), so Dovecot can read and compile what the panel writes there." else bad "the sieve tree is owned by ${owner:-unknown} and could not be handed to vmail, so filters may not run: chown -R vmail:vmail ${SIEVE_DIR}" fi fi fi ################################################################################ # 3. Did the webmail's folder ordering actually reach this server? # # Not a repair - a sidebar order leaves nothing wrong on disk - but worth # asserting, because "the update ran" and "the update replaced the built # frontend" are not the same claim, and this is a built asset rather than a # file the update copies. ################################################################################ webmail_dist=/opt/hitechcloud/mailflow/frontend/dist if [[ ! -d "$webmail_dist" ]]; then ok "No webmail on this server, so its folder ordering is not this server's concern." elif grep -rqs "compareFolderTargets\|specialUseRank" "$webmail_dist" 2>/dev/null; then ok "the webmail's move menus order folders the way its sidebar does." else # Minified output renames locals but not much else; treat a miss as advice # rather than a failure, because the name may simply not have survived. warn "could not find the shared folder-ordering helper in the built webmail. If the move menus still list folders in a different order from the sidebar, rebuild it: hitechcloudcli native-webmail install" fi ################################################################################ # 4. Server backup credentials out of every file an account can read. # # See backup_credentials_step above. Runs on every server: a Docker install's # accounts were given the same template. ################################################################################ backup_credentials_step ################################################################################ # 5. The webmail's sudoers rule names `mailbox` as well as `native-mailbox`. # # See mailbox_sudoers_step above. The CLI was replaced at step 2 of the update, # before this runs, so `mailbox sudoers` is this release's. ################################################################################ mailbox_sudoers_step ################################################################################ # 6. Nobody loses AI when it becomes a plan feature. # # Until 3.9.4 the assistant, Vibe Code and /api/ai/chat were gated on # "dashboard", which every account has. From 3.9.4 they are the plan feature # ai_assistant, and no plan file lists it - so on a server where AI is switched # on, the new panel would take it from every account the moment it started. # # `hitechcloudadmin-engine ai-plans-upgrade` (hitechcloudadmin's # internal/handlers/ai_usage.rs, where the tests for both directions live) # grants it to every plan that exists now and to nothing created later: it # sets each existing plan's own AI switch - the one the plan form edits, in # /etc/hitechcloud/ai/plan_limits.json under the plan's id - to on, unless the # administrator has already set it. No feature set is touched, so a plan # created later has no switch, its feature set decides, and no shipped set # grants AI. With AI switched off it grants nothing. It runs once: a marker in # /etc/hitechcloud/ai records the first run's decision, so a second run # changes nothing. It also rekeys plan_limits.json by plan id. # # The admin panel is replaced at step 3 of the update, before this directory # runs, so the engine here is the one that has the command. The feature lists # the panel has cached in Redis are dropped afterwards, or an account would # keep reading the list without ai_assistant for up to a day. ################################################################################ AI_ENGINE=/usr/local/bin/hitechcloudadmin-engine if [[ ! -x "$AI_ENGINE" ]]; then bad "$AI_ENGINE is missing, so AI could not be kept for the plans that had it. Run it by hand once the admin panel is updated: $AI_ENGINE ai-plans-upgrade" elif [[ "$DRY_RUN" == 1 ]]; then "$AI_ENGINE" ai-plans-upgrade --dry-run 2>&1 | sed 's/^/[3.9.4] /' elif ai_out=$("$AI_ENGINE" ai-plans-upgrade 2>&1); then while IFS= read -r line; do [[ -n "$line" ]] && ok "AI: $line"; done <<< "$ai_out" if command -v hitechcloudcli >/dev/null 2>&1; then hitechcloudcli cache-forget_features >/dev/null 2>&1 \ || warn "could not drop the cached feature lists; accounts see the change within a day. Run: hitechcloudcli cache-forget_features" fi else bad "AI could not be kept for the plans that had it: ${ai_out}. Run it by hand and read the output: $AI_ENGINE ai-plans-upgrade" fi ################################################################################ # 7. The provider key becomes root's alone. # # gateway.conf holds the AI provider's key. It was root:hitechwebmail 0640 # because the webmail read it and called the provider itself. The webmail now # asks the panel over /run/hitechcloud-ai/gateway.sock instead (hitechcloudmail's # panelGateway.js), so nothing but root needs the file. # # Only once the webmail on this server is the version that asks the panel: an # older webmail still reads the file, and taking it away would switch its AI # off with nothing to say why. HiTechAdmin also writes it 0600 on every save. ################################################################################ AI_CONF=/etc/hitechcloud/ai/gateway.conf WEBMAIL_BACKEND=/opt/hitechcloud/mailflow/backend if [[ ! -f "$AI_CONF" ]]; then ok "AI has never been configured here, so there is no provider key to lock down." elif [[ -d "$WEBMAIL_BACKEND" && ! -f "$WEBMAIL_BACKEND/src/services/panelGateway.js" ]]; then warn "the webmail here still reads $AI_CONF itself, so it stays readable by the webmail's group. Update the webmail (hitechcloudcli native-webmail install), then run: chown root:root $AI_CONF && chmod 600 $AI_CONF" elif [[ "$DRY_RUN" == 1 ]]; then echo "[3.9.4] would: chown root:root $AI_CONF && chmod 600 $AI_CONF" elif chown root:root "$AI_CONF" && chmod 600 "$AI_CONF"; then ok "the AI provider key is readable by root alone." else bad "could not make $AI_CONF root-only. Run: chown root:root $AI_CONF && chmod 600 $AI_CONF" fi ################################################################################ # 8. Mailbox storage is a plan figure of its own. # # A plan had one "disk" figure and two things read it. setquota applied it to # the account's home - websites, files - and the Email Pro tiers sold it as # their mail storage ("2 GB", "25 GB"...), which nothing applied to mail at # all: mailboxes live under /var/mail/vhosts, owned by vmail, outside every # account's home and so outside its quota, and every tier's per-mailbox quota # is '0'. An Email Pro account's mail was unlimited. # # plans.mail_storage_gb is the account-wide mailbox storage: every mailbox of # every domain the account owns, together. 0 is no account-wide limit, which # is what every plan had in effect until now. The migration keeps that for # every plan except the eight Email Pro tiers, which take the figure they were # sold with - the owner's decision: that figure means mailbox storage, and it # is enforced from this release on existing accounts too, with no grace. # Their disk_limit is left as it is; it still limits their home. # # The tiers are found by name: `plan-emailpro add` creates exactly these eight # names and nothing else, so a name is the one thing that marks a row as one. # # Run once, recorded by a marker, not by the column: ALTER TABLE commits on # its own, so a column that exists does not prove the UPDATE after it ran. ################################################################################ MAIL_STORAGE_MARKER=/var/lib/hitechcloud/migrations/3.9.4-mail-storage EMAIL_PRO_TIERS="'Email Pro 1','Email Pro 2','Email Pro 3','Email Pro 4','Email Pro 5','Email Pro 6','Email Pro 7','Email Pro 8'" MAIL_STORAGE_SQL="UPDATE plans SET mail_storage_gb = CAST(REGEXP_SUBSTR(TRIM(disk_limit), '^[0-9]+') AS UNSIGNED) WHERE name IN (${EMAIL_PRO_TIERS}) AND mail_storage_gb = 0 AND TRIM(disk_limit) REGEXP '^[0-9]+';" mail_storage_migrate() { if [[ -f "$MAIL_STORAGE_MARKER" ]]; then ok "plans.mail_storage_gb already migrated." return 0 fi if ! mysql -N -B -e "SELECT 1 FROM panel.plans LIMIT 1;" >/dev/null 2>&1; then warn "The panel database could not be read, so plans.mail_storage_gb was not added; re-run this update once it can be." return 0 fi local present present=$(mysql -N -B -e "SELECT COUNT(*) FROM information_schema.columns WHERE table_schema='panel' AND table_name='plans' AND column_name='mail_storage_gb';" 2>/dev/null) if [[ "$DRY_RUN" == 1 ]]; then [[ "$present" == "0" ]] && echo "[3.9.4] would: ALTER TABLE plans ADD COLUMN mail_storage_gb INT NOT NULL DEFAULT 0" echo "[3.9.4] would: give the Email Pro tiers their storage figure as mailbox storage" return 0 fi # Guarded by information_schema rather than IF NOT EXISTS: MariaDB has # that spelling and MySQL does not (see version/3.0.7/UPDATE.sh). if [[ "$present" == "0" ]] && ! mysql panel -e "ALTER TABLE plans ADD COLUMN mail_storage_gb INT NOT NULL DEFAULT 0;"; then bad "plans.mail_storage_gb could not be added, so no plan can sell mailbox storage and Email Pro mail stays unlimited." return 0 fi if ! mysql panel -e "$MAIL_STORAGE_SQL"; then bad "the Email Pro tiers could not be given their mailbox storage; their mail stays unlimited. Re-run this update." return 0 fi mkdir -p "$(dirname "$MAIL_STORAGE_MARKER")" && date -u +%FT%TZ > "$MAIL_STORAGE_MARKER" local tiers tiers=$(mysql -N -B panel -e "SELECT CONCAT(name, ' = ', mail_storage_gb, ' GB') FROM plans WHERE mail_storage_gb > 0 ORDER BY id;" 2>/dev/null | paste -sd, - | sed 's/,/, /g') ok "plans.mail_storage_gb added. Plans with an account-wide mailbox storage limit: ${tiers:-none}. Every other plan has none, as before." } mail_storage_migrate # Who is already over. The scheduled `mailusage enforce` starts refusing their # mail within five minutes; this is the list, in the update log, before it # does - and the same report the admin panel's Email Settings page shows. if [[ "$DRY_RUN" != 1 ]] && [[ -f "$MAIL_STORAGE_MARKER" ]] && mail_configured && command -v hitechcloudcli >/dev/null 2>&1; then if over=$(hitechcloudcli mailusage accounts --save 2>&1); then full=$(printf '%s\n' "$over" | grep -c 'FULL - mail refused' || true) if [[ "$full" -gt 0 ]]; then warn "${full} account(s) already hold more mail than their plan's mailbox storage; their mail will be refused (452, the sender retries) until they are under it:" printf '%s\n' "$over" | grep -E '^account |FULL' | sed 's/^/ /' >&2 else ok "no account holds more mail than its plan's mailbox storage." fi else warn "mailbox storage could not be measured yet: ${over}" fi fi ################################################################################ # 9. The firewall filters IPv6 as well as IPv4 (see csf_ipv6_step above). ################################################################################ csf_ipv6_step ################################################################################ # 10. fail2ban's mail and FTP jails, and HAProxy on IPv6 (see # jails_haproxy_step above). ################################################################################ jails_haproxy_step ################################################################################ # 11. The scheduled jobs, the country database and the vhosts (see # schedule_geoip_vhosts_step above). ################################################################################ schedule_geoip_vhosts_step ################################################################################ if [[ "$FAILURES" -gt 0 ]]; then echo -e "[3.9.4] ${RED}$FAILURES check(s) left something undone - see above.${NC}" >&2 exit 1 fi ok "version ${VERSION} applied." exit 0