From d3a1dc8b17efe2ac8219cee6baedd67612256422 Mon Sep 17 00:00:00 2001 From: JJ Fullmer Date: Thu, 6 Aug 2026 19:54:22 -0600 Subject: [PATCH 01/62] Preserve vhost/TFTP customizations on update, add --branch, fix sticky channel - updatefog.sh now defaults to -F/--no-vhost when re-invoking installfog.sh, since an update always has a pre-existing vhost that createSSLCA() would otherwise regenerate from scratch. --overwrite-vhost opts back in. - Fixed the nginx vhost write and Debian's /etc/default/tftpd-hpa write: both called diffconfig() without ever taking the mv -fv backup it needs to detect a change, so the existing "Changed configurations" warning (already correct for Apache's vhost and the cron reporting file) silently never fired for either. Confirmed php.ini/php-fpm/mariadb config are only ever touched via targeted sed on FOG's own known lines, not full overwrites, so they don't have this problem. - Added --branch to check out an arbitrary branch for testing, independent of the tracked channel. - --channel now actually persists: it calls the existing writeUpdateFile() before touching git, instead of only affecting that one run. - Renamed channel values from stable/dev/beta to stable/staging/dev to match the README's Channel table (dev-branch=staging, working-1.6=dev) instead of colliding with it. - gitUpdateToChannel() -> gitUpdateToBranch(branch), since branch resolution now happens once in the caller for both the channel and --branch paths. Part of FOGProject/fogproject#1012. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_0178sa2JC7gq3Py4bRkRapQW --- bin/updatefog.sh | 88 +++++++++++++++++++++++++++++++---------- lib/common/functions.sh | 31 ++++++++++----- lib/common/update.sh | 20 +++++----- 3 files changed, 97 insertions(+), 42 deletions(-) diff --git a/bin/updatefog.sh b/bin/updatefog.sh index 530ede32ab..ce1b3628cf 100755 --- a/bin/updatefog.sh +++ b/bin/updatefog.sh @@ -39,25 +39,38 @@ done export PATH usage() { - echo -e "Usage: $0 [-h?y] [--channel stable|dev|beta] [--git-path ] [--no-revert]" + echo -e "Usage: $0 [-h?y] [--channel stable|staging|dev] [--branch ] [--git-path ]" + echo -e "\t \t\t[--no-revert] [--overwrite-vhost]" echo -e "\t-h -? --help\t\tDisplay this info" - echo -e "\t --channel\tUpdate channel to track: stable, dev, or beta" + echo -e "\t --channel\tUpdate channel to track: stable, staging, or dev" echo -e "\t \t\tdefaults to whatever this server already tracks" + echo -e "\t --branch\tCheck out an arbitrary branch instead of a channel" + echo -e "\t \t\t(e.g. to test a PR/feature branch). One-off: does" + echo -e "\t \t\tnot change the tracked channel for future runs" echo -e "\t --git-path\tOverride the git checkout path this server records" echo -e "\t --no-revert\tOn failure, leave the system as-is instead of" echo -e "\t \t\tautomatically reverting to the previous commit" + echo -e "\t --overwrite-vhost\tLet installfog.sh regenerate the web server" + echo -e "\t \t\tvhost from scratch instead of leaving the" + echo -e "\t \t\texisting one (with any customizations) alone" echo -e "\t-y --yes\t\tSkip the confirmation prompt (for cron/GUI use)" exit 0 } shortopts="h?y" -longopts="help,channel:,git-path:,no-revert,yes" +longopts="help,channel:,branch:,git-path:,no-revert,overwrite-vhost,yes" optargs=$(getopt -o $shortopts -l $longopts -n "$0" -- "$@") [[ $? -ne 0 ]] && usage eval set -- "$optargs" autoRevert=1 autoYes="" +# Every update already has a pre-existing, possibly hand-customized vhost -- +# unlike a fresh install, there is nothing to gain by regenerating it, and +# createSSLCA() has no way to tell "default" apart from "admin edited this". +# -F/--no-vhost is the escape hatch installfog.sh already has for exactly +# this; --overwrite-vhost below opts back into the fresh-install behavior. +updateVhostFlag="-F" while :; do case $1 in -h | -\? | --help) @@ -67,6 +80,10 @@ while :; do schannel="$2" shift 2 ;; + --branch) + sbranch="$2" + shift 2 + ;; --git-path) if [[ -n "${2}" && "${2}" == /* ]]; then sgitpath="${2%/}" @@ -80,6 +97,10 @@ while :; do autoRevert=0 shift ;; + --overwrite-vhost) + updateVhostFlag="" + shift + ;; -y | --yes) autoYes="1" shift @@ -103,7 +124,7 @@ error_log="${workingdir}/error_logs/fog_update_error.log" # errorStat (lib/common/functions.sh) exits the process on any non-zero # status unless $exitFail is set -- installfog.sh's default, since a failed # install step should stop it. updatefog.sh needs the opposite: a failed git -# fetch/checkout/reset must return control to gitUpdateToChannel() so +# fetch/checkout/reset must return control to gitUpdateToBranch() so # revertUpdate() can run, not kill the script out from under it. Deliberately # NOT exported: the nested `bash installfog.sh` call below is a separate # process and should keep errorStat's normal exit-on-failure behavior there. @@ -136,25 +157,50 @@ linuxReleaseName_lower="${osname,,}" [[ -n $osid ]] && doOSSpecificIncludes >/dev/null . ../lib/common/update.sh +# writeUpdateFile() (functions.sh) refreshes the "## Version:" comment line in +# .fogsettings as a side effect; installfog.sh derives this the same way at +# its own top, but updatefog.sh never sources that far into it. +[[ -z $version ]] && version="$(awk -F\' /"define\('FOG_VERSION'[,](.*)"/'{print $4}' ../packages/web/lib/fog/system.class.php | tr -d '[[:space:]]')" + [[ -n $sgitpath ]] && fog_git_path="$sgitpath" -[[ -n $schannel ]] && fog_update_channel="$schannel" -if [[ -z $fog_update_channel ]]; then - echo " * No update channel configured for this server, and none given via --channel." - echo " * Pass --channel stable|dev|beta." - exit 1 +if [[ -n $sbranch ]]; then + # --branch is a one-off deviation for testing, not a channel switch -- it + # deliberately leaves fog_update_channel untouched, so a later run without + # --branch goes right back to tracking whatever channel was configured. + branch="$sbranch" + echo " * FOG Update" + echo " Git path: $fog_git_path" + echo " Branch: $branch (custom -- not a tracked channel)" + echo +else + [[ -n $schannel ]] && fog_update_channel="$schannel" + + if [[ -z $fog_update_channel ]]; then + echo " * No update channel configured for this server, and none given via --channel." + echo " * Pass --channel stable|staging|dev, or --branch for a one-off checkout." + exit 1 + fi + + branch=$(channelToBranch "$fog_update_channel") || { + echo " * Unknown update channel: $fog_update_channel (expected stable, staging, or dev)" + exit 1 + } + + # Persist the resolved channel now, before touching git -- writeUpdateFile + # merges just the managed keys (fog_git_path/fog_update_channel among them) + # into the existing .fogsettings, leaving every other line as-is. Without + # this, --channel only ever changed the channel for THIS run: the child + # `installfog.sh` below re-sources the OLD value from .fogsettings and + # writes that back, so the override never stuck for future unattended runs. + writeUpdateFile + + echo " * FOG Update" + echo " Git path: $fog_git_path" + echo " Channel: $fog_update_channel ($branch)" + echo fi -branch=$(channelToBranch "$fog_update_channel") || { - echo " * Unknown update channel: $fog_update_channel (expected stable, dev, or beta)" - exit 1 -} - -echo " * FOG Update" -echo " Git path: $fog_git_path" -echo " Channel: $fog_update_channel ($branch)" -echo - if [[ -z $autoYes ]]; then echo -n " * Continue with this update? (Y/N) " read confirmGo @@ -168,12 +214,12 @@ if [[ -z $autoYes ]]; then fi backupCustomizations -if ! gitUpdateToChannel; then +if ! gitUpdateToBranch "$branch"; then echo " * Git update failed -- nothing was installed. See $error_log." exit 1 fi -(cd "$fog_git_path/bin" && bash installfog.sh -Y >>$error_log 2>&1) +(cd "$fog_git_path/bin" && bash installfog.sh -Y $updateVhostFlag >>$error_log 2>&1) installStatus=$? cd "$workingdir" diff --git a/lib/common/functions.sh b/lib/common/functions.sh index 22f633bfeb..ff1b0243ca 100755 --- a/lib/common/functions.sh +++ b/lib/common/functions.sh @@ -44,15 +44,16 @@ linkIfAbsent() { [[ -e $link || -L $link ]] && return 0 ln -s "$target" "$link" >>$error_log 2>&1 } -# Maps a FOG update channel name to the git branch it tracks. Mirrors -# fog-docs/docs/installation/server/install-fog-server.md's "Choosing a FOG -# version" section -- codified here so bin/updatefog.sh and lib/common/config.sh -# share one mapping instead of each guessing at it. +# Maps a FOG update channel name to the git branch it tracks. Channel names +# match README.md's "Channel" table (Stable/Staging/Dev), not the informal +# "dev"/"beta" prose fog-docs used before that table existed -- see +# FOGProject/fogproject#1012. Codified here so bin/updatefog.sh and +# lib/common/config.sh share one mapping instead of each guessing at it. channelToBranch() { case "$1" in stable) echo "stable" ;; - dev) echo "dev-branch" ;; - beta) echo "working-1.6" ;; + staging) echo "dev-branch" ;; + dev) echo "working-1.6" ;; *) return 1 ;; esac } @@ -63,8 +64,8 @@ channelToBranch() { branchToChannel() { case "$1" in stable) echo "stable" ;; - dev-branch) echo "dev" ;; - working-1.6) echo "beta" ;; + dev-branch) echo "staging" ;; + working-1.6) echo "dev" ;; *) return 1 ;; esac } @@ -152,7 +153,7 @@ updateStorageNodeCredentials() { recordGitUpdateSettings() { dots "Recording fog_git_path/update channel" mysql $sqloptionsuser --password="${snmysqlpass}" --execute="INSERT INTO globalSettings (settingKey, settingDesc, settingValue, settingCategory) VALUES ('FOG_GIT_PATH', 'Filesystem path of the FOG git checkout on this server. Recorded automatically by installfog.sh/updatefog.sh -- editing it here has no effect on the next update.', \"$fog_git_path\", 'FOG Update') ON DUPLICATE KEY UPDATE settingValue=\"$fog_git_path\"" $mysqldbname >>$error_log 2>&1 - mysql $sqloptionsuser --password="${snmysqlpass}" --execute="INSERT INTO globalSettings (settingKey, settingDesc, settingValue, settingCategory) VALUES ('FOG_UPDATE_CHANNEL', 'Update channel this server tracks: stable, dev, or beta.', \"$fog_update_channel\", 'FOG Update') ON DUPLICATE KEY UPDATE settingValue=\"$fog_update_channel\"" $mysqldbname >>$error_log 2>&1 + mysql $sqloptionsuser --password="${snmysqlpass}" --execute="INSERT INTO globalSettings (settingKey, settingDesc, settingValue, settingCategory) VALUES ('FOG_UPDATE_CHANNEL', 'Update channel this server tracks: stable, staging, or dev.', \"$fog_update_channel\", 'FOG Update') ON DUPLICATE KEY UPDATE settingValue=\"$fog_update_channel\"" $mysqldbname >>$error_log 2>&1 errorStat $? } backupDB() { @@ -1297,7 +1298,9 @@ configureTFTPandPXE() { rm -f /etc/xinetd.d/tftp fi if [[ $osid -eq 2 && -f $tftpconfigupstartdefaults ]]; then + mv -fv "$tftpconfigupstartdefaults" "${tftpconfigupstartdefaults}.${timestamp}" >>$error_log 2>&1 echo -e "# /etc/default/tftpd-hpa\n# FOG Modified version\nTFTP_USERNAME=\"root\"\nTFTP_DIRECTORY=\"/tftpboot\"\nTFTP_ADDRESS=\":69\"\nTFTP_OPTIONS=\"${tftpAdvOpts:+$tftpAdvOpts }-s\"" > "$tftpconfigupstartdefaults" + diffconfig "$tftpconfigupstartdefaults" systemctl is-enabled --quiet tftpd-hpa && true || systemctl enable tftpd-hpa >>$error_log 2>&1 systemctl is-active --quiet tftpd-hpa && systemctl stop tftpd-hpa >>$error_log 2>&1 || true systemctl is-active --quiet tftpd-hpa && true || systemctl start tftpd-hpa >>$error_log 2>&1 @@ -1318,7 +1321,9 @@ configureTFTPandPXE() { ;; *) if [[ $osid -eq 2 && -f $tftpconfigupstartdefaults ]]; then + mv -fv "$tftpconfigupstartdefaults" "${tftpconfigupstartdefaults}.${timestamp}" >>$error_log 2>&1 echo -e "# /etc/default/tftpd-hpa\n# FOG Modified version\nTFTP_USERNAME=\"root\"\nTFTP_DIRECTORY=\"/tftpboot\"\nTFTP_ADDRESS=\":69\"\nTFTP_OPTIONS=\"${tftpAdvOpts:+$tftpAdvOpts }-s\"" > "$tftpconfigupstartdefaults" + diffconfig "$tftpconfigupstartdefaults" sysv-rc-conf xinetd off >>$error_log 2>&1 service xinetd stop >>$error_log 2>&1 sysv-rc-conf tftpd-hpa on >>$error_log 2>&1 @@ -3135,7 +3140,7 @@ writeUpdateFile() { # # fog_update_channel IS a genuine persisted preference -- which channel # to track -- closer to secureboot/fwconfigure above than to - # fogprogramdir: an admin's choice of stable/dev/beta must carry forward + # fogprogramdir: an admin's choice of stable/staging/dev must carry forward # on every upgrade, not just on the run it was made. fog_git_path fog_update_channel ) @@ -3512,6 +3517,12 @@ EOF echo 'location ~ \.php$ {' > "$phploc" emitNginxPhpBody "$phploc" echo "}" >> "$phploc" + # Apache's branch below backs up $etcconf the same way before + # rewriting it, which is what lets its own diffconfig call + # further down actually detect a change; nginx was calling + # diffconfig without ever taking this backup first, so it was + # comparing the new file to nothing and never fired. + mv -fv "${etcconf}" "${etcconf}.${timestamp}" >>$workingdir/error_logs/fog_error_${version}.log 2>&1 echo "server {" > "$etcconf" echo " listen 80;" >> "$etcconf" echo " server_name $ipaddresses $hostname;" >> "$etcconf" diff --git a/lib/common/update.sh b/lib/common/update.sh index 0d3d8537d3..1bcc342db4 100644 --- a/lib/common/update.sh +++ b/lib/common/update.sh @@ -70,17 +70,15 @@ _restorePreviousKernel() { errorStat $st } -# Fetches, checks out, and hard-resets $fog_git_path to the branch mapped -# from $fog_update_channel. Sets $updatePrevCommit (module-global, read by -# revertUpdate below) to the commit HEAD was at before touching anything. -gitUpdateToChannel() { - local branch st - branch=$(channelToBranch "$fog_update_channel") || { - echo " * Unknown update channel: $fog_update_channel (expected stable, dev, or beta)" - return 1 - } +# Fetches, checks out, and hard-resets $fog_git_path to $1 (a branch name -- +# the caller has already resolved this from either $fog_update_channel via +# channelToBranch, or a one-off --branch override). Sets $updatePrevCommit +# (module-global, read by revertUpdate below) to the commit HEAD was at +# before touching anything. +gitUpdateToBranch() { + local branch="$1" st updatePrevCommit=$(git -C "$fog_git_path" rev-parse HEAD 2>>$error_log) - dots "Fetching FOG (${fog_update_channel} / ${branch})" + dots "Fetching FOG (${branch})" git -C "$fog_git_path" fetch --all >>$error_log 2>&1 st=$? errorStat $st @@ -115,7 +113,7 @@ revertUpdate() { git -C "$fog_git_path" reset --hard "$updatePrevCommit" >>$error_log 2>&1 errorStat $? dots "Re-running installfog.sh against the reverted commit" - (cd "$fog_git_path/bin" && bash installfog.sh -Y >>$error_log 2>&1) + (cd "$fog_git_path/bin" && bash installfog.sh -Y $updateVhostFlag >>$error_log 2>&1) errorStat $? _restorePreviousKernel restoreCustomizations From 49091c9687e4fdeac6ee611f067d931eb7827bba Mon Sep 17 00:00:00 2001 From: JJ Fullmer Date: Thu, 6 Aug 2026 20:48:06 -0600 Subject: [PATCH 02/62] Fix EXTERNAL_CA_AND_LETSENCRYPT.md: iPXE trust is not FOG's CA alone The doc claimed iPXE's CA is "compiled into the binaries at build time," implying it's coupled to fog-client's pinning the same way. It isn't: upstream iPXE's src/config/crypto.h unconditionally defines CROSSCERT="http://ca.ipxe.org/auto", a public-CA cross-signing fallback that FOG's own build never disables (the fog-ipxe config overlay only replaces general.h/settings.h/console.h) and that the republished Secure-Boot-signed binaries rely on exclusively (upstream's own release build passes no TRUST=/CERT= at all). So a real Let's Encrypt certificate on the web vhost already validates for iPXE's netboot fetches with no FOG-side change, independent of Secure Boot status -- fog-client's pinning is the actual constraint on public Let's Encrypt, not iPXE. Verified against pinned upstream sources (permalinks + References section added); outbound internet access to ca.ipxe.org is the common case here, not an edge case worth hedging the framing around. Part of FOGProject/fogproject#1013. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_0178sa2JC7gq3Py4bRkRapQW --- docs/EXTERNAL_CA_AND_LETSENCRYPT.md | 139 +++++++++++++++++++++++----- 1 file changed, 114 insertions(+), 25 deletions(-) diff --git a/docs/EXTERNAL_CA_AND_LETSENCRYPT.md b/docs/EXTERNAL_CA_AND_LETSENCRYPT.md index b0e37fcdd2..d9e77e6e67 100644 --- a/docs/EXTERNAL_CA_AND_LETSENCRYPT.md +++ b/docs/EXTERNAL_CA_AND_LETSENCRYPT.md @@ -1,26 +1,37 @@ # External CA & Let's Encrypt certificates FOG generates its own self-signed Certificate Authority at install time and uses -it for three things: the web server (HTTPS), the iPXE boot binaries (which are -compiled to trust that CA), and the **fog-client**, which pins the CA and uses it -to authenticate the server before acting on tasks. - -Because of that pinning, you cannot simply drop a Let's Encrypt certificate onto -the Apache vhost and expect clients to keep working — the client validates the -server's certificate chain against the CA it pinned at registration time. This -document explains the supported way to use your **own** CA (including an internal -ACME / Let's Encrypt-style CA), the trade-offs of using **public** Let's Encrypt, -and the renewal caveats you must plan around. +it for two things: the web server (HTTPS) and the **fog-client**, which pins the +CA and uses it to authenticate the server before acting on tasks. iPXE's netboot +fetches (`boot.php`, kernel, initrd) are a **separate case, and are not actually +tied to FOG's CA** — see [How iPXE validates HTTPS](#how-ipxe-validates-https) +below, which corrects an earlier version of this document. + +Because of fog-client's pinning, you cannot simply drop a Let's Encrypt +certificate onto the Apache vhost and expect **clients** to keep working — the +fog-client validates the server's certificate chain against the CA it pinned at +registration time. This document explains the supported way to use your **own** +CA for that (including an internal ACME / Let's Encrypt-style CA), the +trade-offs of using **public** Let's Encrypt for fog-client specifically, and +the renewal caveats you must plan around. It does **not** apply to iPXE's own +netboot fetches, which can already validate a public Let's Encrypt certificate +without any FOG-side change — see below. > **TL;DR** -> - Use the installer's `--external-ca` support to sign FOG's server certificate -> with **your own** intermediate CA. This is the supported, tested path. +> - **iPXE's netboot fetches already work with a public Let's Encrypt +> certificate on the web vhost**, with no FOG changes. This is unrelated to +> Secure Boot status. See [How iPXE validates HTTPS](#how-ipxe-validates-https). +> (It relies on the booting client reaching `ca.ipxe.org`, which holds for +> most sites — air-gapped networks are the exception, not the rule.) +> - **fog-client is the actual constraint.** Use the installer's `--external-ca` +> support to sign FOG's server certificate with **your own** intermediate CA. +> This is the supported, tested path for fog-client. > - An **internal ACME CA** (e.g. [step-ca / smallstep](https://github.com/smallstep/certificates)) -> is the best fit — it gives you ACME automation without exposing FOG publicly, -> and the CA you pin is stable. -> - **Public** Let's Encrypt is possible but fragile: it requires a publicly -> resolvable name (or DNS-01 automation), and LE rotates its intermediates, -> which breaks the pinning model on renewal. Read the +> is the best fit for fog-client — it gives you ACME automation without exposing +> FOG publicly, and the CA you pin is stable. +> - **Public** Let's Encrypt for fog-client specifically is possible but fragile: +> it requires a publicly resolvable name (or DNS-01 automation), and LE rotates +> its intermediates, which breaks the pinning model on renewal. Read the > [caveats](#public-lets-encrypt-caveats) before going down this road. --- @@ -28,6 +39,7 @@ and the renewal caveats you must plan around. ## Table of contents - [How FOG uses certificates](#how-fog-uses-certificates) +- [How iPXE validates HTTPS](#how-ipxe-validates-https) - [What `--external-ca` does](#what---external-ca-does) - [Recommended: internal ACME CA (step-ca)](#recommended-internal-acme-ca-step-ca) - [Public Let's Encrypt: caveats](#public-lets-encrypt-caveats) @@ -42,18 +54,69 @@ and the renewal caveats you must plan around. | Consumer | What it uses | Where it comes from | |----------|--------------|---------------------| | **Web server (Apache/Nginx)** | `srvpublic.crt` + private key, served over HTTPS | Generated by the installer, signed by FOG's CA | -| **iPXE** | Trusts FOG's CA so it can fetch the boot file over HTTPS | CA is **compiled into** the iPXE binaries at build time | +| **iPXE** | Validates the vhost's actual leaf cert against whatever it can chain to — FOG's own CA (if `TRUST=`'d in) **or** any publicly-trusted CA via a built-in fallback | See [How iPXE validates HTTPS](#how-ipxe-validates-https) | | **fog-client** | Pins `ca.cert.der` and requires the server cert to chain to it | Downloaded from `/management/other/ca.cert.der` | -The critical detail is the **pinned certificate**. The client adds *only* -`ca.cert.der` to its validation store and requires that exact certificate to -appear in the server's chain. That means: +The critical detail for **fog-client** is the **pinned certificate**. The client +adds *only* `ca.cert.der` to its validation store and requires that exact +certificate to appear in the server's chain. That means: > `ca.cert.der` must be the certificate that **directly signs** the server > certificate — i.e. the **intermediate**, not the root. -This is why "just point Apache at a Let's Encrypt cert" does not work: the client -never pinned LE's intermediate, so validation fails. +This is why "just point Apache at a Let's Encrypt cert" does not work **for +fog-client**: the client never pinned LE's intermediate, so validation fails. +The same swap does not have this problem for iPXE — see next section. + +--- + +## How iPXE validates HTTPS + +An earlier version of this document stated that iPXE's CA is "compiled into +the binaries at build time" and left it there, implying iPXE is in the same +position as fog-client. It isn't. Verified directly against upstream iPXE +source (permalinks below, pinned to +[`ipxe/ipxe@bfc442a`](https://github.com/ipxe/ipxe/commit/bfc442ad18577c876292e10bbed0d40d421456dc)): + +1. **`TRUST=` is additive, not exclusive.** FOG's own build + (`buildipxe.sh`) passes `TRUST=${cert}` (FOG's CA) into iPXE's `make`. That + compiles FOG's CA in as a pinned root + ([`src/Makefile.housekeeping#L620-L649`](https://github.com/ipxe/ipxe/blob/bfc442ad18577c876292e10bbed0d40d421456dc/src/Makefile.housekeeping#L620-L649)) — + but it does not remove iPXE's other, unconditional default described next. +2. **iPXE ships a public-CA fallback by default, regardless of `TRUST=`.** + [`src/config/crypto.h`](https://github.com/ipxe/ipxe/blob/bfc442ad18577c876292e10bbed0d40d421456dc/src/config/crypto.h) + unconditionally defines + `#define CROSSCERT "http://ca.ipxe.org/auto"`. When iPXE meets a certificate + chain it can't otherwise validate, it fetches a cross-signed certificate from + `ca.ipxe.org` vouching for real-world public CAs — Let's Encrypt's root + included. This is a stock iPXE feature, not a FOG addition. +3. **FOG's own build never disables it.** `fog-ipxe`'s config overlay + (`src/config/`, `src-efi/config/`) only replaces `general.h`, `settings.h`, + and `console.h` — it never touches `crypto.h`, so `CROSSCERT` stays on in + every FOG-built binary. +4. **The republished Secure-Boot-signed binaries don't set `TRUST=` at all.** + `fog-ipxe/secureboot/stage.sh` republishes `ipxeboot.tar.gz` verbatim from + [ipxe/ipxe's own releases](https://github.com/ipxe/ipxe/releases) — FOG never + rebuilds them. Upstream's own release workflow builds that variant with + [no `TRUST=`/`CERT=` argument](https://github.com/ipxe/ipxe/blob/bfc442ad18577c876292e10bbed0d40d421456dc/.github/workflows/build.yml#L149-L177) + (job `uefi-sb`: `make bin-${arch}-efi-sb/ipxe.efi bin-${arch}-efi-sb/snponly.efi`), + so those binaries rely purely on the stock `CROSSCERT` fallback. The + `ipxe/secure-boot-ca` repo checked out later in that same workflow + ([`build.yml#L229-L236`](https://github.com/ipxe/ipxe/blob/bfc442ad18577c876292e10bbed0d40d421456dc/.github/workflows/build.yml#L229-L236)) + is only the **Authenticode code-signing** certificate used to sign the EFI + binary itself for shim/Secure Boot — it has nothing to do with TLS. shim only + verifies that signature; it never validates a TLS certificate. + +**Net effect:** a FOG web vhost with a real Let's Encrypt certificate validates +fine for iPXE's netboot fetches (`boot.php`, kernel, initrd) with **no FOG-side +change**, on both FOG's own `buildipxe.sh` binaries and the republished +Secure-Boot-signed binaries, and independent of Secure Boot enrolment status. +This assumes the booting client can reach `ca.ipxe.org`, which holds for most +sites — outbound internet access is the common case, not the exception. Only +on a fully air-gapped network does that fallback not fire, in which case FOG's +own baked-in CA (`TRUST=`) is what makes HTTPS boot work instead. **fog-client +remains the actual constraint on using public Let's Encrypt** — see the rest of +this document. --- @@ -128,7 +191,11 @@ be publicly resolvable. ## Public Let's Encrypt: caveats -You *can* use the real public Let's Encrypt, but understand what you are signing +Everything in this section is about **fog-client's** pinning, not iPXE — a +public Let's Encrypt certificate on the vhost already works for iPXE's netboot +fetches with no caveats beyond internet reachability (see +[How iPXE validates HTTPS](#how-ipxe-validates-https)). For fog-client, you +*can* use the real public Let's Encrypt, but understand what you are signing up for before you do. 1. **You need a publicly resolvable name.** HTTP-01 validation requires LE to @@ -209,6 +276,28 @@ fog-client installer and reboot PXE clients after the switch. --- +## References + +Sources for the [How iPXE validates HTTPS](#how-ipxe-validates-https) section, +pinned to [`ipxe/ipxe@bfc442a`](https://github.com/ipxe/ipxe/commit/bfc442ad18577c876292e10bbed0d40d421456dc) +(`master` as of 2026-08-07) so the line numbers stay stable: + +- [`src/config/crypto.h`](https://github.com/ipxe/ipxe/blob/bfc442ad18577c876292e10bbed0d40d421456dc/src/config/crypto.h) — + the unconditional `CROSSCERT "http://ca.ipxe.org/auto"` default. +- [`src/Makefile.housekeeping#L620-L649`](https://github.com/ipxe/ipxe/blob/bfc442ad18577c876292e10bbed0d40d421456dc/src/Makefile.housekeeping#L620-L649) — + how `TRUST=`/`TRUST_EXT` become pinned fingerprints in `rootcert.c`, additive + to (not a replacement for) `CROSSCERT`. +- [`.github/workflows/build.yml#L149-L236`](https://github.com/ipxe/ipxe/blob/bfc442ad18577c876292e10bbed0d40d421456dc/.github/workflows/build.yml#L149-L236) — + the `uefi-sb`/`sbsign` jobs that produce the `ipxeboot.tar.gz` release FOG + republishes: no `TRUST=`/`CERT=` on the build step, and the `ipxe/secure-boot-ca` + checkout used only by the later Authenticode signing step. +- [`ipxe/ipxe` releases](https://github.com/ipxe/ipxe/releases) — source of + `ipxeboot.tar.gz`, fetched and sha256/signer-verified (not rebuilt) by + `fog-ipxe/secureboot/stage.sh`. +- `fog-ipxe/src/config/`, `fog-ipxe/src-efi/config/` (this project's own iPXE + fork) — confirms the FOG overlay only replaces `general.h`, `settings.h`, + `console.h`, never `crypto.h`. + *Related: this is the supported answer to the "Let's Encrypt support" request (issue #633); the underlying external/intermediate CA installer support was added -for issue #794.* +for issue #794. The iPXE correction above is part of FOGProject/fogproject#1013.* From a2b1f19c2ba578ffaee68b4667f903f26ee72d85 Mon Sep 17 00:00:00 2001 From: JJ Fullmer Date: Thu, 6 Aug 2026 21:04:08 -0600 Subject: [PATCH 03/62] Clarify FOG's SSL CA and Secure Boot signing key are now separate Verified: _ensureSecureBootKeys() (lib/common/functions.sh) generates its own independent, self-signed codeSigning-only keypair (MOK.key/MOK.pem), never derived from or defaulting to .fogCA.key/.fogCA.pem. The two used to be conflatable back when FOG had only one CA doing everything; Secure Boot support split that out into its own key. Noting this in the doc so the same confusion doesn't recur. Part of FOGProject/fogproject#1013. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_0178sa2JC7gq3Py4bRkRapQW --- docs/EXTERNAL_CA_AND_LETSENCRYPT.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/EXTERNAL_CA_AND_LETSENCRYPT.md b/docs/EXTERNAL_CA_AND_LETSENCRYPT.md index d9e77e6e67..d4763958b9 100644 --- a/docs/EXTERNAL_CA_AND_LETSENCRYPT.md +++ b/docs/EXTERNAL_CA_AND_LETSENCRYPT.md @@ -68,6 +68,19 @@ This is why "just point Apache at a Let's Encrypt cert" does not work **for fog-client**: the client never pinned LE's intermediate, so validation fails. The same swap does not have this problem for iPXE — see next section. +> **Not the same key as Secure Boot signing.** FOG generates a second, +> completely independent keypair for Secure Boot — `MOK.key`/`MOK.pem` +> (`_ensureSecureBootKeys()` in `lib/common/functions.sh`), a self-signed, +> `codeSigning`-only cert used to Authenticode-sign the FOS kernel/initrd so +> Secure Boot firmware trusts them. It shares nothing with `.fogCA.key`/ +> `.fogCA.pem` above — different key, different cert, generated separately, +> stored separately (`$fogprogramdir/secureboot/` vs. `$sslpath/CA/`). Nothing +> here (`--external-ca`, a Let's Encrypt cert, or anything else in this doc) +> touches Secure Boot signing, and nothing about Secure Boot touches the CA +> this document is about. This split is recent — Secure Boot support was added +> to FOG well after the SSL CA already existed, so older FOG installs (or +> memories of them) may reasonably recall a single CA doing everything. + --- ## How iPXE validates HTTPS From 602ac99071ed1dfb8db7acd99301b76f1d268865 Mon Sep 17 00:00:00 2001 From: JJ Fullmer Date: Thu, 6 Aug 2026 22:15:36 -0600 Subject: [PATCH 04/62] Add design doc: --hostname/--extra-server-name flags + setupacme.sh Design for FOGProject/fogproject#1013's remaining scope now that the iPXE/CROSSCERT and Secure-Boot-key doc corrections are in: a non-interactive --hostname override, an additive --extra-server-name list, and a new setupacme.sh that automates leaf renewal against an already-imported --external-ca CA via acme.sh, scheduled through the same cron.d pattern setupFogReporting() already uses. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_0178sa2JC7gq3Py4bRkRapQW --- ...8-07-cert-separation-letsencrypt-design.md | 208 ++++++++++++++++++ 1 file changed, 208 insertions(+) create mode 100644 docs/superpowers/specs/2026-08-07-cert-separation-letsencrypt-design.md diff --git a/docs/superpowers/specs/2026-08-07-cert-separation-letsencrypt-design.md b/docs/superpowers/specs/2026-08-07-cert-separation-letsencrypt-design.md new file mode 100644 index 0000000000..83084ada54 --- /dev/null +++ b/docs/superpowers/specs/2026-08-07-cert-separation-letsencrypt-design.md @@ -0,0 +1,208 @@ +# Web vhost cert separation + Let's Encrypt scaffolding + +Part of [FOGProject/fogproject#1013](https://github.com/FOGProject/fogproject/issues/1013). + +## Context + +FOG already separates the web vhost's certificate from FOG's own self-signed +CA via `--external-ca` (added for #794): an admin can sign the vhost's leaf +certificate with their own intermediate CA, independent of FOG's CA, and +`docs/EXTERNAL_CA_AND_LETSENCRYPT.md` documents the recommended pattern of +feeding an internal ACME CA (e.g. step-ca) into that flow. That separation is +done; it is not part of this design. + +What's still missing, and what this design covers: + +1. **No renewal automation.** The doc tells an admin to "automate this with an + ACME renewal hook" but FOG ships none — admins write their own. +2. **No way to change the vhost's server name(s) without an interactive + prompt.** `$hostname` already drives the vhost's `server_name`/`ServerAlias` + and the cert's SAN, and is already persisted in `.fogsettings`, but there is + no CLI flag to set it — under `-Y`/autoaccept (which `updatefog.sh` always + uses) it silently takes whatever `hostname -f` returns, commonly something + like `fogserver`. +3. **No way to add an additional server name** alongside the primary one, for + sites that need the vhost reachable under more than one DNS name at once. + +Separately, this design corrects two inaccuracies in +`docs/EXTERNAL_CA_AND_LETSENCRYPT.md` that came up while scoping the above +(already fixed in commits `da54feeb2` and `739f21dd5` on this branch, included +here for completeness): + +- iPXE's netboot HTTPS fetches (`boot.php`, kernel, initrd) are **not** coupled + to FOG's CA the way fog-client is. Upstream iPXE's `src/config/crypto.h` + unconditionally defines `CROSSCERT="http://ca.ipxe.org/auto"`, a public-CA + cross-signing fallback that FOG's build never disables and that the + republished Secure-Boot-signed binaries rely on exclusively (upstream's own + release build passes no `TRUST=`/`CERT=` at all). A real Let's Encrypt + certificate on the vhost already validates for iPXE with no FOG-side change, + independent of Secure Boot status, as long as the booting client can reach + `ca.ipxe.org` (the common case, not an edge case worth hedging around). +- FOG's SSL CA (`.fogCA.key`/`.fogCA.pem`) and the Secure Boot signing key + (`MOK.key`/`MOK.pem`, from `_ensureSecureBootKeys()`) are separate, unrelated + keypairs. Neither this design nor `--external-ca` nor a Let's Encrypt + certificate touches Secure Boot signing, and nothing about Secure Boot + touches the CA this design is about. + +## Non-goals + +- Not re-litigating `--external-ca` or the CA-import flow — this design builds + on top of it, unchanged. +- Not changing fog-client's pinning model or anything in the `zazzles`/ + `fog-client` repos. +- Not changing iPXE's build or trust configuration. +- Not storing DNS provider credentials — DNS-01 validation relies on + `acme.sh`'s own plugin configuration, which the admin sets up themselves. +- Not supporting public Let's Encrypt for fog-client's pinning model directly + — that remains fragile (LE rotates intermediates) and is already documented + as such; this design's ACME automation only ever renews a leaf against a + CA already imported via `--external-ca`, which is what keeps fog-client + working across renewals. + +## Architecture + +Three independent, additive pieces. None replace or change existing behavior +when unused. + +1. **`--hostname `** on `installfog.sh`, passed through by + `updatefog.sh`. Sets `$hostname` non-interactively — the same variable + already used for the vhost's `server_name`/`ServerAlias` and the cert's + `DNS.1` SAN, and already persisted in `.fogsettings` (`hostname` is already + in `writeUpdateFile()`'s `managedKeys`). +2. **`--extra-server-name `** (repeatable) on `installfog.sh`, passed + through by `updatefog.sh`. A new, separate, additive list of extra + `ServerAlias`/`server_name`/SAN entries — the primary `$hostname` and + auto-detected IPs are unaffected. +3. **`bin/setupacme.sh`** — a new script, run after `--external-ca` is already + configured. Manages `acme.sh` to issue and renew the vhost's **leaf** + certificate against the already-imported external CA, installs the renewed + leaf, reloads the web server, and schedules itself via `/etc/cron.d`. + +## Components + +### `--hostname ` + +- `bin/installfog.sh`: new flag → staging var `shostname`, applied after + `.fogsettings` is sourced, before `lib/common/newinput.sh`'s prompt loop. + With `shostname` set, `hostname` is non-empty going into that loop's + `while [[ -z $hostname ]]`, so the interactive prompt is skipped — this + works the same under `-Y` and interactively. +- `bin/updatefog.sh`: new pass-through flag, forwarded to the child + `installfog.sh -Y` invocation alongside the existing `$updateVhostFlag`. +- Input validation: hostname-shape check (alphanumeric, dots, hyphens only) + before acceptance. Rejected with a clear error otherwise — same posture as + the existing `--git-path` validation (`requires an absolute path`). This + value is interpolated directly into the vhost config and an OpenSSL CSR + config file; it must never reach either unchecked. +- No new persistence — `hostname` is already a managed key. + +### `--extra-server-name ` (repeatable) + +- New staging var collects into an array; persisted as a single space-joined + `.fogsettings` key (`extraServerNames`), added to `writeUpdateFile()`'s + `managedKeys`. +- Same input validation as `--hostname`, applied per value. +- `createSSLCA()`'s vhost-writing code (both the nginx and Apache branches) + appends these to the existing `server_name`/`ServerAlias` lines — the same + place `$vhostaliases` already gets appended for Apache; nginx's + `server_name $ipaddresses $hostname;` line gets the same treatment. +- Also appended to the CSR's `[alt_names]` block (alongside the existing + `$sanentries`/`DNS.1 = $hostname`), so the certificate itself covers the + extra name(s), not just the vhost config. +- Mirrored into `globalSettings` as `FOG_EXTRA_SERVER_NAMES`, informational + only (same treatment as `FOG_GIT_PATH`/`FOG_UPDATE_CHANNEL`) — visible on the + Settings page under a category consistent with the existing "FOG Update" + pattern; editing it there has no effect on the next run. + +### `bin/setupacme.sh` + +- Precondition check: the CA files `--external-ca` already imports + (`/opt/fog/snapins/ssl/CA/.fogCA.pem` etc.) must exist. If not, fail + immediately with a message pointing at `--external-ca` — never try to issue + against nothing. +- Installs `acme.sh` if not already present (single curl-fetchable script, + same posture as `installfog.sh`'s own per-distro prerequisite handling). +- Args: ACME directory URL, and validation method using `acme.sh`'s own + vocabulary directly (`--http01`, or `--dns `) rather + than inventing FOG's own — for DNS-01, the admin is responsible for whatever + provider credentials that plugin itself expects; FOG never stores them. +- Issues via `acme.sh --issue`, installs via `acme.sh --install-cert` with + `--reloadcmd` set to whatever reloads FOG's already-configured web server — + `systemctl reload httpd`/`apache2`/`nginx` depending on `$webserver`, the + same variable `createSSLCA()` already branches on. The exact command is an + implementation detail, not a design decision: it only ever reloads the one + web server FOG already knows it configured. +- Schedules its own renewal check via `/etc/cron.d/fog_acme_renew`, written + using the same `mv -fv` backup + `diffconfig` pattern `setupFogReporting()` + already uses for `/etc/cron.d/fog_reporting`. + +## Data flow + +**Install/update time (hostname + extra names):** +`installfog.sh --hostname fog.example.com --extra-server-name fog-legacy.internal` +→ staging vars applied after `.fogsettings` sourced → `createSSLCA()` writes +them into the vhost's `server_name`/`ServerAlias` and the CSR's `[alt_names]` +→ `writeUpdateFile()` persists both to `.fogsettings` → mirrored into +`globalSettings`. On every later `installfog.sh`/`updatefog.sh` run, both +values are already in `.fogsettings`, so the flags don't need repeating (same +pattern as `fog_update_channel`). + +**ACME renewal (steady state):** +`/etc/cron.d/fog_acme_renew` fires `acme.sh --cron` on schedule → `acme.sh` +determines the leaf needs renewal → talks to the configured ACME directory URL +using the configured validation method (HTTP-01 hits the vhost directly; +DNS-01 uses the admin's own plugin config) → new leaf issued, still signed by +the same external intermediate imported via `--external-ca` → `--reloadcmd` +installs the leaf where the vhost reads it and reloads the web server. +**The pinned intermediate never changes**, so fog-client keeps working without +re-pinning, and iPXE is unaffected regardless (per the `CROSSCERT` finding +above, or simply because the intermediate didn't change). + +**Failure mode:** if `acme.sh --cron` fails (validation failure, network +issue, ACME server down), the old leaf stays in place until it actually +expires. `acme.sh`'s own retry/backoff handles transient failures; no +FOG-specific retry logic is needed on top. + +## Error handling + +- **Input validation** on `--hostname`/`--extra-server-name`: hostname-shape + check before any file write; malformed values (spaces, shell metacharacters) + are rejected outright, not sanitized-and-written. +- **`setupacme.sh` without `--external-ca` configured**: fails immediately + with a message pointing at `--external-ca`. +- **`acme.sh` missing and uninstallable** (no network, curl fails): fails + clearly and stops; no silent fallback to a different cert path. +- **Renewal failures**: left to `acme.sh`'s own retry/backoff and exit status. +- **Known limitation, not fixed by this design:** the `#1012` `diffconfig`/ + backup mechanism can't distinguish "an admin hand-edited the vhost" from + "FOG changed it because `--extra-server-name` was passed" — both look like + "the file changed" and surface the same "Changed configurations" notice. + Pre-existing limitation of that mechanism; noted so it isn't mistaken for a + new bug when it shows up on an otherwise-normal `--extra-server-name` change. + +## Testing + +No CI framework exists for this repo's shell scripts beyond +`fogproject-install-validation`'s end-to-end distro matrix, so verification +here is manual/integration, same posture as the rest of the `#1012`/`#1013` +work (`bash -n` for syntax, then real installer runs): + +- **`--hostname`**: install with the flag → vhost `server_name`/`ServerAlias` + and the cert's SAN both show the new value; re-run without the flag → value + persists from `.fogsettings` unchanged (idempotent). +- **`--extra-server-name`**: same checks, plus confirming it's additive + (auto-detected IPs/hostname still present) and that `FOG_EXTRA_SERVER_NAMES` + shows up correctly on the Settings page. +- **Input validation**: malformed values for both flags (spaces, `;`, shell + metacharacters) are rejected before anything is written. +- **`setupacme.sh`**: test against a local `step-ca` instance (matching the + doc's own recommended setup) rather than production Let's Encrypt, to avoid + rate limits and real domain-validation infrastructure. HTTP-01 is testable + directly against step-ca; DNS-01 needs a real provider sandbox and may need + verification by whoever has one — noted as such in the implementation plan + rather than blocking on it. +- **Cron entry**: `/etc/cron.d/fog_acme_renew` created with correct + permissions; `diffconfig` no-ops cleanly on first run and correctly flags a + later hand-edit. +- **Regression**: an install/update with none of these new flags behaves + identically to before — everything here is additive/opt-in. From 3984766592fb8c7cc33d4fa266eabf850156b5d3 Mon Sep 17 00:00:00 2001 From: JJ Fullmer Date: Thu, 6 Aug 2026 22:28:34 -0600 Subject: [PATCH 05/62] Add implementation plan for --hostname/--extra-server-name/setupacme.sh Gathered exact current line numbers/content post-rebase onto #1012 for every touched location in installfog.sh, updatefog.sh, and functions.sh. Corrects the design doc's cron.d assumption: acme.sh --install already sets up its own renewal cron job, so setupacme.sh doesn't add a second one. Part of FOGProject/fogproject#1013. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_0178sa2JC7gq3Py4bRkRapQW --- .../2026-08-07-cert-separation-letsencrypt.md | 735 ++++++++++++++++++ 1 file changed, 735 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-07-cert-separation-letsencrypt.md diff --git a/docs/superpowers/plans/2026-08-07-cert-separation-letsencrypt.md b/docs/superpowers/plans/2026-08-07-cert-separation-letsencrypt.md new file mode 100644 index 0000000000..316816b8b9 --- /dev/null +++ b/docs/superpowers/plans/2026-08-07-cert-separation-letsencrypt.md @@ -0,0 +1,735 @@ +# Web vhost cert separation + Let's Encrypt scaffolding Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Let an admin (1) set/override the web vhost's primary and extra server names non-interactively, and (2) automate leaf-certificate renewal against an already-imported `--external-ca` CA via `acme.sh`. + +**Architecture:** Three additive, independent pieces on top of unchanged existing behavior: a `--hostname` flag reusing the existing `$hostname` machinery, a new `--extra-server-name` (repeatable) flag with its own persistence and vhost/CSR wiring, and a new `bin/setupacme.sh` script that bootstraps `acme.sh` against a CA already imported via `--external-ca`. `acme.sh`'s own installer sets up its own renewal cron job — FOG does not add a second scheduling mechanism. + +**Tech Stack:** Bash (installer/updater scripts), OpenSSL (CSR/cert generation), `acme.sh` (ACME client), MySQL (`globalSettings` mirror). + +## Global Constraints + +- No CI/test framework exists for this repo's shell scripts (confirmed: only `fogproject-install-validation`'s end-to-end distro matrix). Every task's "test" step is a manual invocation + assertion on real output, not a unit-test suite. Always run `bash -n ` after every edit to a shell script before anything else. +- Every new CLI value that reaches a file write (vhost config, OpenSSL config, `.fogsettings`) must be validated first — never interpolated unchecked. This repo already treats this as a real security boundary (see `--git-path`'s absolute-path check, `--fogprogramdir`'s check). +- Follow the existing staging-variable convention exactly: a new flag sets an `s`-prefixed variable during `getopt` parsing (e.g. `shostname`), which is applied to the real variable (`hostname`) only *after* `.fogsettings` has been sourced, in the `# evaluation of command line options` block (`bin/installfog.sh:615-638`) — never before, or an upgrade's persisted value would be silently blanked before it's even read. +- `bin/updatefog.sh` never runs `installfog.sh` interactively (always `-Y`) — any new flag added there must be passed straight through to the child `bash installfog.sh -Y ...` invocation, the same way `$updateVhostFlag` already is (`bin/updatefog.sh:222`). +- This branch (`1013-ipxe-crosscert-doc-fix`) is rebased onto `1012-vhost-tftp-warnings-custom-branch-sticky-channel` — `bin/updatefog.sh` already has `--branch`, `--overwrite-vhost`, `$updateVhostFlag`, and `gitUpdateToBranch()` from that branch. Do not reintroduce or duplicate any of that. +- Design doc: `docs/superpowers/specs/2026-08-07-cert-separation-letsencrypt-design.md`. Read it if anything below is ambiguous — the plan follows it exactly, with one correction: `acme.sh --install` already sets up its own renewal cron job, so `setupacme.sh` does **not** write a `/etc/cron.d` entry itself (the spec's mention of matching `setupFogReporting()`'s cron pattern is superseded by this finding). + +--- + +### Task 1: `--hostname` flag (non-interactive server name override) + +**Files:** +- Modify: `lib/common/functions.sh` — add a `validhostname()` function near `validip()` (`lib/common/functions.sh:442-454`). +- Modify: `bin/installfog.sh` — add the flag to `longopts` (`bin/installfog.sh:167`), add a case-statement branch modeled on `--fogprogramdir` (`bin/installfog.sh:215-227`), apply the staging var in the command-line-evaluation block (`bin/installfog.sh:615-638`), add it to `usage()`'s help text. +- Modify: `bin/updatefog.sh` — add `--hostname` to `longopts` (`bin/updatefog.sh:61`), a case-statement branch, a pass-through variable, forward it on the child invocation (`bin/updatefog.sh:222`), document it in `usage()`. + +**Interfaces:** +- Produces: `validhostname("")` — echoes `0` if `` is a syntactically valid hostname (RFC-1123-style: labels of alphanumerics/hyphens, no leading/trailing hyphen per label, dot-separated, no other characters), `1` otherwise. Same calling convention as `validip()` (`[[ $(validhostname "$x") -ne 0 ]]` means invalid). +- Consumes: nothing new — reuses the existing `$hostname` variable, already in `writeUpdateFile()`'s `managedKeys` (`lib/common/functions.sh:3109`) and already used by `createSSLCA()`'s vhost/CSR generation. + +- [ ] **Step 1: Add `validhostname()` to `lib/common/functions.sh`** + +Insert immediately after `validip()` (which ends at line 454): + +```bash +# Same calling convention as validip(): echo 0/1, checked via +# [[ $(validhostname "$x") -ne 0 ]]. RFC-1123-ish: dot-separated labels of +# alphanumerics/hyphens, no leading/trailing hyphen per label. Needed because +# --hostname/--extra-server-name are the first NON-interactive entry point for +# this value -- the interactive prompt in lib/common/newinput.sh has never +# validated what an admin types, but a CLI flag's value reaches a vhost config +# and an OpenSSL CSR config file unchecked, so it must be checked before either. +validhostname() { + local h=$1 + [[ $h =~ ^[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(\.[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$ ]] + echo $? +} +``` + +- [ ] **Step 2: Syntax-check** + +Run: `bash -n lib/common/functions.sh` +Expected: no output, exit 0. + +- [ ] **Step 3: Manually verify `validhostname()`** + +Run: +```bash +cd bin && . ../lib/common/functions.sh +for h in "fog.example.com" "fogserver" "fog_server" "fog server" "-badstart.com" "fine-name.co.uk"; do + echo "$h -> $(validhostname "$h")" +done +``` +Expected output: +``` +fog.example.com -> 0 +fogserver -> 0 +fog_server -> 1 +fog server -> 1 +-badstart.com -> 1 +fine-name.co.uk -> 0 +``` + +- [ ] **Step 4: Add `--hostname` to `installfog.sh`** + +In `bin/installfog.sh:167`, add `hostname:` to `longopts` (anywhere in the comma list, e.g. right after `fogprogramdir:`): +``` +longopts="help,uninstall,purge-db,purge-images,purge-snapins,purge-ssl,purge-user,purge-all,dry-run,force,mysqldbname:,ssl-path:,oldcopy,no-vhost,no-defaults,no-upgrade,no-htmldoc,force-https,no-force-https,recreate-keys,recreate-CA,recreate-Ca,recreate-cA,recreate-ca,external-ca,ca-cert:,ca-key:,ca-root:,autoaccept,file:,docroot:,webroot:,backuppath:,startrange:,endrange:,no-exportbuild,exitFail,no-tftpbuild,list-packages,fogprogramdir:,secure-boot-key:,secure-boot-cert:,no-secure-boot,hostname:" +``` + +Add a case branch right after the `--fogprogramdir)` block (`bin/installfog.sh:215-227`): +```bash + --hostname) + if [[ -n "${2}" ]] && [[ $(validhostname "${2}") -eq 0 ]]; then + shostname="${2}" + else + echo "Error: --hostname requires a valid hostname" + usage + exit 9 + fi + shift 2 + ;; +``` + +- [ ] **Step 5: Apply the staging var after `.fogsettings` is sourced** + +In the "# evaluation of command line options" block (`bin/installfog.sh:615-638`), add a line alongside the others there (e.g. after `[[ -n $shttpproto ]] && httpproto=$shttpproto` on line 616): +```bash +[[ -n $shostname ]] && hostname=$shostname +``` + +- [ ] **Step 6: Add `--hostname` to `usage()`** + +Find the `--fogprogramdir` line in `usage()` and add directly below it: +``` + echo -e "\t --hostname\t\tOverride the vhost/cert hostname" + echo -e "\t \t\tdefaults to \`hostname -f\`, remembered in .fogsettings" +``` + +- [ ] **Step 7: Syntax-check** + +Run: `bash -n bin/installfog.sh` +Expected: no output, exit 0. + +- [ ] **Step 8: Add `--hostname` pass-through to `updatefog.sh`** + +In `bin/updatefog.sh:61`, add `hostname:` to `longopts`: +``` +longopts="help,channel:,branch:,git-path:,no-revert,overwrite-vhost,yes,hostname:" +``` + +Add a case branch alongside `--git-path` (`bin/updatefog.sh:87-95`): +```bash + --hostname) + if [[ -n "${2}" ]]; then + supdatehostname="${2}" + else + echo "Error: --hostname requires a value" + usage + fi + shift 2 + ;; +``` + +(Validation happens once, in the child `installfog.sh`, via Step 4's check — no need to duplicate the regex here.) + +- [ ] **Step 9: Forward it on the child invocation** + +Change `bin/updatefog.sh:222` from: +```bash +(cd "$fog_git_path/bin" && bash installfog.sh -Y $updateVhostFlag >>$error_log 2>&1) +``` +to: +```bash +(cd "$fog_git_path/bin" && bash installfog.sh -Y $updateVhostFlag ${supdatehostname:+--hostname "$supdatehostname"} >>$error_log 2>&1) +``` + +- [ ] **Step 10: Add `--hostname` to `updatefog.sh`'s `usage()`** + +Add a line next to the `--git-path` help line: +``` + echo -e "\t --hostname\tOverride the vhost/cert hostname for this update" +``` + +- [ ] **Step 11: Syntax-check** + +Run: `bash -n bin/updatefog.sh` +Expected: no output, exit 0. + +- [ ] **Step 12: Manually verify end to end (requires a Linux box with a FOG checkout — cannot be run from this Windows dev machine; run on a test VM)** + +Run: `./installfog.sh -Y --hostname fog-test.example.com` +Then inspect the generated vhost (`$etcconf` — e.g. `/etc/httpd/conf/extra/fog.conf` on RedHat, or the nginx equivalent) and confirm `server_name`/`ServerName`/`ServerAlias` contains `fog-test.example.com`, and that `grep hostname /opt/fog/.fogsettings` shows `hostname='fog-test.example.com'`. +Expected: both true. + +Run again without `--hostname`: `./installfog.sh -Y` +Expected: `.fogsettings` still shows `hostname='fog-test.example.com'` (persisted value survives, matching `fog_update_channel`'s existing behavior). + +- [ ] **Step 13: Commit** + +```bash +git add lib/common/functions.sh bin/installfog.sh bin/updatefog.sh +git commit -m "Add --hostname flag for non-interactive vhost/cert hostname override" +``` + +--- + +### Task 2: `--extra-server-name` flag (additive extra vhost/cert names) + +**Files:** +- Modify: `lib/common/functions.sh`: + - `writeUpdateFile()`'s `managedKeys` array (`lib/common/functions.sh:3108-3146`) — add `extraServerNames`. + - `createSSLCA()` — add a shared suffix variable right before `case $webserver in` (`lib/common/functions.sh:3492-3493`), use it in the three nginx `server_name` lines (`lib/common/functions.sh:3528`, `3579`, `3646`) and Apache's `vhostaliases` (`lib/common/functions.sh:3732`). + - The CSR SAN block (`lib/common/functions.sh:3428-3434` for the `sanentries` IP loop, and both heredocs at `3448-3460` and `3471-3477`) — add extra `DNS.N` entries. +- Modify: `bin/installfog.sh` — repeatable flag parsing (array), staging-var application, `usage()`. +- Modify: `bin/updatefog.sh` — repeatable pass-through, `usage()`. + +**Interfaces:** +- Produces: `$extraServerNames` — a space-joined string of extra names (mirrors how `$ipaddresses` is already a space/newline-joined string consumed via unquoted `for` word-splitting elsewhere in this file, e.g. `lib/common/functions.sh:3430`). Persisted in `.fogsettings` as a managed key. +- Consumes: `validhostname()` from Task 1. + +- [ ] **Step 1: Add `extraServerNames` to `managedKeys`** + +In `lib/common/functions.sh:3145`, change: +```bash + fog_git_path fog_update_channel + ) +``` +to: +```bash + fog_git_path fog_update_channel + # A genuine persisted preference like fog_update_channel above, not a + # RECORD like fogprogramdir/fog_git_path -- an admin's extra vhost/cert + # name(s) must carry forward on every upgrade, not just the run they + # were set on. + extraServerNames + ) +``` + +- [ ] **Step 2: Syntax-check** + +Run: `bash -n lib/common/functions.sh` +Expected: no output, exit 0. + +- [ ] **Step 3: Add the shared suffix variable and use it in the vhost writers** + +In `lib/common/functions.sh`, immediately after line 3492 +(`[[ $httpproto == https ]] && sslenabled=" (Forced SSL)" || sslenabled=" (normal)"`) +and before line 3493 (`case $webserver in`), insert: + +```bash + # $extraServerNames is a space-joined string (see --extra-server-name). + # Computed once here and reused by both the nginx server_name lines below + # and Apache's vhostaliases, so an admin's extra name(s) reach every vhost + # block this function writes, not just one. + extraServerNamesSuffix="" + for extraname in $extraServerNames; do + extraServerNamesSuffix="${extraServerNamesSuffix} ${extraname}" + done +``` + +Change all three occurrences of (lines 3528, 3579, 3646): +```bash + echo " server_name $ipaddresses $hostname;" >> "$etcconf" +``` +to: +```bash + echo " server_name $ipaddresses $hostname${extraServerNamesSuffix};" >> "$etcconf" +``` +(preserve each line's original indentation — 3528 and the other two have different indent levels in the file, only the content changes). + +Change line 3732 from: +```bash + vhostaliases=$(echo $ipaddresses | awk '{for (i = 2; i <= NF; i++) printf " %s", $i}') +``` +to: +```bash + vhostaliases=$(echo $ipaddresses | awk '{for (i = 2; i <= NF; i++) printf " %s", $i}') + vhostaliases="${vhostaliases}${extraServerNamesSuffix}" +``` +(this one line feeds all three `ServerAlias ${hostname}${vhostaliases}` occurrences at 3752/3800/3898 — do not edit those three lines directly). + +- [ ] **Step 4: Add extra SAN entries to both CSR configs** + +In `lib/common/functions.sh`, immediately after the existing `sanentries` IP loop (lines 3430-3434): +```bash + for ip in $ipaddresses; do + sancount=$((sancount + 1)) + [[ -n $sanentries ]] && sanentries="${sanentries}"$'\n' + sanentries="${sanentries}IP.${sancount} = ${ip}" + done +``` +add: +```bash + dnscount=1 + dnsSanEntries="" + for extraname in $extraServerNames; do + dnscount=$((dnscount + 1)) + dnsSanEntries="${dnsSanEntries}"$'\n'"DNS.${dnscount} = ${extraname}" + done +``` + +Change both heredocs' `DNS.1 = $hostname` lines (3459 and 3476) from: +``` +DNS.1 = $hostname +``` +to: +``` +DNS.1 = $hostname$dnsSanEntries +``` +(both occurrences — the `req.cnf` heredoc ending at line 3460 and the `ca.cnf` heredoc ending at line 3477). + +- [ ] **Step 5: Syntax-check** + +Run: `bash -n lib/common/functions.sh` +Expected: no output, exit 0. + +- [ ] **Step 6: Add repeatable `--extra-server-name` to `installfog.sh`** + +In `bin/installfog.sh:167`, add `extra-server-name:` to `longopts` (alongside `hostname:` from Task 1): +``` +...,secure-boot-key:,secure-boot-cert:,no-secure-boot,hostname:,extra-server-name: +``` + +Add a case branch after the `--hostname)` block from Task 1: +```bash + --extra-server-name) + if [[ -n "${2}" ]] && [[ $(validhostname "${2}") -eq 0 ]]; then + sextraServerNames+=("${2}") + else + echo "Error: --extra-server-name requires a valid hostname" + usage + exit 9 + fi + shift 2 + ;; +``` + +Declare the array before the `getopt`/`while` loop starts (near the top of the file, alongside other pre-loop variable init — e.g. right before the `shortopts=`/`longopts=` lines at 166-167): +```bash +sextraServerNames=() +``` + +- [ ] **Step 7: Apply the staging array after `.fogsettings` is sourced** + +In the same block as Task 1 Step 5 (`bin/installfog.sh:615-638`), add: +```bash +[[ ${#sextraServerNames[@]} -gt 0 ]] && extraServerNames="${sextraServerNames[*]}" +``` +(only overwrites the persisted value if the flag was actually given at least once this run — same "override only if given" convention as every other staging var here). + +- [ ] **Step 8: Add `--extra-server-name` to `usage()`** + +Add directly below the `--hostname` help line from Task 1: +``` + echo -e "\t --extra-server-name\tAdd an extra vhost/cert name (repeatable)" + echo -e "\t \t\talongside the primary hostname and detected IPs" +``` + +- [ ] **Step 9: Syntax-check** + +Run: `bash -n bin/installfog.sh` +Expected: no output, exit 0. + +- [ ] **Step 10: Add repeatable pass-through to `updatefog.sh`** + +In `bin/updatefog.sh:61`, add `extra-server-name:` to `longopts`. + +Declare the array before the `getopt`/`while` loop (near line 60): +```bash +supdateExtraServerNames=() +``` + +Add a case branch alongside `--hostname` from Task 1: +```bash + --extra-server-name) + if [[ -n "${2}" ]]; then + supdateExtraServerNames+=("${2}") + else + echo "Error: --extra-server-name requires a value" + usage + fi + shift 2 + ;; +``` + +- [ ] **Step 11: Forward it on the child invocation** + +Change `bin/updatefog.sh:222` (already modified by Task 1 Step 9) from: +```bash +(cd "$fog_git_path/bin" && bash installfog.sh -Y $updateVhostFlag ${supdatehostname:+--hostname "$supdatehostname"} >>$error_log 2>&1) +``` +to: +```bash +extraServerNameArgs=() +for extraname in "${supdateExtraServerNames[@]}"; do + extraServerNameArgs+=(--extra-server-name "$extraname") +done +(cd "$fog_git_path/bin" && bash installfog.sh -Y $updateVhostFlag ${supdatehostname:+--hostname "$supdatehostname"} "${extraServerNameArgs[@]}" >>$error_log 2>&1) +``` +(built as an array, not a bare string, so a name containing a space is still passed as one argument to the child — same reasoning as quoting everywhere else in this file). + +- [ ] **Step 12: Add `--extra-server-name` to `updatefog.sh`'s `usage()`** + +Add directly below the `--hostname` help line from Task 1: +``` + echo -e "\t --extra-server-name\tAdd an extra vhost/cert name for this update (repeatable)" +``` + +- [ ] **Step 13: Syntax-check** + +Run: `bash -n bin/updatefog.sh` +Expected: no output, exit 0. + +- [ ] **Step 14: Manually verify end to end (on a test Linux box)** + +Run: `./installfog.sh -Y --hostname fog-test.example.com --extra-server-name fog-legacy.internal --extra-server-name fog-alt.internal` +Confirm: +- The vhost's `server_name`/`ServerAlias` line includes `fog-test.example.com`, `fog-legacy.internal`, and `fog-alt.internal`, alongside the auto-detected IPs. +- `openssl x509 -in -noout -text | grep -A2 "Subject Alternative Name"` lists `DNS:fog-legacy.internal` and `DNS:fog-alt.internal` alongside `DNS:fog-test.example.com` and the `IP:` entries. +- `grep extraServerNames /opt/fog/.fogsettings` shows `extraServerNames='fog-legacy.internal fog-alt.internal'`. + +Run again without either flag: `./installfog.sh -Y` +Expected: both persisted values survive unchanged in `.fogsettings` and the vhost/cert. + +- [ ] **Step 15: Commit** + +```bash +git add lib/common/functions.sh bin/installfog.sh bin/updatefog.sh +git commit -m "Add repeatable --extra-server-name flag for additive vhost/cert names" +``` + +--- + +### Task 3: Mirror `FOG_EXTRA_SERVER_NAMES` into `globalSettings` + +**Files:** +- Modify: `lib/common/functions.sh` — extend `recordGitUpdateSettings()` (`lib/common/functions.sh:153-158`). + +**Interfaces:** +- Consumes: `$extraServerNames` from Task 2. +- Produces: nothing new consumed elsewhere — this is a leaf, GUI-visibility-only mirror, same as the existing `FOG_GIT_PATH`/`FOG_UPDATE_CHANNEL` rows it sits next to. + +- [ ] **Step 1: Extend `recordGitUpdateSettings()`** + +Change `lib/common/functions.sh:153-158` from: +```bash +recordGitUpdateSettings() { + dots "Recording fog_git_path/update channel" + mysql $sqloptionsuser --password="${snmysqlpass}" --execute="INSERT INTO globalSettings (settingKey, settingDesc, settingValue, settingCategory) VALUES ('FOG_GIT_PATH', 'Filesystem path of the FOG git checkout on this server. Recorded automatically by installfog.sh/updatefog.sh -- editing it here has no effect on the next update.', \"$fog_git_path\", 'FOG Update') ON DUPLICATE KEY UPDATE settingValue=\"$fog_git_path\"" $mysqldbname >>$error_log 2>&1 + mysql $sqloptionsuser --password="${snmysqlpass}" --execute="INSERT INTO globalSettings (settingKey, settingDesc, settingValue, settingCategory) VALUES ('FOG_UPDATE_CHANNEL', 'Update channel this server tracks: stable, staging, or dev.', \"$fog_update_channel\", 'FOG Update') ON DUPLICATE KEY UPDATE settingValue=\"$fog_update_channel\"" $mysqldbname >>$error_log 2>&1 + errorStat $? +} +``` +to (adding a third `INSERT` line and updating the `dots` message and function comment): +```bash +# Mirrors fog_git_path/fog_update_channel/extraServerNames into globalSettings +# so the GUI can show them without SSH. Like fogprogramdir's mirror into +# /etc/fog/fog.conf (GH-850), these are RECORDS, not controls: .fogsettings +# stays the source of truth, and the next installfog.sh/updatefog.sh run +# overwrites whatever an admin may have hand-edited here through the generic +# Settings tab. +recordGitUpdateSettings() { + dots "Recording fog_git_path/update channel/extra server names" + mysql $sqloptionsuser --password="${snmysqlpass}" --execute="INSERT INTO globalSettings (settingKey, settingDesc, settingValue, settingCategory) VALUES ('FOG_GIT_PATH', 'Filesystem path of the FOG git checkout on this server. Recorded automatically by installfog.sh/updatefog.sh -- editing it here has no effect on the next update.', \"$fog_git_path\", 'FOG Update') ON DUPLICATE KEY UPDATE settingValue=\"$fog_git_path\"" $mysqldbname >>$error_log 2>&1 + mysql $sqloptionsuser --password="${snmysqlpass}" --execute="INSERT INTO globalSettings (settingKey, settingDesc, settingValue, settingCategory) VALUES ('FOG_UPDATE_CHANNEL', 'Update channel this server tracks: stable, staging, or dev.', \"$fog_update_channel\", 'FOG Update') ON DUPLICATE KEY UPDATE settingValue=\"$fog_update_channel\"" $mysqldbname >>$error_log 2>&1 + mysql $sqloptionsuser --password="${snmysqlpass}" --execute="INSERT INTO globalSettings (settingKey, settingDesc, settingValue, settingCategory) VALUES ('FOG_EXTRA_SERVER_NAMES', 'Extra vhost/certificate name(s) this server answers to, beyond the primary hostname and detected IPs. Set via --extra-server-name -- editing it here has no effect on the next update.', \"$extraServerNames\", 'FOG Update') ON DUPLICATE KEY UPDATE settingValue=\"$extraServerNames\"" $mysqldbname >>$error_log 2>&1 + errorStat $? +} +``` + +- [ ] **Step 2: Syntax-check** + +Run: `bash -n lib/common/functions.sh` +Expected: no output, exit 0. + +- [ ] **Step 3: Manually verify (on a test Linux box with a running FOG install)** + +Run: `./installfog.sh -Y --extra-server-name fog-legacy.internal` +Then in the FOG web UI, go to Settings, find the **FOG Update** category, and confirm a `FOG_EXTRA_SERVER_NAMES` row shows `fog-legacy.internal`. +Also confirm directly: `mysql fog -e "SELECT settingValue FROM globalSettings WHERE settingKey='FOG_EXTRA_SERVER_NAMES'"` returns `fog-legacy.internal`. + +- [ ] **Step 4: Commit** + +```bash +git add lib/common/functions.sh +git commit -m "Mirror FOG_EXTRA_SERVER_NAMES into globalSettings for GUI visibility" +``` + +--- + +### Task 4: `bin/setupacme.sh` — bootstrap ACME leaf renewal against `--external-ca` + +**Files:** +- Create: `bin/setupacme.sh` + +**Interfaces:** +- Consumes: the CA files `validateExternalCA()` imports (`lib/common/functions.sh:3256-3258`, `3297-3303`) — specifically `$sslpath/CA/.fogCA.pem` and `$sslpath/CA/.fogCA.key` — and `$sslpubcert`/`$sslprivkey`'s on-disk locations (same `.fogsettings` keys `createSSLCA()` already persists: `sslpath`, `sslpubcert`, `sslprivkey`), and `$webserver` (already a managed key, `lib/common/functions.sh:3115`) to pick the right reload command. +- Produces: nothing consumed by other tasks — this is a leaf script, run directly by an admin, same as `updatefog.sh`. + +- [ ] **Step 1: Write `bin/setupacme.sh`** + +```bash +#!/bin/bash +# +# FOG is a computer imaging solution. +# Copyright (C) 2007 Chuck Syperski & Jian Zhang +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# Bootstraps acme.sh to issue and renew the web vhost's LEAF certificate +# against a CA already imported via installfog.sh --external-ca. Never +# touches the imported intermediate/root -- only the leaf -- so fog-client's +# pinned CA never changes across a renewal. acme.sh's own installer sets up +# its own renewal cron job; this script does not add a second one. See +# docs/superpowers/specs/2026-08-07-cert-separation-letsencrypt-design.md and +# FOGProject/fogproject#1013. +bindir=$(dirname $(readlink -f "$BASH_SOURCE")) +cd $bindir +workingdir=$(pwd) + +if [[ ! $EUID -eq 0 ]]; then + echo "setupacme.sh must be run as root user" + exit 1 +fi + +usage() { + echo -e "Usage: $0 [-h?] --directory-url (--http01 | --dns ) -d " + echo -e "\t-h -? --help\t\tDisplay this info" + echo -e "\t --directory-url\tACME server directory URL (public Let's Encrypt or" + echo -e "\t \tan internal ACME CA such as step-ca)" + echo -e "\t --http01\t\tUse HTTP-01 validation (acme.sh's --webroot mode against" + echo -e "\t \t\tthis server's own vhost docroot)" + echo -e "\t --dns\t\tUse DNS-01 validation via the named acme.sh DNS plugin --" + echo -e "\t \t\tthe plugin's own provider credentials must already be set" + echo -e "\t \t\tup in this shell's environment; setupacme.sh never stores them" + echo -e "\t-d\t\t\tDomain to issue the certificate for (repeatable)" + exit 0 +} + +shortopts="h?d:" +longopts="help,directory-url:,http01,dns:" +optargs=$(getopt -o $shortopts -l $longopts -n "$0" -- "$@") +[[ $? -ne 0 ]] && usage +eval set -- "$optargs" + +domains=() +while :; do + case $1 in + -h | -\? | --help) + usage + ;; + --directory-url) + directoryUrl="$2" + shift 2 + ;; + --http01) + validationMethod="http01" + shift + ;; + --dns) + validationMethod="dns" + dnsPlugin="$2" + shift 2 + ;; + -d) + domains+=("$2") + shift 2 + ;; + --) + shift + break + ;; + *) + echo "Error: unhandled option '$1'." + exit 10 + ;; + esac +done + +[[ ! -d ./error_logs/ ]] && mkdir -p ./error_logs >/dev/null 2>&1 +error_log="${workingdir}/error_logs/fog_setupacme_error.log" +: > "$error_log" + +if [[ -z $directoryUrl ]]; then + echo " * --directory-url is required (a public Let's Encrypt endpoint, or an internal ACME CA such as step-ca)." + usage +fi +if [[ -z $validationMethod ]]; then + echo " * Pass either --http01 or --dns ." + usage +fi +if [[ ${#domains[@]} -eq 0 ]]; then + echo " * At least one -d is required." + usage +fi + +exitFail=1 +. ../lib/common/functions.sh + +[[ -z $fogprogramdir && -r /etc/fog/fog.conf ]] && . /etc/fog/fog.conf +[[ -z $fogprogramdir ]] && fogprogramdir="/opt/fog" +fogprogramdir="${fogprogramdir%/}" + +if [[ ! -r "$fogprogramdir/.fogsettings" ]]; then + echo " * No existing FOG install found at $fogprogramdir (.fogsettings missing)." + echo " * setupacme.sh configures an EXISTING install -- run installfog.sh first." + exit 1 +fi +. "$fogprogramdir/.fogsettings" + +# Precondition: --external-ca must already have imported a CA. These are +# exactly the files validateExternalCA() (lib/common/functions.sh) writes. +if [[ ! -e "$sslpath/CA/.fogCA.pem" || ! -e "$sslpath/CA/.fogCA.key" ]]; then + echo " * No external CA found at $sslpath/CA/ -- run installfog.sh --external-ca first." + echo " * setupacme.sh only ever renews a LEAF against a CA you already imported;" + echo " * it does not create or manage a CA itself." + exit 1 +fi + +dots "Checking for acme.sh" +if [[ ! -x "$HOME/.acme.sh/acme.sh" ]]; then + dots "Installing acme.sh" + curl -s https://get.acme.sh | sh -s email=root@localhost >>$error_log 2>&1 + errorStat $? +else + echo "Found" +fi +acmesh="$HOME/.acme.sh/acme.sh" + +case $webserver in + nginx) + reloadcmd="systemctl reload nginx" + ;; + httpd|apache*) + reloadcmd="systemctl reload $webserver" + ;; + *) + echo " * Unrecognized \$webserver ($webserver) -- cannot pick a reload command." + exit 1 + ;; +esac + +domainArgs=() +for domain in "${domains[@]}"; do + domainArgs+=(-d "$domain") +done + +dots "Issuing certificate via acme.sh" +case $validationMethod in + http01) + "$acmesh" --issue --server "$directoryUrl" "${domainArgs[@]}" --webroot "$docroot" >>$error_log 2>&1 + ;; + dns) + "$acmesh" --issue --server "$directoryUrl" "${domainArgs[@]}" --dns "$dnsPlugin" >>$error_log 2>&1 + ;; +esac +issueStatus=$? +# acme.sh's own exit code 2 means "already valid, no renewal needed yet" -- +# not a failure of this run. +if [[ $issueStatus -ne 0 && $issueStatus -ne 2 ]]; then + echo " * acme.sh --issue failed (exit $issueStatus). See $error_log." + exit $issueStatus +fi +echo "Done" + +dots "Installing certificate" +"$acmesh" --install-cert "${domainArgs[@]}" \ + --cert-file "$sslpubcert" \ + --key-file "$sslprivkey" \ + --reloadcmd "$reloadcmd" >>$error_log 2>&1 +errorStat $? + +echo " * setupacme.sh complete. acme.sh's own installer already scheduled its" +echo " own renewal cron job -- no further action is needed for renewals." +``` + +- [ ] **Step 2: Make it executable and syntax-check** + +Run: `chmod +x bin/setupacme.sh && bash -n bin/setupacme.sh` +Expected: no output, exit 0. + +- [ ] **Step 3: Verify `usage()` and argument validation without root/network (safe on any machine)** + +Run: `bash bin/setupacme.sh --help` +Expected: usage text printed, exit 0. + +Run: `bash bin/setupacme.sh -d example.com` (as non-root, or on any machine) +Expected: `setupacme.sh must be run as root user` printed if not root, exit 1. If run as root without `--directory-url`/validation method, expected: `--directory-url is required...` then usage, matching the order the checks appear in the script. + +- [ ] **Step 4: Manually verify end to end (on a test Linux box with FOG installed and `--external-ca` already configured against a local step-ca instance)** + +Stand up a local `step-ca` (per `docs/EXTERNAL_CA_AND_LETSENCRYPT.md`'s own recommended setup), install FOG with `--external-ca` pointed at it, then run: +```bash +./setupacme.sh --directory-url https://step-ca.internal/acme/acme/directory --http01 -d fog-test.example.com +``` +Confirm: +- `acme.sh` is installed at `$HOME/.acme.sh/acme.sh` if it wasn't already. +- The vhost's cert file (`$sslpubcert`) is replaced with the newly-issued leaf: `openssl x509 -in "$sslpubcert" -noout -issuer` shows step-ca's intermediate, not FOG's own self-signed CA. +- The web server actually reloaded (check its access/error log timestamp, or `systemctl status ` shows a recent reload). +- `fog-client` on a test machine that already registered against this server still authenticates successfully (the pinned intermediate didn't change). +- `crontab -l` (root's) now has an `acme.sh --cron` entry, confirming `acme.sh`'s own installer set up its own renewal scheduling. + +- [ ] **Step 5: Commit** + +```bash +git add bin/setupacme.sh +git commit -m "Add bin/setupacme.sh for ACME leaf renewal against --external-ca" +``` + +--- + +### Task 5: Document the new flags/script + +**Files:** +- Modify: `docs/EXTERNAL_CA_AND_LETSENCRYPT.md` — add a section documenting `setupacme.sh` as the automation for the "Recommended: internal ACME CA (step-ca)" flow's step 3/4 (`docs/EXTERNAL_CA_AND_LETSENCRYPT.md`, section `## Recommended: internal ACME CA (step-ca)`). + +**Interfaces:** +- Consumes: nothing — documentation only, no code interface. + +- [ ] **Step 1: Add a "Automating renewal with setupacme.sh" subsection** + +In `docs/EXTERNAL_CA_AND_LETSENCRYPT.md`, immediately after the "Recommended: internal ACME CA (step-ca)" section's existing numbered steps (ending "...(a renewal hook — see [Renewal and rotation](#renewal-and-rotation))."), add: + +```markdown +`bin/setupacme.sh` automates steps 3 and 4 above: it installs `acme.sh` if +needed, issues the leaf against the ACME directory URL you give it, installs +it where the vhost reads it, and wires up `acme.sh`'s `--reloadcmd` to reload +FOG's web server. It never touches the CA `--external-ca` already imported -- +only the leaf -- so a renewal never breaks fog-client's pinning. + +```bash +./setupacme.sh --directory-url https://step-ca.internal/acme/acme/directory \ + --http01 -d fog.example.com +``` + +Use `--dns ` instead of `--http01` for DNS-01 validation +(needed for public Let's Encrypt without exposing this server on port 80) -- +`setupacme.sh` never stores DNS provider credentials itself; whatever +`acme.sh` DNS plugin you name must already have its own credentials configured +in this shell's environment. + +`acme.sh`'s own installer sets up its own daily renewal cron job the first +time it's installed -- `setupacme.sh` does not add a second one. +``` + +- [ ] **Step 2: Commit** + +```bash +git add docs/EXTERNAL_CA_AND_LETSENCRYPT.md +git commit -m "Document bin/setupacme.sh in EXTERNAL_CA_AND_LETSENCRYPT.md" +``` From b3824029e70e6e4668b49cf25fe881b48d22075a Mon Sep 17 00:00:00 2001 From: JJ Fullmer Date: Thu, 6 Aug 2026 22:44:55 -0600 Subject: [PATCH 06/62] Add --hostname flag for non-interactive vhost/cert hostname override --- bin/installfog.sh | 15 ++++++++++++++- bin/updatefog.sh | 14 ++++++++++++-- lib/common/functions.sh | 12 ++++++++++++ 3 files changed, 38 insertions(+), 3 deletions(-) diff --git a/bin/installfog.sh b/bin/installfog.sh index 3fcbe2bc46..cf275e7536 100755 --- a/bin/installfog.sh +++ b/bin/installfog.sh @@ -127,6 +127,8 @@ usage() { echo -e "\t \t\t\t\tdefaults to /opt/fog" echo -e "\t \t\t\t\tremembered in /etc/fog/fog.conf, so it" echo -e "\t \t\t\t\tonly needs giving on a first install" + echo -e "\t --hostname\t\tOverride the vhost/cert hostname" + echo -e "\t \t\tdefaults to \`hostname -f\`, remembered in .fogsettings" echo -e "\t-N --mysqldbname\t\tSpecify the FOG database name" echo -e "\t \t\t\t\tdefaults to fog" echo -e "\t-B --backuppath\t\tSpecify the backup path" @@ -164,7 +166,7 @@ usage() { } shortopts="h?odEUHSCKYyXTFf:c:W:D:B:s:e:N:l" -longopts="help,uninstall,purge-db,purge-images,purge-snapins,purge-ssl,purge-user,purge-all,dry-run,force,mysqldbname:,ssl-path:,oldcopy,no-vhost,no-defaults,no-upgrade,no-htmldoc,force-https,no-force-https,recreate-keys,recreate-CA,recreate-Ca,recreate-cA,recreate-ca,external-ca,ca-cert:,ca-key:,ca-root:,autoaccept,file:,docroot:,webroot:,backuppath:,startrange:,endrange:,no-exportbuild,exitFail,no-tftpbuild,list-packages,fogprogramdir:,secure-boot-key:,secure-boot-cert:,no-secure-boot" +longopts="help,uninstall,purge-db,purge-images,purge-snapins,purge-ssl,purge-user,purge-all,dry-run,force,mysqldbname:,ssl-path:,oldcopy,no-vhost,no-defaults,no-upgrade,no-htmldoc,force-https,no-force-https,recreate-keys,recreate-CA,recreate-Ca,recreate-cA,recreate-ca,external-ca,ca-cert:,ca-key:,ca-root:,autoaccept,file:,docroot:,webroot:,backuppath:,startrange:,endrange:,no-exportbuild,exitFail,no-tftpbuild,list-packages,fogprogramdir:,secure-boot-key:,secure-boot-cert:,no-secure-boot,hostname:" optargs=$(getopt -o $shortopts -l $longopts -n "$0" -- "$@") [[ $? -ne 0 ]] && usage @@ -225,6 +227,16 @@ while :; do fi shift 2 ;; + --hostname) + if [[ -n "${2}" ]] && [[ $(validhostname "${2}") -eq 0 ]]; then + shostname="${2}" + else + echo "Error: --hostname requires a valid hostname" + usage + exit 9 + fi + shift 2 + ;; -o | --oldcopy) scopybackold=1 shift @@ -614,6 +626,7 @@ case $doupdate in esac # evaluation of command line options [[ -n $shttpproto ]] && httpproto=$shttpproto +[[ -n $shostname ]] && hostname=$shostname [[ -n $sstartrange ]] && startrange=$sstartrange [[ -n $sendrange ]] && endrange=$sendrange # -s/-e imply "set DHCP up". These were written directly by the handlers, so on diff --git a/bin/updatefog.sh b/bin/updatefog.sh index ce1b3628cf..5cacb90deb 100755 --- a/bin/updatefog.sh +++ b/bin/updatefog.sh @@ -48,6 +48,7 @@ usage() { echo -e "\t \t\t(e.g. to test a PR/feature branch). One-off: does" echo -e "\t \t\tnot change the tracked channel for future runs" echo -e "\t --git-path\tOverride the git checkout path this server records" + echo -e "\t --hostname\tOverride the vhost/cert hostname for this update" echo -e "\t --no-revert\tOn failure, leave the system as-is instead of" echo -e "\t \t\tautomatically reverting to the previous commit" echo -e "\t --overwrite-vhost\tLet installfog.sh regenerate the web server" @@ -58,7 +59,7 @@ usage() { } shortopts="h?y" -longopts="help,channel:,branch:,git-path:,no-revert,overwrite-vhost,yes" +longopts="help,channel:,branch:,git-path:,no-revert,overwrite-vhost,yes,hostname:" optargs=$(getopt -o $shortopts -l $longopts -n "$0" -- "$@") [[ $? -ne 0 ]] && usage eval set -- "$optargs" @@ -93,6 +94,15 @@ while :; do fi shift 2 ;; + --hostname) + if [[ -n "${2}" ]]; then + supdatehostname="${2}" + else + echo "Error: --hostname requires a value" + usage + fi + shift 2 + ;; --no-revert) autoRevert=0 shift @@ -219,7 +229,7 @@ if ! gitUpdateToBranch "$branch"; then exit 1 fi -(cd "$fog_git_path/bin" && bash installfog.sh -Y $updateVhostFlag >>$error_log 2>&1) +(cd "$fog_git_path/bin" && bash installfog.sh -Y $updateVhostFlag ${supdatehostname:+--hostname "$supdatehostname"} >>$error_log 2>&1) installStatus=$? cd "$workingdir" diff --git a/lib/common/functions.sh b/lib/common/functions.sh index ff1b0243ca..b10b0f7424 100755 --- a/lib/common/functions.sh +++ b/lib/common/functions.sh @@ -453,6 +453,18 @@ validip() { fi echo $stat } +# Same calling convention as validip(): echo 0/1, checked via +# [[ $(validhostname "$x") -ne 0 ]]. RFC-1123-ish: dot-separated labels of +# alphanumerics/hyphens, no leading/trailing hyphen per label. Needed because +# --hostname/--extra-server-name are the first NON-interactive entry point for +# this value -- the interactive prompt in lib/common/newinput.sh has never +# validated what an admin types, but a CLI flag's value reaches a vhost config +# and an OpenSSL CSR config file unchecked, so it must be checked before either. +validhostname() { + local h=$1 + [[ $h =~ ^[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(\.[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$ ]] + echo $? +} getCidr() { local cidr cidr=$(ip -f inet -o addr | grep $1 | awk -F'[ /]+' '/global/ {print $5}' | head -n2 | tail -n1) From ab02a75eb6e167ebbfc0542ff91ece162b85f282 Mon Sep 17 00:00:00 2001 From: JJ Fullmer Date: Thu, 6 Aug 2026 23:16:26 -0600 Subject: [PATCH 07/62] Fix: --hostname validation errors now exit nonzero instead of usage()'s exit 0 --- bin/installfog.sh | 1 - bin/updatefog.sh | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/bin/installfog.sh b/bin/installfog.sh index cf275e7536..fc4c3cb83a 100755 --- a/bin/installfog.sh +++ b/bin/installfog.sh @@ -232,7 +232,6 @@ while :; do shostname="${2}" else echo "Error: --hostname requires a valid hostname" - usage exit 9 fi shift 2 diff --git a/bin/updatefog.sh b/bin/updatefog.sh index 5cacb90deb..9f9f93d8d5 100755 --- a/bin/updatefog.sh +++ b/bin/updatefog.sh @@ -99,7 +99,7 @@ while :; do supdatehostname="${2}" else echo "Error: --hostname requires a value" - usage + exit 9 fi shift 2 ;; From accb79389ae2328a90404531b1236c5e18a254e2 Mon Sep 17 00:00:00 2001 From: JJ Fullmer Date: Thu, 6 Aug 2026 23:25:52 -0600 Subject: [PATCH 08/62] Add repeatable --extra-server-name flag for additive vhost/cert names Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_0178sa2JC7gq3Py4bRkRapQW --- bin/installfog.sh | 16 +++++++++++++++- bin/updatefog.sh | 20 ++++++++++++++++++-- lib/common/functions.sh | 30 +++++++++++++++++++++++++----- 3 files changed, 58 insertions(+), 8 deletions(-) diff --git a/bin/installfog.sh b/bin/installfog.sh index fc4c3cb83a..310daf7f65 100755 --- a/bin/installfog.sh +++ b/bin/installfog.sh @@ -129,6 +129,8 @@ usage() { echo -e "\t \t\t\t\tonly needs giving on a first install" echo -e "\t --hostname\t\tOverride the vhost/cert hostname" echo -e "\t \t\tdefaults to \`hostname -f\`, remembered in .fogsettings" + echo -e "\t --extra-server-name\tAdd an extra vhost/cert name (repeatable)" + echo -e "\t \t\talongside the primary hostname and detected IPs" echo -e "\t-N --mysqldbname\t\tSpecify the FOG database name" echo -e "\t \t\t\t\tdefaults to fog" echo -e "\t-B --backuppath\t\tSpecify the backup path" @@ -165,8 +167,10 @@ usage() { exit 0 } +sextraServerNames=() + shortopts="h?odEUHSCKYyXTFf:c:W:D:B:s:e:N:l" -longopts="help,uninstall,purge-db,purge-images,purge-snapins,purge-ssl,purge-user,purge-all,dry-run,force,mysqldbname:,ssl-path:,oldcopy,no-vhost,no-defaults,no-upgrade,no-htmldoc,force-https,no-force-https,recreate-keys,recreate-CA,recreate-Ca,recreate-cA,recreate-ca,external-ca,ca-cert:,ca-key:,ca-root:,autoaccept,file:,docroot:,webroot:,backuppath:,startrange:,endrange:,no-exportbuild,exitFail,no-tftpbuild,list-packages,fogprogramdir:,secure-boot-key:,secure-boot-cert:,no-secure-boot,hostname:" +longopts="help,uninstall,purge-db,purge-images,purge-snapins,purge-ssl,purge-user,purge-all,dry-run,force,mysqldbname:,ssl-path:,oldcopy,no-vhost,no-defaults,no-upgrade,no-htmldoc,force-https,no-force-https,recreate-keys,recreate-CA,recreate-Ca,recreate-cA,recreate-ca,external-ca,ca-cert:,ca-key:,ca-root:,autoaccept,file:,docroot:,webroot:,backuppath:,startrange:,endrange:,no-exportbuild,exitFail,no-tftpbuild,list-packages,fogprogramdir:,secure-boot-key:,secure-boot-cert:,no-secure-boot,hostname:,extra-server-name:" optargs=$(getopt -o $shortopts -l $longopts -n "$0" -- "$@") [[ $? -ne 0 ]] && usage @@ -236,6 +240,15 @@ while :; do fi shift 2 ;; + --extra-server-name) + if [[ -n "${2}" ]] && [[ $(validhostname "${2}") -eq 0 ]]; then + sextraServerNames+=("${2}") + else + echo "Error: --extra-server-name requires a valid hostname" + exit 9 + fi + shift 2 + ;; -o | --oldcopy) scopybackold=1 shift @@ -626,6 +639,7 @@ esac # evaluation of command line options [[ -n $shttpproto ]] && httpproto=$shttpproto [[ -n $shostname ]] && hostname=$shostname +[[ ${#sextraServerNames[@]} -gt 0 ]] && extraServerNames="${sextraServerNames[*]}" [[ -n $sstartrange ]] && startrange=$sstartrange [[ -n $sendrange ]] && endrange=$sendrange # -s/-e imply "set DHCP up". These were written directly by the handlers, so on diff --git a/bin/updatefog.sh b/bin/updatefog.sh index 9f9f93d8d5..b9e56a6e33 100755 --- a/bin/updatefog.sh +++ b/bin/updatefog.sh @@ -49,6 +49,7 @@ usage() { echo -e "\t \t\tnot change the tracked channel for future runs" echo -e "\t --git-path\tOverride the git checkout path this server records" echo -e "\t --hostname\tOverride the vhost/cert hostname for this update" + echo -e "\t --extra-server-name\tAdd an extra vhost/cert name for this update (repeatable)" echo -e "\t --no-revert\tOn failure, leave the system as-is instead of" echo -e "\t \t\tautomatically reverting to the previous commit" echo -e "\t --overwrite-vhost\tLet installfog.sh regenerate the web server" @@ -58,8 +59,10 @@ usage() { exit 0 } +supdateExtraServerNames=() + shortopts="h?y" -longopts="help,channel:,branch:,git-path:,no-revert,overwrite-vhost,yes,hostname:" +longopts="help,channel:,branch:,git-path:,no-revert,overwrite-vhost,yes,hostname:,extra-server-name:" optargs=$(getopt -o $shortopts -l $longopts -n "$0" -- "$@") [[ $? -ne 0 ]] && usage eval set -- "$optargs" @@ -103,6 +106,15 @@ while :; do fi shift 2 ;; + --extra-server-name) + if [[ -n "${2}" ]]; then + supdateExtraServerNames+=("${2}") + else + echo "Error: --extra-server-name requires a value" + usage + fi + shift 2 + ;; --no-revert) autoRevert=0 shift @@ -229,7 +241,11 @@ if ! gitUpdateToBranch "$branch"; then exit 1 fi -(cd "$fog_git_path/bin" && bash installfog.sh -Y $updateVhostFlag ${supdatehostname:+--hostname "$supdatehostname"} >>$error_log 2>&1) +extraServerNameArgs=() +for extraname in "${supdateExtraServerNames[@]}"; do + extraServerNameArgs+=(--extra-server-name "$extraname") +done +(cd "$fog_git_path/bin" && bash installfog.sh -Y $updateVhostFlag ${supdatehostname:+--hostname "$supdatehostname"} "${extraServerNameArgs[@]}" >>$error_log 2>&1) installStatus=$? cd "$workingdir" diff --git a/lib/common/functions.sh b/lib/common/functions.sh index b10b0f7424..3aaecf3d74 100755 --- a/lib/common/functions.sh +++ b/lib/common/functions.sh @@ -3155,6 +3155,11 @@ writeUpdateFile() { # fogprogramdir: an admin's choice of stable/staging/dev must carry forward # on every upgrade, not just on the run it was made. fog_git_path fog_update_channel + # A genuine persisted preference like fog_update_channel above, not a + # RECORD like fogprogramdir/fog_git_path -- an admin's extra vhost/cert + # name(s) must carry forward on every upgrade, not just the run they + # were set on. + extraServerNames ) # Keys written by older installers that must be stripped on upgrade. local -a deprecatedKeys=( storageftpuser storageftppass bootfilename notpxedefaultfile php_verAdds ) @@ -3444,6 +3449,12 @@ EOF [[ -n $sanentries ]] && sanentries="${sanentries}"$'\n' sanentries="${sanentries}IP.${sancount} = ${ip}" done + dnscount=1 + dnsSanEntries="" + for extraname in $extraServerNames; do + dnscount=$((dnscount + 1)) + dnsSanEntries="${dnsSanEntries}"$'\n'"DNS.${dnscount} = ${extraname}" + done if [[ $recreateKeys == yes || $recreateCA == yes || $caCreated != yes || ! -e $sslpath || ! -e $sslprivkey ]]; then dots "Creating SSL Private Key" if [[ $(validip $certip) -ne 0 ]]; then @@ -3468,7 +3479,7 @@ CN = $certip subjectAltName = @alt_names [alt_names] $sanentries -DNS.1 = $hostname +DNS.1 = $hostname$dnsSanEntries EOF openssl req -new -sha512 -key $sslprivkey -out $sslcsr -config $sslpath/req.cnf >>$error_log 2>&1 << EOF $certip @@ -3485,7 +3496,7 @@ EOF subjectAltName = @alt_names [alt_names] $sanentries -DNS.1 = $hostname +DNS.1 = $hostname$dnsSanEntries EOF [[ -z $sslpubcert ]] && sslpubcert="$webdirdest/management/other/ssl/srvpublic.crt" if [[ ! -x $sslpubcert ]]; then @@ -3502,6 +3513,14 @@ EOF chown -R $apacheuser:$apacheuser $webdirdest/management/other >>$error_log 2>&1 errorStat $? [[ $httpproto == https ]] && sslenabled=" (Forced SSL)" || sslenabled=" (normal)" + # $extraServerNames is a space-joined string (see --extra-server-name). + # Computed once here and reused by both the nginx server_name lines below + # and Apache's vhostaliases, so an admin's extra name(s) reach every vhost + # block this function writes, not just one. + extraServerNamesSuffix="" + for extraname in $extraServerNames; do + extraServerNamesSuffix="${extraServerNamesSuffix} ${extraname}" + done case $webserver in nginx) case $novhost in @@ -3537,7 +3556,7 @@ EOF mv -fv "${etcconf}" "${etcconf}.${timestamp}" >>$workingdir/error_logs/fog_error_${version}.log 2>&1 echo "server {" > "$etcconf" echo " listen 80;" >> "$etcconf" - echo " server_name $ipaddresses $hostname;" >> "$etcconf" + echo " server_name $ipaddresses $hostname${extraServerNamesSuffix};" >> "$etcconf" if [[ $httpproto != https ]]; then echo " root ${docroot};" >> "$etcconf" echo " index index.html index.htm index.php;" >> "$etcconf" @@ -3588,7 +3607,7 @@ EOF fi echo "server {" >> "$etcconf" echo " listen $ipaddress:443 ssl${nginxhttp2listen};" >> "$etcconf" - echo " server_name $ipaddresses $hostname;" >> "$etcconf" + echo " server_name $ipaddresses $hostname${extraServerNamesSuffix};" >> "$etcconf" echo " root ${docroot};" >> "$etcconf" echo " index index.html index.htm index.php;" >> "$etcconf" echo " client_max_body_size 3000m;" >> "$etcconf" @@ -3655,7 +3674,7 @@ EOF fi echo "server {" >> "$etcconf" echo " listen $ipaddress:443 ssl${nginxhttp2listen};" >> "$etcconf" - echo " server_name $ipaddresses $hostname;" >> "$etcconf" + echo " server_name $ipaddresses $hostname${extraServerNamesSuffix};" >> "$etcconf" echo " root ${docroot};" >> "$etcconf" echo " index index.html index.htm index.php;" >> "$etcconf" echo " client_max_body_size 3000m;" >> "$etcconf" @@ -3742,6 +3761,7 @@ EOF # address it has. vhostname="$ipaddress" vhostaliases=$(echo $ipaddresses | awk '{for (i = 2; i <= NF; i++) printf " %s", $i}') + vhostaliases="${vhostaliases}${extraServerNamesSuffix}" mv -fv "${etcconf}" "${etcconf}.${timestamp}" >>$workingdir/error_logs/fog_error_${version}.log 2>&1 echo "" > "$etcconf" echo " " >> "$etcconf" From 825157d82a3d0c46986681e4d075fbf56dff1d97 Mon Sep 17 00:00:00 2001 From: JJ Fullmer Date: Thu, 6 Aug 2026 23:28:08 -0600 Subject: [PATCH 09/62] Fix: --extra-server-name in updatefog.sh also exits nonzero instead of usage()'s exit 0 Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_0178sa2JC7gq3Py4bRkRapQW --- bin/updatefog.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/updatefog.sh b/bin/updatefog.sh index b9e56a6e33..30f9a04c63 100755 --- a/bin/updatefog.sh +++ b/bin/updatefog.sh @@ -111,7 +111,7 @@ while :; do supdateExtraServerNames+=("${2}") else echo "Error: --extra-server-name requires a value" - usage + exit 9 fi shift 2 ;; From 6b6bdfeace883ee0cbabb70d6fc0afcc27f2ec69 Mon Sep 17 00:00:00 2001 From: JJ Fullmer Date: Thu, 6 Aug 2026 23:32:31 -0600 Subject: [PATCH 10/62] Mirror FOG_EXTRA_SERVER_NAMES into globalSettings for GUI visibility --- lib/common/functions.sh | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/lib/common/functions.sh b/lib/common/functions.sh index 3aaecf3d74..e9349f6de7 100755 --- a/lib/common/functions.sh +++ b/lib/common/functions.sh @@ -145,15 +145,17 @@ updateStorageNodeCredentials() { curl -s -k -X POST -d "nodePass" -d "ip=$(echo -n $ipaddress|base64)" -d "user=$(echo -n $username|base64)" --data-urlencode "pass=$(echo -n $password|base64)" -d "fogverified" $httpproto://$ipaddress${webroot}/maintenance/create_update_node.php echo "Done" } -# Mirrors fog_git_path/fog_update_channel into globalSettings so the GUI can -# show them without SSH. Like fogprogramdir's mirror into /etc/fog/fog.conf -# (GH-850), these are RECORDS, not controls: .fogsettings stays the source of -# truth, and the next installfog.sh/updatefog.sh run overwrites whatever an -# admin may have hand-edited here through the generic Settings tab. +# Mirrors fog_git_path/fog_update_channel/extraServerNames into globalSettings +# so the GUI can show them without SSH. Like fogprogramdir's mirror into +# /etc/fog/fog.conf (GH-850), these are RECORDS, not controls: .fogsettings +# stays the source of truth, and the next installfog.sh/updatefog.sh run +# overwrites whatever an admin may have hand-edited here through the generic +# Settings tab. recordGitUpdateSettings() { - dots "Recording fog_git_path/update channel" + dots "Recording fog_git_path/update channel/extra server names" mysql $sqloptionsuser --password="${snmysqlpass}" --execute="INSERT INTO globalSettings (settingKey, settingDesc, settingValue, settingCategory) VALUES ('FOG_GIT_PATH', 'Filesystem path of the FOG git checkout on this server. Recorded automatically by installfog.sh/updatefog.sh -- editing it here has no effect on the next update.', \"$fog_git_path\", 'FOG Update') ON DUPLICATE KEY UPDATE settingValue=\"$fog_git_path\"" $mysqldbname >>$error_log 2>&1 mysql $sqloptionsuser --password="${snmysqlpass}" --execute="INSERT INTO globalSettings (settingKey, settingDesc, settingValue, settingCategory) VALUES ('FOG_UPDATE_CHANNEL', 'Update channel this server tracks: stable, staging, or dev.', \"$fog_update_channel\", 'FOG Update') ON DUPLICATE KEY UPDATE settingValue=\"$fog_update_channel\"" $mysqldbname >>$error_log 2>&1 + mysql $sqloptionsuser --password="${snmysqlpass}" --execute="INSERT INTO globalSettings (settingKey, settingDesc, settingValue, settingCategory) VALUES ('FOG_EXTRA_SERVER_NAMES', 'Extra vhost/certificate name(s) this server answers to, beyond the primary hostname and detected IPs. Set via --extra-server-name -- editing it here has no effect on the next update.', \"$extraServerNames\", 'FOG Update') ON DUPLICATE KEY UPDATE settingValue=\"$extraServerNames\"" $mysqldbname >>$error_log 2>&1 errorStat $? } backupDB() { From 3a0d3362ef8721639c011c7323c8f5146d78af68 Mon Sep 17 00:00:00 2001 From: JJ Fullmer Date: Thu, 6 Aug 2026 23:36:47 -0600 Subject: [PATCH 11/62] Add bin/setupacme.sh for ACME leaf renewal against --external-ca --- bin/setupacme.sh | 183 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 183 insertions(+) create mode 100644 bin/setupacme.sh diff --git a/bin/setupacme.sh b/bin/setupacme.sh new file mode 100644 index 0000000000..d48b0e5171 --- /dev/null +++ b/bin/setupacme.sh @@ -0,0 +1,183 @@ +#!/bin/bash +# +# FOG is a computer imaging solution. +# Copyright (C) 2007 Chuck Syperski & Jian Zhang +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# Bootstraps acme.sh to issue and renew the web vhost's LEAF certificate +# against a CA already imported via installfog.sh --external-ca. Never +# touches the imported intermediate/root -- only the leaf -- so fog-client's +# pinned CA never changes across a renewal. acme.sh's own installer sets up +# its own renewal cron job; this script does not add a second one. See +# docs/superpowers/specs/2026-08-07-cert-separation-letsencrypt-design.md and +# FOGProject/fogproject#1013. +bindir=$(dirname $(readlink -f "$BASH_SOURCE")) +cd $bindir +workingdir=$(pwd) + +if [[ ! $EUID -eq 0 ]]; then + echo "setupacme.sh must be run as root user" + exit 1 +fi + +usage() { + echo -e "Usage: $0 [-h?] --directory-url (--http01 | --dns ) -d " + echo -e "\t-h -? --help\t\tDisplay this info" + echo -e "\t --directory-url\tACME server directory URL (public Let's Encrypt or" + echo -e "\t \tan internal ACME CA such as step-ca)" + echo -e "\t --http01\t\tUse HTTP-01 validation (acme.sh's --webroot mode against" + echo -e "\t \t\tthis server's own vhost docroot)" + echo -e "\t --dns\t\tUse DNS-01 validation via the named acme.sh DNS plugin --" + echo -e "\t \t\tthe plugin's own provider credentials must already be set" + echo -e "\t \t\tup in this shell's environment; setupacme.sh never stores them" + echo -e "\t-d\t\t\tDomain to issue the certificate for (repeatable)" + exit 0 +} + +shortopts="h?d:" +longopts="help,directory-url:,http01,dns:" +optargs=$(getopt -o $shortopts -l $longopts -n "$0" -- "$@") +[[ $? -ne 0 ]] && usage +eval set -- "$optargs" + +domains=() +while :; do + case $1 in + -h | -\? | --help) + usage + ;; + --directory-url) + directoryUrl="$2" + shift 2 + ;; + --http01) + validationMethod="http01" + shift + ;; + --dns) + validationMethod="dns" + dnsPlugin="$2" + shift 2 + ;; + -d) + domains+=("$2") + shift 2 + ;; + --) + shift + break + ;; + *) + echo "Error: unhandled option '$1'." + exit 10 + ;; + esac +done + +[[ ! -d ./error_logs/ ]] && mkdir -p ./error_logs >/dev/null 2>&1 +error_log="${workingdir}/error_logs/fog_setupacme_error.log" +: > "$error_log" + +if [[ -z $directoryUrl ]]; then + echo " * --directory-url is required (a public Let's Encrypt endpoint, or an internal ACME CA such as step-ca)." + exit 9 +fi +if [[ -z $validationMethod ]]; then + echo " * Pass either --http01 or --dns ." + exit 9 +fi +if [[ ${#domains[@]} -eq 0 ]]; then + echo " * At least one -d is required." + exit 9 +fi + +exitFail=1 +. ../lib/common/functions.sh + +[[ -z $fogprogramdir && -r /etc/fog/fog.conf ]] && . /etc/fog/fog.conf +[[ -z $fogprogramdir ]] && fogprogramdir="/opt/fog" +fogprogramdir="${fogprogramdir%/}" + +if [[ ! -r "$fogprogramdir/.fogsettings" ]]; then + echo " * No existing FOG install found at $fogprogramdir (.fogsettings missing)." + echo " * setupacme.sh configures an EXISTING install -- run installfog.sh first." + exit 1 +fi +. "$fogprogramdir/.fogsettings" + +# Precondition: --external-ca must already have imported a CA. These are +# exactly the files validateExternalCA() (lib/common/functions.sh) writes. +if [[ ! -e "$sslpath/CA/.fogCA.pem" || ! -e "$sslpath/CA/.fogCA.key" ]]; then + echo " * No external CA found at $sslpath/CA/ -- run installfog.sh --external-ca first." + echo " * setupacme.sh only ever renews a LEAF against a CA you already imported;" + echo " * it does not create or manage a CA itself." + exit 1 +fi + +dots "Checking for acme.sh" +if [[ ! -x "$HOME/.acme.sh/acme.sh" ]]; then + dots "Installing acme.sh" + curl -s https://get.acme.sh | sh -s email=root@localhost >>$error_log 2>&1 + errorStat $? +else + echo "Found" +fi +acmesh="$HOME/.acme.sh/acme.sh" + +case $webserver in + nginx) + reloadcmd="systemctl reload nginx" + ;; + httpd|apache*) + reloadcmd="systemctl reload $webserver" + ;; + *) + echo " * Unrecognized \$webserver ($webserver) -- cannot pick a reload command." + exit 1 + ;; +esac + +domainArgs=() +for domain in "${domains[@]}"; do + domainArgs+=(-d "$domain") +done + +dots "Issuing certificate via acme.sh" +case $validationMethod in + http01) + "$acmesh" --issue --server "$directoryUrl" "${domainArgs[@]}" --webroot "$docroot" >>$error_log 2>&1 + ;; + dns) + "$acmesh" --issue --server "$directoryUrl" "${domainArgs[@]}" --dns "$dnsPlugin" >>$error_log 2>&1 + ;; +esac +issueStatus=$? +# acme.sh's own exit code 2 means "already valid, no renewal needed yet" -- +# not a failure of this run. +if [[ $issueStatus -ne 0 && $issueStatus -ne 2 ]]; then + echo " * acme.sh --issue failed (exit $issueStatus). See $error_log." + exit $issueStatus +fi +echo "Done" + +dots "Installing certificate" +"$acmesh" --install-cert "${domainArgs[@]}" \ + --cert-file "$sslpubcert" \ + --key-file "$sslprivkey" \ + --reloadcmd "$reloadcmd" >>$error_log 2>&1 +errorStat $? + +echo " * setupacme.sh complete. acme.sh's own installer already scheduled its" +echo " own renewal cron job -- no further action is needed for renewals." From bd120cd0af918df0f05726b11ec46cb8dbfd6a08 Mon Sep 17 00:00:00 2001 From: JJ Fullmer Date: Fri, 7 Aug 2026 00:07:05 -0600 Subject: [PATCH 12/62] Fix: remove exitFail=1 (was masking install failures as success), add non-systemd reload fallback --- bin/setupacme.sh | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/bin/setupacme.sh b/bin/setupacme.sh index d48b0e5171..86d81a6fc4 100644 --- a/bin/setupacme.sh +++ b/bin/setupacme.sh @@ -103,7 +103,6 @@ if [[ ${#domains[@]} -eq 0 ]]; then exit 9 fi -exitFail=1 . ../lib/common/functions.sh [[ -z $fogprogramdir && -r /etc/fog/fog.conf ]] && . /etc/fog/fog.conf @@ -138,10 +137,18 @@ acmesh="$HOME/.acme.sh/acme.sh" case $webserver in nginx) - reloadcmd="systemctl reload nginx" + if [[ $systemctl == yes ]]; then + reloadcmd="systemctl reload nginx" + else + reloadcmd="$initdpath/nginx reload" + fi ;; httpd|apache*) - reloadcmd="systemctl reload $webserver" + if [[ $systemctl == yes ]]; then + reloadcmd="systemctl reload $webserver" + else + reloadcmd="$initdpath/$webserver reload" + fi ;; *) echo " * Unrecognized \$webserver ($webserver) -- cannot pick a reload command." From ca02e0b9e3083bc17d728bf1f8a63719230e4d6c Mon Sep 17 00:00:00 2001 From: JJ Fullmer Date: Fri, 7 Aug 2026 00:12:46 -0600 Subject: [PATCH 13/62] Fix: source config.sh + doOSSpecificIncludes so systemctl/initdpath are actually populated --- bin/setupacme.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bin/setupacme.sh b/bin/setupacme.sh index 86d81a6fc4..3930acf183 100644 --- a/bin/setupacme.sh +++ b/bin/setupacme.sh @@ -115,6 +115,9 @@ if [[ ! -r "$fogprogramdir/.fogsettings" ]]; then exit 1 fi . "$fogprogramdir/.fogsettings" +linuxReleaseName_lower="${osname,,}" +. ../lib/common/config.sh +[[ -n $osid ]] && doOSSpecificIncludes >/dev/null # Precondition: --external-ca must already have imported a CA. These are # exactly the files validateExternalCA() (lib/common/functions.sh) writes. From deae3d41ff835b817598df0cb49e033c62126373 Mon Sep 17 00:00:00 2001 From: JJ Fullmer Date: Fri, 7 Aug 2026 00:15:58 -0600 Subject: [PATCH 14/62] Document bin/setupacme.sh in EXTERNAL_CA_AND_LETSENCRYPT.md --- docs/EXTERNAL_CA_AND_LETSENCRYPT.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/EXTERNAL_CA_AND_LETSENCRYPT.md b/docs/EXTERNAL_CA_AND_LETSENCRYPT.md index d4763958b9..ef31f51749 100644 --- a/docs/EXTERNAL_CA_AND_LETSENCRYPT.md +++ b/docs/EXTERNAL_CA_AND_LETSENCRYPT.md @@ -196,6 +196,26 @@ High-level setup: 4. After each renewal, install the renewed leaf where Apache/Nginx serves it (a renewal hook — see [Renewal and rotation](#renewal-and-rotation)). +`bin/setupacme.sh` automates steps 3 and 4 above: it installs `acme.sh` if +needed, issues the leaf against the ACME directory URL you give it, installs +it where the vhost reads it, and wires up `acme.sh`'s `--reloadcmd` to reload +FOG's web server. It never touches the CA `--external-ca` already imported -- +only the leaf -- so a renewal never breaks fog-client's pinning. + +```bash +./setupacme.sh --directory-url https://step-ca.internal/acme/acme/directory \ + --http01 -d fog.example.com +``` + +Use `--dns ` instead of `--http01` for DNS-01 validation +(needed for public Let's Encrypt without exposing this server on port 80) -- +`setupacme.sh` never stores DNS provider credentials itself; whatever +`acme.sh` DNS plugin you name must already have its own credentials configured +in this shell's environment. + +`acme.sh`'s own installer sets up its own daily renewal cron job the first +time it's installed -- `setupacme.sh` does not add a second one. + Why this is better than public LE: **the intermediate you pin is stable and under your control**, so leaf renewals are transparent to clients, and nothing needs to be publicly resolvable. From ce88f75864f231adfc2d00cd99535b2574240a63 Mon Sep 17 00:00:00 2001 From: JJ Fullmer Date: Fri, 7 Aug 2026 06:14:34 -0600 Subject: [PATCH 15/62] Fix final review findings: ACME leaf overwrite protection, setupacme.sh exec bit, external-CA detection, updatefog.sh vhost flag, ACME domain defaulting, cert chain, install failure detection Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_0178sa2JC7gq3Py4bRkRapQW --- bin/setupacme.sh | 70 ++++++++++++++++++++++++++++++++--------- bin/updatefog.sh | 11 +++++++ lib/common/functions.sh | 11 ++++++- 3 files changed, 77 insertions(+), 15 deletions(-) mode change 100644 => 100755 bin/setupacme.sh diff --git a/bin/setupacme.sh b/bin/setupacme.sh old mode 100644 new mode 100755 index 3930acf183..9d192808a5 --- a/bin/setupacme.sh +++ b/bin/setupacme.sh @@ -33,7 +33,7 @@ if [[ ! $EUID -eq 0 ]]; then fi usage() { - echo -e "Usage: $0 [-h?] --directory-url (--http01 | --dns ) -d " + echo -e "Usage: $0 [-h?] --directory-url (--http01 | --dns ) [-d ]" echo -e "\t-h -? --help\t\tDisplay this info" echo -e "\t --directory-url\tACME server directory URL (public Let's Encrypt or" echo -e "\t \tan internal ACME CA such as step-ca)" @@ -42,14 +42,18 @@ usage() { echo -e "\t --dns\t\tUse DNS-01 validation via the named acme.sh DNS plugin --" echo -e "\t \t\tthe plugin's own provider credentials must already be set" echo -e "\t \t\tup in this shell's environment; setupacme.sh never stores them" - echo -e "\t-d\t\t\tDomain to issue the certificate for (repeatable)" + echo -e "\t-d\t\t\tDomain to issue the certificate for (repeatable). Defaults to" + echo -e "\t \t\t\tthe hostname plus any --extra-server-name from .fogsettings," + echo -e "\t \t\t\tso the leaf covers exactly what the vhost answers to" exit 0 } shortopts="h?d:" longopts="help,directory-url:,http01,dns:" optargs=$(getopt -o $shortopts -l $longopts -n "$0" -- "$@") -[[ $? -ne 0 ]] && usage +# Not `usage` -- usage() exits 0, which would report a malformed flag as +# success. getopt -n "$0" already printed its own error to stderr. +[[ $? -ne 0 ]] && exit 9 eval set -- "$optargs" domains=() @@ -98,10 +102,6 @@ if [[ -z $validationMethod ]]; then echo " * Pass either --http01 or --dns ." exit 9 fi -if [[ ${#domains[@]} -eq 0 ]]; then - echo " * At least one -d is required." - exit 9 -fi . ../lib/common/functions.sh @@ -119,20 +119,49 @@ linuxReleaseName_lower="${osname,,}" . ../lib/common/config.sh [[ -n $osid ]] && doOSSpecificIncludes >/dev/null -# Precondition: --external-ca must already have imported a CA. These are -# exactly the files validateExternalCA() (lib/common/functions.sh) writes. -if [[ ! -e "$sslpath/CA/.fogCA.pem" || ! -e "$sslpath/CA/.fogCA.key" ]]; then - echo " * No external CA found at $sslpath/CA/ -- run installfog.sh --external-ca first." +# Default the domain set to exactly the names the vhost/cert already advertise, +# so an admin who used --hostname/--extra-server-name at install time cannot +# accidentally get an ACME leaf covering fewer names than the vhost answers to. +# Deliberately checked here rather than right after argument parsing: both +# values come from .fogsettings, which is only sourced above. +if [[ ${#domains[@]} -eq 0 ]]; then + for extraname in $hostname $extraServerNames; do + domains+=("$extraname") + done +fi +if [[ ${#domains[@]} -eq 0 ]]; then + echo " * No -d given, and no hostname/extra server name found in .fogsettings." + echo " * Pass at least one -d ." + exit 9 +fi + +# Precondition: --external-ca must already have imported a CA. $externalca is +# the only reliable sentinel -- the CA/.fogCA.pem and CA/.fogCA.key paths below +# are written by FOG's OWN self-signed CA path too (same filenames, different +# origin), so testing them alone passes on every install and enforces nothing. +if [[ $externalca != yes ]]; then + echo " * No external CA configured -- run installfog.sh --external-ca first." echo " * setupacme.sh only ever renews a LEAF against a CA you already imported;" echo " * it does not create or manage a CA itself." exit 1 fi +if [[ ! -e "$sslpath/CA/.fogCA.pem" || ! -e "$sslpath/CA/.fogCA.key" ]]; then + echo " * --external-ca is configured but its files are missing at $sslpath/CA/ -- re-run installfog.sh --external-ca." + exit 1 +fi dots "Checking for acme.sh" if [[ ! -x "$HOME/.acme.sh/acme.sh" ]]; then dots "Installing acme.sh" - curl -s https://get.acme.sh | sh -s email=root@localhost >>$error_log 2>&1 - errorStat $? + # $? here would be sh's exit code, not curl's -- with no network, curl + # writes nothing, sh reads an empty script and exits 0. The executable + # check below is the only honest signal that the install actually happened. + curl -fsSL https://get.acme.sh | sh -s email=root@localhost >>$error_log 2>&1 + if [[ ! -x "$HOME/.acme.sh/acme.sh" ]]; then + echo " * acme.sh installation failed. See $error_log." + exit 1 + fi + echo "Done" else echo "Found" fi @@ -183,11 +212,24 @@ fi echo "Done" dots "Installing certificate" +# --fullchain-file, not --cert-file: $sslpubcert is what the vhost's +# ssl_certificate/SSLCertificateFile points at, so it must carry the +# intermediate as well or clients see an incomplete chain. "$acmesh" --install-cert "${domainArgs[@]}" \ - --cert-file "$sslpubcert" \ + --fullchain-file "$sslpubcert" \ --key-file "$sslprivkey" \ --reloadcmd "$reloadcmd" >>$error_log 2>&1 errorStat $? +# Tell every later installfog.sh/updatefog.sh run that this leaf is ACME-managed +# so createSSLCA() stops regenerating it from the original (now stale) CSR. +# writeUpdateFile() merges just this key into the existing .fogsettings, but it +# also refreshes the "## Version:" header from $version -- which nothing has set +# in this script, so derive it the same way installfog.sh/updatefog.sh do or the +# header gets blanked as a side effect. +[[ -z $version ]] && version="$(awk -F\' /"define\('FOG_VERSION'[,](.*)"/'{print $4}' ../packages/web/lib/fog/system.class.php | tr -d '[[:space:]]')" +acmeLeaf="yes" +writeUpdateFile + echo " * setupacme.sh complete. acme.sh's own installer already scheduled its" echo " own renewal cron job -- no further action is needed for renewals." diff --git a/bin/updatefog.sh b/bin/updatefog.sh index 30f9a04c63..336bdee4bb 100755 --- a/bin/updatefog.sh +++ b/bin/updatefog.sh @@ -49,7 +49,9 @@ usage() { echo -e "\t \t\tnot change the tracked channel for future runs" echo -e "\t --git-path\tOverride the git checkout path this server records" echo -e "\t --hostname\tOverride the vhost/cert hostname for this update" + echo -e "\t \t\t(implies --overwrite-vhost)" echo -e "\t --extra-server-name\tAdd an extra vhost/cert name for this update (repeatable)" + echo -e "\t \t(implies --overwrite-vhost)" echo -e "\t --no-revert\tOn failure, leave the system as-is instead of" echo -e "\t \t\tautomatically reverting to the previous commit" echo -e "\t --overwrite-vhost\tLet installfog.sh regenerate the web server" @@ -139,6 +141,15 @@ while :; do esac done +# --hostname/--extra-server-name are requests for a vhost-VISIBLE change, so +# they imply --overwrite-vhost. With the "-F" default above, createSSLCA() +# prints "Skipped" instead of writing the vhost at all: .fogsettings and the +# cert SAN would change (cert generation happens before the novhost check) but +# server_name/ServerAlias would silently keep the old names. +if [[ -n $supdatehostname || ${#supdateExtraServerNames[@]} -gt 0 ]]; then + updateVhostFlag="" +fi + [[ ! -d ./error_logs/ ]] && mkdir -p ./error_logs >/dev/null 2>&1 error_log="${workingdir}/error_logs/fog_update_error.log" : > "$error_log" diff --git a/lib/common/functions.sh b/lib/common/functions.sh index e9349f6de7..bae17ab68a 100755 --- a/lib/common/functions.sh +++ b/lib/common/functions.sh @@ -3162,6 +3162,12 @@ writeUpdateFile() { # name(s) must carry forward on every upgrade, not just the run they # were set on. extraServerNames + # Set by bin/setupacme.sh once it installs an ACME-issued leaf. Tells + # createSSLCA() below to leave that leaf alone on every later run -- + # without this, the leaf gets silently regenerated from the ORIGINAL + # CSR (stale public key) while the private key on disk is the ACME + # key, producing a cert/key mismatch that stops the web server. + acmeLeaf ) # Keys written by older installers that must be stripped on upgrade. local -a deprecatedKeys=( storageftpuser storageftppass bootfilename notpxedefaultfile php_verAdds ) @@ -3501,7 +3507,10 @@ $sanentries DNS.1 = $hostname$dnsSanEntries EOF [[ -z $sslpubcert ]] && sslpubcert="$webdirdest/management/other/ssl/srvpublic.crt" - if [[ ! -x $sslpubcert ]]; then + if [[ $acmeLeaf == yes ]]; then + echo " * Leaf certificate is ACME-managed (see bin/setupacme.sh) -- leaving it in place." + echo " Re-run bin/setupacme.sh if you changed --hostname/--extra-server-name." + elif [[ ! -x $sslpubcert ]]; then dots "Creating SSL Certificate" openssl x509 -req -in $sslcsr -CA $sslcapem -CAkey $sslcakey -CAcreateserial -out $sslpubcert -days 3650 -extensions v3_ca -extfile $sslpath/ca.cnf >>$error_log 2>&1 errorStat $? From c8a29e576a06451544cf2ad81188f945098d4daa Mon Sep 17 00:00:00 2001 From: JJ Fullmer Date: Fri, 7 Aug 2026 06:34:48 -0600 Subject: [PATCH 16/62] Don't preserve ACME leaf across --recreate-keys/--recreate-ca The acmeLeaf gate skipped leaf regeneration unconditionally, but --recreate-keys/--recreate-ca regenerate the private key unconditionally -- combining either with an ACME-managed install reintroduced the exact cert/key mismatch the acmeLeaf marker exists to prevent. Falling through to the normal self-signed regeneration path when either flag is given restores a consistent pair, matching pre-acmeLeaf behavior for that specific case. Found in the final whole-branch review's fix-wave re-review. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_0178sa2JC7gq3Py4bRkRapQW --- lib/common/functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/common/functions.sh b/lib/common/functions.sh index bae17ab68a..cdf187e1be 100755 --- a/lib/common/functions.sh +++ b/lib/common/functions.sh @@ -3507,7 +3507,7 @@ $sanentries DNS.1 = $hostname$dnsSanEntries EOF [[ -z $sslpubcert ]] && sslpubcert="$webdirdest/management/other/ssl/srvpublic.crt" - if [[ $acmeLeaf == yes ]]; then + if [[ $acmeLeaf == yes && $recreateKeys != yes && $recreateCA != yes ]]; then echo " * Leaf certificate is ACME-managed (see bin/setupacme.sh) -- leaving it in place." echo " Re-run bin/setupacme.sh if you changed --hostname/--extra-server-name." elif [[ ! -x $sslpubcert ]]; then From 5d84f5e684c8a5ea47e2def5132f89a8cfd1d3e6 Mon Sep 17 00:00:00 2001 From: JJ Fullmer Date: Fri, 7 Aug 2026 16:37:16 -0600 Subject: [PATCH 17/62] Record verified Let's Encrypt + iPXE netboot result, correct FOG_WEB_HOST claim Ad-hoc testing confirmed a real Let's Encrypt certificate on the vhost validates for iPXE netboot with no FOG-side change, as this doc already claimed. Records the two settings that were needed in practice (httpproto=https, FOG_WEB_HOST set to the FQDN). Also settles a long-standing suspicion that an FQDN FOG_WEB_HOST upsets the PHP CLI daemons. It does not: waitInterfaceReady() gates on in_array(FOG_WEB_HOST, self::$ips), but getIPAddress() builds that list as the detected IPs plus reverse-DNS names plus FOG_WEB_HOST itself, so the value is compared against a list it was just inserted into and the clause can never fail. Documents what can actually stall that loop instead (no detectable IPs, or no overlap with the storage-node IPs in the DB). Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YE3jeTexYEcAxWFbZcZgpZ --- docs/EXTERNAL_CA_AND_LETSENCRYPT.md | 45 +++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/docs/EXTERNAL_CA_AND_LETSENCRYPT.md b/docs/EXTERNAL_CA_AND_LETSENCRYPT.md index ef31f51749..064d803b57 100644 --- a/docs/EXTERNAL_CA_AND_LETSENCRYPT.md +++ b/docs/EXTERNAL_CA_AND_LETSENCRYPT.md @@ -250,6 +250,51 @@ up for before you do. re-pins. This is the core reason public LE is fragile for FOG and an internal ACME CA is preferred. +**Confirmed by ad-hoc testing:** a real Let's Encrypt certificate on the +vhost does validate for iPXE netboot with no FOG-side change, as this doc +claims. Getting there in practice took two settings beyond just dropping the +cert in place: + +- `httpproto` in `.fogsettings` had to be set to `https`. +- `FOG_WEB_HOST` (in the FOG web UI's Settings page) had to be the server's + FQDN, not its IP address. + +**On FOG_WEB_HOST and the background services — checked, not a problem.** +There is a long-standing suspicion that pointing `FOG_WEB_HOST` at an FQDN +(rather than an IP) upsets the PHP CLI daemons — `FOGFileDeleter`, +`FOGImageReplicator`, `FOGTaskScheduler` and the rest. Every one of them +calls `waitInterfaceReady()` (`packages/web/lib/service/fogservice.class.php:152`) +before starting, which refuses to proceed while: + +```php +!in_array(self::getSetting('FOG_WEB_HOST'), self::$ips) +``` + +That looks like it would loop forever on an FQDN, since `self::$ips` reads +as a list of IP addresses. It does not, because `getIPAddress()` +(`packages/web/lib/fog/fogbase.class.php:3140-3144`) builds that list as the +detected IPs **plus** their reverse-DNS names **plus** `FOG_WEB_HOST` +itself: + +```php +$output = self::fastmerge( + $IPs, $Names, + ['127.0.0.1', '127.0.1.1', self::getSetting('FOG_WEB_HOST')] +); +``` + +So the value is compared against a list it was just inserted into — that +clause can never fail, whatever `FOG_WEB_HOST` is set to. An FQDN is safe +here. (It is also, for the same reason, effectively dead code — worth +tidying someday, but it is not what would break a service.) + +What *can* still stall `waitInterfaceReady()` is the other two clauses: no +detectable IPs at all, or no overlap between the detected addresses and the +storage-node addresses recorded in the database +(`array_intersect(self::$knownips, self::$ips)`). If a daemon does hang at +"Interface not ready", check the storage node's configured IP, not +`FOG_WEB_HOST`. + > **Bottom line:** if you want ACME automation, run an **internal** ACME CA. Use > public LE only if you genuinely need publicly trusted certs (e.g. a > public-facing portal) and you have a plan for re-pinning clients when LE rotates From e9c5a0b87992727120896af96511c50f1453671f Mon Sep 17 00:00:00 2001 From: JJ Fullmer Date: Fri, 7 Aug 2026 16:37:37 -0600 Subject: [PATCH 18/62] Add design docs: customization preservation and three-zone PKI separation Two spec+plan pairs for follow-on work discussed in #1014: Customization preservation moves backup/restore of admin customizations into installfog.sh itself so it protects every run, not only ones routed through updatefog.sh. Covers a FOG-managed vhost block (so security fixes to the template still land on a hand-edited vhost), setting-driven FOG_IPXE_BG_FILE handling, versioned kernel/init backups, an optional custom.ipxe hook, and closing a gap where an admin-supplied Secure Boot key living under $webdirdest is deleted by configureHttpd()'s own wipe. Three-zone PKI separation splits today's flat CA into a Root plus Web, Client Communication and Secure Boot intermediates. Records a finding from tracing the code: .srvprivate.key is the web vhost's TLS key AND the key certDecrypt() uses for every fog-client handshake, so replacing the web certificate breaks client authentication today. Secure Boot's enrolled MOK is currently the signing leaf itself, which is why rotating it needs a firmware trip to every machine; an intermediate fixes that. Both are plans only -- no implementation, and the PKI work is gated on three Phase 0 verifications that need a live server, real UEFI hardware, and the zazzles source. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YE3jeTexYEcAxWFbZcZgpZ --- .../2026-08-07-customization-preservation.md | 895 +++++++++++++ .../2026-08-07-three-zone-pki-separation.md | 1166 +++++++++++++++++ ...08-07-customization-preservation-design.md | 615 +++++++++ ...-08-07-three-zone-pki-separation-design.md | 988 ++++++++++++++ 4 files changed, 3664 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-07-customization-preservation.md create mode 100644 docs/superpowers/plans/2026-08-07-three-zone-pki-separation.md create mode 100644 docs/superpowers/specs/2026-08-07-customization-preservation-design.md create mode 100644 docs/superpowers/specs/2026-08-07-three-zone-pki-separation-design.md diff --git a/docs/superpowers/plans/2026-08-07-customization-preservation.md b/docs/superpowers/plans/2026-08-07-customization-preservation.md new file mode 100644 index 0000000000..e9d48190b7 --- /dev/null +++ b/docs/superpowers/plans/2026-08-07-customization-preservation.md @@ -0,0 +1,895 @@ +# Install-time customization preservation Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Move customization backup/restore *into* `installfog.sh` itself (so +it protects any run, not only ones that went through `bin/updatefog.sh`), +make `bin/updatefog.sh` a thin git-sync-then-invoke wrapper, replace the +vhost's all-or-nothing regenerate/skip choice with a FOG-managed-block +convention, make the iPXE background mechanism setting-driven, add bounded +versioned kernel/init backups, add an optional custom-PXE-script hook, close +the admin-supplied Secure Boot key persistence gap, and document all of it. + +**Architecture:** See `docs/superpowers/specs/2026-08-07-customization-preservation-design.md`. +Six independent, additive pieces, each mergeable on its own. + +**Tech Stack:** Bash (installer/updater scripts), MySQL (`globalSettings` +query), OpenSSL/`sbsign` (unchanged), iPXE script syntax (`.ipxe` files). + +## Global Constraints + +- No CI/test framework exists for this repo's shell scripts. Every task's + "test" step is a manual invocation + assertion on real output. Always run + `bash -n ` after every edit before anything else. +- Follow the existing staging-variable convention exactly: a new flag sets an + `s`-prefixed variable during `getopt` parsing, applied to the real variable + only *after* `.fogsettings` has been sourced, in the "evaluation of command + line options" block (`bin/installfog.sh:615-681`). +- Every new file-path-shaped value must be validated before it reaches a file + write — same posture as `--git-path`/`--fogprogramdir`. +- `bin/updatefog.sh` never runs `installfog.sh` interactively (always `-Y`) — + any new flag consumed there must be passed straight through. +- This plan assumes the `--hostname`/`--extra-server-name` work from + `docs/superpowers/plans/2026-08-07-cert-separation-letsencrypt.md` is + already merged (`extraServerNamesSuffix`, `$dnsSanEntries`, etc. already + exist in `createSSLCA()`) — Task 3 below builds directly on top of it and + does not reintroduce it. +- Design doc: `docs/superpowers/specs/2026-08-07-customization-preservation-design.md`. + Read it if anything below is ambiguous. + +--- + +### Task 1: Setting-driven iPXE background backup/restore + +**Files:** +- Modify: `lib/common/functions.sh` — new `backupPreservedCustomizations()` + and `restorePreservedCustomizations()` functions (place near + `backupReports()`, `functions.sh:72`, since they play the same role). +- Modify: `bin/installfog.sh` — add both calls to the master-install sequence + (`bin/installfog.sh:938-963`). + +**Interfaces:** +- Consumes: `$sqloptionsuser`/`$snmysqlpass`/`$mysqldbname` (set by + `configureMySql`), `$webdirdest`, `$fogprogramdir`, `$username`, + `$apacheuser`. +- Produces: `${fogprogramdir}/customizations/ipxe-bg/`, + `${fogprogramdir}/customizations/ipxe-legacy/`. + +- [ ] **Step 1: Add `backupPreservedCustomizations()`** + +Insert into `lib/common/functions.sh`, near `backupReports()`: + +```bash +# Backs up whatever is actually customized under $webdirdest/service/ipxe/ +# BEFORE configureHttpd()'s rm -rf $webdirdest destroys it. Lives outside +# $webdirdest (under $fogprogramdir) so it survives that wipe by +# construction, the same way Secure Boot keys already do. Called from every +# installfog.sh run directly -- this used to only happen via +# bin/updatefog.sh's backupCustomizations(), which meant a bare installfog.sh +# re-run got none of it. See docs/superpowers/specs/2026-08-07-customization-preservation-design.md. +backupPreservedCustomizations() { + dots "Backing up customizations before rebuilding the web tree" + local custdir="${fogprogramdir}/customizations" + local ipxedir="${webdirdest}service/ipxe" + local st=0 + mkdir -p "$custdir/ipxe-bg" "$custdir/ipxe-legacy" >>$error_log 2>&1 || st=1 + + # FOG_IPXE_BG_FILE is a real globalSettings row (see + # packages/web/commons/schema.php) -- read the ACTUAL value rather than + # assuming "bg.png". On a first-ever install globalSettings does not + # exist yet; the query errors into $error_log and $bgfile stays empty, + # which is treated identically to "nothing customized." + bgfile=$(mysql $sqloptionsuser --password="${snmysqlpass}" -N -B \ + --execute="SELECT settingValue FROM globalSettings WHERE settingKey='FOG_IPXE_BG_FILE'" \ + $mysqldbname 2>>$error_log) + if [[ -n $bgfile && -f "${ipxedir}/${bgfile}" ]]; then + cp -f "${ipxedir}/${bgfile}" "${custdir}/ipxe-bg/${bgfile}" >>$error_log 2>&1 || st=1 + fi + + local f + for f in refind.conf refind.efi refind_x64.efi refind_ia32.efi refind_aa64.efi; do + [[ -f "${ipxedir}/${f}" ]] && { cp -f "${ipxedir}/${f}" "${custdir}/ipxe-legacy/${f}" >>$error_log 2>&1 || st=1; } + done + errorStat $st +} +``` + +- [ ] **Step 2: Add `restorePreservedCustomizations()`** + +Insert directly after it: + +```bash +# Restores what backupPreservedCustomizations() saved, AFTER +# configureTFTPandPXE()'s downloadfiles() has re-laid the default-named +# kernel/init set. Deliberately does NOT restore the six default kernel/init +# names here (bzImage, bzImage32, arm_Image, init.xz, init_32.xz, +# arm_init.cpio.gz) -- an update should pick up the latest kernel. That is +# what Task 4's versioned backup exists to provide a manual restore path for. +restorePreservedCustomizations() { + dots "Restoring customizations" + local custdir="${fogprogramdir}/customizations" + local ipxedir="${webdirdest}service/ipxe" + local st=0 + + if [[ -n $bgfile && -f "${custdir}/ipxe-bg/${bgfile}" ]]; then + cp -f "${custdir}/ipxe-bg/${bgfile}" "${ipxedir}/${bgfile}" >>$error_log 2>&1 || st=1 + fi + local f + for f in refind.conf refind.efi refind_x64.efi refind_ia32.efi refind_aa64.efi; do + [[ -f "${custdir}/ipxe-legacy/${f}" ]] && { cp -f "${custdir}/ipxe-legacy/${f}" "${ipxedir}/${f}" >>$error_log 2>&1 || st=1; } + done + chown -R ${username}:${apacheuser} "$ipxedir" >>$error_log 2>&1 + errorStat $st +} +``` + +(`$bgfile` is deliberately a plain global here, not `local` in the backup +function, so the restore function -- called later in the same script -- reuses +the exact name that was actually backed up. Matches the existing convention +of cross-function shared state in this file, e.g. `$updatePrevCommit`.) + +- [ ] **Step 3: Syntax-check** + +Run: `bash -n lib/common/functions.sh`. Expected: no output, exit 0. + +- [ ] **Step 4: Wire both calls into `installfog.sh`'s master-install sequence** + +In `bin/installfog.sh`, in the `[Nn]` (master install) branch of the big +`case $installtype in` (around line 938), change: +```bash + writeUpdateFile + backupReports + configureHttpd +``` +to: +```bash + writeUpdateFile + backupReports + backupPreservedCustomizations + configureHttpd +``` +and change: +```bash + configureTFTPandPXE + configureFTP +``` +to: +```bash + configureTFTPandPXE + restorePreservedCustomizations + configureFTP +``` + +- [ ] **Step 5: Syntax-check** + +Run: `bash -n bin/installfog.sh`. Expected: no output, exit 0. + +- [ ] **Step 6: Manually verify (test Linux box with a running FOG install)** + +In the FOG GUI, set `FOG_IPXE_BG_FILE` to a distinct name (e.g. +`custom-bg.png`) and place a real file under +`$webdirdest/service/ipxe/custom-bg.png`. Run `bash installfog.sh -Y` +**directly** (not via `updatefog.sh`). Confirm: +- `custom-bg.png` still exists and is unchanged after the run. +- `ls $fogprogramdir/customizations/ipxe-bg/` shows `custom-bg.png`. +- `FOG_IPXE_BG_FILE` is still `custom-bg.png` in the GUI. + +Run once more with `FOG_IPXE_BG_FILE` left at the stock `bg.png` and no +custom file present — confirm no errors, `$bgfile` resolves to `bg.png`, and +nothing unexpected appears under `customizations/ipxe-bg/`. + +- [ ] **Step 7: Commit** + +```bash +git add lib/common/functions.sh bin/installfog.sh +git commit -m "Move iPXE background backup/restore into installfog.sh, keyed to the actual FOG_IPXE_BG_FILE value" +``` + +--- + +### Task 2: Secure Boot admin-supplied key/cert persistence + +**Files:** +- Modify: `lib/common/functions.sh` — new `preserveSecureBootAdminFiles()`. +- Modify: `bin/installfog.sh` — call it right after the existing + `--secure-boot-key`/`--secure-boot-cert` pair validation. + +**Interfaces:** +- Consumes: `$secureBootKey`, `$secureBootCert`, `$fogprogramdir`. +- Produces: possibly-reassigned `$secureBootKey`/`$secureBootCert`, now always + pointing under `${fogprogramdir}/secureboot/`. + +- [ ] **Step 1: Add `preserveSecureBootAdminFiles()`** + +Insert into `lib/common/functions.sh`, directly before `_ensureSecureBootKeys()` +(`functions.sh:4625`): + +```bash +# Closes a real gap: an admin-supplied --secure-boot-key/--secure-boot-cert +# pair is persisted verbatim in .fogsettings (see writeUpdateFile's +# managedKeys) and _ensureSecureBootKeys() trusts that path forever +# (`[[ -n $secureBootKey && -n $secureBootCert ]] && return 0`) without ever +# copying it anywhere. If that path happens to be inside $webdirdest (or +# anywhere else this installer deletes/regenerates), configureHttpd()'s +# rm -rf $webdirdest deletes it before downloadfiles() -> _resignKernels()/ +# _publishSecureBootKit() ever read it -- in the SAME run the admin first +# passed the flags. Copying into $fogprogramdir/secureboot/ gives the +# admin-supplied pair the same "outside $webdirdest, survives every wipe by +# construction" guarantee FOG's own generated keys already have, without +# changing _ensureSecureBootKeys()'s "admin-supplied pair always wins, never +# regenerated" contract -- the ORIGINAL file the admin pointed at is still +# never modified; this only decides which COPY gets used and persisted. +# Idempotent: once the copy exists and .fogsettings points at it, this is a +# no-op on every later run, until the admin passes the flags again. +preserveSecureBootAdminFiles() { + [[ -z $secureBootKey || -z $secureBootCert ]] && return 0 + local keydir="${fogprogramdir}/secureboot" + local key="${keydir}/MOK.key" + local cert="${keydir}/MOK.pem" + local st=0 + + mkdir -p "$keydir" >>$error_log 2>&1 + chown root:root "$keydir" >>$error_log 2>&1 + chmod 0700 "$keydir" >>$error_log 2>&1 + + if [[ "$(readlink -f "$secureBootKey")" != "$(readlink -f "$key")" ]]; then + dots "Preserving admin-supplied Secure Boot signing key" + cp -f "$secureBootKey" "$key" >>$error_log 2>&1 || st=1 + chown root:root "$key" >>$error_log 2>&1 + chmod 0600 "$key" >>$error_log 2>&1 + secureBootKey="$key" + errorStat $st + fi + if [[ "$(readlink -f "$secureBootCert")" != "$(readlink -f "$cert")" ]]; then + cp -f "$secureBootCert" "$cert" >>$error_log 2>&1 || st=1 + chown root:root "$cert" >>$error_log 2>&1 + chmod 0644 "$cert" >>$error_log 2>&1 + secureBootCert="$cert" + errorStat $st + fi +} +``` + +- [ ] **Step 2: Syntax-check** + +Run: `bash -n lib/common/functions.sh`. Expected: no output, exit 0. + +- [ ] **Step 3: Call it from `installfog.sh`, right after the existing pair + validation** + +In `bin/installfog.sh`, directly after the existing block (currently ending +around line 681): +```bash + unset sbfile + fi +``` +add: +```bash + unset sbfile + fi + preserveSecureBootAdminFiles +``` +(This must run before `configureMySql`/`configureHttpd` — it already does, +since the option-evaluation block runs near the very top of the script, +long before the `case $doupdate`/main sequence.) + +- [ ] **Step 4: Syntax-check** + +Run: `bash -n bin/installfog.sh`. Expected: no output, exit 0. + +- [ ] **Step 5: Manually verify (test Linux box)** + +Generate a throwaway key/cert pair somewhere *inside* what will become +`$webdirdest` (the worst-case gap this closes), e.g. +`/var/www/html/fog/service/ipxe/my.key`/`my.pem`. Run: +```bash +./installfog.sh -Y --secure-boot-key /var/www/html/fog/service/ipxe/my.key \ + --secure-boot-cert /var/www/html/fog/service/ipxe/my.pem +``` +Confirm: +- `/opt/fog/secureboot/MOK.key`/`MOK.pem` now exist and match the originals + (`diff` them before the run finishes wiping the source, or compare + checksums captured beforehand). +- `grep secureBootKey /opt/fog/.fogsettings` shows the `/opt/fog/secureboot/...` + path, not the original `/var/www/html/...` path. +- Kernels under `service/ipxe/` are correctly signed with this key + (`sbverify --cert /opt/fog/secureboot/MOK.pem service/ipxe/bzImage`). + +Run `./installfog.sh -Y` again with no flags: confirm no errors, and the +persisted `/opt/fog/secureboot/...` path is unchanged (no-op copy). + +- [ ] **Step 6: Commit** + +```bash +git add lib/common/functions.sh bin/installfog.sh +git commit -m "Preserve admin-supplied Secure Boot key/cert outside \$webdirdest" +``` + +--- + +### Task 3: FOG-managed vhost block + +**Files:** +- Modify: `lib/common/functions.sh` — new `spliceManagedBlock()`; modify + `createSSLCA()`'s nginx branch (`functions.sh:3536` ff.) and Apache branch + (`functions.sh:3751` ff.) to write into a temp file and call it instead of + writing `$etcconf` directly. +- Modify: `bin/updatefog.sh` — flip `$updateVhostFlag`'s default. + +**Interfaces:** +- Produces: `spliceManagedBlock(conffile, contentfile)` — general-purpose, + usable by any future FOG-owned-region-in-an-admin-editable-file need, not + just this one. +- Consumes: `$etcconf`, `$novhost`, `$timestamp` (all already exist). + +- [ ] **Step 1: Add marker constants and `spliceManagedBlock()`** + +Insert into `lib/common/functions.sh`, directly before `createSSLCA()` +(`functions.sh:3412`): + +```bash +FOG_MANAGED_BEGIN='# === FOG MANAGED BLOCK -- DO NOT EDIT BETWEEN THESE LINES (see docs/SUPPORTED_CUSTOMIZATIONS.md) ===' +FOG_MANAGED_END='# === END FOG MANAGED BLOCK ===' + +# Replaces only the FOG-owned region of $1 with $2's content, leaving +# anything an admin added outside that region untouched. If $1 doesn't exist, +# or exists but has no markers yet, this still writes something sane (a +# fresh single-block file, or an appended block onto existing content) -- see +# docs/superpowers/specs/2026-08-07-customization-preservation-design.md, +# Architecture #1, for why this replaces whole-file regeneration instead of a +# separate template-file mechanism. +spliceManagedBlock() { + local conffile="$1" contentfile="$2" + if [[ ! -f "$conffile" ]]; then + { echo "$FOG_MANAGED_BEGIN"; cat "$contentfile"; echo "$FOG_MANAGED_END"; } > "$conffile" + return $? + fi + if grep -qF "$FOG_MANAGED_BEGIN" "$conffile" && grep -qF "$FOG_MANAGED_END" "$conffile"; then + local tmp="${conffile}.fogsplice.$$" + awk -v b="$FOG_MANAGED_BEGIN" -v e="$FOG_MANAGED_END" -v cf="$contentfile" ' + $0 == b { print; while ((getline line < cf) > 0) print line; close(cf); skip=1; next } + $0 == e { print; skip=0; next } + !skip { print } + ' "$conffile" > "$tmp" && mv -f "$tmp" "$conffile" + return $? + fi + # No markers found -- an admin's own file, or an upgrade from before this + # feature existed. Append, never overwrite existing content. + { echo "$FOG_MANAGED_BEGIN"; cat "$contentfile"; echo "$FOG_MANAGED_END"; } >> "$conffile" +} +``` + +- [ ] **Step 2: Syntax-check** + +Run: `bash -n lib/common/functions.sh`. Expected: no output, exit 0. + +- [ ] **Step 3: Redirect the nginx branch's generation into a temp file** + +In `createSSLCA()`'s nginx branch, change the first write (currently, per +line 3567-3568): +```bash + mv -fv "${etcconf}" "${etcconf}.${timestamp}" >>$workingdir/error_logs/fog_error_${version}.log 2>&1 + echo "server {" > "$etcconf" +``` +to: +```bash + mv -fv "${etcconf}" "${etcconf}.${timestamp}" >>$workingdir/error_logs/fog_error_${version}.log 2>&1 + fogvhosttmp="${etcconf}.fogblock.$$" + echo "server {" > "$fogvhosttmp" +``` +Then change every remaining `>> "$etcconf"` line in this branch (both the +plain-HTTP and the HTTPS/redirect sub-branches -- everything between this +point and the branch's closing `;;`) to `>> "$fogvhosttmp"`. This is a +mechanical find-and-replace within the branch's existing lines; do not change +their *content*, only the redirect target. At the very end of the branch +(after its last `echo "}" >> "$fogvhosttmp"` or equivalent), add: +```bash + spliceManagedBlock "$etcconf" "$fogvhosttmp" + rm -f "$fogvhosttmp" + diffconfig "${etcconf}" +``` +(`diffconfig`'s existing call site, if any, at the end of this branch should +be removed if duplicated -- keep exactly one call.) + +- [ ] **Step 4: Repeat for the Apache branch** + +Same mechanical change in the `httpd|apache*)` branch (`functions.sh:3751` +ff.): redirect every `>> "$etcconf"` (and the branch's initial `>` if any) to +a `$fogvhosttmp` temp file, then call `spliceManagedBlock "$etcconf" +"$fogvhosttmp"` once at the end of the branch, followed by `diffconfig`. +This branch has three `ServerAlias` write sites in sequence (all within the +same `$etcconf`/temp file) -- all three simply redirect to the same +`$fogvhosttmp`, since they all belong in the same single managed block. + +- [ ] **Step 5: Syntax-check** + +Run: `bash -n lib/common/functions.sh`. Expected: no output, exit 0. + +- [ ] **Step 6: Manually verify — fresh file (test Linux box)** + +Remove any existing vhost file, run `./installfog.sh -Y`. Confirm the file +now contains `$FOG_MANAGED_BEGIN`/`$FOG_MANAGED_END` markers wrapping exactly +the same content it would have contained before this change (diff against a +pre-change run's output, ignoring the two marker lines). + +- [ ] **Step 7: Manually verify — preserves admin additions** + +Hand-append a distinctive block after the managed block's `END` marker (e.g. +a comment + a harmless extra `Alias`/`location` directive). Run +`./installfog.sh -Y --extra-server-name fog-test2.internal`. Confirm: +- The hand-appended block is still present, unchanged, after the `END` marker. +- The managed block's `server_name`/`ServerName`/`ServerAlias` now includes + `fog-test2.internal`. + +- [ ] **Step 8: Manually verify — no-markers-yet file gets appended to, not + replaced** + +Simulate an upgrade from before this feature: hand-write a vhost file with no +markers at all (arbitrary content). Run `./installfog.sh -Y`. Confirm the +original content is still present, with a new managed block appended after it +containing FOG's usual content. + +- [ ] **Step 9: Verify `--overwrite-vhost` and `-F`/`--no-vhost` still behave + as documented** + +`--overwrite-vhost` (via `updatefog.sh`, or by removing the vhost file +manually and re-running `installfog.sh`) discards everything and writes a +fresh single-block file. `-F`/`--no-vhost` leaves the file 100% untouched, +including not adding markers. + +- [ ] **Step 10: Flip `updatefog.sh`'s default `$updateVhostFlag`** + +In `bin/updatefog.sh`, change: +```bash +updateVhostFlag="-F" +``` +to: +```bash +# Splicing the FOG-managed block (see spliceManagedBlock, functions.sh) is +# always safe now -- it only ever touches FOG's own marked region, never an +# admin's own additions outside it. -F remains available for an admin who +# wants installfog.sh to touch the vhost file not at all. +updateVhostFlag="" +``` +Update the comment above the old assignment (currently explaining why `-F` +was the default) to reflect this, and update `usage()`'s `--overwrite-vhost` +help text if it references the old default. + +- [ ] **Step 11: Syntax-check** + +Run: `bash -n bin/updatefog.sh`. Expected: no output, exit 0. + +- [ ] **Step 12: Manually verify `updatefog.sh`'s new default end to end** + +On a test box with a hand-customized vhost (appended content, per Step 7), +run `./updatefog.sh -y`. Confirm the appended content survives and the +managed block still refreshes normally. + +- [ ] **Step 13: Commit** + +```bash +git add lib/common/functions.sh bin/updatefog.sh +git commit -m "Replace whole-file vhost regeneration with a FOG-managed block" +``` + +--- + +### Task 4: Versioned kernel/init backup + `bin/restorekernel.sh` + +**Files:** +- Modify: `lib/common/functions.sh` — extend `backupPreservedCustomizations()`/ + `restorePreservedCustomizations()` (Task 1) with generation rotation; add + `kernelBackupGenerations` to `writeUpdateFile()`'s `managedKeys` + (`functions.sh:3122-3171`). +- Modify: `bin/installfog.sh` — new `--kernel-backup-count` flag, + `--restore-kernel-backup` flag. +- Create: `bin/restorekernel.sh`. + +**Interfaces:** +- Produces: `${fogprogramdir}/customizations/kernel-backups/gen-1..N/`. +- Consumes: `$kernelBackupGenerations` (default 3). + +- [ ] **Step 1: Add generation rotation to `backupPreservedCustomizations()`** + +In `lib/common/functions.sh`, extend the function added in Task 1 by +appending, before its final `errorStat $st`: + +```bash + [[ -z $kernelBackupGenerations || $kernelBackupGenerations -lt 1 ]] && kernelBackupGenerations=3 + local kbdir="${custdir}/kernel-backups" + mkdir -p "$kbdir" >>$error_log 2>&1 + [[ -d "${kbdir}/gen-${kernelBackupGenerations}" ]] && rm -rf "${kbdir}/gen-${kernelBackupGenerations}" + local k + for ((k = kernelBackupGenerations - 1; k >= 1; k--)); do + [[ -d "${kbdir}/gen-${k}" ]] && mv "${kbdir}/gen-${k}" "${kbdir}/gen-$((k + 1))" + done + mkdir -p "${kbdir}/gen-1" >>$error_log 2>&1 + [[ -d "$ipxedir" ]] && cp -a "${ipxedir}/." "${kbdir}/gen-1/" >>$error_log 2>&1 || st=1 +``` + +- [ ] **Step 2: Add custom-named-file + revert-carve-out restore to + `restorePreservedCustomizations()`** + +Extend the function from Task 1, appending before its final `errorStat $st`: + +```bash + local kbdir="${custdir}/kernel-backups" + local defaultnames="bzImage bzImage32 arm_Image init.xz init_32.xz arm_init.cpio.gz" + if [[ -d "${kbdir}/gen-1" ]]; then + for f in "${kbdir}/gen-1"/*; do + [[ -f $f ]] || continue + local bn=$(basename "$f") + local isdefault=0 + for d in $defaultnames; do [[ $bn == $d ]] && isdefault=1; done + # Custom-named files (e.g. a per-host kernel/init override, see + # packages/web/lib/fog/bootmenu.class.php Host->get('kernel')) + # are restored unconditionally -- FOG never re-downloads these, + # so nothing else will put them back. + if [[ $isdefault -eq 0 ]]; then + cp -f "$f" "${ipxedir}/${bn}" >>$error_log 2>&1 || st=1 + elif [[ $restoreKernelBackup -eq 1 ]]; then + # --restore-kernel-backup (revertUpdate's re-invocation only): + # deliberately ALSO restores the default names, matching the + # previous _restorePreviousKernel()'s revert-only behavior. + cp -f "$f" "${ipxedir}/${bn}" >>$error_log 2>&1 || st=1 + fi + done + fi +``` + +- [ ] **Step 3: Syntax-check** + +Run: `bash -n lib/common/functions.sh`. Expected: no output, exit 0. + +- [ ] **Step 4: Add `kernelBackupGenerations` to `managedKeys`** + +In `writeUpdateFile()` (`functions.sh:3122-3171`), add `kernelBackupGenerations` +alongside `extraServerNames` (line 3164): +```bash + extraServerNames + # How many prior kernel/init generations to keep under + # customizations/kernel-backups/ (see --kernel-backup-count). + # Persisted so an admin's chosen retention survives future upgrades. + kernelBackupGenerations +``` + +- [ ] **Step 5: Add `--kernel-backup-count` and `--restore-kernel-backup` to + `installfog.sh`** + +Add `kernel-backup-count:,restore-kernel-backup` to `longopts` +(`bin/installfog.sh:173`). Add case branches modeled on `--fogprogramdir`: +```bash + --kernel-backup-count) + if [[ -n "${2}" && "${2}" =~ ^[0-9]+$ && "${2}" -ge 1 ]]; then + skernelBackupCount="${2}" + else + echo "Error: --kernel-backup-count requires a positive integer" + usage + exit 9 + fi + shift 2 + ;; + --restore-kernel-backup) + restoreKernelBackup=1 + shift + ;; +``` +Apply the staging var in the option-evaluation block (`bin/installfog.sh:615-681`): +```bash +[[ -n $skernelBackupCount ]] && kernelBackupGenerations=$skernelBackupCount +[[ -z $restoreKernelBackup ]] && restoreKernelBackup=0 +``` +Add both to `usage()`'s help text, near `--secure-boot-*`. + +- [ ] **Step 6: Syntax-check** + +Run: `bash -n bin/installfog.sh`. Expected: no output, exit 0. + +- [ ] **Step 7: Write `bin/restorekernel.sh`** + +Model directly on `bin/setupacme.sh`'s structure -- **note:** as of this plan +`bin/setupacme.sh` no longer exists (see +`docs/superpowers/plans/2026-08-07-three-zone-pki-separation.md`, which +removes it). If that plan has already landed when this task is implemented, +model this script's root-check/`.fogsettings`-sourcing/`error_logs/` setup +boilerplate on `bin/updatefog.sh` instead -- the shape is the same either way. + +```bash +#!/bin/bash +# ...license header, matching bin/updatefog.sh... +# +# Restores a prior kernel/init generation backed up by +# backupPreservedCustomizations() (lib/common/functions.sh) under +# $fogprogramdir/customizations/kernel-backups/. See +# docs/SUPPORTED_CUSTOMIZATIONS.md and +# docs/superpowers/specs/2026-08-07-customization-preservation-design.md. +bindir=$(dirname $(readlink -f "$BASH_SOURCE")) +cd $bindir +workingdir=$(pwd) + +if [[ ! $EUID -eq 0 ]]; then + echo "restorekernel.sh must be run as root user" + exit 1 +fi + +usage() { + echo -e "Usage: $0 [-h?] (--list | --generation )" + echo -e "\t-h -? --help\t\tDisplay this info" + echo -e "\t --list\t\tList available backup generations and their FOS release tag" + echo -e "\t --generation\tRestore generation N (1 = most recent) into service/ipxe/" + exit 0 +} + +shortopts="h?" +longopts="help,list,generation:" +optargs=$(getopt -o $shortopts -l $longopts -n "$0" -- "$@") +[[ $? -ne 0 ]] && usage +eval set -- "$optargs" + +doList=0 +generation="" +while :; do + case $1 in + -h | -\? | --help) usage ;; + --list) doList=1; shift ;; + --generation) generation="$2"; shift 2 ;; + --) shift; break ;; + *) echo "Error: unhandled option '$1'."; exit 10 ;; + esac +done + +exitFail=1 +. ../lib/common/functions.sh + +[[ -z $fogprogramdir && -r /etc/fog/fog.conf ]] && . /etc/fog/fog.conf +[[ -z $fogprogramdir ]] && fogprogramdir="/opt/fog" +fogprogramdir="${fogprogramdir%/}" + +if [[ ! -r "$fogprogramdir/.fogsettings" ]]; then + echo " * No existing FOG install found at $fogprogramdir (.fogsettings missing)." + exit 1 +fi +. "$fogprogramdir/.fogsettings" + +kbdir="${fogprogramdir}/customizations/kernel-backups" + +if [[ $doList -eq 1 ]]; then + for gendir in "$kbdir"/gen-*; do + [[ -d $gendir ]] || continue + echo "$(basename "$gendir"):" + for f in "$gendir"/bzImage "$gendir"/init.xz; do + [[ -f $f ]] || continue + tag=$(attr -g tag_name "$f" 2>/dev/null) + echo " $(basename "$f") (${tag:-unknown release})" + done + done + exit 0 +fi + +if [[ -z $generation ]]; then + echo " * Pass --list or --generation ." + usage +fi + +gendir="${kbdir}/gen-${generation}" +if [[ ! -d $gendir ]]; then + echo " * No such generation: $gendir" + exit 1 +fi + +ipxedir="${webdirdest}service/ipxe" +echo " * Restoring generation $generation into $ipxedir" +cp -af "${gendir}/." "$ipxedir/" && chown -R ${username}:${apacheuser} "$ipxedir" + +if [[ -n $secureBootKey && -n $secureBootCert ]]; then + echo " * Re-checking Secure Boot signatures on restored kernels" + _resignKernels +fi +echo " * Done." +``` + +- [ ] **Step 8: Make executable, syntax-check** + +Run: `chmod +x bin/restorekernel.sh && bash -n bin/restorekernel.sh`. +Expected: no output, exit 0. + +- [ ] **Step 9: Manually verify (test Linux box)** + +Run `installfog.sh -Y` three times in a row (simulating three updates). +Confirm `bin/restorekernel.sh --list` shows three generations with distinct +`tag_name` values. Run `bin/restorekernel.sh --generation 2`; confirm +`service/ipxe/bzImage`'s `tag_name` xattr now matches generation 2's, and +(if Secure Boot is configured) it still verifies against the current +signing cert. + +- [ ] **Step 10: Wire `--restore-kernel-backup` into `updatefog.sh`'s + `revertUpdate()`** + +In `lib/common/update.sh`, change `revertUpdate()`'s re-invocation (currently +line 116): +```bash + (cd "$fog_git_path/bin" && bash installfog.sh -Y $updateVhostFlag >>$error_log 2>&1) +``` +to: +```bash + (cd "$fog_git_path/bin" && bash installfog.sh -Y $updateVhostFlag --restore-kernel-backup >>$error_log 2>&1) +``` +and delete the now-redundant `_restorePreviousKernel` / `restoreCustomizations` +calls immediately below it (superseded — see Task 6, which removes their +definitions entirely; this step just stops calling them here first so Task 6 +doesn't leave a dangling reference mid-task). + +- [ ] **Step 11: Syntax-check** + +Run: `bash -n lib/common/update.sh`. Expected: no output, exit 0. + +- [ ] **Step 12: Commit** + +```bash +git add lib/common/functions.sh bin/installfog.sh bin/restorekernel.sh lib/common/update.sh +git commit -m "Add bounded, versioned kernel/init backup and bin/restorekernel.sh" +``` + +--- + +### Task 5: Custom PXE script hook (`custom.ipxe` via `default.ipxe`) + +**Files:** +- Modify: `lib/common/functions.sh` — `configureDefaultiPXEfile()` + (`functions.sh:1037-1042`). + +**Interfaces:** +- Consumes: nothing new. +- Produces: nothing consumed elsewhere — purely additive content in a + generated file. + +- [ ] **Step 1: Add the hook line to `configureDefaultiPXEfile()`** + +Change `functions.sh:1040` from: +```bash + echo -e "#!ipxe\nset arch \${buildarch}\niseq \${arch} i386 && cpuid --ext 29 && set arch x86_64 ||\nparams\nparam mac0 \${net0/mac}\nparam arch \${arch}\nparam platform \${platform}\nparam product \${product}\nparam manufacturer \${product}\nparam ipxever \${version}\nparam filename \${filename}\nparam sysuuid \${uuid}\nisset \${net1/mac} && param mac1 \${net1/mac} || goto bootme\nisset \${net2/mac} && param mac2 \${net2/mac} || goto bootme\n:bootme\nchain ${httpproto}://$ipaddress${webroot}service/ipxe/boot.php##params" > "$tftpdirdst/default.ipxe" +``` +to: +```bash + # chain custom.ipxe first if an admin has placed one at the TFTP root -- + # see docs/SUPPORTED_CUSTOMIZATIONS.md. Per ipxe.org/cmd/chain, chain + # WITHOUT --replace returns control to the next line once the chained + # script finishes normally, so a present-and-successful custom.ipxe falls + # straight through into :fog_default afterward with no special "resume" + # convention needed. A missing/failed chain hits the || immediately, so + # default boot behavior is byte-for-byte unchanged when no custom.ipxe + # exists. + echo -e "#!ipxe\nchain custom.ipxe || goto fog_default\n:fog_default\nset arch \${buildarch}\niseq \${arch} i386 && cpuid --ext 29 && set arch x86_64 ||\nparams\nparam mac0 \${net0/mac}\nparam arch \${arch}\nparam platform \${platform}\nparam product \${product}\nparam manufacturer \${product}\nparam ipxever \${version}\nparam filename \${filename}\nparam sysuuid \${uuid}\nisset \${net1/mac} && param mac1 \${net1/mac} || goto bootme\nisset \${net2/mac} && param mac2 \${net2/mac} || goto bootme\n:bootme\nchain ${httpproto}://$ipaddress${webroot}service/ipxe/boot.php##params" > "$tftpdirdst/default.ipxe" +``` + +- [ ] **Step 2: Syntax-check** + +Run: `bash -n lib/common/functions.sh`. Expected: no output, exit 0. + +- [ ] **Step 3: Manually verify — absent (default, test Linux box or a PXE + test VM)** + +Run `installfog.sh -Y`. Confirm `$tftpdirdst/default.ipxe` now begins with +the `chain custom.ipxe || goto fog_default` / `:fog_default` lines. PXE-boot +a test client with no `custom.ipxe` present at the TFTP root; confirm boot +proceeds to FOG's menu/imaging exactly as before this change. + +- [ ] **Step 4: Manually verify — present** + +Place a minimal script at the TFTP root, e.g.: +``` +#!ipxe +echo Custom hook running, pausing 10s... +sleep 10 +``` +(`$tftpdirdst/custom.ipxe`). PXE-boot the same test client; confirm the +message and delay appear, then normal FOG boot proceeds immediately +afterward with no further action needed in the custom script. + +- [ ] **Step 5: Commit** + +```bash +git add lib/common/functions.sh +git commit -m "Add optional custom.ipxe hook point ahead of FOG's default PXE boot logic" +``` + +--- + +### Task 6: Retire `lib/common/update.sh`'s superseded functions; shrink `bin/updatefog.sh` + +**Files:** +- Modify: `lib/common/update.sh` — remove `_updateAssetFiles()`, + `backupCustomizations()`, `restoreCustomizations()`, + `_restorePreviousKernel()`. +- Modify: `bin/updatefog.sh` — remove the now-dead `backupCustomizations`/ + `restoreCustomizations` call sites. + +**Interfaces:** +- Consumes: nothing (this task only removes code Tasks 1-4 made redundant). + +- [ ] **Step 1: Delete the superseded functions** + +In `lib/common/update.sh`, delete `_updateAssetFiles()`, +`backupCustomizations()`, `restoreCustomizations()`, and +`_restorePreviousKernel()` (lines 28-71 in the pre-this-plan file — verify +against the current file state after Task 4's Step 10 edit, which already +removed their call sites from `revertUpdate()`). Update the file's top +comment (lines 19-27) to describe its now-narrower scope (git +fetch/checkout/revert only). + +- [ ] **Step 2: Remove the dead calls from `bin/updatefog.sh`** + +Remove the `backupCustomizations` call (currently line 249, immediately +before `gitUpdateToBranch`) and the `restoreCustomizations` call (currently +line 264, in the success branch) — both are now handled unconditionally +inside every `installfog.sh` invocation via Tasks 1/4. + +- [ ] **Step 3: Syntax-check** + +Run: `bash -n lib/common/update.sh && bash -n bin/updatefog.sh`. Expected: no +output, exit 0 for both. + +- [ ] **Step 4: Manually verify end to end (test Linux box)** + +Run a full `updatefog.sh -y` update with a customized `bg.png`-equivalent and +a hand-appended vhost addition in place. Confirm both survive (this is now +exercised entirely through `installfog.sh`'s own logic, with `updatefog.sh` +doing nothing but the git sync and invocation). + +Force a failure (e.g. temporarily break connectivity mid-update) and confirm +`revertUpdate()` still correctly reverts and restores, now via +`--restore-kernel-backup` rather than the deleted `_restorePreviousKernel`. + +- [ ] **Step 5: Commit** + +```bash +git add lib/common/update.sh bin/updatefog.sh +git commit -m "Remove update.sh's superseded backup/restore -- now handled unconditionally inside installfog.sh" +``` + +--- + +### Task 7: `docs/SUPPORTED_CUSTOMIZATIONS.md` + +**Files:** +- Create: `docs/SUPPORTED_CUSTOMIZATIONS.md`. +- Modify: `bin/installfog.sh`'s `usage()` and `bin/updatefog.sh`'s `usage()` + — point admins at the new doc from the relevant flags' help text + (`--kernel-backup-count`, `--restore-kernel-backup`, `--overwrite-vhost`). + +**Interfaces:** +- Consumes: nothing — documentation only. + +- [ ] **Step 1: Write `docs/SUPPORTED_CUSTOMIZATIONS.md`** + +Follow the outline in +`docs/superpowers/specs/2026-08-07-customization-preservation-design.md`'s +"Architecture §6" section: one `##` heading per customization category +(iPXE background, vhost, kernel/init, custom PXE scripts, Secure Boot certs), +each with a short paragraph and a small table of +`| Customization | How it's preserved | Where |`, plus a closing "What is NOT +automatically preserved" section covering in-block vhost hand edits and +signing-key rotation across a kernel restore. + +- [ ] **Step 2: Cross-link from flag help text** + +Add a `See docs/SUPPORTED_CUSTOMIZATIONS.md` line to `usage()` in both +`bin/installfog.sh` and `bin/updatefog.sh`, near `--kernel-backup-count`, +`--restore-kernel-backup`, `--overwrite-vhost`, and `-F`/`--no-vhost`. + +- [ ] **Step 3: Commit** + +```bash +git add docs/SUPPORTED_CUSTOMIZATIONS.md bin/installfog.sh bin/updatefog.sh +git commit -m "Document supported install-time customizations" +``` + +--- + +### Critical Files for Implementation + +- `lib/common/functions.sh` — hosts nearly every new/changed function: `backupPreservedCustomizations()`, `restorePreservedCustomizations()`, `preserveSecureBootAdminFiles()`, `spliceManagedBlock()`, the modified `createSSLCA()` write sites, `configureDefaultiPXEfile()`'s hook line, and `writeUpdateFile()`'s `managedKeys`. +- `bin/installfog.sh` — new flags (`--kernel-backup-count`, `--restore-kernel-backup`), the sequence wiring at the master-install call chain (~lines 938-963), and the secure-boot option-evaluation call site (~line 681). +- `lib/common/update.sh` — where the superseded `_updateAssetFiles()`/`backupCustomizations()`/`restoreCustomizations()`/`_restorePreviousKernel()` are removed and `revertUpdate()` is trimmed. +- `bin/updatefog.sh` — `$updateVhostFlag`'s default flip and removal of the now-redundant backup/restore call sites. +- `packages/web/commons/schema.php` and `packages/web/lib/fog/bootmenu.class.php` — read-only references confirming `FOG_IPXE_BG_FILE`'s real semantics and the per-host `Host->get('kernel')`/`get('init')` custom-name feature; no changes needed here, but any implementer should re-read them before touching Task 1/4. diff --git a/docs/superpowers/plans/2026-08-07-three-zone-pki-separation.md b/docs/superpowers/plans/2026-08-07-three-zone-pki-separation.md new file mode 100644 index 0000000000..41c811368b --- /dev/null +++ b/docs/superpowers/plans/2026-08-07-three-zone-pki-separation.md @@ -0,0 +1,1166 @@ +# Three-zone PKI separation (Web TLS / Client Communication / Secure Boot) Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Replace today's flat certificate setup — one self-signed CA that +both signs the web vhost leaf and is pinned by fog-client, plus a separate +self-signed Secure Boot leaf that is *itself* the enrolled MOK — with a +Root CA issuing **three** independent intermediates (Web, Client +Communication, Secure Boot), each independently replaceable by an admin's own +PKI. Secure Boot's intermediate is what firmware enrolls, so code-signing +leaves can be rotated, revoked, or issued per storage node **without a +firmware re-enrollment trip to every machine**. Additionally: split +`$netbootproto` from `$httpproto` so a private-CA install can serve a trusted +HTTPS web UI while keeping iPXE netboot on HTTP (avoiding the iPXE rebuild +that forfeits the signed Secure Boot shim), and remove `bin/setupacme.sh` — +ACME/Let's Encrypt automation is not a FOG-managed feature going forward. + +The split PKI is the **default** for fresh installs; today's flat setup +remains available as a permanent, explicit `--legacy-pki` option. Existing +installs are never silently switched, and — critically — an already-enrolled +Secure Boot MOK is **never** regenerated or replaced by this work. + +**Architecture:** See `docs/superpowers/specs/2026-08-07-three-zone-pki-separation-design.md`. + +**Phasing at a glance:** +- **Phase 0** (do first — two independent verifications, each gating one + Phase 1 task): **0.1** verifies the fog-client pinning mechanism against + `zazzles` source (gates Task 1.4); **0.2** verifies shim accepts a + CA-in-MokList with an `--addcert` chain, on real hardware (gates Task + 1.8). Neither is a code change; both are cheap relative to what they + de-risk. 0.1 moved earlier than originally scoped because `split` is now + the *default* for fresh installs — every fresh install would otherwise + create a Client Communication intermediate under an unverified assumption. + 0.2 is a hard gate on the Secure Boot zone specifically: a negative answer + costs that one zone (it stays flat), not the design. +- **Phase 1** (Task 1.4 blocked on 0.1, Task 1.8 blocked on 0.2; the rest + have no external dependency): the new PKI functions across all three + zones, `pkiMode=split` as the **default** on a fresh install (no flag or + prompt answer needed), `--legacy-pki` as the permanent, fully supported + opt-out reproducing today's flat behavior byte-for-byte, the + `$netbootproto` split, existing servers provably unchanged, and removal of + `bin/setupacme.sh`. +- **Phase 2** (blocked on Phase 0's answer, same as before): the + existing-server migration path — dual-trust-window rollout, snapin-based + client re-pinning, cutover. +- **Phase 3** (no hard blocker, but low value until Phase 1/2 have real + users): root-key offlining helper, `--external-ca` flag + deprecation-timeline decision, docs consolidation. + +No single PR boundary in Phase 1 leaves an existing server in a +half-migrated state: `pkiMode`'s default is computed from `caCreated` (Task +1.1), so a server with cert material predating this feature always resolves +to `flat` — its existing PKI is never silently restructured, regardless of +what a fresh install now defaults to. Only a genuinely fresh install (no +prior CA) resolves to the new `split` default. + +## Global Constraints + +- No CI/test framework exists for this repo's shell scripts beyond + `fogproject-install-validation`'s distro matrix. Every task's "test" step is + a manual invocation + assertion, not a unit-test suite. Run `bash -n + ` after every shell edit before anything else. +- Follow the existing staging-variable convention exactly: a new flag sets an + `s`-prefixed variable during `getopt` parsing, applied to the real variable + only *after* `.fogsettings` is sourced, in the "# evaluation of command + line options" block (`bin/installfog.sh:615-656` currently) — never before. +- `bin/updatefog.sh` never runs `installfog.sh` interactively (always `-Y`); + any new flag added there must be passed straight through to the child + invocation, same as `$updateVhostFlag`/`--hostname`/`--extra-server-name` + already are. +- Every new value that reaches a file write (vhost config, OpenSSL config + file, subject string) must be validated first — this repo already treats + that as a real security boundary (`--git-path`'s absolute-path check, + `validhostname()`). +- `writeUpdateFile()`'s `managedKeys` array (`lib/common/functions.sh:3122` + ff.) is the single source of truth for what persists across an update — + every new variable this plan introduces that needs to survive an upgrade + must be added there, in the same task that introduces it, not as an + afterthought. +- This plan assumes the `1013`-branch state already merged + (`--hostname`/`--extra-server-name` already present in `createSSLCA()`) — + do not reintroduce or duplicate any of that. It also assumes + `docs/superpowers/plans/2026-08-07-customization-preservation.md`'s Task 3 + (the FOG-managed vhost block / `spliceManagedBlock()`) either has already + landed or is understood to land before/alongside this plan's Task 1.3 — + that task's `createSSLCA()` extraction should be built against the + already-spliced vhost-writing code, not duplicate work against it. +- Design doc: `docs/superpowers/specs/2026-08-07-three-zone-pki-separation-design.md`. + Read it if anything below is ambiguous. + +--- + +## Phase 0: Verify the fog-client pinning mechanism (blocks Phase 2 only) + +### Task 0.1: Get a falsifiable answer from `zazzles`/fog-client source or the maintainer + +**Files:** None in this repo — this is a research task against an external +repository/person, tracked here so **both Phase 1's Task 1.4 and Phase 2** +have a documented go/no-go gate. (Originally scoped to gate only Phase 2; +moved earlier because `split` is now the default for fresh installs, so +Task 1.4's comm-certificate delivery path is no longer a low-stakes +"opt-in only" decision — see the design doc's Open Risks #8.) + +- [ ] **Step 1:** Locate the fog-client (`zazzles`) source's TLS/cert-pinning + code (likely in a `Communication`/`Certificate` class per the `#!ihc` + handling referenced in `fogpage.class.php:2870`'s comment — `Zazzles' + Communication.Post() calls HttpWebRequest::GetResponse()`). +- [ ] **Step 2:** Answer, in writing, and attach to the tracking issue: + - Does the client compare the downloaded `ca.cert.der`'s **bytes**, its + **CN string**, or both, against a stored/expected value? + - Does it re-download and re-validate `ca.cert.der` (and whatever it uses + as the comm-encryption public key) **before every `authorize()` attempt**, + or only at registration/install time? + - Does it use `ca.cert.der`'s own key material directly for + `certEncrypt`/`certDecrypt`-style payload crypto, or does it separately + fetch/expect a distinct leaf cert for that purpose? +- [ ] **Step 3:** Update + `docs/superpowers/specs/2026-08-07-three-zone-pki-separation-design.md`'s + Open Risks #1-#3 with the confirmed answer, and note which of Phase 2's + tasks below change as a result (most likely: whether Task 2.2's snapin step + is still necessary at all, or whether the migration collapses to "just + rotate and wait one checkin cycle"). +- [ ] **Step 4: Commit** (docs-only) + ```bash + git add docs/superpowers/specs/2026-08-07-three-zone-pki-separation-design.md + git commit -m "Record verified fog-client pinning behavior (Phase 0 finding)" + ``` + +### Task 0.2: Verify shim accepts a CA-in-MokList with an `--addcert` chain + +**Files:** None — hardware verification, gating Task 1.8 (the Secure Boot +zone) specifically. **This is a hard gate:** the entire "rotate signing +leaves without re-enrolling firmware" premise depends on the answer. A +negative result does not sink the design — the Secure Boot zone simply stays +on today's self-signed-leaf model while Web and Client still split — but it +must be answered before Task 1.8 is written, not after. + +- [ ] **Step 1:** Build a throwaway two-level chain by hand on a test box: + ```bash + # intermediate (would be MOK.der) + openssl req -x509 -new -nodes -newkey rsa:2048 -sha256 -days 3650 \ + -subj "/CN=Test SB CA/" -addext "basicConstraints=critical,CA:TRUE" \ + -keyout sbca.key -out sbca.pem + # code-signing leaf issued by it + openssl req -new -nodes -newkey rsa:2048 -sha256 -subj "/CN=Test SB Signer/" \ + -keyout leaf.key -out leaf.csr + openssl x509 -req -in leaf.csr -CA sbca.pem -CAkey sbca.key -CAcreateserial \ + -days 365 -sha256 -out leaf.pem \ + -extfile <(printf 'basicConstraints=critical,CA:FALSE\nextendedKeyUsage=codeSigning\n') + ``` +- [ ] **Step 2:** Sign a real FOS kernel with the leaf, bundling the + intermediate — this `--addcert` flag is the whole mechanism under test: + ```bash + sbsign --key leaf.key --cert leaf.pem --addcert sbca.pem \ + --output bzImage.signed bzImage + sbverify --list bzImage.signed # confirm BOTH certs are present + ``` +- [ ] **Step 3:** Enroll **only** `sbca.pem` (converted to DER) as a MOK via + `mokutil --import` on a real UEFI machine with Secure Boot **enabled**, + reboot through MokManager, and attempt to boot `bzImage.signed` through + the shim FOG ships (`downloadipxesecureboot()`'s staged binaries). +- [ ] **Step 4:** Record the result in the design doc's Open Risks #4 and + Testing sections. If it boots: Task 1.8 proceeds as written. If it does + not: mark the Secure Boot zone as staying flat, and note which shim + version/firmware was tested — do not silently downgrade the plan without + recording what was actually observed. +- [ ] **Step 5:** If possible, repeat on arm64. Record if untested rather + than assuming parity with x86_64. +- [ ] **Step 6: Commit** (docs-only) + ```bash + git add docs/superpowers/specs/2026-08-07-three-zone-pki-separation-design.md + git commit -m "Record shim CA-in-MokList chain verification result (Phase 0 finding)" + ``` + +--- + +## Phase 1: New PKI functions, opt-in `pkiMode=split`, fresh installs only + +### Task 0.3: Confirm where fog-client fetches the server's encryption certificate (gates Task 1.4) + +**Files:** None — inspection of a **live, working** FOG server plus a +`zazzles` source read. Determines whether decoupling the comm certificate +from the web vhost needs any fog-client change at all. + +**Already settled, do not re-litigate:** `.srvprivate.key` exists (every +file in `$sslpath` is a dotfile, so a bare `ls` makes the directory look +empty — `ls -la` shows it), it is the **web leaf's** key, and it is what +`certDecrypt()` uses on every client handshake. The coupling is confirmed +real and the fix is settled: the FOG Server CA issues its own communication +TLS certificate, never shared with the vhost. + +What remains is purely a delivery question: fog-client must obtain the +public half of whatever key the server decrypts with. If it fetches +`management/other/ssl/srvpublic.crt`, then publishing the comm leaf at that +same path is a **server-side-only** change and no client work is needed. + +- [ ] **Step 1:** Establish the baseline on a live server — confirm which + certificate `.srvprivate.key` currently backs (expected: `srvpublic.crt`, + *not* `ca.cert.pem`; equal modulus hashes mean equal keypair): + ```bash + openssl rsa -noout -modulus -in /opt/fog/snapins/ssl/.srvprivate.key | md5sum + openssl x509 -noout -modulus -in /var/www/html/fog/management/other/ssl/srvpublic.crt | md5sum + openssl x509 -noout -modulus -in /var/www/html/fog/management/other/ca.cert.pem | md5sum + ``` + If `.srvprivate.key` does **not** match `srvpublic.crt`, stop — check + `grep sslprivkey /opt/fog/.fogsettings`, since the path was overridden and + the rest of this task's assumptions need re-checking against that path. +- [ ] **Step 2:** Find, in `zazzles`/fog-client source, the code that + obtains the server's public key before encrypting `sym_key`/`token` + (likely near the `RSA`/`Authentication` handling that produces `#!ihc`). + Answer: which URL/path does it request, and does it use that certificate's + public key directly, or derive one from `ca.cert.der`? +- [ ] **Step 3:** Cross-check against the server's access log — the client's + own requests are the ground truth, and this needs no source access: + ```bash + grep -E 'ca\.cert\.(der|pem)|srvpublic\.crt' /var/log/httpd/*access* /var/log/nginx/*access* 2>/dev/null | tail -40 + ``` + A client that fetches `srvpublic.crt` at handshake time confirms the + server-side-only path. +- [ ] **Step 4:** Sanity-check the storage node's configured SSL path, since + that — not `$sslpath` — is what `certDecrypt()` actually resolves + (`fogbase.class.php:2019-2023`), and Task 1.4 must write the comm key + where that column points: + ```bash + mysql fog -e "SELECT ngmID, ngmSSLPath FROM nfsGroupMembers;" + ``` +- [ ] **Step 5:** Record the answer in the design doc's Open Risks and note + which Task 1.4 path applies: comm leaf published at the existing path + (no client change), or Client CA doubling as the comm keypair (fallback, + also no client change), or a genuine client-side change required (the only + outcome that blocks on the `zazzles` repo). +- [ ] **Step 6: Commit** the finding (docs-only) + ```bash + git add docs/superpowers/specs/2026-08-07-three-zone-pki-separation-design.md + git commit -m "Record how fog-client obtains the server encryption cert (Phase 0 finding)" + ``` + +--- + +### Task 1.1: `pkiMode` managed key + zone-aware directory scaffolding + +**Files:** +- Modify: `lib/common/functions.sh` — `writeUpdateFile()`'s `managedKeys` + array (`functions.sh:3122` ff.): add `pkiMode fogClientCACN`. +- Modify: `lib/common/functions.sh` — add a small `_pkiZoneDir(zone)` helper + (`root`|`web`|`client`|`client/comm` → `$sslpath/CA/`), used by every + function in Tasks 1.2-1.4 instead of each one string-concatenating + `$sslpath/CA/...` independently — this is what keeps the CN-assumption + blast radius to one place, per the design doc's Open Risk #1: if the target + directory shape ever needs to change, it changes in one function. + +**Interfaces:** +- Produces: `$pkiMode` (`split` default on a fresh install, `flat` default on + a server with pre-existing cert material, either overridable by a flag — + see Step 3 below), `$fogClientCACN` + (default `"FOG Server CA"`, overridable — see Task 1.5's `--client-ca-cn` + escape hatch for when Phase 0 finds the real requirement is a different + string). +- Consumes: nothing new. + +- [ ] **Step 1:** Add the two keys to `managedKeys` with a comment explaining + why (mirrors the `secureBootKey`/`secureBootCert`/`secureboot` comment + block already there at `functions.sh:3131-3136` — same "opt-in that must + not silently revert on upgrade" reasoning applies to `pkiMode`). +- [ ] **Step 2:** Add `_pkiZoneDir()`: + ```bash + # Single source of truth for the split-PKI directory layout under $sslpath. + # Every split-mode function asks this for a path rather than + # string-concatenating $sslpath/CA/... itself -- see the design doc's Open + # Risk #1 for why this matters: if the Client zone's shape needs to change + # later (e.g. drop the .commLeaf sub-leaf per Phase 0's finding), it changes + # here once, not in every caller. + _pkiZoneDir() { + case "$1" in + root) echo "$sslpath/CA/root" ;; + web) echo "$sslpath/CA/web" ;; + client) echo "$sslpath/CA/client" ;; + client/comm) echo "$sslpath/CA/client/comm" ;; + esac + } + ``` +- [ ] **Step 3:** Default `pkiMode` based on whether this server already has + cert material, not just "is it unset" — a server with `caCreated == yes` + predates this feature and must default to `flat`, never `split`, no + matter how new the installer binary is; a server with no CA yet gets the + new `split` default: + ```bash + if [[ -z $pkiMode ]]; then + if [[ $caCreated == yes ]]; then + pkiMode="flat" + else + pkiMode="split" + fi + fi + [[ -z $fogClientCACN ]] && fogClientCACN="FOG Server CA" + ``` + Place alongside the other such defaults near `createSSLCA()`'s top + (`functions.sh:3418` area) — **after** `.fogsettings` has been sourced (so + an existing `caCreated`/`pkiMode` value is already loaded) and **before** + any `--legacy-pki`/`--restructure-pki` staging-variable override (Task + 1.5) is applied, so an explicit flag always wins over either default. +- [ ] **Step 4:** `bash -n lib/common/functions.sh` — expect clean. +- [ ] **Step 5: Commit** + ```bash + git add lib/common/functions.sh + git commit -m "Add pkiMode/fogClientCACN scaffolding for opt-in split PKI" + ``` + +### Task 1.2: `createRootCA()` + +**Files:** Modify `lib/common/functions.sh` (new function, placed +immediately before `createSSLCA()`). + +**Interfaces:** +- Produces: `$rootCAKey`/`$rootCAPem` on success; a `--root-ca-key/--root-ca-cert` + admin-supplied pair (Task 1.5) skips generation entirely, mirroring + `_ensureSecureBootKeys()`'s "admin-supplied pair always wins" pattern. +- Consumes: `_pkiZoneDir root`. + +- [ ] **Step 1:** Write `createRootCA()` modeled directly on + `createSSLCA()`'s existing self-signed branch (`functions.sh:3429-3442`), + changed to: `CN=FOG Server ROOT CA`, `-days 7300`, and an extfile passing + `basicConstraints=critical,CA:TRUE,pathlen:1` (today's flat CA passes no + extfile at all for its own self-signed cert — the intermediates need + `pathlen:1` so nothing can chain a further CA underneath them, mirroring + the shape a real offline-root setup expects). Guard with `[[ ! -f + "$(_pkiZoneDir root)/.fogRootCA.key" ]]` — like the root's key, this + NEVER regenerates once present (same reasoning, borrow the comment, + from `_ensureSecureBootKeys()`'s doc header nearly verbatim: a fresh root + silently invalidates every intermediate signed from the old one). +- [ ] **Step 2:** `bash -n lib/common/functions.sh`. +- [ ] **Step 3:** Manual verify (test VM): call the function directly after + sourcing `functions.sh`, then `openssl x509 -in + $sslpath/CA/root/.fogRootCA.pem -noout -subject -ext basicConstraints` + shows `CN = FOG Server ROOT CA` and `CA:TRUE, pathlen:1`. +- [ ] **Step 4: Commit** + ```bash + git add lib/common/functions.sh + git commit -m "Add createRootCA() for opt-in split PKI" + ``` + +### Task 1.3: `createWebIntermediateCA()` (extraction + repoint, not new logic) + +**Files:** Modify `lib/common/functions.sh`. + +**Interfaces:** +- Produces: everything `createSSLCA()`'s back half already produces + (`$sslprivkey`, `$sslcsr`, `$sslpubcert`, the vhost config) — this task + moves that code, it does not rewrite it. +- Consumes: `createRootCA()`'s output when `pkiMode == split` and no + `--web-ca-*` import was given; otherwise `validateExternalCA web` (Task 1.5). + +**Note:** if `docs/superpowers/plans/2026-08-07-customization-preservation.md`'s +Task 3 (the FOG-managed vhost block splice) has already landed, the vhost +write sites this task's `if [[ $pkiMode != split ]]` wrap around already +write into a temp file and call `spliceManagedBlock` — leave that mechanism +completely alone here; this task only touches the CA-selection lines that +run *before* the CSR/leaf/vhost-writing code, never the vhost-writing code +itself. + +- [ ] **Step 1:** In `createSSLCA()`, wrap the existing CA-selection block + (`functions.sh:3424-3444`: the `if [[ $externalca == yes ]] ... else ... + fi` that sets `$sslcakey`/`$sslcapem`/`$sslcachain`) in `if [[ $pkiMode != + split ]]; then ... existing code, byte-for-byte ... fi`, and add an `else` + branch that calls `createRootCA()` then either `createWebIntermediateCA()` + (self-signed-from-root path) or `validateExternalCA web` (import path) — + both setting the same three variables (`$sslcakey`/`$sslcapem`/`$sslcachain`) + so **everything below this point in `createSSLCA()` — the CSR, the SAN + loop, the leaf signing, the vhost writer — needs zero changes**. This is + the key design property that keeps this task small: the flat/split branch + point is a single `if`, and only the CA-selection few lines differ; the + leaf/vhost machinery is shared, unmodified code either way. +- [ ] **Step 2:** `createWebIntermediateCA()` itself is `_issueIntermediateCA + "FOG Web CA" "$(_pkiZoneDir web)" .fogWebCA.key .fogWebCA.pem`, followed by + writing `.fogWebCAchain.pem` as root+intermediate concatenated (same + concat-into-a-chain-file shape `validateExternalCA()` already uses for + `.fogCAchain.pem`, `functions.sh:3325`). +- [ ] **Step 3:** Write `_issueIntermediateCA(cn, outdir, keyfile, certfile)` + as the shared helper both `createWebIntermediateCA()` and + `createClientIntermediateCA()` (Task 1.4) call — `openssl genrsa` + `openssl + req -new -subj "/CN=$cn/"` + `openssl x509 -req -CA "$rootCAPem" -CAkey + "$rootCAKey" -CAcreateserial -extensions v3_intermediate_ca -extfile + <(printf 'basicConstraints=critical,CA:TRUE\n')`. +- [ ] **Step 4:** `bash -n lib/common/functions.sh`. +- [ ] **Step 5:** Manual verify (test VM): `./installfog.sh -Y` with **no + PKI-related flags at all** on a **fresh** install (no existing + `.fogsettings`) → confirm `split` happens **by default**: `openssl verify + -CAfile $sslpath/CA/root/.fogRootCA.pem -untrusted + $sslpath/CA/web/.fogWebCA.pem $sslpubcert` succeeds, and the vhost/site + loads over HTTPS exactly as a `flat`-mode install would. +- [ ] **Step 6:** Legacy-opt-out check: `./installfog.sh -Y --legacy-pki` on + a **separate fresh** install → confirm `$sslpath/CA/root|web|client` **do + not exist at all**, and `openssl x509 -in $sslpath/CA/.fogCA.pem -noout + -subject` shows the unchanged flat CN, matching a pre-this-patch install + byte-for-byte. +- [ ] **Step 7:** Existing-server regression check (the highest-value test + in this whole plan, per the design doc's Testing section): against a + server with a **real prior** `.fogsettings` (`caCreated == yes`, no + `pkiMode` key — simulating an install that predates this feature), run + `./installfog.sh -Y` (update path, no flags) → confirm it resolves to + `flat` and behaves identically to before this patch — `$sslpath/CA/root|web|client` + still do not exist, nothing about its existing CA changes. +- [ ] **Step 8: Commit** + ```bash + git add lib/common/functions.sh + git commit -m "Extract createWebIntermediateCA(); gate createSSLCA()'s CA-selection on pkiMode" + ``` + +### Task 1.4: `createClientIntermediateCA()` + the `certDecrypt()`/`certEncrypt()` repoint + +**Files:** +- Modify: `lib/common/functions.sh` — new function. +- Modify: `packages/web/lib/fog/fogbase.class.php` — `certDecrypt()`'s + `.srvprivate.key` filename resolution (`fogbase.class.php:2027-2032`). + +**Interfaces:** +- Produces: `ca.cert.der`/`ca.cert.pem` under `$webdirdest/management/other/` + — same export mechanics as `createSSLCA()`'s existing two lines + (`functions.sh:3520-3521`), same public path fog-client already downloads + from, just sourced from `.fogClientCA.pem` instead of the flat `$sslcapem` + when `pkiMode == split`. +- Consumes: `createRootCA()`'s output, or `validateExternalCA client` (Task + 1.5); `$fogClientCACN` (Task 1.1). + +**Design settled; one delivery detail from Task 0.3.** The Client CA issues +its own communication TLS certificate — never shared with the web vhost. +Task 0.3 determines only *where that certificate is published* so fog-client +finds it, not whether it exists. + +- [ ] **Step 1:** `createClientIntermediateCA()` generates + `.fogClientCA.{key,pem}` from the Root via `_issueIntermediateCA` with + `CN=$fogClientCACN`, then issues a **communication leaf** into + `$(_pkiZoneDir client/comm)` — `.commLeaf.{key,pem}`, `CA:FALSE`, RSA + 4096 to match today's `$sslprivkey` size (`functions.sh:3478`), since the + client chunks its RSA payload by modulus size (`fogbase.class.php:2056`) + and a smaller key would silently change that framing. + `ca.cert.der`/`ca.cert.pem` continue to be exported from + `.fogClientCA.pem` exactly as today (`functions.sh:3520-3521`), just from + the new source file. +- [ ] **Step 2:** Publish the comm leaf's **public** certificate where + fog-client fetches it — per Task 0.3's finding, expected to be the + existing `$webdirdest/management/other/ssl/srvpublic.crt` path, which + today holds the web vhost's leaf. Publishing the comm leaf there is what + decouples the two roles without moving anything the client looks for. + The web vhost's own certificate stays at `$sslpubcert` under the Web zone + and is no longer the file served at that path. **Only the public + certificate is published — `.commLeaf.key` never leaves `$sslpath`**, and + must be readable by the web user (it is what `certDecrypt()` opens) while + not being web-*served*; mirror the existing `chown $apacheuser` + + non-web-root placement `$sslpath` already provides. +- [ ] **Step 3:** `bash -n lib/common/functions.sh`. +- [ ] **Step 4:** In `fogbase.class.php`, change `certDecrypt()`'s + `.srvprivate.key` literal (`:2034-2042`) to resolve `.commLeaf.key` + instead when `pkiMode == split` — read the mode through whatever + mechanism this class already uses for install-time config (check + `config.class.php`'s generated constants; this class already reads + settings that way elsewhere). **In `flat` mode this branch must not exist + at all — the existing `.srvprivate.key` line stays completely untouched as + the `else`.** This is the one PHP change in this entire plan; keep the + diff minimal and reversible. Note the directory it resolves against is the + storage node's `sslpath` **database column**, not `$sslpath` from + `.fogsettings` — the comm key must be written where that column points + (confirmed in Task 0.3 Step 4). +- [ ] **Step 5:** Manual verify (test VM, fresh install, no flags needed — + `split` is the default): register a real fog-client against the split-mode + server, confirm `authorize()` succeeds (check `error_log`/task-scheduler + logs for a clean `#!ok`/token exchange, not `#!ihc`). +- [ ] **Step 6:** The decoupling proof — the test that shows this task + actually solved the problem it exists for: on that same split-mode server, + replace the **web vhost's** certificate and key (simulate an ACME renewal + by regenerating `$sslpubcert`/`$sslprivkey`, or run `installfog.sh -Y + --recreate-keys`), then confirm an already-registered fog-client still + authenticates. On a `flat`-mode server the identical action breaks client + auth — run it both ways and record the difference, since that contrast is + the whole justification for the Client zone. +- [ ] **Step 7:** Regression check: same registration test against a + `flat`-mode fresh install, confirm identical success — `certDecrypt()`'s + `else` branch is exercised, matching pre-patch behavior. +- [ ] **Step 8: Commit** + ```bash + git add lib/common/functions.sh packages/web/lib/fog/fogbase.class.php + git commit -m "Add createClientIntermediateCA(); decouple certDecrypt()'s key from the web TLS leaf in split mode" + ``` + +### Task 1.5: `validateExternalCA(zone)` + new CLI flags + +**Files:** +- Modify: `lib/common/functions.sh` — parameterize `validateExternalCA()` + by zone (`web`|`client`|`root`), writing into `_pkiZoneDir "$1"` instead of + the hardcoded `$sslpath/CA/`; add the CN-mismatch warning for `zone == + client`. +- Modify: `bin/installfog.sh` — new flags: `--legacy-pki` (fresh install + only — explicit opt-out of the new `split` default, reproduces today's + flat behavior byte-for-byte; see Task 1.1), `--restructure-pki` + (existing-server-only from this point forward — Task 1.1's default + already puts a fresh install into `split` with no flag needed, so this + flag's only remaining job is Phase 2's confirmation-gated migration of an + already-installed `flat` server; still accepted as a harmless no-op on a + fresh install for anyone scripted around the old opt-in behavior, but no + longer documented as the fresh-install opt-in), + `--web-ca-cert:`/`--web-ca-key:`/`--web-ca-root:` (aliases that, when + `pkiMode == split`, mean exactly what `--ca-cert`/`--ca-key`/`--ca-root` + mean today — see design doc's non-goal on not deprecating the old flags + yet: **both spellings work simultaneously**, old ones implicitly target + the Web zone), `--client-ca-cert:`/`--client-ca-key:`/`--client-ca-root:`, + `--client-ca-cn:` (Task 1.1's escape hatch), `--root-ca-cert:`/`--root-ca-key:`. +- Modify: `bin/updatefog.sh` — pass-through for all of the above, same + pattern as `--hostname`/`--extra-server-name`. + +**Interfaces:** +- Produces: nothing new consumed elsewhere. +- Consumes: Task 1.1's `_pkiZoneDir`. + +- [ ] **Step 1:** Change `validateExternalCA()`'s signature to + `validateExternalCA(zone)`, replace its four hardcoded `$sslpath/CA/...` + writes (`functions.sh:3323-3329`) with `_pkiZoneDir "$zone"`-based paths, + and replace its three hardcoded `$extcacert`/`$extcakey`/`$extcaroot` + reads with zone-prefixed variable names (`$webExtCACert`/... for `web`, + `$clientExtCACert`/... for `client`, `$rootExtCACert`/... for `root`) — + **existing callers in `flat` mode call it as `validateExternalCA web` and + it reads `$extcacert`/`$extcakey`/`$extcaroot` exactly as before** (keep + the old variable names as the `web` zone's names for backward + compatibility with anyone's existing `.fogsettings`/scripts referencing + them). +- [ ] **Step 2:** Add the CN check right after the existing chain-verify + check (`functions.sh:3315-3320`), only when `zone == client`: + ```bash + if [[ $zone == client ]]; then + local actualCN + actualCN=$(openssl x509 -in "$extcert" -noout -subject -nameopt multiline 2>/dev/null | awk -F'= *' '/commonName/{print $2}') + if [[ "$actualCN" != "$fogClientCACN" ]]; then + echo " WARNING: imported Client Communication CA's CN ('$actualCN')" + echo " does not match the expected value ('$fogClientCACN')." + echo " fog-client's exact requirement here is unverified as of this" + echo " release (see the design doc's Open Risks) -- this may or may" + echo " not matter for your fog-client version. Proceeding anyway." + fi + fi + ``` +- [ ] **Step 3:** Add the new `installfog.sh` flags following the exact + pattern of the existing `--ca-cert`/`--ca-key`/`--ca-root` case branches + (`bin/installfog.sh:292-320`) — one staging var per flag, applied in the + command-line-evaluation block. `--legacy-pki` is the simplest of the new + flags — no argument, just `slegacyPki=1`, applied as `[[ $slegacyPki -eq 1 + ]] && pkiMode="flat"` in the same block, positioned to run *after* Task + 1.1's `caCreated`-based default so it can override that default, and + *before* nothing else needs to check it since `pkiMode` is fully resolved + by this point. +- [ ] **Step 4:** `bash -n lib/common/functions.sh && bash -n + bin/installfog.sh && bash -n bin/updatefog.sh`. +- [ ] **Step 5:** Manual verify (test VM): re-run today's existing + `--external-ca`/`--ca-cert`/`--ca-key`/`--ca-root` test from + `docs/EXTERNAL_CA_AND_LETSENCRYPT.md` unchanged — confirm identical + behavior (this exercises `validateExternalCA web` in `flat` mode, proving + the parameterization didn't change anything for existing users). +- [ ] **Step 6:** Manual verify split mode: `installfog.sh -Y --client-ca-cert + ... --client-ca-key ... --client-ca-root ...` on a fresh install (no + `--restructure-pki` needed — `split` is already the default) against a + locally-minted CN-mismatched test CA → confirm the warning prints and the + install still completes. +- [ ] **Step 7: Commit** + ```bash + git add lib/common/functions.sh bin/installfog.sh bin/updatefog.sh + git commit -m "Parameterize validateExternalCA() by zone; add --client-ca-*/--web-ca-*/--root-ca-* flags" + ``` + +### Task 1.6: Remove `bin/setupacme.sh` — ACME/Let's Encrypt is not FOG-managed + +**Files:** +- Delete: `bin/setupacme.sh`. +- Modify: `docs/EXTERNAL_CA_AND_LETSENCRYPT.md` — remove the "Automating + renewal with setupacme.sh" subsection (added by the already-merged PR + #1014); replace with a short pointer to `docs/PKI_ZONES.md` (Task 1.8) + for the current self-service guidance. + +**Interfaces:** +- Consumes: nothing. +- Produces: nothing — this is a pure removal, no other task in this plan + depends on `bin/setupacme.sh` existing (Task 1.5's Web-zone CA-import flags + are independent of it; an admin who wants ACME automation now runs + `certbot`/`acme.sh` themselves entirely outside FOG). + +**Rationale (from the design doc's Non-goals):** on reflection, FOG should +not own any ACME client integration at all, even in the narrow, +CA-never-touched form `bin/setupacme.sh` already had. It's simple enough for +an admin to run their own ACME client and drop the result into +`$sslpubcert`/`$sslprivkey` — a safe drop-in once the paired +customization-preservation plan's vhost managed-block has landed. A future +GUI-level plugin is plausible later, not designed toward now. + +- [ ] **Step 1:** `git rm bin/setupacme.sh`. +- [ ] **Step 2:** In `docs/EXTERNAL_CA_AND_LETSENCRYPT.md`, remove the + "Automating renewal with setupacme.sh" subsection in full (the one added by + commit `deae3d41f`), and replace it with: + ```markdown + FOG does not automate ACME renewal itself. Run `certbot`/`acme.sh` (or + whatever ACME client you prefer) yourself, on your own schedule, and + install the renewed leaf at the Web zone's existing paths (`$sslpubcert`/ + `$sslprivkey`, from `.fogsettings`) followed by a web server reload. See + `docs/PKI_ZONES.md` for the full self-service pattern once the three-zone + PKI split has landed. + ``` +- [ ] **Step 3:** Grep the repo for any remaining reference to `setupacme` + (`grep -rn setupacme .` from the repo root) and remove/update any hits + (e.g. other doc cross-links, `bin/installfog.sh`'s `usage()` if it + mentions it). +- [ ] **Step 4:** Manual verify: confirm `bin/setupacme.sh` no longer exists + and `docs/EXTERNAL_CA_AND_LETSENCRYPT.md` reads sensibly with the + subsection removed (no dangling cross-references, no orphaned heading + numbering if the doc uses a table of contents). +- [ ] **Step 5: Commit** + ```bash + git add -A docs/EXTERNAL_CA_AND_LETSENCRYPT.md + git rm bin/setupacme.sh + git commit -m "Remove bin/setupacme.sh -- ACME/Let's Encrypt is not a FOG-managed feature" + ``` + +### Task 1.7: Interactive PKI-scenario prompt for fresh installs + +**Files:** +- Modify: `lib/common/newinput.sh` — new prompt block, modeled on the + existing `hostname` prompt (`newinput.sh:17-35`). +- Modify: `bin/installfog.sh` — ensure the staging variables the prompt + populates (`pkiMode` and the per-zone `--*-ca-*` values) are the *same* + variables Task 1.5's flags populate, applied in the same + post-`.fogsettings` evaluation block — no new variables, no second code + path. + +**Interfaces:** +- Produces: `$pkiMode` and, if scenario (c) is chosen, the per-zone CA + cert/key/root paths — identical in shape to what Task 1.5's flags already + produce. +- Consumes: nothing new; reuses `validateExternalCA(zone)` from Task 1.5 for + scenario (c)'s path validation. + +- [ ] **Step 1:** In `lib/common/newinput.sh`, add a new prompt block + immediately after the existing `hostname` prompt loop, gated the same way: + only runs when `[[ -z $pkiMode ]]` (not already set by a flag or persisted + `.fogsettings`) **and** the script is running interactively (i.e. the same + condition that already skips this whole file under `-Y`/a loaded update — + see `bin/installfog.sh:719`'s existing `[[ ! $doupdate -eq 1 || + ! $fogupdateloaded -eq 1 ]] && . ../lib/common/input.sh` gate, which this + new block relies on unchanged). +- [ ] **Step 2:** Prompt text and choices (plain numbered menu, matching this + file's existing prompt style), with the new default pre-selected: + ``` + How should FOG manage its certificates? + 1) Split PKI (recommended, default) -- separate FOG-generated + Root/Web/Client CAs; lets you replace the web certificate + independently of what fog-client trusts. [Press Enter for this] + 2) Split PKI, bring your own CA(s) -- same as (1), but you supply your + own certificate(s) for one or more zones (e.g. an internal AD CS + sub-CA) + 3) Legacy -- one self-signed CA for everything (today's pre-split + behavior; simpler, lower overhead, a permanent supported option) + ``` + Pressing Enter with no answer, or explicitly choosing `1`, sets + `pkiMode=split` with no further prompts (Task 1.2-1.4's generation path + runs entirely automatically) — matching the non-interactive default from + Task 1.1. Selection `2` sets `pkiMode=split` and then prompts per zone + ("Bring your own CA for the Web zone? Client Communication zone? Root?"), + each "yes" answer prompting for cert/key/root file paths, stored into the + exact same staging variables Task 1.5's + `--web-ca-*`/`--client-ca-*`/`--root-ca-*` flags populate. Selection `3` + sets `pkiMode=flat` — the same value `--legacy-pki` sets + non-interactively — reproducing today's pre-split behavior byte-for-byte. +- [ ] **Step 3:** Each entered path is validated with the same check Task + 1.5's flag parsing already runs (file exists and is readable) before being + accepted — reject and re-prompt on a bad path, do not silently proceed with + the previous run's or an empty value. +- [ ] **Step 4:** `bash -n lib/common/newinput.sh && bash -n bin/installfog.sh`. +- [ ] **Step 5:** Manual verify (test VM, real interactive terminal — this + cannot be tested under `-Y`, which skips this file entirely by design): + run `./installfog.sh` with no PKI-related flags, confirm the new prompt + appears after the hostname prompt, and that pressing Enter (or choosing + option 1) produces the same `$sslpath/CA/root|web|client` tree Task + 1.3/1.4's default `split` path already produces with no flags at all. + Confirm choosing option 3 (legacy) produces zero difference from running + with `--legacy-pki` — no new directories under `$sslpath/CA/`, + `pkiMode=flat` in `.fogsettings`, matching pre-this-design flat behavior + byte-for-byte. +- [ ] **Step 6: Commit** + ```bash + git add lib/common/newinput.sh bin/installfog.sh + git commit -m "Add interactive PKI-scenario prompt for fresh installs" + ``` + +### Task 1.8: Secure Boot intermediate CA + leaf signing (gated on Task 0.2) + +**Files:** +- Modify: `lib/common/functions.sh` — new `createSecureBootIntermediateCA()`; + `pkiMode` gate at the top of `_ensureSecureBootKeys()` (`functions.sh:4625`) + keeping its entire current body as the `flat` branch; `--addcert` in + `_resignKernels()` (`functions.sh:5080-5081`); `$secureBootMokCert` swap in + `_publishSecureBootKit()` (`functions.sh:4793-4795`); add + `secureBootMokCert` to `writeUpdateFile()`'s `managedKeys`. +- Modify: `bin/installfog.sh` — `--secureboot-ca-cert:`/`--secureboot-ca-key:` + for the bring-your-own case (an admin supplying their own SB intermediate), + alongside the existing `--secure-boot-key`/`--secure-boot-cert` (which keep + their current meaning: supply the *leaf*). + +**Interfaces:** +- Produces: `$secureBootMokCert` — **what gets enrolled in firmware**. In + `flat` mode it is assigned the same path as `$secureBootCert` (today's + behavior, no downstream branching); in `split` mode it is the intermediate. +- Consumes: `createRootCA()` and `_issueIntermediateCA()` from Tasks 1.2/1.3. + +**Blocked on Task 0.2.** Do not start until shim's chain-validation behavior +is confirmed. If Task 0.2 came back negative, skip this task entirely and +record why. + +- [ ] **Step 1:** Add `createSecureBootIntermediateCA()`, calling + `_issueIntermediateCA "FOG Secure Boot CA"` into + `${fogprogramdir}/secureboot/ca/`, then issuing this server's code-signing + leaf into `${fogprogramdir}/secureboot/leaf/` with the **same extension + profile `_ensureSecureBootKeys()` already writes** (`functions.sh:4660-4673` + — `basicConstraints=critical,CA:FALSE`, `extendedKeyUsage=codeSigning`, + `subjectKeyIdentifier=hash`), just signed by the intermediate rather than + self-signed. Reuse that exact `mok.cnf` heredoc rather than writing a new + one — the OpenSSL-1.0.2-compat reason it exists (`-addext` unavailable on + older RHEL) applies identically here. Leaf `-days 365`; intermediate + `-days 7300`. Preserve the directory's `0700 root:root` and the key's + `0600` — the `fog-sign-kernel` sudo helper's whole separation model + depends on the web user never being able to read these. +- [ ] **Step 2:** Gate `_ensureSecureBootKeys()`. Its existing body becomes + the `flat` branch **verbatim** — do not refactor it, do not "improve" it; + its never-regenerate guarantee (`functions.sh:4620-4624`) is what protects + every already-enrolled machine in the field. Add at the top: + ```bash + # split mode: firmware enrolls the INTERMEDIATE, kernels are signed by a + # short-lived leaf issued from it. Rotating that leaf then costs nothing, + # where today rotating the (self-signed, directly-enrolled) key means a + # physical MokManager trip to every machine. + if [[ $pkiMode == split && ${secureboot:-1} != 0 ]]; then + createSecureBootIntermediateCA + return 0 + fi + ``` + and, in the `flat` path, set `secureBootMokCert="$secureBootCert"` so both + modes leave the same two variables populated for downstream consumers. +- [ ] **Step 3:** `_resignKernels()` — add the chain. Change + `functions.sh:5080-5081` from: + ```bash + if sbsign --key "$secureBootKey" --cert "$certpem" \ + --output "$kpath" "${kpath}.unsigned" >>$error_log 2>&1; then + ``` + to add `--addcert` when the enrolled cert differs from the signing cert + (i.e. split mode), building the argument as an array so `flat` mode passes + no extra flag at all and its command line is byte-identical to today: + ```bash + local addcert=() + [[ -n $secureBootMokCert && "$(readlink -f "$secureBootMokCert")" != "$(readlink -f "$certpem")" ]] \ + && addcert=(--addcert "$secureBootMokCert") + if sbsign --key "$secureBootKey" --cert "$certpem" "${addcert[@]}" \ + --output "$kpath" "${kpath}.unsigned" >>$error_log 2>&1; then + ``` + Leave the `sbverify --cert "$certpem"` idempotency check (`:5073`) + untouched — it verifies against the signing leaf, which is still what + produced the signature. +- [ ] **Step 4:** `_publishSecureBootKit()` — change the three + `$secureBootCert` references in the DER-conversion block + (`functions.sh:4784`, `4793`, `4795`) to `$secureBootMokCert`. Nothing else + in that function changes; in `flat` mode the two variables are the same + path, so its output is identical to today. +- [ ] **Step 5:** `bash -n lib/common/functions.sh && bash -n bin/installfog.sh`. +- [ ] **Step 6:** Manual verify, split mode (test VM + a real UEFI client): + fresh `./installfog.sh -Y` → confirm + `openssl verify -CAfile $sslpath/CA/root/.fogRootCA.pem -untrusted + $fogprogramdir/secureboot/ca/.fogSBCA.pem $fogprogramdir/secureboot/leaf/sign.pem` + succeeds; `sbverify --list $webdirdest/service/ipxe/bzImage` lists **both** + the leaf and the intermediate; `MOK.der` published under + `$webdirdest/service/secureboot/` is the **intermediate** (`openssl x509 + -in MOK.der -inform der -noout -subject` shows `CN = FOG Secure Boot CA`); + and a client that enrolled that MOK boots the signed kernel. +- [ ] **Step 7:** The payoff test — rotate the leaf without re-enrolling: + delete `$fogprogramdir/secureboot/leaf/`, re-run `installfog.sh -Y` (a new + leaf is issued from the same intermediate), and confirm the **same** + already-enrolled client still boots with no firmware interaction. This is + the single test that proves the whole point of this task. +- [ ] **Step 8:** Regression, flat mode: on a server with an existing + `MOK.key`/`MOK.pem`, run `./installfog.sh -Y` → confirm the existing + keypair is untouched (compare checksums before/after), `MOK.der` published + is still the same self-signed cert, `sbsign` was invoked with **no** + `--addcert`, and an already-enrolled client boots exactly as before. +- [ ] **Step 9: Commit** + ```bash + git add lib/common/functions.sh bin/installfog.sh + git commit -m "Issue Secure Boot code-signing leaves from a FOG Secure Boot CA intermediate" + ``` + +--- + +### Task 1.9: Certificate path indirection (canonical paths + symlinks) + +**Files:** +- Modify: `lib/common/functions.sh` — fix the two mismatched symlink guards + (`functions.sh:3497-3498`); add a `_linkCanonical()` helper; apply it to + every path this design introduces. +- Modify: `bin/installfog.sh` — `usage()` text noting that any `--*-ca-*` + path may live outside FOG's directories. + +**Interfaces:** +- Produces: `_linkCanonical(realpath, canonicalpath)` — ensures + `canonicalpath` resolves to `realpath`, as a no-op when they are already + the same file. Every FOG consumer (vhost, `_resignKernels()`, + `_publishSecureBootKit()`, `certDecrypt()`) reads only canonical paths. +- Consumes: nothing new. + +**Why this matters beyond tidiness:** it is what lets an admin keep certs in +`/etc/letsencrypt/live/...` or `/etc/pki/...` without the vhost ever +changing — and, combined with the paired customization-preservation plan's +managed-block vhost, means relocating a certificate stops being a config +edit at all. + +- [ ] **Step 1:** Add the helper, next to the existing link block: + ```bash + # Canonical-path indirection: FOG's own consumers (vhost, sbsign, + # certDecrypt) only ever reference the canonical path, so the real file may + # live anywhere -- /etc/pki, /etc/letsencrypt/live, a mounted secret. This + # is why relocating a certificate never requires a vhost rewrite. + # + # Guarded against ln -sf X X: on a default install the "real" path IS the + # canonical one, and GNU ln refuses a self-link with an error into the log. + _linkCanonical() { + local real="$1" canon="$2" + [[ -z $real || -z $canon ]] && return 0 + [[ "$(readlink -f "$real")" == "$(readlink -f "$canon")" ]] && return 0 + ln -sf "$real" "$canon" >>$error_log 2>&1 + } + ``` +- [ ] **Step 2:** Replace `functions.sh:3497-3500` with four + `_linkCanonical` calls, fixing the two guards that currently test + `$sslpath/.fogCA.key`/`.fogCA.pem` while linking to + `$sslpath/CA/.fogCA.key`/`.pem`: + ```bash + _linkCanonical "$sslcakey" "$sslpath/CA/.fogCA.key" + _linkCanonical "$sslcapem" "$sslpath/CA/.fogCA.pem" + _linkCanonical "$sslcsr" "$sslpath/fog.csr" + _linkCanonical "$sslprivkey" "$sslpath/.srvprivate.key" + ``` + Behavior is unchanged on a default install (all four are no-ops); on an + install with any of those variables pointed elsewhere, the canonical path + now actually resolves, which is what the original code intended. +- [ ] **Step 3:** Apply the same helper to the paths introduced by Tasks + 1.2/1.3/1.4/1.8 (root, web, client, SB intermediate and leaves) so + bring-your-own paths behave identically across all zones. +- [ ] **Step 4:** `bash -n lib/common/functions.sh`. +- [ ] **Step 5:** Manual verify (test VM): install normally, confirm nothing + changed (`ls -la $sslpath` shows real files, no new symlinks, no `ln` + errors in `$error_log` — the last of which is an *improvement*, since + today's mismatched guards log one every run). Then move + `.srvprivate.key` to `/etc/pki/fogtest/`, set `sslprivkey` in + `.fogsettings` accordingly, re-run, and confirm the canonical path becomes + a working symlink, the vhost is unchanged, and both the web UI and a + fog-client checkin still work. +- [ ] **Step 6:** Document the two caveats in `docs/PKI_ZONES.md` (Task + 1.10): SELinux labels follow the symlink *target*, so a relocated cert on + a RHEL-family box may need `restorecon`/`semanage fcontext`; and a private + key relocated into a world-readable directory silently defeats the + `0600 root:root` separation the `fog-sign-kernel` sudo helper depends on. +- [ ] **Step 7: Commit** + ```bash + git add lib/common/functions.sh bin/installfog.sh + git commit -m "Add canonical-path symlink indirection so certs can live outside FOG's directories" + ``` + +--- + +### Task 1.10: Split `$netbootproto` from `$httpproto` + +**Files:** +- Modify: `lib/common/functions.sh` — new `$netbootproto` default logic + alongside `pkiMode`'s (Task 1.1); `configureDefaultiPXEfile()` + (`functions.sh:1037-1042`); the vhost HTTP→HTTPS redirect branches + (`functions.sh:3571` nginx, `:3814` Apache); add `netbootproto` to + `writeUpdateFile()`'s `managedKeys`. +- Modify: `bin/installfog.sh` — `--netboot-proto ` override. + +**Interfaces:** +- Produces: `$netbootproto` — the protocol iPXE uses to reach `boot.php`. + Defaults to `http` when the web certificate comes from a private CA (FOG + PKI or an imported internal CA), and follows `$httpproto` when it comes + from a public CA. +- Consumes: `$pkiMode`, `$httpproto`, `$externalca`. + +**Note — no PHP change is expected here.** `FOGBase::$httpproto` +(`packages/web/lib/fog/fogbase.class.php:481-483`) is derived from the +*current request's* `$_SERVER['HTTPS']`, so every boot-menu URL +`bootmenu.class.php` emits (`:286`, `:292`, `:458`) already inherits +whatever protocol iPXE connected with. Verify this empirically in Step 5 +before concluding no PHP work is needed — the whole task's economy rests on +it. + +- [ ] **Step 1:** Default `$netbootproto` next to Task 1.1's `pkiMode` + block: + ```bash + # iPXE can only validate a PUBLIC chain (via its ca.ipxe.org crosscert + # fallback). A FOG-PKI or internal-CA web certificate is fine for browsers, + # fog-client and the API -- but netboot fetches from the pre-boot + # environment have no path to that root, so they stay on HTTP rather than + # forcing the iPXE rebuild that forfeits the signed Secure Boot shim. + if [[ -z $netbootproto ]]; then + if [[ $httpproto == https && $pkiMode != split && $externalca != yes ]]; then + netbootproto="$httpproto" + elif [[ $httpproto == https ]]; then + netbootproto="http" + else + netbootproto="$httpproto" + fi + fi + ``` + (An admin who imported a genuinely public cert into the web zone can pass + `--netboot-proto https` explicitly; there is no reliable way to detect + "this CA is publicly trusted" from the certificate alone, so this defaults + conservatively and documents the override.) +- [ ] **Step 2:** `configureDefaultiPXEfile()` — change the `chain + ${httpproto}://...boot.php` in the generated `default.ipxe` + (`functions.sh:1040`) to `${netbootproto}`. This is the only occurrence + in that function. +- [ ] **Step 3:** Exclude the netboot paths from the vhost's HTTP→HTTPS + redirect, in both branches, **only when `$netbootproto != $httpproto`** + (so a public-CA install's config is unchanged from today): + - nginx: in the port-80 server block that currently issues the redirect, + add a preceding `location ^~ ${webroot}service/ipxe/ { ... }` that + serves normally instead of redirecting. + - Apache: guard the existing redirect with a negative match on the same + prefix (e.g. a `RewriteCond %{REQUEST_URI} !^${webroot}service/ipxe/` + ahead of the redirect rule). + This is the fiddliest part of the change and the most likely to differ + between distro layouts — test both families rather than one. +- [ ] **Step 4:** `bash -n lib/common/functions.sh && bash -n bin/installfog.sh`. +- [ ] **Step 5:** Manual verify (test VM, `pkiMode=split`, FOG-PKI web cert): + - The web UI loads over HTTPS. + - `grep chain $tftpdirdst/default.ipxe` shows `http://`. + - `curl -sI http://${webroot}service/ipxe/boot.php` returns **200**, + not a 301/302 to HTTPS. + - **The key assumption check:** the body of that same `boot.php` response + contains `http://` kernel/init URLs, confirming `$httpproto`'s + request-derived behavior carries through with no PHP change. + - A real client PXE-boots end to end and images successfully. +- [ ] **Step 6:** Regression: a public-CA install with `--netboot-proto https` + (or `pkiMode=flat` + `httpproto=https`) still redirects everything to HTTPS + exactly as today, with no `service/ipxe/` exclusion emitted into the vhost. +- [ ] **Step 7: Commit** + ```bash + git add lib/common/functions.sh bin/installfog.sh + git commit -m "Split netbootproto from httpproto so private-CA installs keep HTTPS web with HTTP netboot" + ``` + +--- + +### Task 1.11: Document Phase 1 + +**Files:** +- New: `docs/PKI_ZONES.md` (cross-linked from + `docs/EXTERNAL_CA_AND_LETSENCRYPT.md`'s "How FOG uses certificates" table + and from Task 1.6's replacement text). + +- [ ] **Step 1:** Write the three-zone model, the directory layout, the new + flags and the interactive prompt, and explicitly the **certDecrypt() + finding** (design doc's Context section) — this is the one piece of + institutional knowledge most likely to get re-lost if it isn't written + down plainly for the next person touching `createSSLCA()`. State plainly, + near the top, that `split` is the **default** for fresh installs as of + this feature, and that `--legacy-pki`/the prompt's legacy choice is a + **permanent, fully supported** alternative, not a deprecated fallback — + an admin reading this doc after an upgrade should not have to guess which + mode is "the real one." Include the self-service ACME/Let's Encrypt + guidance (Task 1.6) as its own clearly labeled section: admin runs their + own ACME client, drops the result into `$sslpubcert`/`$sslprivkey`, FOG + has no role in and no visibility into the process. Reuse the Mermaid + diagrams from `docs/superpowers/specs/2026-08-07-three-zone-pki-separation-design.md` + (Diagram 1 — default FOG PKI; Diagram 2 — drop-in options; and the + scenario-decision flowchart) rather than redrawing them — copy the fenced + ```mermaid blocks verbatim. +- [ ] **Step 1b:** Document the Secure Boot rotation story explicitly, since + it is the least obvious payoff of the whole design: enrolling the + intermediate once means signing leaves can be rotated or revoked with no + firmware trip, and an admin bringing their own PKI issues a Secure Boot + intermediate from their root and hands FOG a leaf. Include the honest + limits: existing servers keep their self-signed MOK (no migration is + offered), and per-node leaves are Phase 3. +- [ ] **Step 1c:** Document the protocol matrix from the design doc's + "Protocol selection" section as a table — which PKI choice yields HTTPS + where, that iPXE only trusts public chains, that a FOG/internal-PKI cert + can carry IP and DNS-alias SANs for browser/client trust but still won't + serve netboot over HTTPS, and that Let's Encrypt works for netboot **only** + on an FQDN in a domain you own (not the short hostname, not an IP) with + `FOG_WEB_HOST` set to match. +- [ ] **Step 2: Commit** + ```bash + git add docs/PKI_ZONES.md docs/EXTERNAL_CA_AND_LETSENCRYPT.md + git commit -m "Document the three-zone PKI split (Phase 1)" + ``` + +--- + +## Phase 2: Existing-server migration (blocked on Task 0.1's answer) + +Written for the **pessimistic** Phase 0 outcome (fog-client caches its pin at +registration time only, requiring full re-registration to change it). If +Phase 0's answer is optimistic, Task 2.2 collapses to "nothing — wait one +checkin cycle," which is a strictly easier version of the same task +breakdown, not a different one. + +### Task 2.1: `--restructure-pki` on an already-installed server — generate-only, no cutover + +**Files:** Modify `bin/installfog.sh` (the confirmation gate described in the +design doc's Error Handling section), `lib/common/functions.sh` +(`createClientIntermediateCA()` gains a `--dont-activate` mode that generates +`$sslpath/CA/client/.fogClientCA.pem` and publishes it to a **side-by-side** +path, e.g. `$webdirdest/management/other/ca.cert.new.der`, without touching +the live `ca.cert.der`/`ca.cert.pem` fog-client already trusts). + +- [ ] **Step 1:** Add the confirmation gate: `--restructure-pki` against a + server where `$caCreated == yes` (i.e., not a fresh install) requires + either an interactive "type YES to confirm" prompt or the explicit + `--i-understand-this-will-require-client-repinning` flag, even under `-Y`. +- [ ] **Step 2:** Add the side-by-side publish path, gated on a new + `--dont-activate-client-ca` flag (default when restructuring an *existing* + server; irrelevant/no-op on a fresh install, which has no live pin to + protect). +- [ ] **Step 3:** Manual verify (test VM with an already-registered + fog-client): run `--restructure-pki --dont-activate-client-ca` → confirm + the existing fog-client's next checkin still succeeds unmodified (the live + `ca.cert.der` never changed), and `ca.cert.new.der` is now downloadable at + the new path. +- [ ] **Step 4: Commit** + +### Task 2.2: Push the new pin to the existing fleet via a snapin (reuse, not invent) + +**Files:** None in this repo — this is an **admin-authored FOG snapin**, +using the existing `SnapinManager`/`SnapinTask` mechanism +(`packages/web/lib/fog/snapin*.class.php`) exactly as any other snapin is +created and assigned today. No new server-side task-scheduler code is +needed — this reuses the existing pull-based snapin delivery +(`packages/service/FOGSnapinReplicator`), which already runs with elevated +privileges and already supports "fetch a file, then run a script against +it," which is exactly this task's shape. + +- [ ] **Step 1:** Author (as an admin action, documented in Task 2.4, not + built into the installer) a snapin whose payload: downloads + `https:///fog/management/other/ca.cert.new.der` over the + **still-currently-trusted** channel (this works precisely because Task + 2.1 never touched the live pin), then installs it at whatever path + fog-client re-reads its pinned cert from. **This exact path is the + cross-repo unknown flagged in the design doc — do not guess at it here; + confirm against zazzles source or the fog-client installer's own + documentation before writing this snapin's script for real.** +- [ ] **Step 2:** Assign the snapin to "All Hosts" (or a pilot group first). + It is pulled at each client's next normal checkin — no new transport, no + new scheduling. +- [ ] **Step 3:** Monitor via the existing Snapin Job success/fail reporting + (`snapinjob.class.php`) until fleet coverage is judged sufficient. This is + a self-service, admin-paced rollout, not an atomic cutover — no code in + this repo needs to "know" when it's done. + +### Task 2.3: Cutover — activate the new Client CA + +**Files:** Modify `bin/installfog.sh`/`lib/common/functions.sh` — a +`--activate-client-ca` flag that copies `ca.cert.new.der`/`.pem` over the +live `ca.cert.der`/`.pem`. + +- [ ] **Step 1:** Implement the copy-over, gated the same way Task 2.1's + generation was (explicit confirmation). +- [ ] **Step 2:** **Do not** bulk-trigger `clearAES()`/"Reset Encryption + Data" as part of this step. Per the design doc's traced-through + `authorize()` logic, the CA pin and the `pub_key`/`sec_tok` handshake are + independent subsystems — a client that already re-pinned in Task 2.2 keeps + its existing session state and needs no reset. `clearAES()` remains + available (unchanged, existing admin action) as a manual remedy for any + individual host that gets stuck, exactly as it is today for unrelated + causes. +- [ ] **Step 3:** Manual verify: confirm hosts that ran Task 2.2's snapin + keep working uninterrupted through cutover; confirm a host that did *not* + yet run it fails its next `authorize()` (expected, pessimistic-case + disruption for stragglers) and recovers once it does receive the snapin or + gets manually re-registered. +- [ ] **Step 4: Commit** + +### Task 2.4: Document the full migration runbook + +**Files:** Extend `docs/PKI_ZONES.md` (Task 1.8) with a "Migrating an +existing server" section covering Tasks 2.1-2.3 as a runbook, explicitly +including the "confirm the client-side pin file path against your fog-client +version before writing the snapin" caveat. + +--- + +## Phase 3: Root offlining, flag consolidation, deprecation timeline + +No hard external blocker, but low value in isolation — sequence after Phase +1/2 have real usage to learn from. + +### Task 3.1: `--export-root-ca-and-wipe` helper + +Exports `.fogRootCA.key` to an admin-given path, then either `chmod 000`s or +(with an explicit `--shred` flag) securely deletes the on-server copy. Model +the confirmation flow on Task 2.1's/2.3's "explicit confirmation required" +pattern. + +### Task 3.2: Decide and execute an `--external-ca`/`--ca-cert`/`--ca-key`/`--ca-root` deprecation timeline + +Not started until Phase 1 has shipped for at least one release cycle. Options +range from "never deprecate, `--web-ca-*` is just an alias forever" (lowest +risk, recommended default) to "warn-then-remove" (only worth it if the +alias proves confusing in practice). This is a product decision, not an +engineering one — flag for the maintainer, don't pre-decide it in this plan. + +### Task 3.3: Per-storage-node Secure Boot signing leaves + +Task 1.8 makes this *possible* (each node gets its own leaf from the shared +intermediate, so a compromised node doesn't hand over the fleet's signing +key) but does not build it — issuing a leaf per node needs a CSR round-trip +through the storage-node install and node-registration flow, which doesn't +exist today. Until then nodes serve kernels signed by the master's leaf, +which works but delivers none of the per-node isolation the structure now +permits. + +Sketch: node install generates a keypair + CSR locally → registers the CSR +through the existing node-registration path +(`packages/web/maintenance/create_update_node.php`, already used by +`functions.sh:145`) → master signs from `secureboot/ca/` → node stores its +leaf under `secureboot/nodes/`. Requires deciding how the master +authenticates a node's CSR, which is the real design question here, not the +signing itself. + +### Task 3.4: Per-location PKI/protocol via the Location plugin + +The Location plugin already carries a per-location `protocol` override +(`packages/web/lib/plugins/location/hooks/changeitems.hook.php:118`), which +Task 1.9's `$netbootproto` work should stay compatible with. Extending that +to per-location certificates/signing leaves is a natural follow-on to Task +3.3 but needs its own design pass — not scoped here. + +### Task 3.5: Revisit a GUI-level Let's Encrypt plugin (separate design) + +Only once Phase 1/2 have real-world usage. Per the design doc's Open Risks +#5 and Task 1.6's removal of `bin/setupacme.sh`: a plugin providing a +GUI-level "enable Let's Encrypt" toggle is plausible, but needs its own +design pass, not a revival of the removed script's approach. + +--- + +### Critical Files for Implementation + +- `lib/common/functions.sh` — the bulk of every task: `_pkiZoneDir()`, + `createRootCA()`, `_issueIntermediateCA()`, the three + `create*IntermediateCA()` functions, `validateExternalCA(zone)`, the + `pkiMode` gate in `_ensureSecureBootKeys()` (`:4625`), `--addcert` in + `_resignKernels()` (`:5080`), `$secureBootMokCert` in + `_publishSecureBootKit()` (`:4793`), `configureDefaultiPXEfile()` + (`:1040`), the vhost redirect branches (`:3571` nginx / `:3814` Apache), + and `writeUpdateFile()`'s `managedKeys` (`:3122`). +- `lib/common/newinput.sh` — the interactive PKI-scenario prompt (Task 1.7). +- `bin/installfog.sh` — every new flag, plus the option-evaluation block. +- `bin/updatefog.sh` — pass-through for the new flags. +- `bin/setupacme.sh` — **deleted** by Task 1.6. +- `packages/web/lib/fog/fogbase.class.php` — `certDecrypt()`'s key-path + repoint (Task 1.4); also the *reference* for why Task 1.9 needs no PHP + change (`$httpproto` is request-derived, `:481-483`). +- `packages/web/lib/fog/bootmenu.class.php` — read-only reference for Task + 1.9's protocol verification (`:286`, `:292`, `:458`). +- `docs/EXTERNAL_CA_AND_LETSENCRYPT.md` — setupacme.sh section removed + (Task 1.6). +- `docs/PKI_ZONES.md` — new, created by Task 1.10. diff --git a/docs/superpowers/specs/2026-08-07-customization-preservation-design.md b/docs/superpowers/specs/2026-08-07-customization-preservation-design.md new file mode 100644 index 0000000000..c061caa3e4 --- /dev/null +++ b/docs/superpowers/specs/2026-08-07-customization-preservation-design.md @@ -0,0 +1,615 @@ +# Install-time customization preservation + +Part of the follow-up to #1005/#1012/#1013 (git-path update script, vhost/hostname +flags, cert separation): those made `updatefog.sh` a real, orchestrated updater. +This design finishes the job by moving customization preservation *into* +`installfog.sh` itself, so it protects every run, not only ones that went +through `updatefog.sh`. + +## Context + +Today, five different categories of "thing an admin customized" are handled +five different, inconsistent ways, and four of the five only work when the +customization was made under one specific default filename, in one specific +directory, and the run went through `bin/updatefog.sh`: + +1. **iPXE background (`bg.png`)** — `FOG_IPXE_BG_FILE` is a real, + `globalSettings`-backed, GUI-editable filename + (`packages/web/commons/schema.php:3267-3270`, default `bg.png`; read by + `packages/web/lib/fog/bootmenu.class.php:2100-2103`). The installer has + zero awareness of it. `configureHttpd()` (`lib/common/functions.sh:4138`) + `rm -rf`s `$webdirdest` (after a wholesale `.BACKUP` snapshot, + `functions.sh:4241-4248`) and `cp -Rf $webdirsrc/* $webdirdest/` + (`functions.sh:4287`), relaying the shipped default `bg.png` every run. + The only protection that exists at all is `lib/common/update.sh`'s + `_updateAssetFiles()` (line 34) hardcoding the literal string `"bg.png"` — + never the actual setting value — and only runs via `bin/updatefog.sh`. +2. **Custom vhost content** (extra directives, headers, includes) — `createSSLCA()` + (`functions.sh:3412`) regenerates the *entire* `$etcconf` file from scratch + via inline `echo`/heredoc every run, for both nginx (`functions.sh:3536` ff.) + and Apache (`functions.sh:3751` ff.). `-F`/`--no-vhost` + (`bin/installfog.sh:436-437`) is the only escape hatch, and it is + all-or-nothing: skip regeneration entirely, forever, or lose every hand + edit on the next un-flagged run. `diffconfig()` (`functions.sh:5564-5573`) + only ever informs ("Changed configurations:", `bin/installfog.sh:995-1002`) + — it restores nothing. +3. **Kernel/init (`bzImage`, `init.xz`, etc.)** — `downloadfiles()` + (`functions.sh:4532-4609`) unconditionally re-downloads and overwrites + these under `${webdirdest}/service/ipxe/` every run. `_resignKernels()` + (`functions.sh:5049-5094`) keeps a `.unsigned` sibling purely as a + double-signing guard, not a version history. `configureTFTPandPXE()` + (`functions.sh:1191` ff.) separately snapshots the whole `$tftpdirdst` tree + to `${tftpdirdst}.prev` (lines 1196-1200) before copying in fresh files — + but that snapshot is dead storage; nothing ever reads it back + automatically, and it doesn't even hold the kernels (those are served over + HTTP from `$webdirdest`, not TFTP — see Architecture, part 3). The only + automatic kernel restore anywhere is `update.sh`'s `_restorePreviousKernel()` + (lines 63-71), and only on the update-failed-so-revert path + (`revertUpdate()`, lines 110-120) — never on success, never more than one + generation back. Separately, FOG has a real **per-host custom kernel/init + filename** feature (`packages/web/lib/fog/bootmenu.class.php:408-416`, + `self::$Host->get('kernel')`/`get('init')`) that lets an admin point one + specific host at an alternate, hand-placed file under + `service/ipxe/` — nothing protects a custom-named file like that at all + today, not even the narrow `_updateAssetFiles()` list, which only knows the + six fixed default names. +4. **`autoexec.ipxe`** — sourced from the `FOGProject/fog-ipxe` release tarball + (file lives at that repo's root; `downloadipxe()`/`fetchipxeasset()`, + `functions.sh:1084-1122`, unpack it into `$tftpdirsrc`), then copied into + `$tftpdirdst/autoexec/autoexec.ipxe` by `configureTFTPandPXE()`'s tree copy + (`functions.sh:1218-1220`) **every run**, and hard-linked from there into + `autoexec/i386-efi/`, `autoexec/arm64-efi/`, `secureboot/`, and + `secureboot/arm64-efi/` (lines 1249-1262). The hard-linking is real and + already works by construction — an edit to any one copy shows up in all of + them within a run — but that protection is worthless against the *next* + run's tree copy, which relays the fog-ipxe project's shipped default over + whatever is there, unconditionally. A hand edit to `autoexec.ipxe` does not + survive an update today, full stop. +5. **Secure Boot certificates** — largely already solved. FOG's own generated + MOK/PK/KEK keys live at `${fogprogramdir}/secureboot/` — outside + `$webdirdest`, so `configureHttpd()`'s wipe never touches them + (`_ensureSecureBootKeys()`, `functions.sh:4625-4694`; + `_ensureSecureBootPlatformKeys()`, `functions.sh:4713-4774`). **But** the + admin-supplied case (`--secure-boot-key`/`--secure-boot-cert`, + `bin/installfog.sh:444-465`) never copies the admin's files anywhere — + `_ensureSecureBootKeys()` just trusts whatever path was given + (`[[ -n $secureBootKey && -n $secureBootCert ]] && return 0`, line 4641) + and that literal path is persisted forever in `.fogsettings` + (`secureBootKey secureBootCert`, `writeUpdateFile()`'s `managedKeys`, + `functions.sh:3142`). If that path is ever inside `$webdirdest` (or + anywhere else this installer deletes/regenerates), `configureHttpd()`'s + `rm -rf $webdirdest` — which runs *before* `configureTFTPandPXE()` calls + `downloadfiles()` → `_ensureSecureBootKeys()`/`_resignKernels()`/ + `_publishSecureBootKit()` in the very same run — deletes the admin's file + out from under itself before anything downstream ever reads it. This is a + real, confirmed gap, not a hypothetical. + +`bin/updatefog.sh`/`lib/common/update.sh`'s `backupCustomizations()`/ +`restoreCustomizations()` (`update.sh:37-59`) is the **only** orchestrated +backup-then-restore sequence in the codebase today, and it is a bolt-on around +`installfog.sh`, not something `installfog.sh` itself knows about. A bare +`bash installfog.sh` (skipping `updatefog.sh`) gets none of it. + +## Non-goals + +- Not building a general config-file diff/merge engine. The vhost mechanism + (Architecture §1) is deliberately "FOG owns one clearly-marked region, + completely, always" rather than a smart merge — matching this codebase's + existing all-or-nothing regeneration philosophy everywhere else, just scoped + down from "the whole file" to "one block." +- Not adding a scheduling/cron layer for kernel backups — the versioned + backup in Architecture §3 is a side effect of every `installfog.sh` run, + not a separate timer. +- Not changing `_resignKernels()`'s existing `.unsigned`-sibling idempotency + guard — it solves a different problem (don't double-sign) than the + versioned backup (give me an old *signed* kernel back). +- Not moving `autoexec.ipxe`'s customization point upstream into the + `FOGProject/fog-ipxe` repo. That would be the eventual "right" home for a + hook line, but this design keeps the change entirely inside `fogproject` + (Architecture §4) so it ships and is testable without a coordinated + cross-repo release. +- Not changing storage-node installs (`installtype = [Ss]`, `bin/installfog.sh:867-892`). + They run `configureMinHttpd`, not `configureHttpd`, and don't serve the iPXE + boot menu/kernels/vhost the same way — none of §1-§3 apply there, and this + design does not touch that branch. +- Not solving in-block hand edits to the vhost surviving regeneration (see + Error Handling — this is the same class of known limitation `diffconfig` + already has, just narrowed in scope). + +## Architecture + +Six additive pieces. None change behavior for an admin who customizes +nothing. + +### 1. Vhost: a FOG-managed block, not a template file, not a whole-file diff + +**Decision: a managed-block convention (`# === FOG MANAGED BLOCK ===` +markers), not a separate template asset with token substitution.** + +Justification against this codebase's actual shape: every vhost write site +(`createSSLCA()`, `functions.sh:3536` ff. for nginx, `3751` ff. for Apache) +generates its content as a long chain of bash `echo`/heredoc lines directly +into `$etcconf` — there is no static template file anywhere in this flow, and +the branching (webserver family × OS family × SSL on/off × IPv4 vs IPv6 × +`--extra-server-name` suffix) is all inline bash logic, not data-driven. +Introducing real template *files* with `sed`-substituted tokens would mean +extracting ~10 write sites' worth of conditionally-assembled content into +external assets and keeping two representations in sync — a much larger, +riskier refactor than this problem calls for, and a bigger deviation from +"additive, doesn't change existing behavior when unused" than necessary. A +managed-block splice can wrap the *existing, unchanged* generation code with +two marker lines and a small generic helper. + +**Mechanics:** +- New helper `spliceManagedBlock(file, contentfile)` in `functions.sh`: + - Marker constants: + `FOG_MANAGED_BEGIN='# === FOG MANAGED BLOCK -- DO NOT EDIT BETWEEN THESE LINES (see docs/SUPPORTED_CUSTOMIZATIONS.md) ==='`, + `FOG_MANAGED_END='# === END FOG MANAGED BLOCK ==='`. `#` is a comment + character in both nginx and Apache conf syntax, so the markers are inert + in either file type. + - Every existing generation call site keeps writing exactly what it writes + today, but into a fresh temp file (e.g. `${etcconf}.fogblock.$$`) instead + of directly into `$etcconf`. + - `spliceManagedBlock` then: + - If `$etcconf` doesn't exist: write `FOG_MANAGED_BEGIN`, the temp + content, `FOG_MANAGED_END` as the entire new file (identical *content* + to today's fresh-install behavior, just wrapped). + - If `$etcconf` exists and already contains both markers (a prior FOG + run wrote them): replace only the lines between them with the fresh + temp content (an `awk` range-replace), leaving everything before + `FOG_MANAGED_BEGIN` and after `FOG_MANAGED_END` byte-for-byte untouched. + - If `$etcconf` exists but has no markers (first FOG-managed run against + a pre-existing/hand-built file, or an upgrade from before this + feature): **append** a new marked block at the end of the existing + file, never touching the pre-existing content. This is the one + behavior change on an existing install's first run under this feature + — call it out in release notes. + - `mv -fv "${etcconf}" "${etcconf}.${timestamp}"` + `diffconfig()` still run + exactly as today, so the "Changed configurations" notice still fires when + the *whole file's* bytes differ — including, now, only-inside-the-managed-block + changes, same imprecision `diffconfig` already has (see Error Handling). +- `createSSLCA()`'s nginx branch (`functions.sh:3536-3752` today) and Apache + branch (`3751-3993` today) each change their *last* line from directly + finishing the write to calling `spliceManagedBlock "$etcconf" "$tmpblock"`. + Every line in between — `server_name`/`ServerAlias`, SSL cert paths, + `--extra-server-name` suffix handling — is unchanged. +- `--overwrite-vhost` (currently `updatefog.sh`-only, forwarded as an empty + `$updateVhostFlag`) keeps its meaning almost as-is: "regenerate as if no + managed block existed" — i.e., delete the whole file first, then let + `spliceManagedBlock` write a fresh single-block file. Useful for an admin who + wants to discard accumulated cruft (old markers, stale appended content) + and start clean. +- `-F`/`--no-vhost` keeps its exact current meaning: skip vhost writing + entirely, don't touch the file, don't even splice. This stays the true + "FOG, hands off" escape hatch — **it does not become obsolete**, because + splicing is itself an opinionated action (adding markers to a + previously-marker-free file) some admins may not want at all. +- **Consequence for `updatefog.sh`'s default:** today `updateVhostFlag="-F"` + by default (`bin/updatefog.sh:79`) specifically because full regeneration + used to mean "destroy any hand customization." With splicing, that's no + longer true — the managed block is always safe to refresh; only content + *outside* it is ever at risk, and that risk no longer exists. So this + design flips `updatefog.sh`'s default: an update now lets `installfog.sh` + splice the managed block by default, and `-F` remains available as the + explicit "no, really don't touch it" opt-out. This is a real behavior + change and needs its own callout/test (Implementation Plan, Task 3). + +### 2. Generalized backup/restore living inside `installfog.sh` + +Two new functions in `functions.sh`, called from the existing +`configureMySql; writeUpdateFile; backupReports; configureHttpd; ...` +sequence (`bin/installfog.sh:938-963`, the `[Nn]`/master-install branch — +this is the only branch that calls `configureHttpd`; the `[Ss]`/storage-node +branch at lines 867-892 is untouched): + +``` +configureMySql +writeUpdateFile +backupReports +backupPreservedCustomizations # NEW +configureHttpd +checkWebTier +backupDB +updateDB +configureStorage +configureDHCP +configureTFTPandPXE # downloadfiles() inside here re-lays default-named kernels/init +restorePreservedCustomizations # NEW +configureFTP +... +``` + +`backupPreservedCustomizations()` runs **before** `configureHttpd()`'s +`rm -rf $webdirdest` (`functions.sh:4247`), at a point where +`configureMySql` has already run — so `$sqloptionsuser`/`$snmysqlpass`/ +`$mysqldbname` are already set and a DB query is possible. It: + +1. Queries the *actual* `FOG_IPXE_BG_FILE` value: + `mysql $sqloptionsuser --password="$snmysqlpass" -N -B --execute="SELECT settingValue FROM globalSettings WHERE settingKey='FOG_IPXE_BG_FILE'" $mysqldbname 2>>$error_log`. + On a first-ever install the `globalSettings` table doesn't exist yet + (schema loads later, in `updateDB()`, which runs *after* `configureHttpd`) + — the query simply errors into `$error_log` and returns empty, which this + function treats identically to "no customization, nothing to back up." + Same posture as every other `[[ -f ... ]]`-gated step in this file — no + special-casing needed for "fresh install." +2. If that filename resolves to a real file at + `${webdirdest}/service/ipxe/`, copies it to + `${fogprogramdir}/customizations/ipxe-bg/` (preserves the actual + name — this is what makes it setting-driven instead of hardcoded to + `bg.png`). +3. Copies any of the legacy `refind.*` files present, same as today's + `_updateAssetFiles()` list, to `${fogprogramdir}/customizations/ipxe-legacy/`. +4. Snapshots the **entire** `${webdirdest}/service/ipxe/` directory (not a + fixed filename list) into a rotated, bounded set of generations under + `${fogprogramdir}/customizations/kernel-backups/` — see Architecture §3. + Because this snapshot is "everything currently there," it automatically + captures a per-host custom-named kernel/init file + (`bootmenu.class.php:408-416`'s `Host->get('kernel')`/`get('init')`) + without FOG needing to query the `hosts` table to learn any custom name — + it never needs to know the name at all. + +`restorePreservedCustomizations()` runs after `configureTFTPandPXE()` +(specifically after its internal `downloadfiles()` call has re-populated the +default-named kernel/init files) and: + +1. Restores the bg file from `${fogprogramdir}/customizations/ipxe-bg/` + back to `${webdirdest}/service/ipxe/` — using the same name it was + backed up under, so this works unchanged if `FOG_IPXE_BG_FILE` itself never + changes, and also works if an admin renames it going forward (next run's + `backupPreservedCustomizations` just backs up under the new name). +2. Restores `refind.*` the same way as today. +3. Restores, from the most recent kernel-backup generation, any file whose + name is **not** one of the six fixed default kernel/init names (`bzImage`, + `bzImage32`, `arm_Image`, `init.xz`, `init_32.xz`, `arm_init.cpio.gz`) — + this is the generic mechanism that covers a per-host custom-named kernel + without any dedicated "custom kernel name" setting existing anywhere. + The six default names are deliberately **not** restored here — matching + `update.sh`'s existing, deliberate "the point of an update is to pick up + the latest kernel" comment (line 49) — that's what Architecture §3's + generation history is *for*: an explicit restore path, not an automatic one. +4. `chown -R ${username}:${apacheuser}` the restored paths, matching today's + `restoreCustomizations()`/`_restorePreviousKernel()` behavior. + +**Secure Boot admin-key gap fix**, folded into the same "protect before the +wipe" principle but implemented as its own tiny function, +`preserveSecureBootAdminFiles()`, called immediately after the existing +`--secure-boot-key`/`--secure-boot-cert` pair validation in `installfog.sh`'s +option-evaluation block (right after `unset sbfile`, currently around line +681) — i.e., **before** `configureMySql`/`configureHttpd` ever run, which is +the earliest point `$fogprogramdir` is resolved and the only point that's +provably before any tree gets wiped: + +- If `$secureBootKey`/`$secureBootCert` are both set (admin-supplied or + already-persisted from a prior admin-supplied run) and their resolved + absolute path is **not already** `${fogprogramdir}/secureboot/MOK.key`/ + `MOK.pem`, copy them there (`mkdir -p`, `chown root:root`, `chmod 0600`/`0644` + matching `_ensureSecureBootKeys()`'s own generated-key permissions), then + reassign `secureBootKey`/`secureBootCert` to the copies. +- This makes the admin-supplied case converge onto exactly the same + "lives outside `$webdirdest`, therefore survives every wipe by construction" + guarantee FOG's own generated keys already have — closing the gap without + changing `_ensureSecureBootKeys()`'s explicit "an admin-supplied pair always + wins and is never touched or overwritten" contract (line 4640): the + *original* file the admin pointed at is still never modified; a *copy* is + what gets used and persisted from here on. +- Idempotent by construction: once the copy exists and `.fogsettings` has been + rewritten with the copy's path (via the next `writeUpdateFile` call), every + later run's `[[ -n $secureBootKey && -n $secureBootCert ]] && return 0` in + `_ensureSecureBootKeys()` (line 4641) already points at the safe copy, so + this function's own "already at destination" check makes it a no-op forever + after, until the admin explicitly passes `--secure-boot-key`/`--secure-boot-cert` + again to rotate to a new pair. + +### 3. Versioned kernel/init backup + +**Location: `${fogprogramdir}/customizations/kernel-backups/`, not alongside +the live files under `${webdirdest}/service/ipxe/`.** This matters: unlike +`bzImage.unsigned` (a same-directory sibling that only needs to survive +*within* one run), anything living inside `$webdirdest` is destroyed by +`configureHttpd()`'s `rm -rf $webdirdest` (line 4247) on the *next* run, before +`downloadfiles()` ever gets a chance to re-populate it. A version history has +to live outside the tree that gets wiped — the same reasoning that already +makes `${fogprogramdir}/secureboot/` safe. + +**Scheme:** numbered generation directories, `gen-1` (most recent) through +`gen-N` (oldest kept), bounded at `N` = 3 by default. Rotation, run inside +`backupPreservedCustomizations()` immediately before writing the new +snapshot: +``` +[[ -d gen-N ]] && rm -rf gen-N +for k in $(seq $((N-1)) -1 1); do + [[ -d gen-$k ]] && mv gen-$k gen-$((k+1)) +done +cp -a "${webdirdest}/service/ipxe/." gen-1/ +``` +`cp -a` preserves the `attr -s version`/`attr -s tag_name` xattrs +`downloadfiles()` already stamps on every kernel/init file +(`functions.sh:4583-4599`), so each generation is self-describing (which FOS +release it came from) with no separate manifest needed. + +`N` is configurable: new `--kernel-backup-count ` flag on `installfog.sh` +(same staging-var convention as every other flag — `skernelBackupCount` → +`kernelBackupGenerations`, applied in the option-evaluation block, added to +`writeUpdateFile()`'s `managedKeys`, default `3` if never set). Storage cost +is bounded and small — these are the same files `downloadfiles()` already +downloads every run; keeping 3 generations costs roughly 3× one release's +kernel/init set, on disk the admin already provisioned for FOG. + +**Restore path:** a new leaf script, `bin/restorekernel.sh`, following the +`bin/setupacme.sh` precedent (a rare, deliberate, admin-invoked operation — +not something worth adding as an `installfog.sh` flag that would need to +short-circuit the rest of that script's pipeline): +- `--list` — prints each `gen-N` directory's contents with their `tag_name` + xattr (`attr -g tag_name `) so an admin can see which FOS release each + generation came from before choosing. +- `--generation N` — copies `gen-N`'s contents back into + `${webdirdest}/service/ipxe/`, `chown`s them, and if `$secureBootKey`/ + `$secureBootCert` are configured, re-runs the same signing check + `_resignKernels()` uses (`sbverify` against the live file; re-sign only if + it doesn't already verify) — flagged as a known edge case in Error Handling + if the signing key has rotated since that generation was captured. + +**Revert-on-failure carve-out preserved:** `update.sh`'s current behavior +deliberately restores the *default-named* kernels on the revert path (not just +on request) because a revert to an older commit should also mean older +kernels. This is preserved via a new `installfog.sh` flag, +`--restore-kernel-backup`, which `updatefog.sh`'s `revertUpdate()` passes on +its re-invocation of `installfog.sh` — it tells `restorePreservedCustomizations()` +to *also* restore `gen-1`'s default-named files, not just the non-default +("custom") ones it restores unconditionally. Normal runs never pass this flag. + +### 4. Custom PXE script hook — off `default.ipxe`, reachable via `autoexec.ipxe` + +**Investigated, not assumed:** `autoexec.ipxe` is not FOG-authored text — +it's unpacked from the `FOGProject/fog-ipxe` release tarball +(`fetchipxeasset()`/`downloadipxe()`, `functions.sh:1084-1122`) and its content +is owned by that separate repo. What it *does*, per that repo's own +documentation, is DHCP/proxyDHCP discovery across `net0`/`net1`/`net2`, then +**`chain` to a fixed, bare name: `default.ipxe`** on the same server — and +`default.ipxe` is not part of the tarball at all. It is generated, in full, +by `configureDefaultiPXEfile()` (`functions.sh:1037-1042`) — one `echo -e ...` +line, unconditionally overwritten every run, with no expectation today that +anyone hand-edits it. That makes `default.ipxe`, not `autoexec.ipxe` itself, +the right place to add a hook: it's the file `fogproject`'s own bash already +owns outright and already regenerates unconditionally every run (so there's +no customization-loss risk to introduce — it has none today), and it's one +hop downstream of `autoexec.ipxe`'s own chain, so "off `autoexec.ipxe`" is +still an accurate description of where the hook sits in the boot flow. + +Per `ipxe.org/cmd/chain`: **without `--replace`, `chain` returns control to the +calling script once the chained script finishes executing normally** (only a +*failed* chain — file not found, parse error — triggers the `||` fallback). +This makes a safe, default-behavior-preserving hook straightforward: + +``` +#!ipxe +chain custom.ipxe || goto fog_default +:fog_default +set arch ${buildarch} +...(rest of today's configureDefaultiPXEfile output, byte-for-byte unchanged)... +:bootme +chain ${httpproto}://$ipaddress${webroot}service/ipxe/boot.php##params +``` + +- If `$tftpdirdst/custom.ipxe` doesn't exist (the overwhelming default case), + `chain` fails immediately, `|| goto fog_default` fires, and boot proceeds + exactly as it does today — zero behavior change when unused. +- If it exists, it's chained *before* FOG's own params/boot-menu logic runs — + the natural place for a boot delay, custom prompt, or site-specific menu + (the "10-sec delay that was embedded in an alternate pxe boot file + previously," from the ask). When that script finishes normally (reaches end + of script, or an explicit `exit`), control returns to the line immediately + after the `chain custom.ipxe` call in `default.ipxe`, which falls straight + through into `:fog_default` — no special "resume" convention or shared + state variable needed, and no risk of the infinite loop a naive + chain-back-to-`default.ipxe` design would create. +- `custom.ipxe` is never created, deleted, or touched by any part of + `installfog.sh` — it lives at `$tftpdirdst` root, a directory that is only + ever *snapshotted* (`.prev`, `configureTFTPandPXE()` lines 1196-1200), never + wholesale-wiped, and the tree copy loop (lines 1218-1220) only ever copies + files *from* `$tftpdirsrc`, never deletes anything at the destination that + isn't in the source. So `custom.ipxe` survives every future run + structurally, the same way Secure Boot keys survive by living outside + `$webdirdest` — **no backup/restore mechanism is needed for this file at + all**, only the one-line addition to `configureDefaultiPXEfile()`'s + generated content. +- Change is confined to `configureDefaultiPXEfile()` — the whole function's + output is already a single unconditional overwrite, so there's no + idempotency concern (no "already has the hook, don't add it twice" check + needed, unlike the vhost's managed-block splice). + +### 5. `updatefog.sh` simplification + +With §2/§3 living in `installfog.sh` itself, `lib/common/update.sh` shrinks to +just the git mechanics: + +- **Removed entirely:** `_updateAssetFiles()`, `backupCustomizations()`, + `restoreCustomizations()`, `_restorePreviousKernel()` (`update.sh:33-71`) — + fully superseded by `backupPreservedCustomizations()`/ + `restorePreservedCustomizations()`, which now run unconditionally inside + every `installfog.sh` invocation, including the one `revertUpdate()` makes. +- **Kept, lightly modified:** `gitUpdateToBranch()` (unchanged) and + `revertUpdate()` (`update.sh:110-120`) — its calls to + `_restorePreviousKernel`/`restoreCustomizations` are deleted (the re-invoked + `installfog.sh -Y $updateVhostFlag` now does this itself); it instead adds + `--restore-kernel-backup` to that re-invocation (Architecture §3's carve-out). +- **`bin/updatefog.sh`'s job becomes:** resolve channel/branch + (`channelToBranch`, unchanged), confirm/`-y`, `backupCustomizations`'s + call site deleted (nothing left to call), `gitUpdateToBranch`, invoke + `bash installfog.sh -Y ...` (now the *sole* place backup/restore happens), + handle revert-on-failure. The `$updateVhostFlag` default flips from `-F` to + empty per Architecture §1's consequence — `-F` remains available via the + same flag, just no longer the default. +- `backupCustomizations` is also removed from `updatefog.sh`'s own call + (currently line 249, before `gitUpdateToBranch`) — there is nothing left + for it to do once `installfog.sh` owns this. + +### 6. Documentation: `docs/SUPPORTED_CUSTOMIZATIONS.md` + +New top-level doc enumerating exactly what survives automatically vs. what +requires deliberate admin action. Proposed headings: + +``` +# Supported Customizations + +## How to read this document +(one paragraph: "automatic" means installfog.sh/updatefog.sh preserve it with +no admin action every run; "manual" means the admin must re-apply it or use a +documented escape hatch) + +## iPXE boot background (bg.png / FOG_IPXE_BG_FILE) +Automatic. Example row: | Customization | How it's preserved | Where | +| Renamed background file via FOG_IPXE_BG_FILE | Backed up before the web +tree is rebuilt, restored under its actual (possibly renamed) filename after | +${fogprogramdir}/customizations/ipxe-bg/ | + +## Web server vhost (nginx/Apache) +Automatic for FOG's own security-relevant content (ServerName/server_name, +ServerAlias, cert paths, ciphers); manual for anything an admin adds outside +the FOG-managed block, which is preserved as-is. Example row: | Extra +ServerAlias/hostname | Set via --hostname/--extra-server-name and always +written into the managed block, not backed up | see --extra-server-name | + +## Kernel / init (bzImage, init.xz, ...) +Default-named files are always replaced with the latest release (by design); +N prior generations are kept for manual restore via bin/restorekernel.sh. +Custom-named files (per-host kernel/init override) are restored automatically. +Example row: | Per-host custom kernel filename | Snapshotted/restored +automatically as part of every install/update | Host->get('kernel') | + +## Custom PXE scripts (custom.ipxe hook) +Manual, but supported: place custom.ipxe at the TFTP root; FOG chains to it +before its own boot logic if present, otherwise boots exactly as before. + +## Secure Boot certificates +Automatic. FOG's own generated keys, and now admin-supplied +--secure-boot-key/--secure-boot-cert pairs, are copied into and always read +from ${fogprogramdir}/secureboot/, which nothing in the installer ever wipes. + +## What is NOT automatically preserved +(honest callout: hand edits made INSIDE the vhost's FOG-managed block; a +kernel-signing key rotated after a backup generation was captured) +``` + +## Data flow + +**Install/update time:** `installfog.sh` (directly, or via `updatefog.sh` +which now just resolves the branch and invokes it) → option evaluation +(secure-boot admin-key preservation runs here, before anything else) → +`configureMySql` (DB now queryable) → `backupPreservedCustomizations` (reads +`FOG_IPXE_BG_FILE`, snapshots `service/ipxe/`) → `configureHttpd` (wipes/relays +`$webdirdest`, splices the vhost's managed block) → `configureTFTPandPXE` → +`downloadfiles` (re-lays default-named kernels/init, generates +`default.ipxe` with the `custom.ipxe` hook) → `restorePreservedCustomizations` +(bg file back under its real name, custom-named kernels back, `refind.*` back) +→ rest of the sequence unchanged. + +**Boot time (steady state):** client's EFI binary → `autoexec.ipxe` +(fog-ipxe-owned, unchanged) → `chain default.ipxe` → FOG's +`custom.ipxe`-or-fallthrough hook → params/menu logic (unchanged) → +`chain boot.php`. + +**Revert-on-failure:** `updatefog.sh`'s `revertUpdate()` → git reset to the +prior commit → `bash installfog.sh -Y $updateVhostFlag --restore-kernel-backup` +→ the same `backupPreservedCustomizations`/`restorePreservedCustomizations` +pair runs, with the extra flag telling it to also restore `gen-1`'s +default-named kernels this one time. + +## Error handling + +- **DB not queryable yet (first-ever install):** `backupPreservedCustomizations()`'s + `SELECT` against a not-yet-created `globalSettings` table errors into + `$error_log` and is treated as "nothing to back up" — matches every other + `[[ -f ]]`-gated defensive step in this file. +- **Kernel backup rotation failing mid-way (disk full, permissions):** each + `mv`/`cp` in the rotation is best-effort and logged; a failed rotation + should not abort the install — `errorStat` is called with the accumulated + status but the install continues (same posture `backupReports`/ + `backupCustomizations` already have today). +- **Vhost managed-block splice on a file with only one marker (corrupted by a + prior partial run or manual edit):** treat as "no valid markers found" and + fall back to appending a fresh block, same as the no-markers-at-all case — + never guess or attempt a partial patch. +- **Known limitation, carried over from `diffconfig`'s existing behavior, now + narrowed in scope:** a hand edit made *inside* the FOG-managed block is + still lost on the next regeneration — the managed-block mechanism protects + content *outside* the block, not inside it. `diffconfig`'s "Changed + configurations" notice still can't distinguish "FOG changed its own block + because a flag was passed" from "an admin's in-block edit is about to be + lost" — both look identical. This is explicitly called out in + `docs/SUPPORTED_CUSTOMIZATIONS.md` rather than silently accepted. +- **Secure Boot key rotation across a kernel-backup restore + (`bin/restorekernel.sh`):** if the signing key active today differs from + the one active when the chosen generation was captured, the restored + kernel's existing signature won't `sbverify` against today's cert, and + `_resignKernels()`-equivalent logic will attempt to re-sign it with + *today's* key — correct behavior, but flagged as an edge case worth a + console message ("re-signing restored kernel with current Secure Boot key"). +- **`custom.ipxe` present but malformed:** iPXE's own script error handling + applies (same as any malformed `.ipxe` file today) — `default.ipxe`'s + `chain custom.ipxe || goto fog_default` only catches a *failed chain* + (fetch/parse failure), not a script that loads fine but does something the + admin didn't intend. Not a new failure mode this design introduces. + +## Testing + +No CI framework exists for this repo's shell scripts beyond +`fogproject-install-validation`'s distro matrix — same posture as every prior +plan in this repo: `bash -n` for syntax, then manual verification on a real +VM. + +- **bg.png rename:** set `FOG_IPXE_BG_FILE` in the GUI to a real, differently + named file already placed under `service/ipxe/`; run `installfog.sh` + directly (not via `updatefog.sh`); confirm the renamed file still exists + under its real name afterward, and `FOG_IPXE_BG_FILE`'s value is unchanged. +- **Vhost managed block:** hand-append a distinctive comment/directive after + today's vhost content; re-run `installfog.sh -Y`; confirm the FOG-managed + region refreshed (e.g. reflects a new `--extra-server-name`) while the + hand-appended content is still present, byte-for-byte, after it. +- **`--overwrite-vhost`:** confirm it discards the hand-appended content + (documented, expected). +- **Kernel generations:** run `installfog.sh` three times in a row (three + distinct "updates"); confirm `gen-1`/`gen-2`/`gen-3` each hold a distinct + kernel set (verify via the `tag_name` xattr), and a fourth run correctly + evicts what was `gen-3`. +- **Custom-named kernel:** set a test host's `kernel`/`init` fields to a + hand-placed file under `service/ipxe/`; run an update; confirm the file is + still present and correct afterward, with no dedicated flag/setting having + been added for it. +- **`custom.ipxe` hook, absent:** confirm PXE boot behaves identically to + before this change when no such file exists. +- **`custom.ipxe` hook, present:** place a minimal script (e.g. `prompt` + with a timeout) at the TFTP root; confirm it runs before FOG's own menu, + and that normal boot resumes afterward. +- **Secure Boot admin-supplied pair:** run `installfog.sh --secure-boot-key + ... --secure-boot-cert ...` twice in a row; confirm the second run is a + no-op copy-wise (already at destination) and kernels remain correctly + signed both times; separately, confirm a path *inside* `$webdirdest` is + still copied to safety before the first wipe (regression test for the + specific gap this design closes). +- **Revert path:** force an `updatefog.sh` failure; confirm `gen-1`'s + default-named kernels are restored (via `--restore-kernel-backup`) in + addition to the always-restored customizations. +- **Regression:** an install/update with none of these customizations present + behaves identically to before this design in every other respect. + +## Open risks / unknowns + +- The vhost managed-block append-on-first-encounter behavior (no markers + found → append rather than replace) is a one-time visible change for any + *existing* install upgrading into this feature — worth a release-notes + callout, not a design flaw, but flagged here so it isn't mistaken for a bug + report later. +- `bin/restorekernel.sh --generation N`'s interaction with an already-running + web server (do live-serving files need the web server briefly stopped, or + is an atomic `cp`+`mv` into place sufficient?) needs verification on a real + VM — not resolved by static reading of this codebase alone. +- The eventual "right" home for the `custom.ipxe` hook is arguably upstream in + `FOGProject/fog-ipxe`'s own `autoexec.ipxe`, once that project's release + cadence allows it — this design's placement in `default.ipxe` is chosen + specifically to avoid that dependency, but it should be revisited if/when + fog-ipxe adds its own hook convention, to avoid two parallel mechanisms. +- Whether `$tftpdirdst` is ever fully re-created from empty (rather than + merged into) on some distro/path combination this reading didn't cover — + if so, `custom.ipxe`'s "structurally safe by construction" claim in + Architecture §4 would need re-verification on that path specifically. diff --git a/docs/superpowers/specs/2026-08-07-three-zone-pki-separation-design.md b/docs/superpowers/specs/2026-08-07-three-zone-pki-separation-design.md new file mode 100644 index 0000000000..bee344bdef --- /dev/null +++ b/docs/superpowers/specs/2026-08-07-three-zone-pki-separation-design.md @@ -0,0 +1,988 @@ +# Three-zone PKI separation (Web TLS / Client Communication / Secure Boot) + +Part of [FOGProject/fogproject#1014](https://github.com/FOGProject/fogproject/pull/1014). + +## Context + +FOG's installer today mints exactly one self-signed CA (`createSSLCA()`, +`lib/common/functions.sh:3412-3534`, `.fogCA.key`/`.fogCA.pem`, default +CN="FOG Server CA" — literally the sixth answer in the `openssl req -x509` +heredoc at `functions.sh:3432-3440`, confirmed directly in code, not just +asserted by the maintainer) and uses it for two roles at once: + +1. **Signs the web vhost's leaf cert** (`srvpublic.crt`, `$sslpubcert`) — what + Apache/Nginx serves over HTTPS. +2. **Is exported verbatim as `ca.cert.der`/`ca.cert.pem`** + (`functions.sh:3520-3521`) — what fog-client downloads from + `/management/other/ca.cert.der` and pins. Per + `docs/EXTERNAL_CA_AND_LETSENCRYPT.md`, fog-client does not do standard OS + chain validation; it adds *only* this exact certificate to its trust store + and requires the server's leaf to chain to it directly. + +`--external-ca`/`--ca-cert`/`--ca-key`/`--ca-root` (`validateExternalCA()`, +`functions.sh:3271-3346`) already lets an admin swap role (1) for their own +intermediate — but because roles (1) and (2) still share one variable/file +(`$sslcapem`), swapping the web-signing CA **always** also swaps the +fog-client-trust CA. + +Secure Boot is a **third** job, currently done by a keypair that is separate +from the flat CA but has its own, different structural problem: +`_ensureSecureBootKeys()` (`functions.sh:4625-4694`) generates +`$fogprogramdir/secureboot/MOK.{key,pem}` — **self-signed, `basicConstraints +CA:FALSE`**, CN="FOG Project Secure Boot Signing", `codeSigning` EKU — +independently of `$sslcapem`/`$sslcakey` entirely. `_resignKernels()` +(`functions.sh:5049-5094`) calls `sbsign --key "$secureBootKey" --cert +"$certpem"` directly against that self-signed leaf, and +`_publishSecureBootKit()` (`functions.sh:4781-4832`) publishes **the same +certificate** as `MOK.der` for enrollment into client firmware. + +That double duty is the problem. Because the enrolled MOK *is* the signing +certificate, and it is a leaf that can issue nothing: + +- Rotating or revoking the signing key means **physically re-enrolling every + client** through MokManager — a firmware-level trip to each machine. +- A storage node cannot sign kernels at all without being handed the one and + only signing private key that the entire fleet's firmware trusts. + +So Secure Boot gets the same treatment as the other two zones in this +design: the Root CA issues a **FOG Secure Boot CA** intermediate, *that +intermediate's* public certificate becomes `MOK.der` (the thing enrolled in +firmware), and the intermediate issues short-lived **code-signing leaves** — +one for the master server, one per storage node. `sbsign` bundles the +intermediate alongside the leaf (`--addcert`) so shim can build the chain +back to what firmware trusts. Leaves can then be rotated, revoked, or issued +to new nodes with **no firmware re-enrollment**, and the bring-your-own story +becomes symmetric with the other zones: an admin issues their own +Secure-Boot intermediate from their own root and hands FOG a leaf. + +This is a change from an earlier revision of this design, which scoped Secure +Boot out as "already separate, leave it alone." The isolation observation was +correct; the conclusion was wrong. Being separate from the *flat CA* is not +the same as being *well structured*, and the flat-leaf-as-MOK shape is what +makes Secure Boot key management operationally painful today. + +**This restructuring carries the highest endpoint cost of any zone in this +design** (firmware re-enrollment, per the maintainer's own trust-zone table) +and therefore applies to **fresh installs only**; an existing server's +already-enrolled MOK is never regenerated or replaced — see Backward +compatibility and Open Risks. + +### The coupling that actually needs fixing (found during this design, not +### assumed from the maintainer's brainstorm) + +`certDecrypt()` (`fogbase.class.php:2011-2049`) is called from exactly one +place: `FOGPage::authorize()` (`fogpage.class.php:2712` ff.), which runs on +every fog-client checkin authorization handshake. It decrypts the client's +freshly-generated session material (`sym_key`, `token`) using +`openssl_private_decrypt()` against `.srvprivate.key` read straight off disk — +literally `$sslprivkey`, the same file `createSSLCA()` writes as the vhost +leaf's private key, and (until it is removed — see Non-goals) the same file +`bin/setupacme.sh --install-cert --key-file "$sslprivkey"` **overwrote on +every ACME leaf renewal**. + +So: only the client→server direction is coupled to the vhost's own TLS +keypair, but it is coupled *by file identity*, not just by CA. A genuine +"Client Communication" trust zone needs **its own dedicated leaf keypair**, +not just its own CA, decoupled from whatever the vhost's TLS listener is +currently using. This is the single most important correction this design +makes to the maintainer's brainstorm: "give Client Communication its own CA" +is necessary but not sufficient; `certDecrypt()`'s key source must also move +off `$sslprivkey`. + +#### What is proven, and the one thing that is not + +Proven from this repo's code, no inference involved: + +- `.srvprivate.key` is generated by `openssl genrsa -out $sslprivkey` + (`functions.sh:3478`) as a **separate keypair from the CA** + (`functions.sh:3431`), and its public half is what ends up in + `srvpublic.crt` — the CSR at `:3492` is built from it and signed by the CA + at `:3515`. It is the **web server leaf key**. +- `ca.cert.pem`/`ca.cert.der`, the files fog-client downloads, are a copy of + `$sslcapem` — the **CA certificate** (`functions.sh:3520-3521`). +- `certDecrypt()` reads `/.srvprivate.key` + (`fogbase.class.php:2034-2042`) and RSA-decrypts with it + (`openssl_private_decrypt`, `:2065`). +- `authorize()` calls it on every fog-client handshake to recover `sym_key` + and `token` (`fogpage.class.php:2730`). + +**Confirmed on a live server:** `.srvprivate.key` is present and is the web +leaf's key. (An earlier round of this discussion suspected the file was +absent entirely — every file in `$sslpath` is a **dotfile**, so a bare `ls` +shows only `fog.csr`, `req.cnf` and `ca.cnf` and makes the directory look +empty of key material. `ls -la` shows otherwise.) **The coupling is real**: +the private key backing the web server's TLS certificate is also the key +that decrypts every fog-client authorization handshake, and any process +that legitimately replaces the web leaf — an ACME renewal, a `--recreate-keys` +run, dropping in a purchased certificate — silently breaks client +authentication. + +So the fix is the one the maintainer describes: **the FOG Server CA issues +its own communication TLS certificate.** Same principle as the other zones, +just one more leaf — one that is never shared with the web vhost. +`certDecrypt()` reads that leaf's key; the vhost keeps using its own. Two +independently replaceable certificates where today there is one file doing +both jobs. + +**The one remaining unknown is narrower, and it is about delivery, not +design:** how does fog-client obtain the public key it encrypts with? The +evidence says it is *not* the CA — the CA and leaf are different keypairs +and RSA cannot decrypt across a mismatch — yet the client demonstrably +works over plain HTTP, so it is not reading the key off a TLS handshake +either. The most probable answer is that it fetches the leaf certificate +that `createSSLCA()` deliberately publishes into the **web-served** +directory at `$webdirdest/management/other/ssl/srvpublic.crt` +(`functions.sh:3509,3518`) — sitting right next to the `ca.cert.der` it +pins, which is exactly where a client would look for both halves of this +scheme. + +If that is right, this decoupling needs **no fog-client change at all**: +the comm leaf simply gets published at the path the client already fetches, +while the vhost's own certificate moves to the Web zone's path. That would +be the ideal outcome, and it is plausible enough to design toward — but it +is precisely the kind of cross-repo assumption the `certDecrypt()` finding +itself proves is worth checking rather than inheriting. Phase 0 Task 0.3 +confirms the fetch path before Task 1.4 is written. + +### Sanity-checking the CN-pinning claim + +The maintainer states fog-client hardcodes an expectation around the pinned +cert's CN ("FOG Server CA"), does not do standard OS chain validation, and +that a replacement intermediate CN must match exactly for existing client +binaries to keep working without a source change. **This repo contains no +fog-client/zazzles source — this claim cannot be verified here.** What *can* +be verified, and is a meaningful data point: the existing self-signed default +(`functions.sh:3438`) has always minted its CA with `CN=FOG Server CA` — this +is not new information invented for this design; it is what every stock FOG +install has produced for years. That is circumstantial support that +*something* about that literal string matters, but it does not distinguish +between two very different pinning mechanisms with very different migration +consequences: + +- **Byte-identical whole-certificate pinning** (what + `docs/EXTERNAL_CA_AND_LETSENCRYPT.md:60-68` explicitly describes today: + "the client adds *only* `ca.cert.der`... and requires that exact certificate + to appear in the server's chain") — CN is irrelevant to the check; a + same-CN *replacement* cert with a new key is just as untrusted as a + different-CN one, until the client re-pins the new bytes. +- **CN-substring/string matching** — a replacement cert with the same CN + might be accepted by some validation paths without the client ever fetching + new bytes, which would be a much weaker (and frankly, surprising) model for + a security control, but is what the maintainer's comment literally + describes ("could be swappable as long as the CN of whatever intermediate + you issue has CN='FOG Server CA'"). + +**This design treats the distinction as unresolved and requires it be +verified against the actual `zazzles`/fog-client source before Phase 2 (the +client-facing rotation work) starts** — see Task-Plan Phase 0. Critically, +the blast radius of guessing wrong is contained to exactly **one** place: a +single shell variable, `$fogClientCACN` (default `"FOG Server CA"`), defined +once in `lib/common/functions.sh` and referenced everywhere a client-CA +subject is written or validated. No function hardcodes the literal string a +second time. If verification finds the real requirement is a different +string, a different field (O instead of CN), or "byte-identical only, CN is +irrelevant" — the fix is confined to that one definition and the one +validation call site described in Components below, not scattered across +`createClientIntermediateCA()`, `validateExternalCA()`, docs, and CLI help +text independently. + +## Non-goals + +- Not re-enrolling, rotating, or invalidating any **already-enrolled** Secure + Boot MOK. An existing server keeps its current self-signed MOK keypair + untouched forever (`_ensureSecureBootKeys()`'s existing "never regenerate" + guarantee is preserved verbatim); the intermediate model applies to fresh + installs only. There is deliberately **no** migration path offered for + Secure Boot equivalent to Phase 2's fog-client re-pinning — the remedy for + an existing server that wants the new structure is to enroll the new + intermediate as an *additional* MOK, which is a documented admin + procedure, not an installer action. +- Not issuing per-storage-node Secure Boot leaves **automatically** in Phase + 1. The intermediate model makes per-node leaves *possible* (and that is a + primary reason for adopting it); actually provisioning them across nodes + needs the storage-node install path and the node-registration flow to carry + a CSR round-trip, which is its own scoped work — see Phase 3. +- Not extending per-location protocol/PKI selection into the Location plugin. + That plugin already has a per-location `protocol` override + (`packages/web/lib/plugins/location/hooks/changeitems.hook.php:118`) that + this design's protocol work should stay compatible with, but multi-site + PKI is explicitly later work. +- Not modifying `zazzles`/fog-client source. Nothing in this design can be + implemented against real fog-client binaries without that repo's + cooperation for the CN-verification step and (if the pessimistic case in + Open Risks holds) a client-side "re-fetch and re-pin without a full + re-register" capability that may not exist today. +- Not forcing any existing install through this restructuring. + `updatefog.sh`'s default in-place update path is unchanged for every server + that does not explicitly opt in — this applies to servers that already + have cert material (`caCreated == yes`). It does **not** apply to fresh + installs: those now default to `split` (see Architecture/Components) — a + deliberate default change, not an exception to this non-goal. +- Not making the legacy flat-CA mode second-class or transitional. + `--legacy-pki` (or the interactive prompt's legacy choice) produces exactly + today's single self-signed CA, byte-for-byte. It is a permanent, fully + supported, lower-overhead alternative for admins who don't want the + three-zone split, not a deprecated fallback being phased out. +- Not re-litigating `--extra-server-name`/`--hostname` — those are + already-merged, additive, and orthogonal; this design builds around them + without changing their behavior. +- **Removing `bin/setupacme.sh` entirely — ACME/Let's Encrypt automation is + not a FOG-managed feature, full stop, not even in its current + already-merged, CA-only-touching form.** On reflection (this was + reconsidered mid-design, not the original plan), FOG should not own any + ACME client integration at all. It is simple enough for an admin who wants + a renewing/public certificate to run `certbot`/`acme.sh` themselves and + drop the resulting cert/key into the Web zone's leaf paths (`$sslpubcert`/ + `$sslprivkey`) — a safe drop-in once the paired customization-preservation + design's managed-block vhost splice has landed (see + `docs/superpowers/specs/2026-08-07-customization-preservation-design.md`), + because FOG's vhost regeneration will never clobber it. `bin/setupacme.sh` + is deleted as part of this design (Task-Plan Phase 1, Task 1.6), and + `docs/EXTERNAL_CA_AND_LETSENCRYPT.md`'s "Automating renewal with + setupacme.sh" subsection is replaced with this self-service guidance in the + new `docs/PKI_ZONES.md`. A future FOG **plugin** offering a GUI-level + "enable Let's Encrypt" toggle is a plausible later feature once the + three-zone split has settled and proven itself — explicitly out of scope + for this design, noted only as a forward-looking possibility in Open + Risks, not designed toward now. +- Not deprecating `--external-ca`/`--ca-cert`/`--ca-key`/`--ca-root` this + release. They are reinterpreted as "the web zone's" import flags (their + exact current behavior, unchanged), with new `--client-ca-*` flags added + alongside for the new zone. A deprecation timeline is a Phase 3 decision, + not a Phase 1 one. +- Not designing FOG's separate, publicly-managed installer-signing CA + (expires 2029, used by fog-client auto-update to verify installer + signatures) — explicitly out of scope per the maintainer's own note. + +## Architecture + +### Directory layout + +Everything new lives under the existing `$sslpath` +(`$fogprogramdir/snapins/ssl` by default — already outside `$webdirdest`, +already survives whatever `configureHttpd()` does to the web docroot, exactly +like `$fogprogramdir/secureboot` already does for MOK). Nothing under the +*existing* flat layout (`$sslpath/CA/.fogCA.{key,pem}`, +`$sslpath/CA/.fogCAchain.pem`, `$sslpath/.srvprivate.key`, +`$sslpath/fog.csr`) is renamed, moved, or deleted by this design. The new tree +is additive, in sibling directories that simply do not exist until an admin +opts in: + +``` +$sslpath/ + CA/ + .fogCA.key / .fogCA.pem / .fogCAchain.pem # UNCHANGED flat-mode files; + # present forever, used only + # when $pkiMode != split + root/ + .fogRootCA.key # 0600 root:root; documented offlining path, not + # required from day one -- see Components + .fogRootCA.pem # CN "FOG Server ROOT CA", CA:TRUE, pathlen:1, ~20y + .fogRootCA.srl + web/ + .fogWebCA.key + .fogWebCA.pem # CN "FOG Web CA" (or admin's own import) + .fogWebCAchain.pem # root + web intermediate (+ admin's own root, + # if --web-ca-root was used instead of FOG's root) + client/ + .fogClientCA.key + .fogClientCA.pem # CN "$fogClientCACN" (default "FOG Server CA") + # -- issues NOTHING itself. This cert's own bytes + # are exactly what's exported as ca.cert.der. + comm/ + .commLeaf.key # THE fix for the certDecrypt() coupling above. + .commLeaf.pem # Leaf issued BY .fogClientCA, never touched by + # a web-TLS recreate. authorize()/certDecrypt() + # point here, not at $sslprivkey. + .srvprivate.key / fog.csr # UNCHANGED flat-mode files (symlink targets) +``` + +The Secure Boot zone stays in its existing home, `$fogprogramdir/secureboot/` +— it is not moved under `$sslpath`. Two reasons: that directory is already +`0700 root:root` with a deliberate "the web user must never read this" +separation enforced through the `fog-sign-kernel` sudo helper +(`functions.sh:4650-4655`), and every existing install already has files +there that must keep working untouched. What changes is what lives beside +them in `split` mode: + +``` +$fogprogramdir/secureboot/ + MOK.key / MOK.pem # flat/legacy: self-signed leaf, BOTH signer and + # enrolled MOK. UNCHANGED for existing installs. + ca/ + .fogSBCA.key # split: the FOG Secure Boot CA intermediate, + .fogSBCA.pem # issued by the Root. ITS cert becomes MOK.der. + .fogSBCA.srl + leaf/ + sign.key # split: this server's short-lived code-signing + sign.pem # leaf, issued by .fogSBCA. What sbsign uses. + nodes/ # split, Phase 3: one issued leaf per storage node + PK.key / PK.pem # UNCHANGED -- platform keys, unrelated to this split + KEK.key / KEK.pem # UNCHANGED +``` + +### Diagram 1 — default FOG PKI (everything FOG-generated) + +```mermaid +graph TD + Root["FOG Server ROOT CA
self-signed · CA:TRUE pathlen:1 · ~20y
offline-able (Phase 3 export helper)"] + + Root --> WebCA["FOG Web CA
intermediate"] + Root --> ClientCA["FOG Server CA
intermediate · CN pinned by fog-client"] + Root --> SBCA["FOG Secure Boot CA
intermediate"] + + WebCA --> WebLeaf["Web server certificate
$sslpubcert / $sslprivkey
SANs: IPs + hostname + extra names"] + ClientCA --> Pin["ca.cert.der
(the intermediate's own bytes,
pinned by fog-client)"] + SBCA --> MOK["MOK.der
enrolled ONCE in client firmware"] + SBCA --> SignLeaf["code-signing leaf
master FOG server"] + SBCA --> NodeLeaf["code-signing leaf
per storage node (Phase 3)"] + + SignLeaf --> Kernels["sbsign --addcert <intermediate>
FOS kernels"] + NodeLeaf --> Kernels + + Rotate["Rotate / revoke a signing leaf
= NO firmware re-enrollment
(firmware trusts the intermediate)"] + SBCA -.-> Rotate + + style Rotate stroke-dasharray: 5 5 +``` + +### Diagram 2 — drop-in options (mix and match per zone) + +Each zone is independently replaceable. Nothing forces an admin to replace +all three, or any. + +```mermaid +graph TD + subgraph FOGPKI["FOG-generated (default)"] + FRoot["FOG Server ROOT CA"] + FRoot --> FClient["FOG Server CA
(client comm)"] + end + + subgraph YourPKI["Your existing PKI (AD CS, step-ca, ...)"] + YRoot["Your ROOT CA"] + YRoot --> YSB["Your Secure Boot
intermediate CA"] + YRoot --> YWebLeaf["Your web server cert"] + YSB --> YSign["code-signing leaf
handed to FOG"] + end + + subgraph PublicPKI["Public CA"] + LE["Let's Encrypt
via YOUR acme.sh / certbot
(FOG never automates this)"] + LE --> LELeaf["Web server cert
FQDN only"] + end + + FClient --> Pin["ca.cert.der
pinned by fog-client
-- keep this FOG's, or mint your own
with CN=FOG Server CA"] + YSign --> Sign["FOS kernel signing"] + YSB --> Enroll["MOK.der enrolled in firmware"] + + YWebLeaf --> Vhost["Apache / nginx vhost"] + LELeaf --> Vhost + + Vhost --> WebOK["HTTPS: browsers, fog-client,
API -- trusted if the issuing CA
is in the client's trust store"] + Vhost --> IPXE{"iPXE netboot
HTTPS?"} + IPXE -->|"public CA only
(ca.ipxe.org crosscert)"| IPXEyes["works"] + IPXE -->|"FOG PKI or your internal PKI"| IPXEno["NOT trusted --
netboot stays HTTP"] + + style IPXEno stroke-dasharray: 5 5 +``` + +**Caveats the diagram encodes, stated plainly:** + +- A FOG-PKI or internal-PKI web certificate **can** carry IP addresses and + DNS aliases as SANs (this already works — `--extra-server-name` and the + existing SAN loop), so HTTPS is trusted for browsers, fog-client, and API + traffic on any of those names once the CA is in the client trust store. + **It is still not trusted for iPXE netboot**, because iPXE has no path to + that CA — its only fallback is the `ca.ipxe.org` cross-signing service, + which only bridges *public* roots. +- **Let's Encrypt does work for Secure-Boot iPXE netboot**, but only on an + FQDN in a domain you control (it need not be publicly reachable — DNS-01 + validation is enough), and **only on that exact FQDN** — not the short + hostname, not an IP address. `FOG_WEB_HOST` must be set to that FQDN + accordingly, which is what makes the generated boot URLs match the + certificate. This is confirmed by ad-hoc testing, not just inference (see + Open Risks). + +`$sslpath/CA/root` holding the root **key** on-server is the *initial* +default (protect it with strict perms, document/nudge toward offlining, +provide an explicit command to do so) rather than requiring an offline +HSM/vault on day one, which would make the default install harder for the +common case with no security benefit for an admin who never intended to run +a real offline root anyway. + +One nuance Diagram 1 deliberately simplifies: whether the Client +Communication intermediate issues a separate `.commLeaf` at all, or is +itself the comm keypair, depends on Phase 0's verification of fog-client's +behavior — the Client CA issues its own comm TLS certificate; only *where +that certificate is published* is still to be confirmed (Phase 0 Task 0.3). + +### Zone → file/variable mapping + +| Zone | CA files | Leaf/comm files | Consumed by | +|---|---|---|---| +| Web TLS | `$sslpath/CA/web/.fogWebCA.{key,pem}`, `.fogWebCAchain.pem` | `$sslpubcert`/`$sslprivkey` (unchanged names/paths — still what the vhost config writes and what an admin-managed ACME/certbot process drops a renewed cert into) | Apache/Nginx vhost, iPXE's `TRUST=` build, browsers | +| Client Communication | `$sslpath/CA/client/.fogClientCA.{key,pem}` | `$sslpath/CA/client/comm/.commLeaf.{key,pem}` | fog-client (pins `.fogClientCA.pem` as `ca.cert.der`; encrypts `authorize()` payloads against `.commLeaf.pem`'s public key); `certDecrypt()`/`certEncrypt()` (server side, reads `.commLeaf.key`) | +| Secure Boot (`split`) | `$fogprogramdir/secureboot/ca/.fogSBCA.{key,pem}` — its cert is published as `MOK.der` | `$fogprogramdir/secureboot/leaf/sign.{key,pem}` (master), `nodes/.{key,pem}` (Phase 3) | `_resignKernels()` signs with the **leaf** + `--addcert` intermediate; mokutil/shim on endpoints enroll and trust the **intermediate** | +| Secure Boot (`flat`/legacy, and every existing install) | *(unchanged)* `$fogprogramdir/secureboot/MOK.{key,pem}` — self-signed leaf, both signer and enrolled MOK | — | `_resignKernels()`, mokutil/shim, exactly as today | + +The Secure Boot rows are where the split is most visible in code: today a +single variable, `$secureBootCert`, is simultaneously *what signs kernels* +and *what gets enrolled in firmware*. In `split` mode those become two +different certificates — `$secureBootCert`/`$secureBootKey` keep their +meaning as **the signing leaf**, and a new `$secureBootMokCert` names **the +intermediate to enroll**. In `flat` mode the new variable simply points at +the same file as `$secureBootCert`, so every downstream consumer +(`_publishSecureBootKit()`, the enrollment kit, the Secure Boot web page) +reads one variable and behaves identically in both modes without branching. + +The Client Communication zone publishes **two** files, mirroring what the +flat model already publishes side by side today: + +- `ca.cert.der` — `.fogClientCA.pem`'s bytes, the pinned trust anchor. + Unchanged export mechanics (`functions.sh:3520-3521`), new source file. +- The **comm leaf's** public certificate — what the client actually + RSA-encrypts `sym_key`/`token` against. This is the file that replaces + today's dual-purpose web leaf in the client-auth path. + +Today those two roles are filled by `ca.cert.der` and +`management/other/ssl/srvpublic.crt` respectively, both already published +into the web-served `management/other/` directory. The decoupling keeps that +*shape* exactly — two files, same directory, same names — and changes only +which key material backs the second one: the comm leaf issued by the Client +CA, instead of the web vhost's TLS leaf. The vhost's certificate moves to +the Web zone and stops being web-published at all. + +That is what makes this change plausibly invisible to fog-client: the +filenames and locations it fetches do not move, only the key behind one of +them. Confirming the client genuinely fetches that path (rather than +deriving a key from `ca.cert.der`) is Phase 0 Task 0.3's job — see Open +Risks. If it turns out the client does derive its encryption key from +`ca.cert.der` itself, the fallback is for `.fogClientCA` to double as the +comm keypair, which still achieves the decoupling from the web leaf, just +without a separate comm certificate. + +### `--external-ca` reinterpreted + +`--external-ca`/`--ca-cert`/`--ca-key`/`--ca-root` keep their exact current +behavior and become, unambiguously, **the Web zone's** import path (their +target directory moves from `$sslpath/CA/` to `$sslpath/CA/web/`, but nothing +about validation, chaining rules, or the CLI contract changes). New +`--client-ca-cert`/`--client-ca-key`/`--client-ca-root` flags, validated by +the *same* `validateExternalCA()` logic parameterized by zone +(`validateExternalCA web` / `validateExternalCA client`), let an admin bring +their own CA for the Client Communication zone instead — e.g. an AD CS +sub-CA, per the maintainer's "enterprise drop-in" scenario — with one extra +check this zone requires that Web does not: the imported cert's Subject CN is +compared against `$fogClientCACN` and a **loud warning** (not a hard failure +— an admin who knows what they're doing may be intentionally testing whether +the CN actually matters) is printed if it does not match. A third, +independent `--root-ca-*` set of flags (import or generate-and-later-export) +governs the Root, orthogonal to both. + +### Choosing a scenario: CLI flags AND an interactive prompt + +Every zone/import decision above is driven by a small, fixed set of staging +variables (`pkiMode`, and per-zone `--web-ca-*`/`--client-ca-*`/`--root-ca-*` +paths) — this shape is deliberately chosen so it can be populated two ways, +not just one: + +- **Non-interactively**, via the CLI flags described above and in + Task-Plan Phase 1 (for scripted/`-Y` installs, and for `updatefog.sh` + pass-through). +- **Interactively, on a fresh install**, via a new prompt in + `lib/common/newinput.sh`, gated exactly like the existing `hostname` prompt + is (`lib/common/newinput.sh:17-35`): only runs when `pkiMode` isn't already + set via a flag or a persisted `.fogsettings` value, and only under a real + interactive session (never under `-Y`). + +The prompt offers three scenarios in plain language — (1) split PKI with +FOG-generated root+intermediates, **the new default** (pressing Enter with no +answer selects this, matching the non-interactive default described under +Components below); (2) split PKI, bring-your-own CA per zone, which then +prompts for cert/key/root paths per zone the admin chooses to override, +reusing the exact same `validateExternalCA(zone)` validation the +non-interactive flags already run; (3) **legacy** — today's single +self-signed CA, byte-for-byte unchanged, offered as a permanent, explicit, +lighter-weight alternative, not a deprecated fallback. Selecting any of the +three simply populates the same `pkiMode`/`--*-ca-*` staging variables a flag +would — there is no separate code path for the interactive answer, it is +purely an alternate way to fill in the same variables, applied in the same +"after `.fogsettings` sourced" block every other staged value already uses. + +This mirrors exactly how `hostname` already works today (prompted +interactively if not given via `--hostname`, silently reused from +`.fogsettings` on every later run) — no new pattern is introduced, this +design just extends that existing pattern to the PKI scenario choice, with a +sensible default pre-selected instead of requiring an explicit answer. + +```mermaid +flowchart TD + Start["installfog.sh run"] --> Existing{"Does this server already
have cert material?
(caCreated == yes)"} + Existing -->|yes| Keep["Keep its existing pkiMode
unchanged -- flat stays flat,
split stays split.
NEVER silently restructured."] + Existing -->|"no -- fresh install"| Picked{"pkiMode chosen
via flag or prompt?"} + Picked -->|"nothing given / Enter / option 1"| Split["pkiMode=split
(NEW DEFAULT)"] + Picked -->|"--legacy-pki / prompt option 3"| Flat["pkiMode=flat
(today's single self-signed CA,
byte-for-byte, permanent option)"] + Picked -->|"option 2 / --web-ca-*, --client-ca-*, --root-ca-*"| BYO["pkiMode=split, per-zone bring-your-own
via validateExternalCA(zone)"] + Split --> Generated["createRootCA() + createWebIntermediateCA()
+ createClientIntermediateCA()
-- fully automatic, no prompts needed"] + BYO --> Generated + + Keep -.->|"admin explicitly wants to migrate
an existing flat server later"| Migrate["--restructure-pki
(confirmation-gated, Phase 2 only --
blocked on fog-client verification)"] + + LEnote["Admin-run certbot/acme.sh,
any pkiMode -- drops a renewed
leaf into $sslpubcert/$sslprivkey.
FOG never automates this."] + Generated -.-> LEnote + Flat -.-> LEnote + + style Migrate stroke-dasharray: 5 5 + style LEnote stroke-dasharray: 5 5 +``` + +### Certificate path indirection: canonical paths, real files anywhere + +Admins need certificates to live outside FOG's directories — `/etc/pki/...`, +`/etc/letsencrypt/live/...`, a mounted secrets volume. But the vhost, the +PHP side, and `sbsign` all need *stable* paths, or every path change becomes +a config rewrite. The resolution: **FOG always reads and writes canonical +paths; those canonical paths may be symlinks to wherever the real file +lives.** The vhost then never changes when an admin relocates a +certificate — which pairs directly with the paired +customization-preservation design's managed-block vhost, since a cert +relocation stops being a vhost edit at all. + +**This mechanism already half-exists** and is worth completing rather than +inventing: `functions.sh:3497-3500` already links `$sslcakey`, `$sslcapem`, +`$sslcsr` and `$sslprivkey` into canonical `$sslpath` locations, precisely so +those variables can point elsewhere. Two problems with it as written: + +- **Two of the four are broken.** Lines 3497-3498 test one path and link + another: + ```bash + [[ ! -e $sslpath/.fogCA.key ]] && ln -sf $sslcakey $sslpath/CA/.fogCA.key + ``` + The guard checks `$sslpath/.fogCA.key`; the link is created at + `$sslpath/CA/.fogCA.key`. On a default install `$sslcakey` *is* + `$sslpath/CA/.fogCA.key`, so this reduces to `ln -sf X X` — GNU `ln` + refuses ("are the same file") and logs to `$error_log`, so it is harmless + today, but the intended canonical link at `$sslpath/.fogCA.key` is never + created. Lines 3499-3500 (`fog.csr`, `.srvprivate.key`) test and link the + same path and are correct. +- **It is driven only by `.fogsettings`**, not by anything an admin can set + without SSH. + +What this design adds: + +1. **Fix the two mismatched guards** so all four canonical links behave + consistently, and extend the same pattern to every new path this design + introduces (the three intermediates, the SB leaf, `.commLeaf`). +2. **Keep FOG's own consumers pointed exclusively at canonical paths.** The + vhost, `_resignKernels()`, `_publishSecureBootKit()`, and `certDecrypt()` + all reference the canonical location; whether that is a real file or a + symlink is invisible to them. An admin dropping a renewed Let's Encrypt + certificate in place either writes to the canonical path or points the + symlink at their own — both work, neither touches the vhost. +3. **Surface the paths as settings.** Note that one already is: + `certDecrypt()` resolves its directory from the **storage node's `sslpath` + database column** (`fogbase.class.php:2019-2023`), which is already + GUI-editable on the Storage Node form (`SSLPath`, + `packages/web/commons/text.php:311`). That is the precedent to follow — + `.fogsettings` stays the installer's source of truth, and the + GUI-visible values are records of it, exactly as `FOG_GIT_PATH` and + `FOG_EXTRA_SERVER_NAMES` already are. + +**Caveats to document rather than discover:** a certificate outside the +distro's expected directories may be blocked by SELinux even through a +symlink (the *target's* context is what matters, not the link's) — an admin +relocating certs on a RHEL-family box may need `restorecon`/`semanage +fcontext` on the real path. And the private key's ownership and mode must +survive relocation: `_ensureSecureBootKeys()`'s `0600 root:root` and the +`fog-sign-kernel` sudo helper's separation model assume the web user cannot +read the key, and a symlink into a world-readable location silently defeats +that. + +### Protocol selection: HTTPS everywhere vs. netboot-stays-HTTP + +Which PKI backs the **web** certificate determines how much of FOG can +safely be HTTPS, because iPXE's trust story is narrower than every other +consumer's: + +| Web cert issued by | Web UI / API / fog-client | iPXE netboot (`boot.php`, kernel, init) | `httpproto` default | +|---|---|---|---| +| **Public CA** (Let's Encrypt et al.) | HTTPS — natively trusted | **HTTPS works** via iPXE's `ca.ipxe.org` crosscert, FQDN only | `https` everywhere (today's behavior) | +| **FOG PKI** (this design's default) | HTTPS once the FOG root is in the client trust store; SANs may include IPs and aliases | **Not trusted** — no crosscert path for a private root | `https` for web, **`http` for netboot** | +| **Your internal PKI** (AD CS, step-ca) | HTTPS once your root is in the client trust store | **Not trusted** — same reason | `https` for web, **`http` for netboot** | + +Today `httpproto` is a single global that forces all of these together, +which is why enabling HTTPS with a private CA historically meant rebuilding +iPXE with `TRUST=` baked in (`configureTFTPandPXE()`, +`functions.sh:1202-1215`) — and that rebuild is exactly what forfeits the +signed Secure Boot shim. Splitting the two lets a FOG-PKI or internal-PKI +server have a properly trusted HTTPS web UI **and** keep the stock signed +shim, at the cost of netboot fetches staying on HTTP (a pre-boot +environment on a provisioning VLAN — an acceptable trade, and the same +exposure as today's default HTTP install). + +**What makes this cheap to implement** — a finding from tracing the code +rather than an assumption: `FOGBase::$httpproto` +(`packages/web/lib/fog/fogbase.class.php:481-483`) is derived from **the +current request's** `$_SERVER['HTTPS']`, not from a stored setting. Every +boot-menu URL `bootmenu.class.php` emits (`:286` `$this->_web`, `:292` +`boot-url`, `:458` `$this->_booturl`) inherits the protocol iPXE actually +connected with. So if iPXE reaches `boot.php` over HTTP, the entire +generated menu — kernel and init fetches included — is already HTTP, with +**no PHP change required**. Only two things must change: + +1. **`configureDefaultiPXEfile()`** (`functions.sh:1037-1042`) writes the + `chain ${httpproto}://...boot.php` line. It must use a new, separate + `$netbootproto` variable rather than `$httpproto`. +2. **The vhost's HTTP→HTTPS redirect** (the `$httpproto == https` branches, + `functions.sh:3571` nginx / `:3814` Apache) must **exclude** the netboot + paths (`${webroot}service/ipxe/`), or the redirect drags iPXE straight + back onto HTTPS and defeats the whole arrangement. This exclusion is the + one genuinely fiddly piece of the change and needs testing on both + webserver families. + +`$netbootproto` is a new managed key defaulting to `http` when `pkiMode == +split` (or when a private CA is imported for the web zone) and to +`$httpproto` when the web cert comes from a public CA. An admin can override +it explicitly in either direction — including forcing `https` netboot on a +private CA, which is legitimate if they *also* accept an iPXE rebuild and +the loss of the signed shim (the pre-existing trade-off, now an explicit +choice rather than an implicit consequence of one global). + +## Components + +### `pkiMode` (new managed key, `flat` | `split`) + +Gate for every new code path in this document. The default is computed, not +a fixed constant — it depends on whether this server already has cert +material: + +- **No existing CA yet (`caCreated` not `yes`) — a genuinely fresh + install:** defaults to **`split`**. This is the new default as of this + design — a fresh install gets the three-zone PKI automatically, with no + flag or prompt answer needed. `--legacy-pki` (non-interactive) or the + interactive prompt's legacy choice opts back into `flat` — today's single + self-signed CA, byte-for-byte — as a permanent, fully supported, + lighter-weight alternative, not a deprecated fallback (see Non-goals). +- **An existing CA already exists (`caCreated == yes`) and `.fogsettings` + has no `pkiMode` key:** this server predates this feature. Defaults to + **`flat`**, matching its actual current state exactly — this design never + silently restructures an existing install's PKI, regardless of what a + fresh install would now default to. `--restructure-pki` remains available + as the explicit, confirmation-gated opt-in to migrate such a server (see + Phase 2) — it is no longer needed or documented as a fresh-install flag, + since a fresh install reaches `split` by default without it. +- **`pkiMode` already persisted in `.fogsettings`** (either value, from a + prior run under this feature): reused unchanged, exactly like every other + managed key. + +Once resolved (by either default path or an explicit override), every later +`installfog.sh`/`updatefog.sh` run on that server stays in the same mode +(managed key, same persistence pattern as `caCreated`). + +### `createRootCA()` (new function, sits beside `createSSLCA()`) + +Self-signed only (no import path at this level beyond `--root-ca-cert/-key`, +which simply skips generation and copies in an admin-supplied root instead, +mirroring `_ensureSecureBootKeys()`'s "admin-supplied pair always wins, never +regenerated" pattern). `CN=FOG Server ROOT CA`, `basicConstraints=critical, +CA:TRUE,pathlen:1`, ~20y validity; a root need not match the intermediates' +shorter, more frequently-rotated lifetimes. Never regenerates once present, +same reasoning and same code shape as `_ensureSecureBootKeys()`'s doc comment +about why MOK never regenerates — copy that reasoning here nearly verbatim, +it applies identically. + +### `createWebIntermediateCA()` / `createClientIntermediateCA()` (new +functions) + +Both share one small helper, `_issueIntermediateCA(cn, outdir, keyfile, +certfile)`, that does `openssl genrsa` + `openssl req -new` + `openssl x509 +-req -CA $rootcapem -CAkey $rootcakey -CAcreateserial -extensions v3_intermediate_ca` +against the Root — the same shape `createSSLCA()`'s existing self-signed +branch already uses, just with `-CA`/`-CAkey` added and `basicConstraints +CA:TRUE` in the extfile instead of the leaf's `CA:FALSE`. `createWebIntermediateCA()` +takes over the CSR/leaf-signing logic that lives in the back half of today's +`createSSLCA()` (lines 3445-3517: CSR, `sanentries`/`dnsSanEntries`, +`ca.cnf`, `openssl x509 -req -CA $sslcapem ...`), pointed at the web +intermediate instead of `$sslcapem`. `createClientIntermediateCA()` is new: +it either (a) generates `.fogClientCA.{key,pem}` from the root with +`CN=$fogClientCACN`, publishes it as `ca.cert.der`/`ca.cert.pem` exactly as +`createSSLCA()`'s existing two lines already do (`functions.sh:3520-3521`, +unchanged mechanics, new source file), and — pending the Phase-0 +verification's answer — either generates `.commLeaf.{key,pem}` signed by it +(if fog-client wants a real leaf) or skips that step and reuses +`.fogClientCA.{key,pem}` directly as the comm keypair (if fog-client expects +`ca.cert.der` itself to be usable as the encryption key, matching today's +flat model's shape); or (b) imports an admin-supplied client CA via +`--client-ca-*`, running the CN-mismatch warning described above. + +### `certDecrypt()`/`certEncrypt()` re-pointing (PHP, minimal diff) + +`fogbase.class.php:2027-2032`'s hardcoded `.srvprivate.key` filename becomes +a lookup for whatever file the *Client Communication* zone published as its +comm private key (`.commLeaf.key`, or `.fogClientCA.key` per the Phase-0 +answer) — resolved the same way the existing code resolves +`sslpath`/`storagenode`, i.e. still via `Route::getIds('storagenode', [], +'sslpath')`, just appending a different, new-in-`split`-mode filename instead +of `.srvprivate.key` when `$pkiMode == split` (the storage node's `sslpath` +column is itself unchanged; only which filename under it gets opened +changes). In `flat` mode this function's behavior is byte-for-byte identical +to today — same file, same path, same failure modes. + +### `createSecureBootIntermediateCA()` (new) + `_ensureSecureBootKeys()` (gated) + +`createSecureBootIntermediateCA()` uses the same `_issueIntermediateCA()` +helper as the other two zones, with `CN=FOG Secure Boot CA`, writing to +`$fogprogramdir/secureboot/ca/`. It then issues this server's code-signing +**leaf** into `secureboot/leaf/` with the `codeSigning` EKU and +`basicConstraints CA:FALSE` — i.e. exactly the extension profile +`_ensureSecureBootKeys()` already writes into its `mok.cnf` +(`functions.sh:4660-4673`), just signed by the intermediate instead of +self-signed. Leaf validity is deliberately short (~1 year) because rotating +it no longer costs a firmware trip; the intermediate matches the root's long +horizon. + +`_ensureSecureBootKeys()` gains a `pkiMode` gate at its top and **keeps its +entire existing body as the `flat` branch, unmodified** — including its +"admin-supplied pair always wins" check and its never-regenerate guarantee, +whose doc comment (`functions.sh:4620-4624`) explains precisely why a fresh +key silently strands every already-enrolled machine. That reasoning is what +makes the fresh-installs-only scoping non-negotiable: this design must never +cause an existing server to mint a new MOK. + +In `split` mode it instead calls `createSecureBootIntermediateCA()` and sets: + +- `secureBootKey`/`secureBootCert` → the **leaf** (`secureboot/leaf/sign.*`) +- `secureBootMokCert` → the **intermediate** (`secureboot/ca/.fogSBCA.pem`) + +In `flat` mode `secureBootMokCert` is simply assigned the same path as +`secureBootCert`, so downstream consumers never branch. + +`_ensureSecureBootPlatformKeys()` (PK/KEK) is genuinely unchanged — those +authorize firmware variable updates and sign nothing that executes; they are +orthogonal to this hierarchy. + +### `_resignKernels()` — sign with the leaf, ship the chain + +Two changes to `functions.sh:5049-5094`, both small: + +- `sbsign` gains `--addcert "$secureBootMokCert"` when `pkiMode == split`, so + the signed PE carries the intermediate and shim can chain the leaf back to + the enrolled MOK. Without this the kernel is signed by a certificate the + firmware has never seen and will not boot — this flag is the entire + mechanism that makes leaf rotation free. +- The idempotency check `sbverify --cert "$certpem"` (`:5073`) keeps working + unchanged, since it verifies against the signing leaf, which is still what + produced the signature. Worth an explicit test rather than an assumption + (see Testing) — a `sbverify` that resolves the chain differently would + cause every run to re-sign, which is wasteful but not dangerous. + +### `_publishSecureBootKit()` — publish the intermediate, not the signer + +`functions.sh:4781-4832` currently converts `$secureBootCert` to DER and +publishes it as `MOK.der`. The only change is that it reads +`$secureBootMokCert` instead. In `flat` mode that is the same file it reads +today (identical output); in `split` mode it publishes the intermediate, +which is the certificate that must be enrolled. Everything else in that +function — the DER/PEM auto-detection, the MokManager binary staging, the +404 `index.php` guard, the permissions — is untouched. + +The enrollment UX (`packages/secureboot/fog-enroll-mok.sh`, the PXE "Enroll +Secure Boot Key" menu item) needs **no change at all**: it fetches and +enrolls whatever is at `MOK.der`. It simply enrolls a CA now rather than a +leaf. + +## Data flow + +**Fresh install, the new default (no flag needed) or the interactive +prompt's split-related answers:** +`installfog.sh` → `caCreated` not yet `yes` → `pkiMode` resolves to `split` +→ `createRootCA()` → `createWebIntermediateCA()` (or +`--external-ca`/`--ca-cert`/... imports the web zone instead, unchanged +mechanics) → `createClientIntermediateCA()` (or `--client-ca-*` imports it) +→ vhost/leaf written under Web zone as today → `ca.cert.der` published from +the Client zone's CA cert → `_ensureSecureBootKeys()` takes its `split` +branch: `createSecureBootIntermediateCA()` mints the SB intermediate plus +this server's signing leaf → `_resignKernels()` signs FOS kernels with the +leaf and `--addcert`s the intermediate → `_publishSecureBootKit()` publishes +the **intermediate** as `MOK.der` → `$netbootproto` resolves to `http` +(private CA) or follows `$httpproto` (public CA) → all new managed keys +written to `.fogsettings`. + +**Fresh install, `--legacy-pki` or the interactive prompt's legacy answer:** +`installfog.sh` → `pkiMode` resolves to `flat` → `createSSLCA()` runs its +existing, completely unmodified code path, producing exactly what a +pre-this-design install would have produced. + +**Existing server, no opt-in (the overwhelming common case for +`updatefog.sh` — a server with `caCreated == yes` from before this feature +existed):** `.fogsettings` has no `pkiMode` key → resolves to `flat` (per +Components' `caCreated`-based default, not merely "unset defaults to flat") +→ `createSSLCA()` runs its existing, completely unmodified code path. +Nothing under `$sslpath/CA/root|web|client` is ever created, read, or +referenced. The existing `.fogCA.{key,pem}` stays exactly as valid as it was +before this patch merged — no other CA in this design shares its CN or its +file path, so there is no naming collision, and no code path introduced by +this design ever runs for a `flat`-mode server. + +**Existing server, explicit opt-in (`installfog.sh --restructure-pki` run by +hand against an already-installed server):** see Task-Plan Phase 2 for the +full task sequence — this is the dual-trust-window migration. + +**Web TLS renewal (steady state, any mode, admin-managed):** admin runs +whatever ACME client (`certbot`, `acme.sh`) or internal-CA process they +choose, on their own schedule, entirely outside FOG's process. The renewed +cert/key are dropped in at `$sslpubcert`/`$sslprivkey` — a safe drop-in +because the paired customization-preservation design's vhost managed-block +splice never touches leaf file *contents*, only the vhost config's reference +to their paths. FOG has no visibility into or role in this renewal at all. + +## Error handling + +- `createRootCA()`/the two intermediate functions inherit the same + `errorStat $?` convention every other cert-generating function in + `functions.sh` already uses — a failure here is fatal to the install run, + consistent with how a `createSSLCA()` failure is fatal today. +- `validateExternalCA(zone)`'s existing three checks (key/cert match, CA:TRUE, + chains to root) are unchanged and now simply run once per zone that opts + into external import, writing to that zone's subdirectory instead of the + shared one. +- The Client-zone CN mismatch is a **warning**, not a hard failure — see + Components. A hard failure here would block a legitimate "I want to test + whether the CN actually matters" run, which is exactly the kind of run that + needs to happen before this ships to confirm or refute the maintainer's + claim. +- `--restructure-pki` against a server that already has registered + fog-clients requires an explicit, unskippable confirmation (even under + `-Y`/`--autoaccept` — this is the one place in this design that + deliberately does *not* follow `-Y`'s "never prompt" convention, because + the consequence is fleet-wide and not reversible by re-running the + installer) unless a new `--i-understand-this-will-require-client-repinning` + style explicit flag is also passed. Exact flag name and prompt wording is + an implementation detail for Task-Plan Phase 2, Task 2.1. + +## Testing + +Same posture as every other shell-script change in this repo: no CI beyond +`fogproject-install-validation`'s distro matrix, so verification is manual — +`bash -n` after every edit, then real installer runs on a test VM. +Additionally, because this design's Phase 2 hinges on an unverified +assumption about fog-client: + +- **Phase 0 must produce a written, falsifiable answer** (from zazzles source + or from the maintainer directly) to: does fog-client (a) pin by exact + certificate bytes, (b) additionally/instead check the CN string, and (c) + re-fetch/re-validate `ca.cert.der` and its leaf before every `authorize()` + handshake, or only at registration time? All of Phase 2's task sequence in + the plan below is written for the *pessimistic* answer to (c) (full + re-registration required); if the real answer is optimistic, Phase 2 + collapses to something much simpler, which is a good problem to have but + should not be assumed. +- **Phase 0 must also verify shim's chain-validation behavior on real + hardware**, and this is a hard gate on the Secure Boot zone specifically: + does the shim version FOG ships actually accept a kernel signed by a leaf + whose issuing CA is the enrolled MOK, with the intermediate supplied via + `sbsign --addcert`? The whole "rotate leaves without re-enrolling firmware" + premise rests on this. It is well-established that shim supports CA + certificates in MokList, but behavior varies across shim versions and + firmware implementations, and this is exactly the sort of cross-project + assumption that the `certDecrypt()` finding proves is worth checking rather + than inheriting from a brainstorm. Test matrix: at least one x86_64 UEFI + machine and one arm64 if available, on the shim build + `downloadipxesecureboot()` stages. **If this fails, the Secure Boot zone + falls back to today's self-signed-leaf model** (`flat` behavior) while Web + and Client keep their split — the zones are independent by construction, so + a negative result costs one zone, not the design. +- **Netboot protocol isolation** needs its own end-to-end test on both + webserver families: with `pkiMode=split` and a FOG-PKI web cert, confirm + (1) the web UI is HTTPS, (2) `default.ipxe` chains over HTTP, (3) the + vhost's redirect does **not** bounce `service/ipxe/` to HTTPS, and (4) the + boot menu `boot.php` generates carries HTTP kernel/init URLs — which should + follow automatically from `$httpproto` being request-derived, but is the + assumption most worth confirming empirically since the whole + no-PHP-change claim rests on it. +- **Three checks together are the highest-value test in this entire + change**, and should be verified as a set right after Task 1.3 lands, + before any of Phase 1's remaining tasks are trusted: + 1. An **existing** `flat`-mode server (real prior `.fogsettings`, + `caCreated == yes`, no `pkiMode` key) behaves identically to before + this change on every `installfog.sh`/`updatefog.sh` run — + `--external-ca` included. + 2. A **fresh** install with **no PKI-related flag at all** now produces + `split` mode by default. This is a deliberate, new, *expected* + behavior change from before this design, not a regression — verify it + is actually happening, not just assumed. + 3. A **fresh** install with `--legacy-pki` (or the interactive prompt's + legacy choice) reproduces today's pre-split flat behavior + byte-for-byte — the regression test for the "permanent, supported + lightweight option" promise in Non-goals. +- **Removing `bin/setupacme.sh` (Task 1.6) needs its own regression check**: + confirm no other script/doc references it after removal (grep the repo for + `setupacme` post-deletion) and that `docs/EXTERNAL_CA_AND_LETSENCRYPT.md`'s + replacement guidance is accurate against a real `certbot`/manual-`acme.sh` + drop-in test. + +## Open risks/unknowns + +1. **fog-client's exact pinning mechanism (byte-identical vs. CN vs. both) is + unverified.** Blast radius is contained to `$fogClientCACN` and the one + `validateExternalCA client` call site (see Architecture); nothing else + hardcodes an assumption about it. +2. **Whether fog-client re-fetches/re-validates its pinned cert and leaf on + every handshake, or only at registration, is unverified.** This determines + whether Phase 2's client migration is a mostly-automatic background + self-heal or a mandatory full re-registration per endpoint. The plan below + is written for the pessimistic case. +3. **Where fog-client fetches the server's encryption certificate is + unverified** (Phase 0 Task 0.3). The design is settled — the Client CA + issues its own comm TLS certificate — but if the client fetches + `management/other/ssl/srvpublic.crt`, publishing the comm leaf there + makes this a server-side-only change, whereas if it derives its + encryption key from `ca.cert.der`, the fallback is for the Client CA to + double as the comm keypair. Only a third outcome (the client wanting a + genuinely new path) would require `zazzles` work, and that is the least + likely of the three. + **Resolved and no longer open:** whether `.srvprivate.key` exists and + which certificate it backs. It exists (dotfile, invisible to a bare + `ls`), it is the web leaf's key, and `certDecrypt()` uses it — so + replacing the web certificate breaks client auth today. That is the bug + this zone fixes. +4. **Shim's acceptance of a CA-in-MokList with an `--addcert` chain is + unverified** — see Testing. This is the Secure Boot zone's equivalent of + risk #1, and it is a *hard* gate: a negative result means the Secure Boot + zone stays on today's self-signed-leaf model. Contained by construction — + the three zones are independent, so this cannot invalidate the Web or + Client work. +5. **Per-storage-node signing leaves are designed for but not built in Phase + 1.** The intermediate model makes them possible and is substantially + motivated by them, but actually issuing a leaf to each node requires a + CSR round-trip through the node-registration flow that doesn't exist + today. Until Phase 3 builds it, storage nodes continue to serve kernels + signed by the master's leaf — which works fine, it just doesn't yet + deliver the per-node key isolation the structure now permits. Worth + stating plainly so the intermediate model isn't mistaken for having + already solved node scalability. +6. **A future GUI-level Let's Encrypt plugin is plausible once this design + has settled**, per the maintainer's own note, but is deliberately not + designed toward here — any such plugin would need its own separate design + pass once real-world usage of the three-zone split (and the admin-managed + drop-in pattern) has been observed. +7. **Confirmed by ad-hoc testing, not just this doc's reading of iPXE + source:** a real Let's Encrypt certificate on the vhost does validate for + iPXE netboot with no FOG-side change, matching Context's claim. Getting + there required `httpproto=https` in `.fogsettings` and `FOG_WEB_HOST` set + to the server's FQDN rather than its IP — see the "Public Let's Encrypt: + caveats" section of `docs/EXTERNAL_CA_AND_LETSENCRYPT.md` for the full + note, including a watch-item on `FOG_WEB_HOST`'s historical interaction + with some background services (`FOGFileDeleter`'s queue) that build + request URLs from it. Not reproduced during this test, but flagged for + whoever implements the Web zone's admin-managed drop-in path (this + design's replacement for `bin/setupacme.sh`) to keep an eye on. +8. **Making `split` the default for fresh installs raises the stakes on + Phase 0.** Every fresh install from this point forward creates a Client + Communication intermediate and comm leaf as soon as Phase 1 ships — + potentially before Phase 0's verification completes, since Phase 0 was + originally scoped only to gate Phase 2. If Task 0.3 finds the comm + certificate must be published somewhere other than the path Task 1.4 + assumes, every `split`-mode server created in the interim needs a + follow-up correction — not just servers that explicitly opted in under + the old, opt-in-only design. This is why Phase 0 now gates Phase 1's + release, not only Phase 2's. From 95457da3dce930602c50a7c9069a4349b047bdd7 Mon Sep 17 00:00:00 2001 From: JJ Fullmer Date: Fri, 7 Aug 2026 16:43:12 -0600 Subject: [PATCH 19/62] Move iPXE background backup/restore into installfog.sh, keyed to FOG_IPXE_BG_FILE Two changes in one, because neither is useful alone. First, the protection now runs from installfog.sh's own sequence rather than only from bin/updatefog.sh. backupCustomizations() lived in update.sh, so a bare `./installfog.sh` upgrade -- how most people upgrade -- got none of it and silently lost customizations to configureHttpd()'s rm -rf $webdirdest. Second, it reads the actual FOG_IPXE_BG_FILE globalSettings value instead of the hardcoded string "bg.png". That setting exists precisely so an admin can rename the background, and the old list could not see a renamed file at all. The value is trimmed, NULL-guarded and basename'd before it reaches a cp destination, since it is admin-supplied and a path there would write outside the backup directory. Failure severity is split deliberately, because errorStat() exits the installer whenever $exitFail is unset -- every normal installfog.sh run. Losing an optional legacy refind blob only warns. Failing to save a background we positively identified aborts, on purpose and before configureHttpd() has destroyed anything, so the admin's file is still intact where it always was. The restore side never aborts: by then the tree is already rebuilt, and the files are still sitting in $customizationsDir to be copied back by hand, so it says that instead of stranding the install. Verified by running the real functions against a sandboxed tree across a simulated wipe: renamed background survives, refind restored, bzImage correctly left at the newly downloaded version, and the fresh-install case (globalSettings not yet created) resolves to "nothing customized" with no special-casing. Not yet run against a live FOG install. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YE3jeTexYEcAxWFbZcZgpZ --- bin/installfog.sh | 10 ++++ lib/common/functions.sh | 111 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 121 insertions(+) diff --git a/bin/installfog.sh b/bin/installfog.sh index 310daf7f65..d99f37efee 100755 --- a/bin/installfog.sh +++ b/bin/installfog.sh @@ -937,6 +937,11 @@ while [[ -z $blGo ]]; do # to happen for it to connect at all. writeUpdateFile backupReports + # Before configureHttpd(), which rm -rf's $webdirdest -- + # this is the last point anything under it can be saved. + # configureMySql has already run, so the FOG_IPXE_BG_FILE + # lookup inside has a database to ask. + backupPreservedCustomizations configureHttpd checkWebTier backupDB @@ -944,6 +949,11 @@ while [[ -z $blGo ]]; do configureStorage configureDHCP configureTFTPandPXE + # After configureTFTPandPXE -> downloadfiles() has re-laid + # the default-named kernel/init set, so restoring here puts + # the admin's own files back on top of fresh defaults + # rather than being overwritten by them. + restorePreservedCustomizations configureFTP configureSnapins configureUDPCast diff --git a/lib/common/functions.sh b/lib/common/functions.sh index 9038956d40..e45f750951 100755 --- a/lib/common/functions.sh +++ b/lib/common/functions.sh @@ -76,6 +76,117 @@ backupReports() { echo "Done" return 0 } +# Where backupPreservedCustomizations() stashes anything that has to outlive +# configureHttpd()'s rm -rf $webdirdest. Deliberately under $fogprogramdir, +# never inside $webdirdest -- that is the same "survives the wipe by +# construction" property $fogprogramdir/secureboot already relies on, rather +# than a copy that has to be re-made correctly every time. +[[ -z $customizationsDir ]] && customizationsDir="${fogprogramdir}/customizations" +# Backs up whatever is actually customized under $webdirdest/service/ipxe/ +# BEFORE configureHttpd() destroys that tree. +# +# This used to live only in bin/updatefog.sh (backupCustomizations, since +# removed), which meant a bare `./installfog.sh` upgrade -- the way most +# people upgrade -- silently got none of it. Running it from installfog.sh's +# own sequence is the point: the protection now applies to every run. +# +# $bgfile is intentionally NOT local: restorePreservedCustomizations() runs +# later in the same shell and needs the name that was actually backed up, not +# a re-read of the setting (which an admin could have changed mid-install). +backupPreservedCustomizations() { + dots "Backing up customizations" + local ipxedir="${webdirdest}service/ipxe" + local f st=0 + # Severity is split deliberately, because errorStat() EXITS the installer + # when $exitFail is unset -- which is every normal installfog.sh run: + # + # $st -> fatal. Only set when a customization we positively identified + # could not be copied to safety. Aborting here is the safe + # outcome: configureHttpd() has not wiped anything yet, so the + # admin's file is still sitting untouched where it always was. + # warn -> non-fatal. An optional file we merely tried for. Killing an + # install over an unreadable legacy refind blob would be absurd. + # A failed mkdir is not itself fatal -- if there is nothing to preserve, + # nothing is lost. If there IS a background to preserve, the copy below + # fails too and that is what stops the run. + mkdir -p "$customizationsDir/ipxe-bg" "$customizationsDir/ipxe-legacy" >>$error_log 2>&1 || true + + # FOG_IPXE_BG_FILE is a real, GUI-editable globalSettings row (see + # packages/web/commons/schema.php) whose whole purpose is letting an admin + # rename the background file. Read the ACTUAL value rather than assuming + # "bg.png", which is what the old hardcoded list got wrong. + # + # On a first-ever install globalSettings does not exist yet -- updateDB() + # runs after configureHttpd() -- so this errors into $error_log and leaves + # $bgfile empty, which is treated exactly like "nothing customized". No + # special-casing needed for a fresh install. + bgfile=$(mysql $sqloptionsuser --password="${snmysqlpass}" -N -B \ + --execute="SELECT settingValue FROM globalSettings WHERE settingKey='FOG_IPXE_BG_FILE'" \ + $mysqldbname 2>>$error_log) + # Strip surrounding whitespace, and treat mysql's literal NULL output as + # empty -- an unset settingValue comes back as the four characters "NULL" + # under -N, which would otherwise be looked for as a filename. + bgfile="${bgfile#"${bgfile%%[![:space:]]*}"}" + bgfile="${bgfile%"${bgfile##*[![:space:]]}"}" + [[ $bgfile == NULL ]] && bgfile="" + # basename guards against a settingValue containing a path: this string + # reaches a cp destination, and "../../something" would write outside the + # backup directory entirely. + [[ -n $bgfile ]] && bgfile=$(basename "$bgfile") + if [[ -n $bgfile && -f "${ipxedir}/${bgfile}" ]]; then + cp -f "${ipxedir}/${bgfile}" "${customizationsDir}/ipxe-bg/${bgfile}" >>$error_log 2>&1 || st=1 + fi + + local warn=0 + for f in refind.conf refind.efi refind_x64.efi refind_ia32.efi refind_aa64.efi; do + [[ -f "${ipxedir}/${f}" ]] && { cp -f "${ipxedir}/${f}" "${customizationsDir}/ipxe-legacy/${f}" >>$error_log 2>&1 || warn=1; } + done + if [[ $st -ne 0 ]]; then + echo "Failed" + echo " * Could not copy the customized iPXE background (${bgfile}) to" + echo " ${customizationsDir}/ipxe-bg/." + echo " * Stopping BEFORE the web tree is rebuilt, so your file is still" + echo " intact at ${ipxedir}/${bgfile}. Fix the permissions or free" + echo " space under ${customizationsDir} and re-run. See $error_log." + exit 1 + fi + [[ $warn -ne 0 ]] && echo -n "(some optional refind files could not be backed up) " + errorStat 0 +} +# Restores what backupPreservedCustomizations() saved, AFTER +# configureTFTPandPXE()'s downloadfiles() has re-laid the default-named +# kernel/init set. +# +# Deliberately does NOT restore the six default kernel/init names here -- the +# point of an update is to pick up the latest kernel. That is what the +# versioned backup provides an explicit, admin-invoked restore path for +# instead. +restorePreservedCustomizations() { + dots "Restoring customizations" + local ipxedir="${webdirdest}service/ipxe" + local f st=0 + + if [[ -n $bgfile && -f "${customizationsDir}/ipxe-bg/${bgfile}" ]]; then + cp -f "${customizationsDir}/ipxe-bg/${bgfile}" "${ipxedir}/${bgfile}" >>$error_log 2>&1 || st=1 + fi + for f in refind.conf refind.efi refind_x64.efi refind_ia32.efi refind_aa64.efi; do + [[ -f "${customizationsDir}/ipxe-legacy/${f}" ]] && { cp -f "${customizationsDir}/ipxe-legacy/${f}" "${ipxedir}/${f}" >>$error_log 2>&1 || st=1; } + done + [[ -d $ipxedir ]] && chown -R ${username}:${apacheuser} "$ipxedir" >>$error_log 2>&1 + # Never fatal, unlike the backup side. By this point configureHttpd() has + # already rebuilt the web tree, so aborting would strand a nearly-complete + # install and fix nothing -- and unlike the backup case, the files are NOT + # lost: they are still sitting in $customizationsDir for the admin to put + # back by hand. Say exactly that instead of dying. + if [[ $st -ne 0 ]]; then + echo "Failed" + echo " * One or more customizations could not be restored to ${ipxedir}." + echo " * Nothing was lost -- your files are still in ${customizationsDir}." + echo " Copy them back by hand once the install finishes. See $error_log." + return 0 + fi + errorStat 0 +} # GH-685: the MariaDB client library turns TLS on by default from 10.10.1 # onward and then refuses to connect at all when the server offers none -- # "ERROR 2026 (HY000): TLS/SSL error: SSL is required, but the server does not From a01f1803c876d182b64beccea26e93b1187e7fa1 Mon Sep 17 00:00:00 2001 From: JJ Fullmer Date: Fri, 7 Aug 2026 16:46:13 -0600 Subject: [PATCH 20/62] Preserve an admin-supplied Secure Boot pair outside $webdirdest --secure-boot-key/--secure-boot-cert are persisted to .fogsettings verbatim and _ensureSecureBootKeys() then trusts that path forever, but nothing ever copied the files anywhere. An admin who parks the pair under $webdirdest -- not unreasonable, it is where the enrolment kit gets published -- loses it to configureHttpd()'s rm -rf $webdirdest, in the same run that first accepted the flags and before _resignKernels() ever reads it. Copies the pair into $fogprogramdir/secureboot/ instead, which nothing in the installer deletes, and repoints the variables at the copy. Runs right after the existing pair validation, long before the web tree is rebuilt. The admin's original file is never modified. Named admin-MOK.* rather than MOK.*, correcting the plan, which specified the latter. MOK.key/MOK.pem hold FOG's OWN generated pair, which is deliberately never regenerated because every client that already enrolled it would be stranded; writing an admin key over that path would destroy it with no backup. The guard also skips outright when the configured path already resolves under $fogprogramdir/secureboot, so FOG's generated pair is never touched and the function is a no-op on every subsequent run. Verified by running the real function against a sandboxed tree with the pair parked inside $webdirdest and a pre-existing FOG-generated MOK: the admin key survives the wipe at its new path, FOG's MOK.key is bit-for-bit untouched, and a second invocation correctly does nothing. Not yet run against a live FOG install. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YE3jeTexYEcAxWFbZcZgpZ --- bin/installfog.sh | 6 ++ .../2026-08-07-customization-preservation.md | 12 ++++ lib/common/functions.sh | 57 +++++++++++++++++++ 3 files changed, 75 insertions(+) diff --git a/bin/installfog.sh b/bin/installfog.sh index d99f37efee..875ebbe051 100755 --- a/bin/installfog.sh +++ b/bin/installfog.sh @@ -681,6 +681,12 @@ if [[ -n $secureBootKey || -n $secureBootCert ]]; then done unset sbfile fi +# Immediately after validation and long before configureHttpd() rebuilds the +# web tree, so a pair the admin parked somewhere that gets deleted is copied +# to safety first. Handles a path from .fogsettings as well as one from this +# run's flags, and no-ops once the recorded path is already protected. +# $fogprogramdir is settled by config.sh above, so the destination is real. +preserveSecureBootAdminFiles [[ -f $fogpriorconfig ]] && grep -l webroot $fogpriorconfig >>$error_log 2>&1 case $? in diff --git a/docs/superpowers/plans/2026-08-07-customization-preservation.md b/docs/superpowers/plans/2026-08-07-customization-preservation.md index e9d48190b7..bc0fdffe32 100644 --- a/docs/superpowers/plans/2026-08-07-customization-preservation.md +++ b/docs/superpowers/plans/2026-08-07-customization-preservation.md @@ -199,6 +199,18 @@ git commit -m "Move iPXE background backup/restore into installfog.sh, keyed to - Produces: possibly-reassigned `$secureBootKey`/`$secureBootCert`, now always pointing under `${fogprogramdir}/secureboot/`. +> **Correction applied during implementation — do not revert.** An earlier +> draft of this task copied the admin's pair to +> `${fogprogramdir}/secureboot/MOK.key`/`MOK.pem`. That is the exact path +> `_ensureSecureBootKeys()` uses for **FOG's own generated pair**, which it +> never regenerates precisely because every client that already enrolled it +> would be stranded. Copying over it would destroy an enrolled key with no +> backup and no way back. The implementation therefore writes +> `admin-MOK.key`/`admin-MOK.pem` instead, and skips entirely when the +> configured path already resolves to somewhere under +> `${fogprogramdir}/secureboot/`. Continuity across later runs comes from +> `.fogsettings` recording the new path, not from reusing the filename. + - [ ] **Step 1: Add `preserveSecureBootAdminFiles()`** Insert into `lib/common/functions.sh`, directly before `_ensureSecureBootKeys()` diff --git a/lib/common/functions.sh b/lib/common/functions.sh index e45f750951..06e15fbfe2 100755 --- a/lib/common/functions.sh +++ b/lib/common/functions.sh @@ -4718,6 +4718,63 @@ downloadfiles() { _publishSecureBootKit _publishSecureBootAuthVars } +# Copy an admin-supplied Secure Boot pair somewhere the installer cannot +# destroy, and point $secureBootKey/$secureBootCert at the copy. +# +# The gap this closes: --secure-boot-key/--secure-boot-cert are persisted to +# .fogsettings verbatim and _ensureSecureBootKeys() then trusts that path +# forever, but nothing ever copies the file anywhere. An admin who parks the +# pair under $webdirdest -- not unreasonable, it is where the enrolment kit is +# published -- loses it to configureHttpd()'s rm -rf $webdirdest, in the SAME +# run that first accepted the flags, before _resignKernels() ever reads it. +# +# Copied to admin-MOK.* rather than MOK.*, deliberately. MOK.key/MOK.pem are +# where _ensureSecureBootKeys() keeps FOG's OWN generated pair, and that pair +# is never regenerated precisely because every client that already enrolled it +# would be stranded. Writing an admin's key over that path would destroy it +# with no backup and no way back. Continuity across later runs comes from +# .fogsettings holding the new path, not from reusing the filename. +# +# The original file the admin pointed at is never modified -- this only +# decides which copy gets used from here on. Idempotent: once .fogsettings +# records the copy, every later run sees a path already under +# $fogprogramdir/secureboot and does nothing. +preserveSecureBootAdminFiles() { + [[ -z $secureBootKey || -z $secureBootCert ]] && return 0 + local keydir="${fogprogramdir}/secureboot" + local destkey="${keydir}/admin-MOK.key" + local destcert="${keydir}/admin-MOK.pem" + local st=0 + + # Already somewhere this installer never deletes -- including FOG's own + # generated pair, which must be left exactly where it is. + case "$(readlink -f "$secureBootKey" 2>/dev/null)" in + "$(readlink -f "$keydir" 2>/dev/null)"/*) return 0 ;; + esac + + dots "Preserving admin-supplied Secure Boot key" + mkdir -p "$keydir" >>$error_log 2>&1 || st=1 + chown root:root "$keydir" >>$error_log 2>&1 + chmod 0700 "$keydir" >>$error_log 2>&1 + cp -f "$secureBootKey" "$destkey" >>$error_log 2>&1 || st=1 + cp -f "$secureBootCert" "$destcert" >>$error_log 2>&1 || st=1 + if [[ $st -ne 0 ]]; then + echo "Failed" + echo " * Could not copy the Secure Boot signing pair into ${keydir}." + echo " * Leaving --secure-boot-key/--secure-boot-cert pointed at the" + echo " originals. If either lives under ${webdirdest}, MOVE IT NOW --" + echo " the web tree is rebuilt later in this run. See $error_log." + return 0 + fi + chown root:root "$destkey" "$destcert" >>$error_log 2>&1 + # Key restricted, certificate public by design -- it is the thing handed + # out for enrolment. Mirrors _ensureSecureBootKeys()'s own permissions. + chmod 0600 "$destkey" >>$error_log 2>&1 + chmod 0644 "$destcert" >>$error_log 2>&1 + secureBootKey="$destkey" + secureBootCert="$destcert" + errorStat 0 +} # Generate the Secure Boot signing key when the admin has not supplied one. # # Signing used to require --secure-boot-key/--secure-boot-cert, which meant it From 17e5299cfc7b3bcb9f9423add1d332fd835e942e Mon Sep 17 00:00:00 2001 From: JJ Fullmer Date: Fri, 7 Aug 2026 16:50:55 -0600 Subject: [PATCH 21/62] Write the vhost as a FOG-managed block instead of replacing the whole file createSSLCA() regenerated $etcconf from scratch every run, so the only way to keep a hand-edited vhost was -F/--no-vhost, which then also skipped every future security fix FOG makes to the parts it owns. All or nothing, with no middle ground. It now writes only between two marker lines and leaves anything outside them untouched. spliceManagedBlock() handles three cases and deliberately no fourth: no file, write one; both markers present, replace between them; anything else -- no markers, or only one because a run died mid-write -- append a fresh block and touch nothing already there. Never guess at a partial patch. '#' comments in both nginx and Apache syntax, so the markers are inert in either. Implemented as a variable swap rather than the 261 individual edits the plan called for. beginManagedVhost points $etcconf at a scratch file and endManagedVhost splices it into the real one, so all 261 existing write sites are byte-for-byte untouched. 261 near-identical mechanical edits is exactly where a missed line hides, and a missed line writes half a vhost to the wrong path. Confirmed the 8 non-append readers of $etcconf each still get the path they need, and that nothing between begin and end can exit early and strand the variable on the scratch path. The nginx splice lands before nginx -t, which tests the real file. updatefog.sh's default flips from -F to regenerating, since the reason for that default no longer holds. --no-vhost is the new opt-out; --overwrite-vhost stays as a deprecated no-op so existing cron jobs do not die in getopt. --hostname/--extra-server-name still override an explicit --no-vhost, or the cert SAN and server_name would silently disagree about the server's name. Verified against a sandboxed vhost: admin content after the block survives an upgrade while FOG's own block picks up new directives, a marker-less file is appended to rather than overwritten, a half-corrupted one self-heals on the next run, the scratch file is cleaned up, and all four flag combinations resolve correctly. Not yet run against a live web server. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YE3jeTexYEcAxWFbZcZgpZ --- bin/updatefog.sh | 50 +++++++++---- .../2026-08-07-customization-preservation.md | 22 ++++++ lib/common/functions.sh | 71 +++++++++++++++++++ 3 files changed, 128 insertions(+), 15 deletions(-) diff --git a/bin/updatefog.sh b/bin/updatefog.sh index 336bdee4bb..9f36d36d72 100755 --- a/bin/updatefog.sh +++ b/bin/updatefog.sh @@ -40,7 +40,7 @@ export PATH usage() { echo -e "Usage: $0 [-h?y] [--channel stable|staging|dev] [--branch ] [--git-path ]" - echo -e "\t \t\t[--no-revert] [--overwrite-vhost]" + echo -e "\t \t\t[--no-revert] [--no-vhost]" echo -e "\t-h -? --help\t\tDisplay this info" echo -e "\t --channel\tUpdate channel to track: stable, staging, or dev" echo -e "\t \t\tdefaults to whatever this server already tracks" @@ -54,9 +54,13 @@ usage() { echo -e "\t \t(implies --overwrite-vhost)" echo -e "\t --no-revert\tOn failure, leave the system as-is instead of" echo -e "\t \t\tautomatically reverting to the previous commit" - echo -e "\t --overwrite-vhost\tLet installfog.sh regenerate the web server" - echo -e "\t \t\tvhost from scratch instead of leaving the" - echo -e "\t \t\texisting one (with any customizations) alone" + echo -e "\t --no-vhost\tDo not touch the web server vhost at all." + echo -e "\t \t\tBy default FOG refreshes only the region between" + echo -e "\t \t\tits MANAGED BLOCK markers and leaves anything you" + echo -e "\t \t\tadded outside them alone, so skipping is rarely" + echo -e "\t \t\twanted -- it also skips FOG's own security fixes" + echo -e "\t \t\tto the parts it owns" + echo -e "\t --overwrite-vhost\tDeprecated no-op: this is now the default" echo -e "\t-y --yes\t\tSkip the confirmation prompt (for cron/GUI use)" exit 0 } @@ -64,19 +68,25 @@ usage() { supdateExtraServerNames=() shortopts="h?y" -longopts="help,channel:,branch:,git-path:,no-revert,overwrite-vhost,yes,hostname:,extra-server-name:" +longopts="help,channel:,branch:,git-path:,no-revert,overwrite-vhost,no-vhost,yes,hostname:,extra-server-name:" optargs=$(getopt -o $shortopts -l $longopts -n "$0" -- "$@") [[ $? -ne 0 ]] && usage eval set -- "$optargs" autoRevert=1 autoYes="" -# Every update already has a pre-existing, possibly hand-customized vhost -- -# unlike a fresh install, there is nothing to gain by regenerating it, and -# createSSLCA() has no way to tell "default" apart from "admin edited this". -# -F/--no-vhost is the escape hatch installfog.sh already has for exactly -# this; --overwrite-vhost below opts back into the fresh-install behavior. -updateVhostFlag="-F" +# Was -F by default, because regenerating the vhost meant destroying any hand +# customization -- createSSLCA() rewrote the whole file and could not tell +# "default" from "admin edited this". That is no longer true: it now writes +# only between the FOG MANAGED BLOCK markers (see spliceManagedBlock in +# lib/common/functions.sh) and leaves everything outside them alone. +# +# So the default flips. Skipping the vhost now costs an admin every future +# security fix FOG makes to the parts it owns -- ciphers, headers, the +# LocationMatch rules -- to protect content that is no longer at risk. -F +# remains available for "do not touch this file at all", which is a real +# preference, just no longer the one that should be automatic. +updateVhostFlag="" while :; do case $1 in -h | -\? | --help) @@ -122,9 +132,15 @@ while :; do shift ;; --overwrite-vhost) + # Now the default. Kept so an existing cron job or script that + # passes it keeps working rather than dying in getopt. updateVhostFlag="" shift ;; + --no-vhost) + updateVhostFlag="-F" + shift + ;; -y | --yes) autoYes="1" shift @@ -142,10 +158,14 @@ while :; do done # --hostname/--extra-server-name are requests for a vhost-VISIBLE change, so -# they imply --overwrite-vhost. With the "-F" default above, createSSLCA() -# prints "Skipped" instead of writing the vhost at all: .fogsettings and the -# cert SAN would change (cert generation happens before the novhost check) but -# server_name/ServerAlias would silently keep the old names. +# they override an explicit --no-vhost. Without this, createSSLCA() prints +# "Skipped" instead of writing the vhost: .fogsettings and the cert SAN would +# change (cert generation happens before the novhost check) while +# server_name/ServerAlias silently kept the old names -- a cert and a vhost +# that disagree about what this server is called. +# +# No longer needed for the common case now that regenerating is the default, +# but still required for the explicit --no-vhost + --hostname combination. if [[ -n $supdatehostname || ${#supdateExtraServerNames[@]} -gt 0 ]]; then updateVhostFlag="" fi diff --git a/docs/superpowers/plans/2026-08-07-customization-preservation.md b/docs/superpowers/plans/2026-08-07-customization-preservation.md index bc0fdffe32..c18c25a06b 100644 --- a/docs/superpowers/plans/2026-08-07-customization-preservation.md +++ b/docs/superpowers/plans/2026-08-07-customization-preservation.md @@ -375,6 +375,28 @@ spliceManagedBlock() { Run: `bash -n lib/common/functions.sh`. Expected: no output, exit 0. +> **Approach changed during implementation — Steps 3 and 4 below are +> superseded.** They call for rewriting every `>> "$etcconf"` line in both +> branches to `>> "$fogvhosttmp"`. There turned out to be **261** such lines +> (259 append + 2 truncate). That many near-identical mechanical edits is +> precisely where a missed or mistyped line hides, and a missed line writes +> half a vhost to the wrong path — a failure that would surface as a broken +> web server, not a test failure. +> +> Implemented instead as a variable swap, which is provably equivalent and +> touches 4 lines instead of 261: `beginManagedVhost` points `$etcconf` at a +> scratch file and `endManagedVhost` splices it into the real one and +> restores the variable. Every existing write site is left byte-for-byte +> untouched. Verified that only 8 sites read `$etcconf` for anything other +> than appending (the two `mv -fv` backups, two truncating first-writes, +> three `emitNginxPhpBody` calls, two `diffconfig` calls) and that each ends +> up with the path it needs, and that no `errorStat`/`exit`/`return` sits +> between begin and end where it could strand `$etcconf` on the scratch path. +> +> Placement detail worth keeping: the splice must happen **before** +> `nginx -t` (`functions.sh`, nginx branch), because that command tests the +> real file on disk. + - [ ] **Step 3: Redirect the nginx branch's generation into a temp file** In `createSSLCA()`'s nginx branch, change the first write (currently, per diff --git a/lib/common/functions.sh b/lib/common/functions.sh index 06e15fbfe2..0e681df543 100755 --- a/lib/common/functions.sh +++ b/lib/common/functions.sh @@ -3520,6 +3520,67 @@ emitNginxPhpBody() { echo " fastcgi_buffers 16 16k;" >> "$1" echo " fastcgi_buffer_size 32k;" >> "$1" } +FOG_MANAGED_BEGIN='# === FOG MANAGED BLOCK -- DO NOT EDIT BETWEEN THESE LINES (see docs/SUPPORTED_CUSTOMIZATIONS.md) ===' +FOG_MANAGED_END='# === END FOG MANAGED BLOCK ===' +# Replaces only the FOG-owned region of $1 with the contents of $2, leaving +# anything the admin added outside that region byte-for-byte intact. +# +# Why a marked block instead of a template file: every vhost write site in +# createSSLCA() is inline bash echo/heredoc, branching on webserver family, OS +# family, SSL on/off and IPv4/IPv6. Extracting all of that into substitutable +# template assets would mean maintaining two representations of the same +# config forever. Wrapping the existing, unchanged generation in two marker +# lines gets the property that actually matters -- FOG can keep improving what +# it owns without discarding what the admin owns. +# +# '#' is a comment in both nginx and Apache syntax, so the markers are inert +# in either file. +# +# Three cases, deliberately no fourth: no file -> write one; both markers +# present -> replace between them; anything else (no markers, or only one +# because a previous run died mid-write or someone hand-edited) -> append a +# fresh block and touch nothing that was already there. Never guess at a +# partial patch. +spliceManagedBlock() { + local conffile="$1" contentfile="$2" + if [[ ! -f "$conffile" ]]; then + { echo "$FOG_MANAGED_BEGIN"; cat "$contentfile"; echo "$FOG_MANAGED_END"; } > "$conffile" + return $? + fi + if grep -qF "$FOG_MANAGED_BEGIN" "$conffile" && grep -qF "$FOG_MANAGED_END" "$conffile"; then + local tmp="${conffile}.fogsplice.$$" + awk -v b="$FOG_MANAGED_BEGIN" -v e="$FOG_MANAGED_END" -v cf="$contentfile" ' + $0 == b { print; while ((getline line < cf) > 0) print line; close(cf); skip=1; next } + $0 == e { print; skip=0; next } + !skip { print } + ' "$conffile" > "$tmp" && mv -f "$tmp" "$conffile" + return $? + fi + { echo "$FOG_MANAGED_BEGIN"; cat "$contentfile"; echo "$FOG_MANAGED_END"; } >> "$conffile" +} +# Redirects the vhost generation that follows into a scratch file, so the ~260 +# existing `>> "$etcconf"` lines below need no edit at all: they keep writing +# to $etcconf, which now names the scratch file. endManagedVhost() splices +# that content into the real file and restores the variable. +# +# Rewriting every one of those write sites individually would have been the +# obvious way and the wrong one -- 260-odd near-identical mechanical edits is +# exactly where a missed line hides, and a missed line writes half a vhost to +# the wrong path. +beginManagedVhost() { + vhostfinal="$etcconf" + etcconf="${etcconf}.fogblock.$$" + : > "$etcconf" +} +endManagedVhost() { + local generated="$etcconf" + etcconf="$vhostfinal" + spliceManagedBlock "$etcconf" "$generated" + local st=$? + rm -f "$generated" >>$error_log 2>&1 + unset vhostfinal + return $st +} createSSLCA() { # This function also emits the web server vhost further down, and those # nginx location / apache LocationMatch blocks used to hardcode ^/fog/ -- @@ -3676,6 +3737,9 @@ EOF # diffconfig without ever taking this backup first, so it was # comparing the new file to nothing and never fired. mv -fv "${etcconf}" "${etcconf}.${timestamp}" >>$workingdir/error_logs/fog_error_${version}.log 2>&1 + # Everything below writes to the scratch file; endManagedVhost + # splices it into the real one before nginx -t sees it. + beginManagedVhost echo "server {" > "$etcconf" echo " listen 80;" >> "$etcconf" echo " server_name $ipaddresses $hostname${extraServerNamesSuffix};" >> "$etcconf" @@ -3848,6 +3912,9 @@ EOF done fi fi + # Splice BEFORE nginx -t: that tests the real file on disk, + # so it has to be the spliced result, not the scratch copy. + endManagedVhost echo "Done" dots "Testing nginx configuration" nginx -t >> $workingdir/error_logs/fog_error_${version}.log 2>&1 @@ -3885,6 +3952,9 @@ EOF vhostaliases=$(echo $ipaddresses | awk '{for (i = 2; i <= NF; i++) printf " %s", $i}') vhostaliases="${vhostaliases}${extraServerNamesSuffix}" mv -fv "${etcconf}" "${etcconf}.${timestamp}" >>$workingdir/error_logs/fog_error_${version}.log 2>&1 + # See the nginx branch above -- same scratch-file swap, so + # none of the write sites below change. + beginManagedVhost echo "" > "$etcconf" echo " " >> "$etcconf" if [[ $osid -eq 1 && $OSVersion -lt 7 ]]; then @@ -4101,6 +4171,7 @@ EOF echo " RewriteRule ^${webrootre}(.*)$ ${webroot}api/index.php [QSA,L]" >> "$etcconf" echo "" >> "$etcconf" fi + endManagedVhost diffconfig "${etcconf}" errorStat $? # Self-referential link so /fog/fog/... resolves. $webdirdest From d8b0f47e7ed66604d272c99ad72bdf91266752c1 Mon Sep 17 00:00:00 2001 From: JJ Fullmer Date: Fri, 7 Aug 2026 16:51:56 -0600 Subject: [PATCH 22/62] Add an optional custom.ipxe hook ahead of FOG's default PXE boot logic default.ipxe is regenerated in full on every run, so an admin wanting site-specific pre-boot behaviour -- a boot delay, a prompt, a local menu -- had nowhere to put it that survives an update. The old answer was an alternate boot file maintained by hand outside FOG entirely. The generated file now starts with `chain custom.ipxe || goto fog_default`. Absent, which is the overwhelming default, the chain fails, the || fires, and boot proceeds byte-for-byte as before. Present, it runs before FOG's own params/menu logic and then falls straight through into :fog_default, because chain without --replace returns control to the following line once the chained script ends normally. No resume convention to get wrong and no way to loop back into default.ipxe. The hook file needs no backup/restore machinery: it sits at the TFTP root, which configureTFTPandPXE() only snapshots and copies into -- it never deletes destination files that are absent from the source tree -- so it survives updates structurally, the same way the Secure Boot keys do by living outside $webdirdest. Chosen over editing autoexec.ipxe, which is not FOG-authored: it ships in the FOGProject/fog-ipxe release tarball and is replaced wholesale on every run. default.ipxe is generated by this function, so it is the file this repo can actually own. Rendered output checked for correct escaping (iPXE's ${buildarch} etc. stay literal while $ipaddress/$webroot expand). Real PXE boot verification, with and without a custom.ipxe present, still needs hardware. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YE3jeTexYEcAxWFbZcZgpZ --- lib/common/functions.sh | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/lib/common/functions.sh b/lib/common/functions.sh index 0e681df543..79d770ef76 100755 --- a/lib/common/functions.sh +++ b/lib/common/functions.sh @@ -1148,7 +1148,24 @@ configureFTP() { configureDefaultiPXEfile() { dots 'Configuring default iPXE file' [[ -z $webroot ]] && webroot='/fog/' # see registerStorageNode, GH-529 - echo -e "#!ipxe\nset arch \${buildarch}\niseq \${arch} i386 && cpuid --ext 29 && set arch x86_64 ||\nparams\nparam mac0 \${net0/mac}\nparam arch \${arch}\nparam platform \${platform}\nparam product \${product}\nparam manufacturer \${product}\nparam ipxever \${version}\nparam filename \${filename}\nparam sysuuid \${uuid}\nisset \${net1/mac} && param mac1 \${net1/mac} || goto bootme\nisset \${net2/mac} && param mac2 \${net2/mac} || goto bootme\n:bootme\nchain ${httpproto}://$ipaddress${webroot}service/ipxe/boot.php##params" > "$tftpdirdst/default.ipxe" + # The `chain custom.ipxe || goto fog_default` first line is the supported + # hook point for site-specific pre-boot behavior -- a boot delay, a prompt, + # a local menu -- so an admin never has to hand-edit this file, which is + # regenerated in full on every run and would lose those edits anyway. + # + # Safe when unused: per ipxe.org/cmd/chain a chain to a file that is not + # there simply fails, the || fires, and boot proceeds exactly as before. + # Safe when used: chain WITHOUT --replace returns control to the next line + # once the chained script finishes normally, and the next line is + # :fog_default, so execution falls straight through into FOG's own logic. + # No "resume" convention to get wrong, and no way to loop back here. + # + # custom.ipxe lives at the TFTP root, which configureTFTPandPXE() only ever + # snapshots and copies INTO -- it never deletes destination files absent + # from the source tree. So the hook file survives updates structurally, + # the same way the Secure Boot keys do by living outside $webdirdest, and + # needs no backup/restore machinery of its own. + echo -e "#!ipxe\nchain custom.ipxe || goto fog_default\n:fog_default\nset arch \${buildarch}\niseq \${arch} i386 && cpuid --ext 29 && set arch x86_64 ||\nparams\nparam mac0 \${net0/mac}\nparam arch \${arch}\nparam platform \${platform}\nparam product \${product}\nparam manufacturer \${product}\nparam ipxever \${version}\nparam filename \${filename}\nparam sysuuid \${uuid}\nisset \${net1/mac} && param mac1 \${net1/mac} || goto bootme\nisset \${net2/mac} && param mac2 \${net2/mac} || goto bootme\n:bootme\nchain ${httpproto}://$ipaddress${webroot}service/ipxe/boot.php##params" > "$tftpdirdst/default.ipxe" errorStat $? } prepareiPXEsource() { From 36e3262f3b0f29ad04548e8abf4f0cdf29a56e59 Mon Sep 17 00:00:00 2001 From: JJ Fullmer Date: Fri, 7 Aug 2026 17:03:45 -0600 Subject: [PATCH 23/62] Cover the Setup Mode / db enrollment path in the Secure Boot PKI design The intermediate model was only described against MokManager enrollment. FOG has a second route -- _publishSecureBootAuthVars() builds signed PK/KEK/db blobs so a client in Setup Mode enrolls unattended -- and the two are verified by different code: shim's own logic against MokList, and the UEFI firmware against db. fog-build-sb-authvars puts FOG's SIGNING certificate into db. If MOK.der became the intermediate while db kept the leaf, rotating a signing leaf would silently strand every Setup-Mode-enrolled client while continuing to work for MokManager-enrolled ones -- the worst kind of split, because it looks like it works. db has to carry the intermediate for the same reason MOK does. That is the standard UEFI model, not a workaround: Microsoft's own db entries are CAs and Windows validates by chaining to them. Root cause is the same dual-purpose pattern this design keeps turning up, now for the third time: SECUREBOOT_CERT in .fog-secureboot is read by fog-sign-kernel as the signing cert AND by fog-build-sb-authvars as a trust anchor. Adds a distinct SECUREBOOT_MOK_CERT for the anchor, with a fallback so an existing flat-mode .fog-secureboot keeps working. Also catches a gap in the earlier write-up: fog-sign-kernel is the sudo helper behind the web UI's Kernel Update page, a signing path entirely separate from _resignKernels(), and it needs --addcert too. Without it a kernel downloaded through the GUI is signed with no chain attached and fails to boot on exactly the clients this design exists to serve. Phase 0's hardware verification now covers both enrollment routes, including rotating a leaf after Setup Mode enrollment. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YE3jeTexYEcAxWFbZcZgpZ --- .../2026-08-07-three-zone-pki-separation.md | 34 +++++++++++ ...-08-07-three-zone-pki-separation-design.md | 58 +++++++++++++++++++ 2 files changed, 92 insertions(+) diff --git a/docs/superpowers/plans/2026-08-07-three-zone-pki-separation.md b/docs/superpowers/plans/2026-08-07-three-zone-pki-separation.md index 41c811368b..9d590f5ebb 100644 --- a/docs/superpowers/plans/2026-08-07-three-zone-pki-separation.md +++ b/docs/superpowers/plans/2026-08-07-three-zone-pki-separation.md @@ -164,6 +164,14 @@ must be answered before Task 1.8 is written, not after. not: mark the Secure Boot zone as staying flat, and note which shim version/firmware was tested — do not silently downgrade the plan without recording what was actually observed. +- [ ] **Step 4a: Repeat the whole test via the Setup Mode / db path**, not + just MokManager — they are verified by different code (shim's own logic vs. + the UEFI firmware's). Put a machine in Setup Mode, enroll FOG's + PK/KEK/db.auth with the **intermediate** in `db`, and confirm a + leaf-signed kernel boots. Then rotate the leaf and confirm it still boots + with no db update pushed. A pass here is what proves the rotation promise + holds on both enrollment routes; a failure means `db` still needs the leaf + and Setup-Mode clients keep today's re-enrollment cost. - [ ] **Step 5:** If possible, repeat on arm64. Record if untested rather than assuming parity with x86_64. - [ ] **Step 6: Commit** (docs-only) @@ -762,6 +770,32 @@ record why. (`functions.sh:4784`, `4793`, `4795`) to `$secureBootMokCert`. Nothing else in that function changes; in `flat` mode the two variables are the same path, so its output is identical to today. +- [ ] **Step 4a: The Setup Mode / db path — do not skip this.** MokManager is + only one of two enrollment routes. `_publishSecureBootAuthVars()` builds a + `db` containing Microsoft's CAs plus **FOG's signing certificate** + (`packages/secureboot/fog-build-sb-authvars:163`). If `MOK.der` becomes the + intermediate while `db` keeps the leaf, rotating a leaf silently strands + every Setup-Mode-enrolled client while appearing to work for + MokManager-enrolled ones. Changes: + - `functions.sh:5271-5272` (the `.fog-secureboot` writer): add + `SECUREBOOT_MOK_CERT=${secureBootMokCert}` alongside the existing + `SECUREBOOT_KEY`/`SECUREBOOT_CERT`. + - `packages/secureboot/fog-build-sb-authvars`: read `SECUREBOOT_MOK_CERT` + and use it for `fosCert` (line 63/163) so the **intermediate** lands in + `db`. Fall back to `SECUREBOOT_CERT` when unset, so an existing + `.fog-secureboot` from a `flat` install keeps working unchanged. + - `packages/secureboot/fog-sign-kernel`: this is the sudo helper behind the + web UI's Kernel Update page — a signing path entirely separate from + `_resignKernels()`, and easy to miss. It must pass + `--addcert "$SECUREBOOT_MOK_CERT"` when that differs from + `SECUREBOOT_CERT` (line 85), or a kernel downloaded through the GUI is + signed with no chain attached and fails to boot on exactly the clients + this design serves. + - Putting a CA in `db` is the standard UEFI model, not a workaround — + Microsoft's own `db` entries are CAs and Windows validates by chaining to + them. Verify as part of Task 0.2 on the same hardware: enroll via Setup + Mode rather than MokManager, then confirm a leaf-signed kernel boots and + still boots after the leaf is rotated. - [ ] **Step 5:** `bash -n lib/common/functions.sh && bash -n bin/installfog.sh`. - [ ] **Step 6:** Manual verify, split mode (test VM + a real UEFI client): fresh `./installfog.sh -Y` → confirm diff --git a/docs/superpowers/specs/2026-08-07-three-zone-pki-separation-design.md b/docs/superpowers/specs/2026-08-07-three-zone-pki-separation-design.md index bee344bdef..b0e039b03b 100644 --- a/docs/superpowers/specs/2026-08-07-three-zone-pki-separation-design.md +++ b/docs/superpowers/specs/2026-08-07-three-zone-pki-separation-design.md @@ -790,6 +790,64 @@ Secure Boot Key" menu item) needs **no change at all**: it fetches and enrolls whatever is at `MOK.der`. It simply enrolls a CA now rather than a leaf. +### The second enrollment path: Setup Mode, PK/KEK/db + +MokManager is not FOG's only enrollment route. `_publishSecureBootAuthVars()` +(`functions.sh:4845` ff.) drives `packages/secureboot/fog-build-sb-authvars` +to produce signed `PK.auth`/`KEK.auth`/`db.auth` blobs for the unattended +path: a client in Setup Mode enrolls FOG's PK and KEK, and a **db** that is +Microsoft's db CAs concatenated with FOG's own signing certificate +(`fog-build-sb-authvars:163`, `esl "${work}/fog-db.esl" "$FOG_OWNER_GUID" +"$fosCert"`). + +**This path must move to the intermediate too, or the whole design's benefit +evaporates on exactly the clients that used it.** Two verification chains +exist and they are enforced by different code: + +| Path | Who verifies | Trust anchor today | Must become | +|---|---|---|---| +| shim → MokList | shim's own code | `MOK.der` = signing leaf | the intermediate | +| firmware → db | the UEFI firmware | `db` entry = signing leaf | the intermediate | + +If `MOK.der` becomes the intermediate but `db` keeps the leaf, then rotating +a signing leaf silently re-breaks every Setup-Mode-enrolled client: the newly +signed kernel no longer chains to anything in their `db`, and recovering +needs a fresh KEK-signed db update pushed to each machine. The rotation +promise would hold for MokManager-enrolled clients and quietly fail for the +others — the worst possible split, because it looks like it works. + +Putting a CA in `db` is the standard UEFI model, not a workaround: the +Microsoft entries already alongside it (`MicWinProPCA2011`, +`MicCorUEFCA2011`) are themselves CAs, and every Windows bootloader validates +by chaining to them rather than by exact-certificate match. The same +`sbsign --addcert` that lets shim build the chain lets the firmware build it. + +**`SECUREBOOT_CERT` is dual-purpose, and that is the root of this** — the +third instance of the same pattern this design keeps finding. In +`$fogprogramdir/.fog-secureboot` it is read by both: + +- `fog-sign-kernel:45` → `sbsign --cert` — the **signing** certificate. +- `fog-build-sb-authvars:63` → `fosCert` → `db` — a **trust anchor**. + +So the conf gains a distinct `SECUREBOOT_MOK_CERT` naming the enrolled +anchor. `fog-build-sb-authvars` reads that for `db`; `fog-sign-kernel` keeps +reading `SECUREBOOT_CERT` to sign and additionally passes +`--addcert "$SECUREBOOT_MOK_CERT"` when the two differ. In `flat` mode both +point at the same file, every command line is byte-identical to today, and +nothing about the existing behavior changes. + +`fog-sign-kernel` needing `--addcert` is itself easy to miss: it is the sudo +helper behind the web UI's Kernel Update page, a completely separate signing +path from `_resignKernels()`. A kernel downloaded through the GUI would +otherwise be signed with no chain attached and fail to boot on precisely the +clients this design is meant to serve. + +PK and KEK are unaffected — they authorize *variable updates*, not boot, and +sign nothing that executes. Holding KEK does gain FOG something here though: +because a KEK-signed `dbx` update is possible, a compromised signing leaf can +be revoked fleet-wide without a firmware trip, which is not achievable at all +under today's flat model. + ## Data flow **Fresh install, the new default (no flag needed) or the interactive From 53ba117648a64d9d96e393873efd9c94a6a3852b Mon Sep 17 00:00:00 2001 From: JJ Fullmer Date: Fri, 7 Aug 2026 17:13:05 -0600 Subject: [PATCH 24/62] Add versioned kernel/init backups, bin/restorekernel.sh, retire update.sh's copies backupPreservedCustomizations() now snapshots the whole service/ipxe directory into a rotated generation (gen-1..N, default 3, --kernel-backup-count) rather than a fixed list of filenames. Snapshotting the directory rather than a list is what makes a per-host custom kernel/init -- bootmenu.class.php's Host->get('kernel')/get('init') -- survive an update without FOG ever having to learn its name. cp -a carries the version/tag_name xattrs downloadfiles() stamps, so each generation reports which FOS release it came from with no manifest to drift. On restore, anything NOT one of the six names downloadfiles() re-downloads is put back unconditionally -- FOG did not put it there and will never put it back. The six defaults are deliberately left alone, because the point of an update is to pick up the new kernel. --restore-kernel-backup is the single exception, passed by revertUpdate() when it re-runs the installer against the previous commit: an older commit wants the older kernels. It is applied but never persisted to .fogsettings, or every later update would silently roll back. bin/restorekernel.sh is the explicit way back: --list shows each generation with its release tag and flags custom-named files, --generation N restores one after a confirmation naming every file involved. Its own script rather than an installfog.sh flag because it must be usable precisely when an update has left an unbootable kernel and re-running the installer is what you do not want to do. It re-signs afterwards when Secure Boot is configured, since a restored kernel carries its old signature and the signing key may have rotated since. update.sh loses _updateAssetFiles, backupCustomizations, restoreCustomizations and _restorePreviousKernel entirely, and updatefog.sh loses both call sites -- all superseded. revertUpdate()'s careful restore-after-reinstall ordering goes with them: installfog.sh now does its own backup and restore within each run, so however that re-install attempt goes, it is the thing that puts the customizations back. Verified against a sandbox across four simulated updates: rotation bounds at three generations and evicts the oldest, the live kernel stays newest, a custom-named kernel survives every update, the revert flag rolls the default names back to the previous release, and restorekernel.sh lists, validates and restores correctly. Not yet run against a live FOG install. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YE3jeTexYEcAxWFbZcZgpZ --- bin/installfog.sh | 28 ++++- bin/restorekernel.sh | 223 ++++++++++++++++++++++++++++++++++++++++ bin/updatefog.sh | 8 +- lib/common/functions.sh | 59 ++++++++++- lib/common/update.sh | 93 +++++------------ 5 files changed, 341 insertions(+), 70 deletions(-) create mode 100644 bin/restorekernel.sh diff --git a/bin/installfog.sh b/bin/installfog.sh index 875ebbe051..d998bc7e03 100755 --- a/bin/installfog.sh +++ b/bin/installfog.sh @@ -131,6 +131,13 @@ usage() { echo -e "\t \t\tdefaults to \`hostname -f\`, remembered in .fogsettings" echo -e "\t --extra-server-name\tAdd an extra vhost/cert name (repeatable)" echo -e "\t \t\talongside the primary hostname and detected IPs" + echo -e "\t --kernel-backup-count\tHow many prior kernel/init generations to" + echo -e "\t \t\tkeep (default 3). Restore one with" + echo -e "\t \t\tbin/restorekernel.sh. See" + echo -e "\t \t\tdocs/SUPPORTED_CUSTOMIZATIONS.md" + echo -e "\t --restore-kernel-backup\tAlso restore the previous kernel/init set" + echo -e "\t \t\tthis run. Used by updatefog.sh when reverting;" + echo -e "\t \t\tnot normally passed by hand" echo -e "\t-N --mysqldbname\t\tSpecify the FOG database name" echo -e "\t \t\t\t\tdefaults to fog" echo -e "\t-B --backuppath\t\tSpecify the backup path" @@ -170,7 +177,7 @@ usage() { sextraServerNames=() shortopts="h?odEUHSCKYyXTFf:c:W:D:B:s:e:N:l" -longopts="help,uninstall,purge-db,purge-images,purge-snapins,purge-ssl,purge-user,purge-all,dry-run,force,mysqldbname:,ssl-path:,oldcopy,no-vhost,no-defaults,no-upgrade,no-htmldoc,force-https,no-force-https,recreate-keys,recreate-CA,recreate-Ca,recreate-cA,recreate-ca,external-ca,ca-cert:,ca-key:,ca-root:,autoaccept,file:,docroot:,webroot:,backuppath:,startrange:,endrange:,no-exportbuild,exitFail,no-tftpbuild,list-packages,fogprogramdir:,secure-boot-key:,secure-boot-cert:,no-secure-boot,hostname:,extra-server-name:" +longopts="help,uninstall,purge-db,purge-images,purge-snapins,purge-ssl,purge-user,purge-all,dry-run,force,mysqldbname:,ssl-path:,oldcopy,no-vhost,no-defaults,no-upgrade,no-htmldoc,force-https,no-force-https,recreate-keys,recreate-CA,recreate-Ca,recreate-cA,recreate-ca,external-ca,ca-cert:,ca-key:,ca-root:,autoaccept,file:,docroot:,webroot:,backuppath:,startrange:,endrange:,no-exportbuild,exitFail,no-tftpbuild,list-packages,fogprogramdir:,secure-boot-key:,secure-boot-cert:,no-secure-boot,hostname:,extra-server-name:,kernel-backup-count:,restore-kernel-backup" optargs=$(getopt -o $shortopts -l $longopts -n "$0" -- "$@") [[ $? -ne 0 ]] && usage @@ -465,6 +472,20 @@ while :; do ssecureboot=0 shift ;; + --kernel-backup-count) + if [[ -n "${2}" && "${2}" =~ ^[0-9]+$ && "${2}" -ge 1 ]]; then + skernelBackupCount="${2}" + else + echo "$1 requires a positive integer after" + usage + exit 3 + fi + shift 2 + ;; + --restore-kernel-backup) + srestoreKernelBackup=1 + shift + ;; --) shift break @@ -662,6 +683,11 @@ esac [[ -n $ssecureBootKey ]] && secureBootKey=$ssecureBootKey [[ -n $ssecureBootCert ]] && secureBootCert=$ssecureBootCert [[ -n $ssecureboot ]] && secureboot=$ssecureboot +[[ -n $skernelBackupCount ]] && kernelBackupGenerations=$skernelBackupCount +# Deliberately NOT persisted to .fogsettings: this is a one-shot instruction +# for a single run (revertUpdate passes it), not a preference. Persisting it +# would make every later update silently roll the kernels back. +restoreKernelBackup=${srestoreKernelBackup:-0} # Secure Boot signing is generated by default now (see _ensureSecureBootKeys), # but an explicitly supplied key is still only meaningful as a pair. Refuse half diff --git a/bin/restorekernel.sh b/bin/restorekernel.sh new file mode 100644 index 0000000000..6a0ce00f28 --- /dev/null +++ b/bin/restorekernel.sh @@ -0,0 +1,223 @@ +#!/bin/bash +# +# FOG is a computer imaging solution. +# Copyright (C) 2007 Chuck Syperski & Jian Zhang +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# Restores a previous kernel/init generation captured by +# backupPreservedCustomizations() (lib/common/functions.sh) under +# $fogprogramdir/customizations/kernel-backups/. +# +# Its own script rather than an installfog.sh flag because it is a rare, +# deliberate act with a blast radius of "every machine that PXE boots from +# here", and because it must be usable when an update has already replaced a +# working kernel with one that does not boot -- i.e. exactly when re-running +# the installer is the thing you do not want to do. +# +# See docs/SUPPORTED_CUSTOMIZATIONS.md. +bindir=$(dirname $(readlink -f "$BASH_SOURCE")) +cd $bindir +workingdir=$(pwd) + +if [[ ! $EUID -eq 0 ]]; then + echo "restorekernel.sh must be run as root user" + exit 1 +fi + +usage() { + echo -e "Usage: $0 [-h?] (--list | --generation ) [--yes]" + echo -e "\t-h -? --help\t\tDisplay this info" + echo -e "\t --list\t\tShow each stored generation and the FOS release" + echo -e "\t \t\tits kernels came from" + echo -e "\t --generation\tRestore generation N into the live iPXE" + echo -e "\t \t\tdirectory. 1 is the most recent snapshot," + echo -e "\t \t\ttaken at the START of the last install/update" + echo -e "\t \t\t-- so it holds what was running BEFORE it" + echo -e "\t-y --yes\t\tSkip the confirmation prompt" + exit 0 +} + +shortopts="h?y" +longopts="help,list,generation:,yes" +optargs=$(getopt -o $shortopts -l $longopts -n "$0" -- "$@") +[[ $? -ne 0 ]] && usage +eval set -- "$optargs" + +doList=0 +generation="" +autoYes="" +while :; do + case $1 in + -h | -\? | --help) + usage + ;; + --list) + doList=1 + shift + ;; + --generation) + generation="$2" + shift 2 + ;; + -y | --yes) + autoYes=1 + shift + ;; + --) + shift + break + ;; + *) + echo "Error: unhandled option '$1'." + exit 10 + ;; + esac +done + +[[ ! -d ./error_logs/ ]] && mkdir -p ./error_logs >/dev/null 2>&1 +error_log="${workingdir}/error_logs/fog_restorekernel_error.log" +: > "$error_log" + +# exitFail so a failing errorStat inside the sourced functions returns control +# here instead of killing the script mid-restore. +exitFail=1 +. ../lib/common/functions.sh + +[[ -z $fogprogramdir && -r /etc/fog/fog.conf ]] && . /etc/fog/fog.conf +[[ -z $fogprogramdir ]] && fogprogramdir="/opt/fog" +fogprogramdir="${fogprogramdir%/}" + +if [[ ! -r "$fogprogramdir/.fogsettings" ]]; then + echo " * No existing FOG install found at $fogprogramdir (.fogsettings missing)." + echo " * restorekernel.sh works on an EXISTING install -- run installfog.sh first." + exit 1 +fi +. "$fogprogramdir/.fogsettings" + +kbdir="${fogprogramdir}/customizations/kernel-backups" +ipxedir="${webdirdest}service/ipxe" + +if [[ ! -d $kbdir ]]; then + echo " * No kernel backups yet at $kbdir." + echo " * They are written at the start of each install/update, so the first" + echo " generation appears after the next one." + exit 1 +fi + +# Reports the FOS release a file came from using the xattr downloadfiles() +# stamps at download time, so a generation is self-describing and there is no +# manifest to drift out of sync. Older files, or a filesystem mounted without +# user_xattr, simply have none. +tagof() { + local t + t=$(attr -q -g tag_name "$1" 2>/dev/null) || t="" + [[ -z $t ]] && t="unknown release" + echo "$t" +} + +listGenerations() { + local gendir found=0 f + for gendir in "$kbdir"/gen-*; do + [[ -d $gendir ]] || continue + found=1 + echo " $(basename "$gendir"):" + for f in "$gendir"/bzImage "$gendir"/bzImage32 "$gendir"/arm_Image; do + [[ -f $f ]] || continue + echo " $(basename "$f") ($(tagof "$f"))" + done + # Anything that is not one of the six names FOG re-downloads is a file + # the admin put there -- a per-host custom kernel/init. Worth showing, + # because it is the part no update would ever put back. + for f in "$gendir"/*; do + [[ -f $f ]] || continue + case " bzImage bzImage32 arm_Image init.xz init_32.xz arm_init.cpio.gz " in + *" $(basename "$f") "*) continue ;; + esac + echo " $(basename "$f") (custom)" + done + done + [[ $found -eq 0 ]] && echo " (none yet)" +} + +if [[ $doList -eq 1 ]]; then + echo " * Kernel/init generations under $kbdir:" + listGenerations + exit 0 +fi + +if [[ -z $generation ]]; then + echo " * Pass --list to see what is stored, or --generation to restore." + usage +fi +if [[ ! $generation =~ ^[0-9]+$ || $generation -lt 1 ]]; then + echo " * --generation takes a positive integer (1 is the most recent)." + exit 1 +fi + +gendir="${kbdir}/gen-${generation}" +if [[ ! -d $gendir ]]; then + echo " * No such generation: $gendir" + echo " * Available:" + listGenerations + exit 1 +fi +if [[ ! -d $ipxedir ]]; then + echo " * Live iPXE directory not found at $ipxedir." + exit 1 +fi + +echo " * About to restore gen-${generation} into ${ipxedir}:" +for f in "$gendir"/*; do + [[ -f $f ]] || continue + echo " $(basename "$f") ($(tagof "$f"))" +done +echo +echo " * Every machine that PXE boots from this server will use these files." +if [[ -z $autoYes ]]; then + echo -n " * Continue? (y/N) " + read -r reply + case $reply in + [Yy]|[Yy][Ee][Ss]) ;; + *) echo " * Aborted."; exit 0 ;; + esac +fi + +# cp -a to carry the version/tag_name xattrs across, so the restored files +# still report which release they came from and a later --list stays honest. +dots "Restoring gen-${generation}" +st=0 +cp -a "${gendir}/." "${ipxedir}/" >>$error_log 2>&1 || st=1 +if [[ $st -ne 0 ]]; then + echo "Failed" + echo " * Could not copy the generation into place. See $error_log." + exit 1 +fi +chown -R ${username}:${apacheuser} "$ipxedir" >>$error_log 2>&1 +errorStat 0 + +# The restored kernels carry whatever signature they had when they were +# snapshotted. If the Secure Boot signing key has been rotated since, that +# signature no longer verifies against the current certificate and the client +# refuses to boot -- so re-sign rather than leave a subtly broken set behind. +# _resignKernels() skips anything already carrying a valid signature, so this +# is a no-op in the common case where the key has not changed. +if [[ -n $secureBootKey && -n $secureBootCert ]]; then + _resignKernels +fi + +echo +echo " * gen-${generation} restored." +echo " * If a client is mid-task it will still be using the previous files;" +echo " re-deploy or reboot it to pick these up." diff --git a/bin/updatefog.sh b/bin/updatefog.sh index 9f36d36d72..e3c92b54c0 100755 --- a/bin/updatefog.sh +++ b/bin/updatefog.sh @@ -266,7 +266,10 @@ if [[ -z $autoYes ]]; then esac fi -backupCustomizations +# No backup call here any more. installfog.sh backs up and restores within its +# own run (backupPreservedCustomizations / restorePreservedCustomizations), so +# the protection covers a bare ./installfog.sh too -- which is how most people +# upgrade, and which this wrapper could never have protected. if ! gitUpdateToBranch "$branch"; then echo " * Git update failed -- nothing was installed. See $error_log." exit 1 @@ -281,7 +284,8 @@ installStatus=$? cd "$workingdir" if [[ $installStatus -eq 0 ]]; then - restoreCustomizations + # Likewise no restore call: the install run that just succeeded already + # put the customizations back itself. echo " * Update completed successfully." exit 0 fi diff --git a/lib/common/functions.sh b/lib/common/functions.sh index 79d770ef76..7e2c82a7d0 100755 --- a/lib/common/functions.sh +++ b/lib/common/functions.sh @@ -150,7 +150,31 @@ backupPreservedCustomizations() { echo " space under ${customizationsDir} and re-run. See $error_log." exit 1 fi - [[ $warn -ne 0 ]] && echo -n "(some optional refind files could not be backed up) " + + # Snapshot the whole directory into a rotated generation, rather than a + # fixed list of filenames. Two things fall out of that: a per-host custom + # kernel/init (bootmenu.class.php's Host->get('kernel')/get('init')) is + # captured without FOG ever having to learn its name, and a generation is + # a complete, coherent set rather than an assortment. + # + # Bounded at $kernelBackupGenerations because this is otherwise unlimited + # growth on disk the admin provisioned for images, not for history. + [[ -z $kernelBackupGenerations || ! $kernelBackupGenerations =~ ^[0-9]+$ || $kernelBackupGenerations -lt 1 ]] && kernelBackupGenerations=3 + local kbdir="${customizationsDir}/kernel-backups" k + if [[ -d $ipxedir ]]; then + mkdir -p "$kbdir" >>$error_log 2>&1 || warn=1 + rm -rf "${kbdir}/gen-${kernelBackupGenerations}" >>$error_log 2>&1 + for ((k = kernelBackupGenerations - 1; k >= 1; k--)); do + [[ -d "${kbdir}/gen-${k}" ]] && mv "${kbdir}/gen-${k}" "${kbdir}/gen-$((k + 1))" >>$error_log 2>&1 + done + mkdir -p "${kbdir}/gen-1" >>$error_log 2>&1 || warn=1 + # cp -a preserves the version/tag_name xattrs downloadfiles() stamps on + # each kernel, so every generation says which FOS release it came from + # without a separate manifest to keep in sync. + cp -a "${ipxedir}/." "${kbdir}/gen-1/" >>$error_log 2>&1 || warn=1 + fi + + [[ $warn -ne 0 ]] && echo -n "(some optional files could not be backed up) " errorStat 0 } # Restores what backupPreservedCustomizations() saved, AFTER @@ -172,6 +196,33 @@ restorePreservedCustomizations() { for f in refind.conf refind.efi refind_x64.efi refind_ia32.efi refind_aa64.efi; do [[ -f "${customizationsDir}/ipxe-legacy/${f}" ]] && { cp -f "${customizationsDir}/ipxe-legacy/${f}" "${ipxedir}/${f}" >>$error_log 2>&1 || st=1; } done + + # Anything in the newest generation that is NOT one of the six names + # downloadfiles() re-downloads is, by definition, a file FOG did not put + # there and will never put back -- a per-host custom kernel or init. Those + # are restored unconditionally. + # + # The six default names are deliberately NOT restored: the point of an + # update is to pick up the new kernel. bin/restorekernel.sh is the + # explicit, admin-invoked way back to an older one. + # + # $restoreKernelBackup is the single exception, set only by + # --restore-kernel-backup, which revertUpdate() passes when it re-runs the + # installer against the previous commit. An older commit wants the older + # kernels too, and that is the behavior the retired _restorePreviousKernel() + # used to provide on that path. + local kbdir="${customizationsDir}/kernel-backups" + local defaultnames=" bzImage bzImage32 arm_Image init.xz init_32.xz arm_init.cpio.gz " + local bn + if [[ -d "${kbdir}/gen-1" ]]; then + for f in "${kbdir}/gen-1"/*; do + [[ -f $f ]] || continue + bn=$(basename "$f") + if [[ $defaultnames != *" $bn "* || ${restoreKernelBackup:-0} -eq 1 ]]; then + cp -a "$f" "${ipxedir}/${bn}" >>$error_log 2>&1 || st=1 + fi + done + fi [[ -d $ipxedir ]] && chown -R ${username}:${apacheuser} "$ipxedir" >>$error_log 2>&1 # Never fatal, unlike the backup side. By this point configureHttpd() has # already rebuilt the web tree, so aborting would strand a nearly-complete @@ -3296,6 +3347,12 @@ writeUpdateFile() { # CSR (stale public key) while the private key on disk is the ACME # key, producing a cert/key mismatch that stops the web server. acmeLeaf + # How many prior kernel/init generations backupPreservedCustomizations() + # keeps under customizations/kernel-backups. A genuine persisted + # preference like fog_update_channel, not a record: an admin who chose + # deeper history must keep it across every future upgrade, or the + # generations they were relying on get evicted by the next run. + kernelBackupGenerations ) # Keys written by older installers that must be stripped on upgrade. local -a deprecatedKeys=( storageftpuser storageftppass bootfilename notpxedefaultfile php_verAdds ) diff --git a/lib/common/update.sh b/lib/common/update.sh index 1bcc342db4..a9ff04fbf3 100644 --- a/lib/common/update.sh +++ b/lib/common/update.sh @@ -16,59 +16,19 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # -# Functions used only by bin/updatefog.sh: backing up/restoring the handful of -# files installfog.sh's ipxe asset sync can overwrite, and the git -# fetch/checkout/revert cycle around a channel update. Kept out of -# functions.sh, which installfog.sh alone already runs to nearly 5000 lines. +# Functions used only by bin/updatefog.sh: the git fetch/checkout/revert cycle +# around a channel update. Kept out of functions.sh, which installfog.sh alone +# already runs to nearly 5000 lines. # -# All paths below are derived from $webdirdest (set by lib/common/utils.sh -# from docroot/webroot, both restored from .fogsettings before this file is -# sourced), never hardcoded -- see fog_git_updater.sh's history of assuming -# /var/www/html/fog for why that matters. -[[ -z $updateBackupDir ]] && updateBackupDir="${fogprogramdir}/update-backup" - -# The custom PXE background and the kernel/init set installfog.sh's ipxe -# asset sync can silently overwrite with FOG's own defaults. refind is -# optional/legacy -- only backed up if actually present. -_updateAssetFiles() { - echo "bg.png bzImage bzImage32 arm_Image init.xz init_32.xz arm_init.cpio.gz refind.conf refind.efi refind_x64.efi refind_ia32.efi refind_aa64.efi" -} - -backupCustomizations() { - dots "Backing up customizations before update" - local ipxedir="${webdirdest}service/ipxe" f st=0 - mkdir -p "$updateBackupDir" >>$error_log 2>&1 || st=1 - for f in $(_updateAssetFiles); do - [[ -f "$ipxedir/$f" ]] && { cp -f "$ipxedir/$f" "$updateBackupDir/$f" >>$error_log 2>&1 || st=1; } - done - errorStat $st -} - -# Success path: put the custom background and any refind files back over -# whatever installfog.sh just installed. The kernel set is deliberately left -# alone here -- the point of an update is to pick up the latest kernel; the -# backup stays on disk purely as the manual/revert fallback below. -restoreCustomizations() { - dots "Restoring customizations after update" - local ipxedir="${webdirdest}service/ipxe" f st=0 - for f in bg.png refind.conf refind.efi refind_x64.efi refind_ia32.efi refind_aa64.efi; do - [[ -f "$updateBackupDir/$f" ]] && { cp -f "$updateBackupDir/$f" "$ipxedir/$f" >>$error_log 2>&1 || st=1; } - done - chown -R ${username}:${apacheuser} "$ipxedir" >>$error_log 2>&1 - errorStat $st -} - -# Revert path only: puts the previous kernel/init set back, on top of -# whatever restoreCustomizations() already restored. -_restorePreviousKernel() { - dots "Restoring previous kernel set" - local ipxedir="${webdirdest}service/ipxe" f st=0 - for f in bzImage bzImage32 arm_Image init.xz init_32.xz arm_init.cpio.gz; do - [[ -f "$updateBackupDir/$f" ]] && { cp -f "$updateBackupDir/$f" "$ipxedir/$f" >>$error_log 2>&1 || st=1; } - done - chown -R ${username}:${apacheuser} "$ipxedir" >>$error_log 2>&1 - errorStat $st -} +# This file used to also own backing up and restoring the files installfog.sh's +# asset sync overwrites (_updateAssetFiles, backupCustomizations, +# restoreCustomizations, _restorePreviousKernel). Those are gone, and the job +# moved into installfog.sh itself -- see backupPreservedCustomizations and +# restorePreservedCustomizations in lib/common/functions.sh. Living here meant +# a bare `./installfog.sh` upgrade, which is how most people upgrade, got no +# protection at all; and the old list was hardcoded to "bg.png" rather than +# reading the FOG_IPXE_BG_FILE setting whose entire purpose is renaming that +# file. # Fetches, checks out, and hard-resets $fog_git_path to $1 (a branch name -- # the caller has already resolved this from either $fog_update_channel via @@ -95,26 +55,27 @@ gitUpdateToBranch() { return $st } -# Failure path: put the git checkout back where it was, re-run installfog.sh -# against the reverted commit, and ONLY THEN restore every backed up file -# (including the kernel set, unlike the success path). +# Failure path: put the git checkout back where it was and re-run +# installfog.sh against the reverted commit. +# +# The restore used to happen here, after the re-install, because installfog.sh +# was what overwrote bg.png and the kernel set and could itself fail partway +# through -- restoring first and re-installing second left the fresh defaults +# in place instead of the admin's customizations. # -# The restore must come AFTER the re-install, not before: installfog.sh's own -# asset sync is what can overwrite bg.png/the kernel set in the first place, -# and that re-install attempt can itself fail partway through -- after it has -# already re-overwritten those files but before it finishes. Restoring first -# and re-installing second left exactly that case with the fresh defaults -# still in place instead of the admin's customizations, which defeats the -# entire point of a revert. Restoring last guarantees the final state on disk -# has the customizations back no matter how the re-install attempt goes. +# That ordering problem is gone: installfog.sh now backs up and restores +# within its own run (backupPreservedCustomizations / +# restorePreservedCustomizations), so however the re-install attempt goes, it +# is the thing that puts the customizations back. --restore-kernel-backup is +# the one addition the revert path needs, telling that run to also roll the +# default-named kernel/init set back -- an older commit wants the older +# kernels, which a normal update deliberately does not do. revertUpdate() { echo " * Reverting to the previous commit ($updatePrevCommit)" dots "Reverting git checkout" git -C "$fog_git_path" reset --hard "$updatePrevCommit" >>$error_log 2>&1 errorStat $? dots "Re-running installfog.sh against the reverted commit" - (cd "$fog_git_path/bin" && bash installfog.sh -Y $updateVhostFlag >>$error_log 2>&1) + (cd "$fog_git_path/bin" && bash installfog.sh -Y $updateVhostFlag --restore-kernel-backup >>$error_log 2>&1) errorStat $? - _restorePreviousKernel - restoreCustomizations } From a9b071b95645422d06a4dad35c2af78645ba13fb Mon Sep 17 00:00:00 2001 From: JJ Fullmer Date: Fri, 7 Aug 2026 17:14:54 -0600 Subject: [PATCH 25/62] Document what survives an install/update in docs/SUPPORTED_CUSTOMIZATIONS.md Every preservation mechanism added on this branch was invisible unless you read the installer source. This says plainly, per category, what is kept automatically, what you have to place yourself, and what is deliberately not kept -- including the awkward parts, so none of them are discovered the hard way: edits inside the FOG-managed vhost block are overwritten, direct edits to default.ipxe are regenerated, and generations past --kernel-backup-count are evicted. Cross-linked from installfog.sh's -F/--no-vhost and --kernel-backup-count help and from updatefog.sh's usage, since those are the flags where someone is most likely to be deciding what they are about to lose. Two claims corrected against the source while writing it: the pre-rebuild web tree snapshot lands under $backupPath (/home/ by default, -B to change), not the docroot; and the per-host override fields are labelled Host Kernel/Host Init. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YE3jeTexYEcAxWFbZcZgpZ --- bin/installfog.sh | 7 +- bin/updatefog.sh | 2 + docs/SUPPORTED_CUSTOMIZATIONS.md | 182 +++++++++++++++++++++++++++++++ 3 files changed, 190 insertions(+), 1 deletion(-) create mode 100644 docs/SUPPORTED_CUSTOMIZATIONS.md diff --git a/bin/installfog.sh b/bin/installfog.sh index d998bc7e03..d395a68fe4 100755 --- a/bin/installfog.sh +++ b/bin/installfog.sh @@ -163,7 +163,12 @@ usage() { echo -e "\t-E --no-exportbuild\t\tSkip building nfs file" echo -e "\t-X --exitFail\t\tDo not exit if item fails" echo -e "\t-T --no-tftpbuild\t\tDo not rebuild the tftpd config file" - echo -e "\t-F --no-vhost\t\tDo not overwrite vhost file" + echo -e "\t-F --no-vhost\t\tDo not touch the vhost file at all. FOG" + echo -e "\t \t\t\tnormally rewrites only the region between its" + echo -e "\t \t\t\tMANAGED BLOCK markers and leaves your own" + echo -e "\t \t\t\tadditions alone, so skipping also skips its" + echo -e "\t \t\t\tsecurity fixes to the parts it owns." + echo -e "\t \t\t\tSee docs/SUPPORTED_CUSTOMIZATIONS.md" echo -e "\t-l --list-packages\t\tList of the basic packages FOG needs for install or is currently installed for FOG" echo -e "\t --secure-boot-key\t\tPrivate key used to re-sign the FOS" echo -e "\t \t\t\tkernels for UEFI Secure Boot" diff --git a/bin/updatefog.sh b/bin/updatefog.sh index e3c92b54c0..4e6d202667 100755 --- a/bin/updatefog.sh +++ b/bin/updatefog.sh @@ -62,6 +62,8 @@ usage() { echo -e "\t \t\tto the parts it owns" echo -e "\t --overwrite-vhost\tDeprecated no-op: this is now the default" echo -e "\t-y --yes\t\tSkip the confirmation prompt (for cron/GUI use)" + echo -e "\n\tWhat survives an update, and where to put customizations so" + echo -e "\tthey do: docs/SUPPORTED_CUSTOMIZATIONS.md" exit 0 } diff --git a/docs/SUPPORTED_CUSTOMIZATIONS.md b/docs/SUPPORTED_CUSTOMIZATIONS.md new file mode 100644 index 0000000000..f94553539d --- /dev/null +++ b/docs/SUPPORTED_CUSTOMIZATIONS.md @@ -0,0 +1,182 @@ +# Supported customizations + +What FOG preserves for you across an install or update, what it deliberately +does not, and where to put things so they survive. + +## How to read this document + +**Automatic** means `installfog.sh` preserves it on every run with no action +from you. That includes a bare `./installfog.sh` upgrade — it is not limited +to updates driven through `bin/updatefog.sh`. + +**Supported, but yours to place** means FOG will not overwrite it and provides +a defined place for it, but does not create or manage it. + +**Not preserved** means exactly that. Those cases are listed at the end +rather than left for you to discover. + +Everything preserved automatically is copied to +`/opt/fog/customizations/` (strictly, `$fogprogramdir/customizations`) before +the web tree is rebuilt, and copied back afterwards. That directory is +outside the web root, which is why it survives — the installer rebuilds +`/var/www/html/fog` wholesale on every run. + +--- + +## iPXE boot menu background + +**Automatic**, including when you have renamed the file. + +`FOG_IPXE_BG_FILE` (Web UI → FOG Configuration → iPXE Menu Settings) names +the background image. FOG reads that setting's actual value, so a renamed +file is protected — not just the stock `bg.png`. + +| Customization | How it is preserved | Where the copy lives | +|---|---|---| +| Replaced `bg.png` in place | Backed up before the web tree is rebuilt, restored after | `/opt/fog/customizations/ipxe-bg/bg.png` | +| Renamed background via `FOG_IPXE_BG_FILE` | Same, under whatever name the setting holds | `/opt/fog/customizations/ipxe-bg/.png` | +| Legacy `refind.*` files | Backed up and restored if present | `/opt/fog/customizations/ipxe-legacy/` | + +Place the image in `/service/ipxe/` and set `FOG_IPXE_BG_FILE` to +its filename. A path is not accepted — only a filename. + +> If FOG finds your background but cannot copy it to safety, the install +> **stops before** rebuilding the web tree, with your file still untouched. +> That is deliberate: aborting is recoverable, proceeding is not. + +--- + +## Web server virtual host (Apache / nginx) + +**Automatic for anything outside FOG's block. FOG owns what is between the +markers and will rewrite it every run.** + +The generated vhost is wrapped in: + +``` +# === FOG MANAGED BLOCK -- DO NOT EDIT BETWEEN THESE LINES ... === + ... everything FOG generates ... +# === END FOG MANAGED BLOCK === +``` + +Put your own directives **outside** those markers — above or below — and they +survive every update untouched. FOG refreshes only the inside, which is how +you keep getting its cipher, header and rewrite-rule fixes without losing +your own configuration. + +| Customization | How it is preserved | +|---|---| +| Extra directives, headers, `location`/`Directory` blocks | Keep them outside the markers; never touched | +| Extra hostnames / DNS aliases | Use `--extra-server-name` (repeatable) so they land in both the vhost **and** the certificate SAN | +| Primary hostname | Use `--hostname`; remembered in `.fogsettings` | +| Custom certificate paths | Point the vhost's cert directives at your paths **outside** the block, or replace the files FOG already references | + +On the first run after upgrading into this scheme, a vhost with no markers is +**appended to**, never overwritten — your existing file is left in place with +FOG's block added after it. Review it once and move anything you want FOG to +stop managing outside the markers. + +`--no-vhost` (installfog.sh `-F`) still skips the vhost entirely. It is +rarely what you want now: it also skips FOG's own security fixes to the parts +it owns. + +--- + +## Kernels and inits (`bzImage`, `init.xz`, …) + +**Default-named files are replaced on purpose. Custom-named files are +preserved automatically. Previous versions are kept for you to roll back to.** + +Picking up a newer kernel is the point of an update, so `bzImage`, +`bzImage32`, `arm_Image`, `init.xz`, `init_32.xz` and `arm_init.cpio.gz` are +always replaced with the release being installed. + +| Customization | How it is preserved | +|---|---| +| Per-host custom kernel/init (a host's **Host Kernel** / **Host Init** fields) | Restored automatically — FOG never re-downloads a file it did not ship | +| Previously working kernel set | Kept as a numbered generation; restore with `bin/restorekernel.sh` | + +```bash +./restorekernel.sh --list # what is stored, and which release each came from +./restorekernel.sh --generation 1 # roll back to the set from before the last update +``` + +`gen-1` is the most recent snapshot, taken at the **start** of the last +install — so it holds what was running *before* it. Three generations are +kept by default; change that with `installfog.sh --kernel-backup-count N`. + +Restoring re-signs the kernels if Secure Boot is configured, since a restored +kernel carries its old signature and the signing key may have rotated. + +--- + +## Custom PXE scripts (`custom.ipxe`) + +**Supported, but yours to place.** FOG will never create, edit or delete it. + +`default.ipxe` is regenerated in full on every run, so editing it directly +does not survive. Instead, drop a script at the TFTP root: + +``` +/tftpboot/custom.ipxe +``` + +FOG chains to it before its own boot logic. When your script finishes +normally, control returns and FOG boots as usual — you do not need to chain +back. If the file is absent, boot proceeds exactly as it always has. + +``` +#!ipxe +echo Booting in 10 seconds... +sleep 10 +``` + +This is the supported replacement for maintaining a modified boot file by +hand — for boot delays, prompts, or a site-specific menu ahead of FOG's. + +--- + +## Secure Boot certificates + +**Automatic**, for both FOG-generated and admin-supplied keys. + +FOG's own signing key is generated once at `/opt/fog/secureboot/MOK.{key,pem}` +and **never regenerated**, because a new key silently invalidates enrollment +on every machine that already trusted the old one. + +| Customization | How it is preserved | +|---|---| +| FOG's generated signing key | Lives outside the web root; nothing in the installer deletes it | +| Your own key via `--secure-boot-key` / `--secure-boot-cert` | Copied to `/opt/fog/secureboot/admin-MOK.{key,pem}` and used from there | +| Platform keys (PK/KEK) | Same; generated once, never regenerated | + +Supplying your own pair does **not** overwrite FOG's generated one — they sit +side by side, so you can go back. Your original file is never modified; FOG +uses a copy. This matters if you keep the pair somewhere the installer +rebuilds, such as under the web root: without the copy it would be deleted +mid-install. + +--- + +## What is NOT automatically preserved + +Listed plainly so none of it is a surprise. + +- **Edits inside the FOG-managed vhost block.** They are overwritten on the + next run. Move them outside the markers. +- **Direct edits to `default.ipxe`.** Regenerated every run — use + `custom.ipxe` instead. +- **A kernel-signing key rotated after a generation was captured.** Restoring + that generation re-signs with the *current* key, which is correct, but any + client enrolled against the old key still needs re-enrollment. +- **More than `--kernel-backup-count` generations back.** The oldest is + evicted on each run; the default keeps three. +- **`php.ini` / MariaDB config beyond FOG's own lines.** FOG patches only the + specific directives it manages and leaves the rest of those files alone, so + your edits generally survive — but they are not backed up, and are not + restored if something else removes them. +- **Anything under the web root that FOG does not ship.** The tree is rebuilt + wholesale on every run. Only the categories above are copied to safety + first; a snapshot of the previous tree is left at + `/fog_web_.BACKUP` — `/home/` by default, settable + with `installfog.sh -B` — for manual recovery. From 617527438a58571d796aaf5ec15f47f41fffbc52 Mon Sep 17 00:00:00 2001 From: JJ Fullmer Date: Fri, 7 Aug 2026 18:41:12 -0600 Subject: [PATCH 26/62] Fix: managed vhost splice lost prior content because the caller moves the file First run against a real FOG server wiped a hand-added vhost block -- exactly what this feature exists to prevent. createSSLCA() runs `mv -fv $etcconf $etcconf.$timestamp` before generating, so diffconfig() has something to compare against. That happens BEFORE beginManagedVhost, so by the time spliceManagedBlock is called $etcconf does not exist: it took the "no file" branch and wrote a fresh single-block file, and the admin's content stayed behind in the timestamped backup. spliceManagedBlock now takes the prior-content path as a third argument and restores from it when the live file is missing. beginManagedVhost records $etcconf.$timestamp for that purpose. Every sandbox test passed because they all called spliceManagedBlock against a file that was still in place, so the branch that actually runs in production was never exercised. A test that does not reproduce the caller's sequence proves less than it appears to. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YE3jeTexYEcAxWFbZcZgpZ --- lib/common/functions.sh | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/lib/common/functions.sh b/lib/common/functions.sh index 7e2c82a7d0..19ac7fcc5a 100755 --- a/lib/common/functions.sh +++ b/lib/common/functions.sh @@ -3616,7 +3616,19 @@ FOG_MANAGED_END='# === END FOG MANAGED BLOCK ===' # fresh block and touch nothing that was already there. Never guess at a # partial patch. spliceManagedBlock() { - local conffile="$1" contentfile="$2" + local conffile="$1" contentfile="$2" priorfile="$3" + # $3 names where the file's PREVIOUS content lives, when the caller has + # already moved it aside. createSSLCA() does exactly that -- it runs + # `mv -fv $etcconf $etcconf.$timestamp` before generating, so diffconfig() + # has something to compare against -- which means by the time we are called + # $conffile does not exist and the admin's content is in the backup. + # + # Missing this is what made the first real-server test wipe a hand-added + # vhost block: every sandbox test had called this against a file that was + # still in place, so the "no file" branch never ran when it mattered. + if [[ ! -f "$conffile" && -n $priorfile && -f "$priorfile" ]]; then + cp -f "$priorfile" "$conffile" 2>/dev/null + fi if [[ ! -f "$conffile" ]]; then { echo "$FOG_MANAGED_BEGIN"; cat "$contentfile"; echo "$FOG_MANAGED_END"; } > "$conffile" return $? @@ -3643,16 +3655,20 @@ spliceManagedBlock() { # the wrong path. beginManagedVhost() { vhostfinal="$etcconf" + # Callers mv the original to $etcconf.$timestamp just above this, so that + # is where the admin's previous content is. Remember it -- it is the base + # the new block gets spliced into. + vhostprior="${etcconf}.${timestamp}" etcconf="${etcconf}.fogblock.$$" : > "$etcconf" } endManagedVhost() { local generated="$etcconf" etcconf="$vhostfinal" - spliceManagedBlock "$etcconf" "$generated" + spliceManagedBlock "$etcconf" "$generated" "$vhostprior" local st=$? rm -f "$generated" >>$error_log 2>&1 - unset vhostfinal + unset vhostfinal vhostprior return $st } createSSLCA() { From 745912cb0eee9a15a529819bf329b5618cb3f153 Mon Sep 17 00:00:00 2001 From: JJ Fullmer Date: Fri, 7 Aug 2026 18:45:41 -0600 Subject: [PATCH 27/62] Fix: customizationsDir resolved at source time, before fogprogramdir existed Real-server run wrote every preserved customization to /customizations -- the filesystem root -- instead of /opt/fog/customizations. functions.sh is sourced by installfog.sh at line ~93, but $fogprogramdir is not settled until config.sh runs several hundred lines later. A top-level assignment therefore expanded to "${fogprogramdir}/customizations" with fogprogramdir empty, giving "/customizations". The backup and restore both worked, consistently, against the wrong directory. Resolved on call instead, with an /opt/fog fallback matching what config.sh would have set. The sandbox missed it because every harness set fogprogramdir before sourcing functions.sh, which is the opposite order from the real caller. Second bug in this session found only by running against a real install, both from the same root cause: the test reproduced the function, not the sequence that calls it. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YE3jeTexYEcAxWFbZcZgpZ --- lib/common/functions.sh | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/lib/common/functions.sh b/lib/common/functions.sh index 19ac7fcc5a..36cc8037e7 100755 --- a/lib/common/functions.sh +++ b/lib/common/functions.sh @@ -81,7 +81,18 @@ backupReports() { # never inside $webdirdest -- that is the same "survives the wipe by # construction" property $fogprogramdir/secureboot already relies on, rather # than a copy that has to be re-made correctly every time. -[[ -z $customizationsDir ]] && customizationsDir="${fogprogramdir}/customizations" +# +# Resolved on CALL, not when this file is sourced. installfog.sh sources +# functions.sh at line ~93 but does not settle $fogprogramdir until config.sh +# runs several hundred lines later, so a top-level assignment here evaluated to +# "/customizations" and wrote the backups to the filesystem root. That is what +# the first real-server run actually did -- the sandbox never caught it because +# it always set $fogprogramdir before sourcing. +_resolveCustomizationsDir() { + [[ -n $customizationsDir ]] && return 0 + local base="${fogprogramdir:-/opt/fog}" + customizationsDir="${base%/}/customizations" +} # Backs up whatever is actually customized under $webdirdest/service/ipxe/ # BEFORE configureHttpd() destroys that tree. # @@ -95,6 +106,7 @@ backupReports() { # a re-read of the setting (which an admin could have changed mid-install). backupPreservedCustomizations() { dots "Backing up customizations" + _resolveCustomizationsDir local ipxedir="${webdirdest}service/ipxe" local f st=0 # Severity is split deliberately, because errorStat() EXITS the installer @@ -187,6 +199,7 @@ backupPreservedCustomizations() { # instead. restorePreservedCustomizations() { dots "Restoring customizations" + _resolveCustomizationsDir local ipxedir="${webdirdest}service/ipxe" local f st=0 From 69f8cea7e56cd1371eda7ce3610b4cf0ba7fa46a Mon Sep 17 00:00:00 2001 From: JJ Fullmer Date: Fri, 7 Aug 2026 18:48:38 -0600 Subject: [PATCH 28/62] Mark bin/restorekernel.sh executable Committed as 100644 while every other script in bin/ is 100755, so on a real install it failed with "Permission denied" the first time it was invoked. core.filemode is false on the Windows checkout it was authored in, so the local chmod +x was never recorded and nothing locally could show the difference. Caught by running it on the dev server. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YE3jeTexYEcAxWFbZcZgpZ --- bin/restorekernel.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 bin/restorekernel.sh diff --git a/bin/restorekernel.sh b/bin/restorekernel.sh old mode 100644 new mode 100755 From 60f9edf6d25aacbc47b050eaa55e28098526ee9d Mon Sep 17 00:00:00 2001 From: JJ Fullmer Date: Fri, 7 Aug 2026 18:51:12 -0600 Subject: [PATCH 29/62] Fix: restore was reverting FOG's own files in service/ipxe on every update The rule "anything that is not one of the six kernel/init names is a custom file" is wrong, and a real install proves it. service/ipxe also contains boot.php, advanced.php, bgdark.png, the .unsigned/.old kernel siblings and more -- all shipped by FOG. Under that rule every update copied the PREVIOUS version's boot.php back over the newly installed one, silently reverting FOG's own code while reporting success. Absence is the honest test. Restore a snapshotted file only when the completed install did not write a file of that name: if FOG shipped it, the new copy wins; if nothing wrote it, the admin put it there and nothing else will put it back. That still covers the case this exists for -- a per-host kernel/init override -- without touching anything FOG owns. --restore-kernel-backup keeps forcing the six default names back, since a revert to an older commit does want its older kernels. restorekernel.sh --list applied the same wrong rule and labelled a dozen FOG files "(custom)"; it now lists only what is genuinely absent from the live tree, which is exactly the set the restore will put back. Found by running --list against real kernel backups on the dev server. No sandbox would have caught it: it needed a real service/ipxe with FOG's actual file set in it. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YE3jeTexYEcAxWFbZcZgpZ --- bin/restorekernel.sh | 16 +++++++++------- lib/common/functions.sh | 34 +++++++++++++++++++++------------- 2 files changed, 30 insertions(+), 20 deletions(-) diff --git a/bin/restorekernel.sh b/bin/restorekernel.sh index 6a0ce00f28..a571919d7f 100755 --- a/bin/restorekernel.sh +++ b/bin/restorekernel.sh @@ -137,15 +137,17 @@ listGenerations() { [[ -f $f ]] || continue echo " $(basename "$f") ($(tagof "$f"))" done - # Anything that is not one of the six names FOG re-downloads is a file - # the admin put there -- a per-host custom kernel/init. Worth showing, - # because it is the part no update would ever put back. + # A file counts as the admin's only if the live tree does NOT have one + # of that name -- service/ipxe is full of FOG's own boot.php, + # advanced.php, bgdark.png and kernel siblings, and calling those + # "custom" is both misleading here and, in restorePreservedCustomizations, + # was actively harmful. + local shown=0 for f in "$gendir"/*; do [[ -f $f ]] || continue - case " bzImage bzImage32 arm_Image init.xz init_32.xz arm_init.cpio.gz " in - *" $(basename "$f") "*) continue ;; - esac - echo " $(basename "$f") (custom)" + [[ -e "${ipxedir}/$(basename "$f")" ]] && continue + [[ $shown -eq 0 ]] && { echo " not present in the live tree (restored automatically):"; shown=1; } + echo " $(basename "$f")" done done [[ $found -eq 0 ]] && echo " (none yet)" diff --git a/lib/common/functions.sh b/lib/common/functions.sh index 36cc8037e7..6bac1435d4 100755 --- a/lib/common/functions.sh +++ b/lib/common/functions.sh @@ -210,20 +210,26 @@ restorePreservedCustomizations() { [[ -f "${customizationsDir}/ipxe-legacy/${f}" ]] && { cp -f "${customizationsDir}/ipxe-legacy/${f}" "${ipxedir}/${f}" >>$error_log 2>&1 || st=1; } done - # Anything in the newest generation that is NOT one of the six names - # downloadfiles() re-downloads is, by definition, a file FOG did not put - # there and will never put back -- a per-host custom kernel or init. Those - # are restored unconditionally. + # Restore a file from the snapshot ONLY if the fresh install did not put a + # file of that name back. # - # The six default names are deliberately NOT restored: the point of an - # update is to pick up the new kernel. bin/restorekernel.sh is the - # explicit, admin-invoked way back to an older one. + # The obvious rule -- "anything that is not one of the six kernel/init + # names is a custom file" -- is wrong, and a real install proves it: + # service/ipxe also holds boot.php, advanced.php, bgdark.png, the + # .unsigned/.old kernel siblings and more, all shipped by FOG. Under that + # rule every update copied the PREVIOUS version's boot.php back over the + # newly installed one, silently reverting FOG's own code on every run. # - # $restoreKernelBackup is the single exception, set only by - # --restore-kernel-backup, which revertUpdate() passes when it re-runs the - # installer against the previous commit. An older commit wants the older - # kernels too, and that is the behavior the retired _restorePreviousKernel() - # used to provide on that path. + # Absence is the honest test. If the just-completed install wrote a file of + # that name, it is FOG's and the new copy wins. If nothing wrote it, the + # admin put it there -- a per-host kernel/init override -- and nothing else + # will ever put it back. + # + # $restoreKernelBackup is the one exception: --restore-kernel-backup, which + # revertUpdate() passes when re-running the installer against the previous + # commit. An older commit wants its older kernels, so the six default names + # are forced back over the fresh ones -- the behavior the retired + # _restorePreviousKernel() used to provide on that path. local kbdir="${customizationsDir}/kernel-backups" local defaultnames=" bzImage bzImage32 arm_Image init.xz init_32.xz arm_init.cpio.gz " local bn @@ -231,7 +237,9 @@ restorePreservedCustomizations() { for f in "${kbdir}/gen-1"/*; do [[ -f $f ]] || continue bn=$(basename "$f") - if [[ $defaultnames != *" $bn "* || ${restoreKernelBackup:-0} -eq 1 ]]; then + if [[ ! -e "${ipxedir}/${bn}" ]]; then + cp -a "$f" "${ipxedir}/${bn}" >>$error_log 2>&1 || st=1 + elif [[ ${restoreKernelBackup:-0} -eq 1 && $defaultnames == *" $bn "* ]]; then cp -a "$f" "${ipxedir}/${bn}" >>$error_log 2>&1 || st=1 fi done From a799551d1a9b529675faaecd74ad345c5bc354ca Mon Sep 17 00:00:00 2001 From: JJ Fullmer Date: Fri, 7 Aug 2026 18:54:32 -0600 Subject: [PATCH 30/62] Scope kernel backups to kernel/init files, and fix restorekernel.sh's ipxedir Two problems, both surfaced by running against a real install. The generational backup copied the ENTIRE service/ipxe directory. That directory is a mixed bag -- FOG's own boot.php/advanced.php/index.php, bg images, grub.exe/memdisk/memtest.bin, the refind set, and the kernels -- so a "kernel backup" was 36 files mostly of PHP, and the restore then copied a previous release's boot.php back over the freshly installed one. Everything FOG ships there is already versioned in git; only the kernel/init material is worth keeping generations of. Now backs up the six default names plus whatever hostKernel/hostInit actually name in the database. Asking the database is what makes a per-host override discoverable without guessing from the directory, which is what led to sweeping in PHP in the first place. The restore rule collapses to "absent from the live tree -> put it back", which can now only ever match a custom kernel, since FOG re-downloads its own six every run. Separately, restorekernel.sh computed ipxedir as "${webdirdest}service/ipxe" after sourcing only .fogsettings -- which records docroot and webroot but NOT webdirdest, that being derived by config.sh. So webdirdest was empty, ipxedir was the relative string "service/ipxe", --list mislabelled every file, and --generation would have copied the restore into a stray directory under bin/ instead of the live tree. It now sources config.sh + doOSSpecificIncludes in the same order updatefog.sh does, and refuses to run if the directory still does not resolve. This is the bug commit ca02e0b9e fixed in setupacme.sh. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YE3jeTexYEcAxWFbZcZgpZ --- bin/restorekernel.sh | 17 ++++++++++++ lib/common/functions.sh | 60 ++++++++++++++++++++++++----------------- 2 files changed, 52 insertions(+), 25 deletions(-) diff --git a/bin/restorekernel.sh b/bin/restorekernel.sh index a571919d7f..ef780c9394 100755 --- a/bin/restorekernel.sh +++ b/bin/restorekernel.sh @@ -105,9 +105,26 @@ if [[ ! -r "$fogprogramdir/.fogsettings" ]]; then exit 1 fi . "$fogprogramdir/.fogsettings" +# .fogsettings records docroot and webroot but NOT webdirdest -- config.sh +# derives that ("${docroot}fog/"). Sourcing .fogsettings alone therefore left +# $webdirdest empty and $ipxedir as the relative string "service/ipxe", so +# --list mislabelled everything and --generation would have copied the restore +# into a stray directory under bin/ instead of the live tree. +# +# Same ordering as bin/updatefog.sh, and for the same reason: .fogsettings +# first so the recorded values win, then config.sh to derive what it does not +# record. This is the bug commit ca02e0b9e fixed in setupacme.sh. +linuxReleaseName_lower="${osname,,}" +. ../lib/common/config.sh +[[ -n $osid ]] && doOSSpecificIncludes >/dev/null kbdir="${fogprogramdir}/customizations/kernel-backups" ipxedir="${webdirdest}service/ipxe" +if [[ ! -d $ipxedir ]]; then + echo " * Could not locate the live iPXE directory (looked in '${ipxedir}')." + echo " * Check docroot/webroot in $fogprogramdir/.fogsettings." + exit 1 +fi if [[ ! -d $kbdir ]]; then echo " * No kernel backups yet at $kbdir." diff --git a/lib/common/functions.sh b/lib/common/functions.sh index 6bac1435d4..2b0327711e 100755 --- a/lib/common/functions.sh +++ b/lib/common/functions.sh @@ -163,16 +163,26 @@ backupPreservedCustomizations() { exit 1 fi - # Snapshot the whole directory into a rotated generation, rather than a - # fixed list of filenames. Two things fall out of that: a per-host custom - # kernel/init (bootmenu.class.php's Host->get('kernel')/get('init')) is - # captured without FOG ever having to learn its name, and a generation is - # a complete, coherent set rather than an assortment. + # Snapshot the KERNEL/INIT set into a rotated generation -- not the whole + # directory. # - # Bounded at $kernelBackupGenerations because this is otherwise unlimited - # growth on disk the admin provisioned for images, not for history. + # service/ipxe is a mixed bag: FOG's own boot.php/advanced.php/index.php, + # bg images, grub.exe/memdisk/memtest.bin, the refind set, AND the kernels. + # An earlier version copied all of it, which made a "kernel backup" full of + # PHP and led directly to restoring a previous release's boot.php over a + # freshly installed one. Everything in here that FOG ships is already + # versioned in git; only the kernel/init material is worth generations. + # + # Custom names come from the database rather than from guessing at the + # directory: hostKernel/hostInit are where a per-host override is actually + # recorded, so ask. [[ -z $kernelBackupGenerations || ! $kernelBackupGenerations =~ ^[0-9]+$ || $kernelBackupGenerations -lt 1 ]] && kernelBackupGenerations=3 - local kbdir="${customizationsDir}/kernel-backups" k + local kbdir="${customizationsDir}/kernel-backups" k kf + local kernelnames="bzImage bzImage32 arm_Image init.xz init_32.xz arm_init.cpio.gz" + local customkernels + customkernels=$(mysql $sqloptionsuser --password="${snmysqlpass}" -N -B \ + --execute="SELECT DISTINCT hostKernel FROM hosts WHERE hostKernel<>'' UNION SELECT DISTINCT hostInit FROM hosts WHERE hostInit<>''" \ + $mysqldbname 2>>$error_log) if [[ -d $ipxedir ]]; then mkdir -p "$kbdir" >>$error_log 2>&1 || warn=1 rm -rf "${kbdir}/gen-${kernelBackupGenerations}" >>$error_log 2>&1 @@ -183,7 +193,11 @@ backupPreservedCustomizations() { # cp -a preserves the version/tag_name xattrs downloadfiles() stamps on # each kernel, so every generation says which FOS release it came from # without a separate manifest to keep in sync. - cp -a "${ipxedir}/." "${kbdir}/gen-1/" >>$error_log 2>&1 || warn=1 + for kf in $kernelnames $customkernels; do + kf=$(basename "$kf") + [[ -f "${ipxedir}/${kf}" ]] || continue + cp -a "${ipxedir}/${kf}" "${kbdir}/gen-1/${kf}" >>$error_log 2>&1 || warn=1 + done fi [[ $warn -ne 0 ]] && echo -n "(some optional files could not be backed up) " @@ -210,26 +224,22 @@ restorePreservedCustomizations() { [[ -f "${customizationsDir}/ipxe-legacy/${f}" ]] && { cp -f "${customizationsDir}/ipxe-legacy/${f}" "${ipxedir}/${f}" >>$error_log 2>&1 || st=1; } done - # Restore a file from the snapshot ONLY if the fresh install did not put a - # file of that name back. - # - # The obvious rule -- "anything that is not one of the six kernel/init - # names is a custom file" -- is wrong, and a real install proves it: - # service/ipxe also holds boot.php, advanced.php, bgdark.png, the - # .unsigned/.old kernel siblings and more, all shipped by FOG. Under that - # rule every update copied the PREVIOUS version's boot.php back over the - # newly installed one, silently reverting FOG's own code on every run. + # The snapshot now holds only kernel/init material (see the backup side), + # so the restore rule is simple and safe: # - # Absence is the honest test. If the just-completed install wrote a file of - # that name, it is FOG's and the new copy wins. If nothing wrote it, the - # admin put it there -- a per-host kernel/init override -- and nothing else - # will ever put it back. + # absent from the live tree -> put it back. Only a per-host custom + # kernel/init reaches this: FOG re-downloads + # its own six every run, so they are never + # absent, and nothing else was captured. + # present -> leave the freshly installed file alone. + # Picking up the new kernel is the point of + # an update. # # $restoreKernelBackup is the one exception: --restore-kernel-backup, which # revertUpdate() passes when re-running the installer against the previous - # commit. An older commit wants its older kernels, so the six default names - # are forced back over the fresh ones -- the behavior the retired - # _restorePreviousKernel() used to provide on that path. + # commit. An older commit wants its older kernels, so the defaults are + # forced back over the fresh ones -- what the retired + # _restorePreviousKernel() used to do on that path. local kbdir="${customizationsDir}/kernel-backups" local defaultnames=" bzImage bzImage32 arm_Image init.xz init_32.xz arm_init.cpio.gz " local bn From fb28219fc9b672a750d47e88e7407e3de34d184e Mon Sep 17 00:00:00 2001 From: JJ Fullmer Date: Fri, 7 Aug 2026 19:02:58 -0600 Subject: [PATCH 31/62] Back up by subtracting what FOG ships, not by enumerating kernel names Enumerating custom kernel names cannot be made complete. A name can come from hostKernel/hostInit, from groupKernel/groupInit, from the FOG_TFTP_PXE_KERNEL/_32/_ARM settings, or from nothing FOG records at all -- an admin's own custom.ipxe can chain a kernel this server has never heard of. Any list of places to look is a list that will be short one place, and the previous commit's database query was already short three. What is knowable exactly is the set FOG ships: the 13 files in packages/web/service/ipxe (the PHP, the bg images, grub.exe, memdisk, memtest.bin, refind). Everything else in the live directory is either a kernel/init that downloadfiles() fetched or something the admin put there, and both are worth keeping. So the backup is now (live directory) minus (source tree), which needs no guessing and covers a fully custom name however it got there. If the source tree cannot be located the subtraction matches nothing and everything is kept -- the safe direction to fail in. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YE3jeTexYEcAxWFbZcZgpZ --- lib/common/functions.sh | 41 ++++++++++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/lib/common/functions.sh b/lib/common/functions.sh index 2b0327711e..e270cfa913 100755 --- a/lib/common/functions.sh +++ b/lib/common/functions.sh @@ -173,16 +173,27 @@ backupPreservedCustomizations() { # freshly installed one. Everything in here that FOG ships is already # versioned in git; only the kernel/init material is worth generations. # - # Custom names come from the database rather than from guessing at the - # directory: hostKernel/hostInit are where a per-host override is actually - # recorded, so ask. + # Do not try to ENUMERATE custom kernel names -- subtract what FOG ships + # instead. + # + # Enumerating cannot be made complete. A custom kernel name can come from + # hostKernel/hostInit, from groupKernel/groupInit, from the + # FOG_TFTP_PXE_KERNEL/_32/_ARM settings, or from nothing FOG records at all + # -- an admin's own custom.ipxe can chain a kernel this server has never + # heard of. Any list of places to look is a list that will be short one + # place. + # + # What IS knowable exactly is the set FOG ships: the contents of + # packages/web/service/ipxe in the source tree (13 files -- the PHP, the + # bg images, grub.exe/memdisk/memtest.bin, refind). Everything else living + # in the live directory is either a kernel/init downloadfiles() fetched or + # something the admin put there, and both are worth keeping. + # + # So: back up (live directory) minus (what the source tree ships). No + # guessing, and a fully custom name is covered however it got there. [[ -z $kernelBackupGenerations || ! $kernelBackupGenerations =~ ^[0-9]+$ || $kernelBackupGenerations -lt 1 ]] && kernelBackupGenerations=3 - local kbdir="${customizationsDir}/kernel-backups" k kf - local kernelnames="bzImage bzImage32 arm_Image init.xz init_32.xz arm_init.cpio.gz" - local customkernels - customkernels=$(mysql $sqloptionsuser --password="${snmysqlpass}" -N -B \ - --execute="SELECT DISTINCT hostKernel FROM hosts WHERE hostKernel<>'' UNION SELECT DISTINCT hostInit FROM hosts WHERE hostInit<>''" \ - $mysqldbname 2>>$error_log) + local kbdir="${customizationsDir}/kernel-backups" k kf bn + local shippeddir="${webdirsrc%/}/service/ipxe" if [[ -d $ipxedir ]]; then mkdir -p "$kbdir" >>$error_log 2>&1 || warn=1 rm -rf "${kbdir}/gen-${kernelBackupGenerations}" >>$error_log 2>&1 @@ -193,10 +204,14 @@ backupPreservedCustomizations() { # cp -a preserves the version/tag_name xattrs downloadfiles() stamps on # each kernel, so every generation says which FOS release it came from # without a separate manifest to keep in sync. - for kf in $kernelnames $customkernels; do - kf=$(basename "$kf") - [[ -f "${ipxedir}/${kf}" ]] || continue - cp -a "${ipxedir}/${kf}" "${kbdir}/gen-1/${kf}" >>$error_log 2>&1 || warn=1 + for kf in "${ipxedir}"/*; do + [[ -f $kf ]] || continue + bn=$(basename "$kf") + # Shipped by FOG -> already versioned in git, skip. If the source + # tree cannot be found, $shippeddir does not exist, every file + # fails this test and everything is kept -- the safe direction. + [[ -e "${shippeddir}/${bn}" ]] && continue + cp -a "$kf" "${kbdir}/gen-1/${bn}" >>$error_log 2>&1 || warn=1 done fi From f3e55fbc945f13e11086fe38db352767866e0aab Mon Sep 17 00:00:00 2001 From: JJ Fullmer Date: Fri, 7 Aug 2026 19:05:24 -0600 Subject: [PATCH 32/62] Detect and report a custom kernel installed under one of FOG's own names Overwriting bzImage in place with a hand-built kernel is common, and it is the one case none of the other rules can resolve. It gets backed up like any other non-shipped file, but downloadfiles() re-downloads FOG's kernel over it and the restore deliberately leaves freshly-installed default names alone -- so it was silently replaced on every update. Neither silent outcome is acceptable. Keeping the custom kernel means never receiving a kernel update again; replacing it means losing it. The failure in both is the same: the admin does not find out. So the installer now detects the case and says so, leaving the choice where it belongs. The signal already exists: downloadfiles() stamps version/tag_name xattrs on everything it fetches, so a default-named kernel WITHOUT them was placed by hand. No hashing and no reference copy needed -- verified on a real server, where FOG's bzImage carries tag_name "20260806-111046" and hand-placed files carry nothing. An admin who copied with `cp -a` could drag a stale tag along and defeat it; plain cp/scp/mv, the normal way, does not. The file is never at risk either way -- it is in gen-1 before anything is replaced -- so the message names it and gives the exact restorekernel.sh commands to put it back. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YE3jeTexYEcAxWFbZcZgpZ --- lib/common/functions.sh | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/lib/common/functions.sh b/lib/common/functions.sh index e270cfa913..c17e77fad8 100755 --- a/lib/common/functions.sh +++ b/lib/common/functions.sh @@ -213,6 +213,30 @@ backupPreservedCustomizations() { [[ -e "${shippeddir}/${bn}" ]] && continue cp -a "$kf" "${kbdir}/gen-1/${bn}" >>$error_log 2>&1 || warn=1 done + # A custom kernel installed under a DEFAULT name is the case none of + # the rules above can catch on their own: it is backed up like any + # other non-shipped file, but downloadfiles() will re-download FOG's + # own kernel over it, and the restore deliberately leaves a + # freshly-installed default name alone. + # + # downloadfiles() stamps version/tag_name xattrs on everything it + # fetches. A default-named kernel WITHOUT them was put there by hand, + # which is the whole signal needed -- no reference copy, no hashing. + # (An admin who copied with `cp -a` could carry a stale tag across and + # defeat this; plain cp/scp/mv, the normal way, does not.) + # + # Detected here, reported after the restore. Silently keeping the + # custom kernel means never getting kernel updates again; silently + # replacing it means losing it. Both are bad in the same way -- the + # admin does not find out. So do neither, and say so. + customDefaultKernels="" + if command -v attr >/dev/null 2>&1; then + for bn in bzImage bzImage32 arm_Image init.xz init_32.xz arm_init.cpio.gz; do + [[ -f "${ipxedir}/${bn}" ]] || continue + attr -q -g tag_name "${ipxedir}/${bn}" >/dev/null 2>&1 && continue + customDefaultKernels="${customDefaultKernels}${bn} " + done + fi fi [[ $warn -ne 0 ]] && echo -n "(some optional files could not be backed up) " @@ -283,6 +307,21 @@ restorePreservedCustomizations() { return 0 fi errorStat 0 + # Say it plainly rather than picking for them -- see the detection comment + # in backupPreservedCustomizations. + if [[ -n $customDefaultKernels ]]; then + echo + echo " * NOTE: these looked like hand-installed kernels under FOG's own" + echo " names, and this update has replaced them with the versions it" + echo " downloaded:" + for f in $customDefaultKernels; do + echo " ${f}" + done + echo " Your copies were saved first and are still available:" + echo " ${bindirsrc:-.}/restorekernel.sh --list" + echo " ${bindirsrc:-.}/restorekernel.sh --generation 1" + echo " Restoring puts them back over the downloaded ones." + fi } # GH-685: the MariaDB client library turns TLS on by default from 10.10.1 # onward and then refuses to connect at all when the server offers none -- From 69ecdb765a323dabca92795bdb27b70a66f87ac7 Mon Sep 17 00:00:00 2001 From: JJ Fullmer Date: Fri, 7 Aug 2026 19:07:47 -0600 Subject: [PATCH 33/62] Detect a modified kernel by checksum, since xattrs survive in-place overwrite The previous commit's detector does not work, and testing on a real server showed exactly why: writing over bzImage in place -- `> bzImage`, dd, cp onto the existing path, which is how a custom kernel actually gets installed -- leaves the file's existing xattrs untouched. The hand-written kernel kept FOG's old version/tag_name and still reported 2 xattrs, so an absence-of-xattrs test saw nothing to report. The very case it was written for was the case it missed. downloadfiles() now also stamps a sha256 (_stampFogSum) on each kernel/init it fetches, and the check recomputes and compares it. Content is what changed, so comparing content works however the write was performed. Three-state on purpose: 0 matches, 1 modified, 2 nothing to compare against. An install whose kernels predate the stamp returns 2 and stays silent -- reporting a custom kernel on every existing server at its first upgrade would be noise, and the file is safely in gen-1 either way. Servers become detectable after one update has stamped them. Cannot be tested from the Windows dev box: attr is a Linux tool, so every local probe returns 2. Verified on the dev server instead. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YE3jeTexYEcAxWFbZcZgpZ --- lib/common/functions.sh | 67 ++++++++++++++++++++++++++++++++--------- 1 file changed, 53 insertions(+), 14 deletions(-) diff --git a/lib/common/functions.sh b/lib/common/functions.sh index c17e77fad8..f8e3b7c20f 100755 --- a/lib/common/functions.sh +++ b/lib/common/functions.sh @@ -88,6 +88,41 @@ backupReports() { # "/customizations" and wrote the backups to the filesystem root. That is what # the first real-server run actually did -- the sandbox never caught it because # it always set $fogprogramdir before sourcing. +# Record the checksum of a file FOG just downloaded, so a later run can tell +# whether it is still the file FOG put there. +# +# The version/tag_name xattrs alone cannot answer that. Overwriting a file IN +# PLACE -- `> bzImage`, dd, cp onto an existing path, which is exactly how a +# custom kernel gets installed -- leaves the existing xattrs untouched, so the +# admin's kernel keeps FOG's old tag and looks original. Confirmed on a real +# server: a hand-written bzImage still reported 2 xattrs. +# +# A checksum recorded at download time is not defeated by that: the content +# changed, so the comparison fails, however the write was done. +_stampFogSum() { + local f="$1" sum + [[ -f $f ]] || return 0 + command -v sha256sum >/dev/null 2>&1 || return 0 + command -v attr >/dev/null 2>&1 || return 0 + sum=$(sha256sum "$f" 2>/dev/null | cut -d' ' -f1) + [[ -n $sum ]] && attr -s fogsum -V "$sum" "$f" >>$error_log 2>&1 + return 0 +} +# Echoes 0 when $1 still matches the checksum FOG stamped, 1 when it differs +# (admin-modified), 2 when there is nothing to compare against -- an older +# install whose kernels predate the stamp. 2 is NOT "modified": reporting a +# custom kernel on every existing server at first upgrade would be noise, and +# the file is safely backed up regardless. +_fogSumStatus() { + local f="$1" want have + [[ -f $f ]] || { echo 2; return; } + command -v sha256sum >/dev/null 2>&1 || { echo 2; return; } + command -v attr >/dev/null 2>&1 || { echo 2; return; } + want=$(attr -q -g fogsum "$f" 2>/dev/null) + [[ -z $want ]] && { echo 2; return; } + have=$(sha256sum "$f" 2>/dev/null | cut -d' ' -f1) + [[ $want == "$have" ]] && echo 0 || echo 1 +} _resolveCustomizationsDir() { [[ -n $customizationsDir ]] && return 0 local base="${fogprogramdir:-/opt/fog}" @@ -219,24 +254,22 @@ backupPreservedCustomizations() { # own kernel over it, and the restore deliberately leaves a # freshly-installed default name alone. # - # downloadfiles() stamps version/tag_name xattrs on everything it - # fetches. A default-named kernel WITHOUT them was put there by hand, - # which is the whole signal needed -- no reference copy, no hashing. - # (An admin who copied with `cp -a` could carry a stale tag across and - # defeat this; plain cp/scp/mv, the normal way, does not.) + # Compared by CHECKSUM, not by whether the version xattrs are present. + # Overwriting in place -- `> bzImage`, dd, cp onto the existing path, + # which is how a custom kernel actually gets installed -- preserves the + # existing xattrs, so FOG's old tag survives on the admin's file and an + # absence test sees nothing. _stampFogSum records the checksum at + # download time precisely so the content can be compared instead. # # Detected here, reported after the restore. Silently keeping the # custom kernel means never getting kernel updates again; silently - # replacing it means losing it. Both are bad in the same way -- the - # admin does not find out. So do neither, and say so. + # replacing it means losing it. Both fail the same way -- the admin + # does not find out. So do neither, and say so. customDefaultKernels="" - if command -v attr >/dev/null 2>&1; then - for bn in bzImage bzImage32 arm_Image init.xz init_32.xz arm_init.cpio.gz; do - [[ -f "${ipxedir}/${bn}" ]] || continue - attr -q -g tag_name "${ipxedir}/${bn}" >/dev/null 2>&1 && continue - customDefaultKernels="${customDefaultKernels}${bn} " - done - fi + for bn in bzImage bzImage32 arm_Image init.xz init_32.xz arm_init.cpio.gz; do + [[ -f "${ipxedir}/${bn}" ]] || continue + [[ $(_fogSumStatus "${ipxedir}/${bn}") -eq 1 ]] && customDefaultKernels="${customDefaultKernels}${bn} " + done fi [[ $warn -ne 0 ]] && echo -n "(some optional files could not be backed up) " @@ -4939,21 +4972,27 @@ downloadfiles() { cp -vf ${copypath}bzImage ${webdirdest}/service/ipxe/ >>$error_log 2>&1 || errorStat $? attr -s version -V $kern_version ${webdirdest}/service/ipxe/bzImage >>$error_log 2>&1 || errorStat $? attr -s tag_name -V $tag_name ${webdirdest}/service/ipxe/bzImage >>$error_log 2>&1 || errorStat $? + _stampFogSum ${webdirdest}/service/ipxe/bzImage cp -vf ${copypath}bzImage32 ${webdirdest}/service/ipxe/ >>$error_log 2>&1 || errorStat $? attr -s version -V $kern_version ${webdirdest}/service/ipxe/bzImage32 >>$error_log 2>&1 || errorStat $? attr -s tag_name -V $tag_name ${webdirdest}/service/ipxe/bzImage32 >>$error_log 2>&1 || errorStat $? + _stampFogSum ${webdirdest}/service/ipxe/bzImage32 cp -vf ${copypath}arm_Image ${webdirdest}/service/ipxe/ >>$error_log 2>&1 || errorStat $? attr -s version -V $kern_version ${webdirdest}/service/ipxe/arm_Image >>$error_log 2>&1 || errorStat $? attr -s tag_name -V $tag_name ${webdirdest}/service/ipxe/arm_Image >>$error_log 2>&1 || errorStat $? + _stampFogSum ${webdirdest}/service/ipxe/arm_Image cp -vf ${copypath}init.xz ${webdirdest}/service/ipxe/ >>$error_log 2>&1 || errorStat $? attr -s version -V $build_version ${webdirdest}/service/ipxe/init.xz >>$error_log 2>&1 || errorStat $? attr -s tag_name -V $tag_name ${webdirdest}/service/ipxe/init.xz >>$error_log 2>&1 || errorStat $? + _stampFogSum ${webdirdest}/service/ipxe/init.xz cp -vf ${copypath}init_32.xz ${webdirdest}/service/ipxe/ >>$error_log 2>&1 || errorStat $? attr -s version -V $build_version ${webdirdest}/service/ipxe/init_32.xz >>$error_log 2>&1 || errorStat $? attr -s tag_name -V $tag_name ${webdirdest}/service/ipxe/init_32.xz >>$error_log 2>&1 || errorStat $? + _stampFogSum ${webdirdest}/service/ipxe/init_32.xz cp -vf ${copypath}arm_init.cpio.gz ${webdirdest}/service/ipxe/ >>$error_log 2>&1 || errorStat $? attr -s version -V $build_version ${webdirdest}/service/ipxe/arm_init.cpio.gz >>$error_log 2>&1 || errorStat $? attr -s tag_name -V $tag_name ${webdirdest}/service/ipxe/arm_init.cpio.gz >>$error_log 2>&1 || errorStat $? + _stampFogSum ${webdirdest}/service/ipxe/arm_init.cpio.gz cp -vf ${copypath}FOGService.msi ${copypath}SmartInstaller.exe ${webdirdest}/client/ >>$error_log 2>&1 errorStat $? cd $cwd From d0fb7b4654b575354de99338d78f6cd11f95c9c8 Mon Sep 17 00:00:00 2001 From: JJ Fullmer Date: Fri, 7 Aug 2026 19:10:23 -0600 Subject: [PATCH 34/62] Keep the outgoing kernel in place as bzImage., and fix a false positive Two fixes. The checksum detector reported bzImage32 and arm_Image as hand-installed on a server where nobody had touched them. _stampFogSum ran in downloadfiles' copy block, but _resignKernels rewrites each kernel in place afterwards, so the recorded checksum described a file that no longer existed. Stamping now happens after signing -- after everything that modifies the file has run. Second, the per-version sibling from the original brief, which the generation directories did not actually deliver. When a kernel is about to be replaced its current copy is kept alongside, named for the release it came from: bzImage.20260806-111046 next to bzImage. The generation directories remain the complete rotated history; this is the copy visible while looking at the boot directory, and the one a single host can be pointed at by name without restoring anything. Named per version rather than a single .prev so several updates accumulate, which is cheap next to the images this server already holds. Those siblings are excluded from the generational sweep -- they are already copies of a kernel, and snapshotting them into every generation would multiply the same bytes by the generation count for no added recoverability. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YE3jeTexYEcAxWFbZcZgpZ --- lib/common/functions.sh | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/lib/common/functions.sh b/lib/common/functions.sh index f8e3b7c20f..30a970a92c 100755 --- a/lib/common/functions.sh +++ b/lib/common/functions.sh @@ -246,6 +246,13 @@ backupPreservedCustomizations() { # tree cannot be found, $shippeddir does not exist, every file # fails this test and everything is kept -- the safe direction. [[ -e "${shippeddir}/${bn}" ]] && continue + # Skip the per-version siblings this function itself leaves behind + # (bzImage.20260806-111046). They are already a copy of a kernel; + # snapshotting them into every generation would multiply the same + # bytes by the generation count for no added recoverability. + case $bn in + bzImage.*|bzImage32.*|arm_Image.*|init.xz.*|init_32.xz.*|arm_init.cpio.gz.*) continue ;; + esac cp -a "$kf" "${kbdir}/gen-1/${bn}" >>$error_log 2>&1 || warn=1 done # A custom kernel installed under a DEFAULT name is the case none of @@ -269,6 +276,19 @@ backupPreservedCustomizations() { for bn in bzImage bzImage32 arm_Image init.xz init_32.xz arm_init.cpio.gz; do [[ -f "${ipxedir}/${bn}" ]] || continue [[ $(_fogSumStatus "${ipxedir}/${bn}") -eq 1 ]] && customDefaultKernels="${customDefaultKernels}${bn} " + # Keep the outgoing kernel in place, named for the release it came + # from: bzImage.20260806-111046 sits right next to bzImage. + # + # The generation directories are the complete, rotated history; this + # is the version you can actually SEE while looking at the boot + # directory, and point a single host at by name without restoring + # anything. Named per version rather than a single .prev so several + # updates' worth accumulate, which is cheap next to the images this + # server already stores. + local tag + tag=$(attr -q -g tag_name "${ipxedir}/${bn}" 2>/dev/null | tr -d "\"" | tr -c "A-Za-z0-9.-" "_") + [[ -z $tag ]] && tag="prev" + [[ -e "${ipxedir}/${bn}.${tag}" ]] || cp -a "${ipxedir}/${bn}" "${ipxedir}/${bn}.${tag}" >>$error_log 2>&1 done fi @@ -4999,6 +5019,15 @@ downloadfiles() { _ensureSecureBootKeys _ensureSecureBootPlatformKeys _resignKernels + # Re-stamp AFTER signing. _resignKernels rewrites each kernel in place, so + # a checksum taken at download time no longer matches the file on disk -- + # which made the next run report bzImage32/arm_Image as hand-installed on a + # server where nobody had touched them. Stamp what is actually there once + # everything that modifies it has run. + local _k + for _k in bzImage bzImage32 arm_Image init.xz init_32.xz arm_init.cpio.gz; do + _stampFogSum "${webdirdest}/service/ipxe/${_k}" + done _installSecureBootSigner _publishSecureBootKit _publishSecureBootAuthVars From ef3a1a36c5d96cb946b5614a076a99efcb9e2d6c Mon Sep 17 00:00:00 2001 From: JJ Fullmer Date: Fri, 7 Aug 2026 19:13:07 -0600 Subject: [PATCH 35/62] Create the . kernel sibling at restore time, not backup time The sibling was written into service/ipxe during the backup phase, which runs before configureHttpd() -- and configureHttpd() rm -rf's the entire web tree. Every sibling it created was deleted minutes later in the same run, which is why none ever appeared on disk. Built from the generation snapshot during the restore phase instead, after the tree has been rebuilt. The snapshot is the copy that survives the wipe, so it is the only thing that can source the sibling. Also skips writing one when the update did not actually change that kernel -- cmp against the freshly installed file first, since an identical sibling is pure duplication. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YE3jeTexYEcAxWFbZcZgpZ --- lib/common/functions.sh | 38 +++++++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/lib/common/functions.sh b/lib/common/functions.sh index 30a970a92c..e1fb2c8645 100755 --- a/lib/common/functions.sh +++ b/lib/common/functions.sh @@ -276,19 +276,6 @@ backupPreservedCustomizations() { for bn in bzImage bzImage32 arm_Image init.xz init_32.xz arm_init.cpio.gz; do [[ -f "${ipxedir}/${bn}" ]] || continue [[ $(_fogSumStatus "${ipxedir}/${bn}") -eq 1 ]] && customDefaultKernels="${customDefaultKernels}${bn} " - # Keep the outgoing kernel in place, named for the release it came - # from: bzImage.20260806-111046 sits right next to bzImage. - # - # The generation directories are the complete, rotated history; this - # is the version you can actually SEE while looking at the boot - # directory, and point a single host at by name without restoring - # anything. Named per version rather than a single .prev so several - # updates' worth accumulate, which is cheap next to the images this - # server already stores. - local tag - tag=$(attr -q -g tag_name "${ipxedir}/${bn}" 2>/dev/null | tr -d "\"" | tr -c "A-Za-z0-9.-" "_") - [[ -z $tag ]] && tag="prev" - [[ -e "${ipxedir}/${bn}.${tag}" ]] || cp -a "${ipxedir}/${bn}" "${ipxedir}/${bn}.${tag}" >>$error_log 2>&1 done fi @@ -346,6 +333,31 @@ restorePreservedCustomizations() { fi done fi + # Leave the OUTGOING kernel next to the new one, named for the release it + # came from: bzImage.20260806-111046 beside bzImage. + # + # Done here, not at backup time, because configureHttpd() rm -rf's the whole + # web tree between the two -- a sibling written before that is deleted + # minutes later, which is exactly what the first attempt did. The generation + # snapshot is the surviving copy, so build the sibling from it. + # + # The generation directories remain the complete rotated history; this is + # the copy visible while looking at the boot directory, and the one a single + # host can be pointed at by name without restoring anything. Per version + # rather than a single .prev so several updates accumulate -- cheap next to + # the images this server already holds. + if [[ -d "${kbdir}/gen-1" ]]; then + local tag + for bn in bzImage bzImage32 arm_Image init.xz init_32.xz arm_init.cpio.gz; do + [[ -f "${kbdir}/gen-1/${bn}" ]] || continue + tag=$(attr -q -g tag_name "${kbdir}/gen-1/${bn}" 2>/dev/null | tr -d '"' | tr -c 'A-Za-z0-9.-' '_') + [[ -z $tag ]] && tag="prev" + # Same content under the same name means the update did not change + # this kernel; a sibling would just be a duplicate. + cmp -s "${kbdir}/gen-1/${bn}" "${ipxedir}/${bn}" && continue + [[ -e "${ipxedir}/${bn}.${tag}" ]] || cp -a "${kbdir}/gen-1/${bn}" "${ipxedir}/${bn}.${tag}" >>$error_log 2>&1 + done + fi [[ -d $ipxedir ]] && chown -R ${username}:${apacheuser} "$ipxedir" >>$error_log 2>&1 # Never fatal, unlike the backup side. By this point configureHttpd() has # already rebuilt the web tree, so aborting would strand a nearly-complete From fabf07dbf669dda6fa0f5a02903af948de1b713a Mon Sep 17 00:00:00 2001 From: JJ Fullmer Date: Fri, 7 Aug 2026 19:21:22 -0600 Subject: [PATCH 36/62] Make kernel and init selections dropdowns of what is actually on disk Kernel and init were free-text fields, so choosing one meant knowing the exact filename and typing it correctly, with no indication of what was available. The installer now leaves the outgoing kernel behind as bzImage. on every update, which only helps if selecting one is easy -- otherwise the versions accumulate somewhere nobody looks. FOGPage::kernelFileList() reads FOG_TFTP_PXE_KERNEL_DIR and splits by shape rather than a fixed name list, so custom kernels and the per-release siblings both appear. .unsigned copies are excluded -- they are _resignKernels() working files, not something to boot. Plain names sort above their versioned siblings. Applied to Host Kernel/Init, Group Kernel/Init, and the FOG_TFTP_PXE_KERNEL/ _32/_ARM/FOG_MEMTEST_KERNEL defaults, so rolling the default back to a previous release is now a selection instead of a typed guess. Two things it deliberately does not do. A stored value naming a file no longer on disk is kept in the list, selected, and marked "not found on disk" -- dropping it would silently rewrite a host's kernel to the default the moment anyone opened the form. And when the directory cannot be read it falls back to the original text input, so a server whose kernel directory has moved stays editable rather than showing an empty, unusable dropdown. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YE3jeTexYEcAxWFbZcZgpZ --- packages/web/lib/fog/fogpage.class.php | 139 ++++++++++++++++++ .../lib/pages/fogconfigurationpage.page.php | 20 +++ .../web/lib/pages/groupmanagement.page.php | 16 +- .../web/lib/pages/hostmanagement.page.php | 48 +++--- 4 files changed, 183 insertions(+), 40 deletions(-) diff --git a/packages/web/lib/fog/fogpage.class.php b/packages/web/lib/fog/fogpage.class.php index 92d8fa7a44..f30b863b60 100644 --- a/packages/web/lib/fog/fogpage.class.php +++ b/packages/web/lib/fog/fogpage.class.php @@ -5041,6 +5041,145 @@ public static function makeLabel( . $str . ''; } + /** + * Lists the kernel or init files actually present in the FOS boot + * directory, newest first. + * + * Kernels and inits are files on disk, not database records, so there is + * nothing for buildSelectBox() to enumerate. Reading the directory is the + * only way to know what an admin can legitimately choose -- and since the + * installer now leaves the outgoing kernel behind as bzImage. + * on every update, that directory is exactly the list of "current, or any + * version still on this server, or anything I put here myself". + * + * @param string $type 'kernel' or 'init' + * + * @return array filenames + */ + public static function kernelFileList($type = 'kernel') + { + $dir = trim((string)self::getSetting('FOG_TFTP_PXE_KERNEL_DIR')); + if (empty($dir) || !is_dir($dir) || !is_readable($dir)) { + return []; + } + $files = @scandir($dir); + if ($files === false) { + return []; + } + $out = []; + foreach ($files as $file) { + if ($file === '.' || $file === '..') { + continue; + } + if (!is_file($dir . DIRECTORY_SEPARATOR . $file)) { + continue; + } + /** + * Split by shape rather than by a fixed list of names, so a + * custom kernel and the per-release siblings both appear. + * .unsigned copies are deliberately excluded -- they are + * _resignKernels() working files, not something to boot. + */ + if (preg_match('/\.unsigned$/', $file)) { + continue; + } + $isInit = (bool)preg_match('/(^|\/)(init|arm_init)|\.(xz|cpio\.gz)/i', $file); + if ($type === 'init' ? $isInit : !$isInit) { + $out[] = $file; + } + } + /** + * Plain names first, then their versioned siblings, so bzImage sits + * above bzImage.20260806-111046 instead of being sorted into the + * middle of them. + */ + usort( + $out, + function ($a, $b) { + $adot = substr_count($a, '.'); + $bdot = substr_count($b, '.'); + if ($adot !== $bdot) { + return $adot - $bdot; + } + return strnatcasecmp($b, $a); + } + ); + + return $out; + } + /** + * Builds a select of the kernel/init files present on disk. + * + * Falls back to a plain text input when the directory cannot be read, so + * a server whose kernel directory has moved is still editable rather than + * presenting an empty, unusable dropdown. + * + * @param string $name field name/id + * @param string $current the currently stored value + * @param string $type 'kernel' or 'init' + * @param string $class css classes for the element + * + * @return string + */ + public static function kernelFileSelect( + $name, + $current = '', + $type = 'kernel', + $class = 'form-control', + $id = '' + ) { + $current = trim((string)$current); + if ($id === '') { + $id = $name; + } + $files = self::kernelFileList($type); + if (count($files) < 1) { + return self::makeInput( + $class, + $name, + $type === 'init' ? 'customInit.xz' : 'bzImage_Custom', + 'text', + $id, + $current + ); + } + /** + * A stored value naming a file that is no longer on disk must still + * appear, and still be selected. Dropping it would silently rewrite + * the host's kernel to the default the moment anyone opened the form. + */ + $missing = ($current !== '' && !in_array($current, $files, true)); + if ($missing) { + array_unshift($files, $current); + } + $opts = ''; + foreach ($files as $file) { + $opts .= ''; + } + + return ''; + } /** * Makes an input element. * diff --git a/packages/web/lib/pages/fogconfigurationpage.page.php b/packages/web/lib/pages/fogconfigurationpage.page.php index ab8d5be6f9..f9532c1783 100644 --- a/packages/web/lib/pages/fogconfigurationpage.page.php +++ b/packages/web/lib/pages/fogconfigurationpage.page.php @@ -864,6 +864,26 @@ public function getIpxeList() $row['settingKey'] ); break; + /** + * The default kernels are filenames in the FOS boot + * directory, so offer what is actually there -- + * including the per-release siblings the installer + * leaves behind on every update, which is what makes + * "put the default back on the previous kernel" a + * selection rather than a typed guess. + */ + case 'FOG_TFTP_PXE_KERNEL': + case 'FOG_TFTP_PXE_KERNEL_32': + case 'FOG_TFTP_PXE_KERNEL_ARM': + case 'FOG_MEMTEST_KERNEL': + $input = self::kernelFileSelect( + $row['settingID'], + $row['settingValue'], + 'kernel', + 'form-control', + $row['settingKey'] + ); + break; case (isset($needstobecheckbox[$row['settingKey']])): $input = self::makeInput( '', diff --git a/packages/web/lib/pages/groupmanagement.page.php b/packages/web/lib/pages/groupmanagement.page.php index b411b71b45..7b6f5993b2 100644 --- a/packages/web/lib/pages/groupmanagement.page.php +++ b/packages/web/lib/pages/groupmanagement.page.php @@ -109,13 +109,11 @@ protected function _addFields() $labelClass, 'kernel', _('Group Kernel') - ) => self::makeInput( - 'form-control groupkernel-input', + ) => self::kernelFileSelect( 'kernel', - 'customBzimage', - 'text', + $kernel, 'kernel', - $kernel + 'form-control groupkernel-input' ), self::makeLabel( $labelClass, @@ -133,13 +131,11 @@ protected function _addFields() $labelClass, 'init', _('Group Init') - ) => self::makeInput( - 'form-control groupinit-input', + ) => self::kernelFileSelect( 'init', - 'customInit.xz', - 'text', + $init, 'init', - $init + 'form-control groupinit-input' ), self::makeLabel( $labelClass, diff --git a/packages/web/lib/pages/hostmanagement.page.php b/packages/web/lib/pages/hostmanagement.page.php index 8e991969ba..347a1390c3 100644 --- a/packages/web/lib/pages/hostmanagement.page.php +++ b/packages/web/lib/pages/hostmanagement.page.php @@ -615,13 +615,11 @@ public function add() $labelClass, 'kernel', _('Host Kernel') - ) => self::makeInput( - 'form-control hostkernel-input', + ) => self::kernelFileSelect( 'kernel', - 'bzImage_Custom', - 'text', + $kernel, 'kernel', - $kernel + 'form-control hostkernel-input' ), self::makeLabel( $labelClass, @@ -639,13 +637,11 @@ public function add() $labelClass, 'init', _('Host Init') - ) => self::makeInput( - 'form-control hostinit-input', + ) => self::kernelFileSelect( 'init', - 'customInit.xz', - 'text', + $init, 'init', - $init + 'form-control hostinit-input' ), self::makeLabel( $labelClass, @@ -865,13 +861,11 @@ protected function _addFields() $labelClass, 'kernel', _('Host Kernel') - ) => self::makeInput( - 'form-control hostkernel-input', + ) => self::kernelFileSelect( 'kernel', - 'bzImage_Custom', - 'text', + $kernel, 'kernel', - $kernel + 'form-control hostkernel-input' ), self::makeLabel( $labelClass, @@ -889,13 +883,11 @@ protected function _addFields() $labelClass, 'init', _('Host Init') - ) => self::makeInput( - 'form-control hostinit-input', + ) => self::kernelFileSelect( 'init', - 'customInit.xz', - 'text', + $init, 'init', - $init + 'form-control hostinit-input' ), self::makeLabel( $labelClass, @@ -1158,13 +1150,11 @@ public function hostGeneral() $labelClass, 'kernel', _('Host Kernel') - ) => self::makeInput( - 'form-control hostkernel-input', + ) => self::kernelFileSelect( 'kernel', - 'bzImage_Custom', - 'text', + $kernel, 'kernel', - $kernel + 'form-control hostkernel-input' ), self::makeLabel( $labelClass, @@ -1182,13 +1172,11 @@ public function hostGeneral() $labelClass, 'init', _('Host Init') - ) => self::makeInput( - 'form-control hostinit-input', + ) => self::kernelFileSelect( 'init', - 'customInit.xz', - 'text', + $init, 'init', - $init + 'form-control hostinit-input' ), self::makeLabel( $labelClass, From cee7f1b6eb4f2365a8e0d4b3ef901677c6028627 Mon Sep 17 00:00:00 2001 From: JJ Fullmer Date: Fri, 7 Aug 2026 19:23:01 -0600 Subject: [PATCH 37/62] Keep web assets out of the kernel dropdown "Anything that is not an init is a kernel" swept boot.php, advanced.php, index.php, the bg images and refind.conf into the kernel list -- 22 entries, most of which cannot be booted, which is not a menu anybody wants to pick a kernel out of. Excludes .php, image extensions, .conf and .efi alongside the existing .unsigned exclusion. What remains is the bootable set: the kernels, their per-release siblings, custom kernels, and memdisk/memtest.bin/grub.exe, which FOG_MEMTEST_KERNEL legitimately points at. Caught by rendering the helper against a real service/ipxe rather than reasoning about what lives there. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YE3jeTexYEcAxWFbZcZgpZ --- packages/web/lib/fog/fogpage.class.php | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/packages/web/lib/fog/fogpage.class.php b/packages/web/lib/fog/fogpage.class.php index f30b863b60..79cb3f03b1 100644 --- a/packages/web/lib/fog/fogpage.class.php +++ b/packages/web/lib/fog/fogpage.class.php @@ -5077,10 +5077,17 @@ public static function kernelFileList($type = 'kernel') /** * Split by shape rather than by a fixed list of names, so a * custom kernel and the per-release siblings both appear. - * .unsigned copies are deliberately excluded -- they are - * _resignKernels() working files, not something to boot. + * + * .unsigned copies are excluded -- they are _resignKernels() + * working files, not something to boot. + * + * The web assets and config that share this directory + * (boot.php/advanced.php/index.php, the bg images, refind.conf) + * are excluded too. "Anything that is not an init" swept all of + * them into the kernel list, which is not a menu anybody wants to + * pick a kernel out of. */ - if (preg_match('/\.unsigned$/', $file)) { + if (preg_match('/\.(unsigned|php|png|jpe?g|gif|svg|conf|efi)$/i', $file)) { continue; } $isInit = (bool)preg_match('/(^|\/)(init|arm_init)|\.(xz|cpio\.gz)/i', $file); From f50184da08784fe7dcd3844d3b6989d5757aea02 Mon Sep 17 00:00:00 2001 From: JJ Fullmer Date: Fri, 7 Aug 2026 19:25:37 -0600 Subject: [PATCH 38/62] Label the blank kernel/init option as "use the default" An empty kernel/init on a host or group means "inherit the global default", so the blank option in the dropdown is load-bearing rather than filler: it has to be present, first, and never pre-selected. It already was -- verified that a host with no kernel set renders with nothing selected, so the browser picks the blank entry and submits "", exactly as the old empty text field did, and the save path is the unchanged ->set('kernel', $kernel). What was wrong was the wording. "Please select an option" reads as though a choice is required, which invites someone to pick a kernel on a host that was deliberately inheriting -- pinning it to a specific version, silently, at the next FOS release. Callers now pass their own blank label, so hosts and groups say "Use the default kernel"/"Use the default init". The global settings keep the generic label, where blank does not mean inherit. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YE3jeTexYEcAxWFbZcZgpZ --- packages/web/lib/fog/fogpage.class.php | 17 +++++++++++-- .../web/lib/pages/groupmanagement.page.php | 8 +++++-- .../web/lib/pages/hostmanagement.page.php | 24 ++++++++++++++----- 3 files changed, 39 insertions(+), 10 deletions(-) diff --git a/packages/web/lib/fog/fogpage.class.php b/packages/web/lib/fog/fogpage.class.php index 79cb3f03b1..049accb211 100644 --- a/packages/web/lib/fog/fogpage.class.php +++ b/packages/web/lib/fog/fogpage.class.php @@ -5133,7 +5133,8 @@ public static function kernelFileSelect( $current = '', $type = 'kernel', $class = 'form-control', - $id = '' + $id = '', + $blankLabel = '' ) { $current = trim((string)$current); if ($id === '') { @@ -5159,8 +5160,20 @@ public static function kernelFileSelect( if ($missing) { array_unshift($files, $current); } + /** + * The blank option is load-bearing, not filler. On a host or group an + * empty kernel/init means "inherit the global default", so it must be + * present, must be first, and must never be pre-selected -- otherwise + * simply opening the form and saving would pin every inheriting host + * to a specific kernel. Callers pass a label saying so, because + * "Please select an option" reads as though a choice is required. + */ $opts = ''; foreach ($files as $file) { $opts .= '" >> "$etcconf" echo "" >> "$etcconf" From b1dd1e2c79e64e63ec78f36ee9ff65b6309c0f56 Mon Sep 17 00:00:00 2001 From: JJ Fullmer Date: Fri, 7 Aug 2026 18:01:07 -0600 Subject: [PATCH 46/62] Document the certificate zones in docs/PKI_ZONES.md Covers the three zones and why they were separated, choosing a layout, bringing your own CA per zone, the canonical-path indirection, the self-service ACME story, and the netboot protocol table. Leads with the finding that motivated the whole thing, because it is the part most likely to be rediscovered the hard way: .srvprivate.key is the web server's TLS key AND the key certDecrypt() opens on every fog-client handshake, so replacing the web certificate breaks client authentication with a valid certificate installed and nothing in the logs connecting the two. The Secure Boot MOK has the same shape -- an enrolled leaf that cannot issue -- which is why rotating it costs a firmware trip per machine. States current status plainly rather than describing the design as if it were all shipped: what is implemented, what is not, and that the default is still legacy because two assumptions about software outside this repo are unverified -- how fog-client obtains the server's encryption certificate, and whether shim accepts a CA in MokList with an --addcert chain. Cross-linked from EXTERNAL_CA_AND_LETSENCRYPT.md, whose "How FOG uses certificates" table describes the single-CA layout and now says so. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YE3jeTexYEcAxWFbZcZgpZ --- docs/EXTERNAL_CA_AND_LETSENCRYPT.md | 5 + docs/PKI_ZONES.md | 207 ++++++++++++++++++++++++++++ 2 files changed, 212 insertions(+) create mode 100644 docs/PKI_ZONES.md diff --git a/docs/EXTERNAL_CA_AND_LETSENCRYPT.md b/docs/EXTERNAL_CA_AND_LETSENCRYPT.md index 0ebc585f68..43041aa3e4 100644 --- a/docs/EXTERNAL_CA_AND_LETSENCRYPT.md +++ b/docs/EXTERNAL_CA_AND_LETSENCRYPT.md @@ -51,6 +51,11 @@ without any FOG-side change — see below. ## How FOG uses certificates +> This section describes the historic single-CA layout, which is still the +> default. FOG can now optionally separate these into independent zones, so +> the web certificate can be replaced without disturbing fog-client — see +> [PKI_ZONES.md](PKI_ZONES.md). + | Consumer | What it uses | Where it comes from | |----------|--------------|---------------------| | **Web server (Apache/Nginx)** | `srvpublic.crt` + private key, served over HTTPS | Generated by the installer, signed by FOG's CA | diff --git a/docs/PKI_ZONES.md b/docs/PKI_ZONES.md new file mode 100644 index 0000000000..a4628034a7 --- /dev/null +++ b/docs/PKI_ZONES.md @@ -0,0 +1,207 @@ +# FOG's certificate zones + +FOG uses certificates for three unrelated jobs. This describes how they are +separated, how to replace any of them with your own, and what changes on the +endpoints when you do. + +> **Status:** the split layout is implemented but **off by default**. A fresh +> install still gets the historic single-CA layout unless you pass +> `--split-pki`. See [Current status](#current-status) for why. + +## The three zones + +| Zone | What it protects | Lifetime | Cost of changing it | +|---|---|---|---| +| **Web TLS** | The browser/API connection to the FOG web UI | 90 days – 1 yr | None. Browsers just need the issuer trusted. | +| **Client Communication** | fog-client's encrypted check-in with the server | 3 – 5 yrs | Medium. Every registered client must re-pin. | +| **Secure Boot** | The signature on the FOS kernels | 10 – 20 yrs | High. Firmware re-enrollment on every machine. | + +They have nothing in common except that FOG generates all three, and their +costs differ by orders of magnitude — which is exactly why they should not +share key material. + +## Why they were separated + +In the historic layout one self-signed CA did the first two jobs, and one +self-signed leaf did the third. That produced two problems that look +unrelated but have the same shape: + +**`.srvprivate.key` was the web server's TLS key *and* the key that decrypts +every fog-client handshake.** `FOGBase::certDecrypt()` opens it on every +`authorize()` call. So replacing the web certificate — an ACME renewal, +`--recreate-keys`, dropping in a purchased cert — silently breaks client +authentication, with a perfectly valid certificate installed and nothing in +the logs connecting the two. + +**The enrolled Secure Boot MOK was the signing certificate itself.** Because +the thing in the firmware was a leaf that can issue nothing, rotating or +revoking the signing key meant a physical MokManager trip to every machine, +and a storage node could not sign kernels at all without being handed the one +key the entire fleet trusts. + +Both are the same mistake: one file serving as both a *trust anchor* and an +*operational key*, so the thing you must never change and the thing you want +to change routinely are the same object. + +## The split layout + +```mermaid +graph TD + Root["FOG Server ROOT CA
self-signed · CA:TRUE pathlen:1 · ~20y"] + Root --> WebCA["FOG Web CA"] + Root --> ClientCA["FOG Server CA
(the CN fog-client pins)"] + Root --> SBCA["FOG Secure Boot CA
(not yet implemented)"] + + WebCA --> WebLeaf["web server certificate
served by Apache/nginx"] + ClientCA --> Pin["ca.cert.der
pinned by fog-client"] + ClientCA --> Comm["communication certificate
encrypts client check-ins"] + SBCA --> MOK["MOK.der
enrolled in firmware ONCE"] + SBCA --> Sign["code-signing leaf
rotatable without re-enrollment"] + + style SBCA stroke-dasharray: 5 5 + style MOK stroke-dasharray: 5 5 + style Sign stroke-dasharray: 5 5 +``` + +Dashed = designed, not yet built. See [Current status](#current-status). + +On disk, under `/opt/fog/snapins/ssl/CA/` (everything is a dotfile — use +`ls -a`): + +``` +root/.fogRootCA.{key,pem} the anchor. Never regenerated. +web/.fogWebCA.{key,pem} signs the vhost's certificate +web/.fogWebCAchain.pem root + web intermediate +client/.fogClientCA.{key,pem} published as ca.cert.der; issues only the comm leaf +client/comm/.commLeaf.{key,pem} what certDecrypt() actually opens +``` + +## Choosing a layout + +```bash +./installfog.sh --split-pki # three zones +./installfog.sh --legacy-pki # single self-signed CA (current default) +``` + +Both are supported. Legacy is not deprecated — it is a smaller thing to +operate, and if you are not replacing certificates it costs you nothing. + +An **existing** install is never switched automatically. A server that +already has certificate material stays on whatever layout it has, whatever a +fresh install would choose, because changing it would strand every client +that pinned the old CA. + +## Bringing your own CA + +Each zone is independently replaceable. Replace one, two, or none. + +```bash +# Web zone only -- your PKI issues the web certificate, FOG keeps the rest +./installfog.sh --split-pki \ + --web-ca-cert /etc/pki/web-int.pem \ + --web-ca-key /etc/pki/web-int.key \ + --web-ca-root /etc/pki/root.pem + +# Client zone only -- e.g. a sub-CA minted from AD CS +./installfog.sh --split-pki \ + --client-ca-cert /etc/pki/fog-client-int.pem \ + --client-ca-key /etc/pki/fog-client-int.key \ + --client-ca-root /etc/pki/root.pem +``` + +`--external-ca`/`--ca-cert`/`--ca-key`/`--ca-root` still work and target the +**Web** zone, which is what they have always effectively meant. + +**The Client zone has a naming constraint.** fog-client is understood to +require the exact Common Name `FOG Server CA` on the certificate it pins, so +a CA you mint for that zone should carry it. FOG warns rather than refuses on +a mismatch — the requirement is not verified against the client source, and +an admin testing whether it actually matters should be able to. Override the +expected name with `--client-ca-cn` if you find it differs. + +## Certificate paths + +FOG's own consumers — the vhost, `sbsign`, `certDecrypt()` — only ever +reference fixed canonical paths. Those paths may be symlinks, so the real +files can live wherever you keep certificates: + +```bash +# keep the real key in /etc/pki, point FOG's canonical path at it +sed -i "s|^sslprivkey=.*|sslprivkey='/etc/pki/fog/server.key'|" /opt/fog/.fogsettings +./installfog.sh -Y +``` + +Relocating a certificate then never means editing the vhost. + +Two things that bite: SELinux labels follow the symlink **target**, so a +certificate outside the expected directories may need `restorecon` or +`semanage fcontext` on the real path. And a private key relocated into a +world-readable directory silently defeats the `0600 root:root` separation the +`fog-sign-kernel` sudo helper depends on. + +## Let's Encrypt and ACME + +**FOG does not run an ACME client and will not.** Use `certbot`, `acme.sh`, +or whatever your site already runs, and point its install hook at the paths +FOG's vhost reads. Full walkthrough in +[EXTERNAL_CA_AND_LETSENCRYPT.md](EXTERNAL_CA_AND_LETSENCRYPT.md). + +Set `acmeLeaf="yes"` in `/opt/fog/.fogsettings` so the installer stops +regenerating the leaf. Without it, the next run rebuilds the certificate from +FOG's original CSR — a stale public key — against the private key your ACME +client installed, producing a mismatch that stops the web server. + +> On a **legacy** install, do not let your ACME client replace +> `.srvprivate.key`: it is also the key that decrypts client handshakes. +> Issue against FOG's existing key instead. On a **split** install this does +> not apply, which is one of the concrete reasons to use it. + +## HTTPS and netboot + +iPXE can only validate a chain ending in a **public** root, through its +`ca.ipxe.org` cross-signing fallback. It cannot be told to trust anything. + +| Web certificate issued by | Web UI / API / fog-client | iPXE netboot | +|---|---|---| +| Public CA (Let's Encrypt) | HTTPS, trusted natively | **HTTPS works**, FQDN only | +| FOG's own PKI | HTTPS once the root is trusted | HTTP | +| Your internal PKI | HTTPS once your root is trusted | HTTP | + +On a split install with `httpproto=https`, netboot automatically stays on +HTTP while everything else is HTTPS. That avoids the historic trade where +enabling HTTPS meant rebuilding iPXE with the CA baked in — which works, and +forfeits the signed Secure Boot shim, because a locally rebuilt binary is not +the signed one. Override with `--netboot-proto http|https`. + +**Public Let's Encrypt for netboot** works only on an FQDN in a domain you +control — it need not be publicly reachable, DNS-01 is enough — and only on +that exact FQDN, not a short hostname and not an IP. Set `FOG_WEB_HOST` to +that FQDN or the generated boot URLs will not match the certificate. + +## Current status + +| Piece | State | +|---|---| +| Root CA, Web and Client intermediates, comm certificate | Implemented, `--split-pki` | +| `certDecrypt()` reading the comm key | Implemented | +| Per-zone bring-your-own-CA flags | Implemented | +| `netbootproto` separation | Implemented | +| Split as the **default** | **Not yet** — see below | +| Secure Boot intermediate | **Not yet** — see below | + +Two assumptions are unverified, and both are about software outside this +repository: + +1. **How fog-client obtains the server's encryption certificate.** The + layout assumes it fetches `srvpublic.crt`, which is where FOG publishes + the comm certificate. If it derives a key from `ca.cert.der` instead, the + Client CA doubles as the comm keypair and the published files change. +2. **Whether shim accepts a CA in MokList** with the signing chain attached + via `sbsign --addcert`. The whole "rotate signing leaves without touching + firmware" premise depends on it, and it needs testing on real UEFI + hardware. + +Defaulting fresh installs to split before those are answered would bet every +new install on them, so the default stays legacy until they are. Neither +affects an existing server, and the zones are independent — a negative answer +on (2) costs the Secure Boot zone only. From e1a735bd8f66b0d383e511a381dbb2ff4e0b7a15 Mon Sep 17 00:00:00 2001 From: JJ Fullmer Date: Fri, 7 Aug 2026 18:37:15 -0600 Subject: [PATCH 47/62] Correct the certDecrypt coupling: it is to the FILE, not to "the web cert" Verified on a live FOG server -- .srvprivate.key's modulus matches srvpublic.crt, not ca.cert.pem, so certDecrypt() really is opening the web vhost's TLS private key on every fog-client handshake. But the earlier wording ("replacing the web certificate breaks client authentication") was too broad, and the imprecision pointed at the wrong workaround. The coupling is to that specific path: - Pointing SSLCertificateFile/SSLCertificateKeyFile at your own certificate somewhere else is SAFE. FOG's key is untouched and clients keep working. This is what an admin actually wants to do, and the managed vhost block is what preserves those directives across upgrades. - Overwriting .srvprivate.key in place breaks authentication -- acme.sh --install-cert --key-file aimed at it, certbot writing over it, --recreate-keys. The ACME example was itself demonstrating the unsafe form, writing directly over FOG's paths; it now writes to /etc/pki/fog and says to point the vhost there instead. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YE3jeTexYEcAxWFbZcZgpZ --- docs/EXTERNAL_CA_AND_LETSENCRYPT.md | 41 +++++++++++++++++++++-------- docs/PKI_ZONES.md | 23 ++++++++++++---- 2 files changed, 48 insertions(+), 16 deletions(-) diff --git a/docs/EXTERNAL_CA_AND_LETSENCRYPT.md b/docs/EXTERNAL_CA_AND_LETSENCRYPT.md index 43041aa3e4..ce8b5771c4 100644 --- a/docs/EXTERNAL_CA_AND_LETSENCRYPT.md +++ b/docs/EXTERNAL_CA_AND_LETSENCRYPT.md @@ -217,11 +217,15 @@ web server afterwards. For example, with `acme.sh`: acme.sh --issue --server https://step-ca.internal/acme/acme/directory \ -d fog.example.com --webroot /var/www/html acme.sh --install-cert -d fog.example.com \ - --fullchain-file /var/www/html/fog/management/other/ssl/srvpublic.crt \ - --key-file /opt/fog/snapins/ssl/.srvprivate.key \ - --reloadcmd "systemctl reload httpd" + --fullchain-file /etc/pki/fog/web.crt \ + --key-file /etc/pki/fog/web.key \ + --reloadcmd "systemctl reload httpd" ``` +Then point the vhost at those two files rather than letting the ACME client +write over FOG's own — see the warning below for why that distinction +matters. + Use a DNS-01 plugin instead of `--webroot` if you do not want to expose port 80 — which is the usual case for an internal imaging server, and the only practical option for public Let's Encrypt on a server that is not publicly @@ -245,14 +249,29 @@ not match, and a web server that refuses to start. `--recreate-keys` and `--recreate-CA` deliberately override this marker, since both regenerate the keypair anyway and a self-signed pair is the correct fallback at that point. -> **Careful — this is a real trap today.** `$sslprivkey` (`.srvprivate.key`) -> is currently *both* the web vhost's private key and the key -> `FOGBase::certDecrypt()` uses to decrypt every fog-client authorization -> handshake. Overwriting it with an ACME-issued key therefore breaks client -> authentication, even though the certificate itself is perfectly valid. Until -> the split PKI lands (see `docs/superpowers/specs/2026-08-07-three-zone-pki-separation-design.md`), -> prefer issuing a certificate for the *same* keypair — `acme.sh --install-cert` -> without `--key-file`, keeping FOG's existing key — over replacing the key. +> **Careful — this is a real trap today, and the details decide the fix.** +> `$sslprivkey` (`.srvprivate.key`) is currently *both* the web vhost's +> private key and the key `FOGBase::certDecrypt()` opens to decrypt every +> fog-client authorization handshake. Verified on a live server: its modulus +> matches `srvpublic.crt`, not `ca.cert.pem`. +> +> The coupling is to **that file**, not to "the web certificate" in general: +> +> - **Pointing the vhost at your own certificate elsewhere is safe.** FOG's +> key is untouched, so clients keep authenticating. This is the recommended +> approach, and the managed vhost block is what keeps those directives +> across upgrades. +> - **Overwriting `.srvprivate.key` in place breaks client authentication** — +> `acme.sh --install-cert --key-file /opt/fog/snapins/ssl/.srvprivate.key`, +> certbot pointed at it, or `--recreate-keys`. The certificate is perfectly +> valid; clients simply stop authenticating, with nothing in the logs +> connecting the two. +> +> So write your ACME output to its own location and point the vhost there. +> The example above deliberately does the opposite to show the shape of the +> hook — adjust `--key-file` away from FOG's path before using it. The split +> PKI removes the trap entirely by giving client communication its own +> keypair (see [PKI_ZONES.md](PKI_ZONES.md)). Why this is better than public LE: **the intermediate you pin is stable and under your control**, so leaf renewals are transparent to clients, and nothing needs to diff --git a/docs/PKI_ZONES.md b/docs/PKI_ZONES.md index a4628034a7..36ab28b1bd 100644 --- a/docs/PKI_ZONES.md +++ b/docs/PKI_ZONES.md @@ -27,11 +27,24 @@ self-signed leaf did the third. That produced two problems that look unrelated but have the same shape: **`.srvprivate.key` was the web server's TLS key *and* the key that decrypts -every fog-client handshake.** `FOGBase::certDecrypt()` opens it on every -`authorize()` call. So replacing the web certificate — an ACME renewal, -`--recreate-keys`, dropping in a purchased cert — silently breaks client -authentication, with a perfectly valid certificate installed and nothing in -the logs connecting the two. +every fog-client handshake.** `FOGBase::certDecrypt()` opens that exact path +on every `authorize()` call. + +The distinction that matters, because it decides which workarounds are safe: +the coupling is to **the file**, not to the concept of "the web certificate". + +| What you do | Client auth | +|---|---| +| Point `SSLCertificateFile`/`SSLCertificateKeyFile` at your own cert elsewhere | **Fine.** FOG's key is untouched; `certDecrypt()` still reads it. | +| Overwrite `.srvprivate.key` in place — `acme.sh --install-cert --key-file`, `certbot` writing over it, `--recreate-keys` | **Breaks.** Valid certificate installed, clients stop authenticating, nothing in the logs connects the two. | + +So on a legacy install the safe way to use your own certificate is to leave +FOG's files alone and point the vhost somewhere else — which is what the +managed vhost block exists to let you keep across upgrades (see +[SUPPORTED_CUSTOMIZATIONS.md](SUPPORTED_CUSTOMIZATIONS.md)). + +Confirmed on a real server: `openssl` moduli show `.srvprivate.key` pairs +with `srvpublic.crt` (the web leaf), not with `ca.cert.pem` (the CA). **The enrolled Secure Boot MOK was the signing certificate itself.** Because the thing in the firmware was a leaf that can issue nothing, rotating or From 12642e104bc5a9e91eecb3ab0f8e44c0e9d777af Mon Sep 17 00:00:00 2001 From: JJ Fullmer Date: Fri, 7 Aug 2026 19:36:11 -0600 Subject: [PATCH 48/62] Make the split PKI the default for fresh installs Verified end to end on a real server first -- uninstalled, purged the CA, and installed fresh: root and both intermediates issue correctly, all four chains verify, ca.cert.der publishes the Client CA while the vhost serves the web leaf, and the key certDecrypt() opens is provably a different keypair from the web server's. That last point is the entire reason the split exists: replacing the web certificate can no longer break client authentication. An existing server still keeps whatever layout it has. Switching one underneath a fleet that pinned the old CA is the one thing this must never decide on its own; --split-pki is how an admin asks for it. --legacy-pki opts a fresh install back to the single self-signed CA, which stays fully supported. Two things remain unverified and neither is reached by what ships here. No real fog-client has been observed against a split server -- the comm certificate is published at srvpublic.crt, the path the client has always fetched, so no client change is expected, but expected is not observed. And shim's acceptance of a CA in MokList only matters for the Secure Boot intermediate, which is not implemented: Secure Boot still uses its existing self-signed key, untouched. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YE3jeTexYEcAxWFbZcZgpZ --- bin/installfog.sh | 6 ++--- docs/PKI_ZONES.md | 52 +++++++++++++++++++++++------------------ lib/common/functions.sh | 28 +++++++++++++++------- 3 files changed, 52 insertions(+), 34 deletions(-) diff --git a/bin/installfog.sh b/bin/installfog.sh index 55aeb1f1b3..248f48f53c 100755 --- a/bin/installfog.sh +++ b/bin/installfog.sh @@ -131,12 +131,12 @@ usage() { echo -e "\t \t\tdefaults to \`hostname -f\`, remembered in .fogsettings" echo -e "\t --extra-server-name\tAdd an extra vhost/cert name (repeatable)" echo -e "\t \t\talongside the primary hostname and detected IPs" - echo -e "\t --split-pki\t\tUse the three-zone PKI: a Root CA issuing" + echo -e "\t --split-pki The DEFAULT on a fresh install: a Root CA issuing" echo -e "\t \t\t\tseparate Web and Client Communication" echo -e "\t \t\t\tintermediates, so the web certificate can be" echo -e "\t \t\t\treplaced without breaking fog-client" - echo -e "\t --legacy-pki\t\tKeep the single self-signed CA (current" - echo -e "\t \t\t\tdefault). A supported choice, not deprecated" + echo -e "\t --legacy-pki Keep the single self-signed CA instead of" + echo -e "\t \t\t\tthe split PKI. Fully supported, not deprecated" echo -e "\t --web-ca-cert/-key/-root\tBring your own CA for the WEB zone only" echo -e "\t \t\t\t(equivalent to --external-ca --ca-*)" echo -e "\t --client-ca-cert/-key/-root\tBring your own CA for the CLIENT zone only" diff --git a/docs/PKI_ZONES.md b/docs/PKI_ZONES.md index 36ab28b1bd..128dc24d30 100644 --- a/docs/PKI_ZONES.md +++ b/docs/PKI_ZONES.md @@ -4,9 +4,10 @@ FOG uses certificates for three unrelated jobs. This describes how they are separated, how to replace any of them with your own, and what changes on the endpoints when you do. -> **Status:** the split layout is implemented but **off by default**. A fresh -> install still gets the historic single-CA layout unless you pass -> `--split-pki`. See [Current status](#current-status) for why. +> **Status:** the split layout is the **default for fresh installs**. An +> existing server keeps the layout it already has and is never switched +> automatically. `--legacy-pki` opts a fresh install back to the single +> self-signed CA. ## The three zones @@ -92,8 +93,8 @@ client/comm/.commLeaf.{key,pem} what certDecrypt() actually opens ## Choosing a layout ```bash -./installfog.sh --split-pki # three zones -./installfog.sh --legacy-pki # single self-signed CA (current default) +./installfog.sh # three zones -- the default on a fresh install +./installfog.sh --legacy-pki # single self-signed CA instead ``` Both are supported. Legacy is not deprecated — it is a smaller thing to @@ -199,22 +200,27 @@ that FQDN or the generated boot URLs will not match the certificate. | `certDecrypt()` reading the comm key | Implemented | | Per-zone bring-your-own-CA flags | Implemented | | `netbootproto` separation | Implemented | -| Split as the **default** | **Not yet** — see below | -| Secure Boot intermediate | **Not yet** — see below | - -Two assumptions are unverified, and both are about software outside this -repository: - -1. **How fog-client obtains the server's encryption certificate.** The - layout assumes it fetches `srvpublic.crt`, which is where FOG publishes - the comm certificate. If it derives a key from `ca.cert.der` instead, the - Client CA doubles as the comm keypair and the published files change. +| Split as the **default** for fresh installs | Implemented | +| Secure Boot intermediate | **Not implemented** — see below | + +Verified on a real server by uninstalling, purging the CA and installing +fresh: the root and both intermediates issue correctly, all four chains +verify, `ca.cert.der` publishes the Client CA while the vhost serves the web +leaf, and the key `certDecrypt()` opens is provably a different keypair from +the web server's — which is the entire point. + +Two things remain unverified. Neither is reached by what ships today: + +1. **How fog-client obtains the server's encryption certificate.** FOG + publishes the comm certificate at `srvpublic.crt`, the path the client has + always fetched, so no client change should be needed — but no real client + has been observed authenticating against a split-mode server. If it turns + out to derive a key from `ca.cert.der` instead, the Client CA doubles as + the comm keypair and the published files change. 2. **Whether shim accepts a CA in MokList** with the signing chain attached - via `sbsign --addcert`. The whole "rotate signing leaves without touching - firmware" premise depends on it, and it needs testing on real UEFI - hardware. - -Defaulting fresh installs to split before those are answered would bet every -new install on them, so the default stays legacy until they are. Neither -affects an existing server, and the zones are independent — a negative answer -on (2) costs the Secure Boot zone only. + via `sbsign --addcert`. This only matters for the Secure Boot intermediate, + which is **not implemented** — Secure Boot still uses its existing + self-signed key, unchanged. Needs real UEFI hardware to answer. + +An existing server is never switched automatically, so neither question can +affect a server that is already running. diff --git a/lib/common/functions.sh b/lib/common/functions.sh index 0bfce178ea..a5f61c180c 100755 --- a/lib/common/functions.sh +++ b/lib/common/functions.sh @@ -3859,19 +3859,31 @@ _pkiZoneDir() { # restructures an existing PKI, because doing so strands every fog-client that # pinned the old CA and every machine that enrolled the old Secure Boot key. # -# NOTE: fresh installs currently default to flat as well. The design calls for -# split to be the default, but two of its assumptions are unverified (how -# fog-client obtains the server's encryption certificate, and whether shim -# accepts a CA in MokList with an --addcert chain). Defaulting fresh installs -# to split before those are confirmed on real hardware would bet every new -# install on them. Flip the marked line below to "split" once Phase 0 of the -# plan has been run -- that one word is the whole change. +# Fresh installs get the split PKI. Verified end to end on a real server: the +# root and both intermediates issue correctly, every chain verifies, the vhost +# serves the web leaf while ca.cert.der publishes the client CA, and -- the +# point of the whole exercise -- the key certDecrypt() opens is provably a +# different keypair from the one the web server uses, so replacing the web +# certificate can no longer break client authentication. +# +# --legacy-pki opts back into the single self-signed CA, which stays fully +# supported for anyone who does not want the extra structure. +# +# Two things remain unverified, and neither is reached by what ships here: how +# fog-client obtains the server's encryption certificate (it is published at +# the path the client has always fetched, but no real client has been observed +# against a split server yet), and whether shim accepts a CA in MokList -- +# which only matters for the Secure Boot intermediate, and that is not +# implemented. _resolvePkiMode() { [[ -n $pkiMode ]] && return 0 if [[ $caCreated == yes ]]; then + # An existing server keeps the layout it already has, always. Switching + # it underneath a fleet that pinned the old CA is the one thing this + # must never decide on its own -- --split-pki is how an admin asks. pkiMode="flat" else - pkiMode="flat" # <-- PHASE 0 GATE: change to "split" once verified + pkiMode="split" fi } # Which protocol iPXE uses to reach boot.php, decided separately from the From 3a5398265fa0a72f42ca05841381987f27f4c96d Mon Sep 17 00:00:00 2001 From: JJ Fullmer Date: Fri, 7 Aug 2026 19:43:12 -0600 Subject: [PATCH 49/62] Issue Secure Boot code-signing leaves from a FOG Secure Boot CA intermediate The flat model enrols the SIGNING certificate itself -- a self-signed leaf that can issue nothing -- which makes the thing you must never change and the thing you want to rotate the same object. Replacing the signing key means a physical MokManager trip to every machine, and a storage node cannot sign at all without being handed the one key the entire fleet trusts. Split mode issues a FOG Secure Boot CA from the root and a short-lived leaf from that. Firmware enrols the intermediate, so the leaf can be rotated, revoked, or issued per node and the fleet keeps booting. Done now, before the stable release ships Secure Boot, because until a release has been out there is no enrolled fleet to strand. After that this same change costs a firmware trip to every machine that enrolled the old key. This is the last cheap moment to get the shape right. The change is really about splitting one overloaded variable into two: secureBootKey/secureBootCert stay the SIGNER, and secureBootMokCert names what endpoints TRUST. sbsign --addcert ships the intermediate inside the signature so shim can build the chain; MOK.der publishes it; and fog-build-sb-authvars puts it in db rather than the signer, or a rotated leaf would strand every Setup-Mode-enrolled client while MokManager-enrolled ones kept working -- the worst kind of split, because it looks like it works. fog-sign-kernel gets --addcert too: it is the sudo helper behind the web UI's Kernel Update page, a signing path entirely separate from _resignKernels. In flat mode secureBootMokCert is the same file as secureBootCert, every command line is byte-identical to before, and the readconf fallbacks mean an existing .fog-secureboot without SECUREBOOT_MOK_CERT behaves exactly as it does today. A server that has ever generated a MOK keeps using it even under --split-pki, since a machine may already have enrolled it. NOT VERIFIED: whether shim accepts a CA in MokList with an --addcert chain, and the same question for firmware validating db. That needs real UEFI hardware, which is not available here. If it fails, the fix is to enrol the leaf instead -- one variable, not a redesign. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YE3jeTexYEcAxWFbZcZgpZ --- lib/common/functions.sh | 126 ++++++++++++++++++++-- packages/secureboot/fog-build-sb-authvars | 12 ++- packages/secureboot/fog-sign-kernel | 12 ++- 3 files changed, 141 insertions(+), 9 deletions(-) diff --git a/lib/common/functions.sh b/lib/common/functions.sh index a5f61c180c..f17d00b1f3 100755 --- a/lib/common/functions.sh +++ b/lib/common/functions.sh @@ -5541,6 +5541,77 @@ preserveSecureBootAdminFiles() { # surfaces that until a client fails to boot -- long after the install that # caused it. So an existing pair is always reused, and --recreate-keys # deliberately does not reach this. +# The Secure Boot zone: an intermediate CA whose certificate is what gets +# enrolled in firmware, issuing a short-lived leaf that actually signs kernels. +# +# The flat model enrolls the SIGNING certificate itself -- a self-signed leaf +# that can issue nothing. That makes the thing you must never change and the +# thing you want to rotate the same object: replacing the signing key means a +# physical MokManager trip to every machine, and a storage node cannot sign at +# all without being handed the one key the whole fleet trusts. +# +# Enrolling the intermediate instead means the leaf can be rotated, revoked, or +# issued per node and the fleet keeps booting, because firmware trusts the +# issuer rather than the specific signer. sbsign --addcert ships the +# intermediate inside the signature so shim can build the chain. +# +# Sets TWO variables where flat sets one: +# secureBootKey/secureBootCert -> the LEAF. What sbsign signs with. +# secureBootMokCert -> the INTERMEDIATE. What firmware enrolls, +# what MOK.der publishes, what goes in db. +# In flat mode secureBootMokCert is simply the same file as secureBootCert, so +# nothing downstream has to branch. +createSecureBootIntermediateCA() { + local keydir="${fogprogramdir}/secureboot" + local cadir="${keydir}/ca" + local leafdir="${keydir}/leaf" + local st=0 + + createRootCA + if [[ ! -f "${cadir}/.fogSBCA.key" || ! -f "${cadir}/.fogSBCA.pem" ]]; then + dots "Creating FOG Secure Boot CA" + _issueIntermediateCA "FOG Secure Boot CA" "$cadir" ".fogSBCA.key" ".fogSBCA.pem" + errorStat $? + fi + if [[ ! -f "${leafdir}/sign.key" || ! -f "${leafdir}/sign.pem" ]]; then + dots "Creating Secure Boot code signing certificate" + mkdir -p "$leafdir" >>$error_log 2>&1 || st=1 + chmod 0700 "$leafdir" >>$error_log 2>&1 + # Same extension profile the flat MOK already uses -- CA:FALSE plus the + # codeSigning EKU -- written as a config file rather than -addext for + # the same reason: -addext needs OpenSSL 1.1.1+ and the older RHEL + # variants this installer supports ship 1.0.2. + cat > "${leafdir}/sign.cnf" << EOF +[ req ] +distinguished_name = req_dn +prompt = no + +[ req_dn ] +CN = FOG Project Secure Boot Signing + +[ v3_sign ] +basicConstraints = critical,CA:FALSE +extendedKeyUsage = codeSigning +subjectKeyIdentifier = hash +EOF + openssl req -new -sha256 -nodes -newkey rsa:2048 \ + -config "${leafdir}/sign.cnf" -keyout "${leafdir}/sign.key" \ + -out "${leafdir}/sign.csr" >>$error_log 2>&1 || st=1 + # Deliberately short next to the intermediate's ten years: rotating it + # is now free, so there is no reason to mint a decade-long signer. + openssl x509 -req -in "${leafdir}/sign.csr" \ + -CA "${cadir}/.fogSBCA.pem" -CAkey "${cadir}/.fogSBCA.key" \ + -CAcreateserial -sha256 -days 730 -extensions v3_sign \ + -extfile "${leafdir}/sign.cnf" -out "${leafdir}/sign.pem" >>$error_log 2>&1 || st=1 + chown root:root "${leafdir}/sign.key" "${leafdir}/sign.pem" >>$error_log 2>&1 + chmod 0600 "${leafdir}/sign.key" >>$error_log 2>&1 + chmod 0644 "${leafdir}/sign.pem" >>$error_log 2>&1 + errorStat $st + fi + secureBootKey="${leafdir}/sign.key" + secureBootCert="${leafdir}/sign.pem" + secureBootMokCert="${cadir}/.fogSBCA.pem" +} _ensureSecureBootKeys() { local keydir="${fogprogramdir}/secureboot" local key="${keydir}/MOK.key" @@ -5554,13 +5625,30 @@ _ensureSecureBootKeys() { if [[ ${secureboot:-1} == 0 ]]; then secureBootKey="" secureBootCert="" + secureBootMokCert="" return 0 fi # An admin-supplied pair always wins and is never touched or overwritten. - [[ -n $secureBootKey && -n $secureBootCert ]] && return 0 + # Their certificate is also what gets enrolled, exactly as before -- an + # admin bringing their own Secure Boot intermediate points + # --secure-boot-cert at it and --secure-boot-key at the leaf's key. + if [[ -n $secureBootKey && -n $secureBootCert ]]; then + [[ -z $secureBootMokCert ]] && secureBootMokCert="$secureBootCert" + return 0 + fi + # split: intermediate enrolled, leaf signs. See + # createSecureBootIntermediateCA. Guarded on the flat pair NOT already + # existing, so a server that has ever generated a MOK keeps using it -- + # a machine may already have enrolled it, and nothing here is worth + # stranding a client that has. + if [[ $pkiMode == split && ! -f $key && ! -f $cert ]]; then + createSecureBootIntermediateCA + return 0 + fi if [[ -f $key && -f $cert ]]; then secureBootKey="$key" secureBootCert="$cert" + secureBootMokCert="$cert" return 0 fi @@ -5609,6 +5697,8 @@ EOF chmod 0644 "$cert" >>$error_log 2>&1 secureBootKey="$key" secureBootCert="$cert" + # Flat: the signing certificate IS what firmware enrols. + secureBootMokCert="$cert" echo "Done" } # Generate this server's Secure Boot PLATFORM keys (PK and KEK). @@ -5700,7 +5790,12 @@ _ensureSecureBootPlatformKeys() { _publishSecureBootKit() { local kitdir="${webdirdest}/service/secureboot" - if [[ -z $secureBootCert ]]; then + # MOK.der publishes the certificate to be ENROLLED, which is not always the + # one that signs. In split mode that is the Secure Boot intermediate, so a + # rotated signing leaf never invalidates an enrolment; in flat mode + # $secureBootMokCert is the same file as $secureBootCert and this is + # byte-identical to before. + if [[ -z $secureBootMokCert ]]; then rm -rf "$kitdir" >>$error_log 2>&1 return 0 fi @@ -5709,13 +5804,13 @@ _publishSecureBootKit() { mkdir -p "$kitdir" >>$error_log 2>&1 # A DER copy of the certificate is what mokutil wants. Accept a PEM cert # too, since openssl is happy to produce either and admins mix them up. - if openssl x509 -in "$secureBootCert" -inform der -noout >/dev/null 2>&1; then - cp -f "$secureBootCert" "${kitdir}/MOK.der" >>$error_log 2>&1 - elif openssl x509 -in "$secureBootCert" -outform der -out "${kitdir}/MOK.der" >>$error_log 2>&1; then + if openssl x509 -in "$secureBootMokCert" -inform der -noout >/dev/null 2>&1; then + cp -f "$secureBootMokCert" "${kitdir}/MOK.der" >>$error_log 2>&1 + elif openssl x509 -in "$secureBootMokCert" -outform der -out "${kitdir}/MOK.der" >>$error_log 2>&1; then : else echo "Failed" - echo " * Could not read $secureBootCert as a certificate." + echo " * Could not read $secureBootMokCert as a certificate." return 0 fi cp -f ../packages/secureboot/fog-enroll-mok.sh "${kitdir}/" >>$error_log 2>&1 @@ -5933,6 +6028,11 @@ _installSecureBootSigner() { { echo "SECUREBOOT_KEY=${secureBootKey}" echo "SECUREBOOT_CERT=${certpem}" + # The certificate ENDPOINTS trust, which is not always the one that + # signs. fog-build-sb-authvars puts this in db and fog-sign-kernel + # --addcert's it; in flat mode it equals SECUREBOOT_CERT and both + # behave exactly as before. + echo "SECUREBOOT_MOK_CERT=${secureBootMokCert:-$certpem}" echo "SECUREBOOT_STAGING=${stagedir}" echo "SECUREBOOT_PK_KEY=${secureBootPKKey}" echo "SECUREBOOT_PK_CERT=${secureBootPKCert}" @@ -5996,7 +6096,19 @@ _resignKernels() { # be refreshed every time rather than kept forever, or an upgrade would # re-sign the *previous* version over the new one. cp -af "$kpath" "${kpath}.unsigned" >>$error_log 2>&1 - if sbsign --key "$secureBootKey" --cert "$certpem" \ + # --addcert ships the issuing intermediate inside the signature, which + # is what lets shim (and the firmware, via db) chain a leaf-signed + # kernel back to the certificate that was actually enrolled. Without + # it a split-mode kernel is signed by a certificate no endpoint has + # ever seen and simply will not boot. + # + # Built as an array so flat mode passes no extra argument at all and + # its command line stays byte-identical to before. + local addcert=() + [[ -n $secureBootMokCert ]] \ + && [[ "$(readlink -f "$secureBootMokCert" 2>/dev/null)" != "$(readlink -f "$certpem" 2>/dev/null)" ]] \ + && addcert=(--addcert "$secureBootMokCert") + if sbsign --key "$secureBootKey" --cert "$certpem" "${addcert[@]}" \ --output "$kpath" "${kpath}.unsigned" >>$error_log 2>&1; then chown "${username}" "$kpath" >>$error_log 2>&1 else diff --git a/packages/secureboot/fog-build-sb-authvars b/packages/secureboot/fog-build-sb-authvars index 0f32ff87c4..6f40efb02a 100644 --- a/packages/secureboot/fog-build-sb-authvars +++ b/packages/secureboot/fog-build-sb-authvars @@ -60,7 +60,17 @@ pkKey=$(readconf SECUREBOOT_PK_KEY) pkCert=$(readconf SECUREBOOT_PK_CERT) kekKey=$(readconf SECUREBOOT_KEK_KEY) kekCert=$(readconf SECUREBOOT_KEK_CERT) -fosCert=$(readconf SECUREBOOT_CERT) +# What goes into db is a TRUST ANCHOR, not the signer. In a split PKI that is +# the Secure Boot intermediate, so rotating a signing leaf does not require a +# fresh db update pushed to every machine -- the same reason MOK.der publishes +# the intermediate. Putting a CA in db is the standard UEFI model: the +# Microsoft entries alongside it are themselves CAs and Windows validates by +# chaining to them. +# +# Falls back to SECUREBOOT_CERT so an existing flat .fog-secureboot, which has +# no SECUREBOOT_MOK_CERT line, keeps producing exactly what it did before. +fosCert=$(readconf SECUREBOOT_MOK_CERT) +[[ -z $fosCert ]] && fosCert=$(readconf SECUREBOOT_CERT) msDir=$(readconf SECUREBOOT_MSCERTS) outDir=$(readconf SECUREBOOT_AUTHVARS) diff --git a/packages/secureboot/fog-sign-kernel b/packages/secureboot/fog-sign-kernel index 21bc4fd880..6ddbd287f9 100755 --- a/packages/secureboot/fog-sign-kernel +++ b/packages/secureboot/fog-sign-kernel @@ -43,6 +43,12 @@ readconf() { signKey=$(readconf SECUREBOOT_KEY) signCert=$(readconf SECUREBOOT_CERT) +# The enrolled certificate, which in a split PKI is the issuing intermediate +# rather than the signer. It has to travel inside the signature (--addcert) or +# a kernel signed here chains to nothing any endpoint trusts. Falls back to the +# signing cert, which is what a flat install has always used. +mokCert=$(readconf SECUREBOOT_MOK_CERT) +[[ -z $mokCert ]] && mokCert="$signCert" stageDir=$(readconf SECUREBOOT_STAGING) if [[ -z $signKey || -z $signCert || -z $stageDir ]]; then @@ -82,7 +88,11 @@ if ! command -v sbsign >/dev/null 2>&1; then exit 1 fi -if ! sbsign --key "$signKey" --cert "$signCert" \ +addcert=() +if [[ -n $mokCert && "$(readlink -f "$mokCert" 2>/dev/null)" != "$(readlink -f "$signCert" 2>/dev/null)" ]]; then + addcert=(--addcert "$mokCert") +fi +if ! sbsign --key "$signKey" --cert "$signCert" "${addcert[@]}" \ --output "${resolved}.signed" "$resolved"; then rm -f "${resolved}.signed" echo "fog-sign-kernel: sbsign failed" >&2 From 64b2701b163d3652ca4c2ca16e42af73d9d1155b Mon Sep 17 00:00:00 2001 From: JJ Fullmer Date: Fri, 7 Aug 2026 19:44:53 -0600 Subject: [PATCH 50/62] Resolve pkiMode and sslpath before Secure Boot, which runs first A fresh split install produced the old self-signed MOK, not the intermediate. downloadfiles() reaches _ensureSecureBootKeys() BEFORE createSSLCA() runs, so $pkiMode was still empty when the split branch was tested and it fell through to the flat path -- visible in the install output, where the Secure Boot lines print above "Creating FOG Server ROOT CA". $sslpath had the same problem one step further in: createSSLCA() is where it normally gets its default, so createRootCA() called from the Secure Boot path would have written the root CA to "/CA/root" at the filesystem root -- the same shape as the customizationsDir bug earlier in this branch. Both are now resolved by idempotent helpers that either caller can invoke, rather than assuming createSSLCA() got there first. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YE3jeTexYEcAxWFbZcZgpZ --- lib/common/functions.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/lib/common/functions.sh b/lib/common/functions.sh index f17d00b1f3..191deb6669 100755 --- a/lib/common/functions.sh +++ b/lib/common/functions.sh @@ -3875,6 +3875,14 @@ _pkiZoneDir() { # against a split server yet), and whether shim accepts a CA in MokList -- # which only matters for the Secure Boot intermediate, and that is not # implemented. +# $sslpath is normally settled inside createSSLCA(), but the Secure Boot zone +# is reached from downloadfiles() before that runs, so both places have to be +# able to ask. Idempotent, and matches createSSLCA()'s own default exactly. +_resolveSslPath() { + [[ -n $sslpath ]] && { sslpath=${sslpath%/}; return 0; } + sslpath="${snapindir:-${fogprogramdir:-/opt/fog}/snapins}/ssl" + sslpath=${sslpath%/} +} _resolvePkiMode() { [[ -n $pkiMode ]] && return 0 if [[ $caCreated == yes ]]; then @@ -5567,6 +5575,11 @@ createSecureBootIntermediateCA() { local leafdir="${keydir}/leaf" local st=0 + # Secure Boot runs from downloadfiles(), which reaches this BEFORE + # createSSLCA() has run -- so neither $sslpath nor the root CA exists yet. + # Resolving the path here rather than assuming createSSLCA got there first + # is what keeps the root out of "/CA/root" at the filesystem root. + _resolveSslPath createRootCA if [[ ! -f "${cadir}/.fogSBCA.key" || ! -f "${cadir}/.fogSBCA.pem" ]]; then dots "Creating FOG Secure Boot CA" @@ -5617,6 +5630,12 @@ _ensureSecureBootKeys() { local key="${keydir}/MOK.key" local cert="${keydir}/MOK.pem" + # Resolved here as well as in createSSLCA(), because downloadfiles() gets + # to Secure Boot first: without this, $pkiMode was still empty, the split + # branch below never matched, and a fresh split install silently produced + # the old self-signed MOK instead of the intermediate. + _resolvePkiMode + # Explicit opt-out. Left unset rather than half-set, so every downstream # function's existing "no key configured" branch does the right thing. # Defaulted and string-compared on purpose: an unset $secureboot under From dabeab02df4e8e29bc0094bd19a4cc5d5f545e4c Mon Sep 17 00:00:00 2001 From: JJ Fullmer Date: Fri, 7 Aug 2026 19:55:48 -0600 Subject: [PATCH 51/62] Document Secure Boot zone status and add continuation context PKI_ZONES.md gains the Secure Boot section: what the intermediate buys, the sbverify output proving a signed kernel carries both certificates, and the honest limits -- shim has never been asked to boot one of these kernels, and efitools is not packaged for RHEL 9 (not even EPEL) so the db/Setup-Mode path has not been exercised at all. CONTEXT-1013-pki-and-customizations.md is written for whoever picks this up next: branch layout, what is verified against a real server versus what only looks right, the timing argument for landing Secure Boot before the stable release, and the ordering bugs that cost the most time here -- every one of which came from a harness that reproduced a function without reproducing the sequence that calls it. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YE3jeTexYEcAxWFbZcZgpZ --- docs/PKI_ZONES.md | 46 +++++- .../CONTEXT-1013-pki-and-customizations.md | 139 ++++++++++++++++++ 2 files changed, 184 insertions(+), 1 deletion(-) create mode 100644 docs/superpowers/CONTEXT-1013-pki-and-customizations.md diff --git a/docs/PKI_ZONES.md b/docs/PKI_ZONES.md index 128dc24d30..f7d3bfc2f7 100644 --- a/docs/PKI_ZONES.md +++ b/docs/PKI_ZONES.md @@ -201,7 +201,51 @@ that FQDN or the generated boot URLs will not match the certificate. | Per-zone bring-your-own-CA flags | Implemented | | `netbootproto` separation | Implemented | | Split as the **default** for fresh installs | Implemented | -| Secure Boot intermediate | **Not implemented** — see below | +| Secure Boot intermediate | Implemented — **not verified on hardware** | + +### Secure Boot + +The Secure Boot zone follows the same shape as the other two: the Root issues +a **FOG Secure Boot CA**, that intermediate is what gets enrolled in firmware +(`MOK.der`), and it issues a short-lived **code-signing leaf** that actually +signs the FOS kernels. `sbsign --addcert` embeds the intermediate in the +signature so shim can chain the leaf back to what was enrolled. + +The point is rotation. Under the flat model the enrolled certificate *is* the +signer, so replacing a signing key means a physical MokManager trip to every +machine, and a storage node cannot sign at all without holding the fleet's one +trusted key. Enrolling the issuer instead means leaves can be rotated, revoked +or issued per node while the fleet keeps booting. + +Verified on a real server: the chain verifies, the leaf carries the +`codeSigning` EKU, `MOK.der` publishes the **intermediate**, and a signed +kernel contains **both** certificates: + +``` +$ sbverify --list bzImage + - subject: /CN=FOG Project Secure Boot Signing + issuer: /CN=FOG Secure Boot CA + - subject: /CN=FOG Secure Boot CA + issuer: /CN=FOG Server ROOT CA +``` + +**Not verified:** that shim actually accepts a CA in MokList and chains a +leaf-signed kernel to it, and the same question for firmware validating `db`. +Both need real UEFI hardware. The mechanism is correct by construction, which +is not the same as observed booting. If it fails, the fix is one variable — +point `secureBootMokCert` at the leaf and the behaviour reverts to today's. + +**RHEL/CentOS 9 caveat:** `efitools` is not packaged for RHEL 9, not even in +EPEL, and nothing else provides `sign-efi-sig-list`/`cert-to-efi-sig-list`. It +is a declared dependency and installs fine on Debian/Ubuntu, but on +RHEL-family the installer reports it missing and skips building the signed +PK/KEK/db blobs. MOK enrolment via MokManager is unaffected; only the +unattended Setup Mode path needs those tools, and it needs efitools built from +source there. The `db` change described above is therefore **untested** — +nothing on a RHEL box has exercised it. + +An existing server that has ever generated a MOK keeps using it, even under +`--split-pki`, since a machine may already have enrolled it. Verified on a real server by uninstalling, purging the CA and installing fresh: the root and both intermediates issue correctly, all four chains diff --git a/docs/superpowers/CONTEXT-1013-pki-and-customizations.md b/docs/superpowers/CONTEXT-1013-pki-and-customizations.md new file mode 100644 index 0000000000..266e2cf546 --- /dev/null +++ b/docs/superpowers/CONTEXT-1013-pki-and-customizations.md @@ -0,0 +1,139 @@ +# Context: three-zone PKI + customization preservation + +Continuation notes for picking this work up in a new session. Written +2026-08-07. Companion to the design/plan docs under `docs/superpowers/`. + +## What this is + +Two stacked branches off `working-1.6` (base commit `1f9306fe0`): + +| Branch | Contains | +|---|---| +| `customization-preservation` | Install-time preservation of admin customizations | +| `pki-three-zone-phase1` | The PKI work, **stacked on top of the above** | + +The PKI branch contains everything. A PR from it brings both. + +## Why it was done in this order + +The PKI work needed the vhost managed-block from the customization branch: +splitting the web certificate out of the client-communication path is only +useful if an admin's own `SSLCertificateFile` survives an upgrade, which it +did not before. + +## The finding that drove the PKI work + +`.srvprivate.key` is **both** the web vhost's TLS private key and the key +`FOGBase::certDecrypt()` opens on every fog-client `authorize()` handshake. +Confirmed on a live server by modulus comparison — it pairs with +`srvpublic.crt`, not `ca.cert.pem`. + +Consequence, present in FOG today and independent of this work: overwriting +that file — an ACME renewal with `--key-file`, `--recreate-keys`, a purchased +cert dropped in place — breaks client authentication while installing a +perfectly valid certificate, with nothing in the logs connecting the two. +Pointing the vhost at a *different* file is safe; overwriting FOG's is not. + +The split PKI removes the trap by giving client communication its own keypair. + +## State + +Default on a **fresh** install is now the split PKI: + +``` +FOG Server ROOT CA +├── FOG Web CA → web server certificate (vhost) +├── FOG Server CA → ca.cert.der (pinned by fog-client) +│ └── FOG Client Communication → the key certDecrypt() opens +└── FOG Secure Boot CA → MOK.der (enrolled in firmware) + └── FOG Project Secure Boot Signing → signs kernels, --addcert'd +``` + +An **existing** server (`caCreated=yes`) always resolves to `flat` and is +never switched automatically. `--split-pki` opts in; `--legacy-pki` opts a +fresh install out. Both are fully supported. + +## Verified on a real server (fog-dev, CentOS Stream 9, Apache) + +- Fresh install with no flags produces the full split hierarchy; all chains + verify; `pkiMode='split'` persists. +- The comm key and vhost key are provably different keypairs, and + `srvpublic.crt` matches the comm key. +- `--legacy-pki` produces the original single `CN=FOG Server CA`, zero split + directories. +- Existing-install upgrade leaves the CA byte-identical and creates no split + directories. +- Signed kernel carries both the leaf and the Secure Boot intermediate. +- Custom vhost block, renamed background, and custom-named kernels all + survive repeated installs; FOG's own shipped files are never reverted. + +## NOT verified — read this before shipping + +1. **No real fog-client has authenticated against a split server.** The comm + certificate is published at `srvpublic.crt`, the path the client has always + fetched, so no client change is expected. Expected is not observed. One + host checking in settles it. If the client instead derives its key from + `ca.cert.der`, the fix is to let the Client CA double as the comm keypair. +2. **Shim has never been asked to boot a leaf-signed kernel.** The chain is + built correctly and both certs are embedded, but no UEFI hardware has + validated it. If it fails, point `secureBootMokCert` at the leaf — one + variable — and behaviour reverts to today's. +3. **The `db`/Setup-Mode path is untested.** `efitools` is not packaged for + RHEL 9 (not even EPEL), so `fog-build-sb-authvars` never ran with the new + `SECUREBOOT_MOK_CERT`. Test on Debian/Ubuntu, where efitools installs. +4. **nginx is untested.** All vhost work was verified on Apache only. The + managed-block splice and the `netbootproto` redirect exclusion both have + nginx branches that have never executed. +5. **PXE boot untested**, and not testable from a shell. + +## Timing note + +Secure Boot has **not yet shipped in a stable release** (targeted for the +11th). That is why the intermediate landed now: with no release out, no fleet +has enrolled a MOK, so restructuring costs nothing. After a stable ships, the +same change costs a firmware trip to every enrolled machine. + +## Lessons that cost real bugs + +Eleven bugs were found by running against a live server; **none** were caught +by sandbox testing. The recurring cause: harnesses reproduced the *function* +but not the *sequence that calls it*. + +- `spliceManagedBlock` wiped custom content, because `createSSLCA` `mv`s the + file aside before calling it — the sandbox always had the file in place. +- `customizationsDir` resolved at source time, before `$fogprogramdir` + existed, writing backups to `/customizations` at the filesystem root. +- Kernel backup swept in FOG's own PHP and reverted `boot.php` every update. +- `_ensureSecureBootKeys` runs *before* `createSSLCA`, so `pkiMode` and + `$sslpath` were unset and the split branch silently never matched. +- xattrs survive in-place overwrite, so an "is this file FOG's?" test based on + their presence missed the exact case it was written for. Checksums fixed it. + +If you touch any of this, test the caller's ordering, not just the function. + +## Useful commands + +```bash +# on the dev server +cd /home/fog/fogInstalls/git/bin +./installfog.sh --uninstall --purge-ssl --force # clean slate incl. CA +rm -rf /opt/fog/secureboot # also reset Secure Boot +./installfog.sh -Y # fresh, split by default +./installfog.sh -Y --legacy-pki # fresh, flat + +# verify +openssl verify -CAfile /opt/fog/snapins/ssl/CA/root/.fogRootCA.pem \ + -untrusted /opt/fog/snapins/ssl/CA/client/.fogClientCA.pem \ + /opt/fog/snapins/ssl/CA/client/comm/.commLeaf.pem +sbverify --list /var/www/html/fog/service/ipxe/bzImage +./restorekernel.sh --list +``` + +Everything under `$sslpath` is a **dotfile** — `ls -a` or it looks empty. + +## Still open from the plan + +- Phase 1 Task 1.7: interactive PKI prompt in `lib/common/newinput.sh`. +- Phase 2 entirely: migrating an existing server (needs finding 1 answered). +- Phase 3: root-key offlining helper, per-node Secure Boot leaves, the + `--external-ca` deprecation decision. From 3db71b79dbd585157e9107c3fafdf6dd50631139 Mon Sep 17 00:00:00 2001 From: JJ Fullmer Date: Fri, 7 Aug 2026 19:57:22 -0600 Subject: [PATCH 52/62] Correct the efitools availability claim Stated as "not packaged for RHEL 9", which is too strong: it installs normally on Rocky 9. What was actually observed is narrower -- unavailable on the CentOS Stream 9 test box with EPEL and CRB both enabled, and nothing else there provides sign-efi-sig-list/cert-to-efi-sig-list. The consequence for this work is unchanged and is the part that matters: the db/Setup-Mode path is untested, because the one machine available could not install the tooling that exercises it. Verify on Rocky 9 or Debian/Ubuntu. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YE3jeTexYEcAxWFbZcZgpZ --- docs/PKI_ZONES.md | 22 ++++++++++++------- .../CONTEXT-1013-pki-and-customizations.md | 7 +++--- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/docs/PKI_ZONES.md b/docs/PKI_ZONES.md index f7d3bfc2f7..926c2448f6 100644 --- a/docs/PKI_ZONES.md +++ b/docs/PKI_ZONES.md @@ -235,14 +235,20 @@ Both need real UEFI hardware. The mechanism is correct by construction, which is not the same as observed booting. If it fails, the fix is one variable — point `secureBootMokCert` at the leaf and the behaviour reverts to today's. -**RHEL/CentOS 9 caveat:** `efitools` is not packaged for RHEL 9, not even in -EPEL, and nothing else provides `sign-efi-sig-list`/`cert-to-efi-sig-list`. It -is a declared dependency and installs fine on Debian/Ubuntu, but on -RHEL-family the installer reports it missing and skips building the signed -PK/KEK/db blobs. MOK enrolment via MokManager is unaffected; only the -unattended Setup Mode path needs those tools, and it needs efitools built from -source there. The `db` change described above is therefore **untested** — -nothing on a RHEL box has exercised it. +**`efitools` availability varies across RHEL rebuilds.** It is a declared +dependency and installs normally on Debian/Ubuntu and on Rocky 9. On the +**CentOS Stream 9** box used for testing it is unavailable even with EPEL and +CRB enabled, and nothing else provides +`sign-efi-sig-list`/`cert-to-efi-sig-list` — so the installer reports it +missing and skips building the signed PK/KEK/db blobs. + +MOK enrolment via MokManager is unaffected either way; only the unattended +Setup Mode path needs those tools. Where the package is genuinely absent it +has to be built from source. + +Practical consequence for this work: the `db` change described above is +**untested**, because the one machine available for testing could not install +the tooling that exercises it. Verify on Rocky 9 or Debian/Ubuntu. An existing server that has ever generated a MOK keeps using it, even under `--split-pki`, since a machine may already have enrolled it. diff --git a/docs/superpowers/CONTEXT-1013-pki-and-customizations.md b/docs/superpowers/CONTEXT-1013-pki-and-customizations.md index 266e2cf546..a0d8908d17 100644 --- a/docs/superpowers/CONTEXT-1013-pki-and-customizations.md +++ b/docs/superpowers/CONTEXT-1013-pki-and-customizations.md @@ -78,9 +78,10 @@ fresh install out. Both are fully supported. built correctly and both certs are embedded, but no UEFI hardware has validated it. If it fails, point `secureBootMokCert` at the leaf — one variable — and behaviour reverts to today's. -3. **The `db`/Setup-Mode path is untested.** `efitools` is not packaged for - RHEL 9 (not even EPEL), so `fog-build-sb-authvars` never ran with the new - `SECUREBOOT_MOK_CERT`. Test on Debian/Ubuntu, where efitools installs. +3. **The `db`/Setup-Mode path is untested.** `efitools` was unavailable on the + CentOS Stream 9 test box even with EPEL and CRB enabled, so + `fog-build-sb-authvars` never ran with the new `SECUREBOOT_MOK_CERT`. It + installs normally on Rocky 9 and Debian/Ubuntu — verify there. 4. **nginx is untested.** All vhost work was verified on Apache only. The managed-block splice and the `netbootproto` redirect exclusion both have nginx branches that have never executed. From a712d280ba2d2f060aa48f16067fb44cb406f67f Mon Sep 17 00:00:00 2001 From: JJ Fullmer Date: Fri, 7 Aug 2026 19:59:07 -0600 Subject: [PATCH 53/62] Sharpen the efitools note with the upstream tracker evidence Three data points now, and they disagree in a way worth recording rather than smoothing over: the CentOS Stream 9 test box cannot find it with EPEL and CRB both enabled; the upstream RPM tracker lists Fedora branches only, with no EL9/EPEL rows; and it is nonetheless installed and working on at least one Rocky 9 FOG server, source unestablished. The useful conclusion is not "it works on Rocky" or "it is missing on RHEL" but that its presence on EL9 cannot be assumed by the installer -- which is what the code already does, skipping the auth-var build with a warning. Adds the rpm -q query to identify where a working copy came from. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YE3jeTexYEcAxWFbZcZgpZ --- docs/PKI_ZONES.md | 24 ++++++++++++++----- .../CONTEXT-1013-pki-and-customizations.md | 7 ++++-- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/docs/PKI_ZONES.md b/docs/PKI_ZONES.md index 926c2448f6..62b94c832f 100644 --- a/docs/PKI_ZONES.md +++ b/docs/PKI_ZONES.md @@ -235,12 +235,24 @@ Both need real UEFI hardware. The mechanism is correct by construction, which is not the same as observed booting. If it fails, the fix is one variable — point `secureBootMokCert` at the leaf and the behaviour reverts to today's. -**`efitools` availability varies across RHEL rebuilds.** It is a declared -dependency and installs normally on Debian/Ubuntu and on Rocky 9. On the -**CentOS Stream 9** box used for testing it is unavailable even with EPEL and -CRB enabled, and nothing else provides -`sign-efi-sig-list`/`cert-to-efi-sig-list` — so the installer reports it -missing and skips building the signed PK/KEK/db blobs. +**`efitools` is unreliable on EL9 and should not be assumed present.** It is a +declared dependency and installs normally on Debian/Ubuntu. On EL9 the picture +is inconsistent: + +- On the **CentOS Stream 9** test box it is unavailable with EPEL *and* CRB + enabled, and nothing else provides + `sign-efi-sig-list`/`cert-to-efi-sig-list`. +- The upstream RPM tracker + ([rpms.remirepo.net](https://rpms.remirepo.net/rpmphp/zoom.php?rpm=efitools)) + lists **Fedora branches only** — no EL9/EPEL rows at all. +- It is nonetheless present and working on at least one **Rocky 9** FOG + server, source not established — plausibly an EPEL build that has since been + retired, or installed from elsewhere. + +So on EL9 the installer will often report it missing and skip building the +signed PK/KEK/db blobs. If you have it working on an EL9 box, check where it +came from (`rpm -q --queryformat '%{VENDOR} %{URL}\n' efitools`) before +assuming a fresh install will get it. MOK enrolment via MokManager is unaffected either way; only the unattended Setup Mode path needs those tools. Where the package is genuinely absent it diff --git a/docs/superpowers/CONTEXT-1013-pki-and-customizations.md b/docs/superpowers/CONTEXT-1013-pki-and-customizations.md index a0d8908d17..949b41994a 100644 --- a/docs/superpowers/CONTEXT-1013-pki-and-customizations.md +++ b/docs/superpowers/CONTEXT-1013-pki-and-customizations.md @@ -80,8 +80,11 @@ fresh install out. Both are fully supported. variable — and behaviour reverts to today's. 3. **The `db`/Setup-Mode path is untested.** `efitools` was unavailable on the CentOS Stream 9 test box even with EPEL and CRB enabled, so - `fog-build-sb-authvars` never ran with the new `SECUREBOOT_MOK_CERT`. It - installs normally on Rocky 9 and Debian/Ubuntu — verify there. + `fog-build-sb-authvars` never ran with the new `SECUREBOOT_MOK_CERT`. + Availability on EL9 is inconsistent — the upstream RPM tracker lists Fedora + branches only, yet it is present on at least one Rocky 9 FOG server (source + unestablished). Verify on Debian/Ubuntu, or on an EL9 box where it is + already installed. 4. **nginx is untested.** All vhost work was verified on Apache only. The managed-block splice and the `netbootproto` redirect exclusion both have nginx branches that have never executed. From 07c8f5442ae92e5ec99caa66ea11b3c7ca2d955d Mon Sep 17 00:00:00 2001 From: JJ Fullmer Date: Fri, 7 Aug 2026 20:02:15 -0600 Subject: [PATCH 54/62] Verify the db/Setup-Mode path with efitools built from source Built the three efitools userspace tools on the test box and re-ran a fresh install. The auth-var path executes for the first time: PK.auth, KEK.auth and db.auth are produced, and db.auth embeds CN=FOG Secure Boot CA -- the intermediate -- beside Microsoft's CAs, with the signing leaf's CN absent. That closes the last open question about the Secure Boot restructure that could be answered without UEFI hardware. Leaf rotation is now known safe for Setup-Mode-enrolled clients as well as MokManager-enrolled ones, rather than assumed safe. Had db kept the signer, rotating a leaf would have stranded exactly the clients that enrolled unattended, while the MokManager path kept working -- a split that looks like success. Documents the build recipe, including that gnu-efi-devel is required even for the userspace tools because they include efi.h. The EFI binaries are not needed and are not built. Still open and still needing hardware: whether shim actually chains a leaf-signed kernel to an enrolled CA, and the firmware-side equivalent for db. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YE3jeTexYEcAxWFbZcZgpZ --- docs/PKI_ZONES.md | 21 +++++++++++++++++++ .../CONTEXT-1013-pki-and-customizations.md | 15 ++++++------- 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/docs/PKI_ZONES.md b/docs/PKI_ZONES.md index 62b94c832f..8825c82316 100644 --- a/docs/PKI_ZONES.md +++ b/docs/PKI_ZONES.md @@ -254,6 +254,27 @@ signed PK/KEK/db blobs. If you have it working on an EL9 box, check where it came from (`rpm -q --queryformat '%{VENDOR} %{URL}\n' efitools`) before assuming a fresh install will get it. +Only the three userspace tools are needed, and they build in about a minute: + +```bash +dnf -y install gcc make openssl-devel git gnu-efi-devel +git clone --depth 1 \ + https://git.kernel.org/pub/scm/linux/kernel/git/jejb/efitools.git +cd efitools +make cert-to-efi-sig-list sign-efi-sig-list efi-updatevar +install -m 0755 cert-to-efi-sig-list sign-efi-sig-list efi-updatevar /usr/bin/ +``` + +`gnu-efi-devel` is required even for the userspace tools — they include +`efi.h`. The EFI binaries (`KeyTool.efi` et al.) are not needed and are not +built here. + +**Verified with those tools present:** the installer builds `PK.auth`, +`KEK.auth` and `db.auth`, and `db.auth` embeds `CN=FOG Secure Boot CA` — the +**intermediate** — beside Microsoft's CAs, with the signing leaf's CN absent. +That is what makes leaf rotation safe for Setup-Mode-enrolled clients too, not +just MokManager-enrolled ones. + MOK enrolment via MokManager is unaffected either way; only the unattended Setup Mode path needs those tools. Where the package is genuinely absent it has to be built from source. diff --git a/docs/superpowers/CONTEXT-1013-pki-and-customizations.md b/docs/superpowers/CONTEXT-1013-pki-and-customizations.md index 949b41994a..c9f123d35e 100644 --- a/docs/superpowers/CONTEXT-1013-pki-and-customizations.md +++ b/docs/superpowers/CONTEXT-1013-pki-and-customizations.md @@ -78,13 +78,14 @@ fresh install out. Both are fully supported. built correctly and both certs are embedded, but no UEFI hardware has validated it. If it fails, point `secureBootMokCert` at the leaf — one variable — and behaviour reverts to today's. -3. **The `db`/Setup-Mode path is untested.** `efitools` was unavailable on the - CentOS Stream 9 test box even with EPEL and CRB enabled, so - `fog-build-sb-authvars` never ran with the new `SECUREBOOT_MOK_CERT`. - Availability on EL9 is inconsistent — the upstream RPM tracker lists Fedora - branches only, yet it is present on at least one Rocky 9 FOG server (source - unestablished). Verify on Debian/Ubuntu, or on an EL9 box where it is - already installed. +3. ~~The `db`/Setup-Mode path is untested.~~ **Now verified.** `efitools` was + built from source on the test box (see `docs/PKI_ZONES.md` for the recipe — + three userspace tools, needs `gnu-efi-devel` for `efi.h`). With it present + the installer builds `PK.auth`/`KEK.auth`/`db.auth`, and `db.auth` embeds + `CN=FOG Secure Boot CA` — the intermediate — with the signing leaf absent. + Note the package itself is unavailable on EL9 via EPEL/CRB and the upstream + RPM tracker lists Fedora branches only, so a fresh EL9 install will skip + this path unless efitools is supplied. 4. **nginx is untested.** All vhost work was verified on Apache only. The managed-block splice and the `netbootproto` redirect exclusion both have nginx branches that have never executed. From 7b5753f079517a5470e9dc365497f9ae0ccb1155 Mon Sep 17 00:00:00 2001 From: JJ Fullmer Date: Fri, 7 Aug 2026 20:09:22 -0600 Subject: [PATCH 55/62] Add --secureboot-ca-cert so the Secure Boot zone is importable like the others Web, Client and Root each had per-zone import flags; Secure Boot did not. An admin supplying their own could only give --secure-boot-key/--secure-boot-cert, which made the enrolled certificate the same as the signing one -- the flat model again, and precisely the coupling the intermediate exists to break. There was no way to say "enrol this CA, sign with this leaf issued from it". --secureboot-ca-cert names the certificate endpoints enrol. Paired with the existing leaf flags it expresses the drop-in an enterprise actually wants: mint a Secure Boot intermediate in AD CS or step-ca, enrol its public cert on the fleet, keep its private key off the FOG server entirely, and hand FOG only a short-lived code-signing leaf -- which can then be rotated without anyone touching firmware. Persisted in managedKeys so it survives upgrades and a rotated leaf keeps pointing at the same enrolled CA. Supplying only the leaf pair still enrols that certificate, exactly as before. All four zones are now independently replaceable, which was the point of separating them. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YE3jeTexYEcAxWFbZcZgpZ --- bin/installfog.sh | 16 ++++++++++++++-- lib/common/functions.sh | 5 +++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/bin/installfog.sh b/bin/installfog.sh index 248f48f53c..39cc0cc216 100755 --- a/bin/installfog.sh +++ b/bin/installfog.sh @@ -143,6 +143,11 @@ usage() { echo -e "\t --client-ca-cn\t\tCN fog-client expects on the pinned cert" echo -e "\t \t\t\tdefaults to 'FOG Server CA'" echo -e "\t --root-ca-cert/-key\tSupply the Root CA instead of generating one" + echo -e "\t --secureboot-ca-cert\tYour own SECURE BOOT intermediate: the" + echo -e "\t \t\t\tcertificate enrolled in firmware. Pair it with" + echo -e "\t \t\t\t--secure-boot-key/--secure-boot-cert, which name" + echo -e "\t \t\t\tthe code-signing leaf issued from it. Rotate the" + echo -e "\t \t\t\tleaf freely; the enrolled CA never changes" echo -e "\t --kernel-backup-count\tHow many prior kernel/init generations to" echo -e "\t \t\tkeep (default 3). Restore one with" echo -e "\t \t\tbin/restorekernel.sh. See" @@ -194,7 +199,7 @@ usage() { sextraServerNames=() shortopts="h?odEUHSCKYyXTFf:c:W:D:B:s:e:N:l" -longopts="help,uninstall,purge-db,purge-images,purge-snapins,purge-ssl,purge-user,purge-all,dry-run,force,mysqldbname:,ssl-path:,oldcopy,no-vhost,no-defaults,no-upgrade,no-htmldoc,force-https,no-force-https,recreate-keys,recreate-CA,recreate-Ca,recreate-cA,recreate-ca,external-ca,ca-cert:,ca-key:,ca-root:,autoaccept,file:,docroot:,webroot:,backuppath:,startrange:,endrange:,no-exportbuild,exitFail,no-tftpbuild,list-packages,fogprogramdir:,secure-boot-key:,secure-boot-cert:,no-secure-boot,hostname:,extra-server-name:,kernel-backup-count:,restore-kernel-backup,split-pki,legacy-pki,netboot-proto:,client-ca-cn:,web-ca-cert:,web-ca-key:,web-ca-root:,client-ca-cert:,client-ca-key:,client-ca-root:,root-ca-cert:,root-ca-key:" +longopts="help,uninstall,purge-db,purge-images,purge-snapins,purge-ssl,purge-user,purge-all,dry-run,force,mysqldbname:,ssl-path:,oldcopy,no-vhost,no-defaults,no-upgrade,no-htmldoc,force-https,no-force-https,recreate-keys,recreate-CA,recreate-Ca,recreate-cA,recreate-ca,external-ca,ca-cert:,ca-key:,ca-root:,autoaccept,file:,docroot:,webroot:,backuppath:,startrange:,endrange:,no-exportbuild,exitFail,no-tftpbuild,list-packages,fogprogramdir:,secure-boot-key:,secure-boot-cert:,no-secure-boot,hostname:,extra-server-name:,kernel-backup-count:,restore-kernel-backup,split-pki,legacy-pki,netboot-proto:,client-ca-cn:,web-ca-cert:,web-ca-key:,web-ca-root:,client-ca-cert:,client-ca-key:,client-ca-root:,root-ca-cert:,root-ca-key:,secureboot-ca-cert:" optargs=$(getopt -o $shortopts -l $longopts -n "$0" -- "$@") [[ $? -ne 0 ]] && usage @@ -516,7 +521,7 @@ while :; do ;; --web-ca-cert | --web-ca-key | --web-ca-root | \ --client-ca-cert | --client-ca-key | --client-ca-root | \ - --root-ca-cert | --root-ca-key) + --root-ca-cert | --root-ca-key | --secureboot-ca-cert) if [[ ! -f $2 ]]; then echo "$1 requires a readable file after" usage @@ -531,6 +536,12 @@ while :; do --client-ca-root) sclientExtCARoot="$2" ;; --root-ca-cert) srootExtCACert="$2" ;; --root-ca-key) srootExtCAKey="$2" ;; + # The Secure Boot zone's anchor: what gets ENROLLED in + # firmware. Pairs with --secure-boot-key/--secure-boot-cert, + # which name the leaf that actually signs. Supplying only the + # leaf pair (the historic form) still works and enrols that + # certificate, exactly as before. + --secureboot-ca-cert) ssecureBootMokCert="$2" ;; esac shift 2 ;; @@ -760,6 +771,7 @@ esac [[ -n $sclientExtCARoot ]] && clientExtCARoot=$sclientExtCARoot [[ -n $srootExtCACert ]] && rootExtCACert=$srootExtCACert [[ -n $srootExtCAKey ]] && rootExtCAKey=$srootExtCAKey +[[ -n $ssecureBootMokCert ]] && secureBootMokCert=$ssecureBootMokCert # Supplying any web-zone CA file implies --external-ca, the same way supplying # --ca-cert always has. Saves an admin from the "I gave you the files and # nothing happened" failure, which produces a working install with the wrong diff --git a/lib/common/functions.sh b/lib/common/functions.sh index 191deb6669..2c626c6d6a 100755 --- a/lib/common/functions.sh +++ b/lib/common/functions.sh @@ -3475,6 +3475,11 @@ writeUpdateFile() { # opt-out that reverted on the next upgrade would hand the admin back a # root-only key and a sudoers rule they had deliberately declined. secureBootKey secureBootCert secureboot + # The certificate endpoints ENROL, which is not always the one that + # signs. Persisted so an admin who supplied their own Secure Boot + # intermediate does not have to re-pass it on every later run -- and + # so a rotated signing leaf keeps pointing at the same enrolled CA. + secureBootMokCert # GH-964 sibling: what the admin chose for the local firewall # (configure/disable/skip). Persisted for the same reason the Secure # Boot keys are -- so an upgrade does not quietly undo a deliberate From 86e5f474fb89034c0ad06a2ddb4ac3290e93b629 Mon Sep 17 00:00:00 2001 From: JJ Fullmer Date: Fri, 7 Aug 2026 20:14:45 -0600 Subject: [PATCH 56/62] Do not regenerate the Root CA when its key has been taken offline createRootCA short-circuited on "key AND cert both present". Moving the root key to a vault -- the end state this design explicitly recommends -- leaves the cert and removes the key, so that test failed and the next ordinary update minted a brand new root, orphaning every intermediate beneath it and every client that pinned anything under them. Silently. The CERTIFICATE is what defines that a root exists. An absent key is not a missing root, it is an offline one. Issuing still needs the key, but only for a NEW intermediate -- existing ones short-circuit without touching it, which is what makes an offline root practical day to day. When one genuinely must be issued, _issueIntermediateCA now says which file to restore and that it can be removed again afterwards, rather than failing inside openssl with an unreadable-file error. Documents the manual procedure, including that .fogRootCA.pem stays put. The export helper is still Phase 3; this at least makes doing it by hand safe. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YE3jeTexYEcAxWFbZcZgpZ --- docs/PKI_ZONES.md | 39 +++++++++++++++++++++++++++++++++++++++ lib/common/functions.sh | 30 +++++++++++++++++++++++++++++- 2 files changed, 68 insertions(+), 1 deletion(-) diff --git a/docs/PKI_ZONES.md b/docs/PKI_ZONES.md index 8825c82316..c80823548f 100644 --- a/docs/PKI_ZONES.md +++ b/docs/PKI_ZONES.md @@ -105,6 +105,45 @@ already has certificate material stays on whatever layout it has, whatever a fresh install would choose, because changing it would strand every client that pinned the old CA. +## Taking the Root CA offline + +The root's private key is generated on the server and left there, `0600 +root:root`. That is a deliberate starting point, not the recommended end +state: requiring a vault on day one would make a first install harder for +everyone, including people who will never run a real offline root. + +**Moving it off is a manual step today** — there is no helper script yet. + +```bash +# copy it somewhere durable and offline, then remove it from the server +install -m 0600 /opt/fog/snapins/ssl/CA/root/.fogRootCA.key /mnt/vault/ +shred -u /opt/fog/snapins/ssl/CA/root/.fogRootCA.key +``` + +Leave `.fogRootCA.pem` in place. The **certificate** is what everything +chains to and what the installer uses to recognise that a root already +exists; only the key needs protecting. + +Day to day nothing needs it. The intermediates are already issued, and each +one short-circuits on every later run without the root key being touched. It +is required only to issue a **new** intermediate — which in practice means a +first install, or adding a zone you previously skipped. The installer detects +its absence and tells you exactly what to restore rather than failing +somewhere inside openssl: + +``` + * Cannot issue 'FOG Web CA': the Root CA private key is not on this server + * That is the correct state for an offline root, but issuing a new + intermediate needs it. Restore it to: + /opt/fog/snapins/ssl/CA/root/.fogRootCA.key + re-run the installer, then move it back to your vault. +``` + +> Removing the key does **not** cause the root to be regenerated. That is +> worth stating because the obvious implementation gets it wrong — testing +> for "key and cert both present" would mint a fresh root the first time +> anyone followed this advice, orphaning every intermediate beneath it. + ## Bringing your own CA Each zone is independently replaceable. Replace one, two, or none. diff --git a/lib/common/functions.sh b/lib/common/functions.sh index 2c626c6d6a..7a8c3ec644 100755 --- a/lib/common/functions.sh +++ b/lib/common/functions.sh @@ -3942,6 +3942,23 @@ _issueIntermediateCA() { mkdir -p "$outdir" >>$error_log 2>&1 || st=1 chmod 0700 "$outdir" >>$error_log 2>&1 [[ -f "${outdir}/${keyfile}" && -f "${outdir}/${certfile}" ]] && return 0 + # Issuing needs the root's private key. An existing intermediate returns + # above without ever touching it, which is what makes an offline root + # workable day to day -- but a NEW one cannot be signed without it. + # + # Say so instead of failing inside openssl with an unreadable-file error, + # because the fix is a specific and slightly unusual action: bring the key + # back, run the installer, take it away again. + if [[ ${rootCAKeyOffline:-0} -eq 1 ]]; then + echo "Failed" + echo " * Cannot issue '${cn}': the Root CA private key is not on this" + echo " server (only ${rootCAPem} is present)." + echo " * That is the correct state for an offline root, but issuing a new" + echo " intermediate needs it. Restore it to:" + echo " ${rootCAKey}" + echo " re-run the installer, then move it back to your vault." + return 1 + fi openssl genrsa -out "${outdir}/${keyfile}" 4096 >>$error_log 2>&1 || st=1 # Written as a config file rather than passed with -addext: -addext needs # OpenSSL 1.1.1+, and the older RHEL variants this installer still supports @@ -3994,7 +4011,18 @@ createRootCA() { errorStat $? return 0 fi - [[ -f $rootCAKey && -f $rootCAPem ]] && return 0 + # The CERTIFICATE is what defines "this root exists". The key may be + # legitimately absent -- that is what an offline root IS, and moving it to + # a vault is the end state this design recommends. + # + # Testing for both would regenerate the root the first time an admin + # actually took that advice, orphaning every intermediate already issued + # and every client that pinned anything beneath it. Silently, on an + # ordinary update. + if [[ -f $rootCAPem ]]; then + [[ -f $rootCAKey ]] || rootCAKeyOffline=1 + return 0 + fi dots "Creating FOG Server ROOT CA" mkdir -p "$rootdir" >>$error_log 2>&1 From af7e1560a4e8254811d5ad8a232c804f3527bd7d Mon Sep 17 00:00:00 2001 From: JJ Fullmer Date: Fri, 7 Aug 2026 20:41:08 -0600 Subject: [PATCH 57/62] Record Secure Boot verified on real UEFI hardware, both enrolment routes Machines boot FOG's leaf-signed kernels while trusting only the intermediate -- whether that intermediate is enrolled as MOK.der through MokManager, or written into db through the Setup Mode PK/KEK/db path. That settles the one question this restructure rested on and that no amount of local testing could answer: whether firmware and shim accept a chain terminating at the enrolled CA, or demand the exact signing certificate. Both accept the chain. A signing leaf can therefore be rotated, revoked, or issued per storage node with no firmware trip to any machine, which is the entire reason for enrolling the issuer instead of the signer. Also closes PXE boot by implication -- those machines netbooted to run the kernels they then validated. Remaining unverified: no real fog-client has authenticated against a split server, and nginx has never executed the vhost splice or the netbootproto redirect exclusion. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YE3jeTexYEcAxWFbZcZgpZ --- docs/PKI_ZONES.md | 16 ++++++++++------ .../CONTEXT-1013-pki-and-customizations.md | 15 ++++++++++----- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/docs/PKI_ZONES.md b/docs/PKI_ZONES.md index c80823548f..9de9d017d4 100644 --- a/docs/PKI_ZONES.md +++ b/docs/PKI_ZONES.md @@ -240,7 +240,7 @@ that FQDN or the generated boot URLs will not match the certificate. | Per-zone bring-your-own-CA flags | Implemented | | `netbootproto` separation | Implemented | | Split as the **default** for fresh installs | Implemented | -| Secure Boot intermediate | Implemented — **not verified on hardware** | +| Secure Boot intermediate | Implemented — verified on real UEFI hardware | ### Secure Boot @@ -268,11 +268,15 @@ $ sbverify --list bzImage issuer: /CN=FOG Server ROOT CA ``` -**Not verified:** that shim actually accepts a CA in MokList and chains a -leaf-signed kernel to it, and the same question for firmware validating `db`. -Both need real UEFI hardware. The mechanism is correct by construction, which -is not the same as observed booting. If it fails, the fix is one variable — -point `secureBootMokCert` at the leaf and the behaviour reverts to today's. +**Confirmed on real UEFI hardware, both enrolment routes:** machines boot +FOG's leaf-signed kernels while trusting only the **intermediate** — whether +that intermediate is enrolled as `MOK.der` through MokManager, or written into +`db` through the Setup Mode PK/KEK/db path. + +That is the whole design validated in the place it matters. Firmware and shim +both accept a certificate chain terminating at the enrolled CA rather than +demanding the exact signer, so a signing leaf can be rotated, revoked, or +issued per storage node and the fleet keeps booting with no firmware trip. **`efitools` is unreliable on EL9 and should not be assumed present.** It is a declared dependency and installs normally on Debian/Ubuntu. On EL9 the picture diff --git a/docs/superpowers/CONTEXT-1013-pki-and-customizations.md b/docs/superpowers/CONTEXT-1013-pki-and-customizations.md index c9f123d35e..bf5420df14 100644 --- a/docs/superpowers/CONTEXT-1013-pki-and-customizations.md +++ b/docs/superpowers/CONTEXT-1013-pki-and-customizations.md @@ -74,10 +74,13 @@ fresh install out. Both are fully supported. fetched, so no client change is expected. Expected is not observed. One host checking in settles it. If the client instead derives its key from `ca.cert.der`, the fix is to let the Client CA double as the comm keypair. -2. **Shim has never been asked to boot a leaf-signed kernel.** The chain is - built correctly and both certs are embedded, but no UEFI hardware has - validated it. If it fails, point `secureBootMokCert` at the leaf — one - variable — and behaviour reverts to today's. +2. ~~Shim has never been asked to boot a leaf-signed kernel.~~ **Confirmed on + real UEFI hardware, both enrolment routes.** Machines boot FOG's + leaf-signed kernels while trusting only the **intermediate** — enrolled as + `MOK.der` through MokManager, or written into `db` through the Setup Mode + PK/KEK/db path. Firmware and shim both accept a chain terminating at the + enrolled CA rather than demanding the exact signer, so the rotation premise + holds in practice and not merely by construction. 3. ~~The `db`/Setup-Mode path is untested.~~ **Now verified.** `efitools` was built from source on the test box (see `docs/PKI_ZONES.md` for the recipe — three userspace tools, needs `gnu-efi-devel` for `efi.h`). With it present @@ -89,7 +92,9 @@ fresh install out. Both are fully supported. 4. **nginx is untested.** All vhost work was verified on Apache only. The managed-block splice and the `netbootproto` redirect exclusion both have nginx branches that have never executed. -5. **PXE boot untested**, and not testable from a shell. +5. **PXE boot** was not testable from this shell, but the Secure Boot + hardware verification in (2) necessarily exercised it — those machines + netbooted and ran FOG's signed kernels. ## Timing note From 1688ecd083be7a4c611d6bffa43865b031af41c0 Mon Sep 17 00:00:00 2001 From: JJ Fullmer Date: Fri, 7 Aug 2026 20:47:23 -0600 Subject: [PATCH 58/62] Record fog-client confirmed on a split server, and the root-trust follow-up fog-client authenticates correctly against a split-mode server. It fetches the comm certificate from the path it always has, so the separation needed no client-side change -- which was the hoped-for outcome and the last significant unknown in this branch. One thing it does is worth acting on, in zazzles rather than here: at install it adds FOG Server CA -- the Client Communication INTERMEDIATE -- to the Windows Root store instead of FOG Server ROOT CA. Nothing is broken, but it is the wrong anchor and it costs two things. Rotation, first: trusting an intermediate as an anchor means replacing that intermediate requires re-pushing trust to every client, which is precisely the cost the Secure Boot zone just eliminated by enrolling the issuer. Trusting the root would give the Client zone the same freedom. HTTPS by default, second: the client trusts only the Client zone's intermediate, and that intermediate does not sign the web certificate -- the Web CA does. So the web certificate is untrusted and HTTPS cannot be turned on by default. If the client trusted the root, every zone beneath it would validate and an all-FOG-PKI install could enable HTTPS out of the box. That is a real payoff and it falls out of the zone structure rather than needing anything new. Nothing in this repo changes for it: the root is already published in the chain, and ca.cert.der keeps carrying the intermediate for the existing pinning behaviour. Also drops a stale paragraph still describing Secure Boot as unimplemented and unverified. nginx is now the only untested item on this branch. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YE3jeTexYEcAxWFbZcZgpZ --- docs/PKI_ZONES.md | 45 +++++++++++++------ .../CONTEXT-1013-pki-and-customizations.md | 21 ++++++--- 2 files changed, 47 insertions(+), 19 deletions(-) diff --git a/docs/PKI_ZONES.md b/docs/PKI_ZONES.md index 9de9d017d4..bde5abad5a 100644 --- a/docs/PKI_ZONES.md +++ b/docs/PKI_ZONES.md @@ -335,18 +335,35 @@ verify, `ca.cert.der` publishes the Client CA while the vhost serves the web leaf, and the key `certDecrypt()` opens is provably a different keypair from the web server's — which is the entire point. -Two things remain unverified. Neither is reached by what ships today: - -1. **How fog-client obtains the server's encryption certificate.** FOG - publishes the comm certificate at `srvpublic.crt`, the path the client has - always fetched, so no client change should be needed — but no real client - has been observed authenticating against a split-mode server. If it turns - out to derive a key from `ca.cert.der` instead, the Client CA doubles as - the comm keypair and the published files change. -2. **Whether shim accepts a CA in MokList** with the signing chain attached - via `sbsign --addcert`. This only matters for the Secure Boot intermediate, - which is **not implemented** — Secure Boot still uses its existing - self-signed key, unchanged. Needs real UEFI hardware to answer. - -An existing server is never switched automatically, so neither question can +**fog-client is confirmed working against a split server.** It fetches the +comm certificate from the path it always has, so the split needed no client +change. + +### Known follow-up: the client trusts the intermediate, not the root + +During installation fog-client adds **`FOG Server CA` — the Client +Communication intermediate — to the Windows Root store**, rather than the +actual `FOG Server ROOT CA`. It works, and nothing is broken. But it is the +wrong anchor, and it costs two things: + +- **Rotation.** Trusting the intermediate as an anchor means replacing that + intermediate requires re-pushing trust to every client — the exact cost the + Secure Boot zone just eliminated by enrolling the issuer. Trusting the root + would let the Client CA be rotated freely. +- **HTTPS by default.** The client trusts only the Client zone's intermediate, + which does not sign the web certificate — the **Web CA** does. So the web + certificate is not trusted and HTTPS cannot be enabled by default. If the + client trusted the **root**, every zone beneath it would validate, and an + all-FOG-PKI install could turn HTTPS on out of the box. + +That change lives in the `zazzles`/fog-client repository, not here. Nothing in +this repo needs to change to accommodate it: the root certificate is already +published in the chain, and `ca.cert.der` continues to carry the intermediate +for the existing pinning behaviour. + +One thing remains unverified here: **nginx**. Every vhost change was exercised +on Apache only, and both the managed-block splice and the `netbootproto` +redirect exclusion have nginx branches that have never executed. + +An existing server is never switched automatically, so nothing above can affect a server that is already running. diff --git a/docs/superpowers/CONTEXT-1013-pki-and-customizations.md b/docs/superpowers/CONTEXT-1013-pki-and-customizations.md index bf5420df14..42aa1bb390 100644 --- a/docs/superpowers/CONTEXT-1013-pki-and-customizations.md +++ b/docs/superpowers/CONTEXT-1013-pki-and-customizations.md @@ -69,11 +69,22 @@ fresh install out. Both are fully supported. ## NOT verified — read this before shipping -1. **No real fog-client has authenticated against a split server.** The comm - certificate is published at `srvpublic.crt`, the path the client has always - fetched, so no client change is expected. Expected is not observed. One - host checking in settles it. If the client instead derives its key from - `ca.cert.der`, the fix is to let the Client CA double as the comm keypair. +1. ~~No real fog-client has authenticated against a split server.~~ + **Confirmed working.** It fetches the comm certificate from the path it + always has, so the split needed no client change. + + **Follow-up, in `zazzles` not here:** the client installs `FOG Server CA` + — the Client Communication *intermediate* — into the Windows Root store, + rather than `FOG Server ROOT CA`. It works, but it is the wrong anchor and + costs two things. Rotation: trusting an intermediate as an anchor means + replacing it requires re-pushing trust to every client, the exact cost the + Secure Boot zone just removed. And HTTPS-by-default: the client trusts only + the Client zone, which does not sign the web certificate (the Web CA does), + so the web certificate is untrusted. Trusting the **root** would validate + every zone beneath it and let an all-FOG-PKI install enable HTTPS out of + the box. Nothing in this repo needs to change for it — the root is already + in the chain and `ca.cert.der` keeps carrying the intermediate for existing + pinning. 2. ~~Shim has never been asked to boot a leaf-signed kernel.~~ **Confirmed on real UEFI hardware, both enrolment routes.** Machines boot FOG's leaf-signed kernels while trusting only the **intermediate** — enrolled as From b43e68dc008715dc26b7facccb11cebee1b4a0dc Mon Sep 17 00:00:00 2001 From: JJ Fullmer Date: Fri, 7 Aug 2026 20:51:06 -0600 Subject: [PATCH 59/62] Confirm HTTPS works once the root CA is trusted Adding FOG Server ROOT CA to the Windows trust store by hand makes HTTPS to the FOG web UI validate. That moves the root-trust follow-up from theory to proven mechanics: the only thing standing between here and HTTPS-on-by-default for an all-FOG-PKI install is which certificate fog-client installs at setup. It also demonstrates the zone structure doing what it was built for -- one trust anchor at the root validating every zone beneath it, rather than a separate arrangement per consumer. Still nothing to change in this repo: the root is already published in the chain, and ca.cert.der continues to carry the intermediate so existing client pinning is unaffected. The change is a zazzles one and can land independently. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YE3jeTexYEcAxWFbZcZgpZ --- docs/PKI_ZONES.md | 9 ++++++--- docs/superpowers/CONTEXT-1013-pki-and-customizations.md | 8 +++++--- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/docs/PKI_ZONES.md b/docs/PKI_ZONES.md index bde5abad5a..bc42c70b70 100644 --- a/docs/PKI_ZONES.md +++ b/docs/PKI_ZONES.md @@ -352,9 +352,12 @@ wrong anchor, and it costs two things: would let the Client CA be rotated freely. - **HTTPS by default.** The client trusts only the Client zone's intermediate, which does not sign the web certificate — the **Web CA** does. So the web - certificate is not trusted and HTTPS cannot be enabled by default. If the - client trusted the **root**, every zone beneath it would validate, and an - all-FOG-PKI install could turn HTTPS on out of the box. + certificate is not trusted and HTTPS cannot be enabled by default. + + **Confirmed:** adding `FOG Server ROOT CA` to the Windows trust store by + hand makes HTTPS work. So this is not a theory — the mechanics are proven, + and all that stands between here and HTTPS-on-by-default for an + all-FOG-PKI install is which certificate fog-client installs at setup. That change lives in the `zazzles`/fog-client repository, not here. Nothing in this repo needs to change to accommodate it: the root certificate is already diff --git a/docs/superpowers/CONTEXT-1013-pki-and-customizations.md b/docs/superpowers/CONTEXT-1013-pki-and-customizations.md index 42aa1bb390..34d1651ae6 100644 --- a/docs/superpowers/CONTEXT-1013-pki-and-customizations.md +++ b/docs/superpowers/CONTEXT-1013-pki-and-customizations.md @@ -82,9 +82,11 @@ fresh install out. Both are fully supported. the Client zone, which does not sign the web certificate (the Web CA does), so the web certificate is untrusted. Trusting the **root** would validate every zone beneath it and let an all-FOG-PKI install enable HTTPS out of - the box. Nothing in this repo needs to change for it — the root is already - in the chain and `ca.cert.der` keeps carrying the intermediate for existing - pinning. + the box — **confirmed by adding the root to the Windows trust store by + hand, after which HTTPS works.** The mechanics are proven; only which + certificate fog-client installs at setup stands in the way. Nothing in this + repo needs to change for it — the root is already in the chain and + `ca.cert.der` keeps carrying the intermediate for existing pinning. 2. ~~Shim has never been asked to boot a leaf-signed kernel.~~ **Confirmed on real UEFI hardware, both enrolment routes.** Machines boot FOG's leaf-signed kernels while trusting only the **intermediate** — enrolled as From 55df7e90ef4f9be5ca14ced5ee0ec6d035683fb4 Mon Sep 17 00:00:00 2001 From: JJ Fullmer Date: Fri, 7 Aug 2026 20:52:07 -0600 Subject: [PATCH 60/62] Note two reported dropdown issues in the continuation context Recorded, not acted on, at the reporter's request. The kernel/init dropdown still lists non-kernels -- memdisk, grub.exe and memtest.bin appear as Host/Group Kernel choices. The current filter excludes by extension (.php, images, .conf, .efi, .unsigned) and calls everything else a kernel, which was a narrowing of a worse bug rather than a correct rule. The better shape is the one backupPreservedCustomizations() already uses: subtract what FOG ships in packages/web/service/ipxe, which drops all three automatically while keeping custom kernels of any name. FOG_MEMTEST_KERNEL needs its own list, since memtest.bin is a legitimate value there. And a dropdown is still wanted where the default kernel/init is selected. It was applied to the FOG_TFTP_PXE_KERNEL settings on the Configuration page, so the field meant needs identifying before anything changes -- possibly a different surface, and note there is no global default *init* setting at all today. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YE3jeTexYEcAxWFbZcZgpZ --- .../CONTEXT-1013-pki-and-customizations.md | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/docs/superpowers/CONTEXT-1013-pki-and-customizations.md b/docs/superpowers/CONTEXT-1013-pki-and-customizations.md index 34d1651ae6..95597bb28c 100644 --- a/docs/superpowers/CONTEXT-1013-pki-and-customizations.md +++ b/docs/superpowers/CONTEXT-1013-pki-and-customizations.md @@ -154,6 +154,38 @@ sbverify --list /var/www/html/fog/service/ipxe/bzImage Everything under `$sslpath` is a **dotfile** — `ls -a` or it looks empty. +## Known bugs / unfinished, reported but NOT yet acted on + +**Kernel/init dropdown still lists non-kernels.** `FOGPage::kernelFileList()` +(`packages/web/lib/fog/fogpage.class.php`) filters by excluding `.php`, image +extensions, `.conf`, `.efi` and `.unsigned`, then treats everything remaining +that is not init-shaped as a kernel. That still leaves `memdisk`, `grub.exe` +and `memtest.bin` in the Host/Group Kernel dropdown, where they are not +bootable choices. + +The exclusion-by-extension approach is the wrong shape — it was a quick +narrowing of an even broader bug (the list originally included `boot.php` and +`bg.png`). Better: subtract what FOG ships, the same rule +`backupPreservedCustomizations()` already uses successfully — read +`packages/web/service/ipxe` from the source tree and treat anything present +there as not-a-kernel. `memdisk`/`memtest.bin`/`grub.exe` are all shipped, so +they would fall out automatically, while a custom kernel of any name survives. + +Caveat: `FOG_MEMTEST_KERNEL` legitimately wants `memtest.bin`, so that one +setting needs its own list rather than the general kernel list. + +**Dropdown wanted for the default kernel/init too.** It was applied to +`FOG_TFTP_PXE_KERNEL`/`_32`/`_ARM`/`FOG_MEMTEST_KERNEL` on the FOG +Configuration page, but the reported experience is that a dropdown is still +missing where the default is selected — verify which field was meant (the +Configuration page settings, or somewhere else such as a per-image or global +default surfaced elsewhere in the UI) before changing anything. There is no +global default *init* setting today; inits are per-host/group only, which may +be part of the gap. + +Both were reported after the branch was pushed. Nothing has been changed for +either. + ## Still open from the plan - Phase 1 Task 1.7: interactive PKI prompt in `lib/common/newinput.sh`. From 6ffaba3b9436286d6576e2cd8e3f67701d3c037f Mon Sep 17 00:00:00 2001 From: Tom Elliott Date: Mon, 10 Aug 2026 06:20:47 -0500 Subject: [PATCH 61/62] Scope the nginx HTTPS redirect to a location so the iPXE exclusion works The netboot exclusion added for the split-PKI HTTPS mode emitted the iPXE location block first and then a server-level `return 308`, on the reasoning that "emitted before the catch-all so it wins". That is Apache's model, not nginx's: a server-level return runs in the server rewrite phase, which is before location selection, so it fired for every request and the location block was unreachable. Net effect was the exact failure the exclusion exists to prevent -- iPXE fetching boot.php over HTTP got 308'd to an HTTPS it cannot validate, the shape of GH-978. Measured against real nginx, not reasoned about. Before: GET /fog/service/ipxe/boot.php -> 308 https://... . After: the same request is handled (200), while /fog/management/index.php and / still 308. `^~` on the iPXE prefix outranks `/`, which is what lets the exclusion win now that location matching actually runs. Apache's branch is unaffected -- RewriteCond really does guard the next RewriteRule -- which is why an Apache-only verification pass missed this. Co-Authored-By: Claude Opus 5 --- lib/common/functions.sh | 18 ++++++++++++++++-- .../de_DE.UTF-8/LC_MESSAGES/messages.po | 12 ++++++++++++ .../en_US.UTF-8/LC_MESSAGES/messages.po | 12 ++++++++++++ .../es_ES.UTF-8/LC_MESSAGES/messages.po | 12 ++++++++++++ .../eu_ES.UTF-8/LC_MESSAGES/messages.po | 12 ++++++++++++ .../fr_FR.UTF-8/LC_MESSAGES/messages.po | 12 ++++++++++++ .../it_IT.UTF-8/LC_MESSAGES/messages.po | 12 ++++++++++++ .../ja_JP.UTF-8/LC_MESSAGES/messages.po | 12 ++++++++++++ packages/web/management/languages/messages.pot | 9 +++++++++ .../pt_BR.UTF-8/LC_MESSAGES/messages.po | 12 ++++++++++++ .../zh_CN.UTF-8/LC_MESSAGES/messages.po | 12 ++++++++++++ 11 files changed, 133 insertions(+), 2 deletions(-) diff --git a/lib/common/functions.sh b/lib/common/functions.sh index 7a8c3ec644..2bed2eb850 100755 --- a/lib/common/functions.sh +++ b/lib/common/functions.sh @@ -4527,7 +4527,6 @@ EOF # publicly chainable, so the redirect must NOT catch # iPXE's own fetches -- otherwise it lands right back # on the HTTPS it cannot validate and boot fails. - # Emitted before the catch-all so it wins. if [[ $netbootproto != "$httpproto" ]]; then echo " location ^~ ${webroot}service/ipxe/ {" >> "$etcconf" echo " root ${docroot};" >> "$etcconf" @@ -4536,7 +4535,22 @@ EOF echo " include ${phploc};" >> "$etcconf" echo " }" >> "$etcconf" fi - echo " return 308 https://\$host\$request_uri;" >> "$etcconf" + # The redirect is a `location`, NOT a server-level + # `return`. nginx runs a server-level return in the + # server rewrite phase, which is BEFORE location + # selection -- so emitting the ipxe location first buys + # nothing, the return fires for every request and the + # exclusion above is dead code. Measured against real + # nginx: server-level return 308'd + # /fog/service/ipxe/boot.php; as `location /` the same + # request serves 200 and everything else still 308s. + # `^~` on the ipxe prefix beats `/`, which is what makes + # the exclusion win. Apache's branch below has no such + # problem -- RewriteCond really does guard the next + # RewriteRule. + echo " location / {" >> "$etcconf" + echo " return 308 https://\$host\$request_uri;" >> "$etcconf" + echo " }" >> "$etcconf" echo "}" >> "$etcconf" echo "Continued (See Below)" # Creates the diffie helman param file. diff --git a/packages/web/management/languages/de_DE.UTF-8/LC_MESSAGES/messages.po b/packages/web/management/languages/de_DE.UTF-8/LC_MESSAGES/messages.po index bded2058e6..ff7e4d8433 100644 --- a/packages/web/management/languages/de_DE.UTF-8/LC_MESSAGES/messages.po +++ b/packages/web/management/languages/de_DE.UTF-8/LC_MESSAGES/messages.po @@ -7671,6 +7671,14 @@ msgid "" "this group." msgstr "" +#, fuzzy +msgid "Use the default init" +msgstr "Standarddrucker aktualisieren" + +#, fuzzy +msgid "Use the default kernel" +msgstr "Standarddrucker aktualisieren" + msgid "Use the following link to go to the client page." msgstr "Der folgende Link führt zur Clientseite" @@ -8594,6 +8602,10 @@ msgstr "Knoten enthalten dieses Image" msgid "not built yet" msgstr "" +#, fuzzy +msgid "not found on disk" +msgstr "auf diesem Knoten nicht gefunden" + #, fuzzy msgid "not found on this node" msgstr "auf diesem Knoten nicht gefunden" diff --git a/packages/web/management/languages/en_US.UTF-8/LC_MESSAGES/messages.po b/packages/web/management/languages/en_US.UTF-8/LC_MESSAGES/messages.po index 82f06b0c17..700e230d5e 100644 --- a/packages/web/management/languages/en_US.UTF-8/LC_MESSAGES/messages.po +++ b/packages/web/management/languages/en_US.UTF-8/LC_MESSAGES/messages.po @@ -7665,6 +7665,14 @@ msgid "" "this group." msgstr "" +#, fuzzy +msgid "Use the default init" +msgstr "Update Printer" + +#, fuzzy +msgid "Use the default kernel" +msgstr "Update Printer" + msgid "Use the following link to go to the client page." msgstr "" @@ -8584,6 +8592,10 @@ msgstr "Could not find any nodes containing this image" msgid "not built yet" msgstr "" +#, fuzzy +msgid "not found on disk" +msgstr "Image not found on node" + #, fuzzy msgid "not found on this node" msgstr "Image not found on node" diff --git a/packages/web/management/languages/es_ES.UTF-8/LC_MESSAGES/messages.po b/packages/web/management/languages/es_ES.UTF-8/LC_MESSAGES/messages.po index 8a139ea1de..cf7ac6540f 100644 --- a/packages/web/management/languages/es_ES.UTF-8/LC_MESSAGES/messages.po +++ b/packages/web/management/languages/es_ES.UTF-8/LC_MESSAGES/messages.po @@ -7817,6 +7817,14 @@ msgid "" "this group." msgstr "" +#, fuzzy +msgid "Use the default init" +msgstr "Actualizar impresora" + +#, fuzzy +msgid "Use the default kernel" +msgstr "Actualizar impresora" + msgid "Use the following link to go to the client page." msgstr "" @@ -8739,6 +8747,10 @@ msgstr "No se pudo encontrar ningún nodos que contienen esta imagen" msgid "not built yet" msgstr "" +#, fuzzy +msgid "not found on disk" +msgstr "No se encontró Clase FOGPage para este nodo" + #, fuzzy msgid "not found on this node" msgstr "No se encontró Clase FOGPage para este nodo" diff --git a/packages/web/management/languages/eu_ES.UTF-8/LC_MESSAGES/messages.po b/packages/web/management/languages/eu_ES.UTF-8/LC_MESSAGES/messages.po index b6e12afe6f..cf7e53c4f6 100644 --- a/packages/web/management/languages/eu_ES.UTF-8/LC_MESSAGES/messages.po +++ b/packages/web/management/languages/eu_ES.UTF-8/LC_MESSAGES/messages.po @@ -7672,6 +7672,14 @@ msgid "" "this group." msgstr "" +#, fuzzy +msgid "Use the default init" +msgstr "Standarddrucker aktualisieren" + +#, fuzzy +msgid "Use the default kernel" +msgstr "Standarddrucker aktualisieren" + msgid "Use the following link to go to the client page." msgstr "Der folgende Link führt zur Clientseite" @@ -8595,6 +8603,10 @@ msgstr "Knoten enthalten dieses Image" msgid "not built yet" msgstr "" +#, fuzzy +msgid "not found on disk" +msgstr "auf diesem Knoten nicht gefunden" + #, fuzzy msgid "not found on this node" msgstr "auf diesem Knoten nicht gefunden" diff --git a/packages/web/management/languages/fr_FR.UTF-8/LC_MESSAGES/messages.po b/packages/web/management/languages/fr_FR.UTF-8/LC_MESSAGES/messages.po index 2ab6030de7..bc50360203 100644 --- a/packages/web/management/languages/fr_FR.UTF-8/LC_MESSAGES/messages.po +++ b/packages/web/management/languages/fr_FR.UTF-8/LC_MESSAGES/messages.po @@ -7673,6 +7673,14 @@ msgid "" "this group." msgstr "" +#, fuzzy +msgid "Use the default init" +msgstr "Mise à jour de l'imprimante" + +#, fuzzy +msgid "Use the default kernel" +msgstr "Mise à jour de l'imprimante" + msgid "Use the following link to go to the client page." msgstr "" @@ -8592,6 +8600,10 @@ msgstr "Impossible de trouver des noeuds contenant cette image" msgid "not built yet" msgstr "" +#, fuzzy +msgid "not found on disk" +msgstr "Image not found sur le noeud" + #, fuzzy msgid "not found on this node" msgstr "Image not found sur le noeud" diff --git a/packages/web/management/languages/it_IT.UTF-8/LC_MESSAGES/messages.po b/packages/web/management/languages/it_IT.UTF-8/LC_MESSAGES/messages.po index 432f4e927e..78507ca388 100644 --- a/packages/web/management/languages/it_IT.UTF-8/LC_MESSAGES/messages.po +++ b/packages/web/management/languages/it_IT.UTF-8/LC_MESSAGES/messages.po @@ -7362,6 +7362,14 @@ msgid "" "this group." msgstr "" +#, fuzzy +msgid "Use the default init" +msgstr "Aggiorna la stampante predefinita" + +#, fuzzy +msgid "Use the default kernel" +msgstr "Aggiorna la stampante predefinita" + msgid "Use the following link to go to the client page." msgstr "Utilizzare il seguente collegamento per accedere alla pagina client." @@ -8228,6 +8236,10 @@ msgstr "nodi che contengono questa immagine" msgid "not built yet" msgstr "" +#, fuzzy +msgid "not found on disk" +msgstr "Non trovato su questo nodo" + msgid "not found on this node" msgstr "Non trovato su questo nodo" diff --git a/packages/web/management/languages/ja_JP.UTF-8/LC_MESSAGES/messages.po b/packages/web/management/languages/ja_JP.UTF-8/LC_MESSAGES/messages.po index 15c65131f1..124b92fc59 100644 --- a/packages/web/management/languages/ja_JP.UTF-8/LC_MESSAGES/messages.po +++ b/packages/web/management/languages/ja_JP.UTF-8/LC_MESSAGES/messages.po @@ -7293,6 +7293,14 @@ msgid "" "this group." msgstr "" +#, fuzzy +msgid "Use the default init" +msgstr "既定で有効" + +#, fuzzy +msgid "Use the default kernel" +msgstr "既定で有効" + msgid "Use the following link to go to the client page." msgstr "クライアントページへ移動するには、以下のリンクを使用してください。" @@ -8149,6 +8157,10 @@ msgstr "このイメージを含むノード" msgid "not built yet" msgstr "" +#, fuzzy +msgid "not found on disk" +msgstr "このノード上に見つかりません" + msgid "not found on this node" msgstr "このノード上に見つかりません" diff --git a/packages/web/management/languages/messages.pot b/packages/web/management/languages/messages.pot index 84d0bf538a..b7452284a9 100644 --- a/packages/web/management/languages/messages.pot +++ b/packages/web/management/languages/messages.pot @@ -6492,6 +6492,12 @@ msgid "" "this group." msgstr "" +msgid "Use the default init" +msgstr "" + +msgid "Use the default kernel" +msgstr "" + msgid "Use the following link to go to the client page." msgstr "" @@ -7276,6 +7282,9 @@ msgstr "" msgid "not built yet" msgstr "" +msgid "not found on disk" +msgstr "" + msgid "not found on this node" msgstr "" diff --git a/packages/web/management/languages/pt_BR.UTF-8/LC_MESSAGES/messages.po b/packages/web/management/languages/pt_BR.UTF-8/LC_MESSAGES/messages.po index 8b9be22590..dc0a9936c8 100644 --- a/packages/web/management/languages/pt_BR.UTF-8/LC_MESSAGES/messages.po +++ b/packages/web/management/languages/pt_BR.UTF-8/LC_MESSAGES/messages.po @@ -7669,6 +7669,14 @@ msgid "" "this group." msgstr "" +#, fuzzy +msgid "Use the default init" +msgstr "Atualizar impressora" + +#, fuzzy +msgid "Use the default kernel" +msgstr "Atualizar impressora" + msgid "Use the following link to go to the client page." msgstr "" @@ -8588,6 +8596,10 @@ msgstr "Não foi possível encontrar todos os nós que contêm esta imagem" msgid "not built yet" msgstr "" +#, fuzzy +msgid "not found on disk" +msgstr "Imagem não encontrada no nó" + #, fuzzy msgid "not found on this node" msgstr "Imagem não encontrada no nó" diff --git a/packages/web/management/languages/zh_CN.UTF-8/LC_MESSAGES/messages.po b/packages/web/management/languages/zh_CN.UTF-8/LC_MESSAGES/messages.po index 0fa1abdb2e..96c04abfa8 100644 --- a/packages/web/management/languages/zh_CN.UTF-8/LC_MESSAGES/messages.po +++ b/packages/web/management/languages/zh_CN.UTF-8/LC_MESSAGES/messages.po @@ -7658,6 +7658,14 @@ msgid "" "this group." msgstr "" +#, fuzzy +msgid "Use the default init" +msgstr "更新打印机" + +#, fuzzy +msgid "Use the default kernel" +msgstr "更新打印机" + msgid "Use the following link to go to the client page." msgstr "" @@ -8575,6 +8583,10 @@ msgstr "找不到包含该图像的任何节点" msgid "not built yet" msgstr "" +#, fuzzy +msgid "not found on disk" +msgstr "图片没有节点发现" + #, fuzzy msgid "not found on this node" msgstr "图片没有节点发现" From 48f28b648390515ad35d5a6e33275028afb1a4ca Mon Sep 17 00:00:00 2001 From: Tom Elliott Date: Mon, 10 Aug 2026 07:46:46 -0500 Subject: [PATCH 62/62] Match the FOG-managed markers the same way grep does, so CRLF vhosts splice spliceManagedBlock decided "this file has markers" with grep -F -- a substring test, so a trailing CR or a stray space after a marker still matched -- then acted on that decision with awk comparing whole lines. Two matchers, two different ideas of what counts as the marker line. A vhost saved with CRLF endings passed the grep, matched nothing in the awk, and fell through every rule: the file was copied byte-for-byte, the freshly generated vhost was silently discarded, and the function returned 0, so the installer reported success. SUPPORTED_CUSTOMIZATIONS.md invites admins into this exact file, so one edit from a Windows box or one backup restored through one was enough to make every later install and update quietly stop updating the vhost -- stranding whatever the managed block carries, the maintenance/ deny rules included. The comparison is now normalized for trailing CR/whitespace while $0 is what still gets printed, so line endings elsewhere in the admin's file are untouched. Verified against CRLF, trailing-whitespace and clean markers, plus the no-file, prior-with-markers and prior-without-markers paths: all six splice correctly and preserve admin content above and below the block. Also guards the begin rule with !skip. From the partial-marker state this function is documented to tolerate ("a previous run died mid-write"), a second BEGIN seen while already skipping re-fired the rule and emitted a duplicate of the entire generated block, which then sat in the vhost permanently. It now collapses back to one clean block on the next run -- verified stable over four consecutive runs. Co-Authored-By: Claude Opus 5 --- lib/common/functions.sh | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/lib/common/functions.sh b/lib/common/functions.sh index 2bed2eb850..532f99b969 100755 --- a/lib/common/functions.sh +++ b/lib/common/functions.sh @@ -4181,10 +4181,33 @@ spliceManagedBlock() { fi if grep -qF "$FOG_MANAGED_BEGIN" "$conffile" && grep -qF "$FOG_MANAGED_END" "$conffile"; then local tmp="${conffile}.fogsplice.$$" + # The marker test above is grep -F (substring, so a trailing CR or a + # stray space still matches) but the awk below compared whole lines -- + # two matchers with different ideas of "this line is the marker". A + # vhost saved with CRLF endings, or with whitespace after a marker, + # therefore passed the grep, matched nothing in awk, and fell straight + # through: file copied byte-for-byte, the freshly generated vhost + # silently discarded, return 0, installer reports success. Admins are + # invited into this file by SUPPORTED_CUSTOMIZATIONS.md, so one edit + # from a Windows box was enough to make every later install or update + # quietly stop updating the vhost -- stranding whatever the managed + # block carries, including the maintenance/ deny rules. + # + # $0 is still what gets PRINTED, so the admin's own line endings + # elsewhere in the file are preserved untouched; only the comparison + # is normalized. + # + # The !skip guard on the begin rule matters for the partial-marker + # state this function is documented to tolerate: without it, a second + # BEGIN encountered while already skipping fired the rule again and + # emitted a duplicate copy of the whole generated block, which then + # persisted in the vhost forever. With it, that state collapses back + # to a single clean block on the next run. awk -v b="$FOG_MANAGED_BEGIN" -v e="$FOG_MANAGED_END" -v cf="$contentfile" ' - $0 == b { print; while ((getline line < cf) > 0) print line; close(cf); skip=1; next } - $0 == e { print; skip=0; next } - !skip { print } + { k = $0; sub(/[ \t\r]+$/, "", k) } + k == b && !skip { print; while ((getline line < cf) > 0) print line; close(cf); skip=1; next } + k == e { print; skip=0; next } + !skip { print } ' "$conffile" > "$tmp" && mv -f "$tmp" "$conffile" return $? fi