Capstone: a certificate-expiry and rotation runner

#!/usr/bin/env bash
# certrun.sh plan|apply DESTINATION: the entry point the systemd unit runs as the service account.
# It owns the process: one run per destination, a deadline for the whole run, signals, cleanup,
# the run-level metrics, and a gate that refuses a core that is not the signed release.
# Exit status: the core's contract passed through (0 ok, 1 a person must look at an endpoint or
# at the metrics, 70 internal, 75 temporary, 78 config), plus 2 usage, 75 lock held or deadline
# reached, 77 core refused, 78 no settings or a bad CERTRUN_BUDGET, 70 core killed. Stopped by
# SIGTERM, it cleans up and dies of SIGTERM (143 in a shell).
# CERTRUN_BUDGET: seconds for the whole run (default 60). CERTRUN_VAR: state root (for tests).
set -Eeuo pipefail
shopt -s inherit_errexit
 
here=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P) # /opt/scr-capstone: root's
release=$here/release
wheel=$release/certrun-0.1.0-py3-none-any.whl
var=${CERTRUN_VAR:-/var/lib/scr-capstone} # the service account's: state/ and textfile/
budget=${CERTRUN_BUDGET:-60}
mode='' dest='' run='' child='' stop='' locked=''
 
log() { # log LEVEL MESSAGE: one JSON line on stderr, like the core's
  jq -ncM --arg level "$1" --arg msg "$2" --arg dest "$dest" \
    '{ts: (now | todate), level: $level, logger: "certrun.sh", msg: $msg, destination: $dest}' >&2
}
# shellcheck disable=SC2329  # called from on_exit, the EXIT trap
run_metrics() { # run_metrics STATUS: when the last apply ended and how, rewritten in one rename
  local file=$var/textfile/certrun-$dest-run.prom label="{destination=\"$dest\"}" tmp
  tmp=$(mktemp "$file.XXXXXX") || return
  printf '%s\n' "# TYPE certrun_last_run_seconds gauge" "certrun_last_run_seconds$label $(date +%s)" \
    "# TYPE certrun_last_run_status gauge" "certrun_last_run_status$label $1" >"$tmp"
  chmod 0644 "$tmp" && mv -f -- "$tmp" "$file"
}
# shellcheck disable=SC2329  # called from the EXIT trap
on_exit() { # every way out: the run metrics (an apply that got the lock), then the run directory
  local status=$1
  trap '' TERM INT # a second signal must not cut this short
  [[ -z $stop ]] || status=$((128 + $(kill -l "$stop")))
  [[ $mode != apply || -z $locked ]] || run_metrics "$status" || log warning "run metrics not written"
  [[ -z $run ]] || rm -rf -- "$run"
}
# shellcheck disable=SC2329  # called from the TERM and INT traps
on_signal() { # remember the signal and pass a stop request on to the core, if it runs
  stop=$1
  [[ -z $child ]] || kill -TERM "$child" 2>/dev/null || true
}
reraise() { # die of the signal we were sent; the EXIT trap still runs first
  log warning "stopped by SIG$stop"
  trap - "$stop"
  kill -s "$stop" "$$"
}
usage() {
  echo "usage: certrun.sh plan|apply DESTINATION" >&2
  exit 2
}
trap 'on_exit $?' EXIT
trap 'on_signal TERM' TERM
trap 'on_signal INT' INT
 
(($# == 2)) || usage
[[ $1 == plan || $1 == apply ]] || usage
[[ $2 =~ ^[a-z0-9][a-z0-9-]{0,62}$ ]] || usage # a name, never a path
mode=$1 dest=$2
config=$here/etc/$dest.toml
[[ -f $config ]] || { log error "no settings file $config"; exit 78; }
[[ $budget =~ ^[1-9][0-9]*$ ]] || { log error "CERTRUN_BUDGET must be whole seconds"; exit 78; }
deadline=$((SECONDS + budget))
 
[[ -w $var/state ]] || { log error "$var/state is missing or not writable: see install.sh"; exit 78; }
 
# One run per destination. The core inherits this descriptor on purpose: if this script is
# killed, the lock stays held until the core has gone too. A run refused here did nothing, so it
# leaves the run metrics alone (their age shows that no run gets through).
exec {lock}>>"$var/state/$dest.lock"
flock -n "$lock" || { log warning "another run holds the lock"; exit 75; }
locked=yes
run=$(mktemp -d -t "certrun-$dest.XXXXXX")
 
# The gate: verify the signature on a private copy of the wheel with the pinned key, then check
# the installed core against that copy, with the system python (-I -S: no site, so nothing in the
# venv runs before the check). set -e is off inside a function called in a condition: check each.
verify_core() {
  cp -- "$wheel" "$run/core.whl" || return 1
  cp -- "$wheel.sigstore.json" "$run/core.bundle" || return 1
  "$here/bin/cosign" verify-blob --key "$here/etc/cosign.pub" --insecure-ignore-tlog=true \
    --bundle "$run/core.bundle" "$run/core.whl" || return 1
  /usr/bin/python3 -I -S "$here/verify_core.py" "$run/core.whl" "$release/venv"
}
if ! verify_core >"$run/gate.out" 2>&1; then
  log error "refusing to run: the core is not the signed release"
  cat -- "$run/gate.out" >&2
  exit 77
fi
[[ -z $stop ]] || reraise
 
jq -n --arg mode "$mode" --arg destination "$dest" --arg config "$config" --arg var "$var" \
  --arg run_id "$(date -u +%Y%m%dT%H%M%SZ)-$$" \
  '{mode: $mode, destination: $destination, config: $config, run_id: $run_id, var: $var}' >"$run/job.json"
 
left=$((deadline - SECONDS))
((left > 0)) || { log error "deadline reached before the core started"; exit 75; }
# Only HOME, LANG and PATH reach the core; -I: no PYTHONPATH, no user site, no current
# directory on sys.path. The bytecode is root's, compiled by install.sh next to the source; -B
# writes none at run time (PYTHONDONTWRITEBYTECODE would not do: -I ignores PYTHON* variables).
timeout -k 10 "$left" env -i HOME="$HOME" LANG=C.UTF-8 PATH=/usr/bin:/bin \
  "$release/venv/bin/python" -I -B -m certrun "$run/job.json" &
child=$!
[[ -z $stop ]] || kill -TERM "$child" # a signal that came before $! was known
status=0
wait "$child" || status=$?
while kill -0 "$child" 2>/dev/null; do # a trapped signal ends wait early: wait for the core
  status=0
  wait "$child" || status=$?
done
child=
[[ -z $stop ]] || reraise # the core has finished its own cleanup by now
case $status in
  0 | 1 | 70 | 75 | 78) ;; # the core's own contract
  124)
    log error "deadline of ${budget}s reached: the core was stopped"
    status=75
    ;;
  137)
    if ((SECONDS >= deadline)); then
      log error "deadline of ${budget}s reached: the core ignored SIGTERM and was killed"
      status=75
    else
      log error "the core was killed with SIGKILL before the deadline (out of memory?)"
      status=70
    fi
    ;;
  *)
    log error "the core ended with status $status, which is not in its contract"
    status=70
    ;;
esac
exit "$status"

"""What certrun does on a host, each over its own SSH connection: read the certificate in every
PEM file, replace one file with a backup, put the backup back, and have the server reload.
Blocking (paramiko), so the async code calls these through asyncio.to_thread."""
import shlex
 
from .config import Config, Endpoint, Host
from .errors import TargetError, TransientError
from .ssh import connect, run
 
# Runs in the remote login shell with the new PEM on stdin. The old file is kept as BACKUP (once
# per rotation: a resumed rotation must not back up its own new file). The new file starts as a
# copy of the old one, so it keeps the owner and mode the server expects, and replaces it in one
# rename(2): the server finds the old pair or the new one, never half of each. A symlink is
# refused, because the rename would replace the link instead of the file the server reads.
INSTALL = """set -e
f={path} b={backup}
if [ -L "$f" ]; then echo "$f is a symlink: not replacing it" >&2; exit 1; fi
[ -e "$b" ] || cp -p -- "$f" "$b"
cp -p -- "$f" "$f.certrun-new"
cat >"$f.certrun-new"
mv -f -- "$f.certrun-new" "$f"
"""
 
 
def deployed(host: Host, cfg: Config) -> dict[str, str | Exception]:
    """The SHA-256 of the certificate in each of this host's PEM files, or why it is unknown."""
    found: dict[str, str | Exception] = {}
    with connect(host, cfg) as client:
        for ep in (e for e in cfg.endpoints if e.host == host.name):
            try:
                out = run(client, f"openssl x509 -noout -fingerprint -sha256 -in {shlex.quote(ep.pem_path)}", cfg)
                found[ep.name] = out.strip().split("=", 1)[1].replace(":", "").lower()
            except (TargetError, TransientError) as err:
                found[ep.name] = err
            except (IndexError, ValueError):
                found[ep.name] = TargetError(f"{ep.pem_path}: unexpected openssl output")
    return found
 
 
def install(host: Host, ep: Endpoint, pem: bytes, backup: str, cfg: Config) -> None:
    command = INSTALL.format(path=shlex.quote(ep.pem_path), backup=shlex.quote(backup))
    with connect(host, cfg) as client:
        run(client, command, cfg, data=pem)
 
 
def rollback(host: Host, ep: Endpoint, backup: str, cfg: Config) -> None:
    """Put back the file install() replaced, again in one rename."""
    with connect(host, cfg) as client:
        run(client, f"mv -f -- {shlex.quote(backup)} {shlex.quote(ep.pem_path)}", cfg)
 
 
def forget(host: Host, backup: str, cfg: Config) -> None:
    """The backup holds the old private key: remove it once the new certificate is live."""
    with connect(host, cfg) as client:
        run(client, f"rm -f -- {shlex.quote(backup)}", cfg)
 
 
def reload(host: Host, cfg: Config) -> None:
    with connect(host, cfg) as client:
        run(client, host.reload, cfg)

"""decide(): what to do with one endpoint, from what it serves, what its host has on disk and the
record of any rotation a stopped run left behind. No network and no side effects."""
from dataclasses import dataclass
from datetime import datetime
 
from .config import Config, Endpoint
from .errors import TransientError
from .pending import Pending
from .tlscheck import EXPIRED, Seen
 
 
@dataclass
class Result:
    endpoint: str
    action: str                     # ok, rotate, finish, rollback (plan); rotated, failed (apply)
    detail: str
    temporary: bool = False         # failed, and the next run may succeed
    seen: Seen | None = None
    record: Pending | None = None
    not_after: datetime | None = None
 
 
def failed(ep: Endpoint, err: Exception, seen: Seen | Exception) -> Result:
    not_after = seen.not_after if isinstance(seen, Seen) else None  # still worth a metric
    return Result(ep.name, "failed", str(err), temporary=isinstance(err, TransientError), not_after=not_after)
 
 
def decide(ep: Endpoint, seen: Seen | Exception, disk: str | Exception, rec: Pending | None,
           cfg: Config) -> Result:
    for outcome in (disk, seen):  # the host first: no certificate for a host we cannot trust
        if isinstance(outcome, Exception):
            return failed(ep, outcome, seen)
    assert isinstance(seen, Seen) and isinstance(disk, str)
 
    def act(action: str, detail: str) -> Result:
        return Result(ep.name, action, detail, seen=seen, record=rec, not_after=seen.not_after)
 
    days = seen.days_left()
    if rec and rec.rolled_back:
        return act("failed", f"a deploy was rolled back ({rec.rolled_back}); fix the cause,"
                             f" then delete pending/{ep.name}.json")
    if rec and rec.deployed and seen.key == rec.key:  # deployed, never checked: check it now
        if not seen.problem:
            return act("finish", "a stopped run deployed the new certificate; the audit line is missing")
        return act("rollback", f"a stopped run deployed a certificate that fails its check: {seen.problem}")
    if rec and rec.replacing in (seen.fingerprint, disk):
        return act("rotate", "resume the rotation a stopped run began")
    if rec:
        return act("failed", f"pending/{ep.name}.json replaces {rec.replacing[:12]}, which is gone")
    if disk != seen.fingerprint:
        return act("failed", f"{ep.host} has {disk[:12]} on disk but serves {seen.fingerprint[:12]}:"
                             " a reload is pending or failed")
    if seen.problem and seen.code != EXPIRED:  # a wrong name, an unknown issuer: not ours to fix
        return act("failed", f"serves a certificate that does not verify: {seen.problem}")
    if seen.problem and not seen.expired:  # code 10, but the leaf itself is valid
        return act("failed", f"the chain does not verify ({seen.problem}), but the certificate itself"
                             f" has {days} days left: an issuer's certificate expired")
    if seen.expired:
        return act("rotate", f"EXPIRED {-days} days ago")
    return act("rotate" if days <= cfg.policy.renew_within_days else "ok", f"{days} days left")

"""A rotation in progress, written to disk before the CA is asked: the new private key, the exact
request and its idempotency key, and, from just before the deploy, that the new file may be live
but is not yet checked. A run stopped anywhere after that leaves the record, and the next run
resends the same bytes (the CA answers from its own record) or checks the deployed certificate:
the missing audit line if it verifies, a rollback if it does not.
One file per endpoint, 0600 in a 0700 directory."""
import dataclasses
import hashlib
import json
import os
import secrets
from datetime import UTC, datetime
from pathlib import Path
 
from cryptography import x509
from cryptography.hazmat.primitives import hashes, serialization
from cryptography.hazmat.primitives.asymmetric import ec
from cryptography.x509.oid import NameOID
 
from .config import Endpoint
from .tlscheck import DER, SPKI, Seen
 
PEM = serialization.Encoding.PEM
 
 
@dataclasses.dataclass(frozen=True)
class Pending:
    endpoint: str
    replacing: str        # fingerprint of the certificate being replaced
    replacing_key: str    # and of its public key: the new key must differ
    key: str              # SHA-256 of the new public key
    key_pem: str          # the new private key (unencrypted, like the PEM the server loads)
    request: str          # the request body, resent byte for byte
    idempotency_key: str
    created: str
    deployed: bool = False          # the new file may be on the host: deployed, not yet checked
    rolled_back: str | None = None  # why a deploy of this certificate was undone
 
 
def start(state: Path, ep: Endpoint, seen: Seen) -> Pending:
    """A new key and request for ep, saved before anything is sent."""
    key = ec.generate_private_key(ec.SECP256R1())  # a new key for every rotation, never reused
    subject = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, ep.server_name)])
    csr = (x509.CertificateSigningRequestBuilder().subject_name(subject)
           .add_extension(x509.SubjectAlternativeName([x509.DNSName(ep.server_name)]), critical=False)
           .sign(key, hashes.SHA256()))  # ECDSA signatures are random: a new CSR never repeats
    rec = Pending(
        endpoint=ep.name, replacing=seen.fingerprint, replacing_key=seen.key,
        key=hashlib.sha256(key.public_key().public_bytes(DER, SPKI)).hexdigest(),
        key_pem=key.private_bytes(PEM, serialization.PrivateFormat.PKCS8,
                                  serialization.NoEncryption()).decode(),
        request=json.dumps({"name": ep.name, "csr": csr.public_bytes(PEM).decode()}),
        idempotency_key="certrun-" + secrets.token_hex(16),
        created=datetime.now(UTC).isoformat(timespec="seconds"))
    save(state, rec)
    return rec
 
 
def load(state: Path, name: str) -> Pending | None:
    try:
        return Pending(**json.loads((state / "pending" / f"{name}.json").read_text()))
    except FileNotFoundError:
        return None
 
 
def save(state: Path, rec: Pending) -> None:
    folder = state / "pending"
    folder.mkdir(mode=0o700, parents=True, exist_ok=True)  # parents: 0700 state from install.sh
    tmp = folder / f".{rec.endpoint}.tmp"
    with os.fdopen(os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600), "w") as f:
        json.dump(dataclasses.asdict(rec), f)
        f.flush()
        os.fsync(f.fileno())
    os.replace(tmp, folder / f"{rec.endpoint}.json")
    sync_dir(folder)  # the new name is on disk too, not only the bytes
 
 
def drop(state: Path, name: str) -> None:
    (state / "pending" / f"{name}.json").unlink()
    sync_dir(state / "pending")
 
 
def sync_dir(folder: Path) -> None:
    fd = os.open(folder, os.O_RDONLY | os.O_DIRECTORY)
    try:
        os.fsync(fd)
    finally:
        os.close(fd)

"""The CA API client. Every attempt of one rotation sends the same Idempotency-Key with the same
request bytes, so a request whose answer was lost is answered from the CA's record instead of
issuing a second certificate. Redirects are refused: the bearer token only ever goes to ca.url."""
import asyncio
import http.client
import json
import logging
import random
import ssl
import urllib.error
import urllib.request
 
from .config import CA
from .errors import TargetError, TransientError
from .retry_after import parse_retry_after
 
log = logging.getLogger("certrun.ca")
RETRYABLE = {429, 500, 502, 503, 504}
 
 
class NoRedirects(urllib.request.HTTPRedirectHandler):
    def redirect_request(self, req, fp, code, msg, headers, newurl):  # type: ignore[no-untyped-def]
        return None  # urllib then raises HTTPError for the 3xx, and the token stays here
 
 
def post(ca: CA, token: str, key: str, body: str) -> str:
    """One attempt; blocking. The certificate PEM, or an error that says what should happen next."""
    https = urllib.request.HTTPSHandler(context=ssl.create_default_context(cafile=ca.ca_file))
    opener = urllib.request.build_opener(NoRedirects, https)
    req = urllib.request.Request(
        f"{ca.url}/certificates", data=body.encode(), method="POST",
        headers={"Content-Type": "application/json", "Authorization": f"Bearer {token}",
                 "Idempotency-Key": key})
    try:
        with opener.open(req, timeout=ca.timeout) as resp:
            answer = json.load(resp)
    except urllib.error.HTTPError as err:
        if err.code in RETRYABLE:
            raise TransientError(f"HTTP {err.code}", parse_retry_after(err.headers["Retry-After"])) from None
        moved = f" to {err.headers['Location']}, not followed" if 300 <= err.code < 400 else ""
        raise TargetError(f"the CA answered HTTP {err.code}{moved}") from None
    except urllib.error.URLError as err:
        if isinstance(err.reason, ssl.SSLError):  # the CA's certificate does not verify: not transient
            raise TargetError(f"CA TLS: {err.reason}") from None
        raise TransientError(repr(err.reason)) from None
    except (OSError, http.client.HTTPException, ValueError) as err:  # timeouts, resets, a garbled body
        raise TransientError(repr(err)) from None
    if not isinstance(answer, dict) or not isinstance(answer.get("pem"), str):
        raise TargetError("the CA's answer holds no certificate")
    return answer["pem"]
 
 
async def issue(ca: CA, token: str, key: str, body: str, name: str) -> str:
    for attempt in range(1, ca.attempts + 1):
        try:
            return await asyncio.to_thread(post, ca, token, key, body)
        except TransientError as err:
            if attempt == ca.attempts:
                raise
            wait = random.uniform(0, 0.5 * 2**attempt)  # full jitter: up to 1 s, 2 s, 4 s
            if err.retry_after is not None:
                if err.retry_after > ca.max_retry_after:  # give up now; the next run tries again
                    raise TransientError(f"{err}, Retry-After {err.retry_after:.0f} s is too long") from None
                wait = err.retry_after  # the server said when: not sooner
            log.warning("CA call failed, retrying", extra={"fields": {
                "endpoint": name, "error": str(err), "attempt": attempt, "wait_s": round(wait, 1)}})
            await asyncio.sleep(wait)
    raise AssertionError("not reached: attempts is at least 1")

"""Rotate one endpoint: never issue twice, never deploy blind. The request is on disk before it is
sent (pending.py); the certificate must match the recorded key; the record says "deployed, not yet
checked" before the deploy, which keeps a backup; the server is reloaded and checked; a failed
check is rolled back; only then is the change audited."""
import asyncio
import dataclasses
import hashlib
import logging
from typing import NoReturn
 
from cryptography import x509
 
from . import ca, obs, pending, remote, tlscheck
from .config import Config, Endpoint
from .errors import TargetError, TransientError
from .pending import Pending
from .tlscheck import DER, Seen
 
log = logging.getLogger("certrun")
 
 
async def rotate(ep: Endpoint, rec: Pending | None, seen: Seen, cfg: Config, token: str) -> Seen:
    """rec: the record a stopped run left, resumed as it is; None starts a new rotation."""
    rec = rec or pending.start(cfg.paths.state, ep, seen)
    if rec.key == rec.replacing_key:
        raise TargetError("the new key is the key being replaced; not requesting")
    pem = await ca.issue(cfg.ca, token, rec.idempotency_key, rec.request, ep.name)
    cert = x509.load_pem_x509_certificate(pem.encode())
    if tlscheck.key_id(cert) != rec.key:
        raise TargetError("the CA returned a certificate for another key; not deployed")
    new = hashlib.sha256(cert.public_bytes(DER)).hexdigest()
    host = cfg.hosts[ep.host]
    rec = dataclasses.replace(rec, deployed=True)  # from here the new file may be live, unchecked
    pending.save(cfg.paths.state, rec)
    await asyncio.to_thread(remote.install, host, ep, (pem + rec.key_pem).encode(), backup_of(ep, rec), cfg)
    await asyncio.to_thread(remote.reload, host, cfg)
    try:
        now = await served(ep, cfg, new)
        why = now.problem or (None if now.fingerprint == new else "the reload did not take effect")
    except TransientError as err:  # no answer at all after the reload
        why = str(err)
    if why is None:
        await finish(ep, rec, now, cfg)
        return now
    await roll_back(ep, rec, why, cfg)
 
 
async def roll_back(ep: Endpoint, rec: Pending, why: str, cfg: Config) -> NoReturn:
    """Put the backup back, reload, check. Marked first, so no later run asks the CA again for
    this rotation: a person decides. Also what the next run does with a failing deployed record."""
    pending.save(cfg.paths.state, dataclasses.replace(rec, rolled_back=why))
    host = cfg.hosts[ep.host]
    await asyncio.to_thread(remote.rollback, host, ep, backup_of(ep, rec), cfg)
    await asyncio.to_thread(remote.reload, host, cfg)
    back = await served(ep, cfg, rec.replacing)
    state = "rolled back" if back.fingerprint == rec.replacing else "ROLLBACK FAILED"
    raise TargetError(f"the new certificate failed its check ({why}); {state}, serves {back.fingerprint[:12]}")
 
 
def backup_of(ep: Endpoint, rec: Pending) -> str:
    return f"{ep.pem_path}.certrun-{rec.idempotency_key[-8:]}"  # one backup per rotation
 
 
async def served(ep: Endpoint, cfg: Config, want: str) -> Seen:
    """What ep serves once a reload has taken effect: checked every half second until it serves
    `want`, for at most policy.reload_wait seconds (a graceful reload takes a moment)."""
    loop = asyncio.get_running_loop()
    end = loop.time() + cfg.policy.reload_wait
    while True:
        seen = await tlscheck.check(ep, cfg, asyncio.Semaphore(1))
        if seen.fingerprint == want or loop.time() >= end:
            return seen
        await asyncio.sleep(0.5)
 
 
async def finish(ep: Endpoint, rec: Pending, now: Seen, cfg: Config, recovered: bool = False) -> None:
    """The new certificate is live and verified: remove the backup (it holds the old private key),
    record the change, then delete the record. Anything that stops this half way leaves the record,
    and the next run finishes the job: the backup is never left without a record naming it."""
    try:
        await asyncio.to_thread(remote.forget, cfg.hosts[ep.host], backup_of(ep, rec), cfg)
    except (TargetError, TransientError) as err:
        raise type(err)(f"the new certificate is live; the old key file stays until the next run: {err}") from None
    obs.audit(cfg.paths.audit, {
        "action": "rotate", "endpoint": ep.name, "host": ep.host, "old": rec.replacing,
        "new": now.fingerprint, "old_key": rec.replacing_key, "new_key": rec.key,
        "not_after": now.not_after.isoformat(), "idempotency_key": rec.idempotency_key,
        **({"recovered": True} if recovered else {})})
    pending.drop(cfg.paths.state, ep.name)
    log.info("rotated, audit line written", extra={"fields": {"endpoint": ep.name, "recovered": recovered}})

#!/usr/bin/env bash
# Build one release of the certrun core into release/: the wheel and its signature bundle, a venv
# with the locked dependencies and the wheel, and an SBOM of that venv. The signing key is made
# for this one release in a private temporary directory and deleted when the script ends; only
# its public key survives, as cosign.pub next to release/ (not in it), for install.sh to pin.
# In production the key lives in a KMS or HSM, or signing is keyless: never on the runner.
# Needs bin/uv, bin/cosign and bin/syft (lab/get-tools.sh).
set -euo pipefail
cd "$(dirname "$0")"
whl=certrun-0.1.0-py3-none-any.whl
[[ ! -e release ]] || { echo "release.sh: release/ exists; a release is never overwritten" >&2; exit 1; }
keys=$(mktemp -d)
trap 'rm -rf -- "$keys"' EXIT
 
./bin/uv lock --check -q # the lock still matches pyproject.toml
rm -rf dist
./bin/uv build -q --wheel # dist/$whl
./bin/uv export -q --locked --no-dev --no-emit-project -o dist/requirements.txt # with hashes
 
# Every file is copied (no links into uv's cache), so the release owns its bytes; the venv is
# relocatable because install.sh moves it to /opt.
./bin/uv venv -q --relocatable --python "$(command -v python3)" release/venv
./bin/uv pip install -q --link-mode=copy --python release/venv/bin/python \
  --require-hashes -r dist/requirements.txt
./bin/uv pip install -q --link-mode=copy --python release/venv/bin/python --no-deps "dist/$whl"
cp "dist/$whl" release/
 
# The password protects the key file for the few seconds it exists; only cosign sees it.
pass=$(openssl rand -hex 16)
COSIGN_PASSWORD=$pass ./bin/cosign generate-key-pair --output-key-prefix "$keys/cosign" 2>/dev/null
COSIGN_PASSWORD=$pass ./bin/cosign sign-blob --yes --key "$keys/cosign.key" \
  --signing-config offline-signing.json --bundle "release/$whl.sigstore.json" "release/$whl"
cp "$keys/cosign.pub" cosign.pub
chmod -R u=rwX,go=rX release # readable by the service account; install.sh makes it root's
./bin/syft scan -q dir:release/venv -o cyclonedx-json=release/sbom.cdx.json
printf 'release: %s, %s components in the SBOM; signing key deleted, public key in cosign.pub\n' \
  "$whl" "$(jq '.components | length' release/sbom.cdx.json)"

"""Is the installed core the signed wheel? Every file the wheel's RECORD lists for the certrun
package must be installed in VENV with that hash, and nothing else may sit in the package
directory. Run by certrun.sh after cosign has verified the wheel, with the system python -I -S,
so nothing in the venv (a .pth file in site-packages runs at startup) runs before the check.
Standard library only. The dependencies are not checked: root ownership of the release is what
keeps them as installed. Usage: python3 -I -S verify_core.py WHEEL VENV   exit 0 same files, 1 not"""
import base64
import csv
import hashlib
import io
import sys
import zipfile
from pathlib import Path
 
PACKAGE = "certrun/"
 
 
def main(wheel: str, venv: str) -> int:
    site = next(Path(venv).glob("lib/python3*/site-packages"), None)
    if site is None:
        print(f"verify_core: no site-packages in {venv}", file=sys.stderr)
        return 1
    with zipfile.ZipFile(wheel) as zf:
        record = next(n for n in zf.namelist() if n.endswith(".dist-info/RECORD"))
        rows = csv.reader(io.StringIO(zf.read(record).decode()))
        signed = {path: digest for path, digest, _size in rows if path.startswith(PACKAGE)}
    problems = []
    for path, digest in sorted(signed.items()):
        algorithm, _, expected = digest.partition("=")
        try:
            data = (site / path).read_bytes()
        except OSError:
            problems.append(f"{path}: missing")
            continue
        actual = base64.urlsafe_b64encode(hashlib.new(algorithm, data).digest()).rstrip(b"=")
        if actual.decode() != expected:
            problems.append(f"{path}: differs from the signed wheel")
    for file in sorted((site / PACKAGE).rglob("*")):
        name = file.relative_to(site).as_posix()
        if file.is_file() and "__pycache__" not in file.parts and name not in signed:
            problems.append(f"{name}: not in the signed wheel")
    for problem in problems:
        print(f"verify_core: {problem}", file=sys.stderr)
    return 1 if problems else 0
 
 
if __name__ == "__main__":
    sys.exit(main(sys.argv[1], sys.argv[2]))

#!/usr/bin/env bash
# Install the certrun runner for the service ACCOUNT, as root, from the project directory:
#   /opt/scr-capstone        root:root, 0755: certrun.sh, the release, the gate and its trust
#                            anchors, the settings. The account can run all of it, change none.
#   /var/lib/scr-capstone    state/ (locks, pending rotations, audit; 0700) and textfile/ (0755,
#                            for node_exporter): the only places the account can write.
#   /etc/systemd/system      the unit and timer, with User=ACCOUNT. The timer is not enabled.
# Undo: sudo rm -rf /opt/scr-capstone /var/lib/scr-capstone \
#         /etc/systemd/system/scr-capstone-certrun@.{service,timer} && sudo systemctl daemon-reload
set -euo pipefail
((EUID == 0)) || { echo "install.sh: run as root" >&2; exit 1; }
account=${1:?usage: install.sh ACCOUNT}
group=$(id -gn -- "$account")
opt=/opt/scr-capstone var=/var/lib/scr-capstone sshd=/run/scr-capstone-sshd
cd -- "$(dirname -- "${BASH_SOURCE[0]}")"
[[ ! -e $opt ]] || { echo "install.sh: $opt exists; remove it first" >&2; exit 1; }
 
install -d -m 0755 "$opt" "$opt/bin" "$opt/etc"
install -m 0755 certrun.sh "$opt/"
install -m 0644 verify_core.py "$opt/"
install -m 0755 bin/cosign "$opt/bin/"
cp -R release "$opt/release" && chown -R root:root "$opt/release"
# The bytecode, compiled here by root next to the source, so the account cannot write it (and
# certrun.sh runs python -B: nothing is written at run time). checked-hash: a .pyc is used only
# while it matches the source file beside it, and the gate checks that source.
"$opt/release/venv/bin/python" -I -m compileall -q -j 0 --invalidation-mode checked-hash "$opt/release/venv/lib"
install -m 0644 cosign.pub etc/lab.toml "$opt/etc/" # the pinned signing key, the settings
install -m 0644 lab/pki/ca.pem "$opt/etc/lab-ca.pem"
install -m 0644 "$sshd/known_hosts" "$opt/etc/known_hosts"
# Secrets the account must read and must not change: root's, readable by its group only.
install -m 0640 -g "$group" lab/pki/ca-token "$opt/etc/ca-token"
install -m 0640 -g "$group" "$sshd/id_certrun" "$opt/etc/id_certrun"
 
install -d -m 0755 "$var"
install -d -m 0700 -o "$account" -g "$group" "$var/state"
install -d -m 0755 -o "$account" -g "$group" "$var/textfile"
 
sed "s/^User=.*/User=$account/" systemd/scr-capstone-certrun@.service \
  >/etc/systemd/system/scr-capstone-certrun@.service
install -m 0644 systemd/scr-capstone-certrun@.timer /etc/systemd/system/
systemctl daemon-reload
find "$opt" "$var" -maxdepth 2 -not -path "$opt/release/*" -printf '%M %u:%g %p\n'

# One apply run of certrun for the destination %i, e.g. scr-capstone-certrun@lab.service.
# Started by scr-capstone-certrun@%i.timer, or by hand with systemctl start.
[Unit]
Description=certrun apply %i: renew expiring TLS certificates
 
[Service]
Type=oneshot
# install.sh puts the service account here (in production, a system account of its own)
User=certrun
ExecStart=/opt/scr-capstone/certrun.sh apply %i
Environment=CERTRUN_BUDGET=120
# A stop (shutdown, deploy, an operator) ends the run by SIGTERM. For Type=oneshot that counts
# as a failure unless it is listed here, and a failure is what OnFailure= alerts react to.
SuccessExitStatus=SIGTERM
# SIGTERM to certrun.sh only, which forwards it to the core; SIGKILL to whatever is left once
# certrun.sh has exited, or after TimeoutStopSec. A call running in a thread cannot be cancelled,
# so a stop waits for it: 30 s is above the longest, one SSH call of about 25 s (connect, banner
# and auth 5 s each, then 5 s per idle read of its output and of its errors).
KillMode=mixed
TimeoutStopSec=30
# 75 means "temporary, run again later": retry it in 15 minutes. The other failures need a
# person, not a loop.
Restart=on-failure
RestartSec=15min
RestartPreventExitStatus=1 2 70 77 78
NoNewPrivileges=yes
PrivateTmp=yes
ProtectSystem=strict
ReadWritePaths=/var/lib/scr-capstone

# Twice a day for the destination %i. In production:
#   systemctl enable --now scr-capstone-certrun@lab.timer
[Unit]
Description=certrun apply %i, twice a day
 
[Timer]
OnCalendar=*-*-* 00,12:00:00
# A run missed while the host was down happens at the next boot; many runners spread their
# requests to the CA over half an hour instead of all arriving at 00:00:00.
Persistent=true
RandomizedDelaySec=30min
 
[Install]
WantedBy=timers.target