Capstone: a backup job you can leave running

#!/usr/bin/env bash
# setup.sh: create app/, the sample application the backup job archives. Every secret in
# it is a test value. Run it once, in ~/bash-ops/backup.
set -Eeuo pipefail
mkdir -p app/.ssh app/config/tls app/data/uploads app/logs
printf 'listen=8443\nlog_level=info\n' > app/config/app.conf
printf 'DB_PASSWORD=lab-password-not-secret\n' > app/config/.env
printf 'DB_PASSWORD=lab-prod-password-not-secret\n' > app/config/.env.production
printf 'lab-certificate-not-secret\n' > app/config/tls/server.crt
printf 'lab-key-not-secret\n' > app/config/tls/server.key
printf 'lab-old-key-not-secret\n' > app/config/tls/OLD-SERVER.KEY
printf 'lab-ssh-key-not-secret\n' > app/.ssh/id_ed25519
printf '2026-09-27T08:00:01Z login ok user=alice token=lab-token-not-secret\n' > app/logs/app.log
printf '2026-09-26T08:00:01Z login ok user=bob token=lab-token-not-secret\n' > app/logs/app.log.1
# 2000 order lines, so the data is most of the archive (the size check relies on that)
seq -f '%g,9.50' 1001 3000 > 'app/data/orders 2026.csv'
printf 'invoice 1001\n' > 'app/data/uploads/invoice 1001.txt'
find app -type f | sort

#!/usr/bin/env bash
# backup.sh: archive one directory into DEST as NAME-YYYYmmddTHHMMSSZ.tar.gz with a .sha256
# file, leaving out secrets and logs, and keep only the newest N archives of that job.
#
# Exit status:
#   0    archive published and verified, or --dry-run finished
#   1    the backup failed; nothing new was published (the log says why)
#   2    usage error: bad option or value, or NAME already belongs to another SOURCE in DEST
#   3    another run of this job holds its lock (3 several times in a row: a run is stuck)
#   129, 130, 143  stopped by SIGHUP, SIGINT or SIGTERM (128 + signal number); cleanup ran
# Under timeout(1), as the schedule runs it: 124 the time limit was reached and the run was
# stopped (cleanup ran); 137 it was killed after the grace period (a work.* directory may stay).
set -Eeuo pipefail
shopt -s inherit_errexit nullglob
umask 077 # archives hold your data: new files 0600, new directories 0700
 
# Never archived, compared in lower case: SSH keys, environment files, private keys and
# certificate bundles, logs and rotated logs. A denylist stops only the names it lists.
readonly EXCLUDES=('.ssh' 'id_rsa*' 'id_ecdsa*' 'id_ed25519*' '.env*'
  '*.key' '*.pem' '*.p12' '*.pfx' '*.log' '*.log.*')
 
log() {
  local stamp
  stamp=$(date -u +%Y-%m-%dT%H:%M:%SZ)
  printf '%s backup.sh: %s\n' "$stamp" "$*" >&2
}
 
die() {
  log "ERROR: $*"
  exit 1
}
 
trap 'log "ERROR: line $LINENO: $BASH_COMMAND failed with status $?"' ERR

usage() {
  cat << 'END'
Usage: backup.sh [OPTIONS] [--] SOURCE
Archive the directory SOURCE into DEST without secrets or logs, verify the
archive, publish it, and keep the newest N archives of this job.
 
  -d, --dest DIR      where archives go (default ~/backups); must already exist
      --name NAME     job name used in archive names (default: the last part of
                      SOURCE's path); in one DEST, a NAME belongs to one SOURCE
  -k, --keep N        archives of this job to keep, 2 to 100 (default 7)
      --allow-shrink  publish although the archive is less than half the size
                      of the previous one (after you have checked why)
  -n, --dry-run       show what would be archived and deleted; change nothing
  -h, --help          show this help and exit
 
Exit status: 0 done, 1 backup failed, 2 usage error, 3 another run of this
job holds the lock, 129/130/143 stopped by a signal. Under timeout(1) also
124 (time limit reached) and 137 (killed).
END
}
 
usage_error() {
  printf 'backup.sh: %s\n' "$1" >&2
  printf "Try 'backup.sh --help' for more information.\n" >&2
  exit 2
}

parse_args() {
  dest=$HOME/backups
  name=
  keep=7
  allow_shrink=0
  dry_run=0
  local -a sources=()
  while (($# > 0)); do
    case $1 in
      -h | --help)
        usage
        exit 0
        ;;
      -n | --dry-run) dry_run=1 ;;
      --allow-shrink) allow_shrink=1 ;;
      -d | --dest | --name | -k | --keep)
        (($# >= 2)) || usage_error "option $1 needs a value"
        case $1 in
          -d | --dest) dest=$2 ;;
          --name) name=$2 ;;
          *) keep=$2 ;;
        esac
        shift
        ;;
      --dest=*) dest=${1#--dest=} ;;
      --name=*) name=${1#--name=} ;;
      --keep=*) keep=${1#--keep=} ;;
      --)
        shift
        sources+=("$@")
        break
        ;;
      -?*) usage_error "unknown option: $1" ;;
      *) sources+=("$1") ;;
    esac
    shift
  done
  # At least 2: when the newest archive turns out to be bad, an older one is still there.
  if ! [[ $keep =~ ^[1-9][0-9]{0,2}$ ]] || ((keep < 2 || keep > 100)); then
    usage_error "--keep must be a whole number from 2 to 100, not '$keep'"
  fi
  ((${#sources[@]} == 1)) || usage_error "give exactly one SOURCE directory"
  source_arg=${sources[0]}
}

validate() {
  [[ -d $source_arg ]] || die "not a directory: $source_arg"
  [[ -d $dest ]] || die "destination does not exist: $dest (create it once with mkdir -m 700)"
  src=$(realpath -e -- "$source_arg")
  dest=$(realpath -e -- "$dest")
  [[ $src != / ]] || usage_error "refusing to back up /"
  if [[ $dest == "$src" || $dest == "$src"/* ]]; then
    usage_error "--dest must be outside SOURCE, or tar would read its own archive"
  fi
  name=${name:-${src##*/}}
  # The valid_name allowlist from "Functions, local and logging".
  if ! [[ $name =~ ^[A-Za-z0-9_][A-Za-z0-9._-]*$ ]]; then
    usage_error "cannot use '$name' as a job name; give --name"
  fi
  jobdir=$dest/.$name.job # this job's lock, its SOURCE record and its work directories
  archive=$name-$(date -u +%Y%m%dT%H%M%SZ).tar.gz
  # One list of tar options for the real archive and for the --dry-run listing.
  tar_opts=(--sort=name --ignore-case)
  local pattern
  for pattern in "${EXCLUDES[@]}"; do
    tar_opts+=("--exclude=$pattern")
  done
  tar_opts+=(-C "$(dirname -- "$src")" -- "${src##*/}")
}

work=
cleanup() {
  if [[ -n $work ]]; then rm -rf -- "$work"; fi
}
 
on_signal() {
  log "caught SIG$1, cleaning up"
  trap - "$1"
  kill -s "$1" "$$"
}
 
# One lock per job, inside DEST: every run that writes this job's files, whoever starts it,
# takes the same lock. Other jobs and other destinations are not blocked by it.
take_lock() {
  mkdir -p -- "$jobdir"
  exec 9> "$jobdir/lock"
  if ! flock -n 9; then
    log "another run of job $name holds $jobdir/lock"
    exit 3
  fi
}
 
# In one DEST a job name belongs to one SOURCE: the first run records it, later runs
# compare. Otherwise /srv/site1/app and /srv/site2/app would prune each other's archives.
check_source() {
  local recorded
  if [[ -f $jobdir/source ]]; then
    recorded=$(cat -- "$jobdir/source")
    if [[ $recorded != "$src" ]]; then
      usage_error "job '$name' in $dest backs up $recorded; give $src another --name"
    fi
  elif ((!dry_run)); then
    printf '%s\n' "$src" > "$jobdir/source"
  fi
}

build_archive() {
  log "archiving $src as job $name"
  tar -czf "$work/$archive" "${tar_opts[@]}" || die "tar failed; nothing was published"
}
 
verify_archive() {
  local entry part pattern
  local -a parts
  tar -tzf "$work/$archive" > "$work/listing" || die "cannot read back $archive"
  while IFS= read -r entry; do
    IFS=/ read -r -a parts <<< "$entry"
    for part in "${parts[@]}"; do
      for pattern in "${EXCLUDES[@]}"; do
        # ${part,,} is the name in lower case: tar --ignore-case compared it that way too.
        # shellcheck disable=SC2053  # $pattern is unquoted on purpose: it is a glob
        if [[ ${part,,} == $pattern ]]; then die "excluded name inside the archive: $entry"; fi
      done
    done
  done < "$work/listing"
  (cd -- "$work" && sha256sum -- "$archive" > "$archive.sha256" &&
    sha256sum --check --quiet -- "$archive.sha256") || die "checksum verification failed"
}

# An archive much smaller than the last one usually means SOURCE lost its data (a volume
# that did not mount, a wiped directory). Publishing it would start rotating out good copies.
check_size() {
  local -a old=("$dest/$name"-????????T??????Z.tar.gz)
  ((${#old[@]} > 0)) || return 0 # the first archive of this job
  local last=${old[-1]} new_size last_size
  new_size=$(stat -c %s -- "$work/$archive")
  last_size=$(stat -c %s -- "$last")
  if ((new_size * 2 < last_size && !allow_shrink)); then
    die "new archive: $new_size bytes, ${last##*/}: $last_size; check SOURCE (--allow-shrink once if expected)"
  fi
}
 
publish() {
  sync -- "$work/$archive" "$work/$archive.sha256" # the data on disk before the names appear
  # work is inside DEST, so each mv is a rename within one file system: atomic.
  # The checksum goes first, so every archive a reader can see has its .sha256.
  mv -- "$work/$archive.sha256" "$dest/"
  mv -- "$work/$archive" "$dest/"
  log "published $dest/$archive"
  printf '%s\n' "$dest/$archive"
}

prune() {
  local -a old=()
  local path i excess
  # This job's archives only (the timestamp has a fixed length), never the one just made.
  for path in "$dest/$name"-????????T??????Z.tar.gz; do
    if [[ $path != "$dest/$archive" ]]; then old+=("$path"); fi
  done
  if ((${#old[@]} > 0)) && [[ ${old[-1]} > "$dest/$archive" ]]; then
    log "WARNING: ${old[-1]##*/} is dated after this run; check the clock"
  fi
  # Names sort oldest first: delete from the front until KEEP remain, this run's included.
  excess=$((${#old[@]} + 1 - keep))
  for ((i = 0; i < excess; i++)); do
    if ((dry_run)); then
      printf 'would delete %s\n' "${old[i]}"
    else
      rm -f -- "${old[i]:?}" "${old[i]:?}.sha256"
      log "deleted ${old[i]}"
    fi
  done
}

main() {
  parse_args "$@"
  validate
  if ((dry_run)); then
    check_source
    printf 'would archive into %s/%s:\n' "$dest" "$archive"
    tar -cvf /dev/null "${tar_opts[@]}" || die "cannot read $src"
    prune
    return 0
  fi
  trap cleanup EXIT
  trap 'on_signal HUP' HUP
  trap 'on_signal INT' INT
  trap 'on_signal TERM' TERM
  take_lock
  check_source
  [[ ! -e $dest/$archive ]] || die "$dest/$archive already exists; run again in a second"
  work=$(mktemp -d -- "$jobdir/work.XXXXXX")
  build_archive
  verify_archive
  check_size
  publish
  prune
  log "done: keeping the newest $keep archives of job $name"
}
 
main "$@"

#!/usr/bin/env bash
# stop-run.sh SIGNAL: start a slow backup of app/ into backups/, show that a second run is
# refused while it works, then send SIGNAL to the whole first run and show how it ended.
set -Eeuo pipefail
signal=${1:?usage: stop-run.sh HUP|TERM|KILL}
# Sparse: 2 GiB that take no disk space, but tar needs seconds to read the zeros.
truncate -s 2G app/data/big.img
trap 'rm -f -- app/data/big.img' EXIT
# A script has no job control, so a background command stays in the script's process
# group. setsid starts the backup in a new group whose ID is its PID, $!.
setsid ./backup.sh --dest backups app 2> first.err &
pid=$!
# Wait until tar is running, or until the backup has already ended (at most 10 s).
timeout 10 bash -c "until grep -q archiving first.err || ! kill -0 $pid 2> /dev/null; do
  sleep 0.1
done" || true
if ! grep -q archiving first.err; then
  # It failed before tar, or hangs: stop what is left, then report the backup's own error.
  kill -KILL -- "-$pid" 2> /dev/null || true
  status=0
  wait "$pid" || status=$?
  cat first.err >&2
  echo "stop-run.sh: the backup ended before it started tar: status $status" >&2
  exit "$status"
fi
status=0
./backup.sh --dest backups app || status=$?
echo "second run: status $status"
kill -s "$signal" -- "-$pid" # the minus sign: every process in the group, tar included
status=0
wait "$pid" || status=$?
echo "first run: status $status"
cat first.err

# backup.bats: tests for backup.sh, run from the project directory with: bats --tap test/
setup() {
  script=$BATS_TEST_DIRNAME/../backup.sh
  src=$BATS_TEST_TMPDIR/app
  dest=$BATS_TEST_TMPDIR/backups
  mkdir -p "$src/.ssh" "$src/config" "$src/logs" "$src/data" "$dest"
  printf 'listen=8443\n' > "$src/config/app.conf"
  printf 'DB_PASSWORD=lab-password-not-secret\n' > "$src/config/.env.production"
  printf 'lab-key-not-secret\n' > "$src/config/SERVER.KEY"
  printf 'lab-ssh-key-not-secret\n' > "$src/.ssh/id_ed25519"
  printf 'login ok token=lab-token-not-secret\n' > "$src/logs/app.log.1"
  seq -f '%g,9.50' 1001 3000 > "$src/data/orders 2026.csv"
}
 
teardown() {
  # Stop a slow run that a failed test left behind, then make everything removable.
  if [[ -n ${pid:-} ]]; then kill -KILL -- "-$pid" 2> /dev/null || true; fi
  chmod -R u+rwX -- "$BATS_TEST_TMPDIR"
}
 
# Fails the test unless DEST holds no archive, checksum or work directory.
assert_nothing_published() {
  run find "$dest" -name '*.tar.gz*' -o -name 'work.*'
  [ "$status" -eq 0 ]
  [ "$output" = "" ]
}
 
# Start a slow backup in its own process group; return once tar is running (at most 10 s).
# 3>&- closes the descriptor Bats reads results from, so a stray run cannot hold it.
start_slow_backup() {
  local i
  truncate -s 2G "$src/data/big.img" # sparse: tar reads 2 GiB of zeros, the disk stores none
  setsid "$script" --dest "$dest" -- "$src" 2> "$BATS_TEST_TMPDIR/slow.err" 3>&- &
  pid=$!
  for ((i = 0; i < 100; i++)); do
    if grep -q archiving "$BATS_TEST_TMPDIR/slow.err"; then return 0; fi
    sleep 0.1
  done
  return 1
}

@test "publishes a verified archive without secrets or logs" {
  run "$script" --dest "$dest" -- "$src"
  [ "$status" -eq 0 ]
  run tar -tzf "$dest"/app-*.tar.gz
  [[ $output == *"app/data/orders 2026.csv"* ]]
  [[ ${output,,} != *".env"* && ${output,,} != *".key"* ]]
  [[ $output != *".ssh"* && $output != *".log"* ]]
  cd "$dest"
  run sha256sum --check -- app-*.tar.gz.sha256
  [ "$status" -eq 0 ]
}
 
@test "--dry-run lists the plan and creates nothing" {
  run "$script" --dry-run --dest "$dest" -- "$src"
  [ "$status" -eq 0 ]
  [[ $output == *"app/config/app.conf"* && $output != *".env"* ]]
  run find "$dest" -mindepth 1
  [ "$status" -eq 0 ]
  [ "$output" = "" ]
}
 
@test "usage errors exit 2" {
  run "$script" --keep 1 --dest "$dest" -- "$src"
  [ "$status" -eq 2 ]
  run "$script" --dest "$src/data" -- "$src"
  [ "$status" -eq 2 ]
  [[ $output == *"--dest must be outside SOURCE"* ]]
}
 
@test "a tar failure publishes nothing and leaves no work directory" {
  mkdir "$src/private"
  chmod 000 "$src/private"
  run "$script" --dest "$dest" -- "$src"
  [ "$status" -eq 1 ]
  [[ $output == *"tar failed"* ]]
  assert_nothing_published
}

@test "a second run while the first holds the lock exits 3" {
  start_slow_backup
  run "$script" --dest "$dest" -- "$src"
  kill -TERM -- "-$pid"
  wait "$pid" || true
  [ "$status" -eq 3 ]
  [[ $output == *"holds $dest/.app.job/lock"* ]]
}
 
@test "SIGTERM mid-run exits 143 and leaves nothing behind" {
  start_slow_backup
  kill -TERM -- "-$pid"
  st=0
  wait "$pid" || st=$?
  [ "$st" -eq 143 ]
  assert_nothing_published
}
 
@test "an archive less than half the size of the last one is refused" {
  run "$script" --dest "$dest" -- "$src"
  [ "$status" -eq 0 ]
  rm -- "$src/data/orders 2026.csv"
  sleep 1 # a new second, so the new archive gets a new name
  run "$script" --dest "$dest" -- "$src"
  [ "$status" -eq 1 ]
  [[ $output == *"check SOURCE"* ]]
  run ls "$dest"
  [ "${#lines[@]}" -eq 2 ] # the first archive and its checksum, nothing else
}

@test "two sources named app keep each other's archives" {
  site2=$BATS_TEST_TMPDIR/site2/app
  mkdir -p "$site2"
  printf 'site 2\n' > "$site2/index.html"
  for stamp in 20260101T000000Z 20260102T000000Z 20260103T000000Z; do
    touch "$dest/site2-app-$stamp.tar.gz"
  done
  run "$script" --keep 2 --dest "$dest" -- "$src"
  [ "$status" -eq 0 ]
  run "$script" --keep 2 --dest "$dest" -- "$site2"
  [ "$status" -eq 2 ]
  [[ $output == *"backs up $src"* ]]
  run "$script" --keep 2 --name site2-app --dest "$dest" -- "$site2"
  [ "$status" -eq 0 ]
  run find "$dest" -name 'app-*.tar.gz'
  [ "${#lines[@]}" -eq 1 ] # the first job's archive survived the second job's pruning
  run find "$dest" -name 'site2-app-*.tar.gz'
  [ "${#lines[@]}" -eq 2 ]
}
 
@test "--keep 2 deletes the oldest archives of this job only" {
  for stamp in 20260101T000000Z 20260102T000000Z 20260103T000000Z; do
    touch "$dest/app-$stamp.tar.gz" "$dest/app-$stamp.tar.gz.sha256"
  done
  touch "$dest/app-data-20260101T000000Z.tar.gz" # another job's archive
  run "$script" --keep 2 --dest "$dest" -- "$src"
  [ "$status" -eq 0 ]
  run ls "$dest"
  [ "${#lines[@]}" -eq 5 ]
  [[ $output == *"app-20260103T000000Z.tar.gz"* && $output != *"app-20260102T000000Z"* ]]
  [[ $output == *"app-data-20260101T000000Z.tar.gz"* ]]
}
 
@test "a source named -rf needs --name, and names with spaces survive" {
  mkdir -p "$BATS_TEST_TMPDIR/-rf/sub dir"
  printf 'keep me\n' > "$BATS_TEST_TMPDIR/-rf/sub dir/my notes.txt"
  cd "$BATS_TEST_TMPDIR"
  run "$script" --dest "$dest" -- -rf
  [ "$status" -eq 2 ]
  run "$script" --dest "$dest" --name notes -- -rf
  [ "$status" -eq 0 ]
  run tar -tzf "$dest"/notes-*.tar.gz
  [[ $output == *"-rf/sub dir/my notes.txt"* ]]
}

* * * * * timeout -k 5m 6h "$HOME/bash-ops/backup/backup.sh" --dest "$HOME/bash-ops/backup/backups" "$HOME/bash-ops/backup/app" >> "$HOME/bash-ops/backup/backup.log" 2>&1; echo "backup.sh exit status $?" >> "$HOME/bash-ops/backup/backup.log"