两台 VPS 每天备份到一台廉价存储箱：restic 的空间账、恢复演练与异常巡检

#!/usr/bin/env bash
# backup-to-box.sh — daily restic snapshot of this host onto the 500 GB backup box.
#
#   config : /etc/backup-to-box.conf      (per host: REPO, PATHS, KEEP_*, hooks)
#   excludes: /etc/backup-to-box.exclude
#   status  : /var/lib/backup-to-box/status.env  +  <box>:/home/<备份用户>/status/<host>.env
#   log     : /var/log/backup-to-box.log
#
# Exit 0 = ok, 1 = failure（systemd 单元显示 failed，巡检脚本据此告警）
# Manual run:  BACKUP_CONF=/etc/backup-to-box.conf DRY_RUN=1 /usr/local/sbin/backup-to-box.sh
set -uo pipefail

# --- defaults that a per-host config may override --------------------------
declare -a PATHS=() PG_CONTAINERS=() MYSQL_CONTAINERS=()

CONF="${BACKUP_CONF:-/etc/backup-to-box.conf}"
[[ -r "$CONF" ]] || { echo "FATAL: $CONF unreadable" >&2; exit 2; }
# shellcheck disable=SC1090
source "$CONF"

RESTIC="${RESTIC:-/usr/local/bin/restic}"
REPO="${REPO:?REPO missing in $CONF}"
HOST_TAG="${HOST_TAG:-$(hostname -s)}"
BOX_HOST="${BOX_HOST:-<存储箱别名>}"
BOX_STATUS_DIR="${BOX_STATUS_DIR:-/home/<备份用户>/status}"
BOX_REPO_DIR="${BOX_REPO_DIR:-/home/<备份用户>/restic}"
export RESTIC_PASSWORD_FILE="${RESTIC_PASSWORD_FILE:-/root/.restic-box-pass}"
export RESTIC_CACHE_DIR="${RESTIC_CACHE_DIR:-/var/cache/restic}"
export RESTIC_PROGRESS_FPS=0

STATE_DIR="${STATE_DIR:-/var/lib/backup-to-box}"
DUMP_DIR="$STATE_DIR/dbdumps"
LOG_FILE="${LOG_FILE:-/var/log/backup-to-box.log}"
STATUS_ENV="$STATE_DIR/status.env"
EXCLUDE_FILE="${EXCLUDE_FILE:-/etc/backup-to-box.exclude}"
KEEP_DAILY="${KEEP_DAILY:-7}"
KEEP_WEEKLY="${KEEP_WEEKLY:-4}"
KEEP_MONTHLY="${KEEP_MONTHLY:-6}"
KEEP_YEARLY="${KEEP_YEARLY:-0}"
CHECK_SUBSET="${CHECK_SUBSET:-2%}"                # weekly `restic check --read-data-subset`
BOX_CAP_BYTES="${BOX_CAP_BYTES:-483183820800}"    # 450 GiB self-imposed cap on the 500 GB plan
MIN_FREE_MB="${MIN_FREE_MB:-2048}"
DRY_RUN="${DRY_RUN:-0}"

[[ ${#PATHS[@]} -eq 0 ]] && PATHS=(/root)

mkdir -p "$STATE_DIR" "$DUMP_DIR" "$RESTIC_CACHE_DIR" "$(dirname "$LOG_FILE")"

# ---- single instance ------------------------------------------------------
exec 9>"$STATE_DIR/run.lock"
if ! flock -n 9; then echo "backup-to-box: another run is in progress, exiting" >&2; exit 0; fi

# ---- logging --------------------------------------------------------------
if [ -f "$LOG_FILE" ] && [ "$(stat -c%s "$LOG_FILE" 2>/dev/null || echo 0)" -gt 5242880 ]; then
  mv -f "$LOG_FILE" "$LOG_FILE.1"
fi
STARTED=$(date -Is); T0=$(date +%s)
STATUS=ok; declare -a WARN=()
say()  { printf '%s %s\n' "$(date '+%F %T%z')" "$*" | tee -a "$LOG_FILE"; }
warn() { WARN+=("$*"); say "WARN: $*"; }
fail() { STATUS=fail; say "FAIL: $*"; }

# ---- helpers --------------------------------------------------------------
dump_pg() {
  local c="$1" out="$DUMP_DIR/pg-${1}.dump" tmp="$DUMP_DIR/pg-${1}.dump.tmp"
  rm -f "$tmp"
  if ! docker exec "$c" sh -c 'exec pg_dump -U "${POSTGRES_USER:-postgres}" -d "${POSTGRES_DB:-postgres}" -Fc' >"$tmp" 2>"$DUMP_DIR/pg-${1}.err"; then
    rm -f "$tmp"; warn "pg_dump $c failed: $(tail -1 "$DUMP_DIR/pg-${1}.err" 2>/dev/null)"; return 1
  fi
  mv -f "$tmp" "$out"; say "  dump $c -> $(du -h "$out" | cut -f1)"
}

dump_mysql() {
  local c="$1" out="$DUMP_DIR/mysql-${1}.sql.gz" tmp="$DUMP_DIR/mysql-${1}.sql.gz.tmp"
  rm -f "$tmp"
  if ! docker exec "$c" sh -c '
        if command -v mariadb-dump >/dev/null 2>&1; then D=mariadb-dump; else D=mysqldump; fi
        exec "$D" -uroot -p"${MARIADB_ROOT_PASSWORD:-${MYSQL_ROOT_PASSWORD:-}}" \
             --single-transaction --quick --skip-lock-tables \
             --all-databases --events --routines --triggers' 2>"$DUMP_DIR/mysql-${1}.err" | gzip -1 >"$tmp"; then
    rm -f "$tmp"; warn "mysqldump $c failed: $(tail -1 "$DUMP_DIR/mysql-${1}.err" 2>/dev/null)"; return 1
  fi
  mv -f "$tmp" "$out"; say "  dump $c -> $(du -h "$out" | cut -f1)"
}

on_exit() {
  local rc=$?
  if declare -F post_backup >/dev/null; then post_backup || true; fi
  local dur=$(( $(date +%s) - T0 ))
  say "===== backup-to-box $HOST_TAG finished: STATUS=$STATUS rc=$rc duration=${dur}s ====="
  exit "$rc"
}
trap on_exit EXIT
# make sure the cleanup hook runs even if the session is torn down mid-run
trap 'exit 129' HUP
trap 'exit 143' TERM
trap 'exit 130' INT

say "===== backup-to-box start host=$HOST_TAG repo=$REPO ====="

# ---- pre ------------------------------------------------------------------
avail_mb=$(df -Pm "$STATE_DIR" | awk 'NR==2{print $4}')
[[ "${avail_mb:-0}" -lt "$MIN_FREE_MB" ]] && fail "only ${avail_mb}MB free on $STATE_DIR (need ${MIN_FREE_MB}MB)"

if declare -F pre_backup >/dev/null; then
  say "--- pre_backup hook ---"
  if ! pre_backup; then fail "pre_backup hook failed — aborting before taking a snapshot"; exit 1; fi
fi

say "--- database dumps ---"
for c in "${PG_CONTAINERS[@]}"; do [ -n "$c" ] && dump_pg "$c"; done
for c in "${MYSQL_CONTAINERS[@]}"; do [ -n "$c" ] && dump_mysql "$c"; done

# ---- backup ---------------------------------------------------------------
say "--- restic backup (paths: ${PATHS[*]}) ---"
JSON="$STATE_DIR/last-backup.json"
SNAP_ID=""; ADDED=0; LOGICAL=0; FILES=0
if [[ "$DRY_RUN" == "1" ]]; then
  "$RESTIC" -r "$REPO" backup "${PATHS[@]}" --exclude-file="$EXCLUDE_FILE" \
     --host "$HOST_TAG" --tag daily --dry-run 2>&1 | tail -20 | tee -a "$LOG_FILE"
else
  rm -f "$JSON"
  "$RESTIC" -r "$REPO" backup "${PATHS[@]}" \
      --exclude-file="$EXCLUDE_FILE" \
      --exclude-caches --exclude-if-present .nobackup \
      --host "$HOST_TAG" --tag daily \
      --json >"$JSON" 2>>"$LOG_FILE"
  rc=$?
  if [[ $rc -ne 0 && $rc -ne 3 ]]; then fail "restic backup exited $rc (see $LOG_FILE)"; fi
  [[ $rc -eq 3 ]] && warn "restic reported unreadable source files (exit 3)"
  SNAP_ID=$(jq -r 'select(.message_type=="summary")|.snapshot_id' "$JSON" 2>/dev/null | tail -1)
  ADDED=$(jq -r 'select(.message_type=="summary")|.data_added' "$JSON" 2>/dev/null | tail -1)
  LOGICAL=$(jq -r 'select(.message_type=="summary")|.total_bytes_processed' "$JSON" 2>/dev/null | tail -1)
  FILES=$(jq -r 'select(.message_type=="summary")|.total_files_processed' "$JSON" 2>/dev/null | tail -1)
  [[ -z "${SNAP_ID:-}" || "${SNAP_ID}" == "null" ]] && fail "no snapshot id in restic output"
  say "  snapshot=${SNAP_ID:-none} files=${FILES:-0} scanned=$(numfmt --to=iec "${LOGICAL:-0}") new-data=$(numfmt --to=iec "${ADDED:-0}")"
fi

# ---- retention ------------------------------------------------------------
# Forget runs every day (rolling delete is immediate); the real `prune`
# repack only runs on Sundays, because pack rewriting over SFTP is the
# slowest step by far (measured 7.5 min) and the box has plenty of slack.
if [[ "$STATUS" == "ok" && "$DRY_RUN" != "1" ]]; then
  if [[ "$(date +%u)" == "7" ]]; then
    PRUNE_ARGS=(--prune); PRUNE_NOTE="weekly prune (Sunday)"
  else
    PRUNE_ARGS=(); PRUNE_NOTE="prune deferred to Sunday"
  fi
  say "--- retention: keep-daily=$KEEP_DAILY keep-weekly=$KEEP_WEEKLY keep-monthly=$KEEP_MONTHLY keep-yearly=$KEEP_YEARLY [$PRUNE_NOTE] ---"
  KEEP_ARGS=(--keep-daily "$KEEP_DAILY" --keep-weekly "$KEEP_WEEKLY" --keep-monthly "$KEEP_MONTHLY")
  [[ "$KEEP_YEARLY" -gt 0 ]] && KEEP_ARGS+=(--keep-yearly "$KEEP_YEARLY")
  FORGET_OUT="$STATE_DIR/last-forget.txt"
  if ! "$RESTIC" -r "$REPO" forget "${KEEP_ARGS[@]}" "${PRUNE_ARGS[@]}" --host "$HOST_TAG" >"$FORGET_OUT" 2>&1; then
    warn "restic forget/prune failed (see $FORGET_OUT)"
  else
    SUMMARY=$(grep -E 'snapshots have been removed|no snapshots were removed' "$FORGET_OUT" | tail -1)
    [[ -z "$SUMMARY" ]] && SUMMARY=$(grep -E '^keep [0-9]+ snapshots' "$FORGET_OUT" | tail -1)
    say "  ${SUMMARY:-forget finished}"
  fi
  cat "$FORGET_OUT" >>"$LOG_FILE"
fi

# ---- weekly integrity check ----------------------------------------------
if [[ "$STATUS" == "ok" && "$DRY_RUN" != "1" && "$(date +%u)" == "7" ]]; then
  say "--- weekly restic check (--read-data-subset=$CHECK_SUBSET) ---"
  "$RESTIC" -r "$REPO" check --read-data-subset="$CHECK_SUBSET" >>"$LOG_FILE" 2>&1 || warn "restic check reported problems"
fi

# ---- stats ----------------------------------------------------------------
SNAPSHOTS=0; REPO_RAW=0; BOX_BYTES=0; BOX_TOTAL=0
if [[ "$DRY_RUN" != "1" ]]; then
  SNAPSHOTS=$("$RESTIC" -r "$REPO" snapshots --json 2>/dev/null | jq 'length' 2>/dev/null || echo 0)
  REPO_RAW=$("$RESTIC" -r "$REPO" stats --json 2>/dev/null | jq -r '.total_size' 2>/dev/null || echo 0)
  BOX_BYTES=$(ssh -o BatchMode=yes -o ConnectTimeout=25 "$BOX_HOST" "du -sb $BOX_REPO_DIR/$HOST_TAG 2>/dev/null | cut -f1" 2>/dev/null || echo 0)
  BOX_TOTAL=$(ssh -o BatchMode=yes -o ConnectTimeout=25 "$BOX_HOST" "du -sb $BOX_REPO_DIR 2>/dev/null | cut -f1" 2>/dev/null || echo 0)
  say "  snapshots=$SNAPSHOTS repo-raw-data=$(numfmt --to=iec "${REPO_RAW:-0}") box-repo=$(numfmt --to=iec "${BOX_BYTES:-0}") box-total=$(numfmt --to=iec "${BOX_TOTAL:-0}")"
  if [[ "${BOX_TOTAL:-0}" -gt "$BOX_CAP_BYTES" ]]; then
    warn "backup box usage $(numfmt --to=iec "${BOX_TOTAL:-0}") exceeds the self-imposed cap $(numfmt --to=iec "$BOX_CAP_BYTES") — shorten retention"
  fi
fi

# ---- status file (local + on the box) ------------------------------------
cat > "$STATUS_ENV" <<EOF
HOST=$HOST_TAG
STATUS=$STATUS
STARTED=$STARTED
FINISHED=$(date -Is)
SNAPSHOT=${SNAP_ID:-}
FILES=${FILES:-0}
SCANNED_BYTES=${LOGICAL:-0}
ADDED_BYTES=${ADDED:-0}
REPO_RAW_BYTES=${REPO_RAW:-0}
BOX_BYTES=${BOX_BYTES:-0}
BOX_TOTAL_BYTES=${BOX_TOTAL:-0}
SNAPSHOTS=${SNAPSHOTS:-0}
KEEP_DAILY=$KEEP_DAILY
DURATION_S=$(( $(date +%s) - T0 ))
WARN=${WARN[*]:-}
EOF
if ! ssh -o BatchMode=yes -o ConnectTimeout=25 "$BOX_HOST" "cat > $BOX_STATUS_DIR/$HOST_TAG.env" <"$STATUS_ENV" 2>>"$LOG_FILE"; then
  warn "could not publish status file to the backup box"
fi

if [[ "$STATUS" == "ok" ]]; then exit 0; else exit 1; fi

# /etc/backup-to-box.conf — <本机代号>
# Daily restic snapshot to the 500 GB backup box.
# 每台机器一份，改 REPO / HOST_TAG / PATHS / 容器名单即可

REPO="sftp://<备份用户>@<存储箱地址>:<端口>//home/<备份用户>/restic/<主机代号>"
HOST_TAG="<主机代号>"
BOX_HOST="<存储箱别名>"
EXCLUDE_FILE="/etc/backup-to-box.exclude"

PATHS=(
  /root
  /data
  /etc
  /usr/local
  /var/spool/cron
  /var/lib/docker/volumes
  /var/lib/backup-to-box
)

# databases dumped to /var/lib/backup-to-box/dbdumps before the snapshot
PG_CONTAINERS=(<pg容器1> <pg容器2> <pg容器3> <pg容器4>)   # 换成你自己的 PostgreSQL 容器名
MYSQL_CONTAINERS=(<mysql容器1> <mysql容器2> <mysql容器3>)  # docker ps 里列出来

# retention（容量估算见文中“空间账”一节）
KEEP_DAILY=30
KEEP_WEEKLY=12
KEEP_MONTHLY=12
KEEP_YEARLY=0

# /etc/backup-to-box.exclude — <主机代号>  (restic exclude file: one pattern per line, # = comment)
# Raw database data directories (their contents arrive as pg_dump/mysqldump files instead)
/srv/docker-data/<服务甲>/db
/srv/docker-data/<服务乙>/db
/srv/docker-data/<服务丙>/db
/srv/docker-data/<反向代理>/mysql
# regenerable / bulk material
/srv/docker-data/<反向代理>/data/logs
/srv/docker-data/<反向代理>/letsencrypt-acme-challenge
/srv/docker-data/<在线文档>/onlyoffice-dist
/srv/docker-data/<自建服务>/data/bin
/srv/docker-data/<自建服务>/data/lsp
/srv/docker-data/<自建服务>/data/lazy-packages
/srv/docker-data/<自建服务>/data/logs
/srv/docker-data/<自建服务>/data/models_dev_cache.json
/srv/docker-data/<自建服务>/data/models_dev_cache.json.corrupt
# generic caches / logs
.cache
.npm
node_modules
*.log
*.log.gz
*.pyc
__pycache__
*.tmp
# docker internals that must never travel
/var/lib/docker/volumes/*/backingFsBlockDev
/var/lib/docker/volumes/metadata.db

#!/usr/bin/env bash
# backup-box-watch.sh — anomaly watchdog for the backup box.
# 只在一台机器上跑。 Reads the status files both hosts publish to the box and
# 只有异常才推送到通知通道。
#   * backup reported a failure
#   * last successful run older than MAX_AGE_H
#   * repos together exceed the self-imposed cap (default 450 GiB of a 500 GB plan)
set -uo pipefail

BOX_HOST="${BOX_HOST:-<存储箱别名>}"
STATUS_DIR="${BOX_STATUS_DIR:-/home/<备份用户>/status}"
REPO_DIR="${BOX_REPO_DIR:-/home/<备份用户>/restic}"
HOSTS=(${WATCH_HOSTS:-<主机甲> <主机乙>})
MAX_AGE_H="${MAX_AGE_H:-30}"
CAP_BYTES="${CAP_BYTES:-483183820800}"
TG=/usr/local/sbin/notify.sh    # 换成你自己的通知脚本（收一个参数：消息文本）
LOG=/var/log/backup-box-watch.log

ALERTS=()
log() { printf '%s %s\n' "$(date '+%F %T%z')" "$*" >>"$LOG"; }

total_used=""
for h in "${HOSTS[@]}"; do
  txt=$(ssh -o BatchMode=yes -o ConnectTimeout=25 -o StrictHostKeyChecking=accept-new \
            "$BOX_HOST" "cat $STATUS_DIR/$h.env 2>/dev/null" 2>/dev/null)
  if [[ -z "${txt//[[:space:]]/}" ]]; then
    ALERTS+=("❌ $h：备份箱上没有状态文件（该机备份从未成功或状态目录被清）")
    continue
  fi
  get() { printf '%s\n' "$txt" | sed -n "s/^$1=//p" | head -1; }
  st=$(get STATUS); fin=$(get FINISHED); snaps=$(get SNAPSHOTS); \
  box=$(get BOX_BYTES); warn=$(get WARN)

  if [[ "$st" != "ok" ]]; then
    ALERTS+=("❌ $h：最近一次备份失败（STATUS=$st，WARN=${warn:-none}）")
  fi
  if [[ -n "$fin" ]]; then
    ep=$(date -d "$fin" +%s 2>/dev/null || echo 0)
    if [[ "$ep" -gt 0 ]]; then
      age_h=$(( ( $(date +%s) - ep ) / 3600 ))
      if [[ "$age_h" -gt "$MAX_AGE_H" ]]; then
        ALERTS+=("❌ $h：已 ${age_h}h 没有成功备份（最后成功 $fin）")
      fi
    fi
  fi
  log "host=$h status=$st finished=$fin snapshots=$snaps box_bytes=${box:-?}"
done

used=$(ssh -o BatchMode=yes -o ConnectTimeout=25 "$BOX_HOST" "du -sb $REPO_DIR 2>/dev/null | cut -f1" 2>/dev/null || echo "")
if [[ -n "$used" ]]; then
  total_used=$used
  if [[ "$used" -gt "$CAP_BYTES" ]]; then
    ALERTS+=("⚠️ 备份箱占用 $(numfmt --to=iec "$used") 已超过自设上限 $(numfmt --to=iec "$CAP_BYTES")，请缩短保留策略")
  fi
  log "box total used=$(numfmt --to=iec "$used")"
else
  ALERTS+=("❌ 无法连接备份箱 $BOX_HOST，无法核对备份状态")
fi

if [[ ${#ALERTS[@]} -gt 0 ]]; then
  msg="🗄 备份箱巡检异常 $(date '+%F %H:%M %Z')
$(printf '%s\n' "${ALERTS[@]}")
当前占用：${total_used:+$(numfmt --to=iec "$total_used")}"
  "$TG" "$msg" >>"$LOG" 2>&1
  log "alert sent"
else
  log "all good (box used=$(numfmt --to=iec "${total_used:-0}"))"
fi