backup_OBS/backup.sh

#!/usr/bin/env bash
# 备份并上传到华为云 OBS
# - 从同目录 env.conf 加载配置，支持命令行覆盖
# - 打包多目录（支持 pigz），生成 SHA256 校验，obsutil 分片/并发上传
# - 上传 .tar.gz 以及 .tar.gz.sha256，并下载远端 .sha256 比对确保完整性
# - 成功/失败写入日志并 Telegram 通知
# - 支持 --dry-run / --verify / --keep-local / --concurrency / --rate / --label / --sse
# 退出码：0 成功；2 配置缺失；3 打包失败；4 上传失败；5 校验失败

set -Eeuo pipefail

SCRIPT_PATH="${BASH_SOURCE[0]:-$0}"
SCRIPT_DIR="$(cd "$(dirname -- "$SCRIPT_PATH")" && pwd -P)"
export PATH="$SCRIPT_DIR:$PATH"
cd "$SCRIPT_DIR"

LOG_PREFIX="[BACKUP]"
log_ts() { date '+%Y-%m-%d %H:%M:%S'; }
log() {
  local level="$1"; shift
  local line
  line="$(log_ts) ${LOG_PREFIX}[$level] $*"
  echo "$line" | tee -a "${LOG_FILE:-./backup.log}"
}

# shellcheck disable=SC1091
source "${SCRIPT_DIR}/lib/telegram.sh"

RUNTIME_EXCLUDE_FILE=""

cleanup_tmp() {
  [[ -n "${RUNTIME_EXCLUDE_FILE:-}" ]] && rm -f "$RUNTIME_EXCLUDE_FILE"
}
trap cleanup_tmp EXIT

on_error() {
  local exit_code=$?
  local line_no=${BASH_LINENO[0]:-}
  local tail_log
  log "ERROR" "脚本异常退出（exit=$exit_code, line=${line_no})."
  tail_log="$(tail -n 80 "${LOG_FILE:-./backup.log}" 2>/dev/null || true)"
  tg_backup_error "$LABEL" "$(date +'%Y-%m-%d %H:%M:%S')" "脚本异常退出（exit=${exit_code}, line=${line_no}）" "$tail_log"
  exit "$exit_code"
}
trap on_error ERR

need_cmd() {
  command -v "$1" >/dev/null 2>&1 || {
    log "ERROR" "未找到命令：$1"
    return 1
  }
}

retry() {
  local max="$1"; shift
  local desc="$1"; shift
  local n=1
  until "$@"; do
    local ec=$?
    if (( n >= max )); then
      log "ERROR" "重试用尽：${desc}（exit=$ec）"
      return "$ec"
    fi
    local backoff=$((2**n))
    log "WARN" "步骤失败，将在 ${backoff}s 后第 $((n+1)) 次重试：${desc}（exit=$ec）"
    sleep "$backoff"
    ((n++))
  done
}

CONF_FILE="${SCRIPT_DIR}/env.conf"
if [[ ! -f "$CONF_FILE" ]]; then
  echo "未找到配置文件 ${CONF_FILE}，请先复制/编辑 env.conf" >&2
  exit 2
fi

# shellcheck disable=SC1090
source "$CONF_FILE"

BACKUP_PATHS=()

KEEP_LOCAL=false
DRY_RUN=false
VERIFY_ONLY=false
CLI_LABEL=""
CLI_SSE="none"
CLI_CONCURRENCY=""
CLI_RATE=""
GPG_RECIPIENT=""
GPG_SYM=false

usage() {
  cat <<'USAGE'
用法：
  ./backup.sh [--dry-run] [--verify] [--keep-local]
              [--concurrency=N] [--rate=10MB/s]
              [--label=NAME]
              [--sse=none|kms|aes256]
              [--gpg-recipient=KMS_OR_PGP_ID | --gpg-symmetric]
USAGE
}

while [[ $# -gt 0 ]]; do
  case "$1" in
    --dry-run) DRY_RUN=true; shift;;
    --verify) VERIFY_ONLY=true; shift;;
    --keep-local) KEEP_LOCAL=true; shift;;
    --concurrency=*) CLI_CONCURRENCY="${1#*=}"; shift;;
    --rate=*) CLI_RATE="${1#*=}"; shift;;
    --label=*) CLI_LABEL="${1#*=}"; shift;;
    --sse=*) CLI_SSE="${1#*=}"; shift;;
    --gpg-recipient=*) GPG_RECIPIENT="${1#*=}"; shift;;
    --gpg-symmetric) GPG_SYM=true; shift;;
    -h|--help) usage; exit 0;;
    *) echo "未知参数：$1"; usage; exit 2;;
  esac
done

if [[ -n "$CLI_LABEL" ]]; then
  LABEL="$CLI_LABEL"
  OBS_BACKUP_DIR="backup/${LABEL}"
fi

if [[ -n "$CLI_CONCURRENCY" ]]; then
  OBS_JOBS="$CLI_CONCURRENCY"
fi

OBS_RATE_FLAG=""
if [[ -n "${CLI_RATE:-}" ]]; then
  OBS_RATE_FLAG="--rate=${CLI_RATE}"
fi

DEFAULT_EXCLUDE_FILE="${SCRIPT_DIR}/exclude.list"
USER_EXCLUDE_FILE="${SCRIPT_DIR}/exclude.user.list"
[[ -f "$DEFAULT_EXCLUDE_FILE" ]] || touch "$DEFAULT_EXCLUDE_FILE"
[[ -f "$USER_EXCLUDE_FILE" ]] || cp -f "${SCRIPT_DIR}/exclude.user.list.example" "$USER_EXCLUDE_FILE"

: "${OBS_BUCKET:?缺少 OBS_BUCKET}"
: "${OBS_BACKUP_DIR:?缺少 OBS_BACKUP_DIR}"
: "${LOG_FILE:?缺少 LOG_FILE}"
: "${OBS_PARALLEL:?缺少 OBS_PARALLEL}" "${OBS_JOBS:?缺少 OBS_JOBS}"
: "${OBS_PARTSIZE:?缺少 OBS_PARTSIZE}" "${OBS_THRESHOLD:?缺少 OBS_THRESHOLD}"
: "${RETAIN_DAYS:?缺少 RETAIN_DAYS}"

if [[ -z "${BACKUP_DIRS:-}" && -n "${BACKUP_DIR:-}" ]]; then
  BACKUP_DIRS="$(printf '%s\n' "$BACKUP_DIR")"
fi

if [[ -z "${BACKUP_DIRS:-}" ]]; then
  echo "缺少 BACKUP_DIRS 配置" >&2
  exit 2
fi

while IFS= read -r path; do
  [[ -n "$path" ]] || continue
  [[ "$path" =~ ^[[:space:]]*# ]] && continue
  BACKUP_PATHS+=("$path")
done <<< "$BACKUP_DIRS"

if [[ "${#BACKUP_PATHS[@]}" -eq 0 ]]; then
  echo "BACKUP_DIRS 没有有效目录" >&2
  exit 2
fi

need_cmd tar
need_cmd sha256sum
need_cmd curl
need_cmd bash

if ! command -v obsutil >/dev/null 2>&1; then
  log "ERROR" "未找到 obsutil。请下载 obsutil，并确保其在 PATH 或与脚本同目录。"
  log "ERROR" "首次使用请执行：./obsutil config -interactive"
  exit 2
fi

OBSUTIL_CONFIG_PATH="${HOME}/.obsutilconfig"
if [[ ! -f "$OBSUTIL_CONFIG_PATH" ]]; then
  log "ERROR" "未检测到 obsutil 配置文件：${OBSUTIL_CONFIG_PATH}"
  log "ERROR" "请先执行：./obsutil config -interactive 完成初始化（AK/SK、Endpoint、默认桶）"
  exit 2
fi

if $VERIFY_ONLY; then
  log "INFO" "开始远端可用性检查（ls + stat）"
  set +e
  obsutil ls "obs://${OBS_BUCKET}/${OBS_BACKUP_DIR}/" > /dev/null 2>&1
  rc_ls=$?
  if (( rc_ls != 0 )); then
    log "ERROR" "obsutil 无法列举 obs://${OBS_BUCKET}/${OBS_BACKUP_DIR}/"
    exit 5
  fi
  one_obj="$(obsutil ls "obs://${OBS_BUCKET}/${OBS_BACKUP_DIR}/" 2>/dev/null | awk '/obs:\/\//{print $1; exit}')"
  if [[ -n "$one_obj" ]]; then
    obsutil stat "$one_obj" >/dev/null 2>&1 || { log "ERROR" "stat 失败：$one_obj"; exit 5; }
    log "INFO" "远端可访问，示例对象 stat 正常：$one_obj"
  else
    log "INFO" "远端可访问，但当前前缀下没有对象。"
  fi
  exit 0
fi
set -e

build_tar_file_list() {
  local args=()
  local p abs parent base
  for p in "${BACKUP_PATHS[@]}"; do
    if [[ ! -e "$p" ]]; then
      log "WARN" "路径不存在，跳过：$p"
      continue
    fi
    abs="$(readlink -f "$p")"
    parent="$(dirname "$abs")"
    base="$(basename "$abs")"
    args+=("-C" "$parent" "$base")
  done
  printf '%s\n' "${args[@]}"
}

archive_path_for_abs() {
  # 将真实文件系统绝对路径映射为 tar 归档内路径。
  local abs_input="$1"
  local root abs_root root_base rel

  for root in "${BACKUP_PATHS[@]}"; do
    abs_root="$(readlink -f "$root")"
    root_base="$(basename "$abs_root")"

    if [[ "$abs_input" == "$abs_root" ]]; then
      printf '%s\n' "$root_base"
      return 0
    fi

    if [[ "$abs_input" == "$abs_root"/* ]]; then
      rel="${abs_input#"$abs_root"/}"
      printf '%s/%s\n' "$root_base" "$rel"
      return 0
    fi
  done

  return 1
}

append_exclude_path() {
  local archive_path="$1"
  local path_type="$2"

  printf '%s\n' "$archive_path" >> "$RUNTIME_EXCLUDE_FILE"
  if [[ "$path_type" == "dir" ]]; then
    printf '%s/*\n' "$archive_path" >> "$RUNTIME_EXCLUDE_FILE"
  fi
}

match_user_rule() {
  # 用户规则支持两类写法：
  # 1. 绝对路径，如 /root/data/dify
  # 2. 通配规则，如 */.halo
  # 解析后统一转换成 tar 能识别的实际排除项。
  local rule="$1"
  local matched=0
  local root abs_root root_base entry archive_path path_type
  local rule_tail find_name has_glob

  if [[ "$rule" == /* ]]; then
    if [[ ! -e "$rule" ]]; then
      log "WARN" "用户排除规则未命中任何现有路径：$rule"
      return 0
    fi

    if ! archive_path="$(archive_path_for_abs "$(readlink -f "$rule")")"; then
      log "WARN" "用户排除规则不在任何备份根目录内：$rule"
      return 0
    fi

    if [[ -d "$rule" ]]; then
      path_type="dir"
    else
      path_type="file"
    fi
    append_exclude_path "$archive_path" "$path_type"
    return 0
  fi

  has_glob=false
  if [[ "$rule" == *'*'* || "$rule" == *'?'* || "$rule" == *'['* ]]; then
    has_glob=true
  fi

  for root in "${BACKUP_PATHS[@]}"; do
    abs_root="$(readlink -f "$root")"
    root_base="$(basename "$abs_root")"

    if ! $has_glob; then
      entry=""
      if [[ "$rule" == "$root_base" ]]; then
        entry="$abs_root"
      elif [[ "$rule" == "$root_base"/* ]]; then
        entry="${abs_root}/${rule#"$root_base"/}"
      fi

      if [[ -n "$entry" && -e "$entry" ]]; then
        matched=1
        if [[ -d "$entry" ]]; then
          path_type="dir"
        else
          path_type="file"
        fi
        append_exclude_path "$rule" "$path_type"
      fi
      continue
    fi

    rule_tail="${rule##*/}"
    if [[ "$rule_tail" == *'*'* || "$rule_tail" == *'?'* || "$rule_tail" == *'['* ]]; then
      while IFS= read -r -d '' entry; do
        archive_path="$(archive_path_for_abs "$entry" || true)"
        [[ -n "$archive_path" ]] || continue
        if [[ "$archive_path" == $rule ]]; then
          matched=1
          if [[ -d "$entry" ]]; then
            path_type="dir"
          else
            path_type="file"
          fi
          append_exclude_path "$archive_path" "$path_type"
        fi
      done < <(find "$abs_root" -mindepth 1 -print0 2>/dev/null)
    else
      find_name="$rule_tail"
      while IFS= read -r -d '' entry; do
        archive_path="$(archive_path_for_abs "$entry" || true)"
        [[ -n "$archive_path" ]] || continue
        if [[ "$archive_path" == $rule ]]; then
          matched=1
          if [[ -d "$entry" ]]; then
            path_type="dir"
          else
            path_type="file"
          fi
          append_exclude_path "$archive_path" "$path_type"
        fi
      done < <(find "$abs_root" -mindepth 1 -name "$find_name" -print0 2>/dev/null)
    fi
  done

  if (( matched == 0 )); then
    log "WARN" "用户排除规则未命中任何路径：$rule"
  fi
}

build_runtime_exclude_file() {
  # 默认规则直接交给 tar。
  # 用户规则先解析成真实命中路径，再换算为归档内排除项。
  local rule
  RUNTIME_EXCLUDE_FILE="$(mktemp)"
  cat "$DEFAULT_EXCLUDE_FILE" > "$RUNTIME_EXCLUDE_FILE"

  [[ -f "$USER_EXCLUDE_FILE" ]] || return 0

  while IFS= read -r rule; do
    [[ -n "$rule" ]] || continue
    [[ "$rule" =~ ^[[:space:]]*# ]] && continue
    match_user_rule "$rule"
  done < "$USER_EXCLUDE_FILE"

  awk '!seen[$0]++' "$RUNTIME_EXCLUDE_FILE" > "${RUNTIME_EXCLUDE_FILE}.dedup"
  mv "${RUNTIME_EXCLUDE_FILE}.dedup" "$RUNTIME_EXCLUDE_FILE"
}

check_space() {
  local total=0
  local p sz avail need
  for p in "${BACKUP_PATHS[@]}"; do
    [[ -e "$p" ]] || continue
    sz="$(du -sb --apparent-size "$p" 2>/dev/null | cut -f1)"
    total=$((total + sz))
  done
  avail=$(df -P "$SCRIPT_DIR" | awk 'NR==2{print $4}')
  avail=$((avail * 1024))
  need=$(( total + total/10 + 100*1024*1024 ))
  if (( avail < need )); then
    log "ERROR" "磁盘空间不足。需要约 $need 字节，可用 $avail 字节。"
    return 1
  fi
}

log "INFO" "开始备份：bucket=${OBS_BUCKET} prefix=${OBS_BACKUP_DIR} 并发(-j)=${OBS_JOBS} 分片(-p)=${OBS_PARALLEL}"
check_space || { log "ERROR" "空间检查未通过"; exit 3; }

TS="$(date +%Y%m%d_%H%M%S)"
ARCHIVE_BASE="backup_${TS}.tar.gz"
ARCHIVE_PATH="${SCRIPT_DIR}/${ARCHIVE_BASE}"
SHA_PATH="${ARCHIVE_PATH}.sha256"
USE_PIGZ=false
if command -v pigz >/dev/null 2>&1; then USE_PIGZ=true; fi

pack() {
  local start end
  local tar_args
  start=$(date +%s)
  mapfile -t tar_args < <(build_tar_file_list)
  if [[ "${#tar_args[@]}" -eq 0 ]]; then
    log "ERROR" "没有可打包的路径（BACKUP_DIRS 均不存在）"
    return 1
  fi

  log "INFO" "开始打包（支持默认与用户排除规则），pigz=${USE_PIGZ}"
  build_runtime_exclude_file
  if $USE_PIGZ; then
    tar --warning=no-file-changed \
      --exclude-from="$RUNTIME_EXCLUDE_FILE" \
      -c "${tar_args[@]}" | pigz > "$ARCHIVE_PATH"
  else
    tar --warning=no-file-changed \
      --exclude-from="$RUNTIME_EXCLUDE_FILE" \
      -czf "$ARCHIVE_PATH" "${tar_args[@]}"
  fi
  end=$(date +%s)
  PACK_SECONDS=$((end - start))
  PACK_SIZE="$(stat -c '%s' "$ARCHIVE_PATH")"
  BACKUP_SIZE_HUMAN="$(numfmt --to=iec "$PACK_SIZE")"
  log "INFO" "打包完成：${BACKUP_SIZE_HUMAN} 用时 ${PACK_SECONDS}s"
}

maybe_encrypt() {
  if [[ -n "$GPG_RECIPIENT" ]]; then
    log "INFO" "使用 GPG 收件人加密：$GPG_RECIPIENT"
    need_cmd gpg
    gpg --batch --yes --output "${ARCHIVE_PATH}.gpg" --encrypt --recipient "$GPG_RECIPIENT" "$ARCHIVE_PATH"
    rm -f "$ARCHIVE_PATH"
    ARCHIVE_PATH="${ARCHIVE_PATH}.gpg"
    ARCHIVE_BASE="${ARCHIVE_BASE}.gpg"
  elif $GPG_SYM; then
    log "INFO" "使用 GPG 对称加密"
    need_cmd gpg
    if [[ -n "${GPG_PASSPHRASE:-}" ]]; then
      gpg --batch --yes --passphrase "$GPG_PASSPHRASE" --symmetric --output "${ARCHIVE_PATH}.gpg" "$ARCHIVE_PATH"
    else
      gpg --batch --yes --symmetric --output "${ARCHIVE_PATH}.gpg" "$ARCHIVE_PATH"
    fi
    rm -f "$ARCHIVE_PATH"
    ARCHIVE_PATH="${ARCHIVE_PATH}.gpg"
    ARCHIVE_BASE="${ARCHIVE_BASE}.gpg"
  fi

  PACK_SIZE="$(stat -c '%s' "$ARCHIVE_PATH")"
  BACKUP_SIZE_HUMAN="$(numfmt --to=iec "$PACK_SIZE")"
}

gen_sha() {
  sha256sum "$ARCHIVE_PATH" > "$SHA_PATH"
  log "INFO" "已生成校验文件：${SHA_PATH##*/}"
}

build_obs_flags() {
  local flags=()
  flags+=("-f")
  flags+=("-p=${OBS_PARALLEL}")
  flags+=("-j=${OBS_JOBS}")
  flags+=("-ps=${OBS_PARTSIZE}")
  flags+=("-threshold=${OBS_THRESHOLD}")
  [[ -n "$OBS_RATE_FLAG" ]] && flags+=("$OBS_RATE_FLAG")
  case "$CLI_SSE" in
    aes256|AES256) flags+=("-h=x-obs-server-side-encryption:AES256");;
    kms|KMS)
      if [[ -z "${OBS_KMS_KEY_ID:-}" ]]; then
        log "WARN" "已选择 SSE-KMS，但未设置 OBS_KMS_KEY_ID，将只设置 kms 算法头。"
        flags+=("-h=x-obs-server-side-encryption:kms")
      else
        flags+=("-h=x-obs-server-side-encryption:kms")
        flags+=("-h=x-obs-server-side-encryption-kms-key-id:${OBS_KMS_KEY_ID}")
      fi
      ;;
    none|None|"") ;;
    *) log "WARN" "未知 --sse=$CLI_SSE，忽略。";;
  esac
  printf '%s\n' "${flags[@]}"
}

upload() {
  local start end dst_dir
  local flags
  start=$(date +%s)
  dst_dir="obs://${OBS_BUCKET}/${OBS_BACKUP_DIR}/"
  mapfile -t flags < <(build_obs_flags)
  retry 3 "上传归档 ${ARCHIVE_BASE}" obsutil cp "$ARCHIVE_PATH" "${dst_dir}${ARCHIVE_BASE}" "${flags[@]}"
  retry 3 "上传校验 ${ARCHIVE_BASE}.sha256" obsutil cp "$SHA_PATH" "${dst_dir}${ARCHIVE_BASE}.sha256" "${flags[@]}"
  end=$(date +%s)
  UP_SECONDS=$((end - start))
  if (( UP_SECONDS > 0 )); then
    AVG_SPEED="$(( PACK_SIZE / UP_SECONDS ))"
  else
    AVG_SPEED="$PACK_SIZE"
  fi
}

verify_remote() {
  local tmp_remote dst
  tmp_remote="${SHA_PATH}.remote"
  rm -f "$tmp_remote"
  dst="obs://${OBS_BUCKET}/${OBS_BACKUP_DIR}/${ARCHIVE_BASE}.sha256"
  retry 3 "下载远端校验 ${ARCHIVE_BASE}.sha256" obsutil cp "$dst" "$tmp_remote" -f >/dev/null
  if ! cmp -s "$SHA_PATH" "$tmp_remote"; then
    log "ERROR" "远端 .sha256 与本地不一致"
    return 1
  fi
  rm -f "$tmp_remote"
  log "INFO" "远端完整性校验通过（.sha256 一致）"
}

SECONDS=0
PACK_SECONDS=0
UP_SECONDS=0
AVG_SPEED=0
PACK_SIZE=0
BACKUP_SIZE_HUMAN=""

log "INFO" "准备开始备份。DRY_RUN=${DRY_RUN} KEEP_LOCAL=${KEEP_LOCAL} SSE=${CLI_SSE}"

retry 3 "打包与压缩" pack || { log "ERROR" "打包失败"; exit 3; }
maybe_encrypt
gen_sha

if $DRY_RUN; then
  log "INFO" "Dry-run 模式：不执行上传。归档：${ARCHIVE_BASE} 大小：${BACKUP_SIZE_HUMAN}"
  log "INFO" "本次总用时：${SECONDS}s"
  tg_backup_dryrun "$LABEL" "$(date +'%Y-%m-%d %H:%M:%S')" "$(printf '%s\n' "${BACKUP_PATHS[@]}")" "$(tg_read_user_excludes "$USER_EXCLUDE_FILE")" "$ARCHIVE_BASE" "$BACKUP_SIZE_HUMAN" "${PACK_SECONDS}s"
  exit 0
fi

retry 3 "上传到 OBS" upload || { log "ERROR" "上传失败"; exit 4; }

if ! verify_remote; then
  local_tail_log="$(tail -n 80 "${LOG_FILE:-./backup.log}" 2>/dev/null || true)"
  log "ERROR" "上传后完整性校验失败"
  tg_backup_error "$LABEL" "$(date +'%Y-%m-%d %H:%M:%S')" "上传后完整性校验失败：${ARCHIVE_BASE}" "$local_tail_log"
  exit 5
fi

if ! $KEEP_LOCAL; then
  rm -f "$ARCHIVE_PATH" "$SHA_PATH" || true
  log "INFO" "已删除本地临时归档与校验文件"
fi

TOTAL_SECS=$SECONDS
SPEED_HUMAN="$(numfmt --to=iec "$AVG_SPEED")/s"
log "INFO" "完成：归档大小=${BACKUP_SIZE_HUMAN} 打包=${PACK_SECONDS}s 上传=${UP_SECONDS}s 平均速率=${SPEED_HUMAN} 总用时=${TOTAL_SECS}s"
tg_backup_success "$LABEL" "$(date +'%Y-%m-%d %H:%M:%S')" "$(printf '%s\n' "${BACKUP_PATHS[@]}")" "$(tg_read_user_excludes "$USER_EXCLUDE_FILE")" "$BACKUP_SIZE_HUMAN" "$ARCHIVE_BASE" "obs://${OBS_BUCKET}/${OBS_BACKUP_DIR}/${ARCHIVE_BASE}" "${PACK_SECONDS}s" "${UP_SECONDS}s" "$SPEED_HUMAN"

exit 0