diff --git a/app/.env.example b/app/.env.example index 92931ae..91c963a 100644 --- a/app/.env.example +++ b/app/.env.example @@ -5,7 +5,11 @@ PYTHON_IMAGE=python:3.12-slim BASIC_AUTH_USERNAME=wolfydw BASIC_AUTH_PASSWORD=change-me ADMIN_AUTH_PASSWORD=change-me +INGEST_AUTH_TOKEN=change-this-ingest-token CLASSNOTES_PATH=/data/classnotes.txt ACCOUNTS_PATH=/data/学生课时账户.md ADMIN_TASKS_PATH=/data/admin_tasks.json +COURSE_SUMMARIES_ROOT=/data/course_summaries +COURSE_SUMMARY_STATE_PATH=/data/course_summary_state.json +OPERATION_LOGS_PATH=/data/operation_logs.jsonl diff --git a/app/Dockerfile b/app/Dockerfile index c64e1a1..8a618c2 100644 --- a/app/Dockerfile +++ b/app/Dockerfile @@ -10,6 +10,7 @@ COPY requirements.txt . RUN pip install --no-cache-dir -r requirements.txt COPY app ./app +COPY scripts ./scripts EXPOSE 8000 diff --git a/app/MAINTENANCE.md b/app/MAINTENANCE.md index 9d41eb1..ce5e140 100644 --- a/app/MAINTENANCE.md +++ b/app/MAINTENANCE.md @@ -39,7 +39,7 @@ - `make check`：检查前端 JS 语法和后端 Python 编译。 - `make smoke`：运行轻量前端行为烟测，覆盖排序按钮、组内排序和纠错后排序。 -- `make data-hash`：输出 `classnotes.txt`、`学生课时账户.md`，以及存在时的 `admin_tasks.json` 的 SHA-256。 +- `make data-hash`：输出 `classnotes.txt`、`学生课时账户.md`，以及存在时的 `admin_tasks.json`、`course_summary_state.json` 的 SHA-256。 - `make build`：构建 Docker 镜像。 - `make up`：重启 Docker Compose 服务。 - `make health`：带认证访问 `/api/health`。 @@ -78,9 +78,12 @@ git push origin HEAD:<当前分支> /root/新时空数据/data/classnotes.txt /root/新时空数据/data/学生课时账户.md /root/新时空数据/data/admin_tasks.json +/root/新时空数据/data/course_summaries/ +/root/新时空数据/data/course_summary_state.json +/root/新时空数据/data/operation_logs.jsonl ``` -普通前端和查询类改动不应该改变这些文件。部署前后 `make data-hash` 输出应一致；如果涉及登记 API、课时账户编辑或纠错审核批准，先确认自动备份目录： +普通前端和查询类改动不应该改变这些文件。部署前后 `make data-hash` 输出应一致；如果涉及登记 API、课时账户编辑、课程小结自动入账或审核批准，先确认自动备份目录： ```text /root/新时空数据/data/backups/ @@ -114,6 +117,14 @@ git push origin HEAD:<当前分支> 如果业务数据哈希异常，先不要继续写入数据，优先从 `../data/backups/` 或外部备份恢复。 +## 课程小结迁移维护 + +- VPS 是 `classnotes.txt` 和 `学生课时账户.md` 的唯一正式写入方。 +- 本机 `com.xsk.records.sync` 常驻同步迁移后应停止，避免旧本地数据覆盖 VPS。 +- 本机课程小结采集脚本用 `XSK_INGEST_URL` 和 `XSK_INGEST_TOKEN` 推送批次；失败批次保存在本机 `推送失败队列/`。 +- 管理后台的“课程小结审核”处理低置信或冲突小结；“操作记录”追踪接收、自动入账、重复、失败、审核批准和驳回。 +- 历史小结导入使用 `scripts/import_course_summaries.py`，历史 `classnotes缺失.txt` 只生成审核任务，不自动扣课时。 + ## 提交前检查清单 - `make check` 通过。 diff --git a/app/Makefile b/app/Makefile index 6a58212..0f6a061 100644 --- a/app/Makefile +++ b/app/Makefile @@ -1,7 +1,7 @@ SHELL := /bin/bash APP_PORT ?= 18080 -DATA_FILES := ../data/classnotes.txt ../data/学生课时账户.md $(wildcard ../data/admin_tasks.json) +DATA_FILES := ../data/classnotes.txt ../data/学生课时账户.md $(wildcard ../data/admin_tasks.json) $(wildcard ../data/course_summary_state.json) .PHONY: check smoke data-hash build up ps health deploy logs install-gitea-backup diff --git a/app/README.md b/app/README.md index b56f7a5..938c535 100644 --- a/app/README.md +++ b/app/README.md @@ -1,6 +1,6 @@ # 新时空教务管家 -这是一个面向新时空教务业务的综合管理系统，用于把本机正式业务源中的 `classnotes.txt` 和 `学生课时账户.md` 同步到 VPS，并通过网页查询上课记录、管理课时账户、登记新增记录和审核纠错。 +这是一个面向新时空教务业务的综合管理系统。迁移后 VPS 是正式业务数据主机，负责保存 `classnotes.txt`、`学生课时账户.md`、课程小结库、审核任务和操作记录；本机只保留微信群聊天记录采集/识别，并把课程小结批量推送到 VPS。 ## 目录 @@ -8,7 +8,8 @@ - `MAINTENANCE.md`：日常检查、部署、数据保护和回滚流程。 - `Makefile`：常用维护命令入口。 - `scripts/deploy_to_vps.py`：部署教务管家服务到 VPS。 -- `scripts/sync_to_vps.py`：同步正式数据文件到 VPS。 +- `scripts/import_course_summaries.py`：一次性导入历史课程小结 Markdown。 +- `scripts/sync_to_vps.py`：旧版正式数据同步脚本，迁移后不要继续常驻运行。 - `scripts/smoke_test.js`：轻量前端行为烟测。 - `scripts/install_gitea_backup_hook.py`：安装提交后自动推送到 Gitea 的 Git hook。 - `scripts/install_launch_agent.py`：安装 Mac 开机常驻同步任务。 @@ -42,6 +43,8 @@ XSK_PYTHON_IMAGE='python:3.12-slim' ## 手动同步数据 +迁移完成后不要再用本机 `classnotes.txt` 和 `学生课时账户.md` 覆盖 VPS。下面命令只保留给迁移前或灾难恢复时使用，日常新增课程小结应走 `POST /api/ingest/course-summaries`。 + ```bash XSK_USE_SSHPASS=1 \ XSK_SSH_PASSWORD='填写SSH密码' \ @@ -59,9 +62,58 @@ python3 scripts/sync_to_vps.py --once --use-sshpass /root/新时空数据/data/ ``` +## 课程小结推送 + +VPS 接收接口： + +```text +POST /api/ingest/course-summaries +Header: X-INGEST-TOKEN: +``` + +本机采集脚本默认在 `--write` 时推送到该接口。建议在本机 shell 配置： + +```bash +export XSK_INGEST_URL='http://121.199.172.246:18080/api/ingest/course-summaries' +export XSK_INGEST_TOKEN='填写VPS里的INGEST_AUTH_TOKEN' +``` + +然后运行： + +```bash +python3 /Users/yangdawei/Desktop/新时空业务源数据/新时空课程记录与课时账户/课程小结采集/批量采集微信课程小结.py --write +``` + +推送成功批次会归档到本机 `课程小结采集/推送归档/`，失败批次会进入 `课程小结采集/推送失败队列/`，可用 `--retry-failed` 重试。需要临时恢复旧流程时再加 `--local-write`。 + +## 历史小结导入 + +把本机历史课程小结目录同步或上传到 VPS 后，可在容器内执行一次性导入。历史导入只重建小结库和状态；历史 `classnotes缺失.txt` 默认转为审核任务，不自动扣课时。 + +```bash +cd /root/新时空数据/app +docker compose exec xsk-records-web python scripts/import_course_summaries.py \ + --source /import/课程小结采集 \ + --target /data/course_summaries \ + --state /data/course_summary_state.json \ + --tasks /data/admin_tasks.json \ + --operation-logs /data/operation_logs.jsonl \ + --missing-table /import/课程小结采集/classnotes缺失.txt +``` + ## 数据备份 -通过登记 API 修改数据时，服务会在写入前自动备份本次会改动的业务文件。备份目录位于： +通过登记 API、课程小结自动入账或审核批准修改正式课时数据时，服务会在写入前自动备份本次会改动的业务文件。正式数据和辅助状态位于： + +```text +/root/新时空数据/data/classnotes.txt +/root/新时空数据/data/学生课时账户.md +/root/新时空数据/data/course_summaries/ +/root/新时空数据/data/course_summary_state.json +/root/新时空数据/data/operation_logs.jsonl +``` + +备份目录位于： ```text /root/新时空数据/data/backups/ @@ -138,3 +190,5 @@ docker compose up -d - `GET /api/admin/tasks`：管理后台查看审核任务。 - `POST /api/admin/tasks/{task_id}/approve`：批准纠错并写入正式上课记录。 - `POST /api/admin/tasks/{task_id}/reject`：驳回纠错。 +- `POST /api/ingest/course-summaries`：本机采集脚本批量推送课程小结，使用 `X-INGEST-TOKEN` 鉴权。 +- `GET /api/admin/operation-logs`：管理后台读取小结接收、自动入账、审核批准和驳回记录。 diff --git a/app/app/data.py b/app/app/data.py index 04b035c..94db064 100644 --- a/app/app/data.py +++ b/app/app/data.py @@ -46,6 +46,10 @@ BACKUP_DIR_RE = re.compile(r"^\d{8}-\d{6}-\d{6}-") BACKUP_KEEP_COUNT = 50 ACCOUNT_STATUSES = {"正常", "预警", "欠费", "结课", "退费"} WEEKDAYS = ["星期一", "星期二", "星期三", "星期四", "星期五", "星期六", "星期日"] +UNKNOWN_TEACHERS = {"", "待核对老师", "未知老师"} +UNKNOWN_SUBJECTS = {"", "待核对科目", "未知科目"} +HIGH_CONFIDENCE_VALUES = {"high", "高", "高置信", "true", "1", "yes"} +AUTO_RECOGNITION_SOURCES = {"keyword", "rule", "model", "model_high_confidence", "大模型高置信识别", "关键词"} ROLE_WORDS = { "student": ("学生", "学员", "孩子", "同学"), "teacher": ("老师", "教师"), @@ -847,6 +851,536 @@ def approve_correction_task(tasks_path: Path, classnotes_path: Path, task_id: in return {"task": task_to_dict(task), "backup_id": backup_dir.name} +def safe_filename_part(value: object) -> str: + text = str(value or "").strip() + text = re.sub(r"[\\/:*?\"<>|\s]+", "_", text) + return text.strip("._") or "未命名" + + +def sha1_text(value: str, length: int = 16) -> str: + return hashlib.sha1(value.encode("utf-8")).hexdigest()[:length] + + +def payload_bool(value: object) -> bool: + if isinstance(value, bool): + return value + if isinstance(value, (int, float)): + return value != 0 + text = str(value or "").strip().lower() + return text in {"1", "true", "yes", "y", "高", "高置信", "可信"} + + +def normalize_summary_date(value: object) -> str: + text = str(value or "").strip().replace(".", "-") + if not text: + raise ValueError("课程小结缺少日期") + parsed = datetime.strptime(text, "%Y-%m-%d").date() + return parsed.isoformat() + + +def normalize_time_range_text(value: object) -> str: + text = str(value or "").strip() + if not text: + return "" + match = TIME_RANGE_RE.fullmatch(text) + if not match: + raise ValueError(f"课程小结时间段格式错误: {text}") + start_hour = int(match.group("sh")) + start_minute = int(match.group("sm")) + end_hour = int(match.group("eh")) + end_minute = int(match.group("em")) + if start_hour > 23 or end_hour > 23 or start_minute > 59 or end_minute > 59: + raise ValueError(f"课程小结时间段超出范围: {text}") + if end_hour * 60 + end_minute <= start_hour * 60 + start_minute: + raise ValueError(f"课程小结结束时间必须晚于开始时间: {text}") + return f"{start_hour:02d}:{start_minute:02d}-{end_hour:02d}:{end_minute:02d}" + + +def duration_text_from_minutes(minutes: int) -> str: + return f"{minutes // 60}小时{minutes % 60}分" + + +def duration_minutes_from_time_range(time_range: str) -> int | None: + if not time_range: + return None + match = TIME_RANGE_RE.fullmatch(time_range) + if not match: + return None + start = int(match.group("sh")) * 60 + int(match.group("sm")) + end = int(match.group("eh")) * 60 + int(match.group("em")) + return end - start if end > start else None + + +def duration_minutes_from_summary(summary: dict) -> int | None: + raw_duration = summary.get("duration") or summary.get("duration_text") or "" + if raw_duration: + return int(round(parse_hours_text(str(raw_duration)) * 60)) + for key in ("duration_minutes", "minutes"): + value = summary.get(key) + if value not in (None, ""): + return int(round(float(value))) + for key in ("duration_hours", "hours"): + value = summary.get(key) + if value not in (None, ""): + return int(round(float(value) * 60)) + return duration_minutes_from_time_range(str(summary.get("time_range") or "")) + + +def normalize_course_summary(raw: dict) -> dict: + student = canonical_name(str(raw.get("student") or "").strip()) + teacher = canonical_name(str(raw.get("teacher") or "").strip()) + subject = parse_subject_code(str(raw.get("subject") or "").strip()) + body = str(raw.get("body") or raw.get("content") or "").strip() + if not student: + raise ValueError("课程小结缺少学生") + if not body: + raise ValueError("课程小结缺少正文") + date_iso = normalize_summary_date(raw.get("date_iso") or raw.get("date") or raw.get("class_date")) + time_range = normalize_time_range_text(raw.get("time_range") or raw.get("raw_time") or raw.get("time") or "") + minutes = duration_minutes_from_summary({**raw, "time_range": time_range}) + if time_range and minutes is not None: + time_minutes = duration_minutes_from_time_range(time_range) + if time_minutes is not None and abs(time_minutes - minutes) > 1: + raise ValueError(f"课程小结时间段和时长不一致: {time_range} / {duration_text_from_minutes(minutes)}") + source_id = str(raw.get("source_id") or "").strip() + if not source_id: + source_parts = [ + str(raw.get("db") or ""), + str(raw.get("local_id") or ""), + student, + date_iso, + teacher, + subject, + time_range, + body[:200], + ] + source_id = sha1_text("|".join(source_parts), 24) + summary = { + "source_id": source_id, + "student": student, + "date_iso": date_iso, + "time_range": time_range, + "duration_minutes": minutes, + "duration": duration_text_from_minutes(minutes) if minutes is not None else "", + "teacher": teacher, + "subject": subject, + "group": str(raw.get("group") or "").strip(), + "sender": str(raw.get("sender") or raw.get("sender_name") or "").strip(), + "sender_id": str(raw.get("sender_id") or "").strip(), + "message_time": str(raw.get("message_time") or "").strip(), + "message_date": str(raw.get("message_date") or "").strip(), + "db": str(raw.get("db") or "").strip(), + "local_id": str(raw.get("local_id") or "").strip(), + "title": str(raw.get("title") or "").strip(), + "body": body, + "recognition_source": str(raw.get("recognition_source") or raw.get("source") or "").strip(), + "confidence": str(raw.get("confidence") or "").strip(), + "teacher_trusted": payload_bool(raw.get("teacher_trusted") or raw.get("sender_teacher_trusted")), + "remark": str(raw.get("remark") or "").strip(), + } + if not summary["message_date"] and len(summary["message_time"]) >= 10: + summary["message_date"] = summary["message_time"][:10] + return summary + + +def course_summary_semantic_key(summary: dict) -> str: + body_digest = sha1_text(re.sub(r"\s+", "", str(summary.get("body") or "")), 12) + parts = [ + summary.get("student", ""), + summary.get("date_iso", ""), + summary.get("teacher", ""), + normalize_subject(str(summary.get("subject") or "")), + summary.get("time_range", ""), + str(summary.get("duration_minutes") or ""), + body_digest, + ] + return "|".join(str(part) for part in parts) + + +def default_course_summary_state() -> dict: + return {"version": 1, "seen_source_ids": [], "seen_semantic_keys": [], "batches": []} + + +def read_course_summary_state(path: Path) -> dict: + if not path.exists(): + return default_course_summary_state() + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise ValueError(f"课程小结状态文件 JSON 格式错误: {path}") from exc + if not isinstance(payload, dict): + raise ValueError("课程小结状态文件必须是 JSON 对象") + payload.setdefault("version", 1) + payload.setdefault("seen_source_ids", []) + payload.setdefault("seen_semantic_keys", []) + payload.setdefault("batches", []) + return payload + + +def write_course_summary_state(path: Path, state: dict) -> None: + atomic_write_text(path, json.dumps(state, ensure_ascii=False, indent=2) + "\n") + + +def append_operation_log(path: Path, operation: str, status: str, **fields: object) -> str: + path.parent.mkdir(parents=True, exist_ok=True) + now = datetime.now().isoformat(timespec="seconds") + log_id = f"{datetime.now().strftime('%Y%m%d%H%M%S')}-{sha1_text(json.dumps(fields, ensure_ascii=False, sort_keys=True), 8)}" + row = { + "id": log_id, + "created_at": now, + "operation": operation, + "status": status, + **fields, + } + with path.open("a", encoding="utf-8") as handle: + handle.write(json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n") + return log_id + + +def list_operation_logs(path: Path, limit: int = 100, operation: str = "", status_filter: str = "", student: str = "") -> dict: + rows: list[dict] = [] + if path.exists(): + for raw_line in path.read_text(encoding="utf-8").splitlines(): + if not raw_line.strip(): + continue + try: + item = json.loads(raw_line) + except json.JSONDecodeError: + continue + if operation and item.get("operation") != operation: + continue + if status_filter and item.get("status") != status_filter: + continue + if student and student not in str(item.get("student", "")): + continue + rows.append(item) + rows = rows[-limit:] + rows.reverse() + return {"count": len(rows), "items": rows} + + +def course_summary_path(root: Path, summary: dict) -> Path: + student = safe_filename_part(summary["student"]) + teacher = safe_filename_part(summary["teacher"] or "待核对老师") + subject = safe_filename_part(summary["subject"] or "待核对科目") + return root / student / f"{student}_{teacher}_{subject}.md" + + +def course_summary_heading(summary: dict, existing_headings: set[str]) -> str: + subject = normalize_subject(str(summary.get("subject") or "待核对科目")) + time_range = str(summary.get("time_range") or "").strip() + base = f"{summary['date_iso']} {time_range + ' ' if time_range else ''}{subject}课堂小结" + if base not in existing_headings: + return base + return f"{base}（{sha1_text(str(summary.get('source_id') or '') + str(summary.get('body') or ''), 8)}）" + + +def save_course_summary_markdown(root: Path, summary: dict) -> dict: + path = course_summary_path(root, summary) + existing = path.read_text(encoding="utf-8") if path.exists() else "" + existing_headings = set(re.findall(r"^###\s+(.+)$", existing, flags=re.M)) + existing_compact = re.sub(r"\s+", "", existing) + body = str(summary.get("body") or "").rstrip() + body_compact = re.sub(r"\s+", "", body) + if body_compact and body_compact in existing_compact: + return {"path": str(path), "added": False} + + group_name = str(summary.get("group") or summary["student"]) + heading = course_summary_heading(summary, existing_headings) + lines: list[str] = [] + if not existing.strip(): + lines.extend([f"# {path.stem}", "", f"## {group_name}", ""]) + elif f"## {group_name}" not in existing: + lines.extend(["", f"## {group_name}", ""]) + lines.extend( + [ + f"### {heading}", + "", + f"> 来源ID：`{summary['source_id']}`", + f"> 发送时间：`{summary.get('message_time') or ''}`", + f"> 发送者：`{summary.get('sender') or ''}`", + "", + body, + "", + ] + ) + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a", encoding="utf-8") as handle: + if existing and not existing.endswith("\n"): + handle.write("\n") + handle.write("\n".join(lines).rstrip() + "\n") + return {"path": str(path), "added": True, "heading": heading} + + +def message_date_after_class_date(summary: dict) -> bool: + message_date = str(summary.get("message_date") or "") + if len(message_date) != 10: + return False + try: + return date.fromisoformat(str(summary["date_iso"])) > date.fromisoformat(message_date) + except ValueError: + return False + + +def course_summary_to_class_record_line(summary: dict) -> str: + date_iso = normalize_summary_date(summary.get("date_iso")) + record_date = date_iso.replace("-", ".") + weekday = WEEKDAYS[date.fromisoformat(date_iso).weekday()] + time_range = normalize_time_range_text(summary.get("time_range")) + minutes = summary.get("duration_minutes") + if minutes is None: + raise ValueError("课程小结缺少时长") + duration = duration_text_from_minutes(int(minutes)) + return ( + f"{record_date}-{weekday}-{time_range}-{summary['student']}-" + f"{duration}-{summary['teacher']}-{normalize_subject(str(summary['subject']))}" + ) + + +def auto_register_reasons(summary: dict, classnotes_path: Path, accounts_path: Path) -> tuple[list[str], str]: + reasons: list[str] = [] + confidence = str(summary.get("confidence") or "").lower() + recognition_source = str(summary.get("recognition_source") or "") + if confidence not in HIGH_CONFIDENCE_VALUES and recognition_source not in AUTO_RECOGNITION_SOURCES: + reasons.append("识别置信度不足") + if not summary.get("teacher_trusted"): + reasons.append("发送者老师映射未确认") + if summary.get("teacher") in UNKNOWN_TEACHERS: + reasons.append("老师待核对") + if normalize_subject(str(summary.get("subject") or "")) in UNKNOWN_SUBJECTS: + reasons.append("科目待核对") + if not summary.get("time_range"): + reasons.append("时间段缺失") + if summary.get("duration_minutes") is None: + reasons.append("时长缺失") + if message_date_after_class_date(summary): + reasons.append("课程日期晚于消息发送日期") + + line = "" + try: + line = course_summary_to_class_record_line(summary) + parse_class_record_line(line) + except ValueError as exc: + reasons.append(str(exc)) + + try: + find_account_index(read_accounts(accounts_path), str(summary["student"])) + except ValueError as exc: + reasons.append(str(exc)) + + if line: + existing_lines = {raw.strip() for raw in classnotes_path.read_text(encoding="utf-8").splitlines()} + if line in existing_lines: + reasons.append("classnotes 已存在同一条上课记录") + return reasons, line + + +def create_course_summary_review_task( + tasks_path: Path, + summary: dict, + proposed_line: str, + reasons: list[str], + saved_path: str = "", +) -> dict: + tasks = read_admin_tasks(tasks_path) + now = datetime.now().isoformat(timespec="seconds") + task = { + "id": int(tasks["next_id"]), + "type": "course_summary_review", + "status": "pending", + "created_at": now, + "updated_at": now, + "source_id": summary["source_id"], + "student": summary["student"], + "summary": summary, + "proposed_line": proposed_line, + "reasons": reasons, + "saved_path": saved_path, + } + tasks["next_id"] = int(tasks["next_id"]) + 1 + tasks["items"].append(task) + write_admin_tasks(tasks_path, tasks) + return task_to_dict(task) + + +def approve_course_summary_task( + tasks_path: Path, + classnotes_path: Path, + accounts_path: Path, + task_id: int, +) -> dict: + tasks = read_admin_tasks(tasks_path) + task = find_admin_task(tasks, task_id) + if task.get("type") != "course_summary_review": + raise ValueError("该任务不是课程小结审核") + if task.get("status") not in {"pending", "conflict"}: + raise ValueError("该任务已处理，不能重复批准") + + proposed_line = str(task.get("proposed_line") or "").strip() + if not proposed_line: + summary = dict(task.get("summary") or {}) + proposed_line = course_summary_to_class_record_line(summary) + + try: + result = register_class_record_lines(classnotes_path, accounts_path, line=proposed_line) + except ValueError as exc: + task["status"] = "conflict" + task["updated_at"] = datetime.now().isoformat(timespec="seconds") + task["message"] = str(exc) + write_admin_tasks(tasks_path, tasks) + raise + + task["status"] = "approved" + task["updated_at"] = datetime.now().isoformat(timespec="seconds") + task["reviewed_at"] = task["updated_at"] + task["registered_line"] = proposed_line + task["backup_id"] = result.get("backup_id", "") + write_admin_tasks(tasks_path, tasks) + return {"task": task_to_dict(task), "backup_id": result.get("backup_id", "")} + + +def approve_admin_task( + tasks_path: Path, + classnotes_path: Path, + accounts_path: Path, + task_id: int, +) -> dict: + task = find_admin_task(read_admin_tasks(tasks_path), task_id) + if task.get("type") == "class_record_correction": + return approve_correction_task(tasks_path, classnotes_path, task_id) + if task.get("type") == "course_summary_review": + return approve_course_summary_task(tasks_path, classnotes_path, accounts_path, task_id) + raise ValueError("不支持的审核任务类型") + + +def ingest_course_summaries( + *, + classnotes_path: Path, + accounts_path: Path, + tasks_path: Path, + summaries_root: Path, + state_path: Path, + operation_logs_path: Path, + batch_id: str, + window: dict, + students: list[str], + summaries: list[dict], +) -> dict: + state = read_course_summary_state(state_path) + seen_source_ids = set(str(item) for item in state.get("seen_source_ids", [])) + seen_semantic_keys = set(str(item) for item in state.get("seen_semantic_keys", [])) + result = { + "received": len(summaries), + "saved": 0, + "auto_registered": 0, + "review_pending": 0, + "duplicates": 0, + "rejected": 0, + "operation_log_ids": [], + "items": [], + } + + for raw in summaries: + normalized: dict | None = None + try: + normalized = normalize_course_summary(raw) + source_id = normalized["source_id"] + semantic_key = course_summary_semantic_key(normalized) + if source_id in seen_source_ids or semantic_key in seen_semantic_keys: + result["duplicates"] += 1 + log_id = append_operation_log( + operation_logs_path, + "course_summary_ingest", + "duplicate", + batch_id=batch_id, + source_id=source_id, + student=normalized["student"], + ) + result["operation_log_ids"].append(log_id) + result["items"].append({"source_id": source_id, "status": "duplicate"}) + continue + + saved = save_course_summary_markdown(summaries_root, normalized) + result["saved"] += 1 if saved.get("added") else 0 + + reasons, proposed_line = auto_register_reasons(normalized, classnotes_path, accounts_path) + if reasons: + task = create_course_summary_review_task( + tasks_path, + normalized, + proposed_line, + reasons, + saved_path=str(saved.get("path") or ""), + ) + result["review_pending"] += 1 + status_value = "review" + task_id = task.get("id") + backup_id = "" + else: + register_result = register_class_record_lines(classnotes_path, accounts_path, line=proposed_line) + result["auto_registered"] += 1 + status_value = "auto_registered" + task_id = None + backup_id = str(register_result.get("backup_id") or "") + + seen_source_ids.add(source_id) + seen_semantic_keys.add(semantic_key) + log_id = append_operation_log( + operation_logs_path, + "course_summary_ingest", + status_value, + batch_id=batch_id, + source_id=source_id, + student=normalized["student"], + teacher=normalized.get("teacher", ""), + subject=normalized.get("subject", ""), + proposed_line=proposed_line, + reasons=reasons, + task_id=task_id, + backup_id=backup_id, + saved_path=str(saved.get("path") or ""), + ) + result["operation_log_ids"].append(log_id) + result["items"].append( + { + "source_id": source_id, + "status": status_value, + "task_id": task_id, + "backup_id": backup_id, + "reasons": reasons, + } + ) + except Exception as exc: + result["rejected"] += 1 + source_id = str((normalized or raw).get("source_id") or "") + log_id = append_operation_log( + operation_logs_path, + "course_summary_ingest", + "rejected", + batch_id=batch_id, + source_id=source_id, + student=str((normalized or raw).get("student") or ""), + error=str(exc), + ) + result["operation_log_ids"].append(log_id) + result["items"].append({"source_id": source_id, "status": "rejected", "error": str(exc)}) + + state["seen_source_ids"] = sorted(seen_source_ids) + state["seen_semantic_keys"] = sorted(seen_semantic_keys) + state.setdefault("batches", []).append( + { + "batch_id": batch_id, + "received_at": datetime.now().isoformat(timespec="seconds"), + "window": window, + "students": students, + "result": {key: result[key] for key in ("received", "saved", "auto_registered", "review_pending", "duplicates", "rejected")}, + } + ) + state["batches"] = state["batches"][-200:] + write_course_summary_state(state_path, state) + return result + + def parse_record_date(text: str) -> date: return datetime.strptime(text, "%Y.%m.%d").date() @@ -1065,6 +1599,10 @@ def parse_subject_code(text: str) -> str: return "".join(result) +def normalize_subject(text: str) -> str: + return parse_subject_code(text).strip() + + def detect_subjects(query: str) -> list[str]: subjects = {subject for subject in SUBJECTS if subject in query} for alias, subject in SUBJECT_ALIASES.items(): diff --git a/app/app/main.py b/app/app/main.py index f5f7c2d..e7c7e87 100644 --- a/app/app/main.py +++ b/app/app/main.py @@ -9,7 +9,7 @@ from pathlib import Path import threading from urllib.parse import parse_qs, quote -from fastapi import Depends, FastAPI, HTTPException, Query, Request, status +from fastapi import Depends, FastAPI, Header, HTTPException, Query, Request, status from fastapi.responses import FileResponse, HTMLResponse, JSONResponse, RedirectResponse, Response from fastapi.security import HTTPBasic, HTTPBasicCredentials from pydantic import BaseModel, Field, ValidationError @@ -19,11 +19,14 @@ from .data import ( Account, DuplicateRecordError, Payment, - approve_correction_task, + append_operation_log, + approve_admin_task, account_summary, account_to_dict, create_account, filter_accounts, + ingest_course_summaries, + list_operation_logs, list_admin_tasks, query_records, read_accounts, @@ -41,9 +44,13 @@ STATIC_DIR = APP_DIR / "static" CLASSNOTES_PATH = Path(os.getenv("CLASSNOTES_PATH", "/data/classnotes.txt")) ACCOUNTS_PATH = Path(os.getenv("ACCOUNTS_PATH", "/data/学生课时账户.md")) ADMIN_TASKS_PATH = Path(os.getenv("ADMIN_TASKS_PATH", "/data/admin_tasks.json")) +COURSE_SUMMARIES_ROOT = Path(os.getenv("COURSE_SUMMARIES_ROOT", "/data/course_summaries")) +COURSE_SUMMARY_STATE_PATH = Path(os.getenv("COURSE_SUMMARY_STATE_PATH", "/data/course_summary_state.json")) +OPERATION_LOGS_PATH = Path(os.getenv("OPERATION_LOGS_PATH", "/data/operation_logs.jsonl")) BASIC_AUTH_PASSWORD = os.getenv("BASIC_AUTH_PASSWORD", "") ACCOUNTS_AUTH_PASSWORD = os.getenv("ACCOUNTS_AUTH_PASSWORD") or os.getenv("ACCOUNT_AUTH_PASSWORD", "") ADMIN_AUTH_PASSWORD = os.getenv("ADMIN_AUTH_PASSWORD") or ACCOUNTS_AUTH_PASSWORD +INGEST_AUTH_TOKEN = os.getenv("INGEST_AUTH_TOKEN", "") RECORDS_SESSION_COOKIE = "xsk_records_session" ACCOUNTS_SESSION_COOKIE = "xsk_accounts_session" ADMIN_SESSION_COOKIE = "xsk_admin_session" @@ -82,6 +89,42 @@ class CorrectionSubmitPayload(BaseModel): items: list[CorrectionItemPayload] +class CourseSummaryPayload(BaseModel): + source_id: str = "" + student: str + date_iso: str = "" + date: str = "" + time_range: str = "" + raw_time: str = "" + duration: str = "" + duration_hours: float | None = None + duration_minutes: int | None = None + teacher: str = "" + subject: str = "" + group: str = "" + sender: str = "" + sender_name: str = "" + sender_id: str = "" + message_time: str = "" + message_date: str = "" + db: str = "" + local_id: str | int | None = "" + title: str = "" + body: str + recognition_source: str = "" + confidence: str = "" + teacher_trusted: bool = False + sender_teacher_trusted: bool = False + remark: str = "" + + +class CourseSummaryIngestPayload(BaseModel): + batch_id: str + window: dict = Field(default_factory=dict) + students: list[str] = Field(default_factory=list) + summaries: list[CourseSummaryPayload] = Field(default_factory=list) + + async def read_register_payload(request: Request) -> RegisterLinesPayload: body = await request.body() if not body.strip(): @@ -217,6 +260,16 @@ def verify_any_auth( ) +def verify_ingest_token(x_ingest_token: str = Header(default="")) -> str: + configured_password("课程小结推送", INGEST_AUTH_TOKEN) + if not hmac.compare_digest(x_ingest_token, INGEST_AUTH_TOKEN): + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="课程小结推送 token 不正确", + ) + return "ingest" + + def safe_next_path(value: str | None) -> str: if not value or not value.startswith("/") or value.startswith("//"): return "/" @@ -634,6 +687,27 @@ async def register_payments(request: Request, _user: str = Depends(verify_admin_ return {"ok": True, **result} +@app.post("/api/ingest/course-summaries") +def ingest_course_summary_batch(payload: CourseSummaryIngestPayload, _user: str = Depends(verify_ingest_token)): + try: + with write_lock: + result = ingest_course_summaries( + classnotes_path=CLASSNOTES_PATH, + accounts_path=ACCOUNTS_PATH, + tasks_path=ADMIN_TASKS_PATH, + summaries_root=COURSE_SUMMARIES_ROOT, + state_path=COURSE_SUMMARY_STATE_PATH, + operation_logs_path=OPERATION_LOGS_PATH, + batch_id=payload.batch_id, + window=payload.window, + students=payload.students, + summaries=[item.dict() for item in payload.summaries], + ) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + return {"ok": True, **result} + + @app.get("/api/health") def health(_user: str = Depends(verify_records_auth)): records = load_records() @@ -642,6 +716,9 @@ def health(_user: str = Depends(verify_records_auth)): "ok": True, "classnotes": file_meta(CLASSNOTES_PATH), "accounts": file_meta(ACCOUNTS_PATH), + "course_summaries": file_meta(COURSE_SUMMARIES_ROOT), + "course_summary_state": file_meta(COURSE_SUMMARY_STATE_PATH), + "operation_logs": file_meta(OPERATION_LOGS_PATH), "records_count": len(records), "accounts_count": len(accounts), "account_summary": account_summary(accounts), @@ -748,11 +825,39 @@ def admin_tasks( raise HTTPException(status_code=500, detail=str(exc)) from exc +@app.get("/api/admin/operation-logs") +def admin_operation_logs( + limit: int = Query(100, ge=1, le=500), + operation: str = Query(""), + status_filter: str = Query("", alias="status"), + student: str = Query(""), + _user: str = Depends(verify_admin_auth), +): + return list_operation_logs( + OPERATION_LOGS_PATH, + limit=limit, + operation=operation, + status_filter=status_filter, + student=student, + ) + + @app.post("/api/admin/tasks/{task_id}/approve") def admin_approve_task(task_id: int, _user: str = Depends(verify_admin_auth)): try: with write_lock: - result = approve_correction_task(ADMIN_TASKS_PATH, CLASSNOTES_PATH, task_id) + result = approve_admin_task(ADMIN_TASKS_PATH, CLASSNOTES_PATH, ACCOUNTS_PATH, task_id) + task = result.get("task", {}) + append_operation_log( + OPERATION_LOGS_PATH, + "admin_task_approve", + "approved", + task_id=task_id, + task_type=str(task.get("type") or ""), + student=str(task.get("student") or task.get("corrected", {}).get("student") or ""), + source_id=str(task.get("source_id") or ""), + backup_id=str(result.get("backup_id") or task.get("backup_id") or ""), + ) except ValueError as exc: raise HTTPException(status_code=409, detail=str(exc)) from exc return {"ok": True, **result} @@ -763,6 +868,15 @@ def admin_reject_task(task_id: int, _user: str = Depends(verify_admin_auth)): try: with write_lock: task = reject_admin_task(ADMIN_TASKS_PATH, task_id) + append_operation_log( + OPERATION_LOGS_PATH, + "admin_task_reject", + "rejected", + task_id=task_id, + task_type=str(task.get("type") or ""), + student=str(task.get("student") or task.get("corrected", {}).get("student") or ""), + source_id=str(task.get("source_id") or ""), + ) except ValueError as exc: raise HTTPException(status_code=400, detail=str(exc)) from exc return {"ok": True, "task": task} diff --git a/app/app/static/admin.html b/app/app/static/admin.html index d29c6c3..6defd7f 100644 --- a/app/app/static/admin.html +++ b/app/app/static/admin.html @@ -4,7 +4,7 @@ 管理后台 - +
@@ -23,6 +23,8 @@ @@ -136,6 +138,78 @@ + + + + - + diff --git a/app/app/static/admin.js b/app/app/static/admin.js index 502cd8b..0107591 100644 --- a/app/app/static/admin.js +++ b/app/app/static/admin.js @@ -3,6 +3,8 @@ const refreshBtn = document.querySelector("#refreshBtn"); const panels = { accounts: document.querySelector("#accountsPanel"), reviews: document.querySelector("#reviewsPanel"), + summaries: document.querySelector("#summariesPanel"), + logs: document.querySelector("#logsPanel"), register: document.querySelector("#registerPanel"), }; const accountForm = document.querySelector("#accountForm"); @@ -25,6 +27,15 @@ const editNote = document.querySelector("#editNote"); const reviewStatus = document.querySelector("#reviewStatus"); const reviewMeta = document.querySelector("#reviewMeta"); const reviewRows = document.querySelector("#reviewRows"); +const summaryReviewStatus = document.querySelector("#summaryReviewStatus"); +const summaryReviewMeta = document.querySelector("#summaryReviewMeta"); +const summaryReviewRows = document.querySelector("#summaryReviewRows"); +const logOperation = document.querySelector("#logOperation"); +const logStatus = document.querySelector("#logStatus"); +const logFilterForm = document.querySelector("#logFilterForm"); +const logStudent = document.querySelector("#logStudent"); +const logMeta = document.querySelector("#logMeta"); +const logRows = document.querySelector("#logRows"); const classRegisterForm = document.querySelector("#classRegisterForm"); const classRegisterLines = document.querySelector("#classRegisterLines"); const classRegisterStatus = document.querySelector("#classRegisterStatus"); @@ -64,6 +75,13 @@ function statusClass(status) { return "closed"; } +function taskStatusClass(status) { + if (status === "approved" || status === "auto_registered") return "normal"; + if (status === "rejected" || status === "duplicate") return "closed"; + if (status === "conflict") return "debt"; + return "warning"; +} + function renderPayments(payments) { if (!payments.length) return "暂无"; return payments.map((item) => `${escapeHtml(item.date)}：${fmtHours(item.hours)} 小时`).join("
"); @@ -93,6 +111,8 @@ function setActiveTab(tabName) { }); if (tabName === "accounts") loadAccounts(); if (tabName === "reviews") loadReviews(); + if (tabName === "summaries") loadSummaryReviews(); + if (tabName === "logs") loadOperationLogs(); } async function loadAdminHealth() { @@ -270,13 +290,113 @@ async function loadReviews() { } } -async function reviewTask(taskId, action) { +function renderSummaryInfo(summary) { + const parts = [ + summary.student, + summary.date_iso, + summary.time_range, + summary.teacher, + summary.subject, + ].filter(Boolean); + return `${parts.map(escapeHtml).join("
")}
${escapeHtml(summary.group || "")}`; +} + +function renderSummaryBody(summary) { + const body = String(summary.body || ""); + const preview = body.length > 260 ? `${body.slice(0, 260)}...` : body; + return `
${escapeHtml(preview)}
`; +} + +async function loadSummaryReviews() { + summaryReviewRows.innerHTML = `正在读取`; + const params = new URLSearchParams({ type: "course_summary_review" }); + if (summaryReviewStatus.value) params.set("status", summaryReviewStatus.value); + try { + const data = await fetchJson(`/api/admin/tasks?${params.toString()}`); + summaryReviewMeta.innerHTML = [metric("当前结果", `${data.count} 条`)].join(""); + summaryReviewRows.innerHTML = data.items + .map((item) => { + const canReview = item.status === "pending" || item.status === "conflict"; + const summary = item.summary || {}; + const reasons = Array.isArray(item.reasons) ? item.reasons : []; + return ` + #${escapeHtml(item.id)} + ${escapeHtml(item.status)}${item.message ? `
${escapeHtml(item.message)}` : ""} + ${renderSummaryInfo(summary)} + ${item.proposed_line ? renderReviewLine(item.proposed_line) : "暂无"} + ${renderSummaryBody(summary)} + ${reasons.map(escapeHtml).join("
") || "待人工复核"} + +
+ + +
+ + `; + }) + .join(""); + if (!data.items.length) { + summaryReviewRows.innerHTML = `没有符合条件的课程小结审核项`; + } + } catch (error) { + summaryReviewRows.innerHTML = `读取失败：${escapeHtml(error.message)}`; + } +} + +function renderLogDetail(item) { + const details = []; + if (item.source_id) details.push(`来源：${item.source_id}`); + if (item.teacher || item.subject) details.push(`老师/科目：${item.teacher || ""} ${item.subject || ""}`.trim()); + if (item.proposed_line) details.push(`记录：${item.proposed_line}`); + if (Array.isArray(item.reasons) && item.reasons.length) details.push(`原因：${item.reasons.join("；")}`); + if (item.error) details.push(`错误：${item.error}`); + if (item.backup_id) details.push(`备份：${item.backup_id}`); + if (item.saved_path) details.push(`文件：${item.saved_path}`); + return `
${details.map(escapeHtml).join("
") || "暂无详情"}
`; +} + +async function loadOperationLogs() { + logRows.innerHTML = `正在读取`; + const params = new URLSearchParams({ limit: "200" }); + if (logOperation.value) params.set("operation", logOperation.value); + if (logStatus.value) params.set("status", logStatus.value); + if (logStudent.value.trim()) params.set("student", logStudent.value.trim()); + try { + const data = await fetchJson(`/api/admin/operation-logs?${params.toString()}`); + logMeta.innerHTML = [metric("当前结果", `${data.count} 条`)].join(""); + logRows.innerHTML = data.items + .map((item) => ` + ${escapeHtml(item.created_at || "")} + ${escapeHtml(item.operation || "")} + ${escapeHtml(item.status || "")} + ${escapeHtml(item.student || "")} + ${escapeHtml(item.batch_id || "")}${item.task_id ? `
任务 #${escapeHtml(item.task_id)}` : ""} + ${renderLogDetail(item)} + `) + .join(""); + if (!data.items.length) { + logRows.innerHTML = `没有符合条件的操作记录`; + } + } catch (error) { + logRows.innerHTML = `读取失败：${escapeHtml(error.message)}`; + } +} + +async function runTaskAction(taskId, action, reload) { try { await fetchJson(`/api/admin/tasks/${encodeURIComponent(taskId)}/${action}`, { method: "POST" }); } catch (error) { alert(error.message); } - loadReviews(); + reload(); +} + +async function reviewTask(taskId, action) { + runTaskAction(taskId, action, loadReviews); +} + +async function summaryReviewTask(taskId, action) { + runTaskAction(taskId, action, loadSummaryReviews); } function linesFromTextarea(textarea) { @@ -330,6 +450,19 @@ reviewRows.addEventListener("click", (event) => { if (approve) reviewTask(approve.dataset.taskId, "approve"); if (reject) reviewTask(reject.dataset.taskId, "reject"); }); +summaryReviewStatus.addEventListener("change", loadSummaryReviews); +summaryReviewRows.addEventListener("click", (event) => { + const approve = event.target.closest(".summary-approve"); + const reject = event.target.closest(".summary-reject"); + if (approve) summaryReviewTask(approve.dataset.taskId, "approve"); + if (reject) summaryReviewTask(reject.dataset.taskId, "reject"); +}); +logOperation.addEventListener("change", loadOperationLogs); +logStatus.addEventListener("change", loadOperationLogs); +logFilterForm.addEventListener("submit", (event) => { + event.preventDefault(); + loadOperationLogs(); +}); classRegisterForm.addEventListener("submit", (event) => { submitRegister(event, classRegisterLines, classRegisterStatus, "/api/register/class-records"); }); @@ -340,6 +473,8 @@ refreshBtn.addEventListener("click", () => { loadAdminHealth(); if (!panels.accounts.hidden) loadAccounts(); if (!panels.reviews.hidden) loadReviews(); + if (!panels.summaries.hidden) loadSummaryReviews(); + if (!panels.logs.hidden) loadOperationLogs(); }); loadAdminHealth(); diff --git a/app/app/static/styles.css b/app/app/static/styles.css index 5a2b073..84c1af1 100644 --- a/app/app/static/styles.css +++ b/app/app/static/styles.css @@ -379,6 +379,20 @@ textarea:focus { line-height: 1.4; } +.summary-body, +.log-detail { + max-width: 360px; + white-space: pre-wrap; + overflow-wrap: anywhere; + color: #344054; + font-size: 13px; + line-height: 1.45; +} + +.muted { + color: var(--muted); +} + .correction-toolbar { display: flex; align-items: center; diff --git a/app/docker-compose.yml b/app/docker-compose.yml index 997261e..449464b 100644 --- a/app/docker-compose.yml +++ b/app/docker-compose.yml @@ -13,6 +13,10 @@ services: CLASSNOTES_PATH: ${CLASSNOTES_PATH:-/data/classnotes.txt} ACCOUNTS_PATH: ${ACCOUNTS_PATH:-/data/学生课时账户.md} ADMIN_TASKS_PATH: ${ADMIN_TASKS_PATH:-/data/admin_tasks.json} + COURSE_SUMMARIES_ROOT: ${COURSE_SUMMARIES_ROOT:-/data/course_summaries} + COURSE_SUMMARY_STATE_PATH: ${COURSE_SUMMARY_STATE_PATH:-/data/course_summary_state.json} + OPERATION_LOGS_PATH: ${OPERATION_LOGS_PATH:-/data/operation_logs.jsonl} + INGEST_AUTH_TOKEN: ${INGEST_AUTH_TOKEN:-} ports: - "${APP_PORT:-18080}:8000" volumes: diff --git a/app/scripts/import_course_summaries.py b/app/scripts/import_course_summaries.py new file mode 100644 index 0000000..e5a847d --- /dev/null +++ b/app/scripts/import_course_summaries.py @@ -0,0 +1,264 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import re +import shutil +import sys +from datetime import datetime +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +from app.data import ( # noqa: E402 + append_operation_log, + course_summary_semantic_key, + create_course_summary_review_task, + normalize_course_summary, + read_admin_tasks, + read_course_summary_state, + safe_filename_part, + sha1_text, + write_course_summary_state, +) + + +EXCLUDE_MARKDOWN = { + "微信聊天记录ID映射表.md", + "采集记录表.md", + "课程记录核对表.md", +} +EXCLUDE_PREFIXES = ( + "已采集小结记录", + "classnotes缺失", + "课程小结缺失", + "疑似 classnotes 出错", + "疑似课程小结待复核", + "正文日期晚于发送日期待复核", +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="导入历史课程小结 Markdown 到 VPS 数据目录") + parser.add_argument("--source", required=True, type=Path, help="本机历史课程小结采集目录") + parser.add_argument("--target", default=Path("/data/course_summaries"), type=Path, help="VPS 课程小结正式目录") + parser.add_argument("--state", default=Path("/data/course_summary_state.json"), type=Path, help="课程小结状态文件") + parser.add_argument("--tasks", default=Path("/data/admin_tasks.json"), type=Path, help="管理任务文件") + parser.add_argument("--operation-logs", default=Path("/data/operation_logs.jsonl"), type=Path, help="操作日志文件") + parser.add_argument("--missing-table", type=Path, help="历史 classnotes缺失.txt；不传则尝试 source/classnotes缺失.txt") + parser.add_argument("--dry-run", action="store_true", help="只统计，不写入") + return parser.parse_args() + + +def should_import_markdown(path: Path, source: Path) -> bool: + if path.name in EXCLUDE_MARKDOWN or any(path.name.startswith(prefix) for prefix in EXCLUDE_PREFIXES): + return False + if any(part in {"__pycache__", "临时解析归档", "推送归档", "推送失败队列"} for part in path.relative_to(source).parts): + return False + return path.suffix.lower() == ".md" + + +def split_name_from_path(path: Path) -> tuple[str, str, str]: + parts = path.stem.split("_") + student = path.parent.name if path.parent.name else (parts[0] if parts else "") + teacher = parts[1] if len(parts) >= 2 else "" + subject = parts[2] if len(parts) >= 3 else "" + return student, teacher, subject + + +def iter_markdown_entries(path: Path) -> list[dict]: + text = path.read_text(encoding="utf-8", errors="ignore") + student, teacher, subject = split_name_from_path(path) + group = student + current_title = "" + current_body: list[str] = [] + entries: list[dict] = [] + + def flush() -> None: + if not current_title: + return + body = "\n".join(current_body).strip() + date_match = re.search(r"(\d{4}[.-]\d{2}[.-]\d{2})", current_title) + if not date_match or not body: + return + time_match = re.search(r"(\d{1,2}:\d{2}-\d{1,2}:\d{2})", current_title) + source_seed = f"{path}|{current_title}|{body[:120]}" + entries.append( + { + "source_id": f"history:{sha1_text(source_seed, 24)}", + "student": student, + "date_iso": date_match.group(1).replace(".", "-"), + "time_range": time_match.group(1) if time_match else "", + "teacher": teacher, + "subject": subject, + "group": group, + "title": current_title, + "body": body, + "recognition_source": "history_import", + "confidence": "history", + "teacher_trusted": False, + "remark": f"历史导入：{path}", + } + ) + + for line in text.splitlines(): + if line.startswith("## ") and not line.startswith("### "): + group = line.removeprefix("## ").strip() or group + continue + if line.startswith("### "): + flush() + current_title = line.removeprefix("### ").strip() + current_body = [] + continue + if current_title: + if line.startswith("> 来源ID") or line.startswith("> 发送时间") or line.startswith("> 发送者"): + continue + current_body.append(line) + flush() + return entries + + +def copy_markdown_files(source: Path, target: Path, dry_run: bool) -> tuple[int, int]: + copied = 0 + scanned = 0 + for path in sorted(source.rglob("*.md")): + if not should_import_markdown(path, source): + continue + scanned += 1 + relative = path.relative_to(source) + dest = target / relative + if dest.exists() and dest.read_text(encoding="utf-8", errors="ignore") == path.read_text(encoding="utf-8", errors="ignore"): + continue + copied += 1 + if not dry_run: + dest.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(path, dest) + return scanned, copied + + +def rebuild_state_from_markdown(source: Path, state_path: Path, dry_run: bool) -> tuple[int, int]: + state = read_course_summary_state(state_path) + seen_source_ids = set(str(item) for item in state.get("seen_source_ids", [])) + seen_semantic_keys = set(str(item) for item in state.get("seen_semantic_keys", [])) + imported = 0 + skipped = 0 + for path in sorted(source.rglob("*.md")): + if not should_import_markdown(path, source): + continue + for entry in iter_markdown_entries(path): + try: + summary = normalize_course_summary(entry) + seen_source_ids.add(summary["source_id"]) + seen_semantic_keys.add(course_summary_semantic_key(summary)) + imported += 1 + except ValueError: + skipped += 1 + state["seen_source_ids"] = sorted(seen_source_ids) + state["seen_semantic_keys"] = sorted(seen_semantic_keys) + state.setdefault("batches", []).append( + { + "batch_id": f"history-import-{datetime.now().strftime('%Y%m%d%H%M%S')}", + "received_at": datetime.now().isoformat(timespec="seconds"), + "window": {"mode": "历史导入"}, + "students": [], + "result": {"received": imported, "saved": 0, "auto_registered": 0, "review_pending": 0, "duplicates": 0, "rejected": skipped}, + } + ) + state["batches"] = state["batches"][-200:] + if not dry_run: + write_course_summary_state(state_path, state) + return imported, skipped + + +def parse_missing_table(path: Path) -> list[dict]: + if not path.exists(): + return [] + rows: list[dict] = [] + headers: list[str] = [] + for raw_line in path.read_text(encoding="utf-8", errors="ignore").splitlines(): + line = raw_line.strip() + if not line.startswith("| ") or "---" in line: + continue + cells = [cell.strip() for cell in line.strip("|").split("|")] + if cells and cells[0] == "日期": + headers = cells + continue + if not headers or len(cells) != len(headers): + continue + row = dict(zip(headers, cells, strict=False)) + if row.get("学生"): + rows.append(row) + return rows + + +def import_missing_tasks(missing_table: Path, tasks_path: Path, dry_run: bool) -> int: + rows = parse_missing_table(missing_table) + if not rows: + return 0 + existing = read_admin_tasks(tasks_path) + existing_source_ids = {str(item.get("source_id") or "") for item in existing.get("items", [])} + created = 0 + for row in rows: + source_seed = "|".join(str(row.get(key, "")) for key in ("日期", "学生", "老师", "科目", "来源文件", "备注")) + source_id = f"history-missing:{sha1_text(source_seed, 20)}" + if source_id in existing_source_ids: + continue + summary = { + "source_id": source_id, + "student": row.get("学生", ""), + "date_iso": str(row.get("日期", "")).replace(".", "-"), + "time_range": row.get("时间段", ""), + "duration": row.get("时长", ""), + "teacher": row.get("老师", ""), + "subject": row.get("科目", ""), + "group": row.get("群聊", ""), + "title": "历史 classnotes 缺失", + "body": f"历史 classnotes 缺失项：{source_seed}", + "recognition_source": "history_missing_table", + "confidence": "review", + "teacher_trusted": False, + "remark": row.get("备注", ""), + } + created += 1 + if not dry_run: + create_course_summary_review_task( + tasks_path, + summary, + "", + ["历史 classnotes缺失导入，默认只进入审核，不自动扣课时"], + saved_path=row.get("来源文件", ""), + ) + existing_source_ids.add(source_id) + return created + + +def main() -> None: + args = parse_args() + source = args.source.resolve() + target = args.target.resolve() + missing_table = args.missing_table or (source / "classnotes缺失.txt") + scanned, copied = copy_markdown_files(source, target, args.dry_run) + imported, skipped = rebuild_state_from_markdown(target if target.exists() else source, args.state, args.dry_run) + review_tasks = import_missing_tasks(missing_table, args.tasks, args.dry_run) + if not args.dry_run: + append_operation_log( + args.operation_logs, + "history_course_summary_import", + "completed", + source=str(source), + target=str(target), + scanned_files=scanned, + copied_files=copied, + indexed_summaries=imported, + skipped_summaries=skipped, + review_tasks=review_tasks, + ) + print( + f"历史小结导入完成：扫描 Markdown {scanned} 个，复制 {copied} 个，" + f"索引小结 {imported} 条，跳过 {skipped} 条，生成审核任务 {review_tasks} 条。" + ) + + +if __name__ == "__main__": + main()