diff --git a/app/app/data.py b/app/app/data.py index c010d2d..a8a7bcf 100644 --- a/app/app/data.py +++ b/app/app/data.py @@ -2201,6 +2201,111 @@ def course_summary_duplicate_candidate(item: dict) -> dict: } +def summary_as_duplicate_candidate(summary: dict, saved_path: str = "") -> dict: + body = str(summary.get("body") or "") + return { + "id": str(summary.get("source_id") or sha1_text(json.dumps(summary, ensure_ascii=False, sort_keys=True), 20)), + "title": str(summary.get("title") or ""), + "student": str(summary.get("student") or ""), + "teacher": str(summary.get("teacher") or ""), + "subject": str(summary.get("subject") or ""), + "date_iso": str(summary.get("date_iso") or ""), + "time_range": str(summary.get("time_range") or ""), + "group": str(summary.get("group") or ""), + "source_path": saved_path, + "relative_path": "", + "source_id": str(summary.get("source_id") or ""), + "message_time": str(summary.get("message_time") or ""), + "sender": str(summary.get("sender") or ""), + "body": body, + "body_preview": body[:260] + ("..." if len(body) > 260 else ""), + } + + +def normalize_semantic_key_text(value: str) -> str: + return re.sub(r"\s+", "", value.strip()) + + +def find_course_summary_duplicate_conflicts( + root: Path, + summary: dict, + *, + source_id_duplicate: bool, + semantic_duplicate: bool, +) -> list[dict]: + conflicts: list[dict] = [] + seen_ids: set[str] = set() + summary_source_id = str(summary.get("source_id") or "") + summary_body_key = normalize_semantic_key_text(str(summary.get("body") or "")) + try: + summary_key = course_summary_duplicate_key(summary) + except ValueError: + summary_key = ("", "", "", "", "") + + for item in iter_course_summary_markdown(root): + matched = False + if source_id_duplicate and summary_source_id and str(item.get("source_id") or "") == summary_source_id: + matched = True + if semantic_duplicate: + item_body_key = normalize_semantic_key_text(str(item.get("body") or "")) + try: + item_key = course_summary_duplicate_key(item) + except ValueError: + item_key = ("", "", "", "", "") + matched = matched or ( + item_key == summary_key + and item_body_key == summary_body_key + ) + if not matched: + continue + candidate = course_summary_duplicate_candidate(item) + candidate_id = str(candidate.get("id") or candidate.get("source_id") or "") + if candidate_id in seen_ids: + continue + seen_ids.add(candidate_id) + conflicts.append(candidate) + return conflicts + + +def course_summary_duplicate_review_context( + summaries_root: Path, + summary: dict, + seen_source_ids: set[str], + seen_semantic_keys: set[str], +) -> tuple[list[str], list[dict], bool, bool]: + source_id = str(summary.get("source_id") or "") + semantic_key = course_summary_semantic_key(summary) + source_id_duplicate = source_id in seen_source_ids + semantic_duplicate = semantic_key in seen_semantic_keys + reasons: list[str] = [] + if source_id_duplicate: + reasons.append("来源ID重复,新增课程小结需人工复核") + if semantic_duplicate: + reasons.append("学生、日期、老师、科目、时间和正文均重复,新增课程小结需人工复核") + conflicts = find_course_summary_duplicate_conflicts( + summaries_root, + summary, + source_id_duplicate=source_id_duplicate, + semantic_duplicate=semantic_duplicate, + ) + if not conflicts: + if source_id_duplicate: + reasons.append("状态文件中已存在相同来源ID,但正式小结库未找到对应文件") + if semantic_duplicate: + reasons.append("状态文件中已存在相同语义指纹,但正式小结库未找到对应文件") + return reasons, conflicts, source_id_duplicate, semantic_duplicate + + +def is_duplicate_course_summary_review_task(task: dict) -> bool: + return bool( + task.get("pending_summary_save") + or task.get("duplicate_source_id") + or task.get("duplicate_semantic_key") + or task.get("duplicate_reasons") + or task.get("duplicate_conflicts") + ) + + def course_summary_quality_score(item: dict) -> tuple[int, int, int, str]: title = str(item.get("title") or "") body = str(item.get("body") or "") @@ -2507,17 +2612,21 @@ def course_summary_heading(summary: dict, existing_headings: set[str]) -> str: return f"{base}({sha1_text(str(summary.get('source_id') or '') + str(summary.get('body') or ''), 8)})" -def save_course_summary_markdown(root: Path, summary: dict) -> dict: - path = course_summary_path(root, summary) - existing = path.read_text(encoding="utf-8") if path.exists() else "" +def course_summary_markdown_text( + existing: str, + path: Path, + summary: dict, + *, + allow_same_source_id: bool = False, +) -> tuple[str, bool, str]: matches = list(COURSE_SUMMARY_HEADING_RE.finditer(existing)) existing_headings = {match.group("title").strip() for match in matches} body = str(summary.get("body") or "").rstrip() group_name = str(summary.get("group") or summary["student"]) heading = course_summary_heading(summary, existing_headings) source_id = str(summary.get("source_id") or "").strip() - if source_id and f"来源ID:`{source_id}`" in existing: - return {"path": str(path), "added": False} + if not allow_same_source_id and source_id and f"来源ID:`{source_id}`" in existing: + return existing, False, heading existing_blocks: set[tuple[str, str]] = set() for index, match in enumerate(matches): start = match.end() @@ -2526,7 +2635,7 @@ def save_course_summary_markdown(root: Path, summary: dict) -> dict: block_body = re.sub(r"^(?:>\s+.*\n)+\s*", "", block_body).strip() existing_blocks.add((match.group("title").strip(), re.sub(r"\s+", "", block_body))) if (heading, re.sub(r"\s+", "", body)) in existing_blocks: - return {"path": str(path), "added": False} + return existing, False, heading lines: list[str] = [] if not existing.strip(): lines.extend([f"# {path.stem}", "", f"## {group_name}", ""]) @@ -2544,11 +2653,20 @@ def save_course_summary_markdown(root: Path, summary: dict) -> dict: "", ] ) + prefix = existing + if existing and not existing.endswith("\n"): + prefix += "\n" + return prefix + "\n".join(lines).rstrip() + "\n", True, heading + + +def save_course_summary_markdown(root: Path, summary: dict) -> dict: + path = course_summary_path(root, summary) + existing = path.read_text(encoding="utf-8") if path.exists() else "" + new_text, added, heading = course_summary_markdown_text(existing, path, summary) + if not added: + return {"path": str(path), "added": False, "heading": heading} path.parent.mkdir(parents=True, exist_ok=True) - with path.open("a", encoding="utf-8") as handle: - if existing and not existing.endswith("\n"): - handle.write("\n") - handle.write("\n".join(lines).rstrip() + "\n") + atomic_write_text(path, new_text) return {"path": str(path), "added": True, "heading": heading} @@ -2621,6 +2739,7 @@ def create_course_summary_review_task( proposed_line: str, reasons: list[str], saved_path: str = "", + extra_fields: dict | None = None, ) -> dict: tasks = read_admin_tasks(tasks_path) now = datetime.now().isoformat(timespec="seconds") @@ -2637,6 +2756,8 @@ def create_course_summary_review_task( "reasons": reasons, "saved_path": saved_path, } + if extra_fields: + task.update(extra_fields) tasks["next_id"] = int(tasks["next_id"]) + 1 tasks["items"].append(task) write_admin_tasks(tasks_path, tasks) @@ -2701,6 +2822,8 @@ def editable_course_summary_payload(summary: dict, updates: dict) -> dict: def update_course_summary_review_task( tasks_path: Path, + summaries_root: Path, + state_path: Path, classnotes_path: Path, accounts_path: Path, task_id: int, @@ -2715,6 +2838,39 @@ def update_course_summary_review_task( summary = task.get("summary") or {} updated_summary = editable_course_summary_payload(summary, updates) reasons, proposed_line = auto_register_reasons(updated_summary, classnotes_path, accounts_path) + if is_duplicate_course_summary_review_task(task): + state = read_course_summary_state(state_path) + source_ids = {str(item) for item in state.get("seen_source_ids", [])} + semantic_keys = {str(item) for item in state.get("seen_semantic_keys", [])} + source_ids.discard(str(summary.get("source_id") or "")) + if task.get("semantic_key"): + semantic_keys.discard(str(task.get("semantic_key") or "")) + semantic_keys.discard(course_summary_semantic_key(summary)) + duplicate_reasons, duplicate_conflicts, source_id_duplicate, semantic_duplicate = course_summary_duplicate_review_context( + summaries_root, + updated_summary, + source_ids, + semantic_keys, + ) + merged_reasons = [*duplicate_reasons] + merged_reasons.extend(reason for reason in reasons if reason not in merged_reasons) + reasons = merged_reasons + task["duplicate_reasons"] = duplicate_reasons + task["duplicate_conflicts"] = duplicate_conflicts + task["duplicate_source_id"] = source_id_duplicate + task["duplicate_semantic_key"] = semantic_duplicate + task["duplicate_source"] = summary_as_duplicate_candidate( + updated_summary, + str(course_summary_path(summaries_root, updated_summary)), + ) + task["semantic_key"] = course_summary_semantic_key(updated_summary) + task["pending_summary_save"] = True + task["saved_path"] = str(course_summary_path(summaries_root, updated_summary)) + source_ids.add(str(updated_summary.get("source_id") or "")) + semantic_keys.add(task["semantic_key"]) + state["seen_source_ids"] = sorted(source_ids) + state["seen_semantic_keys"] = sorted(semantic_keys) + write_course_summary_state(state_path, state) now = datetime.now().isoformat(timespec="seconds") task["status"] = "pending" task["updated_at"] = now @@ -2752,6 +2908,7 @@ def link_existing_course_summary_task(tasks_path: Path, classnotes_path: Path, t def approve_course_summary_task( tasks_path: Path, + summaries_root: Path, classnotes_path: Path, accounts_path: Path, task_id: int, @@ -2764,13 +2921,40 @@ def approve_course_summary_task( raise ValueError("该任务已处理,不能重复批准") proposed_line = str(task.get("proposed_line") or "").strip() + summary = task.get("summary") or {} if not proposed_line: - raise ValueError("课程小结信息未补齐,不能直接批准入账") + try: + proposed_line = course_summary_to_class_record_line(summary) + except ValueError as exc: + task["status"] = "conflict" + task["updated_at"] = datetime.now().isoformat(timespec="seconds") + task["message"] = str(exc) + write_admin_tasks(tasks_path, tasks) + raise + if not summary: + raise ValueError("课程小结信息缺失,不能批准入账") try: original_classnotes = classnotes_path.read_text(encoding="utf-8") original_accounts = accounts_path.read_text(encoding="utf-8") original_tasks = json.dumps(tasks, ensure_ascii=False, indent=2) + "\n" + should_save_summary = bool(task.get("pending_summary_save")) + summary_path: Path | None = None + original_summary_text = "" + summary_existed = False + new_summary_text = "" + summary_added = False + summary_heading = "" + if should_save_summary: + summary_path = course_summary_path(summaries_root, summary) + summary_existed = summary_path.exists() + original_summary_text = summary_path.read_text(encoding="utf-8") if summary_existed else "" + new_summary_text, summary_added, summary_heading = course_summary_markdown_text( + original_summary_text, + summary_path, + summary, + allow_same_source_id=True, + ) record = parse_class_record_line(proposed_line) record_line = class_record_to_line(record) existing_lines = {raw.strip() for raw in original_classnotes.splitlines()} @@ -2792,12 +2976,17 @@ def approve_course_summary_task( task["reviewed_at"] = task["updated_at"] task["proposed_line"] = record_line task["registered_line"] = record_line + if summary_path is not None: + task["saved_path"] = str(summary_path) + if summary_heading: + task["summary_heading"] = summary_heading backup_dir = create_data_backup( "admin-approve-course-summary", { accounts_path: original_accounts, classnotes_path: original_classnotes, tasks_path: original_tasks, + **({summary_path: original_summary_text} if summary_path is not None else {}), }, [record_line], ) @@ -2806,11 +2995,18 @@ def approve_course_summary_task( try: atomic_write_text(accounts_path, new_accounts) atomic_write_text(classnotes_path, new_classnotes) + if summary_path is not None and summary_added: + atomic_write_text(summary_path, new_summary_text) atomic_write_text(tasks_path, new_tasks) except Exception: atomic_write_text(accounts_path, original_accounts) atomic_write_text(classnotes_path, original_classnotes) atomic_write_text(tasks_path, original_tasks) + if summary_path is not None: + if summary_existed: + atomic_write_text(summary_path, original_summary_text) + elif summary_path.exists(): + summary_path.unlink() raise except ValueError as exc: task["status"] = "conflict" @@ -2828,6 +3024,7 @@ def approve_course_summary_task( def approve_admin_task( tasks_path: Path, + summaries_root: Path, classnotes_path: Path, accounts_path: Path, task_id: int, @@ -2838,7 +3035,7 @@ def approve_admin_task( if task.get("type") == "class_record_deletion": return approve_deletion_task(tasks_path, classnotes_path, accounts_path, task_id) if task.get("type") == "course_summary_review": - return approve_course_summary_task(tasks_path, classnotes_path, accounts_path, task_id) + return approve_course_summary_task(tasks_path, summaries_root, classnotes_path, accounts_path, task_id) raise ValueError("不支持的审核任务类型") @@ -2879,16 +3076,84 @@ def register_course_summary_texts( seen_source_ids = set(str(item) for item in state.get("seen_source_ids", [])) seen_semantic_keys = set(str(item) for item in state.get("seen_semantic_keys", [])) if source_id in seen_source_ids or semantic_key in seen_semantic_keys: - result["duplicates"] += 1 + duplicate_reasons, duplicate_conflicts, source_id_duplicate, semantic_duplicate = course_summary_duplicate_review_context( + summaries_root, + normalized, + seen_source_ids, + seen_semantic_keys, + ) + reasons, proposed_line = auto_register_reasons(normalized, classnotes_path, accounts_path) + if source_id_duplicate: + duplicate_reason = "来源ID重复,已转入审核" + else: + duplicate_reason = "课程内容与已有课程小结重复,已转入审核" + review_reasons = [duplicate_reason, *duplicate_reasons] + review_reasons.extend(reason for reason in reasons if reason not in review_reasons) + task = create_course_summary_review_task( + tasks_path, + normalized, + proposed_line, + review_reasons, + str(course_summary_path(summaries_root, normalized)), + extra_fields={ + "duplicate_source": summary_as_duplicate_candidate(normalized), + "duplicate_reasons": duplicate_reasons, + "duplicate_conflicts": duplicate_conflicts, + "duplicate_source_id": source_id_duplicate, + "duplicate_semantic_key": semantic_duplicate, + "semantic_key": semantic_key, + "pending_summary_save": True, + }, + ) + seen_source_ids.add(source_id) + seen_semantic_keys.add(semantic_key) + state["seen_source_ids"] = sorted(seen_source_ids) + state["seen_semantic_keys"] = sorted(seen_semantic_keys) + state.setdefault("batches", []).append( + { + "batch_id": f"manual-{datetime.now().strftime('%Y%m%d%H%M%S')}-{sha1_text(source_id, 8)}", + "received_at": now, + "window": {"source": "admin_register", "submitted_at": now}, + "students": [normalized["student"]], + "result": { + "received": 1, + "saved": 0, + "auto_registered": 0, + "review_pending": 1, + "duplicates": 1, + "rejected": 0, + }, + } + ) + state["batches"] = state["batches"][-200:] + write_course_summary_state(state_path, state) log_id = append_operation_log( operation_logs_path, "课程小结登记", - "重复", + "待审核", source_id=source_id, student=normalized["student"], + teacher=normalized.get("teacher", ""), + subject=normalized.get("subject", ""), + proposed_line=proposed_line, + reasons=review_reasons, + task_id=str(task.get("id") or ""), + saved_path=str(course_summary_path(summaries_root, normalized)), + duplicate_source_id=source_id_duplicate, + duplicate_semantic_key=semantic_duplicate, ) + result["review_pending"] += 1 + result["duplicates"] += 1 result["operation_log_ids"].append(log_id) - result["items"].append({"source_id": source_id, "status": "duplicate"}) + result["items"].append( + { + "source_id": source_id, + "status": "review_pending", + "task_id": task.get("id"), + "reasons": review_reasons, + "duplicate_conflicts": duplicate_conflicts, + } + ) continue reasons, proposed_line = auto_register_reasons(normalized, classnotes_path, accounts_path) @@ -2963,7 +3228,6 @@ def register_course_summary_texts( } ) continue - saved = save_course_summary_markdown(summaries_root, normalized) duplicate_tasks = create_course_summary_duplicate_review_tasks( tasks_path, @@ -3088,17 +3352,66 @@ def ingest_course_summaries( source_id = normalized["source_id"] semantic_key = course_summary_semantic_key(normalized) if source_id in seen_source_ids or semantic_key in seen_semantic_keys: + duplicate_reasons, duplicate_conflicts, source_id_duplicate, semantic_duplicate = course_summary_duplicate_review_context( + summaries_root, + normalized, + seen_source_ids, + seen_semantic_keys, + ) + reasons, proposed_line = auto_register_reasons(normalized, classnotes_path, accounts_path) + if source_id_duplicate: + duplicate_reason = "来源ID重复,已转入审核" + else: + duplicate_reason = "课程内容与已有课程小结重复,已转入审核" + review_reasons = [duplicate_reason, *duplicate_reasons] + review_reasons.extend(reason for reason in reasons if reason not in review_reasons) + task = create_course_summary_review_task( + tasks_path, + normalized, + proposed_line, + review_reasons, + str(course_summary_path(summaries_root, normalized)), + extra_fields={ + "duplicate_source": summary_as_duplicate_candidate(normalized), + "duplicate_reasons": duplicate_reasons, + "duplicate_conflicts": duplicate_conflicts, + "duplicate_source_id": source_id_duplicate, + "duplicate_semantic_key": semantic_duplicate, + "semantic_key": semantic_key, + "pending_summary_save": True, + }, + ) + seen_source_ids.add(source_id) + seen_semantic_keys.add(semantic_key) + result["review_pending"] += 1 result["duplicates"] += 1 log_id = append_operation_log( operation_logs_path, "课程小结接收", - "重复", + "待审核", batch_id=batch_id, source_id=source_id, student=normalized["student"], + teacher=normalized.get("teacher", ""), + subject=normalized.get("subject", ""), + proposed_line=proposed_line, + reasons=review_reasons, + task_id=task.get("id"), + saved_path=str(course_summary_path(summaries_root, normalized)), + duplicate_source_id=source_id_duplicate, + duplicate_semantic_key=semantic_duplicate, ) result["operation_log_ids"].append(log_id) - result["items"].append({"source_id": source_id, "status": "duplicate"}) + result["items"].append( + { + "source_id": source_id, + "status": "待审核", + "task_id": task.get("id"), + "backup_id": "", + "reasons": review_reasons, + "duplicate_conflicts": duplicate_conflicts, + } + ) continue saved = save_course_summary_markdown(summaries_root, normalized) diff --git a/app/app/routers/admin.py b/app/app/routers/admin.py index 3572ce5..491564b 100644 --- a/app/app/routers/admin.py +++ b/app/app/routers/admin.py @@ -10,6 +10,7 @@ from ..config import ( ADMIN_TASKS_PATH, CLASSNOTES_PATH, COURSE_SUMMARIES_ROOT, + COURSE_SUMMARY_STATE_PATH, OPERATION_LOGS_PATH, write_lock, ) @@ -129,7 +130,7 @@ def admin_course_summaries( def admin_approve_task(task_id: int, _user: str = Depends(verify_admin_auth)): try: with write_lock: - result = approve_admin_task(ADMIN_TASKS_PATH, CLASSNOTES_PATH, ACCOUNTS_PATH, task_id) + result = approve_admin_task(ADMIN_TASKS_PATH, COURSE_SUMMARIES_ROOT, CLASSNOTES_PATH, ACCOUNTS_PATH, task_id) task = result.get("task", {}) append_operation_log( OPERATION_LOGS_PATH, @@ -140,6 +141,7 @@ def admin_approve_task(task_id: int, _user: str = Depends(verify_admin_auth)): student=str(task.get("student") or task.get("corrected", {}).get("student") or ""), source_id=str(task.get("source_id") or ""), backup_id=str(result.get("backup_id") or task.get("backup_id") or ""), + saved_path=str(task.get("saved_path") or ""), ) except ValueError as exc: raise HTTPException(status_code=409, detail=str(exc)) from exc @@ -179,6 +181,8 @@ def admin_update_course_summary_review(task_id: int, payload: dict, _user: str = with write_lock: result = update_course_summary_review_task( ADMIN_TASKS_PATH, + COURSE_SUMMARIES_ROOT, + COURSE_SUMMARY_STATE_PATH, CLASSNOTES_PATH, ACCOUNTS_PATH, task_id, diff --git a/app/app/static/admin.html b/app/app/static/admin.html index c560d1e..3c3fa74 100644 --- a/app/app/static/admin.html +++ b/app/app/static/admin.html @@ -4,7 +4,7 @@