feat: ingest course summaries on vps
This commit is contained in:
yangdawei
+538
@@ -46,6 +46,10 @@ BACKUP_DIR_RE = re.compile(r"^\d{8}-\d{6}-\d{6}-")
|
||||
BACKUP_KEEP_COUNT = 50
|
||||
ACCOUNT_STATUSES = {"正常", "预警", "欠费", "结课", "退费"}
|
||||
WEEKDAYS = ["星期一", "星期二", "星期三", "星期四", "星期五", "星期六", "星期日"]
|
||||
UNKNOWN_TEACHERS = {"", "待核对老师", "未知老师"}
|
||||
UNKNOWN_SUBJECTS = {"", "待核对科目", "未知科目"}
|
||||
HIGH_CONFIDENCE_VALUES = {"high", "高", "高置信", "true", "1", "yes"}
|
||||
AUTO_RECOGNITION_SOURCES = {"keyword", "rule", "model", "model_high_confidence", "大模型高置信识别", "关键词"}
|
||||
ROLE_WORDS = {
|
||||
"student": ("学生", "学员", "孩子", "同学"),
|
||||
"teacher": ("老师", "教师"),
|
||||
@@ -847,6 +851,536 @@ def approve_correction_task(tasks_path: Path, classnotes_path: Path, task_id: in
|
||||
return {"task": task_to_dict(task), "backup_id": backup_dir.name}
|
||||
|
||||
|
||||
def safe_filename_part(value: object) -> str:
|
||||
text = str(value or "").strip()
|
||||
text = re.sub(r"[\\/:*?\"<>|\s]+", "_", text)
|
||||
return text.strip("._") or "未命名"
|
||||
|
||||
|
||||
def sha1_text(value: str, length: int = 16) -> str:
|
||||
return hashlib.sha1(value.encode("utf-8")).hexdigest()[:length]
|
||||
|
||||
|
||||
def payload_bool(value: object) -> bool:
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
if isinstance(value, (int, float)):
|
||||
return value != 0
|
||||
text = str(value or "").strip().lower()
|
||||
return text in {"1", "true", "yes", "y", "高", "高置信", "可信"}
|
||||
|
||||
|
||||
def normalize_summary_date(value: object) -> str:
|
||||
text = str(value or "").strip().replace(".", "-")
|
||||
if not text:
|
||||
raise ValueError("课程小结缺少日期")
|
||||
parsed = datetime.strptime(text, "%Y-%m-%d").date()
|
||||
return parsed.isoformat()
|
||||
|
||||
|
||||
def normalize_time_range_text(value: object) -> str:
|
||||
text = str(value or "").strip()
|
||||
if not text:
|
||||
return ""
|
||||
match = TIME_RANGE_RE.fullmatch(text)
|
||||
if not match:
|
||||
raise ValueError(f"课程小结时间段格式错误: {text}")
|
||||
start_hour = int(match.group("sh"))
|
||||
start_minute = int(match.group("sm"))
|
||||
end_hour = int(match.group("eh"))
|
||||
end_minute = int(match.group("em"))
|
||||
if start_hour > 23 or end_hour > 23 or start_minute > 59 or end_minute > 59:
|
||||
raise ValueError(f"课程小结时间段超出范围: {text}")
|
||||
if end_hour * 60 + end_minute <= start_hour * 60 + start_minute:
|
||||
raise ValueError(f"课程小结结束时间必须晚于开始时间: {text}")
|
||||
return f"{start_hour:02d}:{start_minute:02d}-{end_hour:02d}:{end_minute:02d}"
|
||||
|
||||
|
||||
def duration_text_from_minutes(minutes: int) -> str:
|
||||
return f"{minutes // 60}小时{minutes % 60}分"
|
||||
|
||||
|
||||
def duration_minutes_from_time_range(time_range: str) -> int | None:
|
||||
if not time_range:
|
||||
return None
|
||||
match = TIME_RANGE_RE.fullmatch(time_range)
|
||||
if not match:
|
||||
return None
|
||||
start = int(match.group("sh")) * 60 + int(match.group("sm"))
|
||||
end = int(match.group("eh")) * 60 + int(match.group("em"))
|
||||
return end - start if end > start else None
|
||||
|
||||
|
||||
def duration_minutes_from_summary(summary: dict) -> int | None:
|
||||
raw_duration = summary.get("duration") or summary.get("duration_text") or ""
|
||||
if raw_duration:
|
||||
return int(round(parse_hours_text(str(raw_duration)) * 60))
|
||||
for key in ("duration_minutes", "minutes"):
|
||||
value = summary.get(key)
|
||||
if value not in (None, ""):
|
||||
return int(round(float(value)))
|
||||
for key in ("duration_hours", "hours"):
|
||||
value = summary.get(key)
|
||||
if value not in (None, ""):
|
||||
return int(round(float(value) * 60))
|
||||
return duration_minutes_from_time_range(str(summary.get("time_range") or ""))
|
||||
|
||||
|
||||
def normalize_course_summary(raw: dict) -> dict:
|
||||
student = canonical_name(str(raw.get("student") or "").strip())
|
||||
teacher = canonical_name(str(raw.get("teacher") or "").strip())
|
||||
subject = parse_subject_code(str(raw.get("subject") or "").strip())
|
||||
body = str(raw.get("body") or raw.get("content") or "").strip()
|
||||
if not student:
|
||||
raise ValueError("课程小结缺少学生")
|
||||
if not body:
|
||||
raise ValueError("课程小结缺少正文")
|
||||
date_iso = normalize_summary_date(raw.get("date_iso") or raw.get("date") or raw.get("class_date"))
|
||||
time_range = normalize_time_range_text(raw.get("time_range") or raw.get("raw_time") or raw.get("time") or "")
|
||||
minutes = duration_minutes_from_summary({**raw, "time_range": time_range})
|
||||
if time_range and minutes is not None:
|
||||
time_minutes = duration_minutes_from_time_range(time_range)
|
||||
if time_minutes is not None and abs(time_minutes - minutes) > 1:
|
||||
raise ValueError(f"课程小结时间段和时长不一致: {time_range} / {duration_text_from_minutes(minutes)}")
|
||||
source_id = str(raw.get("source_id") or "").strip()
|
||||
if not source_id:
|
||||
source_parts = [
|
||||
str(raw.get("db") or ""),
|
||||
str(raw.get("local_id") or ""),
|
||||
student,
|
||||
date_iso,
|
||||
teacher,
|
||||
subject,
|
||||
time_range,
|
||||
body[:200],
|
||||
]
|
||||
source_id = sha1_text("|".join(source_parts), 24)
|
||||
summary = {
|
||||
"source_id": source_id,
|
||||
"student": student,
|
||||
"date_iso": date_iso,
|
||||
"time_range": time_range,
|
||||
"duration_minutes": minutes,
|
||||
"duration": duration_text_from_minutes(minutes) if minutes is not None else "",
|
||||
"teacher": teacher,
|
||||
"subject": subject,
|
||||
"group": str(raw.get("group") or "").strip(),
|
||||
"sender": str(raw.get("sender") or raw.get("sender_name") or "").strip(),
|
||||
"sender_id": str(raw.get("sender_id") or "").strip(),
|
||||
"message_time": str(raw.get("message_time") or "").strip(),
|
||||
"message_date": str(raw.get("message_date") or "").strip(),
|
||||
"db": str(raw.get("db") or "").strip(),
|
||||
"local_id": str(raw.get("local_id") or "").strip(),
|
||||
"title": str(raw.get("title") or "").strip(),
|
||||
"body": body,
|
||||
"recognition_source": str(raw.get("recognition_source") or raw.get("source") or "").strip(),
|
||||
"confidence": str(raw.get("confidence") or "").strip(),
|
||||
"teacher_trusted": payload_bool(raw.get("teacher_trusted") or raw.get("sender_teacher_trusted")),
|
||||
"remark": str(raw.get("remark") or "").strip(),
|
||||
}
|
||||
if not summary["message_date"] and len(summary["message_time"]) >= 10:
|
||||
summary["message_date"] = summary["message_time"][:10]
|
||||
return summary
|
||||
|
||||
|
||||
def course_summary_semantic_key(summary: dict) -> str:
|
||||
body_digest = sha1_text(re.sub(r"\s+", "", str(summary.get("body") or "")), 12)
|
||||
parts = [
|
||||
summary.get("student", ""),
|
||||
summary.get("date_iso", ""),
|
||||
summary.get("teacher", ""),
|
||||
normalize_subject(str(summary.get("subject") or "")),
|
||||
summary.get("time_range", ""),
|
||||
str(summary.get("duration_minutes") or ""),
|
||||
body_digest,
|
||||
]
|
||||
return "|".join(str(part) for part in parts)
|
||||
|
||||
|
||||
def default_course_summary_state() -> dict:
|
||||
return {"version": 1, "seen_source_ids": [], "seen_semantic_keys": [], "batches": []}
|
||||
|
||||
|
||||
def read_course_summary_state(path: Path) -> dict:
|
||||
if not path.exists():
|
||||
return default_course_summary_state()
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
except json.JSONDecodeError as exc:
|
||||
raise ValueError(f"课程小结状态文件 JSON 格式错误: {path}") from exc
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError("课程小结状态文件必须是 JSON 对象")
|
||||
payload.setdefault("version", 1)
|
||||
payload.setdefault("seen_source_ids", [])
|
||||
payload.setdefault("seen_semantic_keys", [])
|
||||
payload.setdefault("batches", [])
|
||||
return payload
|
||||
|
||||
|
||||
def write_course_summary_state(path: Path, state: dict) -> None:
|
||||
atomic_write_text(path, json.dumps(state, ensure_ascii=False, indent=2) + "\n")
|
||||
|
||||
|
||||
def append_operation_log(path: Path, operation: str, status: str, **fields: object) -> str:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
now = datetime.now().isoformat(timespec="seconds")
|
||||
log_id = f"{datetime.now().strftime('%Y%m%d%H%M%S')}-{sha1_text(json.dumps(fields, ensure_ascii=False, sort_keys=True), 8)}"
|
||||
row = {
|
||||
"id": log_id,
|
||||
"created_at": now,
|
||||
"operation": operation,
|
||||
"status": status,
|
||||
**fields,
|
||||
}
|
||||
with path.open("a", encoding="utf-8") as handle:
|
||||
handle.write(json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n")
|
||||
return log_id
|
||||
|
||||
|
||||
def list_operation_logs(path: Path, limit: int = 100, operation: str = "", status_filter: str = "", student: str = "") -> dict:
|
||||
rows: list[dict] = []
|
||||
if path.exists():
|
||||
for raw_line in path.read_text(encoding="utf-8").splitlines():
|
||||
if not raw_line.strip():
|
||||
continue
|
||||
try:
|
||||
item = json.loads(raw_line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if operation and item.get("operation") != operation:
|
||||
continue
|
||||
if status_filter and item.get("status") != status_filter:
|
||||
continue
|
||||
if student and student not in str(item.get("student", "")):
|
||||
continue
|
||||
rows.append(item)
|
||||
rows = rows[-limit:]
|
||||
rows.reverse()
|
||||
return {"count": len(rows), "items": rows}
|
||||
|
||||
|
||||
def course_summary_path(root: Path, summary: dict) -> Path:
|
||||
student = safe_filename_part(summary["student"])
|
||||
teacher = safe_filename_part(summary["teacher"] or "待核对老师")
|
||||
subject = safe_filename_part(summary["subject"] or "待核对科目")
|
||||
return root / student / f"{student}_{teacher}_{subject}.md"
|
||||
|
||||
|
||||
def course_summary_heading(summary: dict, existing_headings: set[str]) -> str:
|
||||
subject = normalize_subject(str(summary.get("subject") or "待核对科目"))
|
||||
time_range = str(summary.get("time_range") or "").strip()
|
||||
base = f"{summary['date_iso']} {time_range + ' ' if time_range else ''}{subject}课堂小结"
|
||||
if base not in existing_headings:
|
||||
return base
|
||||
return f"{base}（{sha1_text(str(summary.get('source_id') or '') + str(summary.get('body') or ''), 8)}）"
|
||||
|
||||
|
||||
def save_course_summary_markdown(root: Path, summary: dict) -> dict:
|
||||
path = course_summary_path(root, summary)
|
||||
existing = path.read_text(encoding="utf-8") if path.exists() else ""
|
||||
existing_headings = set(re.findall(r"^###\s+(.+)$", existing, flags=re.M))
|
||||
existing_compact = re.sub(r"\s+", "", existing)
|
||||
body = str(summary.get("body") or "").rstrip()
|
||||
body_compact = re.sub(r"\s+", "", body)
|
||||
if body_compact and body_compact in existing_compact:
|
||||
return {"path": str(path), "added": False}
|
||||
|
||||
group_name = str(summary.get("group") or summary["student"])
|
||||
heading = course_summary_heading(summary, existing_headings)
|
||||
lines: list[str] = []
|
||||
if not existing.strip():
|
||||
lines.extend([f"# {path.stem}", "", f"## {group_name}", ""])
|
||||
elif f"## {group_name}" not in existing:
|
||||
lines.extend(["", f"## {group_name}", ""])
|
||||
lines.extend(
|
||||
[
|
||||
f"### {heading}",
|
||||
"",
|
||||
f"> 来源ID：`{summary['source_id']}`",
|
||||
f"> 发送时间：`{summary.get('message_time') or ''}`",
|
||||
f"> 发送者：`{summary.get('sender') or ''}`",
|
||||
"",
|
||||
body,
|
||||
"",
|
||||
]
|
||||
)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("a", encoding="utf-8") as handle:
|
||||
if existing and not existing.endswith("\n"):
|
||||
handle.write("\n")
|
||||
handle.write("\n".join(lines).rstrip() + "\n")
|
||||
return {"path": str(path), "added": True, "heading": heading}
|
||||
|
||||
|
||||
def message_date_after_class_date(summary: dict) -> bool:
|
||||
message_date = str(summary.get("message_date") or "")
|
||||
if len(message_date) != 10:
|
||||
return False
|
||||
try:
|
||||
return date.fromisoformat(str(summary["date_iso"])) > date.fromisoformat(message_date)
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
|
||||
def course_summary_to_class_record_line(summary: dict) -> str:
|
||||
date_iso = normalize_summary_date(summary.get("date_iso"))
|
||||
record_date = date_iso.replace("-", ".")
|
||||
weekday = WEEKDAYS[date.fromisoformat(date_iso).weekday()]
|
||||
time_range = normalize_time_range_text(summary.get("time_range"))
|
||||
minutes = summary.get("duration_minutes")
|
||||
if minutes is None:
|
||||
raise ValueError("课程小结缺少时长")
|
||||
duration = duration_text_from_minutes(int(minutes))
|
||||
return (
|
||||
f"{record_date}-{weekday}-{time_range}-{summary['student']}-"
|
||||
f"{duration}-{summary['teacher']}-{normalize_subject(str(summary['subject']))}"
|
||||
)
|
||||
|
||||
|
||||
def auto_register_reasons(summary: dict, classnotes_path: Path, accounts_path: Path) -> tuple[list[str], str]:
|
||||
reasons: list[str] = []
|
||||
confidence = str(summary.get("confidence") or "").lower()
|
||||
recognition_source = str(summary.get("recognition_source") or "")
|
||||
if confidence not in HIGH_CONFIDENCE_VALUES and recognition_source not in AUTO_RECOGNITION_SOURCES:
|
||||
reasons.append("识别置信度不足")
|
||||
if not summary.get("teacher_trusted"):
|
||||
reasons.append("发送者老师映射未确认")
|
||||
if summary.get("teacher") in UNKNOWN_TEACHERS:
|
||||
reasons.append("老师待核对")
|
||||
if normalize_subject(str(summary.get("subject") or "")) in UNKNOWN_SUBJECTS:
|
||||
reasons.append("科目待核对")
|
||||
if not summary.get("time_range"):
|
||||
reasons.append("时间段缺失")
|
||||
if summary.get("duration_minutes") is None:
|
||||
reasons.append("时长缺失")
|
||||
if message_date_after_class_date(summary):
|
||||
reasons.append("课程日期晚于消息发送日期")
|
||||
|
||||
line = ""
|
||||
try:
|
||||
line = course_summary_to_class_record_line(summary)
|
||||
parse_class_record_line(line)
|
||||
except ValueError as exc:
|
||||
reasons.append(str(exc))
|
||||
|
||||
try:
|
||||
find_account_index(read_accounts(accounts_path), str(summary["student"]))
|
||||
except ValueError as exc:
|
||||
reasons.append(str(exc))
|
||||
|
||||
if line:
|
||||
existing_lines = {raw.strip() for raw in classnotes_path.read_text(encoding="utf-8").splitlines()}
|
||||
if line in existing_lines:
|
||||
reasons.append("classnotes 已存在同一条上课记录")
|
||||
return reasons, line
|
||||
|
||||
|
||||
def create_course_summary_review_task(
|
||||
tasks_path: Path,
|
||||
summary: dict,
|
||||
proposed_line: str,
|
||||
reasons: list[str],
|
||||
saved_path: str = "",
|
||||
) -> dict:
|
||||
tasks = read_admin_tasks(tasks_path)
|
||||
now = datetime.now().isoformat(timespec="seconds")
|
||||
task = {
|
||||
"id": int(tasks["next_id"]),
|
||||
"type": "course_summary_review",
|
||||
"status": "pending",
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
"source_id": summary["source_id"],
|
||||
"student": summary["student"],
|
||||
"summary": summary,
|
||||
"proposed_line": proposed_line,
|
||||
"reasons": reasons,
|
||||
"saved_path": saved_path,
|
||||
}
|
||||
tasks["next_id"] = int(tasks["next_id"]) + 1
|
||||
tasks["items"].append(task)
|
||||
write_admin_tasks(tasks_path, tasks)
|
||||
return task_to_dict(task)
|
||||
|
||||
|
||||
def approve_course_summary_task(
|
||||
tasks_path: Path,
|
||||
classnotes_path: Path,
|
||||
accounts_path: Path,
|
||||
task_id: int,
|
||||
) -> dict:
|
||||
tasks = read_admin_tasks(tasks_path)
|
||||
task = find_admin_task(tasks, task_id)
|
||||
if task.get("type") != "course_summary_review":
|
||||
raise ValueError("该任务不是课程小结审核")
|
||||
if task.get("status") not in {"pending", "conflict"}:
|
||||
raise ValueError("该任务已处理，不能重复批准")
|
||||
|
||||
proposed_line = str(task.get("proposed_line") or "").strip()
|
||||
if not proposed_line:
|
||||
summary = dict(task.get("summary") or {})
|
||||
proposed_line = course_summary_to_class_record_line(summary)
|
||||
|
||||
try:
|
||||
result = register_class_record_lines(classnotes_path, accounts_path, line=proposed_line)
|
||||
except ValueError as exc:
|
||||
task["status"] = "conflict"
|
||||
task["updated_at"] = datetime.now().isoformat(timespec="seconds")
|
||||
task["message"] = str(exc)
|
||||
write_admin_tasks(tasks_path, tasks)
|
||||
raise
|
||||
|
||||
task["status"] = "approved"
|
||||
task["updated_at"] = datetime.now().isoformat(timespec="seconds")
|
||||
task["reviewed_at"] = task["updated_at"]
|
||||
task["registered_line"] = proposed_line
|
||||
task["backup_id"] = result.get("backup_id", "")
|
||||
write_admin_tasks(tasks_path, tasks)
|
||||
return {"task": task_to_dict(task), "backup_id": result.get("backup_id", "")}
|
||||
|
||||
|
||||
def approve_admin_task(
|
||||
tasks_path: Path,
|
||||
classnotes_path: Path,
|
||||
accounts_path: Path,
|
||||
task_id: int,
|
||||
) -> dict:
|
||||
task = find_admin_task(read_admin_tasks(tasks_path), task_id)
|
||||
if task.get("type") == "class_record_correction":
|
||||
return approve_correction_task(tasks_path, classnotes_path, task_id)
|
||||
if task.get("type") == "course_summary_review":
|
||||
return approve_course_summary_task(tasks_path, classnotes_path, accounts_path, task_id)
|
||||
raise ValueError("不支持的审核任务类型")
|
||||
|
||||
|
||||
def ingest_course_summaries(
|
||||
*,
|
||||
classnotes_path: Path,
|
||||
accounts_path: Path,
|
||||
tasks_path: Path,
|
||||
summaries_root: Path,
|
||||
state_path: Path,
|
||||
operation_logs_path: Path,
|
||||
batch_id: str,
|
||||
window: dict,
|
||||
students: list[str],
|
||||
summaries: list[dict],
|
||||
) -> dict:
|
||||
state = read_course_summary_state(state_path)
|
||||
seen_source_ids = set(str(item) for item in state.get("seen_source_ids", []))
|
||||
seen_semantic_keys = set(str(item) for item in state.get("seen_semantic_keys", []))
|
||||
result = {
|
||||
"received": len(summaries),
|
||||
"saved": 0,
|
||||
"auto_registered": 0,
|
||||
"review_pending": 0,
|
||||
"duplicates": 0,
|
||||
"rejected": 0,
|
||||
"operation_log_ids": [],
|
||||
"items": [],
|
||||
}
|
||||
|
||||
for raw in summaries:
|
||||
normalized: dict | None = None
|
||||
try:
|
||||
normalized = normalize_course_summary(raw)
|
||||
source_id = normalized["source_id"]
|
||||
semantic_key = course_summary_semantic_key(normalized)
|
||||
if source_id in seen_source_ids or semantic_key in seen_semantic_keys:
|
||||
result["duplicates"] += 1
|
||||
log_id = append_operation_log(
|
||||
operation_logs_path,
|
||||
"course_summary_ingest",
|
||||
"duplicate",
|
||||
batch_id=batch_id,
|
||||
source_id=source_id,
|
||||
student=normalized["student"],
|
||||
)
|
||||
result["operation_log_ids"].append(log_id)
|
||||
result["items"].append({"source_id": source_id, "status": "duplicate"})
|
||||
continue
|
||||
|
||||
saved = save_course_summary_markdown(summaries_root, normalized)
|
||||
result["saved"] += 1 if saved.get("added") else 0
|
||||
|
||||
reasons, proposed_line = auto_register_reasons(normalized, classnotes_path, accounts_path)
|
||||
if reasons:
|
||||
task = create_course_summary_review_task(
|
||||
tasks_path,
|
||||
normalized,
|
||||
proposed_line,
|
||||
reasons,
|
||||
saved_path=str(saved.get("path") or ""),
|
||||
)
|
||||
result["review_pending"] += 1
|
||||
status_value = "review"
|
||||
task_id = task.get("id")
|
||||
backup_id = ""
|
||||
else:
|
||||
register_result = register_class_record_lines(classnotes_path, accounts_path, line=proposed_line)
|
||||
result["auto_registered"] += 1
|
||||
status_value = "auto_registered"
|
||||
task_id = None
|
||||
backup_id = str(register_result.get("backup_id") or "")
|
||||
|
||||
seen_source_ids.add(source_id)
|
||||
seen_semantic_keys.add(semantic_key)
|
||||
log_id = append_operation_log(
|
||||
operation_logs_path,
|
||||
"course_summary_ingest",
|
||||
status_value,
|
||||
batch_id=batch_id,
|
||||
source_id=source_id,
|
||||
student=normalized["student"],
|
||||
teacher=normalized.get("teacher", ""),
|
||||
subject=normalized.get("subject", ""),
|
||||
proposed_line=proposed_line,
|
||||
reasons=reasons,
|
||||
task_id=task_id,
|
||||
backup_id=backup_id,
|
||||
saved_path=str(saved.get("path") or ""),
|
||||
)
|
||||
result["operation_log_ids"].append(log_id)
|
||||
result["items"].append(
|
||||
{
|
||||
"source_id": source_id,
|
||||
"status": status_value,
|
||||
"task_id": task_id,
|
||||
"backup_id": backup_id,
|
||||
"reasons": reasons,
|
||||
}
|
||||
)
|
||||
except Exception as exc:
|
||||
result["rejected"] += 1
|
||||
source_id = str((normalized or raw).get("source_id") or "")
|
||||
log_id = append_operation_log(
|
||||
operation_logs_path,
|
||||
"course_summary_ingest",
|
||||
"rejected",
|
||||
batch_id=batch_id,
|
||||
source_id=source_id,
|
||||
student=str((normalized or raw).get("student") or ""),
|
||||
error=str(exc),
|
||||
)
|
||||
result["operation_log_ids"].append(log_id)
|
||||
result["items"].append({"source_id": source_id, "status": "rejected", "error": str(exc)})
|
||||
|
||||
state["seen_source_ids"] = sorted(seen_source_ids)
|
||||
state["seen_semantic_keys"] = sorted(seen_semantic_keys)
|
||||
state.setdefault("batches", []).append(
|
||||
{
|
||||
"batch_id": batch_id,
|
||||
"received_at": datetime.now().isoformat(timespec="seconds"),
|
||||
"window": window,
|
||||
"students": students,
|
||||
"result": {key: result[key] for key in ("received", "saved", "auto_registered", "review_pending", "duplicates", "rejected")},
|
||||
}
|
||||
)
|
||||
state["batches"] = state["batches"][-200:]
|
||||
write_course_summary_state(state_path, state)
|
||||
return result
|
||||
|
||||
|
||||
def parse_record_date(text: str) -> date:
|
||||
return datetime.strptime(text, "%Y.%m.%d").date()
|
||||
|
||||
@@ -1065,6 +1599,10 @@ def parse_subject_code(text: str) -> str:
|
||||
return "".join(result)
|
||||
|
||||
|
||||
def normalize_subject(text: str) -> str:
|
||||
return parse_subject_code(text).strip()
|
||||
|
||||
|
||||
def detect_subjects(query: str) -> list[str]:
|
||||
subjects = {subject for subject in SUBJECTS if subject in query}
|
||||
for alias, subject in SUBJECT_ALIASES.items():
|
||||
|
||||
Reference in New Issue
Block a user