From 65d37c8093fc0eb6b2ff02b0e20bed0c603867e2 Mon Sep 17 00:00:00 2001 From: shanshi <3316401569@qq.com> Date: Mon, 16 Nov 2026 16:41:07 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=8E=A5=E5=85=A5=E7=BD=91=E5=95=86?= =?UTF-8?q?=E9=93=B6=E8=A1=8C=E9=82=AE=E4=BB=B6=E8=B4=A6=E5=8D=95=E8=87=AA?= =?UTF-8?q?=E5=8A=A8=E5=AF=BC=E5=85=A5=20(#1199)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 独立邮箱机器人读取网商银行加密账单,解密并导入财务台账。 --- scripts/wangshang-mail/README.md | 49 ++ scripts/wangshang-mail/fetch_daily.py | 642 ++++++++++++++++++ scripts/wangshang-mail/import-accounts.ps1 | 74 ++ scripts/wangshang-mail/install-daily-task.ps1 | 69 ++ .../run-wangshang-mail-daily.cmd | 13 + .../run-wangshang-mail-daily.ps1 | 180 +++++ 6 files changed, 1027 insertions(+) create mode 100644 scripts/wangshang-mail/README.md create mode 100644 scripts/wangshang-mail/fetch_daily.py create mode 100644 scripts/wangshang-mail/import-accounts.ps1 create mode 100644 scripts/wangshang-mail/install-daily-task.ps1 create mode 100644 scripts/wangshang-mail/run-wangshang-mail-daily.cmd create mode 100644 scripts/wangshang-mail/run-wangshang-mail-daily.ps1 diff --git a/scripts/wangshang-mail/README.md b/scripts/wangshang-mail/README.md new file mode 100644 index 00000000..17fa16a2 --- /dev/null +++ b/scripts/wangshang-mail/README.md @@ -0,0 +1,49 @@ +# 网商银行邮件账单 + +每天从网易邮箱读取网商银行电子凭证,解压 ZIP,转换交易明细 CSV,并导入财务台账“动账明细”。只处理 `runtime/private/wangshang-mail-accounts.json` 中启用的公司。 + +## 首次配置 + +1. 在网易邮箱开启 IMAP,并生成客户端授权码。 +2. 从公司表生成私有配置: + +```powershell +pwsh -File scripts\wangshang-mail\import-accounts.ps1 +``` + +默认读取 `D:\Desktop\网商数据抓取.xlsx`,生成: + +- `runtime/private/wangshang-mail-accounts.json`:邮箱授权码、公司、账号和解压密码。 +- `runtime/private/wangshang-mail/`:下载账单、导入批次和日志。 + +邮箱授权码只在终端读取,不写入日志。私有配置目录已被 `.gitignore` 忽略。 + +## 手动运行 + +```powershell +# 默认读取昨天邮件中的昨天交易账单 +pwsh -File scripts\wangshang-mail\run-wangshang-mail-daily.ps1 + +# 只下载与解析,不上传 +pwsh -File scripts\wangshang-mail\run-wangshang-mail-daily.ps1 -Date 2026-11-15 -SkipImport + +# 上传并只校验,不写台账 +pwsh -File scripts\wangshang-mail\run-wangshang-mail-daily.ps1 -Date 2026-11-15 -DryRun +``` + +## 每日任务 + +```powershell +pwsh -File scripts\wangshang-mail\install-daily-task.ps1 +``` + +任务名为 `CaiWuXiTong-WangshangMailDaily`,默认每天 10:00 读取当天邮件中的前一日交易明细。电脑休眠后恢复时补跑。 + +## 数据规则 + +- 按附件文件名中的网商账号和交易日匹配,不依赖邮件标题中的脱敏公司名。 +- 每家公司必须恰好匹配一封内容一致的当日账单;缺失或同一天出现不同附件时,该公司不导入。 +- ZIP 解压密码取账号表中的公司密码;账单企业名称和账号必须与配置一致。 +- Excel 使用只读流式方式遍历,并转换成 CSV;导入器按 CSV 流式读取。 +- 先完成全部账户解析,再生成批次;部分账户失败时,成功账户仍可导入,任务以失败状态结束。 +- 每批按“网商银行来源 + 账号 + 日期”替换,最后统一重算一次台账。 diff --git a/scripts/wangshang-mail/fetch_daily.py b/scripts/wangshang-mail/fetch_daily.py new file mode 100644 index 00000000..4ee06075 --- /dev/null +++ b/scripts/wangshang-mail/fetch_daily.py @@ -0,0 +1,642 @@ +#!/usr/bin/env python3 +"""Download and normalize Wangshang Bank daily statements received by email.""" + +from __future__ import annotations + +import argparse +import csv +import email +import email.policy +import hashlib +import imaplib +import io +import json +import os +import re +import ssl +import sys +import warnings +import zipfile +from datetime import date, datetime, timedelta +from email.utils import parsedate_to_datetime +from pathlib import Path +from typing import Any, Iterable +from zoneinfo import ZoneInfo + +try: + import openpyxl +except ImportError as exception: # pragma: no cover - guarded by the wrapper + raise SystemExit("缺少 openpyxl,无法读取网商银行账单 Excel。") from exception + + +SHANGHAI = ZoneInfo("Asia/Shanghai") +MONTHS = ("Jan", "Feb", "Mar", "Apr", "May", "Jun", + "Jul", "Aug", "Sep", "Oct", "Nov", "Dec") +AMOUNT_HEADERS = { + "借方金额(收)", "借方金额(收)", "贷方金额(支)", "贷方金额(支)", + "收入", "收入金额", "支出", "支出金额", "金额", "交易金额", +} +STATUS_HEADERS = {"交易状态", "状态"} +FAILED_STATUS = re.compile(r"关闭|失败|取消|退票") +ACCOUNT_FILE = re.compile(r"(?P\d{10,30})-日账单-(?P\d{8})-交易明细") + + +class StatementError(RuntimeError): + """A statement could not be matched or safely parsed.""" + + +def text(value: Any) -> str: + if value is None: + return "" + if isinstance(value, datetime): + return value.strftime("%Y-%m-%d %H:%M:%S") + if isinstance(value, date): + return value.strftime("%Y-%m-%d") + if isinstance(value, float): + return format(value, ".15g") + return str(value).strip() + + +def normalized(value: Any) -> str: + return re.sub(r"\s+", "", text(value)) + + +def account_digits(value: Any) -> str: + return re.sub(r"\D+", "", text(value)) + + +def mask_account(value: Any) -> str: + digits = account_digits(value) + return f"****{digits[-4:]}" if len(digits) >= 4 else "****" + + +def parse_date(value: str) -> date: + try: + parsed = date.fromisoformat(value) + except ValueError as exception: + raise StatementError("日期必须使用 YYYY-MM-DD 格式。") from exception + if parsed.isoformat() != value: + raise StatementError("日期必须使用 YYYY-MM-DD 格式。") + return parsed + + +def imap_date(value: date) -> str: + return f"{value.day:02d}-{MONTHS[value.month - 1]}-{value.year}" + + +def write_json(path: Path, payload: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text( + json.dumps(payload, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + os.replace(temporary, path) + + +def next_nonempty(row: Iterable[Any], start: int = 1) -> str: + values = list(row) + for value in values[start:]: + current = text(value) + if current: + return current + return "" + + +def locate_row(rows: list[tuple[Any, ...]], *labels: str) -> tuple[int, tuple[Any, ...]]: + expected = set(labels) + for index, row in enumerate(rows): + if expected.issubset({normalized(value) for value in row}): + return index, row + raise StatementError("账号表中的邮箱或公司字段不完整。") + + +def init_config(args: argparse.Namespace) -> int: + warnings.filterwarnings("ignore", category=UserWarning, module="openpyxl") + workbook_path = Path(args.workbook).expanduser().resolve() + if not workbook_path.is_file(): + raise StatementError(f"账号表不存在:{workbook_path}") + + auth_code = os.environ.get(args.auth_code_env, "").strip() + if not auth_code: + raise StatementError(f"环境变量 {args.auth_code_env} 中没有邮箱授权码。") + + workbook = openpyxl.load_workbook(workbook_path, read_only=False, data_only=True) + try: + sheet = workbook.active + rows = [ + tuple(row[: max(12, sheet.max_column)]) + for row in sheet.iter_rows(values_only=True) + ] + _, mail_url_row = locate_row(rows, "邮箱网址") + _, login_row = locate_row(rows, "登录账号") + _, company_header_row = locate_row(rows, "公司名称", "解压密码", "网商银行账号") + finally: + workbook.close() + + mail_url = next_nonempty(mail_url_row) + login_account = next_nonempty(login_row) + if not mail_url or not login_account: + raise StatementError("账号表缺少邮箱网址或登录账号。") + + host = args.imap_host.strip() + if not host: + host = "imap.163.com" if "163.com" in mail_url.lower() else "" + if not host: + raise StatementError("无法从邮箱网址识别 IMAP 服务器,请使用 --imap-host 指定。") + + username = login_account + if "@" not in username and host.endswith("163.com"): + username += "@163.com" + + header_position = rows.index(company_header_row) + columns = { + normalized(value): index + for index, value in enumerate(company_header_row) + if normalized(value) + } + accounts: list[dict[str, Any]] = [] + for row_number, row in enumerate(rows[header_position + 1:], header_position + 2): + company = text(row[columns["公司名称"]]) + password = text(row[columns["解压密码"]]) + bank_account = account_digits(row[columns["网商银行账号"]]) + if not company and not bank_account: + continue + if not company or not password or len(bank_account) < 6: + raise StatementError( + f"账号表第 {row_number} 行缺少公司名称、解压密码或有效网商账号。" + ) + accounts.append({ + "company": company, + "account_short": company, + "bank_account": bank_account, + "password": password, + "enabled": True, + }) + + if not accounts: + raise StatementError("账号表中没有可接入的网商公司。") + if len({item["bank_account"] for item in accounts}) != len(accounts): + raise StatementError("账号表存在重复网商账号。") + + config = { + "version": 1, + "generated_at": datetime.now(SHANGHAI).isoformat(timespec="seconds"), + "mail": { + "host": host, + "port": int(args.imap_port), + "username": username, + "auth_code": auth_code, + "sender": "service@mail.mybank.cn", + "subject": "浙江网商银行电子凭证", + "search_grace_days": 2, + }, + "accounts": accounts, + } + output = Path(args.output).expanduser().resolve() + write_json(output, config) + print(f"已生成网商邮件配置:{len(accounts)} 个公司") + print(f"邮箱:{username},IMAP:{host}:{args.imap_port}") + for item in accounts: + print(f"- {item['company']},账号 {mask_account(item['bank_account'])}") + return 0 + + +def connect_mail(mail: dict[str, Any]) -> imaplib.IMAP4_SSL: + imaplib.Commands["ID"] = ("AUTH",) + connection = imaplib.IMAP4_SSL( + str(mail["host"]), + int(mail.get("port", 993)), + ssl_context=ssl.create_default_context(), + ) + connection.login(str(mail["username"]), str(mail["auth_code"])) + connection._simple_command( + "ID", + '("name" "CaiWuXiTong" "version" "1.0" "vendor" "CaiWuXiTong")', + ) + status, data = connection.select("INBOX", readonly=True) + if status != "OK": + connection.logout() + raise StatementError(f"无法打开邮箱收件箱:{data!r}") + return connection + + +def message_date(message: email.message.Message) -> datetime: + try: + parsed = parsedate_to_datetime(str(message.get("Date", ""))) + if parsed.tzinfo is None: + return parsed.replace(tzinfo=SHANGHAI) + return parsed.astimezone(SHANGHAI) + except (TypeError, ValueError): + return datetime.min.replace(tzinfo=SHANGHAI) + + +def load_candidates( + mail: dict[str, Any], + target_date: date, +) -> list[dict[str, Any]]: + connection = connect_mail(mail) + candidates: list[dict[str, Any]] = [] + since = target_date - timedelta(days=max(1, int(mail.get("search_grace_days", 2)))) + try: + status, data = connection.uid("search", None, "SINCE", imap_date(since)) + if status != "OK": + raise StatementError(f"邮箱搜索失败:{data!r}") + uids = data[0].split() if data and data[0] else [] + sender_match = str(mail.get("sender", "service@mail.mybank.cn")).lower() + subject_match = str(mail.get("subject", "浙江网商银行电子凭证")) + + for uid in uids: + status, parts = connection.uid( + "fetch", + uid, + "(BODY.PEEK[HEADER.FIELDS (FROM SUBJECT DATE)])", + ) + if status != "OK": + continue + raw_header = b"".join(part[1] for part in parts if isinstance(part, tuple)) + header = email.message_from_bytes(raw_header, policy=email.policy.default) + sender = str(header.get("From", "")).lower() + subject = str(header.get("Subject", "")) + if sender_match not in sender or subject_match not in subject: + continue + + status, parts = connection.uid("fetch", uid, "(BODY.PEEK[])") + if status != "OK": + continue + raw = b"".join(part[1] for part in parts if isinstance(part, tuple)) + message = email.message_from_bytes(raw, policy=email.policy.default) + sent_at = message_date(message) + for part in message.walk(): + filename = part.get_filename() + if not filename or not filename.lower().endswith(".zip"): + continue + match = ACCOUNT_FILE.search(filename) + if not match: + continue + file_date = datetime.strptime(match.group("date"), "%Y%m%d").date() + payload = part.get_payload(decode=True) or b"" + candidates.append({ + "uid": uid.decode(), + "sent_at": sent_at, + "subject": str(message.get("Subject", "")), + "filename": filename, + "account": match.group("account"), + "file_date": file_date, + "sha256": hashlib.sha256(payload).hexdigest(), + "payload": payload, + }) + finally: + try: + connection.logout() + except imaplib.IMAP4.error: + pass + return candidates + + +def choose_candidate( + candidates: list[dict[str, Any]], + account: str, + target_date: date, +) -> dict[str, Any]: + matches = [ + item for item in candidates + if item["account"] == account and item["file_date"] == target_date + ] + if not matches: + raise StatementError(f"{target_date.isoformat()} 的账单邮件未找到。") + + hashes = {item["sha256"] for item in matches} + if len(hashes) > 1: + raise StatementError("同一天发现多封内容不同的账单邮件,已停止自动导入。") + return sorted(matches, key=lambda item: item["sent_at"], reverse=True)[0] + + +def summary_values(rows: list[list[str]]) -> dict[str, str]: + values: dict[str, str] = {} + for row in rows: + for index, value in enumerate(row): + label = normalized(value) + if not label: + continue + following = next((text(item) for item in row[index + 1:] if text(item)), "") + if following: + values[label] = following + return values + + +def parse_count(value: Any) -> int: + match = re.search(r"\d+", text(value)) + return int(match.group(0)) if match else 0 + + +def parse_money(value: Any) -> float: + current = text(value).replace(",", "") + if not current or current in {"-", "--"}: + return 0.0 + negative = current.startswith("-") + digits = re.sub(r"[^\d.]", "", current) + if not digits: + return 0.0 + amount = round(float(digits), 2) + return -amount if negative else amount + + +def extract_statement( + candidate: dict[str, Any], + config_account: dict[str, Any], + target_date: date, +) -> tuple[Path, int | None, float | None, float | None]: + password = str(config_account["password"]) + account = str(config_account["bank_account"]) + company = str(config_account["company"]) + try: + with zipfile.ZipFile(io.BytesIO(candidate["payload"])) as archive: + members = [ + item for item in archive.infolist() + if not item.is_dir() and item.filename.lower().endswith(".xlsx") + ] + if len(members) != 1: + raise StatementError("账单 ZIP 中应且只能包含一个 Excel 文件。") + member = members[0] + if Path(member.filename).name != member.filename: + raise StatementError("账单 ZIP 包含不安全文件路径。") + workbook_bytes = archive.read(member, pwd=password.encode("utf-8")) + except RuntimeError as exception: + raise StatementError("账单解压密码不正确。") from exception + except zipfile.BadZipFile as exception: + raise StatementError("账单附件不是有效 ZIP。") from exception + + warnings.filterwarnings("ignore", category=UserWarning, module="openpyxl") + workbook = openpyxl.load_workbook( + io.BytesIO(workbook_bytes), + read_only=True, + data_only=True, + ) + try: + worksheet = workbook.active + worksheet.reset_dimensions() + iterator = worksheet.iter_rows(values_only=True) + preamble: list[list[str]] = [] + headers: list[str] | None = None + data_rows: list[list[str]] = [] + + for row in iterator: + values = [text(value) for value in row] + normalized_values = {normalized(value) for value in values} + if "交易时间" in normalized_values and normalized_values & AMOUNT_HEADERS: + headers = values + break + preamble.append(values) + + if headers is None: + raise StatementError("账单 Excel 缺少交易时间或金额表头。") + + for row in iterator: + values = [text(value) for value in row] + if any(values): + data_rows.append(values) + finally: + workbook.close() + + summary = summary_values(preamble) + statement_company = summary.get("企业名称", "") + statement_account = account_digits(summary.get("企业账号", "")) + if statement_company and normalized(statement_company) != normalized(company): + raise StatementError("账单企业名称与账号表不一致。") + if statement_account and statement_account != account: + raise StatementError("账单企业账号与配置不一致。") + + expected_rows = None + expected_income = None + expected_expense = None + if "借方交易笔数" in summary: + expected_rows = parse_count(summary["借方交易笔数"]) + parse_count( + summary.get("贷方交易笔数", "") + ) + if expected_rows != len(data_rows): + raise StatementError( + f"账单汇总为 {expected_rows} 笔,但明细实际为 {len(data_rows)} 笔。" + ) + if "借方交易金额" in summary: + expected_income = round(abs(parse_money(summary["借方交易金额"])), 2) + if "贷方交易金额" in summary: + expected_expense = round(abs(parse_money(summary["贷方交易金额"])), 2) + + column = {normalized(value): index for index, value in enumerate(headers)} + time_index = next( + (column[item] for item in ("交易时间", "记账时间", "发生时间", "入账时间") + if item in column), + None, + ) + status_index = next( + (column[item] for item in STATUS_HEADERS if item in column), + None, + ) + if time_index is None: + raise StatementError("账单缺少交易时间列。") + + valid_rows: list[list[str]] = [] + for row in data_rows: + current = row[time_index] if time_index < len(row) else "" + if not current or not current.startswith(target_date.isoformat()): + continue + if status_index is not None and status_index < len(row): + if FAILED_STATUS.search(row[status_index]): + continue + valid_rows.append(row) + + output = ( + Path(candidate["output_dir"]) + / f"{account}-{target_date.strftime('%Y%m%d')}-交易明细.csv" + ) + output.parent.mkdir(parents=True, exist_ok=True) + with output.open("w", encoding="utf-8-sig", newline="") as stream: + writer = csv.writer(stream) + writer.writerow(headers[: max(len(headers), 1)]) + writer.writerows(valid_rows) + + if expected_rows is not None and expected_rows != len(valid_rows): + raise StatementError( + f"账单汇总为 {expected_rows} 笔,筛选目标日期后为 {len(valid_rows)} 笔。" + ) + return output, expected_rows, expected_income, expected_expense + + +def build_bundle( + run_directory: Path, + target_date: date, + entries: list[dict[str, Any]], +) -> Path: + bundle_directory = run_directory / "bundle" + bundle_directory.mkdir(parents=True, exist_ok=True) + manifest_accounts: list[dict[str, Any]] = [] + for item in entries: + relative = f"accounts/{item['account'][-4:]}/{item['csv'].name}" + target = bundle_directory / relative + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(item["csv"].read_bytes()) + manifest_accounts.append({ + "company": item["company"], + "account": item["account"], + "account_short": item["account_short"], + "expected_rows": item["expected_rows"], + "expected_income": item["expected_income"], + "expected_expense": item["expected_expense"], + "file": relative, + }) + + manifest = { + "version": 1, + "date": target_date.isoformat(), + "generated_at": datetime.now(SHANGHAI).isoformat(timespec="seconds"), + "accounts": manifest_accounts, + } + manifest_file = bundle_directory / "manifest.json" + write_json(manifest_file, manifest) + + bundle = run_directory / f"wangshang-mail-{target_date.isoformat()}.zip" + with zipfile.ZipFile(bundle, "w", compression=zipfile.ZIP_DEFLATED) as archive: + for path in sorted(bundle_directory.rglob("*")): + if path.is_file(): + archive.write(path, path.relative_to(bundle_directory).as_posix()) + return bundle + + +def fetch(args: argparse.Namespace) -> int: + config_file = Path(args.config).expanduser().resolve() + if not config_file.is_file(): + raise StatementError(f"缺少网商邮件配置:{config_file}") + config = json.loads(config_file.read_text(encoding="utf-8")) + mail = config.get("mail") + accounts = config.get("accounts") + if not isinstance(mail, dict) or not isinstance(accounts, list) or not accounts: + raise StatementError("网商邮件配置格式无效。") + + target_date = parse_date( + args.date or (datetime.now(SHANGHAI).date() - timedelta(days=1)).isoformat() + ) + output_root = Path( + args.out or config_file.parent / "wangshang-mail" / "exports" / target_date.isoformat() + ).expanduser().resolve() + run_directory = ( + Path(args.run_dir).expanduser().resolve() + if args.run_dir + else output_root / datetime.now(SHANGHAI).strftime("%Y%m%d-%H%M%S") + ) + run_directory.mkdir(parents=True, exist_ok=True) + state_file = run_directory / "last-fetch.json" + + selected: list[dict[str, Any]] = [] + for item in accounts: + if not isinstance(item, dict) or not item.get("enabled", True): + continue + if args.company and str(args.company) not in str(item.get("company", "")): + continue + selected.append(item) + selected = selected[: max(1, int(args.limit or 999))] + if not selected: + raise StatementError("没有匹配的已启用网商公司。") + + candidates = load_candidates(mail, target_date) + print(f"邮箱匹配到 {len(candidates)} 个网商账单附件") + + successes: list[dict[str, Any]] = [] + failures: list[dict[str, Any]] = [] + for index, account_config in enumerate(selected, start=1): + company = str(account_config.get("company", "")).strip() + account = account_digits(account_config.get("bank_account")) + print(f"[{index}/{len(selected)}] {company},账号 {mask_account(account)}") + try: + if not company or not account: + raise StatementError("配置缺少公司或账号。") + candidate = choose_candidate(candidates, account, target_date) + account_directory = run_directory / "downloads" / account[-4:] + account_directory.mkdir(parents=True, exist_ok=True) + source_zip = account_directory / Path(candidate["filename"]).name + source_zip.write_bytes(candidate["payload"]) + candidate["output_dir"] = account_directory + csv_file, expected_rows, expected_income, expected_expense = extract_statement( + candidate, + account_config, + target_date, + ) + successes.append({ + "company": company, + "account": account, + "account_short": str(account_config.get("account_short") or company), + "csv": csv_file, + "expected_rows": expected_rows, + "expected_income": expected_income, + "expected_expense": expected_expense, + "message_uid": candidate["uid"], + "attachment": candidate["filename"], + }) + print(f" {expected_rows if expected_rows is not None else '?'} 笔,已解析") + except Exception as exception: + reason = str(exception).replace(str(mail.get("auth_code", "")), "***") + failures.append({ + "company": company, + "account_last4": account[-4:], + "error": reason[:500], + }) + print(f" 失败:{reason}", file=sys.stderr) + + bundle: Path | None = None + if successes: + bundle = build_bundle(run_directory, target_date, successes) + + state = { + "ok": bool(successes) and not failures, + "date": target_date.isoformat(), + "accounts": len(successes), + "failed": len(failures), + "failures": failures, + "bundle": str(bundle) if bundle else "", + "generated_at": datetime.now(SHANGHAI).isoformat(timespec="seconds"), + } + write_json(state_file, state) + print( + f"批次完成:成功 {len(successes)},失败 {len(failures)}," + f"状态文件 {state_file}" + ) + return 0 if state["ok"] else 1 + + +def parser() -> argparse.ArgumentParser: + root = argparse.ArgumentParser(description="网商银行邮件账单自动获取") + commands = root.add_subparsers(dest="command", required=True) + + initialize = commands.add_parser("init", help="从账号表生成私有配置") + initialize.add_argument("--workbook", required=True) + initialize.add_argument("--output", required=True) + initialize.add_argument("--auth-code-env", default="WANGSHANG_MAIL_AUTH_CODE") + initialize.add_argument("--imap-host", default="") + initialize.add_argument("--imap-port", type=int, default=993) + initialize.set_defaults(handler=init_config) + + fetch_command = commands.add_parser("fetch", help="从邮箱获取并生成导入批次") + fetch_command.add_argument("--config", required=True) + fetch_command.add_argument("--date", default="") + fetch_command.add_argument("--out", default="") + fetch_command.add_argument("--run-dir", default="") + fetch_command.add_argument("--company", default="") + fetch_command.add_argument("--limit", type=int, default=999) + fetch_command.set_defaults(handler=fetch) + return root + + +def main() -> int: + args = parser().parse_args() + try: + return int(args.handler(args)) + except StatementError as exception: + print(f"网商银行邮件任务失败:{exception}", file=sys.stderr) + return 2 + except Exception as exception: + print(f"网商银行邮件任务异常:{type(exception).__name__}: {exception}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/wangshang-mail/import-accounts.ps1 b/scripts/wangshang-mail/import-accounts.ps1 new file mode 100644 index 00000000..0c21b88f --- /dev/null +++ b/scripts/wangshang-mail/import-accounts.ps1 @@ -0,0 +1,74 @@ +[CmdletBinding()] +param( + [string] $Workbook = 'D:\Desktop\网商数据抓取.xlsx', + + [string] $Output = '', + + [string] $AuthCode = '', + + [string] $ImapHost = 'imap.163.com', + + [int] $ImapPort = 993 +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +$root = Split-Path -Parent (Split-Path -Parent $PSScriptRoot) +if ([string]::IsNullOrWhiteSpace($Output)) { + $Output = Join-Path $root 'runtime\private\wangshang-mail-accounts.json' +} + +function Resolve-Python { + $configured = $env:WANGSHANG_MAIL_PYTHON + $candidates = @( + $configured, + (Join-Path $env:USERPROFILE '.cache\codex-runtimes\codex-primary-runtime\dependencies\python\python.exe') + ) + foreach ($candidate in $candidates) { + if (-not [string]::IsNullOrWhiteSpace($candidate) -and + (Test-Path -LiteralPath $candidate -PathType Leaf)) { + return $candidate + } + } + + $command = Get-Command python.exe -ErrorAction SilentlyContinue | Select-Object -First 1 + if ($null -ne $command -and -not [string]::IsNullOrWhiteSpace($command.Source)) { + & $command.Source -c 'import openpyxl' 2>$null + if ($LASTEXITCODE -eq 0) { + return $command.Source + } + } + + throw '未找到可用的 Python 和 openpyxl,请设置 WANGSHANG_MAIL_PYTHON' +} + +if ([string]::IsNullOrWhiteSpace($AuthCode)) { + $secure = Read-Host '请输入网易邮箱客户端授权码' -AsSecureString + $pointer = [Runtime.InteropServices.Marshal]::SecureStringToBSTR($secure) + try { + $AuthCode = [Runtime.InteropServices.Marshal]::PtrToStringBSTR($pointer) + } finally { + [Runtime.InteropServices.Marshal]::ZeroFreeBSTR($pointer) + } +} + +if ([string]::IsNullOrWhiteSpace($AuthCode)) { + throw '邮箱客户端授权码不能为空' +} + +$env:WANGSHANG_MAIL_AUTH_CODE = $AuthCode +try { + $python = Resolve-Python + $script = Join-Path $PSScriptRoot 'fetch_daily.py' + & $python -X utf8 $script init ` + --workbook $Workbook ` + --output $Output ` + --imap-host $ImapHost ` + --imap-port $ImapPort + if ($LASTEXITCODE -ne 0) { + throw '生成网商邮件配置失败' + } +} finally { + Remove-Item Env:WANGSHANG_MAIL_AUTH_CODE -ErrorAction SilentlyContinue +} diff --git a/scripts/wangshang-mail/install-daily-task.ps1 b/scripts/wangshang-mail/install-daily-task.ps1 new file mode 100644 index 00000000..f83ade67 --- /dev/null +++ b/scripts/wangshang-mail/install-daily-task.ps1 @@ -0,0 +1,69 @@ +#Requires -Version 5.1 + +[CmdletBinding(SupportsShouldProcess = $true)] +param( + [ValidatePattern('^([01]\d|2[0-3]):[0-5]\d$')] + [string] $Time = '10:00', + + [string] $TaskName = 'CaiWuXiTong-WangshangMailDaily' +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +$cmdPath = (Resolve-Path -LiteralPath (Join-Path $PSScriptRoot 'run-wangshang-mail-daily.cmd')).Path +$currentUser = [Security.Principal.WindowsIdentity]::GetCurrent().Name +$arguments = @('/d', '/c', ('""' + $cmdPath + '""')) -join ' ' + +if (-not $PSCmdlet.ShouldProcess($TaskName, "register daily Wangshang Bank email import at $Time")) { + return +} + +$service = New-Object -ComObject 'Schedule.Service' +$rootFolder = $null +$definition = $null +try { + $service.Connect() + $rootFolder = $service.GetFolder('\') + $definition = $service.NewTask(0) + $definition.RegistrationInfo.Description = 'CaiWuXiTong 网商银行邮件账单自动获取与导入' + $definition.Principal.UserId = $currentUser + $definition.Principal.LogonType = 3 + $definition.Principal.RunLevel = 0 + $definition.Settings.Enabled = $true + $definition.Settings.StartWhenAvailable = $true + $definition.Settings.DisallowStartIfOnBatteries = $false + $definition.Settings.StopIfGoingOnBatteries = $false + $definition.Settings.ExecutionTimeLimit = 'PT2H' + $definition.Settings.MultipleInstances = 2 + $definition.Settings.RestartCount = 1 + $definition.Settings.RestartInterval = 'PT15M' + + $trigger = $definition.Triggers.Create(2) + $trigger.StartBoundary = (Get-Date).Date.AddDays(1).ToString('yyyy-MM-dd') + 'T' + $Time + ':00' + $trigger.DaysInterval = 1 + $trigger.Enabled = $true + + $action = $definition.Actions.Create(0) + $action.Path = $env:ComSpec + $action.Arguments = $arguments + $action.WorkingDirectory = $PSScriptRoot + + $registered = $rootFolder.RegisterTaskDefinition( + $TaskName, + $definition, + 6, + $null, + $null, + 3, + $null + ) + Write-Host "已安装每日网商银行邮件任务:$TaskName,执行时间每天 $Time。" + [void] [Runtime.InteropServices.Marshal]::FinalReleaseComObject($registered) +} finally { + foreach ($item in @($definition, $rootFolder, $service)) { + if ($null -ne $item -and [Runtime.InteropServices.Marshal]::IsComObject($item)) { + [void] [Runtime.InteropServices.Marshal]::FinalReleaseComObject($item) + } + } +} diff --git a/scripts/wangshang-mail/run-wangshang-mail-daily.cmd b/scripts/wangshang-mail/run-wangshang-mail-daily.cmd new file mode 100644 index 00000000..a1ad5203 --- /dev/null +++ b/scripts/wangshang-mail/run-wangshang-mail-daily.cmd @@ -0,0 +1,13 @@ +@echo off +setlocal +set "SCRIPT_DIR=%~dp0" +set "LOG_DIR=%SCRIPT_DIR%..\..\runtime\private\wangshang-mail\logs" +if not exist "%LOG_DIR%" mkdir "%LOG_DIR%" +set "PWSH=%ProgramFiles%\PowerShell\7\pwsh.exe" +if exist "%PWSH%" goto run +set "PWSH=%USERPROFILE%\.cache\codex-runtimes\codex-primary-runtime\dependencies\native\powershell\pwsh.exe" +if exist "%PWSH%" goto run +set "PWSH=pwsh.exe" +:run +"%PWSH%" -NoProfile -ExecutionPolicy Bypass -File "%SCRIPT_DIR%run-wangshang-mail-daily.ps1" >> "%SCRIPT_DIR%..\..\runtime\private\wangshang-mail\logs\scheduler.log" 2>&1 +exit /b %errorlevel% diff --git a/scripts/wangshang-mail/run-wangshang-mail-daily.ps1 b/scripts/wangshang-mail/run-wangshang-mail-daily.ps1 new file mode 100644 index 00000000..dc96c83a --- /dev/null +++ b/scripts/wangshang-mail/run-wangshang-mail-daily.ps1 @@ -0,0 +1,180 @@ +[CmdletBinding()] +param( + [ValidatePattern('^\d{4}-\d{2}-\d{2}$')] + [string] $Date, + + [string] $Company, + + [string] $Server = '218.93.190.75', + + [ValidatePattern('^[A-Za-z0-9._-]+$')] + [string] $SshUser = 'root', + + [switch] $DryRun, + + [switch] $SkipImport +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +$utf8 = [Text.UTF8Encoding]::new($false) +[Console]::InputEncoding = $utf8 +[Console]::OutputEncoding = $utf8 +$OutputEncoding = $utf8 + +$root = Split-Path -Parent (Split-Path -Parent $PSScriptRoot) +$configFile = if ([string]::IsNullOrWhiteSpace($env:WANGSHANG_MAIL_CONFIG_FILE)) { + Join-Path $root 'runtime\private\wangshang-mail-accounts.json' +} else { + $env:WANGSHANG_MAIL_CONFIG_FILE +} +$fetchScript = Join-Path $PSScriptRoot 'fetch_daily.py' +$logDir = Join-Path $root 'runtime\private\wangshang-mail\logs' +New-Item -ItemType Directory -Force -Path $logDir | Out-Null +$logFile = Join-Path $logDir ((Get-Date -Format 'yyyyMMdd') + '.log') + +function Write-Log { + param([string] $Message) + $line = '{0} {1}' -f (Get-Date -Format 'yyyy-MM-dd HH:mm:ss'), $Message + Write-Host $line + Add-Content -LiteralPath $logFile -Value $line -Encoding UTF8 +} + +function Resolve-Python { + $configured = $env:WANGSHANG_MAIL_PYTHON + $candidates = @( + $configured, + (Join-Path $env:USERPROFILE '.cache\codex-runtimes\codex-primary-runtime\dependencies\python\python.exe') + ) + foreach ($candidate in $candidates) { + if (-not [string]::IsNullOrWhiteSpace($candidate) -and + (Test-Path -LiteralPath $candidate -PathType Leaf)) { + return $candidate + } + } + + $command = Get-Command python.exe -ErrorAction SilentlyContinue | Select-Object -First 1 + if ($null -ne $command -and -not [string]::IsNullOrWhiteSpace($command.Source)) { + & $command.Source -c 'import openpyxl' 2>$null + if ($LASTEXITCODE -eq 0) { + return $command.Source + } + } + + throw '未找到可用的 Python 和 openpyxl,请设置 WANGSHANG_MAIL_PYTHON' +} + +if (-not (Test-Path -LiteralPath $configFile -PathType Leaf)) { + throw '缺少 runtime/private/wangshang-mail-accounts.json,请先运行 import-accounts.ps1' +} +if ([string]::IsNullOrWhiteSpace($Date)) { + $Date = (Get-Date).Date.AddDays(-1).ToString('yyyy-MM-dd') +} + +$python = Resolve-Python +$runStamp = Get-Date -Format 'yyyyMMdd-HHmmss' +$runDirectory = Join-Path $root ('runtime\private\wangshang-mail\exports\' + $Date + '\run-' + $runStamp) +New-Item -ItemType Directory -Force -Path $runDirectory | Out-Null +$fetchArgs = @( + $fetchScript, + 'fetch', + ('--config=' + $configFile), + ('--date=' + $Date), + ('--out=' + (Split-Path -Parent $runDirectory)), + ('--run-dir=' + $runDirectory) +) +if (-not [string]::IsNullOrWhiteSpace($Company)) { + $fetchArgs += ('--company=' + $Company) +} + +Write-Log ('开始获取网商银行邮件账单 ' + $Date) +$fetchOutput = (& $python -X utf8 @fetchArgs 2>&1 | Out-String) +$fetchExit = $LASTEXITCODE +if (-not [string]::IsNullOrWhiteSpace($fetchOutput)) { + Write-Log ('获取输出:' + ($fetchOutput.Trim() -replace "`r?`n", ' | ')) +} + +$stateFile = Join-Path $runDirectory 'last-fetch.json' +if (-not (Test-Path -LiteralPath $stateFile -PathType Leaf)) { + throw '网商银行邮件任务没有生成状态文件;未写入财务台账' +} +$state = [IO.File]::ReadAllText($stateFile, [Text.Encoding]::UTF8) | ConvertFrom-Json +if ([string]$state.date -ne $Date) { + throw '网商银行邮件任务状态日期与本次任务不一致' +} +if ([int]$state.accounts -le 0 -or [string]::IsNullOrWhiteSpace([string]$state.bundle)) { + throw '本次没有成功生成网商银行导入批次' +} +$bundleFile = [string]$state.bundle +if (-not (Test-Path -LiteralPath $bundleFile -PathType Leaf)) { + throw '网商银行邮件任务没有生成导入 ZIP' +} + +$hadFailures = $fetchExit -ne 0 -or [int]$state.failed -gt 0 +Write-Log ('获取完成:成功账户 {0},失败账户 {1}' -f $state.accounts, $state.failed) +foreach ($failure in @($state.failures)) { + if ($null -ne $failure) { + Write-Log ('账户失败:{0},尾号 {1},原因 {2}' -f + $failure.company, $failure.account_last4, $failure.error) + } +} + +if ($SkipImport) { + Write-Log '已指定 -SkipImport,未写入财务台账' + if ($hadFailures) { exit 1 } + return +} + +. (Join-Path $root 'scripts\deploy-auth.ps1') +$remoteFile = '/opt/caiwuxitong/runtime/private/wangshang-mail-imports/' + $Date + + '/wangshang-mail-' + (Get-Date -Format 'yyyyMMdd-HHmmss') + '.zip' +$auth = Start-CaiwuDeploymentAuth -Server $Server -SshUser $SshUser +try { + $target = "${SshUser}@${Server}" + $remoteDirectory = [IO.Path]::GetDirectoryName($remoteFile).Replace('\', '/') + $dryRunFlag = if ($DryRun) { ' --dry-run' } else { '' } + $command = @( + 'set -eu', + ('sh /opt/caiwuxitong/run-finance-batch.sh finance-ledger:import-wangshang-bank ' + + "'$remoteFile' --date='$Date'" + $dryRunFlag), + ("rm -f '" + $remoteFile + "'") + ) -join '; ' + $attempts = 3 + $importCompleted = $false + for ($attempt = 1; $attempt -le $attempts; $attempt++) { + try { + & ssh @($auth.Options) $target ("install -d -m 700 '" + $remoteDirectory + "'") + if ($LASTEXITCODE -ne 0) { throw '创建远程目录失败' } + + & scp @($auth.Options) $bundleFile ("${target}:" + $remoteFile) + if ($LASTEXITCODE -ne 0) { throw '上传批次失败' } + + $importOutput = (& ssh @($auth.Options) $target $command 2>&1 | Out-String) + $importExit = $LASTEXITCODE + if (-not [string]::IsNullOrWhiteSpace($importOutput)) { + Write-Log ('导入输出:' + ($importOutput.Trim() -replace "`r?`n", ' | ')) + } + if ($importExit -ne 0) { throw '受限批处理命令返回失败' } + + $importCompleted = $true + break + } catch { + if ($attempt -lt $attempts) { + Write-Log ("远程导入第 {0}/{1} 次失败:{2};10 秒后重试(导入按账号和日期幂等)" -f + $attempt, $attempts, $_.Exception.Message) + Start-Sleep -Seconds 10 + } + } + } + if (-not $importCompleted) { + throw '网商银行邮件账单导入失败;已重试 3 次,原有动账明细未改变' + } +} finally { + Stop-CaiwuDeploymentAuth $auth +} + +Write-Log ($(if ($DryRun) { '网商银行邮件账单校验完成' } else { '网商银行邮件账单已接入财务台账动账明细' })) +if ($hadFailures) { + Write-Log '本次有账户获取失败;成功账户已导入,失败账户保持原数据' + exit 1 +}