#!/usr/bin/env python3 """Download and normalize Wangshang Bank daily statements received by email.""" from __future__ import annotations import argparse import csv import email import email.policy import hashlib import imaplib import io import json import os import re import ssl import sys import warnings import zipfile from datetime import date, datetime, timedelta from email.utils import parsedate_to_datetime from pathlib import Path from typing import Any, Iterable from zoneinfo import ZoneInfo try: import openpyxl except ImportError as exception: # pragma: no cover - guarded by the wrapper raise SystemExit("缺少 openpyxl,无法读取网商银行账单 Excel。") from exception SHANGHAI = ZoneInfo("Asia/Shanghai") MONTHS = ("Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec") AMOUNT_HEADERS = { "借方金额(收)", "借方金额(收)", "贷方金额(支)", "贷方金额(支)", "收入", "收入金额", "支出", "支出金额", "金额", "交易金额", } STATUS_HEADERS = {"交易状态", "状态"} FAILED_STATUS = re.compile(r"关闭|失败|取消|退票") ACCOUNT_FILE = re.compile(r"(?P\d{10,30})-日账单-(?P\d{8})-交易明细") class StatementError(RuntimeError): """A statement could not be matched or safely parsed.""" def text(value: Any) -> str: if value is None: return "" if isinstance(value, datetime): return value.strftime("%Y-%m-%d %H:%M:%S") if isinstance(value, date): return value.strftime("%Y-%m-%d") if isinstance(value, float): return format(value, ".15g") return str(value).strip() def normalized(value: Any) -> str: return re.sub(r"\s+", "", text(value)) def account_digits(value: Any) -> str: return re.sub(r"\D+", "", text(value)) def mask_account(value: Any) -> str: digits = account_digits(value) return f"****{digits[-4:]}" if len(digits) >= 4 else "****" def parse_date(value: str) -> date: try: parsed = date.fromisoformat(value) except ValueError as exception: raise StatementError("日期必须使用 YYYY-MM-DD 格式。") from exception if parsed.isoformat() != value: raise StatementError("日期必须使用 YYYY-MM-DD 格式。") return parsed def imap_date(value: date) -> str: return f"{value.day:02d}-{MONTHS[value.month - 1]}-{value.year}" def write_json(path: Path, payload: dict[str, Any]) -> None: path.parent.mkdir(parents=True, exist_ok=True) temporary = path.with_suffix(path.suffix + ".tmp") temporary.write_text( json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8", ) os.replace(temporary, path) def next_nonempty(row: Iterable[Any], start: int = 1) -> str: values = list(row) for value in values[start:]: current = text(value) if current: return current return "" def locate_row(rows: list[tuple[Any, ...]], *labels: str) -> tuple[int, tuple[Any, ...]]: expected = set(labels) for index, row in enumerate(rows): if expected.issubset({normalized(value) for value in row}): return index, row raise StatementError("账号表中的邮箱或公司字段不完整。") def init_config(args: argparse.Namespace) -> int: warnings.filterwarnings("ignore", category=UserWarning, module="openpyxl") workbook_path = Path(args.workbook).expanduser().resolve() if not workbook_path.is_file(): raise StatementError(f"账号表不存在:{workbook_path}") auth_code = os.environ.get(args.auth_code_env, "").strip() if not auth_code: raise StatementError(f"环境变量 {args.auth_code_env} 中没有邮箱授权码。") workbook = openpyxl.load_workbook(workbook_path, read_only=False, data_only=True) try: sheet = workbook.active rows = [ tuple(row[: max(12, sheet.max_column)]) for row in sheet.iter_rows(values_only=True) ] _, mail_url_row = locate_row(rows, "邮箱网址") _, login_row = locate_row(rows, "登录账号") _, company_header_row = locate_row(rows, "公司名称", "解压密码", "网商银行账号") finally: workbook.close() mail_url = next_nonempty(mail_url_row) login_account = next_nonempty(login_row) if not mail_url or not login_account: raise StatementError("账号表缺少邮箱网址或登录账号。") host = args.imap_host.strip() if not host: host = "imap.163.com" if "163.com" in mail_url.lower() else "" if not host: raise StatementError("无法从邮箱网址识别 IMAP 服务器,请使用 --imap-host 指定。") username = login_account if "@" not in username and host.endswith("163.com"): username += "@163.com" header_position = rows.index(company_header_row) columns = { normalized(value): index for index, value in enumerate(company_header_row) if normalized(value) } accounts: list[dict[str, Any]] = [] for row_number, row in enumerate(rows[header_position + 1:], header_position + 2): company = text(row[columns["公司名称"]]) password = text(row[columns["解压密码"]]) bank_account = account_digits(row[columns["网商银行账号"]]) if not company and not bank_account: continue if not company or not password or len(bank_account) < 6: raise StatementError( f"账号表第 {row_number} 行缺少公司名称、解压密码或有效网商账号。" ) accounts.append({ "company": company, "account_short": company, "bank_account": bank_account, "password": password, "enabled": True, }) if not accounts: raise StatementError("账号表中没有可接入的网商公司。") if len({item["bank_account"] for item in accounts}) != len(accounts): raise StatementError("账号表存在重复网商账号。") config = { "version": 1, "generated_at": datetime.now(SHANGHAI).isoformat(timespec="seconds"), "mail": { "host": host, "port": int(args.imap_port), "username": username, "auth_code": auth_code, "sender": "service@mail.mybank.cn", "subject": "浙江网商银行电子凭证", "search_grace_days": 2, }, "accounts": accounts, } output = Path(args.output).expanduser().resolve() write_json(output, config) print(f"已生成网商邮件配置:{len(accounts)} 个公司") print(f"邮箱:{username},IMAP:{host}:{args.imap_port}") for item in accounts: print(f"- {item['company']},账号 {mask_account(item['bank_account'])}") return 0 def connect_mail(mail: dict[str, Any]) -> imaplib.IMAP4_SSL: imaplib.Commands["ID"] = ("AUTH",) connection = imaplib.IMAP4_SSL( str(mail["host"]), int(mail.get("port", 993)), ssl_context=ssl.create_default_context(), ) connection.login(str(mail["username"]), str(mail["auth_code"])) connection._simple_command( "ID", '("name" "CaiWuXiTong" "version" "1.0" "vendor" "CaiWuXiTong")', ) status, data = connection.select("INBOX", readonly=True) if status != "OK": connection.logout() raise StatementError(f"无法打开邮箱收件箱:{data!r}") return connection def message_date(message: email.message.Message) -> datetime: try: parsed = parsedate_to_datetime(str(message.get("Date", ""))) if parsed.tzinfo is None: return parsed.replace(tzinfo=SHANGHAI) return parsed.astimezone(SHANGHAI) except (TypeError, ValueError): return datetime.min.replace(tzinfo=SHANGHAI) def load_candidates( mail: dict[str, Any], target_date: date, ) -> list[dict[str, Any]]: connection = connect_mail(mail) candidates: list[dict[str, Any]] = [] since = target_date - timedelta(days=max(1, int(mail.get("search_grace_days", 2)))) try: status, data = connection.uid("search", None, "SINCE", imap_date(since)) if status != "OK": raise StatementError(f"邮箱搜索失败:{data!r}") uids = data[0].split() if data and data[0] else [] sender_match = str(mail.get("sender", "service@mail.mybank.cn")).lower() subject_match = str(mail.get("subject", "浙江网商银行电子凭证")) for uid in uids: status, parts = connection.uid( "fetch", uid, "(BODY.PEEK[HEADER.FIELDS (FROM SUBJECT DATE)])", ) if status != "OK": continue raw_header = b"".join(part[1] for part in parts if isinstance(part, tuple)) header = email.message_from_bytes(raw_header, policy=email.policy.default) sender = str(header.get("From", "")).lower() subject = str(header.get("Subject", "")) if sender_match not in sender or subject_match not in subject: continue status, parts = connection.uid("fetch", uid, "(BODY.PEEK[])") if status != "OK": continue raw = b"".join(part[1] for part in parts if isinstance(part, tuple)) message = email.message_from_bytes(raw, policy=email.policy.default) sent_at = message_date(message) for part in message.walk(): filename = part.get_filename() if not filename or not filename.lower().endswith(".zip"): continue match = ACCOUNT_FILE.search(filename) if not match: continue file_date = datetime.strptime(match.group("date"), "%Y%m%d").date() payload = part.get_payload(decode=True) or b"" candidates.append({ "uid": uid.decode(), "sent_at": sent_at, "subject": str(message.get("Subject", "")), "filename": filename, "account": match.group("account"), "file_date": file_date, "sha256": hashlib.sha256(payload).hexdigest(), "payload": payload, }) finally: try: connection.logout() except imaplib.IMAP4.error: pass return candidates def choose_candidate( candidates: list[dict[str, Any]], account: str, target_date: date, ) -> dict[str, Any]: matches = [ item for item in candidates if item["account"] == account and item["file_date"] == target_date ] if not matches: raise StatementError(f"{target_date.isoformat()} 的账单邮件未找到。") hashes = {item["sha256"] for item in matches} if len(hashes) > 1: raise StatementError("同一天发现多封内容不同的账单邮件,已停止自动导入。") return sorted(matches, key=lambda item: item["sent_at"], reverse=True)[0] def summary_values(rows: list[list[str]]) -> dict[str, str]: values: dict[str, str] = {} for row in rows: for index, value in enumerate(row): label = normalized(value) if not label: continue following = next((text(item) for item in row[index + 1:] if text(item)), "") if following: values[label] = following return values def parse_count(value: Any) -> int: match = re.search(r"\d+", text(value)) return int(match.group(0)) if match else 0 def parse_money(value: Any) -> float: current = text(value).replace(",", "") if not current or current in {"-", "--"}: return 0.0 negative = current.startswith("-") digits = re.sub(r"[^\d.]", "", current) if not digits: return 0.0 amount = round(float(digits), 2) return -amount if negative else amount def extract_statement( candidate: dict[str, Any], config_account: dict[str, Any], target_date: date, ) -> tuple[Path, int | None, float | None, float | None]: password = str(config_account["password"]) account = str(config_account["bank_account"]) company = str(config_account["company"]) try: with zipfile.ZipFile(io.BytesIO(candidate["payload"])) as archive: members = [ item for item in archive.infolist() if not item.is_dir() and item.filename.lower().endswith(".xlsx") ] if len(members) != 1: raise StatementError("账单 ZIP 中应且只能包含一个 Excel 文件。") member = members[0] if Path(member.filename).name != member.filename: raise StatementError("账单 ZIP 包含不安全文件路径。") workbook_bytes = archive.read(member, pwd=password.encode("utf-8")) except RuntimeError as exception: raise StatementError("账单解压密码不正确。") from exception except zipfile.BadZipFile as exception: raise StatementError("账单附件不是有效 ZIP。") from exception warnings.filterwarnings("ignore", category=UserWarning, module="openpyxl") workbook = openpyxl.load_workbook( io.BytesIO(workbook_bytes), read_only=True, data_only=True, ) try: worksheet = workbook.active worksheet.reset_dimensions() iterator = worksheet.iter_rows(values_only=True) preamble: list[list[str]] = [] headers: list[str] | None = None data_rows: list[list[str]] = [] for row in iterator: values = [text(value) for value in row] normalized_values = {normalized(value) for value in values} if "交易时间" in normalized_values and normalized_values & AMOUNT_HEADERS: headers = values break preamble.append(values) if headers is None: raise StatementError("账单 Excel 缺少交易时间或金额表头。") for row in iterator: values = [text(value) for value in row] if any(values): data_rows.append(values) finally: workbook.close() summary = summary_values(preamble) statement_company = summary.get("企业名称", "") statement_account = account_digits(summary.get("企业账号", "")) if statement_company and normalized(statement_company) != normalized(company): raise StatementError("账单企业名称与账号表不一致。") if statement_account and statement_account != account: raise StatementError("账单企业账号与配置不一致。") expected_rows = None expected_income = None expected_expense = None if "借方交易笔数" in summary: expected_rows = parse_count(summary["借方交易笔数"]) + parse_count( summary.get("贷方交易笔数", "") ) if expected_rows != len(data_rows): raise StatementError( f"账单汇总为 {expected_rows} 笔,但明细实际为 {len(data_rows)} 笔。" ) if "借方交易金额" in summary: expected_income = round(abs(parse_money(summary["借方交易金额"])), 2) if "贷方交易金额" in summary: expected_expense = round(abs(parse_money(summary["贷方交易金额"])), 2) column = {normalized(value): index for index, value in enumerate(headers)} time_index = next( (column[item] for item in ("交易时间", "记账时间", "发生时间", "入账时间") if item in column), None, ) status_index = next( (column[item] for item in STATUS_HEADERS if item in column), None, ) if time_index is None: raise StatementError("账单缺少交易时间列。") valid_rows: list[list[str]] = [] for row in data_rows: current = row[time_index] if time_index < len(row) else "" if not current or not current.startswith(target_date.isoformat()): continue if status_index is not None and status_index < len(row): if FAILED_STATUS.search(row[status_index]): continue valid_rows.append(row) output = ( Path(candidate["output_dir"]) / f"{account}-{target_date.strftime('%Y%m%d')}-交易明细.csv" ) output.parent.mkdir(parents=True, exist_ok=True) with output.open("w", encoding="utf-8-sig", newline="") as stream: writer = csv.writer(stream) writer.writerow(headers[: max(len(headers), 1)]) writer.writerows(valid_rows) if expected_rows is not None and expected_rows != len(valid_rows): raise StatementError( f"账单汇总为 {expected_rows} 笔,筛选目标日期后为 {len(valid_rows)} 笔。" ) return output, expected_rows, expected_income, expected_expense def build_bundle( run_directory: Path, target_date: date, entries: list[dict[str, Any]], ) -> Path: bundle_directory = run_directory / "bundle" bundle_directory.mkdir(parents=True, exist_ok=True) manifest_accounts: list[dict[str, Any]] = [] for item in entries: relative = f"accounts/{item['account'][-4:]}/{item['csv'].name}" target = bundle_directory / relative target.parent.mkdir(parents=True, exist_ok=True) target.write_bytes(item["csv"].read_bytes()) manifest_accounts.append({ "company": item["company"], "account": item["account"], "account_short": item["account_short"], "expected_rows": item["expected_rows"], "expected_income": item["expected_income"], "expected_expense": item["expected_expense"], "file": relative, }) manifest = { "version": 1, "date": target_date.isoformat(), "generated_at": datetime.now(SHANGHAI).isoformat(timespec="seconds"), "accounts": manifest_accounts, } manifest_file = bundle_directory / "manifest.json" write_json(manifest_file, manifest) bundle = run_directory / f"wangshang-mail-{target_date.isoformat()}.zip" with zipfile.ZipFile(bundle, "w", compression=zipfile.ZIP_DEFLATED) as archive: for path in sorted(bundle_directory.rglob("*")): if path.is_file(): archive.write(path, path.relative_to(bundle_directory).as_posix()) return bundle def fetch(args: argparse.Namespace) -> int: config_file = Path(args.config).expanduser().resolve() if not config_file.is_file(): raise StatementError(f"缺少网商邮件配置:{config_file}") config = json.loads(config_file.read_text(encoding="utf-8")) mail = config.get("mail") accounts = config.get("accounts") if not isinstance(mail, dict) or not isinstance(accounts, list) or not accounts: raise StatementError("网商邮件配置格式无效。") target_date = parse_date( args.date or (datetime.now(SHANGHAI).date() - timedelta(days=1)).isoformat() ) output_root = Path( args.out or config_file.parent / "wangshang-mail" / "exports" / target_date.isoformat() ).expanduser().resolve() run_directory = ( Path(args.run_dir).expanduser().resolve() if args.run_dir else output_root / datetime.now(SHANGHAI).strftime("%Y%m%d-%H%M%S") ) run_directory.mkdir(parents=True, exist_ok=True) state_file = run_directory / "last-fetch.json" selected: list[dict[str, Any]] = [] for item in accounts: if not isinstance(item, dict) or not item.get("enabled", True): continue if args.company and str(args.company) not in str(item.get("company", "")): continue selected.append(item) selected = selected[: max(1, int(args.limit or 999))] if not selected: raise StatementError("没有匹配的已启用网商公司。") candidates = load_candidates(mail, target_date) print(f"邮箱匹配到 {len(candidates)} 个网商账单附件") successes: list[dict[str, Any]] = [] failures: list[dict[str, Any]] = [] for index, account_config in enumerate(selected, start=1): company = str(account_config.get("company", "")).strip() account = account_digits(account_config.get("bank_account")) print(f"[{index}/{len(selected)}] {company},账号 {mask_account(account)}") try: if not company or not account: raise StatementError("配置缺少公司或账号。") candidate = choose_candidate(candidates, account, target_date) account_directory = run_directory / "downloads" / account[-4:] account_directory.mkdir(parents=True, exist_ok=True) source_zip = account_directory / Path(candidate["filename"]).name source_zip.write_bytes(candidate["payload"]) candidate["output_dir"] = account_directory csv_file, expected_rows, expected_income, expected_expense = extract_statement( candidate, account_config, target_date, ) successes.append({ "company": company, "account": account, "account_short": str(account_config.get("account_short") or company), "csv": csv_file, "expected_rows": expected_rows, "expected_income": expected_income, "expected_expense": expected_expense, "message_uid": candidate["uid"], "attachment": candidate["filename"], }) print(f" {expected_rows if expected_rows is not None else '?'} 笔,已解析") except Exception as exception: reason = str(exception).replace(str(mail.get("auth_code", "")), "***") failures.append({ "company": company, "account_last4": account[-4:], "error": reason[:500], }) print(f" 失败:{reason}", file=sys.stderr) bundle: Path | None = None if successes: bundle = build_bundle(run_directory, target_date, successes) state = { "ok": bool(successes) and not failures, "date": target_date.isoformat(), "accounts": len(successes), "failed": len(failures), "failures": failures, "bundle": str(bundle) if bundle else "", "generated_at": datetime.now(SHANGHAI).isoformat(timespec="seconds"), } write_json(state_file, state) print( f"批次完成:成功 {len(successes)},失败 {len(failures)}," f"状态文件 {state_file}" ) return 0 if state["ok"] else 1 def parser() -> argparse.ArgumentParser: root = argparse.ArgumentParser(description="网商银行邮件账单自动获取") commands = root.add_subparsers(dest="command", required=True) initialize = commands.add_parser("init", help="从账号表生成私有配置") initialize.add_argument("--workbook", required=True) initialize.add_argument("--output", required=True) initialize.add_argument("--auth-code-env", default="WANGSHANG_MAIL_AUTH_CODE") initialize.add_argument("--imap-host", default="") initialize.add_argument("--imap-port", type=int, default=993) initialize.set_defaults(handler=init_config) fetch_command = commands.add_parser("fetch", help="从邮箱获取并生成导入批次") fetch_command.add_argument("--config", required=True) fetch_command.add_argument("--date", default="") fetch_command.add_argument("--out", default="") fetch_command.add_argument("--run-dir", default="") fetch_command.add_argument("--company", default="") fetch_command.add_argument("--limit", type=int, default=999) fetch_command.set_defaults(handler=fetch) return root def main() -> int: args = parser().parse_args() try: return int(args.handler(args)) except StatementError as exception: print(f"网商银行邮件任务失败:{exception}", file=sys.stderr) return 2 except Exception as exception: print(f"网商银行邮件任务异常:{type(exception).__name__}: {exception}", file=sys.stderr) return 2 if __name__ == "__main__": raise SystemExit(main())