shanshi 65d37c8093 feat: 接入网商银行邮件账单自动导入 (#1199)
独立邮箱机器人读取网商银行加密账单,解密并导入财务台账。
2026-11-16 16:41:07 +08:00

643 lines
24 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/env python3
"""Download and normalize Wangshang Bank daily statements received by email."""
from __future__ import annotations
import argparse
import csv
import email
import email.policy
import hashlib
import imaplib
import io
import json
import os
import re
import ssl
import sys
import warnings
import zipfile
from datetime import date, datetime, timedelta
from email.utils import parsedate_to_datetime
from pathlib import Path
from typing import Any, Iterable
from zoneinfo import ZoneInfo
try:
import openpyxl
except ImportError as exception: # pragma: no cover - guarded by the wrapper
raise SystemExit("缺少 openpyxl,无法读取网商银行账单 Excel。") from exception
SHANGHAI = ZoneInfo("Asia/Shanghai")
MONTHS = ("Jan", "Feb", "Mar", "Apr", "May", "Jun",
"Jul", "Aug", "Sep", "Oct", "Nov", "Dec")
AMOUNT_HEADERS = {
"借方金额(收)", "借方金额(收)", "贷方金额(支)", "贷方金额(支)",
"收入", "收入金额", "支出", "支出金额", "金额", "交易金额",
}
STATUS_HEADERS = {"交易状态", "状态"}
FAILED_STATUS = re.compile(r"关闭|失败|取消|退票")
ACCOUNT_FILE = re.compile(r"(?P<account>\d{10,30})-日账单-(?P<date>\d{8})-交易明细")
class StatementError(RuntimeError):
"""A statement could not be matched or safely parsed."""
def text(value: Any) -> str:
if value is None:
return ""
if isinstance(value, datetime):
return value.strftime("%Y-%m-%d %H:%M:%S")
if isinstance(value, date):
return value.strftime("%Y-%m-%d")
if isinstance(value, float):
return format(value, ".15g")
return str(value).strip()
def normalized(value: Any) -> str:
return re.sub(r"\s+", "", text(value))
def account_digits(value: Any) -> str:
return re.sub(r"\D+", "", text(value))
def mask_account(value: Any) -> str:
digits = account_digits(value)
return f"****{digits[-4:]}" if len(digits) >= 4 else "****"
def parse_date(value: str) -> date:
try:
parsed = date.fromisoformat(value)
except ValueError as exception:
raise StatementError("日期必须使用 YYYY-MM-DD 格式。") from exception
if parsed.isoformat() != value:
raise StatementError("日期必须使用 YYYY-MM-DD 格式。")
return parsed
def imap_date(value: date) -> str:
return f"{value.day:02d}-{MONTHS[value.month - 1]}-{value.year}"
def write_json(path: Path, payload: dict[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
temporary = path.with_suffix(path.suffix + ".tmp")
temporary.write_text(
json.dumps(payload, ensure_ascii=False, indent=2) + "\n",
encoding="utf-8",
)
os.replace(temporary, path)
def next_nonempty(row: Iterable[Any], start: int = 1) -> str:
values = list(row)
for value in values[start:]:
current = text(value)
if current:
return current
return ""
def locate_row(rows: list[tuple[Any, ...]], *labels: str) -> tuple[int, tuple[Any, ...]]:
expected = set(labels)
for index, row in enumerate(rows):
if expected.issubset({normalized(value) for value in row}):
return index, row
raise StatementError("账号表中的邮箱或公司字段不完整。")
def init_config(args: argparse.Namespace) -> int:
warnings.filterwarnings("ignore", category=UserWarning, module="openpyxl")
workbook_path = Path(args.workbook).expanduser().resolve()
if not workbook_path.is_file():
raise StatementError(f"账号表不存在:{workbook_path}")
auth_code = os.environ.get(args.auth_code_env, "").strip()
if not auth_code:
raise StatementError(f"环境变量 {args.auth_code_env} 中没有邮箱授权码。")
workbook = openpyxl.load_workbook(workbook_path, read_only=False, data_only=True)
try:
sheet = workbook.active
rows = [
tuple(row[: max(12, sheet.max_column)])
for row in sheet.iter_rows(values_only=True)
]
_, mail_url_row = locate_row(rows, "邮箱网址")
_, login_row = locate_row(rows, "登录账号")
_, company_header_row = locate_row(rows, "公司名称", "解压密码", "网商银行账号")
finally:
workbook.close()
mail_url = next_nonempty(mail_url_row)
login_account = next_nonempty(login_row)
if not mail_url or not login_account:
raise StatementError("账号表缺少邮箱网址或登录账号。")
host = args.imap_host.strip()
if not host:
host = "imap.163.com" if "163.com" in mail_url.lower() else ""
if not host:
raise StatementError("无法从邮箱网址识别 IMAP 服务器,请使用 --imap-host 指定。")
username = login_account
if "@" not in username and host.endswith("163.com"):
username += "@163.com"
header_position = rows.index(company_header_row)
columns = {
normalized(value): index
for index, value in enumerate(company_header_row)
if normalized(value)
}
accounts: list[dict[str, Any]] = []
for row_number, row in enumerate(rows[header_position + 1:], header_position + 2):
company = text(row[columns["公司名称"]])
password = text(row[columns["解压密码"]])
bank_account = account_digits(row[columns["网商银行账号"]])
if not company and not bank_account:
continue
if not company or not password or len(bank_account) < 6:
raise StatementError(
f"账号表第 {row_number} 行缺少公司名称、解压密码或有效网商账号。"
)
accounts.append({
"company": company,
"account_short": company,
"bank_account": bank_account,
"password": password,
"enabled": True,
})
if not accounts:
raise StatementError("账号表中没有可接入的网商公司。")
if len({item["bank_account"] for item in accounts}) != len(accounts):
raise StatementError("账号表存在重复网商账号。")
config = {
"version": 1,
"generated_at": datetime.now(SHANGHAI).isoformat(timespec="seconds"),
"mail": {
"host": host,
"port": int(args.imap_port),
"username": username,
"auth_code": auth_code,
"sender": "service@mail.mybank.cn",
"subject": "浙江网商银行电子凭证",
"search_grace_days": 2,
},
"accounts": accounts,
}
output = Path(args.output).expanduser().resolve()
write_json(output, config)
print(f"已生成网商邮件配置:{len(accounts)} 个公司")
print(f"邮箱:{username},IMAP:{host}:{args.imap_port}")
for item in accounts:
print(f"- {item['company']},账号 {mask_account(item['bank_account'])}")
return 0
def connect_mail(mail: dict[str, Any]) -> imaplib.IMAP4_SSL:
imaplib.Commands["ID"] = ("AUTH",)
connection = imaplib.IMAP4_SSL(
str(mail["host"]),
int(mail.get("port", 993)),
ssl_context=ssl.create_default_context(),
)
connection.login(str(mail["username"]), str(mail["auth_code"]))
connection._simple_command(
"ID",
'("name" "CaiWuXiTong" "version" "1.0" "vendor" "CaiWuXiTong")',
)
status, data = connection.select("INBOX", readonly=True)
if status != "OK":
connection.logout()
raise StatementError(f"无法打开邮箱收件箱:{data!r}")
return connection
def message_date(message: email.message.Message) -> datetime:
try:
parsed = parsedate_to_datetime(str(message.get("Date", "")))
if parsed.tzinfo is None:
return parsed.replace(tzinfo=SHANGHAI)
return parsed.astimezone(SHANGHAI)
except (TypeError, ValueError):
return datetime.min.replace(tzinfo=SHANGHAI)
def load_candidates(
mail: dict[str, Any],
target_date: date,
) -> list[dict[str, Any]]:
connection = connect_mail(mail)
candidates: list[dict[str, Any]] = []
since = target_date - timedelta(days=max(1, int(mail.get("search_grace_days", 2))))
try:
status, data = connection.uid("search", None, "SINCE", imap_date(since))
if status != "OK":
raise StatementError(f"邮箱搜索失败:{data!r}")
uids = data[0].split() if data and data[0] else []
sender_match = str(mail.get("sender", "service@mail.mybank.cn")).lower()
subject_match = str(mail.get("subject", "浙江网商银行电子凭证"))
for uid in uids:
status, parts = connection.uid(
"fetch",
uid,
"(BODY.PEEK[HEADER.FIELDS (FROM SUBJECT DATE)])",
)
if status != "OK":
continue
raw_header = b"".join(part[1] for part in parts if isinstance(part, tuple))
header = email.message_from_bytes(raw_header, policy=email.policy.default)
sender = str(header.get("From", "")).lower()
subject = str(header.get("Subject", ""))
if sender_match not in sender or subject_match not in subject:
continue
status, parts = connection.uid("fetch", uid, "(BODY.PEEK[])")
if status != "OK":
continue
raw = b"".join(part[1] for part in parts if isinstance(part, tuple))
message = email.message_from_bytes(raw, policy=email.policy.default)
sent_at = message_date(message)
for part in message.walk():
filename = part.get_filename()
if not filename or not filename.lower().endswith(".zip"):
continue
match = ACCOUNT_FILE.search(filename)
if not match:
continue
file_date = datetime.strptime(match.group("date"), "%Y%m%d").date()
payload = part.get_payload(decode=True) or b""
candidates.append({
"uid": uid.decode(),
"sent_at": sent_at,
"subject": str(message.get("Subject", "")),
"filename": filename,
"account": match.group("account"),
"file_date": file_date,
"sha256": hashlib.sha256(payload).hexdigest(),
"payload": payload,
})
finally:
try:
connection.logout()
except imaplib.IMAP4.error:
pass
return candidates
def choose_candidate(
candidates: list[dict[str, Any]],
account: str,
target_date: date,
) -> dict[str, Any]:
matches = [
item for item in candidates
if item["account"] == account and item["file_date"] == target_date
]
if not matches:
raise StatementError(f"{target_date.isoformat()} 的账单邮件未找到。")
hashes = {item["sha256"] for item in matches}
if len(hashes) > 1:
raise StatementError("同一天发现多封内容不同的账单邮件,已停止自动导入。")
return sorted(matches, key=lambda item: item["sent_at"], reverse=True)[0]
def summary_values(rows: list[list[str]]) -> dict[str, str]:
values: dict[str, str] = {}
for row in rows:
for index, value in enumerate(row):
label = normalized(value)
if not label:
continue
following = next((text(item) for item in row[index + 1:] if text(item)), "")
if following:
values[label] = following
return values
def parse_count(value: Any) -> int:
match = re.search(r"\d+", text(value))
return int(match.group(0)) if match else 0
def parse_money(value: Any) -> float:
current = text(value).replace(",", "")
if not current or current in {"-", "--"}:
return 0.0
negative = current.startswith("-")
digits = re.sub(r"[^\d.]", "", current)
if not digits:
return 0.0
amount = round(float(digits), 2)
return -amount if negative else amount
def extract_statement(
candidate: dict[str, Any],
config_account: dict[str, Any],
target_date: date,
) -> tuple[Path, int | None, float | None, float | None]:
password = str(config_account["password"])
account = str(config_account["bank_account"])
company = str(config_account["company"])
try:
with zipfile.ZipFile(io.BytesIO(candidate["payload"])) as archive:
members = [
item for item in archive.infolist()
if not item.is_dir() and item.filename.lower().endswith(".xlsx")
]
if len(members) != 1:
raise StatementError("账单 ZIP 中应且只能包含一个 Excel 文件。")
member = members[0]
if Path(member.filename).name != member.filename:
raise StatementError("账单 ZIP 包含不安全文件路径。")
workbook_bytes = archive.read(member, pwd=password.encode("utf-8"))
except RuntimeError as exception:
raise StatementError("账单解压密码不正确。") from exception
except zipfile.BadZipFile as exception:
raise StatementError("账单附件不是有效 ZIP。") from exception
warnings.filterwarnings("ignore", category=UserWarning, module="openpyxl")
workbook = openpyxl.load_workbook(
io.BytesIO(workbook_bytes),
read_only=True,
data_only=True,
)
try:
worksheet = workbook.active
worksheet.reset_dimensions()
iterator = worksheet.iter_rows(values_only=True)
preamble: list[list[str]] = []
headers: list[str] | None = None
data_rows: list[list[str]] = []
for row in iterator:
values = [text(value) for value in row]
normalized_values = {normalized(value) for value in values}
if "交易时间" in normalized_values and normalized_values & AMOUNT_HEADERS:
headers = values
break
preamble.append(values)
if headers is None:
raise StatementError("账单 Excel 缺少交易时间或金额表头。")
for row in iterator:
values = [text(value) for value in row]
if any(values):
data_rows.append(values)
finally:
workbook.close()
summary = summary_values(preamble)
statement_company = summary.get("企业名称", "")
statement_account = account_digits(summary.get("企业账号", ""))
if statement_company and normalized(statement_company) != normalized(company):
raise StatementError("账单企业名称与账号表不一致。")
if statement_account and statement_account != account:
raise StatementError("账单企业账号与配置不一致。")
expected_rows = None
expected_income = None
expected_expense = None
if "借方交易笔数" in summary:
expected_rows = parse_count(summary["借方交易笔数"]) + parse_count(
summary.get("贷方交易笔数", "")
)
if expected_rows != len(data_rows):
raise StatementError(
f"账单汇总为 {expected_rows} 笔,但明细实际为 {len(data_rows)} 笔。"
)
if "借方交易金额" in summary:
expected_income = round(abs(parse_money(summary["借方交易金额"])), 2)
if "贷方交易金额" in summary:
expected_expense = round(abs(parse_money(summary["贷方交易金额"])), 2)
column = {normalized(value): index for index, value in enumerate(headers)}
time_index = next(
(column[item] for item in ("交易时间", "记账时间", "发生时间", "入账时间")
if item in column),
None,
)
status_index = next(
(column[item] for item in STATUS_HEADERS if item in column),
None,
)
if time_index is None:
raise StatementError("账单缺少交易时间列。")
valid_rows: list[list[str]] = []
for row in data_rows:
current = row[time_index] if time_index < len(row) else ""
if not current or not current.startswith(target_date.isoformat()):
continue
if status_index is not None and status_index < len(row):
if FAILED_STATUS.search(row[status_index]):
continue
valid_rows.append(row)
output = (
Path(candidate["output_dir"])
/ f"{account}-{target_date.strftime('%Y%m%d')}-交易明细.csv"
)
output.parent.mkdir(parents=True, exist_ok=True)
with output.open("w", encoding="utf-8-sig", newline="") as stream:
writer = csv.writer(stream)
writer.writerow(headers[: max(len(headers), 1)])
writer.writerows(valid_rows)
if expected_rows is not None and expected_rows != len(valid_rows):
raise StatementError(
f"账单汇总为 {expected_rows} 笔,筛选目标日期后为 {len(valid_rows)} 笔。"
)
return output, expected_rows, expected_income, expected_expense
def build_bundle(
run_directory: Path,
target_date: date,
entries: list[dict[str, Any]],
) -> Path:
bundle_directory = run_directory / "bundle"
bundle_directory.mkdir(parents=True, exist_ok=True)
manifest_accounts: list[dict[str, Any]] = []
for item in entries:
relative = f"accounts/{item['account'][-4:]}/{item['csv'].name}"
target = bundle_directory / relative
target.parent.mkdir(parents=True, exist_ok=True)
target.write_bytes(item["csv"].read_bytes())
manifest_accounts.append({
"company": item["company"],
"account": item["account"],
"account_short": item["account_short"],
"expected_rows": item["expected_rows"],
"expected_income": item["expected_income"],
"expected_expense": item["expected_expense"],
"file": relative,
})
manifest = {
"version": 1,
"date": target_date.isoformat(),
"generated_at": datetime.now(SHANGHAI).isoformat(timespec="seconds"),
"accounts": manifest_accounts,
}
manifest_file = bundle_directory / "manifest.json"
write_json(manifest_file, manifest)
bundle = run_directory / f"wangshang-mail-{target_date.isoformat()}.zip"
with zipfile.ZipFile(bundle, "w", compression=zipfile.ZIP_DEFLATED) as archive:
for path in sorted(bundle_directory.rglob("*")):
if path.is_file():
archive.write(path, path.relative_to(bundle_directory).as_posix())
return bundle
def fetch(args: argparse.Namespace) -> int:
config_file = Path(args.config).expanduser().resolve()
if not config_file.is_file():
raise StatementError(f"缺少网商邮件配置:{config_file}")
config = json.loads(config_file.read_text(encoding="utf-8"))
mail = config.get("mail")
accounts = config.get("accounts")
if not isinstance(mail, dict) or not isinstance(accounts, list) or not accounts:
raise StatementError("网商邮件配置格式无效。")
target_date = parse_date(
args.date or (datetime.now(SHANGHAI).date() - timedelta(days=1)).isoformat()
)
output_root = Path(
args.out or config_file.parent / "wangshang-mail" / "exports" / target_date.isoformat()
).expanduser().resolve()
run_directory = (
Path(args.run_dir).expanduser().resolve()
if args.run_dir
else output_root / datetime.now(SHANGHAI).strftime("%Y%m%d-%H%M%S")
)
run_directory.mkdir(parents=True, exist_ok=True)
state_file = run_directory / "last-fetch.json"
selected: list[dict[str, Any]] = []
for item in accounts:
if not isinstance(item, dict) or not item.get("enabled", True):
continue
if args.company and str(args.company) not in str(item.get("company", "")):
continue
selected.append(item)
selected = selected[: max(1, int(args.limit or 999))]
if not selected:
raise StatementError("没有匹配的已启用网商公司。")
candidates = load_candidates(mail, target_date)
print(f"邮箱匹配到 {len(candidates)} 个网商账单附件")
successes: list[dict[str, Any]] = []
failures: list[dict[str, Any]] = []
for index, account_config in enumerate(selected, start=1):
company = str(account_config.get("company", "")).strip()
account = account_digits(account_config.get("bank_account"))
print(f"[{index}/{len(selected)}] {company},账号 {mask_account(account)}")
try:
if not company or not account:
raise StatementError("配置缺少公司或账号。")
candidate = choose_candidate(candidates, account, target_date)
account_directory = run_directory / "downloads" / account[-4:]
account_directory.mkdir(parents=True, exist_ok=True)
source_zip = account_directory / Path(candidate["filename"]).name
source_zip.write_bytes(candidate["payload"])
candidate["output_dir"] = account_directory
csv_file, expected_rows, expected_income, expected_expense = extract_statement(
candidate,
account_config,
target_date,
)
successes.append({
"company": company,
"account": account,
"account_short": str(account_config.get("account_short") or company),
"csv": csv_file,
"expected_rows": expected_rows,
"expected_income": expected_income,
"expected_expense": expected_expense,
"message_uid": candidate["uid"],
"attachment": candidate["filename"],
})
print(f" {expected_rows if expected_rows is not None else '?'} 笔,已解析")
except Exception as exception:
reason = str(exception).replace(str(mail.get("auth_code", "")), "***")
failures.append({
"company": company,
"account_last4": account[-4:],
"error": reason[:500],
})
print(f" 失败:{reason}", file=sys.stderr)
bundle: Path | None = None
if successes:
bundle = build_bundle(run_directory, target_date, successes)
state = {
"ok": bool(successes) and not failures,
"date": target_date.isoformat(),
"accounts": len(successes),
"failed": len(failures),
"failures": failures,
"bundle": str(bundle) if bundle else "",
"generated_at": datetime.now(SHANGHAI).isoformat(timespec="seconds"),
}
write_json(state_file, state)
print(
f"批次完成:成功 {len(successes)},失败 {len(failures)},"
f"状态文件 {state_file}"
)
return 0 if state["ok"] else 1
def parser() -> argparse.ArgumentParser:
root = argparse.ArgumentParser(description="网商银行邮件账单自动获取")
commands = root.add_subparsers(dest="command", required=True)
initialize = commands.add_parser("init", help="从账号表生成私有配置")
initialize.add_argument("--workbook", required=True)
initialize.add_argument("--output", required=True)
initialize.add_argument("--auth-code-env", default="WANGSHANG_MAIL_AUTH_CODE")
initialize.add_argument("--imap-host", default="")
initialize.add_argument("--imap-port", type=int, default=993)
initialize.set_defaults(handler=init_config)
fetch_command = commands.add_parser("fetch", help="从邮箱获取并生成导入批次")
fetch_command.add_argument("--config", required=True)
fetch_command.add_argument("--date", default="")
fetch_command.add_argument("--out", default="")
fetch_command.add_argument("--run-dir", default="")
fetch_command.add_argument("--company", default="")
fetch_command.add_argument("--limit", type=int, default=999)
fetch_command.set_defaults(handler=fetch)
return root
def main() -> int:
args = parser().parse_args()
try:
return int(args.handler(args))
except StatementError as exception:
print(f"网商银行邮件任务失败:{exception}", file=sys.stderr)
return 2
except Exception as exception:
print(f"网商银行邮件任务异常:{type(exception).__name__}: {exception}", file=sys.stderr)
return 2
if __name__ == "__main__":
raise SystemExit(main())