#!/usr/bin/env python3 """Stream a work-platform XLSX into a minimal, auditable monthly shipping input. Uses only the Python standard library. Shared strings stay in a temporary SQLite file with an 8MiB page cache; worksheet rows and CSV output are streamed. No customer names, phone numbers, addresses, notes or settlement freight are copied. """ import argparse import csv import gzip import hashlib import json import re import sqlite3 import tempfile import time import zipfile from datetime import datetime, timedelta from contextlib import ExitStack from functools import lru_cache from pathlib import Path from xml.etree import ElementTree as ET NS = '{http://schemas.openxmlformats.org/spreadsheetml/2006/main}' FIELDS = { 'express_no': '快递单号', 'old_express_no': '换单前单号', 'trade_no': '交易单号', 'product_id': '商品ID', 'barcode': '条形码', 'qty': '数量', 'freight': '运费', 'ship_time': '发货时间', 'package_no': '配货单号', 'unshipped': '未发出', } def digest(path): h = hashlib.sha256() with path.open('rb') as stream: for chunk in iter(lambda: stream.read(1024 * 1024), b''): h.update(chunk) return h.hexdigest() def elements(stream, tag): """Clear parent references as well as cell contents, keeping memory bounded.""" stack = [] with stream: for event, element in ET.iterparse(stream, events=('start', 'end')): if event == 'start': stack.append(element) else: if element.tag == NS + tag: yield element element.clear() if len(stack) > 1: stack[-2].remove(element) stack.pop() def date_value(value): if re.fullmatch(r'\d+(?:\.\d+)?', value): return (datetime(1899, 12, 30) + timedelta(days=float(value))).strftime('%Y-%m-%d %H:%M:%S') value = value.replace('/', '-').replace('T', ' ') for fmt in ('%Y-%m-%d %H:%M:%S', '%Y-%m-%d %H:%M', '%Y-%m-%d'): try: return datetime.strptime(value, fmt).strftime('%Y-%m-%d %H:%M:%S') except ValueError: pass return value # Invalid/missing dates remain explicit and cannot be valued. def prepare(source, month, output, freight_column): started = time.perf_counter() if not re.fullmatch(r'\d{4}-(0[1-9]|1[0-2])', month): raise ValueError('月份须为YYYY-MM') if output.exists() or Path(str(output) + '.json').exists(): raise ValueError('输出已存在,请使用新路径以保留既有导入凭据') output.parent.mkdir(parents=True, exist_ok=True) temp_output = Path(str(output) + '.partial') source_hash = digest(source) with tempfile.TemporaryDirectory(prefix='sb-shipping-') as temp, zipfile.ZipFile(source) as book, ExitStack() as cleanup: db = sqlite3.connect(Path(temp) / 'strings.sqlite') cleanup.callback(db.close) db.execute('PRAGMA journal_mode=OFF') db.execute('PRAGMA synchronous=OFF') db.execute('PRAGMA cache_size=-8192') db.execute('CREATE TABLE strings(id INTEGER PRIMARY KEY,value TEXT)') if 'xl/sharedStrings.xml' in book.namelist(): batch = [] for index, element in enumerate(elements(book.open('xl/sharedStrings.xml'), 'si')): batch.append((index, ''.join(e.text or '' for e in element.iter(NS + 't')))) if len(batch) >= 2000: db.executemany('INSERT INTO strings VALUES (?,?)', batch) batch = [] if batch: db.executemany('INSERT INTO strings VALUES (?,?)', batch) db.commit() @lru_cache(maxsize=30000) def shared_string(index): result = db.execute('SELECT value FROM strings WHERE id=?', (int(index),)).fetchone() if result is None: raise ValueError('共享字符串索引无效') return result[0] def cell(element): if element.get('t') == 'inlineStr': return ''.join(e.text or '' for e in element.iter(NS + 't')) value = element.find(NS + 'v') if value is None or value.text is None: return '' return shared_string(value.text) if element.get('t') == 's' else value.text workbook = ET.fromstring(book.read('xl/workbook.xml')) sheets = workbook.findall(NS + 'sheets/' + NS + 'sheet') matching = [sheet for sheet in sheets if sheet.get('name') == '导出结果'] if len(matching) != 1: raise ValueError('须包含唯一的“导出结果”工作表') rel_id = matching[0].get('{http://schemas.openxmlformats.org/officeDocument/2006/relationships}id') relationships = ET.fromstring(book.read('xl/_rels/workbook.xml.rels')) target = next(rel.get('Target') for rel in relationships if rel.get('Id') == rel_id).lstrip('/') if not target.startswith('xl/'): target = 'xl/' + target count = 0 missing_dates = 0 with gzip.open(temp_output, 'wt', encoding='utf-8', newline='') as stream: writer = csv.writer(stream) writer.writerow(['source_row', *FIELDS]) mapping = None worksheet_rows = elements(book.open(target), 'row') cleanup.callback(worksheet_rows.close) for row in worksheet_rows: row_number = int(row.get('r')) if mapping is None: headers = {} for c in row.findall(NS + 'c'): label = cell(c) headers.setdefault(label, []).append(re.sub(r'\d+$', '', c.get('r'))) fields = dict(FIELDS, freight=freight_column) required = ['express_no', 'trade_no', 'barcode', 'qty', 'freight', 'ship_time'] if any(len(headers.get(fields[key], [])) != 1 for key in required): raise ValueError('必需表头缺失或重复') mapping = {headers[label][0]: key for key, label in fields.items() if label in headers} continue values = {} for c in row.findall(NS + 'c'): key = mapping.get(re.sub(r'\d+$', '', c.get('r'))) if key: values[key] = cell(c).strip() if not any(values.values()): continue date = date_value(values.get('ship_time', '')) if re.fullmatch(r'\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}', date): if date[:7] != month: raise ValueError('发货日期超出所选月份') else: missing_dates += 1 values['ship_time'] = date writer.writerow([row_number, *[values.get(key, '') for key in FIELDS]]) count += 1 shared_string.cache_clear() db.close() if count == 0: raise ValueError('工作表没有发货记录') temp_output.rename(output) meta = dict(schema_version=1, source_month=month, source_filename=source.name, source_sha256=source_hash, data_sha256=digest(output), row_count=count, freight_column=freight_column) Path(str(output) + '.json').write_text(json.dumps(meta, ensure_ascii=False, indent=2), encoding='utf-8') return dict(rows=count, missing_dates=missing_dates, seconds=round(time.perf_counter()-started, 2), output_bytes=output.stat().st_size) if __name__ == '__main__': parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('source', type=Path) parser.add_argument('month') parser.add_argument('output', type=Path) parser.add_argument('--freight-column', choices=['运费', '系统运费'], default='运费') args = parser.parse_args() print(json.dumps(prepare(args.source, args.month, args.output, args.freight_column), ensure_ascii=False))