CaiWuXiTong/scripts/prepare-supplier-bookkeeping-shipping.py

178 lines
7.9 KiB
Python

#!/usr/bin/env python3
"""Stream a work-platform XLSX into a minimal, auditable monthly shipping input.
Uses only the Python standard library. Shared strings stay in a temporary SQLite
file with an 8MiB page cache; worksheet rows and CSV output are streamed. No
customer names, phone numbers, addresses, notes or settlement freight are copied.
"""
import argparse
import csv
import gzip
import hashlib
import json
import re
import sqlite3
import tempfile
import time
import zipfile
from datetime import datetime, timedelta
from contextlib import ExitStack
from functools import lru_cache
from pathlib import Path
from xml.etree import ElementTree as ET
NS = '{http://schemas.openxmlformats.org/spreadsheetml/2006/main}'
FIELDS = {
'express_no': '快递单号', 'old_express_no': '换单前单号',
'trade_no': '交易单号', 'product_id': '商品ID', 'barcode': '条形码',
'qty': '数量', 'freight': '运费', 'ship_time': '发货时间',
'package_no': '配货单号', 'unshipped': '未发出',
}
def digest(path):
h = hashlib.sha256()
with path.open('rb') as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b''):
h.update(chunk)
return h.hexdigest()
def elements(stream, tag):
"""Clear parent references as well as cell contents, keeping memory bounded."""
stack = []
with stream:
for event, element in ET.iterparse(stream, events=('start', 'end')):
if event == 'start':
stack.append(element)
else:
if element.tag == NS + tag:
yield element
element.clear()
if len(stack) > 1:
stack[-2].remove(element)
stack.pop()
def date_value(value):
if re.fullmatch(r'\d+(?:\.\d+)?', value):
return (datetime(1899, 12, 30) + timedelta(days=float(value))).strftime('%Y-%m-%d %H:%M:%S')
value = value.replace('/', '-').replace('T', ' ')
for fmt in ('%Y-%m-%d %H:%M:%S', '%Y-%m-%d %H:%M', '%Y-%m-%d'):
try:
return datetime.strptime(value, fmt).strftime('%Y-%m-%d %H:%M:%S')
except ValueError:
pass
return value # Invalid/missing dates remain explicit and cannot be valued.
def prepare(source, month, output, freight_column):
started = time.perf_counter()
if not re.fullmatch(r'\d{4}-(0[1-9]|1[0-2])', month):
raise ValueError('月份须为YYYY-MM')
if output.exists() or Path(str(output) + '.json').exists():
raise ValueError('输出已存在,请使用新路径以保留既有导入凭据')
output.parent.mkdir(parents=True, exist_ok=True)
temp_output = Path(str(output) + '.partial')
source_hash = digest(source)
with tempfile.TemporaryDirectory(prefix='sb-shipping-') as temp, zipfile.ZipFile(source) as book, ExitStack() as cleanup:
db = sqlite3.connect(Path(temp) / 'strings.sqlite')
cleanup.callback(db.close)
db.execute('PRAGMA journal_mode=OFF')
db.execute('PRAGMA synchronous=OFF')
db.execute('PRAGMA cache_size=-8192')
db.execute('CREATE TABLE strings(id INTEGER PRIMARY KEY,value TEXT)')
if 'xl/sharedStrings.xml' in book.namelist():
batch = []
for index, element in enumerate(elements(book.open('xl/sharedStrings.xml'), 'si')):
batch.append((index, ''.join(e.text or '' for e in element.iter(NS + 't'))))
if len(batch) >= 2000:
db.executemany('INSERT INTO strings VALUES (?,?)', batch)
batch = []
if batch:
db.executemany('INSERT INTO strings VALUES (?,?)', batch)
db.commit()
@lru_cache(maxsize=30000)
def shared_string(index):
result = db.execute('SELECT value FROM strings WHERE id=?', (int(index),)).fetchone()
if result is None:
raise ValueError('共享字符串索引无效')
return result[0]
def cell(element):
if element.get('t') == 'inlineStr':
return ''.join(e.text or '' for e in element.iter(NS + 't'))
value = element.find(NS + 'v')
if value is None or value.text is None:
return ''
return shared_string(value.text) if element.get('t') == 's' else value.text
workbook = ET.fromstring(book.read('xl/workbook.xml'))
sheets = workbook.findall(NS + 'sheets/' + NS + 'sheet')
matching = [sheet for sheet in sheets if sheet.get('name') == '导出结果']
if len(matching) != 1:
raise ValueError('须包含唯一的“导出结果”工作表')
rel_id = matching[0].get('{http://schemas.openxmlformats.org/officeDocument/2006/relationships}id')
relationships = ET.fromstring(book.read('xl/_rels/workbook.xml.rels'))
target = next(rel.get('Target') for rel in relationships if rel.get('Id') == rel_id).lstrip('/')
if not target.startswith('xl/'):
target = 'xl/' + target
count = 0
missing_dates = 0
with gzip.open(temp_output, 'wt', encoding='utf-8', newline='') as stream:
writer = csv.writer(stream)
writer.writerow(['source_row', *FIELDS])
mapping = None
worksheet_rows = elements(book.open(target), 'row')
cleanup.callback(worksheet_rows.close)
for row in worksheet_rows:
row_number = int(row.get('r'))
if mapping is None:
headers = {}
for c in row.findall(NS + 'c'):
label = cell(c)
headers.setdefault(label, []).append(re.sub(r'\d+$', '', c.get('r')))
fields = dict(FIELDS, freight=freight_column)
required = ['express_no', 'trade_no', 'barcode', 'qty', 'freight', 'ship_time']
if any(len(headers.get(fields[key], [])) != 1 for key in required):
raise ValueError('必需表头缺失或重复')
mapping = {headers[label][0]: key for key, label in fields.items() if label in headers}
continue
values = {}
for c in row.findall(NS + 'c'):
key = mapping.get(re.sub(r'\d+$', '', c.get('r')))
if key:
values[key] = cell(c).strip()
if not any(values.values()):
continue
date = date_value(values.get('ship_time', ''))
if re.fullmatch(r'\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}', date):
if date[:7] != month:
raise ValueError('发货日期超出所选月份')
else:
missing_dates += 1
values['ship_time'] = date
writer.writerow([row_number, *[values.get(key, '') for key in FIELDS]])
count += 1
shared_string.cache_clear()
db.close()
if count == 0:
raise ValueError('工作表没有发货记录')
temp_output.rename(output)
meta = dict(schema_version=1, source_month=month, source_filename=source.name,
source_sha256=source_hash, data_sha256=digest(output), row_count=count,
freight_column=freight_column)
Path(str(output) + '.json').write_text(json.dumps(meta, ensure_ascii=False, indent=2), encoding='utf-8')
return dict(rows=count, missing_dates=missing_dates, seconds=round(time.perf_counter()-started, 2), output_bytes=output.stat().st_size)
if __name__ == '__main__':
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('source', type=Path)
parser.add_argument('month')
parser.add_argument('output', type=Path)
parser.add_argument('--freight-column', choices=['运费', '系统运费'], default='运费')
args = parser.parse_args()
print(json.dumps(prepare(args.source, args.month, args.output, args.freight_column), ensure_ascii=False))