178 lines
7.9 KiB
Python
178 lines
7.9 KiB
Python
#!/usr/bin/env python3
|
|
"""Stream a work-platform XLSX into a minimal, auditable monthly shipping input.
|
|
|
|
Uses only the Python standard library. Shared strings stay in a temporary SQLite
|
|
file with an 8MiB page cache; worksheet rows and CSV output are streamed. No
|
|
customer names, phone numbers, addresses, notes or settlement freight are copied.
|
|
"""
|
|
import argparse
|
|
import csv
|
|
import gzip
|
|
import hashlib
|
|
import json
|
|
import re
|
|
import sqlite3
|
|
import tempfile
|
|
import time
|
|
import zipfile
|
|
from datetime import datetime, timedelta
|
|
from contextlib import ExitStack
|
|
from functools import lru_cache
|
|
from pathlib import Path
|
|
from xml.etree import ElementTree as ET
|
|
|
|
NS = '{http://schemas.openxmlformats.org/spreadsheetml/2006/main}'
|
|
FIELDS = {
|
|
'express_no': '快递单号', 'old_express_no': '换单前单号',
|
|
'trade_no': '交易单号', 'product_id': '商品ID', 'barcode': '条形码',
|
|
'qty': '数量', 'freight': '运费', 'ship_time': '发货时间',
|
|
'package_no': '配货单号', 'unshipped': '未发出',
|
|
}
|
|
|
|
|
|
def digest(path):
|
|
h = hashlib.sha256()
|
|
with path.open('rb') as stream:
|
|
for chunk in iter(lambda: stream.read(1024 * 1024), b''):
|
|
h.update(chunk)
|
|
return h.hexdigest()
|
|
|
|
|
|
def elements(stream, tag):
|
|
"""Clear parent references as well as cell contents, keeping memory bounded."""
|
|
stack = []
|
|
with stream:
|
|
for event, element in ET.iterparse(stream, events=('start', 'end')):
|
|
if event == 'start':
|
|
stack.append(element)
|
|
else:
|
|
if element.tag == NS + tag:
|
|
yield element
|
|
element.clear()
|
|
if len(stack) > 1:
|
|
stack[-2].remove(element)
|
|
stack.pop()
|
|
|
|
|
|
def date_value(value):
|
|
if re.fullmatch(r'\d+(?:\.\d+)?', value):
|
|
return (datetime(1899, 12, 30) + timedelta(days=float(value))).strftime('%Y-%m-%d %H:%M:%S')
|
|
value = value.replace('/', '-').replace('T', ' ')
|
|
for fmt in ('%Y-%m-%d %H:%M:%S', '%Y-%m-%d %H:%M', '%Y-%m-%d'):
|
|
try:
|
|
return datetime.strptime(value, fmt).strftime('%Y-%m-%d %H:%M:%S')
|
|
except ValueError:
|
|
pass
|
|
return value # Invalid/missing dates remain explicit and cannot be valued.
|
|
|
|
|
|
def prepare(source, month, output, freight_column):
|
|
started = time.perf_counter()
|
|
if not re.fullmatch(r'\d{4}-(0[1-9]|1[0-2])', month):
|
|
raise ValueError('月份须为YYYY-MM')
|
|
if output.exists() or Path(str(output) + '.json').exists():
|
|
raise ValueError('输出已存在,请使用新路径以保留既有导入凭据')
|
|
output.parent.mkdir(parents=True, exist_ok=True)
|
|
temp_output = Path(str(output) + '.partial')
|
|
source_hash = digest(source)
|
|
with tempfile.TemporaryDirectory(prefix='sb-shipping-') as temp, zipfile.ZipFile(source) as book, ExitStack() as cleanup:
|
|
db = sqlite3.connect(Path(temp) / 'strings.sqlite')
|
|
cleanup.callback(db.close)
|
|
db.execute('PRAGMA journal_mode=OFF')
|
|
db.execute('PRAGMA synchronous=OFF')
|
|
db.execute('PRAGMA cache_size=-8192')
|
|
db.execute('CREATE TABLE strings(id INTEGER PRIMARY KEY,value TEXT)')
|
|
if 'xl/sharedStrings.xml' in book.namelist():
|
|
batch = []
|
|
for index, element in enumerate(elements(book.open('xl/sharedStrings.xml'), 'si')):
|
|
batch.append((index, ''.join(e.text or '' for e in element.iter(NS + 't'))))
|
|
if len(batch) >= 2000:
|
|
db.executemany('INSERT INTO strings VALUES (?,?)', batch)
|
|
batch = []
|
|
if batch:
|
|
db.executemany('INSERT INTO strings VALUES (?,?)', batch)
|
|
db.commit()
|
|
|
|
@lru_cache(maxsize=30000)
|
|
def shared_string(index):
|
|
result = db.execute('SELECT value FROM strings WHERE id=?', (int(index),)).fetchone()
|
|
if result is None:
|
|
raise ValueError('共享字符串索引无效')
|
|
return result[0]
|
|
|
|
def cell(element):
|
|
if element.get('t') == 'inlineStr':
|
|
return ''.join(e.text or '' for e in element.iter(NS + 't'))
|
|
value = element.find(NS + 'v')
|
|
if value is None or value.text is None:
|
|
return ''
|
|
return shared_string(value.text) if element.get('t') == 's' else value.text
|
|
|
|
workbook = ET.fromstring(book.read('xl/workbook.xml'))
|
|
sheets = workbook.findall(NS + 'sheets/' + NS + 'sheet')
|
|
matching = [sheet for sheet in sheets if sheet.get('name') == '导出结果']
|
|
if len(matching) != 1:
|
|
raise ValueError('须包含唯一的“导出结果”工作表')
|
|
rel_id = matching[0].get('{http://schemas.openxmlformats.org/officeDocument/2006/relationships}id')
|
|
relationships = ET.fromstring(book.read('xl/_rels/workbook.xml.rels'))
|
|
target = next(rel.get('Target') for rel in relationships if rel.get('Id') == rel_id).lstrip('/')
|
|
if not target.startswith('xl/'):
|
|
target = 'xl/' + target
|
|
count = 0
|
|
missing_dates = 0
|
|
with gzip.open(temp_output, 'wt', encoding='utf-8', newline='') as stream:
|
|
writer = csv.writer(stream)
|
|
writer.writerow(['source_row', *FIELDS])
|
|
mapping = None
|
|
worksheet_rows = elements(book.open(target), 'row')
|
|
cleanup.callback(worksheet_rows.close)
|
|
for row in worksheet_rows:
|
|
row_number = int(row.get('r'))
|
|
if mapping is None:
|
|
headers = {}
|
|
for c in row.findall(NS + 'c'):
|
|
label = cell(c)
|
|
headers.setdefault(label, []).append(re.sub(r'\d+$', '', c.get('r')))
|
|
fields = dict(FIELDS, freight=freight_column)
|
|
required = ['express_no', 'trade_no', 'barcode', 'qty', 'freight', 'ship_time']
|
|
if any(len(headers.get(fields[key], [])) != 1 for key in required):
|
|
raise ValueError('必需表头缺失或重复')
|
|
mapping = {headers[label][0]: key for key, label in fields.items() if label in headers}
|
|
continue
|
|
values = {}
|
|
for c in row.findall(NS + 'c'):
|
|
key = mapping.get(re.sub(r'\d+$', '', c.get('r')))
|
|
if key:
|
|
values[key] = cell(c).strip()
|
|
if not any(values.values()):
|
|
continue
|
|
date = date_value(values.get('ship_time', ''))
|
|
if re.fullmatch(r'\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}', date):
|
|
if date[:7] != month:
|
|
raise ValueError('发货日期超出所选月份')
|
|
else:
|
|
missing_dates += 1
|
|
values['ship_time'] = date
|
|
writer.writerow([row_number, *[values.get(key, '') for key in FIELDS]])
|
|
count += 1
|
|
shared_string.cache_clear()
|
|
db.close()
|
|
if count == 0:
|
|
raise ValueError('工作表没有发货记录')
|
|
temp_output.rename(output)
|
|
meta = dict(schema_version=1, source_month=month, source_filename=source.name,
|
|
source_sha256=source_hash, data_sha256=digest(output), row_count=count,
|
|
freight_column=freight_column)
|
|
Path(str(output) + '.json').write_text(json.dumps(meta, ensure_ascii=False, indent=2), encoding='utf-8')
|
|
return dict(rows=count, missing_dates=missing_dates, seconds=round(time.perf_counter()-started, 2), output_bytes=output.stat().st_size)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('source', type=Path)
|
|
parser.add_argument('month')
|
|
parser.add_argument('output', type=Path)
|
|
parser.add_argument('--freight-column', choices=['运费', '系统运费'], default='运费')
|
|
args = parser.parse_args()
|
|
print(json.dumps(prepare(args.source, args.month, args.output, args.freight_column), ensure_ascii=False))
|