90 lines
4.0 KiB
Python
90 lines
4.0 KiB
Python
#!/usr/bin/env python3
|
|
"""Run registered imports in the existing, resource-limited batch profile."""
|
|
import json
|
|
import os
|
|
import pathlib
|
|
import shutil
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import uuid
|
|
|
|
|
|
def available_memory_kib():
|
|
meminfo = dict(line.split(':', 1) for line in pathlib.Path('/proc/meminfo').read_text().splitlines())
|
|
return int(meminfo['MemAvailable'].split()[0])
|
|
|
|
|
|
def main():
|
|
arguments = sys.argv[1:]
|
|
registry = json.loads(pathlib.Path(__file__).with_name('finance-batch-imports.json').read_text())
|
|
if not arguments or arguments[0] not in registry:
|
|
print('Unsupported import; register the command in finance-batch-imports.json first.', file=sys.stderr)
|
|
return 64
|
|
|
|
# Keep a GiB available to the host in addition to this import's one-GiB cap.
|
|
if available_memory_kib() < 2 * 1024 * 1024:
|
|
print('Available memory is below 2GiB; no import container started. Retry after other tasks finish.', file=sys.stderr)
|
|
return 75
|
|
|
|
environment = os.environ.copy()
|
|
environment['FINANCE_BATCH_MEMORY_LIMIT'] = '1g'
|
|
environment['RUN_MIGRATIONS'] = '0'
|
|
image = subprocess.check_output(['docker', 'inspect', 'caiwu_app', '--format', '{{.Config.Image}}'], text=True).strip()
|
|
if not image.startswith('caiwu_akaunting:'):
|
|
print('Unsupported active finance image; no import container started.', file=sys.stderr)
|
|
return 65
|
|
environment['IMAGE_TAG'] = image.split(':', 1)[1]
|
|
container_name = 'caiwu_import_' + uuid.uuid4().hex
|
|
command = ['docker', 'compose', '-f', 'compose.jiangling.yml', '--profile', 'batch', 'run', '--rm', '-T', '--no-deps', '--name', container_name]
|
|
|
|
# Copy explicit host input files into a private per-run directory. The source
|
|
# is never modified; the container gets read-only copies readable by www-data.
|
|
with tempfile.TemporaryDirectory(prefix='caiwu-import-') as staging:
|
|
inputs = pathlib.Path(staging) / 'inputs'
|
|
inputs.mkdir(mode=0o755)
|
|
staged_arguments = [arguments[0]]
|
|
staged_count = 0
|
|
for argument in arguments[1:]:
|
|
prefix = ''
|
|
value = argument
|
|
if argument.startswith(('--file=', '--path=')):
|
|
prefix, value = argument.split('=', 1)
|
|
prefix += '='
|
|
source = pathlib.Path(value)
|
|
if not source.is_absolute():
|
|
source = pathlib.Path(os.environ.get('CAIWU_BATCH_CALLER_DIR', os.getcwd())) / source
|
|
if (prefix or not argument.startswith('-')) and source.is_file():
|
|
parent = inputs / str(staged_count)
|
|
parent.mkdir(mode=0o755)
|
|
destination = parent / source.name
|
|
shutil.copyfile(source, destination)
|
|
destination.chmod(0o444)
|
|
staged_arguments.append(prefix + '/run/caiwu-import/' + parent.name + '/' + destination.name)
|
|
staged_count += 1
|
|
else:
|
|
staged_arguments.append(argument)
|
|
if staged_count:
|
|
command += ['--volume', str(inputs) + ':/run/caiwu-import:ro']
|
|
command += ['--entrypoint', 'php', 'batch', '-d', 'memory_limit=512M', 'artisan', *staged_arguments]
|
|
print('Import limits: 1 CPU, 1GiB RAM, no Swap, PHP 512MiB; existing synchronization lock held.', flush=True)
|
|
try:
|
|
return subprocess.run(command, env=environment, timeout=3600).returncode
|
|
except subprocess.TimeoutExpired:
|
|
print('Import exceeded one hour; stop and inspect its checkpoint before retrying.', file=sys.stderr)
|
|
return 124
|
|
except KeyboardInterrupt:
|
|
return 130
|
|
finally:
|
|
# Killing only the Compose client would leave the job consuming RAM.
|
|
subprocess.run(['docker', 'rm', '--force', container_name], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=15)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
def interrupted(signum, frame):
|
|
raise KeyboardInterrupt
|
|
signal.signal(signal.SIGTERM, interrupted)
|
|
signal.signal(signal.SIGHUP, interrupted)
|
|
sys.exit(main())
|