427 lines
20 KiB
Python
427 lines
20 KiB
Python
"""Linux release operations. No SSH dependency; invoked by production48.py.
|
|
|
|
Only the ai-chatbot PM2 application is changed. The original checkout, public
|
|
files, credentials, dependencies and historical builds remain in place.
|
|
"""
|
|
|
|
import argparse
|
|
import datetime
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path, PurePosixPath
|
|
import re
|
|
import shutil
|
|
import socket
|
|
import subprocess
|
|
import tarfile
|
|
import time
|
|
import urllib.request
|
|
|
|
APP = Path("/var/www/ai-chatbot")
|
|
RELEASES = Path("/var/www/ai-chatbot-releases")
|
|
NAME = "ai-chatbot"
|
|
RESERVE = 1024**3
|
|
|
|
|
|
def require(value, message):
|
|
if not value:
|
|
raise RuntimeError(message)
|
|
|
|
|
|
def run(args, cwd=None, stdin=None, timeout=60):
|
|
result = subprocess.run(args, cwd=cwd, input=stdin, text=True,
|
|
capture_output=True, timeout=timeout)
|
|
require(result.returncode == 0, f"{args[0]} failed (exit {result.returncode})")
|
|
return result.stdout
|
|
|
|
|
|
def sha(path):
|
|
digest = hashlib.sha256()
|
|
with open(path, "rb") as stream:
|
|
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def json_write(path, value):
|
|
temporary = path.with_name(path.name + ".tmp")
|
|
with open(temporary, "x", encoding="utf8") as stream:
|
|
os.chmod(temporary, 0o600)
|
|
json.dump(value, stream, indent=2)
|
|
stream.write("\n")
|
|
os.replace(temporary, path)
|
|
|
|
|
|
def safe_extract(archive, destination):
|
|
"""Reject traversal, special files and links escaping a new directory."""
|
|
require(not destination.exists(), "Extraction destination already exists")
|
|
with tarfile.open(archive, "r:gz") as bundle:
|
|
members = bundle.getmembers()
|
|
for member in members:
|
|
path = PurePosixPath(member.name)
|
|
require(not path.is_absolute() and ".." not in path.parts,
|
|
"Unsafe archive member")
|
|
require(member.isdir() or member.isfile() or member.issym(),
|
|
"Special files and hard links are not accepted")
|
|
if member.issym():
|
|
link = PurePosixPath(member.linkname)
|
|
require(not link.is_absolute(), "Absolute archive symlink")
|
|
target = (destination / path.parent / member.linkname).resolve()
|
|
require(target.is_relative_to(destination.resolve()),
|
|
"Archive symlink escapes destination")
|
|
needed = sum(m.size for m in members if m.isfile())
|
|
require(shutil.disk_usage(destination.parent).free > needed + RESERVE,
|
|
"Insufficient space: at least 1 GiB must remain after extraction")
|
|
destination.mkdir(mode=0o700)
|
|
# Extract normal entries first; never follow an archive-created link.
|
|
for member in members:
|
|
if member.issym():
|
|
continue
|
|
path = destination / member.name
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
if member.isdir():
|
|
path.mkdir(exist_ok=True)
|
|
else:
|
|
with bundle.extractfile(member) as source, open(path, "xb") as target:
|
|
shutil.copyfileobj(source, target)
|
|
path.chmod(member.mode & 0o777)
|
|
for member in members:
|
|
if member.issym():
|
|
path = destination / member.name
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.symlink_to(member.linkname)
|
|
|
|
|
|
def release_path(release_id):
|
|
require(re.fullmatch(r"\d{8}T\d{6}Z-[0-9a-f]{12}", release_id),
|
|
"Invalid release ID")
|
|
return RELEASES / release_id
|
|
|
|
|
|
def app_state():
|
|
require(socket.gethostname() == "SfSTE0ttmsycO", "Not the verified production host")
|
|
apps = json.loads(run(["pm2", "jlist"]))
|
|
matching = [app for app in apps if app["name"] == NAME]
|
|
require(len(matching) == 1, "Expected exactly one ai-chatbot PM2 application")
|
|
app = matching[0]
|
|
env = app["pm2_env"]
|
|
cwd = Path(env["pm_cwd"])
|
|
require(cwd == APP or cwd.parent == RELEASES, "Unexpected production cwd")
|
|
require(env["pm_exec_path"] == str(APP / "node_modules/next/dist/bin/next"),
|
|
"Unexpected Next.js executable")
|
|
require(env.get("watch") is False, "PM2 watch must be disabled")
|
|
require(env.get("CANVAS_E2E_LOCAL") != "true", "Production E2E mode is enabled")
|
|
require(env.get("args") == ["start", "-p", "3125", "-H", "127.0.0.1"],
|
|
"Unexpected production listen arguments")
|
|
dist = env.get("NEXT_DIST_DIR", ".next")
|
|
require(re.fullmatch(r"\.next[\w.-]*", dist), "Unsafe active build directory")
|
|
return app, cwd, dist
|
|
|
|
|
|
def status():
|
|
app, cwd, dist = app_state()
|
|
return {"hostname": socket.gethostname(), "pid": app["pid"],
|
|
"status": app["pm2_env"]["status"], "cwd": str(cwd), "distDir": dist,
|
|
"buildId": (cwd / dist / "BUILD_ID").read_text().strip(),
|
|
"freeGiB": round(shutil.disk_usage(APP).free / 1024**3, 2),
|
|
"node": run(["node", "-v"]).strip()}
|
|
|
|
|
|
ENV_LOADER = """
|
|
const {createRequire}=require('node:module');
|
|
const nextRequire=createRequire(require.resolve('next/package.json'));
|
|
nextRequire('@next/env').loadEnvConfig(process.cwd(),false,{info(){},error(){}});
|
|
"""
|
|
|
|
|
|
def build_environment():
|
|
# Only public build-time configuration leaves production.
|
|
source = ENV_LOADER + """
|
|
const extra=['AUTH_URL','NEXTAUTH_URL','PASCAL_EDITOR_URL','CANVAS_DOWNLOAD_CDN_ORIGIN'];
|
|
console.log(JSON.stringify(Object.fromEntries(Object.entries(process.env)
|
|
.filter(([key])=>key.startsWith('NEXT_PUBLIC_')||extra.includes(key)))));
|
|
"""
|
|
app_state()
|
|
return json.loads(run(["node", "-"], cwd=APP, stdin=source))
|
|
|
|
|
|
def job_status():
|
|
source = ENV_LOADER + r"""
|
|
const sql=require('postgres')(process.env.POSTGRES_URL,{max:1,connect_timeout:5,onnotice(){}});
|
|
(async()=>{try{
|
|
const tables=['CanvasImageTask','CanvasCapabilityTask','CanvasAgentRun',
|
|
'EnhanceProTask','AgentPsdRenderTask','VideoStoryboardTask'];const result={};
|
|
await sql.begin('read only',async tx=>{
|
|
await tx.unsafe("set local statement_timeout='5s'");
|
|
for(const table of tables){
|
|
const exists=await tx`select to_regclass(${JSON.stringify(table)}) as name`;
|
|
if(!exists[0].name)throw new Error('Missing task table');
|
|
result[table]=await tx.unsafe('select status,count(*)::int as count,'+
|
|
'count(*) filter (where "updatedAt">now()-interval \'1 hour\')::int as recent '+
|
|
'from "'+table+'" where status not in '+
|
|
'(\'succeeded\',\'failed\',\'completed\',\'cancelled\',\'interrupted\',\'needs_review\') group by status');
|
|
}
|
|
});console.log(JSON.stringify(result));
|
|
}catch(e){console.error('Task preflight failed');process.exitCode=1;}
|
|
finally{await sql.end();}})();
|
|
"""
|
|
return json.loads(run(["node", "-"], cwd=APP, stdin=source, timeout=50))
|
|
|
|
|
|
def inventory_packages(root):
|
|
result = {}
|
|
for path in sorted((root / "node_modules/.pnpm").glob("*/node_modules/*/package.json")):
|
|
result[path.relative_to(root / "node_modules").as_posix()] = sha(path)
|
|
for path in sorted((root / "node_modules/.pnpm").glob("*/node_modules/@*/*/package.json")):
|
|
result[path.relative_to(root / "node_modules").as_posix()] = sha(path)
|
|
return result
|
|
|
|
|
|
def build_manifest(workspace):
|
|
source = workspace / "source"
|
|
metadata = json.loads((workspace / "input.json").read_text())
|
|
result = {"schema": 1, "sourceCommit": metadata["sourceCommit"],
|
|
"sourceTree": metadata["sourceTree"], "releaseId": workspace.name,
|
|
"nodeMajor": 20, "nodeAbi": "115", "pnpm": "9.12.3",
|
|
"buildId": (source / ".next/BUILD_ID").read_text().strip(),
|
|
"lockSha256": sha(source / "pnpm-lock.yaml"),
|
|
"publicFiles": {p.relative_to(source).as_posix(): sha(p)
|
|
for p in sorted((source / "public").rglob("*")) if p.is_file()},
|
|
"directPackages": {name: sha(source / "node_modules" / name / "package.json")
|
|
for name in json.loads((source / "package.json").read_text())["dependencies"]},
|
|
"installedPackages": inventory_packages(source)}
|
|
require(result["installedPackages"], "No dependency inventory")
|
|
json_write(source / "release.json", result)
|
|
with tarfile.open(workspace / "artifact.tar.gz", "w:gz", compresslevel=3) as archive:
|
|
for path in sorted(source.iterdir()):
|
|
if path.name in {"node_modules", "public", ".env.production.local", ".next"}:
|
|
continue
|
|
archive.add(path, arcname=path.name)
|
|
def filter_build(member):
|
|
if member.name == ".next/cache" or member.name.startswith(".next/cache/"):
|
|
return None
|
|
return member
|
|
archive.add(source / ".next", arcname=".next", filter=filter_build)
|
|
return {"releaseId": workspace.name, "sourceCommit": result["sourceCommit"],
|
|
"buildId": result["buildId"], "artifactSha256": sha(workspace / "artifact.tar.gz"),
|
|
"artifactBytes": (workspace / "artifact.tar.gz").stat().st_size}
|
|
|
|
|
|
def smoke(workspace):
|
|
source = workspace / "source"
|
|
expected = (source / ".next/BUILD_ID").read_text().strip()
|
|
with socket.socket() as listener:
|
|
listener.bind(("127.0.0.1", 0))
|
|
port = listener.getsockname()[1]
|
|
environment = os.environ.copy()
|
|
environment.update({"NODE_ENV": "production", "NEXT_DIST_DIR": ".next",
|
|
"PORT": str(port), "POSTGRES_CONNECT_TIMEOUT_SECONDS": "1"})
|
|
with open(workspace / "smoke.log", "w") as log:
|
|
process = subprocess.Popen(["node", "node_modules/next/dist/bin/next", "start",
|
|
"-H", "127.0.0.1", "-p", str(port)],
|
|
cwd=source, env=environment, stdout=log, stderr=log)
|
|
try:
|
|
ready = False
|
|
for _ in range(45):
|
|
require(process.poll() is None, "Isolated candidate exited; inspect smoke.log")
|
|
try:
|
|
with urllib.request.urlopen(f"http://127.0.0.1:{port}/", timeout=3) as response:
|
|
ready = response.status == 200 and expected in response.read().decode("utf8")
|
|
if ready:
|
|
break
|
|
except Exception:
|
|
pass
|
|
time.sleep(1)
|
|
require(ready, "Isolated candidate readiness failed; inspect smoke.log")
|
|
output = run(["env", f"SMOKE_BASE_URL=http://127.0.0.1:{port}", "node",
|
|
"scripts/smoke-routes.mjs"], cwd=source, timeout=100)
|
|
return {"candidateSmoke": "passed", "buildId": expected,
|
|
"checks": output.strip().splitlines()}
|
|
finally:
|
|
process.terminate()
|
|
try:
|
|
process.wait(timeout=20)
|
|
except subprocess.TimeoutExpired:
|
|
process.kill()
|
|
process.wait()
|
|
|
|
|
|
def verify_shared(manifest):
|
|
require(sha(APP / "pnpm-lock.yaml") == manifest["lockSha256"],
|
|
"Dependencies changed: provision a separate reviewed runtime dependency set first")
|
|
for name, digest in manifest["publicFiles"].items():
|
|
path = PurePosixPath(name)
|
|
require(path.parts[0] == "public" and ".." not in path.parts,
|
|
"Invalid public path")
|
|
require((APP / name).is_file() and sha(APP / name) == digest,
|
|
f"Public files differ: {name}; review shared asset publication first")
|
|
actual = inventory_packages(APP)
|
|
missing = [name for name, value in manifest["installedPackages"].items()
|
|
if actual.get(name) != value]
|
|
require(not missing, f"Installed dependency mismatch ({len(missing)} packages)")
|
|
for name, digest in manifest["directPackages"].items():
|
|
require(sha(APP / "node_modules" / name / "package.json") == digest,
|
|
f"Direct dependency link mismatch: {name}")
|
|
require(run(["node", "-p", "process.versions.modules"]).strip() == manifest["nodeAbi"],
|
|
"Node ABI mismatch")
|
|
|
|
|
|
def stage(release_id, digest):
|
|
_, current, dist = app_state()
|
|
archive = RELEASES / (release_id + ".tar.gz")
|
|
require(sha(archive) == digest, "Artifact checksum mismatch")
|
|
destination = release_path(release_id)
|
|
safe_extract(archive, destination)
|
|
manifest = json.loads((destination / "release.json").read_text())
|
|
require(manifest["releaseId"] == release_id, "Artifact release ID mismatch")
|
|
require(manifest["sourceCommit"][:12] == release_id[-12:], "Artifact source mismatch")
|
|
verify_shared(manifest)
|
|
for name in ["node_modules", "public", ".env.local", "tmp", ".model-config-backups"]:
|
|
require(not (destination / name).exists(), f"Unexpected bundled shared path: {name}")
|
|
if (APP / name).exists():
|
|
(destination / name).symlink_to(APP / name)
|
|
# Keep already open browser tabs' content-hashed assets available.
|
|
for path in (current / dist / "static").rglob("*"):
|
|
if not path.is_file():
|
|
continue
|
|
target = destination / ".next/static" / path.relative_to(current / dist / "static")
|
|
if target.exists():
|
|
require(sha(target) == sha(path), "Static asset name collision")
|
|
else:
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
os.link(path, target)
|
|
json_write(destination / "staged.json", {"artifactSha256": digest,
|
|
"previousCwd": str(current), "previousDistDir": dist})
|
|
# The verified unpacked release is retained; the transport copy is redundant.
|
|
archive.unlink()
|
|
return {"staged": release_id, "buildId": manifest["buildId"],
|
|
"freeGiB": round(shutil.disk_usage(APP).free / 1024**3, 2)}
|
|
|
|
|
|
def ecosystem(app, cwd, dist):
|
|
old = app["pm2_env"]
|
|
env = {k: v for k, v in old.items() if k.isupper() and isinstance(v, (str, int, bool))}
|
|
env.update({"NODE_ENV": "production", "PORT": "3125", "PWD": str(cwd), "NEXT_DIST_DIR": dist})
|
|
return {"apps": [{"name": NAME, "cwd": str(cwd), "script": old["pm_exec_path"],
|
|
"args": old["args"], "interpreter": old.get("exec_interpreter", "/usr/bin/node"),
|
|
"exec_mode": "fork", "instances": 1, "watch": False, "env": env,
|
|
"autorestart": old.get("autorestart", True),
|
|
"min_uptime": old.get("min_uptime", "30s"), "max_restarts": old.get("max_restarts", 10),
|
|
"restart_delay": old.get("restart_delay", 3000), "kill_timeout": old.get("kill_timeout", 10000),
|
|
"max_memory_restart": old.get("max_memory_restart", "3G"),
|
|
"error_file": old["pm_err_log_path"], "out_file": old["pm_out_log_path"],
|
|
"merge_logs": True, "log_date_format": "YYYY-MM-DD HH:mm:ss Z"}]}
|
|
|
|
|
|
def healthy(cwd, build_id):
|
|
for _ in range(30):
|
|
try:
|
|
app, active_cwd, _ = app_state()
|
|
require(active_cwd == cwd and app["pm2_env"]["status"] == "online", "Wrong process")
|
|
request = urllib.request.Request("http://127.0.0.1:3125/", headers={"Host": "ai.yydd.ai"})
|
|
with urllib.request.urlopen(request, timeout=5) as response:
|
|
body = response.read().decode("utf8")
|
|
if response.status == 200 and build_id in body:
|
|
return True
|
|
except Exception:
|
|
pass
|
|
time.sleep(2)
|
|
return False
|
|
|
|
|
|
def restart(config):
|
|
run(["pm2", "startOrRestart", str(config), "--only", NAME, "--update-env"], timeout=150)
|
|
|
|
|
|
def activate(release_id):
|
|
destination = release_path(release_id)
|
|
manifest = json.loads((destination / "release.json").read_text())
|
|
staged = json.loads((destination / "staged.json").read_text())
|
|
app, current, dist = app_state()
|
|
require(str(current) == staged["previousCwd"] and dist == staged["previousDistDir"],
|
|
"Production changed after staging")
|
|
require(not (destination / "activation.json").exists(), "Already activated; inspect status")
|
|
verify_shared(manifest)
|
|
jobs = job_status()
|
|
require(not any(row["recent"] for rows in jobs.values() for row in rows),
|
|
"Recent active generation jobs: defer activation")
|
|
before = ecosystem(app, current, dist)
|
|
previous_build = (current / dist / "BUILD_ID").read_text().strip()
|
|
json_write(destination / "previous.pm2.json", before)
|
|
json_write(destination / "candidate.pm2.json", ecosystem(app, destination, ".next"))
|
|
# Backup contains environment metadata and stays root-readable only.
|
|
shutil.copy2(Path.home() / ".pm2/dump.pm2", destination / "pm2-dump.before.json")
|
|
(destination / "pm2-dump.before.json").chmod(0o600)
|
|
receipt = {"sourceCommit": manifest["sourceCommit"], "buildId": manifest["buildId"],
|
|
"previousBuildId": previous_build, "previousCwd": str(current),
|
|
"activatedAtUtc": datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
|
"historicalTaskStates": jobs, "artifactSha256": staged["artifactSha256"],
|
|
"phase": "activating"}
|
|
json_write(destination / "activation.json", receipt)
|
|
try:
|
|
restart(destination / "candidate.pm2.json")
|
|
require(healthy(destination, manifest["buildId"]), "Candidate health check failed")
|
|
run(["env", "SMOKE_BASE_URL=http://127.0.0.1:3125", "node", str(destination / "scripts/smoke-routes.mjs")], cwd=destination,
|
|
timeout=100)
|
|
run(["pm2", "save"])
|
|
except BaseException:
|
|
restart(destination / "previous.pm2.json")
|
|
require(healthy(current, previous_build), "Rollback health check failed; operator required")
|
|
run(["pm2", "save"])
|
|
receipt["phase"] = "rolled-back"
|
|
json_write(destination / "activation.json", receipt)
|
|
raise
|
|
receipt["phase"] = "active"
|
|
json_write(destination / "activation.json", receipt)
|
|
return receipt
|
|
|
|
|
|
def rollback(release_id):
|
|
destination = release_path(release_id)
|
|
receipt = json.loads((destination / "activation.json").read_text())
|
|
_, current, _ = app_state()
|
|
require(current == destination, "Rollback target is not currently active")
|
|
jobs = job_status()
|
|
require(not any(row["recent"] for rows in jobs.values() for row in rows),
|
|
"Recent active generation jobs: defer rollback")
|
|
restart(destination / "previous.pm2.json")
|
|
require(healthy(Path(receipt["previousCwd"]), receipt["previousBuildId"]),
|
|
"Rollback health check failed")
|
|
run(["pm2", "save"])
|
|
receipt["phase"] = "rolled-back"
|
|
json_write(destination / "activation.json", receipt)
|
|
return status()
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("action", choices=["status", "build-env", "jobs", "smoke", "pack", "stage", "activate", "rollback"])
|
|
parser.add_argument("--release")
|
|
parser.add_argument("--digest")
|
|
parser.add_argument("--workspace", type=Path)
|
|
args = parser.parse_args()
|
|
os.umask(0o077)
|
|
if args.action in {"pack", "smoke"}:
|
|
require(socket.gethostname() == "dao02-Default-string", "Not the verified build host")
|
|
require(args.workspace.parent == Path("/opt/ai-chatbot-builds"), "Invalid build workspace")
|
|
result = build_manifest(args.workspace) if args.action == "pack" else smoke(args.workspace)
|
|
elif args.action in {"status", "build-env", "jobs"}:
|
|
result = {"status": status, "build-env": build_environment, "jobs": job_status}[args.action]()
|
|
else:
|
|
import fcntl
|
|
require(RELEASES.is_dir() and not RELEASES.is_symlink(), "Invalid release root")
|
|
with open(RELEASES / ".deploy.lock", "a") as lock:
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
if args.action == "stage":
|
|
result = stage(args.release, args.digest)
|
|
elif args.action == "activate":
|
|
result = activate(args.release)
|
|
else:
|
|
result = rollback(args.release)
|
|
print(json.dumps(result))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|