894 lines
33 KiB
Python
Executable File
894 lines
33 KiB
Python
Executable File
#!/usr/bin/env python3
|
||
import argparse
|
||
import datetime as dt
|
||
import hashlib
|
||
import json
|
||
import os
|
||
import re
|
||
import subprocess
|
||
import time
|
||
import urllib.error
|
||
import urllib.parse
|
||
import urllib.request
|
||
from pathlib import Path
|
||
from typing import Dict, List, Optional, Tuple
|
||
|
||
DEFAULT_APP_TOKEN = "GC2Mb4TXwad8ZusStBecLNg8nCe"
|
||
DEFAULT_TABLE_LINK = "https://acnuoqgha8fk.feishu.cn/base/GC2Mb4TXwad8ZusStBecLNg8nCe"
|
||
OPENCLAW_CONFIG = "/root/.openclaw/openclaw.json"
|
||
SESSIONS_DIR = Path("/root/.openclaw/agents/main/sessions")
|
||
REPORT_DIR = Path("/var/log/openclaw/log-observer")
|
||
|
||
REQUIRED_FIELDS = [
|
||
"记录类型",
|
||
"批次ID",
|
||
"巡检时间",
|
||
"日志范围(分钟)",
|
||
"日志总行数",
|
||
"OpenClaw调用次数",
|
||
"成功回复次数",
|
||
"未完成会话数",
|
||
"错误总数",
|
||
"网络错误数",
|
||
"解析错误数",
|
||
"重复回调数",
|
||
"问题类型",
|
||
"会话ID",
|
||
"问题摘要",
|
||
"建议动作",
|
||
"自动回放状态",
|
||
"自动回放会话",
|
||
"自动回放摘要",
|
||
"状态",
|
||
]
|
||
|
||
ISSUE_PATTERNS = {
|
||
"openclaw_call_failed": re.compile(r"OpenClaw 调用失败|openclaw call timeout|OpenClaw 调用超时", re.I),
|
||
"parse_failed": re.compile(r"结果解析失败|parse openclaw output failed", re.I),
|
||
"network_error": re.compile(r"network exception|keepalive ping timeout|timed out during opening handshake", re.I),
|
||
"reply_failed": re.compile(r"reply failed: session=|failed to send final reply", re.I),
|
||
"traceback": re.compile(r"Traceback \(most recent call last\)", re.I),
|
||
}
|
||
|
||
DIALOG_PATTERNS = {
|
||
"dialog_tool_noise": re.compile(
|
||
r"NO TOOLBeenCalled|without a preceding text-to-speech|tts\s*Completed|结果解析失败|原始输出:",
|
||
re.I,
|
||
),
|
||
"dialog_blocked_response": re.compile(
|
||
r"权限不足|未登录|无法直接从页面抓取|需要登录后的操作|工具版本不支持点击交互|device identity required|网关令牌缺失|需要配对",
|
||
re.I,
|
||
),
|
||
"dialog_meta_leak": re.compile(r"\[plugins\]|Registered\s+\w+\s+tool|runId\"\s*:\s*\"", re.I),
|
||
}
|
||
|
||
CALL_RE = re.compile(r"openclaw call: session=([^\s]+)")
|
||
REPLY_RE = re.compile(r"reply sent: session=([^\s]+)")
|
||
DUP_RE = re.compile(r"duplicate callback ignored", re.I)
|
||
TS_RE = re.compile(r"^(\d{4}-\d{2}-\d{2}T[^\s]+)\s+")
|
||
SESSION_RE = re.compile(r"session=([^\s,]+)")
|
||
|
||
|
||
def _run(cmd: List[str], timeout: int = 120) -> subprocess.CompletedProcess:
|
||
return subprocess.run(cmd, capture_output=True, text=True, timeout=timeout)
|
||
|
||
|
||
def _http_json(method: str, url: str, headers: Dict[str, str], payload: Dict = None) -> Dict:
|
||
data = None
|
||
req_headers = dict(headers)
|
||
if payload is not None:
|
||
data = json.dumps(payload, ensure_ascii=False).encode("utf-8")
|
||
req_headers["Content-Type"] = "application/json; charset=utf-8"
|
||
|
||
req = urllib.request.Request(url, data=data, headers=req_headers, method=method)
|
||
try:
|
||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||
raw = resp.read().decode("utf-8")
|
||
return json.loads(raw) if raw else {}
|
||
except urllib.error.HTTPError as e:
|
||
body = e.read().decode("utf-8", errors="ignore")
|
||
raise RuntimeError(f"HTTP {e.code} {url}: {body[:1000]}") from e
|
||
except Exception as e:
|
||
raise RuntimeError(f"request failed {method} {url}: {e}") from e
|
||
|
||
|
||
def _load_feishu_app() -> Tuple[str, str]:
|
||
app_id = os.getenv("FEISHU_APP_ID", "").strip()
|
||
app_secret = os.getenv("FEISHU_APP_SECRET", "").strip()
|
||
if app_id and app_secret:
|
||
return app_id, app_secret
|
||
|
||
cfg_path = Path(OPENCLAW_CONFIG)
|
||
if not cfg_path.exists():
|
||
raise RuntimeError("missing feishu credentials (env and openclaw.json both unavailable)")
|
||
|
||
cfg = json.loads(cfg_path.read_text(encoding="utf-8"))
|
||
ch = (cfg.get("channels") or {}).get("feishu") or {}
|
||
app_id = (ch.get("appId") or "").strip()
|
||
app_secret = (ch.get("appSecret") or "").strip()
|
||
if not app_id or not app_secret:
|
||
raise RuntimeError("feishu appId/appSecret is empty in openclaw.json")
|
||
return app_id, app_secret
|
||
|
||
|
||
def _tenant_token(app_id: str, app_secret: str) -> str:
|
||
url = "https://open.feishu.cn/open-apis/auth/v3/tenant_access_token/internal"
|
||
res = _http_json("POST", url, headers={}, payload={"app_id": app_id, "app_secret": app_secret})
|
||
if res.get("code") != 0:
|
||
raise RuntimeError(f"feishu auth failed: {res}")
|
||
token = res.get("tenant_access_token")
|
||
if not token:
|
||
raise RuntimeError("feishu auth succeeded but tenant_access_token missing")
|
||
return token
|
||
|
||
|
||
def _create_table(token: str, app_token: str, table_name: str) -> str:
|
||
url = f"https://open.feishu.cn/open-apis/bitable/v1/apps/{app_token}/tables"
|
||
res = _http_json(
|
||
"POST",
|
||
url,
|
||
headers={"Authorization": f"Bearer {token}"},
|
||
payload={"table": {"name": table_name[:100]}},
|
||
)
|
||
if res.get("code") != 0:
|
||
raise RuntimeError(f"create table failed ({table_name}): {res}")
|
||
|
||
data = res.get("data") or {}
|
||
table_id = data.get("table_id") or (data.get("table") or {}).get("table_id") or ""
|
||
if not table_id:
|
||
raise RuntimeError(f"create table succeeded but table_id missing: {res}")
|
||
return table_id
|
||
|
||
|
||
def _list_fields(token: str, app_token: str, table_id: str) -> Dict[str, str]:
|
||
url = f"https://open.feishu.cn/open-apis/bitable/v1/apps/{app_token}/tables/{table_id}/fields?page_size=500"
|
||
res = _http_json("GET", url, headers={"Authorization": f"Bearer {token}"})
|
||
if res.get("code") != 0:
|
||
raise RuntimeError(f"list fields failed: {res}")
|
||
out: Dict[str, str] = {}
|
||
for it in ((res.get("data") or {}).get("items") or []):
|
||
name = it.get("field_name")
|
||
fid = it.get("field_id")
|
||
if name and fid:
|
||
out[name] = fid
|
||
return out
|
||
|
||
|
||
def _create_field(token: str, app_token: str, table_id: str, field_name: str) -> None:
|
||
url = f"https://open.feishu.cn/open-apis/bitable/v1/apps/{app_token}/tables/{table_id}/fields"
|
||
res = _http_json(
|
||
"POST",
|
||
url,
|
||
headers={"Authorization": f"Bearer {token}"},
|
||
payload={"field_name": field_name, "type": 1},
|
||
)
|
||
if res.get("code") != 0:
|
||
raise RuntimeError(f"create field failed ({field_name}): {res}")
|
||
|
||
|
||
def _ensure_fields(token: str, app_token: str, table_id: str) -> None:
|
||
fields = _list_fields(token, app_token, table_id)
|
||
for name in REQUIRED_FIELDS:
|
||
if name not in fields:
|
||
_create_field(token, app_token, table_id, name)
|
||
time.sleep(0.3)
|
||
|
||
|
||
def _create_record(token: str, app_token: str, table_id: str, fields: Dict[str, str]) -> None:
|
||
url = f"https://open.feishu.cn/open-apis/bitable/v1/apps/{app_token}/tables/{table_id}/records"
|
||
res = _http_json(
|
||
"POST",
|
||
url,
|
||
headers={"Authorization": f"Bearer {token}"},
|
||
payload={"fields": fields},
|
||
)
|
||
if res.get("code") != 0:
|
||
raise RuntimeError(f"create record failed: {res}")
|
||
|
||
|
||
def _build_table_link(base_link: str, app_token: str, table_id: str) -> str:
|
||
if not base_link:
|
||
return f"https://acnuoqgha8fk.feishu.cn/base/{app_token}?table={table_id}"
|
||
parsed = urllib.parse.urlsplit(base_link)
|
||
query_pairs = urllib.parse.parse_qsl(parsed.query, keep_blank_values=True)
|
||
query_pairs = [(k, v) for (k, v) in query_pairs if k.lower() not in {"table", "view"}]
|
||
query_pairs.append(("table", table_id))
|
||
query = urllib.parse.urlencode(query_pairs)
|
||
return urllib.parse.urlunsplit((parsed.scheme, parsed.netloc, parsed.path, query, parsed.fragment))
|
||
|
||
|
||
def _extract_text_from_openclaw(stdout: str) -> str:
|
||
raw = (stdout or "").strip()
|
||
if not raw:
|
||
return ""
|
||
|
||
decoder = json.JSONDecoder()
|
||
data = None
|
||
idx = raw.find("{")
|
||
while idx != -1:
|
||
try:
|
||
data, _ = decoder.raw_decode(raw[idx:])
|
||
break
|
||
except Exception:
|
||
idx = raw.find("{", idx + 1)
|
||
|
||
if not isinstance(data, dict):
|
||
return ""
|
||
payloads = ((data.get("result") or {}).get("payloads") or [])
|
||
text_items = [p.get("text", "") for p in payloads if isinstance(p, dict)]
|
||
return "\n".join(t for t in text_items if t).strip()
|
||
|
||
|
||
def _collect_logs(service: str, since_minutes: int) -> List[str]:
|
||
cmd = [
|
||
"journalctl",
|
||
"-u",
|
||
service,
|
||
"--since",
|
||
f"{since_minutes} min ago",
|
||
"--no-pager",
|
||
"-o",
|
||
"short-iso",
|
||
]
|
||
proc = _run(cmd)
|
||
if proc.returncode != 0:
|
||
err = (proc.stderr or proc.stdout or "journalctl failed").strip()
|
||
raise RuntimeError(f"collect logs failed: {err}")
|
||
return [line.rstrip("\n") for line in (proc.stdout or "").splitlines() if line.strip()]
|
||
|
||
|
||
def _extract_session(line: str) -> str:
|
||
for regex in (CALL_RE, REPLY_RE, SESSION_RE):
|
||
m = regex.search(line)
|
||
if m:
|
||
return m.group(1)
|
||
return ""
|
||
|
||
|
||
def _analyze_logs(lines: List[str]) -> Dict:
|
||
call_count = 0
|
||
reply_count = 0
|
||
duplicate_count = 0
|
||
issue_counts = {k: 0 for k in ISSUE_PATTERNS}
|
||
calls_by_session: Dict[str, int] = {}
|
||
replies_by_session: Dict[str, int] = {}
|
||
issues: List[Dict] = []
|
||
|
||
for line in lines:
|
||
m = CALL_RE.search(line)
|
||
if m:
|
||
call_count += 1
|
||
session = m.group(1)
|
||
calls_by_session[session] = calls_by_session.get(session, 0) + 1
|
||
|
||
m = REPLY_RE.search(line)
|
||
if m:
|
||
reply_count += 1
|
||
session = m.group(1)
|
||
replies_by_session[session] = replies_by_session.get(session, 0) + 1
|
||
|
||
if DUP_RE.search(line):
|
||
duplicate_count += 1
|
||
|
||
for issue_type, patt in ISSUE_PATTERNS.items():
|
||
if patt.search(line):
|
||
issue_counts[issue_type] += 1
|
||
ts_match = TS_RE.match(line)
|
||
issues.append(
|
||
{
|
||
"type": issue_type,
|
||
"timestamp": ts_match.group(1) if ts_match else "",
|
||
"line": line[-600:],
|
||
"session": _extract_session(line),
|
||
}
|
||
)
|
||
break
|
||
|
||
unresolved_sessions = []
|
||
for session, cnt in calls_by_session.items():
|
||
rc = replies_by_session.get(session, 0)
|
||
if cnt > rc:
|
||
unresolved_sessions.append({"session": session, "pending_calls": cnt - rc})
|
||
|
||
unresolved_sessions = sorted(unresolved_sessions, key=lambda x: (-x["pending_calls"], x["session"]))
|
||
|
||
return {
|
||
"line_count": len(lines),
|
||
"call_count": call_count,
|
||
"reply_count": reply_count,
|
||
"duplicate_count": duplicate_count,
|
||
"issue_counts": issue_counts,
|
||
"issues": issues,
|
||
"unresolved_sessions": unresolved_sessions,
|
||
}
|
||
|
||
|
||
def _collect_session_ids(lines: List[str]) -> List[str]:
|
||
seen = set()
|
||
out = []
|
||
for line in lines:
|
||
sid = _extract_session(line)
|
||
if sid and sid not in seen:
|
||
seen.add(sid)
|
||
out.append(sid)
|
||
return out
|
||
|
||
|
||
def _parse_iso_utc(value: str) -> Optional[dt.datetime]:
|
||
if not value:
|
||
return None
|
||
try:
|
||
if value.endswith("Z"):
|
||
value = value[:-1] + "+00:00"
|
||
parsed = dt.datetime.fromisoformat(value)
|
||
if parsed.tzinfo is None:
|
||
parsed = parsed.replace(tzinfo=dt.timezone.utc)
|
||
return parsed.astimezone(dt.timezone.utc)
|
||
except Exception:
|
||
return None
|
||
|
||
|
||
def _extract_content_text(content) -> str:
|
||
if isinstance(content, str):
|
||
return content.strip()
|
||
if isinstance(content, list):
|
||
parts = []
|
||
for item in content:
|
||
if not isinstance(item, dict):
|
||
continue
|
||
if item.get("type") != "text":
|
||
continue
|
||
text = item.get("text")
|
||
if text:
|
||
parts.append(str(text))
|
||
return "\n".join(parts).strip()
|
||
return ""
|
||
|
||
|
||
def _normalize_text(text: str) -> str:
|
||
return re.sub(r"[\s，,。.!！?？:：~～`'\"\-_/\\]+", "", text or "").lower()
|
||
|
||
|
||
def _dialog_issue(
|
||
issue_type: str,
|
||
session_id: str,
|
||
timestamp: str,
|
||
user_text: str,
|
||
assistant_text: str,
|
||
) -> Dict:
|
||
user_short = (user_text or "").replace("\n", " ")[:120]
|
||
assistant_short = (assistant_text or "").replace("\n", " ")[:320]
|
||
line = f"user={user_short} | assistant={assistant_short}"
|
||
return {
|
||
"type": issue_type,
|
||
"timestamp": timestamp,
|
||
"line": line,
|
||
"session": session_id,
|
||
}
|
||
|
||
|
||
def _analyze_dialog_anomalies(lines: List[str], since_minutes: int) -> Dict:
|
||
session_ids = _collect_session_ids(lines)
|
||
cutoff = dt.datetime.now(dt.timezone.utc) - dt.timedelta(minutes=since_minutes)
|
||
|
||
issue_counts = {k: 0 for k in DIALOG_PATTERNS}
|
||
issue_counts["dialog_repeat_reply"] = 0
|
||
issue_counts["dialog_large_code_dump"] = 0
|
||
issues: List[Dict] = []
|
||
inspected = 0
|
||
|
||
for session_id in session_ids:
|
||
session_file = SESSIONS_DIR / f"{session_id}.jsonl"
|
||
if not session_file.exists() or not session_file.is_file():
|
||
continue
|
||
|
||
inspected += 1
|
||
last_user_text = ""
|
||
last_assistant_norm = ""
|
||
|
||
try:
|
||
with session_file.open("r", encoding="utf-8") as fh:
|
||
for raw in fh:
|
||
raw = raw.strip()
|
||
if not raw:
|
||
continue
|
||
try:
|
||
obj = json.loads(raw)
|
||
except Exception:
|
||
continue
|
||
|
||
if obj.get("type") != "message":
|
||
continue
|
||
|
||
msg = obj.get("message") or {}
|
||
role = (msg.get("role") or "").lower()
|
||
content_text = _extract_content_text(msg.get("content"))
|
||
if not content_text:
|
||
continue
|
||
|
||
ts_raw = obj.get("timestamp") or ""
|
||
ts_dt = _parse_iso_utc(ts_raw)
|
||
if ts_dt and ts_dt < cutoff:
|
||
continue
|
||
ts_text = ts_raw or ""
|
||
|
||
if role == "user":
|
||
last_user_text = content_text
|
||
continue
|
||
if role != "assistant":
|
||
continue
|
||
|
||
normalized = _normalize_text(content_text)
|
||
for issue_type, patt in DIALOG_PATTERNS.items():
|
||
if patt.search(content_text):
|
||
issue_counts[issue_type] += 1
|
||
issues.append(_dialog_issue(issue_type, session_id, ts_text, last_user_text, content_text))
|
||
break
|
||
|
||
if normalized and last_assistant_norm and normalized == last_assistant_norm and len(normalized) >= 24:
|
||
issue_counts["dialog_repeat_reply"] += 1
|
||
issues.append(_dialog_issue("dialog_repeat_reply", session_id, ts_text, last_user_text, content_text))
|
||
last_assistant_norm = normalized
|
||
|
||
if "```" in content_text:
|
||
code_line_count = sum(1 for ln in content_text.splitlines() if ln.strip())
|
||
if code_line_count >= 24:
|
||
issue_counts["dialog_large_code_dump"] += 1
|
||
issues.append(_dialog_issue("dialog_large_code_dump", session_id, ts_text, last_user_text, content_text))
|
||
except Exception:
|
||
issues.append(
|
||
{
|
||
"type": "dialog_session_parse_failed",
|
||
"timestamp": dt.datetime.now().isoformat(),
|
||
"line": f"session file parse failed: {session_file}",
|
||
"session": session_id,
|
||
}
|
||
)
|
||
|
||
return {
|
||
"dialog_issue_counts": issue_counts,
|
||
"dialog_issues": issues,
|
||
"dialog_inspected_sessions": inspected,
|
||
}
|
||
|
||
|
||
def _last_user_prompt(session_id: str) -> str:
|
||
p = SESSIONS_DIR / f"{session_id}.jsonl"
|
||
if not p.exists() or p.stat().st_size == 0:
|
||
return ""
|
||
|
||
prompt = ""
|
||
try:
|
||
with p.open("r", encoding="utf-8") as fh:
|
||
for raw in fh:
|
||
raw = raw.strip()
|
||
if not raw:
|
||
continue
|
||
try:
|
||
obj = json.loads(raw)
|
||
except Exception:
|
||
continue
|
||
if obj.get("type") != "message":
|
||
continue
|
||
message = obj.get("message") or {}
|
||
role = (message.get("role") or "").lower()
|
||
if role != "user":
|
||
continue
|
||
content = _extract_content_text(message.get("content"))
|
||
if content:
|
||
prompt = content
|
||
except Exception:
|
||
return ""
|
||
return prompt[:1200]
|
||
|
||
|
||
def _is_safe_readonly_prompt(prompt: str) -> bool:
|
||
if not prompt:
|
||
return False
|
||
lowered = prompt.lower()
|
||
dangerous = ["删除", "drop ", "truncate", "update ", "insert ", "修改", "推送", "发送", "写入", "支付", "转账"]
|
||
if any(k in lowered for k in dangerous):
|
||
return False
|
||
read_only = ["查询", "分析", "检查", "统计", "汇总", "搜索", "报表", "售后", "库存", "sku", "店铺"]
|
||
return any(k in lowered for k in read_only)
|
||
|
||
|
||
def _openclaw_replay(prompt: str, timeout: int = 240) -> Tuple[bool, str, str]:
|
||
replay_session = f"observer-replay-{int(time.time())}-{hashlib.sha1(prompt.encode('utf-8')).hexdigest()[:6]}"
|
||
cmd = [
|
||
"/usr/bin/openclaw",
|
||
"agent",
|
||
"--session-id",
|
||
replay_session,
|
||
"--message",
|
||
prompt,
|
||
"--json",
|
||
"--timeout",
|
||
str(timeout),
|
||
]
|
||
proc = _run(cmd, timeout=timeout + 20)
|
||
if proc.returncode != 0:
|
||
err = (proc.stderr or proc.stdout or "openclaw replay failed").strip()
|
||
return False, replay_session, err[:300]
|
||
|
||
text = _extract_text_from_openclaw(proc.stdout or "")
|
||
if not text:
|
||
text = (proc.stdout or "")[:300]
|
||
ok = "调用失败" not in text and "超时" not in text and "解析失败" not in text
|
||
summary = text.replace("\n", " ")[:260]
|
||
return ok, replay_session, summary
|
||
|
||
|
||
def _total_issue_count(analysis: Dict) -> int:
|
||
base = sum((analysis.get("issue_counts") or {}).values())
|
||
dialog = sum((analysis.get("dialog_issue_counts") or {}).values())
|
||
return base + dialog
|
||
|
||
|
||
def _openclaw_observer_comment(analysis: Dict, top_lines: List[str], timeout: int = 180) -> str:
|
||
payload = {
|
||
"summary": {
|
||
"line_count": analysis["line_count"],
|
||
"call_count": analysis["call_count"],
|
||
"reply_count": analysis["reply_count"],
|
||
"duplicate_count": analysis["duplicate_count"],
|
||
"issue_counts": analysis["issue_counts"],
|
||
"dialog_issue_counts": analysis.get("dialog_issue_counts", {}),
|
||
"dialog_inspected_sessions": analysis.get("dialog_inspected_sessions", 0),
|
||
"unresolved_sessions": analysis["unresolved_sessions"][:10],
|
||
},
|
||
"samples": top_lines[:20],
|
||
}
|
||
|
||
prompt = (
|
||
"你是日志巡检分身。根据以下 JSON 生成简短巡检结论，输出三段："
|
||
"\n1) 主要问题（最多3条）"
|
||
"\n2) 对主机器人的修复建议（最多5条，必须可执行）"
|
||
"\n3) 优先级最高的一条修复动作"
|
||
"\n请重点覆盖对话返回异常（如无关输出、工具噪音、重复回复）。"
|
||
"\n请用中文、短句、不要废话。\n\n"
|
||
+ json.dumps(payload, ensure_ascii=False)
|
||
)
|
||
|
||
cmd = [
|
||
"/usr/bin/openclaw",
|
||
"agent",
|
||
"--session-id",
|
||
"log-observer-main",
|
||
"--message",
|
||
prompt,
|
||
"--json",
|
||
"--timeout",
|
||
str(timeout),
|
||
]
|
||
proc = _run(cmd, timeout=timeout + 20)
|
||
if proc.returncode != 0:
|
||
err = (proc.stderr or proc.stdout or "openclaw observer failed").strip()
|
||
return f"分身分析失败: {err[:300]}"
|
||
text = _extract_text_from_openclaw(proc.stdout or "")
|
||
return text or "分身分析完成，但未返回文本。"
|
||
|
||
|
||
def _recommendation(issue_type: str) -> str:
|
||
mapping = {
|
||
"network_error": "检查网络抖动与 websocket keepalive，建议提高重连退避并增加心跳监控。",
|
||
"parse_failed": "统一 OpenClaw 输出解析入口，先剥离插件日志再解码 JSON。",
|
||
"openclaw_call_failed": "检查模型/API可用性与超时阈值，必要时降级到本地简答模式。",
|
||
"reply_failed": "检查 ActionCard 权限，失败时强制文本兜底避免无回复。",
|
||
"traceback": "排查异常栈并加防护，避免同类异常重复触发。",
|
||
"dialog_tool_noise": "在回复前增加输出清洗，拦截 tool 元信息/tts 噪音文本。",
|
||
"dialog_blocked_response": "对登录/权限失败加自动重试和明确下一步，不直接抛给用户。",
|
||
"dialog_meta_leak": "过滤插件注册/运行元数据，用户侧只保留结论。",
|
||
"dialog_repeat_reply": "对同会话相同答复加内容去重，避免重复发送。",
|
||
"dialog_large_code_dump": "非代码任务启用摘要模式，限制长代码块直接外发。",
|
||
"dialog_session_parse_failed": "检查 session 文件格式和异常记录，保证可解析。",
|
||
}
|
||
return mapping.get(issue_type, "检查对应日志并补充防护与重试策略。")
|
||
|
||
|
||
def _write_report(batch_id: str, since_minutes: int, analysis: Dict, observer_text: str, replay_rows: List[Dict], lines: List[str]) -> Path:
|
||
REPORT_DIR.mkdir(parents=True, exist_ok=True)
|
||
report_path = REPORT_DIR / f"{batch_id}.md"
|
||
|
||
issues_top = analysis["issues"][:16]
|
||
unresolved_top = analysis["unresolved_sessions"][:12]
|
||
dialog_top = (analysis.get("dialog_issues") or [])[:16]
|
||
|
||
out = []
|
||
out.append(f"# 日志巡检报告 {batch_id}")
|
||
out.append("")
|
||
out.append(f"- 巡检时间: {dt.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
|
||
out.append(f"- 日志范围: 最近 {since_minutes} 分钟")
|
||
out.append(f"- 日志总行数: {analysis['line_count']}")
|
||
out.append(f"- OpenClaw调用/成功回复: {analysis['call_count']} / {analysis['reply_count']}")
|
||
out.append(f"- 未完成会话: {len(analysis['unresolved_sessions'])}")
|
||
out.append(f"- 错误总数: {_total_issue_count(analysis)}")
|
||
out.append(f"- 对话巡检会话数: {analysis.get('dialog_inspected_sessions', 0)}")
|
||
out.append(f"- 对话内容异常数: {sum((analysis.get('dialog_issue_counts') or {}).values())}")
|
||
out.append("")
|
||
out.append("## 分身结论")
|
||
out.append("")
|
||
out.append(observer_text.strip() or "无")
|
||
out.append("")
|
||
|
||
out.append("## 问题明细")
|
||
out.append("")
|
||
if not issues_top:
|
||
out.append("- 无错误问题")
|
||
else:
|
||
for idx, row in enumerate(issues_top, start=1):
|
||
out.append(f"{idx}. [{row.get('type')}] session={row.get('session') or '-'}")
|
||
out.append(f" - 时间: {row.get('timestamp') or '-'}")
|
||
out.append(f" - 日志: {row.get('line')}")
|
||
out.append(f" - 建议: {_recommendation(row.get('type') or '')}")
|
||
|
||
out.append("")
|
||
out.append("## 对话内容异常")
|
||
out.append("")
|
||
if not dialog_top:
|
||
out.append("- 无")
|
||
else:
|
||
for idx, row in enumerate(dialog_top, start=1):
|
||
out.append(f"{idx}. [{row.get('type')}] session={row.get('session') or '-'}")
|
||
out.append(f" - 时间: {row.get('timestamp') or '-'}")
|
||
out.append(f" - 内容: {row.get('line')}")
|
||
out.append(f" - 建议: {_recommendation(row.get('type') or '')}")
|
||
|
||
out.append("")
|
||
out.append("## 未完成会话")
|
||
out.append("")
|
||
if not unresolved_top:
|
||
out.append("- 无")
|
||
else:
|
||
for row in unresolved_top:
|
||
out.append(f"- {row['session']}: pending_calls={row['pending_calls']}")
|
||
|
||
out.append("")
|
||
out.append("## 自动回放")
|
||
out.append("")
|
||
if not replay_rows:
|
||
out.append("- 未执行")
|
||
else:
|
||
for row in replay_rows:
|
||
out.append(f"- {row['session']} -> {row['status']} ({row['replay_session']}) {row['summary']}")
|
||
|
||
out.append("")
|
||
out.append("## 日志样本")
|
||
out.append("")
|
||
for line in lines[-40:]:
|
||
out.append(f"- {line[-800:]}")
|
||
|
||
report_path.write_text("\n".join(out) + "\n", encoding="utf-8")
|
||
latest = REPORT_DIR / "latest.md"
|
||
latest.write_text(report_path.read_text(encoding="utf-8"), encoding="utf-8")
|
||
return report_path
|
||
|
||
|
||
def _push_to_feishu(
|
||
batch_id: str,
|
||
since_minutes: int,
|
||
analysis: Dict,
|
||
observer_text: str,
|
||
replay_rows: List[Dict],
|
||
app_token: str,
|
||
base_link: str,
|
||
shared_table: bool,
|
||
) -> Tuple[str, str]:
|
||
app_id, app_secret = _load_feishu_app()
|
||
token = _tenant_token(app_id, app_secret)
|
||
|
||
table_name = f"日志巡检-{dt.datetime.now().strftime('%m%d-%H%M')}"
|
||
table_id = ""
|
||
if shared_table:
|
||
parsed = urllib.parse.urlsplit(base_link or "")
|
||
query = dict(urllib.parse.parse_qsl(parsed.query, keep_blank_values=True))
|
||
table_id = query.get("table", "").strip()
|
||
if not table_id:
|
||
raise RuntimeError("shared-table 模式下未提供 table_id")
|
||
else:
|
||
table_id = _create_table(token, app_token, table_name)
|
||
|
||
_ensure_fields(token, app_token, table_id)
|
||
now_text = dt.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||
|
||
_create_record(
|
||
token,
|
||
app_token,
|
||
table_id,
|
||
{
|
||
"记录类型": "summary",
|
||
"批次ID": batch_id,
|
||
"巡检时间": now_text,
|
||
"日志范围(分钟)": str(since_minutes),
|
||
"日志总行数": str(analysis["line_count"]),
|
||
"OpenClaw调用次数": str(analysis["call_count"]),
|
||
"成功回复次数": str(analysis["reply_count"]),
|
||
"未完成会话数": str(len(analysis["unresolved_sessions"])),
|
||
"错误总数": str(_total_issue_count(analysis)),
|
||
"网络错误数": str(analysis["issue_counts"].get("network_error", 0)),
|
||
"解析错误数": str(analysis["issue_counts"].get("parse_failed", 0)),
|
||
"重复回调数": str(analysis["duplicate_count"]),
|
||
"问题类型": "巡检汇总",
|
||
"会话ID": "-",
|
||
"问题摘要": (
|
||
observer_text[:5000]
|
||
+ "\n\n"
|
||
+ f"对话巡检会话数={analysis.get('dialog_inspected_sessions', 0)}, 对话内容异常数={sum((analysis.get('dialog_issue_counts') or {}).values())}"
|
||
)[:8000],
|
||
"建议动作": "按优先级逐条修复，修复后观察下一周期是否下降。",
|
||
"自动回放状态": "已执行" if replay_rows else "未执行",
|
||
"自动回放会话": str(len(replay_rows)),
|
||
"自动回放摘要": "; ".join([f"{r['session']}:{r['status']}" for r in replay_rows])[:8000],
|
||
"状态": "ok",
|
||
},
|
||
)
|
||
|
||
for row in analysis["issues"][:40]:
|
||
_create_record(
|
||
token,
|
||
app_token,
|
||
table_id,
|
||
{
|
||
"记录类型": "issue",
|
||
"批次ID": batch_id,
|
||
"巡检时间": row.get("timestamp") or now_text,
|
||
"日志范围(分钟)": str(since_minutes),
|
||
"日志总行数": "",
|
||
"OpenClaw调用次数": "",
|
||
"成功回复次数": "",
|
||
"未完成会话数": "",
|
||
"错误总数": "",
|
||
"网络错误数": "",
|
||
"解析错误数": "",
|
||
"重复回调数": "",
|
||
"问题类型": row.get("type") or "",
|
||
"会话ID": row.get("session") or "",
|
||
"问题摘要": row.get("line", "")[:8000],
|
||
"建议动作": _recommendation(row.get("type") or ""),
|
||
"自动回放状态": "",
|
||
"自动回放会话": "",
|
||
"自动回放摘要": "",
|
||
"状态": "pending",
|
||
},
|
||
)
|
||
|
||
for row in replay_rows[:20]:
|
||
_create_record(
|
||
token,
|
||
app_token,
|
||
table_id,
|
||
{
|
||
"记录类型": "replay",
|
||
"批次ID": batch_id,
|
||
"巡检时间": now_text,
|
||
"日志范围(分钟)": str(since_minutes),
|
||
"日志总行数": "",
|
||
"OpenClaw调用次数": "",
|
||
"成功回复次数": "",
|
||
"未完成会话数": "",
|
||
"错误总数": "",
|
||
"网络错误数": "",
|
||
"解析错误数": "",
|
||
"重复回调数": "",
|
||
"问题类型": "auto-replay",
|
||
"会话ID": row.get("session") or "",
|
||
"问题摘要": row.get("prompt", "")[:8000],
|
||
"建议动作": "若回放成功，检查主机器人同类问题是否已恢复。",
|
||
"自动回放状态": row.get("status", ""),
|
||
"自动回放会话": row.get("replay_session", ""),
|
||
"自动回放摘要": row.get("summary", "")[:8000],
|
||
"状态": "ok" if row.get("status") == "success" else "failed",
|
||
},
|
||
)
|
||
|
||
table_link = _build_table_link(base_link, app_token, table_id)
|
||
return table_id, table_link
|
||
|
||
|
||
def main() -> int:
|
||
parser = argparse.ArgumentParser(description="OpenClaw log observer")
|
||
parser.add_argument("--service", default="openclaw-dingtalk-bridge.service")
|
||
parser.add_argument("--since-minutes", type=int, default=60)
|
||
parser.add_argument("--push-feishu", action="store_true")
|
||
parser.add_argument("--shared-table", action="store_true")
|
||
parser.add_argument("--app-token", default=DEFAULT_APP_TOKEN)
|
||
parser.add_argument("--table-link", default=DEFAULT_TABLE_LINK)
|
||
parser.add_argument("--replay-failures", action="store_true")
|
||
parser.add_argument("--replay-limit", type=int, default=2)
|
||
args = parser.parse_args()
|
||
|
||
since_minutes = max(5, min(args.since_minutes, 24 * 60))
|
||
batch_id = dt.datetime.now().strftime("%Y%m%d-%H%M%S")
|
||
|
||
try:
|
||
lines = _collect_logs(args.service, since_minutes)
|
||
analysis = _analyze_logs(lines)
|
||
dialog_result = _analyze_dialog_anomalies(lines, since_minutes)
|
||
analysis.update(dialog_result)
|
||
analysis["issues"].extend(dialog_result.get("dialog_issues", []))
|
||
except Exception as e:
|
||
print("STATUS=error")
|
||
print(f"ERROR={e}")
|
||
return 2
|
||
|
||
issue_lines = [row["line"] for row in analysis["issues"][:20]]
|
||
observer_text = _openclaw_observer_comment(analysis, issue_lines)
|
||
|
||
replay_rows: List[Dict] = []
|
||
if args.replay_failures:
|
||
for item in analysis["unresolved_sessions"][: max(0, args.replay_limit)]:
|
||
session_id = item["session"]
|
||
prompt = _last_user_prompt(session_id)
|
||
if not _is_safe_readonly_prompt(prompt):
|
||
replay_rows.append(
|
||
{
|
||
"session": session_id,
|
||
"prompt": prompt[:200],
|
||
"status": "skipped",
|
||
"replay_session": "",
|
||
"summary": "prompt为空或非只读任务，跳过",
|
||
}
|
||
)
|
||
continue
|
||
ok, replay_session, summary = _openclaw_replay(prompt)
|
||
replay_rows.append(
|
||
{
|
||
"session": session_id,
|
||
"prompt": prompt[:500],
|
||
"status": "success" if ok else "failed",
|
||
"replay_session": replay_session,
|
||
"summary": summary,
|
||
}
|
||
)
|
||
|
||
report_path = _write_report(batch_id, since_minutes, analysis, observer_text, replay_rows, lines)
|
||
|
||
table_id = ""
|
||
table_link = ""
|
||
if args.push_feishu:
|
||
try:
|
||
table_id, table_link = _push_to_feishu(
|
||
batch_id=batch_id,
|
||
since_minutes=since_minutes,
|
||
analysis=analysis,
|
||
observer_text=observer_text,
|
||
replay_rows=replay_rows,
|
||
app_token=args.app_token,
|
||
base_link=args.table_link,
|
||
shared_table=args.shared_table,
|
||
)
|
||
except Exception as e:
|
||
print("STATUS=partial")
|
||
print(f"ERROR=push feishu failed: {e}")
|
||
print(f"BATCH_ID={batch_id}")
|
||
print(f"REPORT_PATH={report_path}")
|
||
print(f"LINE_COUNT={analysis['line_count']}")
|
||
print(f"ISSUE_COUNT={_total_issue_count(analysis)}")
|
||
print(f"UNRESOLVED_SESSIONS={len(analysis['unresolved_sessions'])}")
|
||
return 1
|
||
|
||
print("STATUS=ok")
|
||
print(f"BATCH_ID={batch_id}")
|
||
print(f"REPORT_PATH={report_path}")
|
||
print(f"LINE_COUNT={analysis['line_count']}")
|
||
print(f"CALL_COUNT={analysis['call_count']}")
|
||
print(f"REPLY_COUNT={analysis['reply_count']}")
|
||
print(f"ISSUE_COUNT={_total_issue_count(analysis)}")
|
||
print(f"NETWORK_ERRORS={analysis['issue_counts'].get('network_error', 0)}")
|
||
print(f"PARSE_ERRORS={analysis['issue_counts'].get('parse_failed', 0)}")
|
||
print(f"DUPLICATE_CALLBACKS={analysis['duplicate_count']}")
|
||
print(f"DIALOG_INSPECTED_SESSIONS={analysis.get('dialog_inspected_sessions', 0)}")
|
||
print(f"DIALOG_ANOMALIES={sum((analysis.get('dialog_issue_counts') or {}).values())}")
|
||
print(f"UNRESOLVED_SESSIONS={len(analysis['unresolved_sessions'])}")
|
||
print(f"REPLAY_ROWS={len(replay_rows)}")
|
||
if table_id:
|
||
print(f"TABLE_ID={table_id}")
|
||
if table_link:
|
||
print(f"TABLE_LINK={table_link}")
|
||
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main())
|