fix(realtime): 修复服务号实时同步写入解密库并优化桌面端运行时兼容性

服务号实时同步改为直接读取 live biz_message 库,避免 gh_* 会话走空的 wcdb_get_messages。

将同步结果写回 output/databases 解密库,并补充异常降级路径。

修复桌面端日志处理与 wcdb_api.dll 的运行时定位问题。
This commit is contained in:
2977094657
2026-03-28 13:18:22 +08:00
Unverified
parent f4633e672e
commit 673d1f916a
3 changed files with 424 additions and 230 deletions
+64 -8
View File
@@ -41,6 +41,22 @@ class ColoredFormatter(logging.Formatter):
return formatted
def _can_use_logging_stream(stream) -> bool:
try:
if stream is None or getattr(stream, "closed", False):
return False
except Exception:
return False
try:
stream.write("")
stream.flush()
except Exception:
return False
return True
class WeChatLogger:
"""微信解密工具统一日志管理器"""
@@ -64,6 +80,12 @@ class WeChatLogger:
if env_level:
log_level = env_level
console_logging_env = str(os.environ.get("WECHAT_TOOL_ENABLE_CONSOLE_LOG", "") or "").strip().lower()
console_logging_forced = console_logging_env in {"1", "true", "yes", "on"}
console_logging_disabled = console_logging_env in {"0", "false", "no", "off"}
level = getattr(logging, str(log_level or "INFO").upper(), logging.INFO)
# 创建日志目录
now = datetime.now()
from .app_paths import get_output_dir
@@ -73,10 +95,41 @@ class WeChatLogger:
# 设置日志文件名
date_str = now.strftime("%d")
self.log_file = log_dir / f"{date_str}_wechat_tool.log"
desired_log_file = log_dir / f"{date_str}_wechat_tool.log"
root_logger = logging.getLogger()
wants_console_handler = _can_use_logging_stream(sys.stdout)
if getattr(sys, "frozen", False) and not console_logging_forced:
wants_console_handler = False
if console_logging_disabled:
wants_console_handler = False
if WeChatLogger._initialized:
current_log_file = Path(getattr(self, "log_file", desired_log_file))
has_expected_file_handler = False
has_stream_handler = False
for handler in root_logger.handlers:
if isinstance(handler, logging.FileHandler):
try:
if Path(handler.baseFilename).resolve() == desired_log_file.resolve():
has_expected_file_handler = True
except Exception:
if Path(handler.baseFilename) == desired_log_file:
has_expected_file_handler = True
elif isinstance(handler, logging.StreamHandler):
has_stream_handler = True
if (
current_log_file == desired_log_file
and root_logger.level == level
and has_expected_file_handler
and (has_stream_handler or not wants_console_handler)
):
self.log_file = desired_log_file
return self.log_file
self.log_file = desired_log_file
# 清除现有的处理器
root_logger = logging.getLogger()
for handler in root_logger.handlers[:]:
root_logger.removeHandler(handler)
try:
@@ -100,18 +153,20 @@ class WeChatLogger:
# 文件处理器
file_handler = logging.FileHandler(self.log_file, encoding='utf-8')
file_handler.setFormatter(file_formatter)
level = getattr(logging, str(log_level or "INFO").upper(), logging.INFO)
file_handler.setLevel(level)
# 控制台处理器
console_handler = logging.StreamHandler(sys.stdout)
console_handler.setFormatter(console_formatter)
console_handler.setLevel(level)
console_handler = None
if wants_console_handler:
console_handler = logging.StreamHandler(sys.stdout)
console_handler.setFormatter(console_formatter)
console_handler.setLevel(level)
# 配置根日志器
root_logger.setLevel(level)
root_logger.addHandler(file_handler)
root_logger.addHandler(console_handler)
if console_handler is not None:
root_logger.addHandler(console_handler)
# 只为uvicorn日志器添加文件处理器,保持其原有的控制台处理器(带颜色)
uvicorn_logger = logging.getLogger("uvicorn")
@@ -158,7 +213,8 @@ class WeChatLogger:
except Exception:
pass
fastapi_logger.addHandler(file_handler)
fastapi_logger.addHandler(console_handler)
if console_handler is not None:
fastapi_logger.addHandler(console_handler)
fastapi_logger.setLevel(level)
# 记录初始化信息
+326 -222
View File
@@ -1391,6 +1391,299 @@ def _load_contact_top_flags(contact_db_path: Path, usernames: list[str]) -> dict
conn.close()
def _coerce_realtime_blobish_value(value: Any) -> Any:
if value is None:
return None
if isinstance(value, memoryview):
value = value.tobytes()
if isinstance(value, bytearray):
return bytes(value)
if isinstance(value, bytes):
try:
s = value.decode("ascii").strip()
except Exception:
return value
if not s:
return value
b = _hex_to_bytes(s)
if b is not None:
return b
if (len(s) % 2 == 0) and (_HEX_RE.fullmatch(s) is not None):
try:
return bytes.fromhex(s)
except Exception:
return value
return value
if isinstance(value, str):
s = value.strip()
if not s:
return value
b = _hex_to_bytes(s)
if b is not None:
return b
if (len(s) % 2 == 0) and (_HEX_RE.fullmatch(s) is not None):
try:
return bytes.fromhex(s)
except Exception:
return value
return value
return value
def _normalize_realtime_message_item(item: dict[str, Any]) -> dict[str, Any]:
def _pick(*keys: str) -> Any:
return _pick_case_insensitive_value(item, *keys)
message_content = _coerce_realtime_blobish_value(
_pick("message_content", "messageContent", "MessageContent")
)
if message_content is None:
message_content = ""
return {
"local_id": int(_pick("local_id", "localId") or 0),
"server_id": int(_pick("server_id", "serverId", "MsgSvrID") or 0),
"local_type": int(_pick("local_type", "localType", "Type", "type") or 0),
"sort_seq": int(_pick("sort_seq", "sortSeq", "SortSeq") or 0),
"real_sender_id": int(_pick("real_sender_id", "realSenderId") or 0),
"create_time": int(_pick("create_time", "createTime", "CreateTime") or 0),
"message_content": message_content,
"compress_content": _coerce_realtime_blobish_value(
_pick("compress_content", "compressContent", "CompressContent")
),
"packed_info_data": _coerce_realtime_blobish_value(
_pick("packed_info_data", "packedInfoData", "PackedInfoData")
),
"sender_username": str(
_pick("sender_username", "senderUsername", "sender", "SenderUsername") or ""
).strip(),
}
def _collect_realtime_rows_for_session(
*,
trace_id: Optional[str],
account_name: str,
rt_conn: Any,
username: str,
msg_db_path_real: Path,
table_name: str,
max_local_id: int,
max_scan: int,
backfill_limit: int,
) -> dict[str, Any]:
label = f"[{trace_id}]" if trace_id else "[realtime]"
log_fn = logger.info if trace_id else logger.debug
uname = str(username or "").strip()
use_biz_exec_query = uname.startswith("gh_") and ("biz_message" in str(msg_db_path_real.name).lower())
if use_biz_exec_query:
try:
quoted_table = _quote_ident(table_name)
select_cols = (
"local_id",
"server_id",
"local_type",
"sort_seq",
"real_sender_id",
"create_time",
"message_content",
"compress_content",
"packed_info_data",
)
select_sql = ", ".join([_quote_ident(col) for col in select_cols])
if int(max_local_id) > 0:
sql_new = (
f"SELECT {select_sql} FROM {quoted_table} "
f"WHERE local_id > {int(max_local_id)} "
f"ORDER BY local_id ASC LIMIT {int(max_scan)}"
)
else:
sql_new = f"SELECT {select_sql} FROM {quoted_table} ORDER BY local_id DESC LIMIT {int(max_scan)}"
log_fn(
"%s wcdb_exec_query biz account=%s username=%s mode=new_rows max_local_id=%s limit=%s",
label,
account_name,
uname,
int(max_local_id),
int(max_scan),
)
wcdb_t0 = time.perf_counter()
with rt_conn.lock:
raw_new_rows = _wcdb_exec_query(rt_conn.handle, kind="message", path=str(msg_db_path_real), sql=sql_new)
wcdb_ms = (time.perf_counter() - wcdb_t0) * 1000.0
logger.info(
"%s wcdb_exec_query biz done account=%s username=%s mode=new_rows rows=%s ms=%.1f",
label,
account_name,
uname,
len(raw_new_rows or []),
wcdb_ms,
)
if wcdb_ms > 2000:
logger.warning(
"%s wcdb_exec_query biz slow account=%s username=%s mode=new_rows ms=%.1f",
label,
account_name,
uname,
wcdb_ms,
)
normalized_new_rows: list[dict[str, Any]] = []
for item in raw_new_rows or []:
if not isinstance(item, dict):
continue
norm = _normalize_realtime_message_item(item)
if int(norm.get("local_id") or 0) <= 0:
continue
normalized_new_rows.append(norm)
if int(max_local_id) > 0:
new_rows = list(reversed(normalized_new_rows))
else:
new_rows = normalized_new_rows
backfill_rows: list[dict[str, Any]] = []
scanned = len(raw_new_rows or [])
if int(backfill_limit) > 0 and int(max_local_id) > 0:
sql_backfill = (
f"SELECT {select_sql} FROM {quoted_table} "
f"WHERE local_id <= {int(max_local_id)} "
f"ORDER BY local_id DESC LIMIT {int(backfill_limit)}"
)
log_fn(
"%s wcdb_exec_query biz account=%s username=%s mode=backfill limit=%s",
label,
account_name,
uname,
int(backfill_limit),
)
backfill_t0 = time.perf_counter()
with rt_conn.lock:
raw_backfill_rows = _wcdb_exec_query(
rt_conn.handle,
kind="message",
path=str(msg_db_path_real),
sql=sql_backfill,
)
backfill_ms = (time.perf_counter() - backfill_t0) * 1000.0
logger.info(
"%s wcdb_exec_query biz done account=%s username=%s mode=backfill rows=%s ms=%.1f",
label,
account_name,
uname,
len(raw_backfill_rows or []),
backfill_ms,
)
if backfill_ms > 2000:
logger.warning(
"%s wcdb_exec_query biz slow account=%s username=%s mode=backfill ms=%.1f",
label,
account_name,
uname,
backfill_ms,
)
scanned += len(raw_backfill_rows or [])
for item in raw_backfill_rows or []:
if not isinstance(item, dict):
continue
norm = _normalize_realtime_message_item(item)
if int(norm.get("local_id") or 0) <= 0:
continue
backfill_rows.append(norm)
return {
"fetchMode": "biz_exec_query",
"scanned": int(scanned),
"new_rows": new_rows,
"backfill_rows": backfill_rows,
}
except Exception as e:
logger.warning(
"%s wcdb_exec_query biz failed account=%s username=%s err=%s fallback=wcdb_get_messages",
label,
account_name,
uname,
str(e),
)
batch_size = 200
scanned = 0
offset = 0
new_rows: list[dict[str, Any]] = []
backfill_rows: list[dict[str, Any]] = []
reached_existing = False
stop = False
while scanned < int(max_scan):
take = min(batch_size, int(max_scan) - scanned)
log_fn(
"%s wcdb_get_messages account=%s username=%s take=%s offset=%s",
label,
account_name,
uname,
int(take),
int(offset),
)
wcdb_t0 = time.perf_counter()
with rt_conn.lock:
raw_rows = _wcdb_get_messages(rt_conn.handle, uname, limit=take, offset=offset)
wcdb_ms = (time.perf_counter() - wcdb_t0) * 1000.0
log_fn(
"%s wcdb_get_messages done account=%s username=%s rows=%s ms=%.1f",
label,
account_name,
uname,
len(raw_rows or []),
wcdb_ms,
)
if wcdb_ms > 2000:
logger.warning(
"%s wcdb_get_messages slow account=%s username=%s ms=%.1f",
label,
account_name,
uname,
wcdb_ms,
)
if not raw_rows:
break
scanned += len(raw_rows)
offset += len(raw_rows)
for item in raw_rows:
if not isinstance(item, dict):
continue
norm = _normalize_realtime_message_item(item)
lid = int(norm.get("local_id") or 0)
if lid <= 0:
continue
if (not reached_existing) and lid > int(max_local_id):
new_rows.append(norm)
continue
reached_existing = True
if int(backfill_limit) <= 0:
stop = True
break
backfill_rows.append(norm)
if len(backfill_rows) >= int(backfill_limit):
stop = True
break
if stop or len(raw_rows) < take:
break
return {
"fetchMode": "wcdb_get_messages",
"scanned": int(scanned),
"new_rows": new_rows,
"backfill_rows": backfill_rows,
}
@router.post("/api/chat/realtime/sync", summary="实时消息同步到解密库(按会话增量)")
def sync_chat_realtime_messages(
request: Request,
@@ -1511,118 +1804,20 @@ def sync_chat_realtime_messages(
placeholders = ",".join(["?"] * len(insert_cols))
insert_sql = f"INSERT OR IGNORE INTO {quoted_table} ({','.join(insert_cols)}) VALUES ({placeholders})"
def pick(item: dict[str, Any], *keys: str) -> Any:
for k in keys:
if k in item and item[k] is not None:
return item[k]
lk = k.lower()
for kk in item.keys():
if str(kk).lower() == lk and item[kk] is not None:
return item[kk]
return None
def normalize_blob(value: Any) -> Optional[bytes]:
if value is None:
return None
if isinstance(value, memoryview):
return value.tobytes()
if isinstance(value, (bytes, bytearray)):
return bytes(value)
if isinstance(value, str):
s = value.strip()
if s.lower().startswith("0x"):
s = s[2:]
if s and re.fullmatch(r"[0-9a-fA-F]+", s) and (len(s) % 2 == 0):
try:
return bytes.fromhex(s)
except Exception:
return None
return s.encode("utf-8", errors="ignore")
return None
def normalize(item: dict[str, Any]) -> dict[str, Any]:
return {
"local_id": int(pick(item, "local_id", "localId") or 0),
"server_id": int(pick(item, "server_id", "serverId", "MsgSvrID") or 0),
"local_type": int(pick(item, "local_type", "localType", "Type", "type") or 0),
"sort_seq": int(pick(item, "sort_seq", "sortSeq", "SortSeq") or 0),
"real_sender_id": int(pick(item, "real_sender_id", "realSenderId") or 0),
"create_time": int(pick(item, "create_time", "createTime", "CreateTime") or 0),
"message_content": pick(item, "message_content", "messageContent", "MessageContent") or "",
"compress_content": pick(item, "compress_content", "compressContent", "CompressContent"),
"packed_info_data": normalize_blob(pick(item, "packed_info_data", "packedInfoData")),
"sender_username": str(
pick(item, "sender_username", "senderUsername", "sender", "SenderUsername") or ""
).strip(),
}
batch_size = 200
scanned = 0
offset = 0
new_rows: list[dict[str, Any]] = []
backfill_rows: list[dict[str, Any]] = []
reached_existing = False
stop = False
while scanned < int(max_scan):
take = min(batch_size, int(max_scan) - scanned)
logger.info(
"[%s] wcdb_get_messages account=%s username=%s take=%s offset=%s",
trace_id,
account_dir.name,
username,
int(take),
int(offset),
)
wcdb_t0 = time.perf_counter()
with rt_conn.lock:
raw_rows = _wcdb_get_messages(rt_conn.handle, username, limit=take, offset=offset)
wcdb_ms = (time.perf_counter() - wcdb_t0) * 1000.0
logger.info(
"[%s] wcdb_get_messages done account=%s username=%s rows=%s ms=%.1f",
trace_id,
account_dir.name,
username,
len(raw_rows or []),
wcdb_ms,
)
if wcdb_ms > 2000:
logger.warning(
"[%s] wcdb_get_messages slow account=%s username=%s ms=%.1f",
trace_id,
account_dir.name,
username,
wcdb_ms,
)
if not raw_rows:
break
scanned += len(raw_rows)
offset += len(raw_rows)
for item in raw_rows:
if not isinstance(item, dict):
continue
norm = normalize(item)
lid = int(norm.get("local_id") or 0)
if lid <= 0:
continue
if (not reached_existing) and lid > max_local_id:
new_rows.append(norm)
continue
reached_existing = True
if int(backfill_limit) <= 0:
stop = True
break
backfill_rows.append(norm)
if len(backfill_rows) >= int(backfill_limit):
stop = True
break
if stop or len(raw_rows) < take:
break
fetch_result = _collect_realtime_rows_for_session(
trace_id=trace_id,
account_name=account_dir.name,
rt_conn=rt_conn,
username=username,
msg_db_path_real=msg_db_path_real,
table_name=table_name,
max_local_id=max_local_id,
max_scan=int(max_scan),
backfill_limit=int(backfill_limit),
)
scanned = int(fetch_result.get("scanned") or 0)
new_rows = list(fetch_result.get("new_rows") or [])
backfill_rows = list(fetch_result.get("backfill_rows") or [])
inserted = 0
backfilled = 0
@@ -1880,115 +2075,20 @@ def _sync_chat_realtime_messages_for_table(
placeholders = ",".join(["?"] * len(insert_cols))
insert_sql = f"INSERT OR IGNORE INTO {quoted_table} ({','.join(insert_cols)}) VALUES ({placeholders})"
def pick(item: dict[str, Any], *keys: str) -> Any:
for k in keys:
if k in item and item[k] is not None:
return item[k]
lk = k.lower()
for kk in item.keys():
if str(kk).lower() == lk and item[kk] is not None:
return item[kk]
return None
def normalize_blob(value: Any) -> Optional[bytes]:
if value is None:
return None
if isinstance(value, memoryview):
return value.tobytes()
if isinstance(value, (bytes, bytearray)):
return bytes(value)
if isinstance(value, str):
s = value.strip()
if s.lower().startswith("0x"):
s = s[2:]
if s and re.fullmatch(r"[0-9a-fA-F]+", s) and (len(s) % 2 == 0):
try:
return bytes.fromhex(s)
except Exception:
return None
return s.encode("utf-8", errors="ignore")
return None
def normalize(item: dict[str, Any]) -> dict[str, Any]:
return {
"local_id": int(pick(item, "local_id", "localId") or 0),
"server_id": int(pick(item, "server_id", "serverId", "MsgSvrID") or 0),
"local_type": int(pick(item, "local_type", "localType", "Type", "type") or 0),
"sort_seq": int(pick(item, "sort_seq", "sortSeq", "SortSeq") or 0),
"real_sender_id": int(pick(item, "real_sender_id", "realSenderId") or 0),
"create_time": int(pick(item, "create_time", "createTime", "CreateTime") or 0),
"message_content": pick(item, "message_content", "messageContent", "MessageContent") or "",
"compress_content": pick(item, "compress_content", "compressContent", "CompressContent"),
"packed_info_data": normalize_blob(pick(item, "packed_info_data", "packedInfoData")),
"sender_username": str(
pick(item, "sender_username", "senderUsername", "sender", "SenderUsername") or ""
).strip(),
}
batch_size = 200
scanned = 0
offset = 0
new_rows: list[dict[str, Any]] = []
backfill_rows: list[dict[str, Any]] = []
reached_existing = False
stop = False
while scanned < int(max_scan):
take = min(batch_size, int(max_scan) - scanned)
logger.debug(
"[realtime] wcdb_get_messages account=%s username=%s take=%s offset=%s",
account_dir.name,
username,
int(take),
int(offset),
)
wcdb_t0 = time.perf_counter()
with rt_conn.lock:
raw_rows = _wcdb_get_messages(rt_conn.handle, username, limit=take, offset=offset)
wcdb_ms = (time.perf_counter() - wcdb_t0) * 1000.0
logger.debug(
"[realtime] wcdb_get_messages done account=%s username=%s rows=%s ms=%.1f",
account_dir.name,
username,
len(raw_rows or []),
wcdb_ms,
)
if wcdb_ms > 2000:
logger.warning(
"[realtime] wcdb_get_messages slow account=%s username=%s ms=%.1f",
account_dir.name,
username,
wcdb_ms,
)
if not raw_rows:
break
scanned += len(raw_rows)
offset += len(raw_rows)
for item in raw_rows:
if not isinstance(item, dict):
continue
norm = normalize(item)
lid = int(norm.get("local_id") or 0)
if lid <= 0:
continue
if (not reached_existing) and lid > max_local_id:
new_rows.append(norm)
continue
reached_existing = True
if int(backfill_limit) <= 0:
stop = True
break
backfill_rows.append(norm)
if len(backfill_rows) >= int(backfill_limit):
stop = True
break
if stop or len(raw_rows) < take:
break
fetch_result = _collect_realtime_rows_for_session(
trace_id=None,
account_name=account_dir.name,
rt_conn=rt_conn,
username=username,
msg_db_path_real=msg_db_path_real,
table_name=table_name,
max_local_id=max_local_id,
max_scan=int(max_scan),
backfill_limit=int(backfill_limit),
)
scanned = int(fetch_result.get("scanned") or 0)
new_rows = list(fetch_result.get("new_rows") or [])
backfill_rows = list(fetch_result.get("backfill_rows") or [])
inserted = 0
backfilled = 0
@@ -2163,6 +2263,7 @@ def sync_chat_realtime_messages_all(
priority_max_scan: int = 600,
include_hidden: bool = True,
include_official: bool = True,
only_official: bool = False,
backfill_limit: int = 200,
):
"""
@@ -2173,13 +2274,14 @@ def sync_chat_realtime_messages_all(
account_dir = _resolve_account_dir(account)
trace_id = f"rt-syncall-{int(time.time() * 1000)}-{threading.get_ident()}"
logger.info(
"[%s] realtime sync_all start account=%s max_scan=%s priority=%s include_hidden=%s include_official=%s",
"[%s] realtime sync_all start account=%s max_scan=%s priority=%s include_hidden=%s include_official=%s only_official=%s",
trace_id,
account_dir.name,
int(max_scan),
str(priority_username or "").strip(),
bool(include_hidden),
bool(include_official),
bool(only_official),
)
if max_scan < 20:
@@ -2241,6 +2343,8 @@ def sync_chat_realtime_messages_all(
hidden_val = 0
if not include_hidden and hidden_val == 1:
continue
if only_official and not uname.startswith("gh_"):
continue
if not _should_keep_session(uname, include_official=include_official):
continue
+34
View File
@@ -26,6 +26,32 @@ _DEFAULT_WCDB_API_DLL = _NATIVE_DIR / "wcdb_api.dll"
_WCDB_API_DLL_SELECTED: Optional[Path] = None
def _iter_runtime_wcdb_api_dll_paths() -> tuple[Path, ...]:
candidates: list[Path] = []
seen: set[str] = set()
def add_anchor(anchor: str | Path | None) -> None:
if not anchor:
return
try:
base = Path(anchor).resolve()
except Exception:
base = Path(anchor)
candidate = base / "native" / "wcdb_api.dll"
key = str(candidate).replace("/", "\\").rstrip("\\").lower()
if key in seen:
return
seen.add(key)
candidates.append(candidate)
add_anchor(os.environ.get("WECHAT_TOOL_DATA_DIR", "").strip())
add_anchor(Path.cwd())
if getattr(sys, "frozen", False):
add_anchor(Path(sys.executable).resolve().parent)
return tuple(candidates)
def _is_project_wcdb_api_dll_path(path: Path) -> bool:
try:
resolved = path.resolve(strict=False)
@@ -40,6 +66,14 @@ def _is_project_wcdb_api_dll_path(path: Path) -> bool:
if resolved == default_resolved:
return True
for candidate in _iter_runtime_wcdb_api_dll_paths():
try:
if resolved == candidate.resolve(strict=False):
return True
except Exception:
if resolved == candidate:
return True
parts = tuple(str(part).lower() for part in resolved.parts)
allowed_suffixes = (
("backend", "native", "wcdb_api.dll"),