mirror of
https://github.com/LifeArchiveProject/WeChatDataAnalysis.git
synced 2026-06-18 15:54:08 +08:00
fix(realtime): 修复服务号实时同步写入解密库并优化桌面端运行时兼容性
服务号实时同步改为直接读取 live biz_message 库,避免 gh_* 会话走空的 wcdb_get_messages。 将同步结果写回 output/databases 解密库,并补充异常降级路径。 修复桌面端日志处理与 wcdb_api.dll 的运行时定位问题。
This commit is contained in:
@@ -41,6 +41,22 @@ class ColoredFormatter(logging.Formatter):
|
||||
return formatted
|
||||
|
||||
|
||||
def _can_use_logging_stream(stream) -> bool:
|
||||
try:
|
||||
if stream is None or getattr(stream, "closed", False):
|
||||
return False
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
try:
|
||||
stream.write("")
|
||||
stream.flush()
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
class WeChatLogger:
|
||||
"""微信解密工具统一日志管理器"""
|
||||
|
||||
@@ -64,6 +80,12 @@ class WeChatLogger:
|
||||
if env_level:
|
||||
log_level = env_level
|
||||
|
||||
console_logging_env = str(os.environ.get("WECHAT_TOOL_ENABLE_CONSOLE_LOG", "") or "").strip().lower()
|
||||
console_logging_forced = console_logging_env in {"1", "true", "yes", "on"}
|
||||
console_logging_disabled = console_logging_env in {"0", "false", "no", "off"}
|
||||
|
||||
level = getattr(logging, str(log_level or "INFO").upper(), logging.INFO)
|
||||
|
||||
# 创建日志目录
|
||||
now = datetime.now()
|
||||
from .app_paths import get_output_dir
|
||||
@@ -73,10 +95,41 @@ class WeChatLogger:
|
||||
|
||||
# 设置日志文件名
|
||||
date_str = now.strftime("%d")
|
||||
self.log_file = log_dir / f"{date_str}_wechat_tool.log"
|
||||
desired_log_file = log_dir / f"{date_str}_wechat_tool.log"
|
||||
|
||||
root_logger = logging.getLogger()
|
||||
wants_console_handler = _can_use_logging_stream(sys.stdout)
|
||||
if getattr(sys, "frozen", False) and not console_logging_forced:
|
||||
wants_console_handler = False
|
||||
if console_logging_disabled:
|
||||
wants_console_handler = False
|
||||
|
||||
if WeChatLogger._initialized:
|
||||
current_log_file = Path(getattr(self, "log_file", desired_log_file))
|
||||
has_expected_file_handler = False
|
||||
has_stream_handler = False
|
||||
for handler in root_logger.handlers:
|
||||
if isinstance(handler, logging.FileHandler):
|
||||
try:
|
||||
if Path(handler.baseFilename).resolve() == desired_log_file.resolve():
|
||||
has_expected_file_handler = True
|
||||
except Exception:
|
||||
if Path(handler.baseFilename) == desired_log_file:
|
||||
has_expected_file_handler = True
|
||||
elif isinstance(handler, logging.StreamHandler):
|
||||
has_stream_handler = True
|
||||
if (
|
||||
current_log_file == desired_log_file
|
||||
and root_logger.level == level
|
||||
and has_expected_file_handler
|
||||
and (has_stream_handler or not wants_console_handler)
|
||||
):
|
||||
self.log_file = desired_log_file
|
||||
return self.log_file
|
||||
|
||||
self.log_file = desired_log_file
|
||||
|
||||
# 清除现有的处理器
|
||||
root_logger = logging.getLogger()
|
||||
for handler in root_logger.handlers[:]:
|
||||
root_logger.removeHandler(handler)
|
||||
try:
|
||||
@@ -100,18 +153,20 @@ class WeChatLogger:
|
||||
# 文件处理器
|
||||
file_handler = logging.FileHandler(self.log_file, encoding='utf-8')
|
||||
file_handler.setFormatter(file_formatter)
|
||||
level = getattr(logging, str(log_level or "INFO").upper(), logging.INFO)
|
||||
file_handler.setLevel(level)
|
||||
|
||||
# 控制台处理器
|
||||
console_handler = logging.StreamHandler(sys.stdout)
|
||||
console_handler.setFormatter(console_formatter)
|
||||
console_handler.setLevel(level)
|
||||
console_handler = None
|
||||
if wants_console_handler:
|
||||
console_handler = logging.StreamHandler(sys.stdout)
|
||||
console_handler.setFormatter(console_formatter)
|
||||
console_handler.setLevel(level)
|
||||
|
||||
# 配置根日志器
|
||||
root_logger.setLevel(level)
|
||||
root_logger.addHandler(file_handler)
|
||||
root_logger.addHandler(console_handler)
|
||||
if console_handler is not None:
|
||||
root_logger.addHandler(console_handler)
|
||||
|
||||
# 只为uvicorn日志器添加文件处理器,保持其原有的控制台处理器(带颜色)
|
||||
uvicorn_logger = logging.getLogger("uvicorn")
|
||||
@@ -158,7 +213,8 @@ class WeChatLogger:
|
||||
except Exception:
|
||||
pass
|
||||
fastapi_logger.addHandler(file_handler)
|
||||
fastapi_logger.addHandler(console_handler)
|
||||
if console_handler is not None:
|
||||
fastapi_logger.addHandler(console_handler)
|
||||
fastapi_logger.setLevel(level)
|
||||
|
||||
# 记录初始化信息
|
||||
|
||||
@@ -1391,6 +1391,299 @@ def _load_contact_top_flags(contact_db_path: Path, usernames: list[str]) -> dict
|
||||
conn.close()
|
||||
|
||||
|
||||
def _coerce_realtime_blobish_value(value: Any) -> Any:
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, memoryview):
|
||||
value = value.tobytes()
|
||||
if isinstance(value, bytearray):
|
||||
return bytes(value)
|
||||
if isinstance(value, bytes):
|
||||
try:
|
||||
s = value.decode("ascii").strip()
|
||||
except Exception:
|
||||
return value
|
||||
if not s:
|
||||
return value
|
||||
b = _hex_to_bytes(s)
|
||||
if b is not None:
|
||||
return b
|
||||
if (len(s) % 2 == 0) and (_HEX_RE.fullmatch(s) is not None):
|
||||
try:
|
||||
return bytes.fromhex(s)
|
||||
except Exception:
|
||||
return value
|
||||
return value
|
||||
if isinstance(value, str):
|
||||
s = value.strip()
|
||||
if not s:
|
||||
return value
|
||||
b = _hex_to_bytes(s)
|
||||
if b is not None:
|
||||
return b
|
||||
if (len(s) % 2 == 0) and (_HEX_RE.fullmatch(s) is not None):
|
||||
try:
|
||||
return bytes.fromhex(s)
|
||||
except Exception:
|
||||
return value
|
||||
return value
|
||||
return value
|
||||
|
||||
|
||||
def _normalize_realtime_message_item(item: dict[str, Any]) -> dict[str, Any]:
|
||||
def _pick(*keys: str) -> Any:
|
||||
return _pick_case_insensitive_value(item, *keys)
|
||||
|
||||
message_content = _coerce_realtime_blobish_value(
|
||||
_pick("message_content", "messageContent", "MessageContent")
|
||||
)
|
||||
if message_content is None:
|
||||
message_content = ""
|
||||
|
||||
return {
|
||||
"local_id": int(_pick("local_id", "localId") or 0),
|
||||
"server_id": int(_pick("server_id", "serverId", "MsgSvrID") or 0),
|
||||
"local_type": int(_pick("local_type", "localType", "Type", "type") or 0),
|
||||
"sort_seq": int(_pick("sort_seq", "sortSeq", "SortSeq") or 0),
|
||||
"real_sender_id": int(_pick("real_sender_id", "realSenderId") or 0),
|
||||
"create_time": int(_pick("create_time", "createTime", "CreateTime") or 0),
|
||||
"message_content": message_content,
|
||||
"compress_content": _coerce_realtime_blobish_value(
|
||||
_pick("compress_content", "compressContent", "CompressContent")
|
||||
),
|
||||
"packed_info_data": _coerce_realtime_blobish_value(
|
||||
_pick("packed_info_data", "packedInfoData", "PackedInfoData")
|
||||
),
|
||||
"sender_username": str(
|
||||
_pick("sender_username", "senderUsername", "sender", "SenderUsername") or ""
|
||||
).strip(),
|
||||
}
|
||||
|
||||
|
||||
def _collect_realtime_rows_for_session(
|
||||
*,
|
||||
trace_id: Optional[str],
|
||||
account_name: str,
|
||||
rt_conn: Any,
|
||||
username: str,
|
||||
msg_db_path_real: Path,
|
||||
table_name: str,
|
||||
max_local_id: int,
|
||||
max_scan: int,
|
||||
backfill_limit: int,
|
||||
) -> dict[str, Any]:
|
||||
label = f"[{trace_id}]" if trace_id else "[realtime]"
|
||||
log_fn = logger.info if trace_id else logger.debug
|
||||
uname = str(username or "").strip()
|
||||
use_biz_exec_query = uname.startswith("gh_") and ("biz_message" in str(msg_db_path_real.name).lower())
|
||||
|
||||
if use_biz_exec_query:
|
||||
try:
|
||||
quoted_table = _quote_ident(table_name)
|
||||
select_cols = (
|
||||
"local_id",
|
||||
"server_id",
|
||||
"local_type",
|
||||
"sort_seq",
|
||||
"real_sender_id",
|
||||
"create_time",
|
||||
"message_content",
|
||||
"compress_content",
|
||||
"packed_info_data",
|
||||
)
|
||||
select_sql = ", ".join([_quote_ident(col) for col in select_cols])
|
||||
|
||||
if int(max_local_id) > 0:
|
||||
sql_new = (
|
||||
f"SELECT {select_sql} FROM {quoted_table} "
|
||||
f"WHERE local_id > {int(max_local_id)} "
|
||||
f"ORDER BY local_id ASC LIMIT {int(max_scan)}"
|
||||
)
|
||||
else:
|
||||
sql_new = f"SELECT {select_sql} FROM {quoted_table} ORDER BY local_id DESC LIMIT {int(max_scan)}"
|
||||
|
||||
log_fn(
|
||||
"%s wcdb_exec_query biz account=%s username=%s mode=new_rows max_local_id=%s limit=%s",
|
||||
label,
|
||||
account_name,
|
||||
uname,
|
||||
int(max_local_id),
|
||||
int(max_scan),
|
||||
)
|
||||
wcdb_t0 = time.perf_counter()
|
||||
with rt_conn.lock:
|
||||
raw_new_rows = _wcdb_exec_query(rt_conn.handle, kind="message", path=str(msg_db_path_real), sql=sql_new)
|
||||
wcdb_ms = (time.perf_counter() - wcdb_t0) * 1000.0
|
||||
logger.info(
|
||||
"%s wcdb_exec_query biz done account=%s username=%s mode=new_rows rows=%s ms=%.1f",
|
||||
label,
|
||||
account_name,
|
||||
uname,
|
||||
len(raw_new_rows or []),
|
||||
wcdb_ms,
|
||||
)
|
||||
if wcdb_ms > 2000:
|
||||
logger.warning(
|
||||
"%s wcdb_exec_query biz slow account=%s username=%s mode=new_rows ms=%.1f",
|
||||
label,
|
||||
account_name,
|
||||
uname,
|
||||
wcdb_ms,
|
||||
)
|
||||
|
||||
normalized_new_rows: list[dict[str, Any]] = []
|
||||
for item in raw_new_rows or []:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
norm = _normalize_realtime_message_item(item)
|
||||
if int(norm.get("local_id") or 0) <= 0:
|
||||
continue
|
||||
normalized_new_rows.append(norm)
|
||||
|
||||
if int(max_local_id) > 0:
|
||||
new_rows = list(reversed(normalized_new_rows))
|
||||
else:
|
||||
new_rows = normalized_new_rows
|
||||
|
||||
backfill_rows: list[dict[str, Any]] = []
|
||||
scanned = len(raw_new_rows or [])
|
||||
if int(backfill_limit) > 0 and int(max_local_id) > 0:
|
||||
sql_backfill = (
|
||||
f"SELECT {select_sql} FROM {quoted_table} "
|
||||
f"WHERE local_id <= {int(max_local_id)} "
|
||||
f"ORDER BY local_id DESC LIMIT {int(backfill_limit)}"
|
||||
)
|
||||
log_fn(
|
||||
"%s wcdb_exec_query biz account=%s username=%s mode=backfill limit=%s",
|
||||
label,
|
||||
account_name,
|
||||
uname,
|
||||
int(backfill_limit),
|
||||
)
|
||||
backfill_t0 = time.perf_counter()
|
||||
with rt_conn.lock:
|
||||
raw_backfill_rows = _wcdb_exec_query(
|
||||
rt_conn.handle,
|
||||
kind="message",
|
||||
path=str(msg_db_path_real),
|
||||
sql=sql_backfill,
|
||||
)
|
||||
backfill_ms = (time.perf_counter() - backfill_t0) * 1000.0
|
||||
logger.info(
|
||||
"%s wcdb_exec_query biz done account=%s username=%s mode=backfill rows=%s ms=%.1f",
|
||||
label,
|
||||
account_name,
|
||||
uname,
|
||||
len(raw_backfill_rows or []),
|
||||
backfill_ms,
|
||||
)
|
||||
if backfill_ms > 2000:
|
||||
logger.warning(
|
||||
"%s wcdb_exec_query biz slow account=%s username=%s mode=backfill ms=%.1f",
|
||||
label,
|
||||
account_name,
|
||||
uname,
|
||||
backfill_ms,
|
||||
)
|
||||
scanned += len(raw_backfill_rows or [])
|
||||
for item in raw_backfill_rows or []:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
norm = _normalize_realtime_message_item(item)
|
||||
if int(norm.get("local_id") or 0) <= 0:
|
||||
continue
|
||||
backfill_rows.append(norm)
|
||||
|
||||
return {
|
||||
"fetchMode": "biz_exec_query",
|
||||
"scanned": int(scanned),
|
||||
"new_rows": new_rows,
|
||||
"backfill_rows": backfill_rows,
|
||||
}
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"%s wcdb_exec_query biz failed account=%s username=%s err=%s fallback=wcdb_get_messages",
|
||||
label,
|
||||
account_name,
|
||||
uname,
|
||||
str(e),
|
||||
)
|
||||
|
||||
batch_size = 200
|
||||
scanned = 0
|
||||
offset = 0
|
||||
new_rows: list[dict[str, Any]] = []
|
||||
backfill_rows: list[dict[str, Any]] = []
|
||||
reached_existing = False
|
||||
stop = False
|
||||
|
||||
while scanned < int(max_scan):
|
||||
take = min(batch_size, int(max_scan) - scanned)
|
||||
log_fn(
|
||||
"%s wcdb_get_messages account=%s username=%s take=%s offset=%s",
|
||||
label,
|
||||
account_name,
|
||||
uname,
|
||||
int(take),
|
||||
int(offset),
|
||||
)
|
||||
wcdb_t0 = time.perf_counter()
|
||||
with rt_conn.lock:
|
||||
raw_rows = _wcdb_get_messages(rt_conn.handle, uname, limit=take, offset=offset)
|
||||
wcdb_ms = (time.perf_counter() - wcdb_t0) * 1000.0
|
||||
log_fn(
|
||||
"%s wcdb_get_messages done account=%s username=%s rows=%s ms=%.1f",
|
||||
label,
|
||||
account_name,
|
||||
uname,
|
||||
len(raw_rows or []),
|
||||
wcdb_ms,
|
||||
)
|
||||
if wcdb_ms > 2000:
|
||||
logger.warning(
|
||||
"%s wcdb_get_messages slow account=%s username=%s ms=%.1f",
|
||||
label,
|
||||
account_name,
|
||||
uname,
|
||||
wcdb_ms,
|
||||
)
|
||||
if not raw_rows:
|
||||
break
|
||||
|
||||
scanned += len(raw_rows)
|
||||
offset += len(raw_rows)
|
||||
|
||||
for item in raw_rows:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
norm = _normalize_realtime_message_item(item)
|
||||
lid = int(norm.get("local_id") or 0)
|
||||
if lid <= 0:
|
||||
continue
|
||||
if (not reached_existing) and lid > int(max_local_id):
|
||||
new_rows.append(norm)
|
||||
continue
|
||||
|
||||
reached_existing = True
|
||||
if int(backfill_limit) <= 0:
|
||||
stop = True
|
||||
break
|
||||
backfill_rows.append(norm)
|
||||
if len(backfill_rows) >= int(backfill_limit):
|
||||
stop = True
|
||||
break
|
||||
|
||||
if stop or len(raw_rows) < take:
|
||||
break
|
||||
|
||||
return {
|
||||
"fetchMode": "wcdb_get_messages",
|
||||
"scanned": int(scanned),
|
||||
"new_rows": new_rows,
|
||||
"backfill_rows": backfill_rows,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/api/chat/realtime/sync", summary="实时消息同步到解密库(按会话增量)")
|
||||
def sync_chat_realtime_messages(
|
||||
request: Request,
|
||||
@@ -1511,118 +1804,20 @@ def sync_chat_realtime_messages(
|
||||
|
||||
placeholders = ",".join(["?"] * len(insert_cols))
|
||||
insert_sql = f"INSERT OR IGNORE INTO {quoted_table} ({','.join(insert_cols)}) VALUES ({placeholders})"
|
||||
|
||||
def pick(item: dict[str, Any], *keys: str) -> Any:
|
||||
for k in keys:
|
||||
if k in item and item[k] is not None:
|
||||
return item[k]
|
||||
lk = k.lower()
|
||||
for kk in item.keys():
|
||||
if str(kk).lower() == lk and item[kk] is not None:
|
||||
return item[kk]
|
||||
return None
|
||||
|
||||
def normalize_blob(value: Any) -> Optional[bytes]:
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, memoryview):
|
||||
return value.tobytes()
|
||||
if isinstance(value, (bytes, bytearray)):
|
||||
return bytes(value)
|
||||
if isinstance(value, str):
|
||||
s = value.strip()
|
||||
if s.lower().startswith("0x"):
|
||||
s = s[2:]
|
||||
if s and re.fullmatch(r"[0-9a-fA-F]+", s) and (len(s) % 2 == 0):
|
||||
try:
|
||||
return bytes.fromhex(s)
|
||||
except Exception:
|
||||
return None
|
||||
return s.encode("utf-8", errors="ignore")
|
||||
return None
|
||||
|
||||
def normalize(item: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
"local_id": int(pick(item, "local_id", "localId") or 0),
|
||||
"server_id": int(pick(item, "server_id", "serverId", "MsgSvrID") or 0),
|
||||
"local_type": int(pick(item, "local_type", "localType", "Type", "type") or 0),
|
||||
"sort_seq": int(pick(item, "sort_seq", "sortSeq", "SortSeq") or 0),
|
||||
"real_sender_id": int(pick(item, "real_sender_id", "realSenderId") or 0),
|
||||
"create_time": int(pick(item, "create_time", "createTime", "CreateTime") or 0),
|
||||
"message_content": pick(item, "message_content", "messageContent", "MessageContent") or "",
|
||||
"compress_content": pick(item, "compress_content", "compressContent", "CompressContent"),
|
||||
"packed_info_data": normalize_blob(pick(item, "packed_info_data", "packedInfoData")),
|
||||
"sender_username": str(
|
||||
pick(item, "sender_username", "senderUsername", "sender", "SenderUsername") or ""
|
||||
).strip(),
|
||||
}
|
||||
|
||||
batch_size = 200
|
||||
scanned = 0
|
||||
offset = 0
|
||||
new_rows: list[dict[str, Any]] = []
|
||||
backfill_rows: list[dict[str, Any]] = []
|
||||
reached_existing = False
|
||||
stop = False
|
||||
|
||||
while scanned < int(max_scan):
|
||||
take = min(batch_size, int(max_scan) - scanned)
|
||||
logger.info(
|
||||
"[%s] wcdb_get_messages account=%s username=%s take=%s offset=%s",
|
||||
trace_id,
|
||||
account_dir.name,
|
||||
username,
|
||||
int(take),
|
||||
int(offset),
|
||||
)
|
||||
wcdb_t0 = time.perf_counter()
|
||||
with rt_conn.lock:
|
||||
raw_rows = _wcdb_get_messages(rt_conn.handle, username, limit=take, offset=offset)
|
||||
wcdb_ms = (time.perf_counter() - wcdb_t0) * 1000.0
|
||||
logger.info(
|
||||
"[%s] wcdb_get_messages done account=%s username=%s rows=%s ms=%.1f",
|
||||
trace_id,
|
||||
account_dir.name,
|
||||
username,
|
||||
len(raw_rows or []),
|
||||
wcdb_ms,
|
||||
)
|
||||
if wcdb_ms > 2000:
|
||||
logger.warning(
|
||||
"[%s] wcdb_get_messages slow account=%s username=%s ms=%.1f",
|
||||
trace_id,
|
||||
account_dir.name,
|
||||
username,
|
||||
wcdb_ms,
|
||||
)
|
||||
if not raw_rows:
|
||||
break
|
||||
|
||||
scanned += len(raw_rows)
|
||||
offset += len(raw_rows)
|
||||
|
||||
for item in raw_rows:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
norm = normalize(item)
|
||||
lid = int(norm.get("local_id") or 0)
|
||||
if lid <= 0:
|
||||
continue
|
||||
if (not reached_existing) and lid > max_local_id:
|
||||
new_rows.append(norm)
|
||||
continue
|
||||
|
||||
reached_existing = True
|
||||
if int(backfill_limit) <= 0:
|
||||
stop = True
|
||||
break
|
||||
backfill_rows.append(norm)
|
||||
if len(backfill_rows) >= int(backfill_limit):
|
||||
stop = True
|
||||
break
|
||||
|
||||
if stop or len(raw_rows) < take:
|
||||
break
|
||||
fetch_result = _collect_realtime_rows_for_session(
|
||||
trace_id=trace_id,
|
||||
account_name=account_dir.name,
|
||||
rt_conn=rt_conn,
|
||||
username=username,
|
||||
msg_db_path_real=msg_db_path_real,
|
||||
table_name=table_name,
|
||||
max_local_id=max_local_id,
|
||||
max_scan=int(max_scan),
|
||||
backfill_limit=int(backfill_limit),
|
||||
)
|
||||
scanned = int(fetch_result.get("scanned") or 0)
|
||||
new_rows = list(fetch_result.get("new_rows") or [])
|
||||
backfill_rows = list(fetch_result.get("backfill_rows") or [])
|
||||
|
||||
inserted = 0
|
||||
backfilled = 0
|
||||
@@ -1880,115 +2075,20 @@ def _sync_chat_realtime_messages_for_table(
|
||||
|
||||
placeholders = ",".join(["?"] * len(insert_cols))
|
||||
insert_sql = f"INSERT OR IGNORE INTO {quoted_table} ({','.join(insert_cols)}) VALUES ({placeholders})"
|
||||
|
||||
def pick(item: dict[str, Any], *keys: str) -> Any:
|
||||
for k in keys:
|
||||
if k in item and item[k] is not None:
|
||||
return item[k]
|
||||
lk = k.lower()
|
||||
for kk in item.keys():
|
||||
if str(kk).lower() == lk and item[kk] is not None:
|
||||
return item[kk]
|
||||
return None
|
||||
|
||||
def normalize_blob(value: Any) -> Optional[bytes]:
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, memoryview):
|
||||
return value.tobytes()
|
||||
if isinstance(value, (bytes, bytearray)):
|
||||
return bytes(value)
|
||||
if isinstance(value, str):
|
||||
s = value.strip()
|
||||
if s.lower().startswith("0x"):
|
||||
s = s[2:]
|
||||
if s and re.fullmatch(r"[0-9a-fA-F]+", s) and (len(s) % 2 == 0):
|
||||
try:
|
||||
return bytes.fromhex(s)
|
||||
except Exception:
|
||||
return None
|
||||
return s.encode("utf-8", errors="ignore")
|
||||
return None
|
||||
|
||||
def normalize(item: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
"local_id": int(pick(item, "local_id", "localId") or 0),
|
||||
"server_id": int(pick(item, "server_id", "serverId", "MsgSvrID") or 0),
|
||||
"local_type": int(pick(item, "local_type", "localType", "Type", "type") or 0),
|
||||
"sort_seq": int(pick(item, "sort_seq", "sortSeq", "SortSeq") or 0),
|
||||
"real_sender_id": int(pick(item, "real_sender_id", "realSenderId") or 0),
|
||||
"create_time": int(pick(item, "create_time", "createTime", "CreateTime") or 0),
|
||||
"message_content": pick(item, "message_content", "messageContent", "MessageContent") or "",
|
||||
"compress_content": pick(item, "compress_content", "compressContent", "CompressContent"),
|
||||
"packed_info_data": normalize_blob(pick(item, "packed_info_data", "packedInfoData")),
|
||||
"sender_username": str(
|
||||
pick(item, "sender_username", "senderUsername", "sender", "SenderUsername") or ""
|
||||
).strip(),
|
||||
}
|
||||
|
||||
batch_size = 200
|
||||
scanned = 0
|
||||
offset = 0
|
||||
new_rows: list[dict[str, Any]] = []
|
||||
backfill_rows: list[dict[str, Any]] = []
|
||||
reached_existing = False
|
||||
stop = False
|
||||
|
||||
while scanned < int(max_scan):
|
||||
take = min(batch_size, int(max_scan) - scanned)
|
||||
logger.debug(
|
||||
"[realtime] wcdb_get_messages account=%s username=%s take=%s offset=%s",
|
||||
account_dir.name,
|
||||
username,
|
||||
int(take),
|
||||
int(offset),
|
||||
)
|
||||
wcdb_t0 = time.perf_counter()
|
||||
with rt_conn.lock:
|
||||
raw_rows = _wcdb_get_messages(rt_conn.handle, username, limit=take, offset=offset)
|
||||
wcdb_ms = (time.perf_counter() - wcdb_t0) * 1000.0
|
||||
logger.debug(
|
||||
"[realtime] wcdb_get_messages done account=%s username=%s rows=%s ms=%.1f",
|
||||
account_dir.name,
|
||||
username,
|
||||
len(raw_rows or []),
|
||||
wcdb_ms,
|
||||
)
|
||||
if wcdb_ms > 2000:
|
||||
logger.warning(
|
||||
"[realtime] wcdb_get_messages slow account=%s username=%s ms=%.1f",
|
||||
account_dir.name,
|
||||
username,
|
||||
wcdb_ms,
|
||||
)
|
||||
if not raw_rows:
|
||||
break
|
||||
|
||||
scanned += len(raw_rows)
|
||||
offset += len(raw_rows)
|
||||
|
||||
for item in raw_rows:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
norm = normalize(item)
|
||||
lid = int(norm.get("local_id") or 0)
|
||||
if lid <= 0:
|
||||
continue
|
||||
if (not reached_existing) and lid > max_local_id:
|
||||
new_rows.append(norm)
|
||||
continue
|
||||
|
||||
reached_existing = True
|
||||
if int(backfill_limit) <= 0:
|
||||
stop = True
|
||||
break
|
||||
backfill_rows.append(norm)
|
||||
if len(backfill_rows) >= int(backfill_limit):
|
||||
stop = True
|
||||
break
|
||||
|
||||
if stop or len(raw_rows) < take:
|
||||
break
|
||||
fetch_result = _collect_realtime_rows_for_session(
|
||||
trace_id=None,
|
||||
account_name=account_dir.name,
|
||||
rt_conn=rt_conn,
|
||||
username=username,
|
||||
msg_db_path_real=msg_db_path_real,
|
||||
table_name=table_name,
|
||||
max_local_id=max_local_id,
|
||||
max_scan=int(max_scan),
|
||||
backfill_limit=int(backfill_limit),
|
||||
)
|
||||
scanned = int(fetch_result.get("scanned") or 0)
|
||||
new_rows = list(fetch_result.get("new_rows") or [])
|
||||
backfill_rows = list(fetch_result.get("backfill_rows") or [])
|
||||
|
||||
inserted = 0
|
||||
backfilled = 0
|
||||
@@ -2163,6 +2263,7 @@ def sync_chat_realtime_messages_all(
|
||||
priority_max_scan: int = 600,
|
||||
include_hidden: bool = True,
|
||||
include_official: bool = True,
|
||||
only_official: bool = False,
|
||||
backfill_limit: int = 200,
|
||||
):
|
||||
"""
|
||||
@@ -2173,13 +2274,14 @@ def sync_chat_realtime_messages_all(
|
||||
account_dir = _resolve_account_dir(account)
|
||||
trace_id = f"rt-syncall-{int(time.time() * 1000)}-{threading.get_ident()}"
|
||||
logger.info(
|
||||
"[%s] realtime sync_all start account=%s max_scan=%s priority=%s include_hidden=%s include_official=%s",
|
||||
"[%s] realtime sync_all start account=%s max_scan=%s priority=%s include_hidden=%s include_official=%s only_official=%s",
|
||||
trace_id,
|
||||
account_dir.name,
|
||||
int(max_scan),
|
||||
str(priority_username or "").strip(),
|
||||
bool(include_hidden),
|
||||
bool(include_official),
|
||||
bool(only_official),
|
||||
)
|
||||
|
||||
if max_scan < 20:
|
||||
@@ -2241,6 +2343,8 @@ def sync_chat_realtime_messages_all(
|
||||
hidden_val = 0
|
||||
if not include_hidden and hidden_val == 1:
|
||||
continue
|
||||
if only_official and not uname.startswith("gh_"):
|
||||
continue
|
||||
if not _should_keep_session(uname, include_official=include_official):
|
||||
continue
|
||||
|
||||
|
||||
@@ -26,6 +26,32 @@ _DEFAULT_WCDB_API_DLL = _NATIVE_DIR / "wcdb_api.dll"
|
||||
_WCDB_API_DLL_SELECTED: Optional[Path] = None
|
||||
|
||||
|
||||
def _iter_runtime_wcdb_api_dll_paths() -> tuple[Path, ...]:
|
||||
candidates: list[Path] = []
|
||||
seen: set[str] = set()
|
||||
|
||||
def add_anchor(anchor: str | Path | None) -> None:
|
||||
if not anchor:
|
||||
return
|
||||
try:
|
||||
base = Path(anchor).resolve()
|
||||
except Exception:
|
||||
base = Path(anchor)
|
||||
candidate = base / "native" / "wcdb_api.dll"
|
||||
key = str(candidate).replace("/", "\\").rstrip("\\").lower()
|
||||
if key in seen:
|
||||
return
|
||||
seen.add(key)
|
||||
candidates.append(candidate)
|
||||
|
||||
add_anchor(os.environ.get("WECHAT_TOOL_DATA_DIR", "").strip())
|
||||
add_anchor(Path.cwd())
|
||||
if getattr(sys, "frozen", False):
|
||||
add_anchor(Path(sys.executable).resolve().parent)
|
||||
|
||||
return tuple(candidates)
|
||||
|
||||
|
||||
def _is_project_wcdb_api_dll_path(path: Path) -> bool:
|
||||
try:
|
||||
resolved = path.resolve(strict=False)
|
||||
@@ -40,6 +66,14 @@ def _is_project_wcdb_api_dll_path(path: Path) -> bool:
|
||||
if resolved == default_resolved:
|
||||
return True
|
||||
|
||||
for candidate in _iter_runtime_wcdb_api_dll_paths():
|
||||
try:
|
||||
if resolved == candidate.resolve(strict=False):
|
||||
return True
|
||||
except Exception:
|
||||
if resolved == candidate:
|
||||
return True
|
||||
|
||||
parts = tuple(str(part).lower() for part in resolved.parts)
|
||||
allowed_suffixes = (
|
||||
("backend", "native", "wcdb_api.dll"),
|
||||
|
||||
Reference in New Issue
Block a user