feat(wrapped): 新增年度关键词词云卡片(Card #6)

- 新增关键词词云后端计算:消息采样、jieba 分词、关键词权重与例句回看数据。

- Wrapped 年度卡片接入 Card #6,并对该卡片改为按请求重算(不走单卡缓存)。

- 前端新增 storm->cloud 词云动效卡片,支持词点击查看例句及动画阶段隐藏 deck 顶部 UI。

- 统一 wrapped 消息表名解码逻辑,兼容 sqlite text_factory=bytes。

- 新增关键词词云测试并加入 jieba 依赖。
This commit is contained in:
2977094657
2026-02-22 18:59:30 +08:00
parent 9acbfa7582
commit 2dd103814a
12 changed files with 2136 additions and 25 deletions
@@ -77,7 +77,9 @@ def _list_message_tables(conn: sqlite3.Connection) -> list[str]:
for r in rows:
if not r or not r[0]:
continue
name = str(r[0])
name = _decode_sqlite_text(r[0]).strip()
if not name:
continue
ln = name.lower()
if ln.startswith(("msg_", "chat_")):
names.append(name)
@@ -12,6 +12,7 @@ from typing import Any, Optional
from ...chat_search_index import get_chat_search_index_db_path
from ...chat_helpers import (
_build_avatar_url,
_decode_sqlite_text,
_iter_message_db_paths,
_load_contact_rows,
_pick_display_name,
@@ -745,7 +746,9 @@ def _list_message_tables(conn: sqlite3.Connection) -> list[str]:
for r in rows:
if not r or not r[0]:
continue
name = str(r[0])
name = _decode_sqlite_text(r[0]).strip()
if not name:
continue
ln = name.lower()
if ln.startswith(("msg_", "chat_")):
names.append(name)
@@ -11,7 +11,7 @@ from typing import Any, Optional
from pypinyin import lazy_pinyin, Style
from ...chat_helpers import _decode_message_content, _iter_message_db_paths, _quote_ident
from ...chat_helpers import _decode_message_content, _decode_sqlite_text, _iter_message_db_paths, _quote_ident
from ...chat_search_index import get_chat_search_index_db_path
from ...logging_config import get_logger
@@ -467,7 +467,9 @@ def _list_message_tables(conn: sqlite3.Connection) -> list[str]:
for r in rows:
if not r or not r[0]:
continue
name = str(r[0])
name = _decode_sqlite_text(r[0]).strip()
if not name:
continue
ln = name.lower()
if ln.startswith(("msg_", "chat_")):
names.append(name)
@@ -0,0 +1,572 @@
from __future__ import annotations
import hashlib
import logging
import math
import random
import re
import sqlite3
import time
from collections import Counter
from datetime import datetime
from pathlib import Path
from typing import Any
import jieba
from ...chat_helpers import _decode_message_content, _decode_sqlite_text, _iter_message_db_paths, _quote_ident
from ...logging_config import get_logger
logger = get_logger(__name__)
try:
jieba.setLogLevel(logging.ERROR)
except Exception:
pass
_MD5_HEX_RE = re.compile(r"(?i)\b[0-9a-f]{32}\b")
_URL_RE = re.compile(r"(?i)\bhttps?://\S+")
_CTRL_RE = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f]")
_HAS_CJK_RE = re.compile(r"[\u4e00-\u9fff]")
_CJK_SEQ_RE = re.compile(r"[\u4e00-\u9fff]+")
_HAS_ALNUM_RE = re.compile(r"[\u4e00-\u9fffA-Za-z0-9]")
_EN_WORD_RE = re.compile(r"^[A-Za-z]{3,16}$")
_DATEISH_RE = re.compile(
r"^(?:"
r"\d{4}[-/]\d{1,2}[-/]\d{1,2}"
r"|"
r"\d{1,2}:\d{2}"
r"|"
r"\d{1,2}月\d{1,2}日"
r")$"
)
# Small but practical stopword list for chat keywords.
_STOPWORDS_ZH = {
"",
"",
"",
"",
"",
"",
"",
"",
"我们",
"你们",
"他们",
"她们",
"它们",
"",
"",
"这个",
"那个",
"这里",
"那里",
"这样",
"那样",
"就是",
"也是",
"还有",
"因为",
"所以",
"但是",
"如果",
"然后",
"已经",
"可以",
"还是",
"可能",
"不会",
"没有",
"不是",
"一个",
"一下",
"一下子",
"一下下",
"哈哈",
"哈哈哈",
"嘿嘿",
"呜呜",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"可以",
"ok",
"OK",
}
_STOPWORDS_EN = {
"the",
"a",
"an",
"and",
"or",
"but",
"to",
"of",
"in",
"on",
"for",
"with",
"at",
"from",
"as",
"is",
"are",
"was",
"were",
"be",
"been",
"being",
"i",
"me",
"my",
"you",
"your",
"he",
"she",
"it",
"we",
"they",
"them",
"this",
"that",
"these",
"those",
"yeah",
"haha",
"ok",
"okay",
"pls",
"lol",
}
def _year_range_epoch_seconds(year: int) -> tuple[int, int]:
start = int(datetime(int(year), 1, 1).timestamp())
end = int(datetime(int(year) + 1, 1, 1).timestamp())
return start, end
def _stable_seed(account_name: str, year: int) -> int:
s = f"{str(account_name or '').strip()}|{int(year)}|wrapped_keywords"
h = hashlib.sha256(s.encode("utf-8")).hexdigest()
return int(h[:8], 16)
def _list_message_tables(conn: sqlite3.Connection) -> list[str]:
try:
rows = conn.execute("SELECT name FROM sqlite_master WHERE type='table'").fetchall()
except Exception:
return []
names: list[str] = []
for r in rows:
if not r or not r[0]:
continue
name = _decode_sqlite_text(r[0]).strip()
if not name:
continue
ln = name.lower()
if ln.startswith(("msg_", "chat_")):
names.append(name)
return names
def _clean_text(text: str) -> str:
s = str(text or "")
if not s:
return ""
s = s.replace("\u200b", "").replace("\ufeff", "")
s = _CTRL_RE.sub("", s)
s = _URL_RE.sub("", s)
s = re.sub(r"\s+", " ", s).strip()
if not s:
return ""
# XML-like payloads are rarely useful as bubbles/keywords.
if s.startswith("<") or s.startswith('"<'):
return ""
return s
def _is_good_bubble_text(text: str) -> bool:
s = _clean_text(text)
if not s:
return False
# 仅过滤极短噪声,不对消息长度设置上限。
if len(s) < 2:
return False
if _URL_RE.search(s):
return False
if _MD5_HEX_RE.fullmatch(s.replace(" ", "")):
return False
# Avoid pure punctuation / emoji / digits.
if not re.search(r"[\u4e00-\u9fffA-Za-z]", s):
return False
if not _HAS_ALNUM_RE.search(s):
return False
if re.fullmatch(r"[0-9]+", s):
return False
return True
def _is_good_example_text(text: str) -> bool:
s = _clean_text(text)
if not s:
return False
# 仅过滤极短噪声,不对消息长度设置上限。
if len(s) < 4:
return False
if _URL_RE.search(s):
return False
if _MD5_HEX_RE.search(s):
return False
if not re.search(r"[\u4e00-\u9fffA-Za-z]", s):
return False
return True
def _normalize_token(tok: str) -> str:
s = str(tok or "").strip()
if not s:
return ""
if len(s) > 32:
return ""
# Trim punctuation on both sides.
s = re.sub(r"^[^\w\u4e00-\u9fff]+|[^\w\u4e00-\u9fff]+$", "", s, flags=re.UNICODE).strip()
if not s:
return ""
if _MD5_HEX_RE.fullmatch(s) or _MD5_HEX_RE.search(s):
return ""
if _DATEISH_RE.fullmatch(s):
return ""
# Discard if contains obvious long ids (alnum with many digits).
if len(s) >= 18 and re.fullmatch(r"[A-Za-z0-9_-]+", s) and sum(ch.isdigit() for ch in s) >= 6:
return ""
# Remove tokens with digits.
if any(ch.isdigit() for ch in s):
return ""
has_cjk = bool(_HAS_CJK_RE.search(s))
if has_cjk:
if not (2 <= len(s) <= 8):
return ""
if s in _STOPWORDS_ZH:
return ""
return s
if _EN_WORD_RE.fullmatch(s):
low = s.lower()
if low in _STOPWORDS_EN:
return ""
return low
return ""
def extract_keywords_jieba(texts: list[str], *, top_n: int = 40) -> list[dict[str, Any]]:
counter: Counter[str] = Counter()
for raw in texts:
s = _clean_text(raw)
if not s:
continue
try:
toks = jieba.lcut(s, cut_all=False)
except Exception:
toks = []
had_token = False
for tok in toks:
w = _normalize_token(tok)
if not w:
continue
counter[w] += 1
had_token = True
# Fallback for short chat phrases that Jieba often splits into single characters
# (e.g. "在吗" -> ["在","吗"]) which we intentionally filter out.
if not had_token and _HAS_CJK_RE.search(s):
for seg in _CJK_SEQ_RE.findall(s):
if len(seg) < 2:
continue
for i in range(0, len(seg) - 1):
w = _normalize_token(seg[i : i + 2])
if not w:
continue
counter[w] += 1
if not counter:
return []
items = [(w, int(c)) for w, c in counter.items() if int(c) > 1]
if not items:
# If everything is singleton, still provide something.
items = [(w, int(c)) for w, c in counter.items() if int(c) > 0]
items.sort(key=lambda kv: (-kv[1], kv[0]))
items = items[: max(0, int(top_n or 0))]
if not items:
return []
vals = [math.sqrt(max(0, c)) for _, c in items]
minv = min(vals) if vals else 0.0
maxv = max(vals) if vals else 0.0
out: list[dict[str, Any]] = []
for (w, c), v in zip(items, vals):
if maxv <= minv:
weight = 1.0
else:
weight = 0.2 + 0.8 * ((v - minv) / (maxv - minv))
out.append({"word": w, "count": int(c), "weight": round(float(weight), 4)})
return out
def pick_examples(
keywords: list[dict[str, Any]],
message_pool: list[str],
*,
per_word: int = 3,
) -> list[dict[str, Any]]:
uniq_msgs = list(dict.fromkeys([_clean_text(x) for x in (message_pool or []) if _clean_text(x)]))
out: list[dict[str, Any]] = []
for kw in keywords:
word = str(kw.get("word") or "").strip()
if not word:
continue
count = int(kw.get("count") or 0)
hits: list[str] = []
if _HAS_CJK_RE.search(word):
for msg in uniq_msgs:
if len(hits) >= int(per_word):
break
if not _is_good_example_text(msg):
continue
if word in msg:
hits.append(msg)
else:
wlow = word.lower()
for msg in uniq_msgs:
if len(hits) >= int(per_word):
break
if not _is_good_example_text(msg):
continue
if wlow in msg.lower():
hits.append(msg)
out.append({"word": word, "count": int(count), "messages": hits})
return out
def build_keywords_payload(
*,
texts: list[str],
seed: int,
top_n: int = 40,
bubble_limit: int = 180,
examples_per_word: int = 3,
) -> dict[str, Any]:
_ = seed # 保留参数以兼容现有调用/测试;随机采样不再使用固定 seed。
keywords = extract_keywords_jieba(list(texts or []), top_n=top_n)
bubble_candidates = [_clean_text(x) for x in (texts or [])]
bubble_candidates = [x for x in bubble_candidates if _is_good_bubble_text(x)]
bubble_candidates = list(dict.fromkeys(bubble_candidates))
rnd = random.SystemRandom()
rnd.shuffle(bubble_candidates)
bubble_messages = bubble_candidates[: max(0, int(bubble_limit or 0))]
examples = pick_examples(keywords, texts, per_word=examples_per_word)
top_kw = None
if keywords:
top_kw = {"word": str(keywords[0]["word"]), "count": int(keywords[0]["count"])}
return {
"topKeyword": top_kw,
"keywords": keywords,
"bubbleMessages": bubble_messages,
"examples": examples,
}
def _scan_message_pool(
*,
account_dir: Path,
year: int,
outgoing_only: bool,
seed: int,
max_pool: int = 3000,
max_seen: int = 120_000,
) -> tuple[list[str], dict[str, Any]]:
start_ts, end_ts = _year_range_epoch_seconds(int(year))
_ = seed # 保留参数以兼容现有调用;抽样本身使用非确定性随机。
rnd = random.SystemRandom()
db_paths = _iter_message_db_paths(account_dir)
# Prefer chat shards; biz_message often contains service/ads content.
db_paths = [p for p in db_paths if not p.name.lower().startswith("biz_message")]
rnd.shuffle(db_paths)
pool: list[str] = []
seen = 0
t0 = time.time()
for db_path in db_paths:
if not db_path.exists():
continue
conn: sqlite3.Connection | None = None
try:
conn = sqlite3.connect(str(db_path))
conn.row_factory = sqlite3.Row
conn.text_factory = bytes
my_rowid: int | None = None
if outgoing_only:
try:
r = conn.execute(
"SELECT rowid FROM Name2Id WHERE user_name = ? LIMIT 1",
(str(account_dir.name),),
).fetchone()
if r is not None and r[0] is not None:
my_rowid = int(r[0])
except Exception:
my_rowid = None
if my_rowid is None:
continue
tables = _list_message_tables(conn)
if not tables:
continue
rnd.shuffle(tables)
ts_expr = (
"CASE "
"WHEN CAST(create_time AS INTEGER) > 1000000000000 "
"THEN CAST(CAST(create_time AS INTEGER)/1000 AS INTEGER) "
"ELSE CAST(create_time AS INTEGER) "
"END"
)
for table in tables:
if seen >= int(max_seen):
break
qt = _quote_ident(table)
where_sender = ""
params: tuple[Any, ...]
if outgoing_only and my_rowid is not None:
where_sender = " AND CAST(real_sender_id AS INTEGER) = ?"
params = (start_ts, end_ts, int(my_rowid))
else:
params = (start_ts, end_ts)
sql = (
"SELECT message_content, compress_content "
f"FROM {qt} "
"WHERE CAST(local_type AS INTEGER) = 1 "
f" AND {ts_expr} >= ? AND {ts_expr} < ?"
f"{where_sender}"
)
try:
cur = conn.execute(sql, params)
except Exception:
continue
for r in cur:
if seen >= int(max_seen):
break
raw_txt = ""
try:
raw_txt = _decode_message_content(r["compress_content"], r["message_content"]).strip()
except Exception:
raw_txt = ""
cleaned = _clean_text(raw_txt)
if not cleaned:
continue
seen += 1
if len(pool) < int(max_pool):
pool.append(cleaned)
continue
# Reservoir sampling over the accepted stream.
j = rnd.randrange(seen)
if j < int(max_pool):
pool[j] = cleaned
finally:
if conn is not None:
try:
conn.close()
except Exception:
pass
if seen >= int(max_seen):
break
elapsed = time.time() - t0
meta = {
"scannedMessages": int(seen),
"sampledMessages": int(len(pool)),
"sampleRate": round(float(len(pool)) / float(seen), 6) if seen > 0 else 0.0,
"elapsedSec": round(float(elapsed), 3),
}
return pool, meta
def build_card_05_keywords_wordcloud(*, account_dir: Path, year: int) -> dict[str, Any]:
title = "这一年,你把哪些词说了一遍又一遍?"
seed = _stable_seed(str(account_dir.name or ""), int(year))
pool, meta = _scan_message_pool(account_dir=account_dir, year=year, outgoing_only=True, seed=seed)
if len(pool) < 80:
pool, meta = _scan_message_pool(account_dir=account_dir, year=year, outgoing_only=False, seed=seed ^ 0x1234)
payload = build_keywords_payload(texts=pool, seed=seed)
logger.info(
"Wrapped card#6 keywords computed: account=%s year=%s keywords=%s bubble=%s scanned=%s sampled=%s elapsed=%.2fs",
str(account_dir.name or "").strip(),
int(year),
len(payload.get("keywords") or []),
len(payload.get("bubbleMessages") or []),
int(meta.get("scannedMessages") or 0),
int(meta.get("sampledMessages") or 0),
float(meta.get("elapsedSec") or 0.0),
)
return {
"id": 6,
"title": title,
"scope": "global",
"category": "C",
"status": "ok",
"kind": "text/keywords_wordcloud",
"narrative": "你的年度关键词词云",
"data": {
"year": int(year),
**payload,
"meta": {
"scannedMessages": int(meta.get("scannedMessages") or 0),
"sampledMessages": int(meta.get("sampledMessages") or 0),
"sampleRate": float(meta.get("sampleRate") or 0.0),
},
},
}
+39 -13
View File
@@ -8,13 +8,14 @@ from datetime import datetime
from pathlib import Path
from typing import Any, Optional
from ..chat_helpers import _iter_message_db_paths, _quote_ident, _resolve_account_dir
from ..chat_helpers import _decode_sqlite_text, _iter_message_db_paths, _quote_ident, _resolve_account_dir
from ..chat_search_index import get_chat_search_index_db_path
from ..logging_config import get_logger
from .storage import wrapped_cache_dir, wrapped_cache_path
from .cards.card_00_global_overview import build_card_00_global_overview
from .cards.card_01_cyber_schedule import WeekdayHourHeatmap, build_card_01_cyber_schedule, compute_weekday_hour_heatmap
from .cards.card_02_message_chars import build_card_02_message_chars
from .cards.card_05_keywords_wordcloud import build_card_05_keywords_wordcloud
from .cards.card_03_reply_speed import build_card_03_reply_speed
from .cards.card_04_monthly_best_friends_wall import build_card_04_monthly_best_friends_wall
from .cards.card_04_emoji_universe import build_card_04_emoji_universe
@@ -24,9 +25,9 @@ logger = get_logger(__name__)
# We use this number to version the cache filename so adding more cards won't accidentally serve
# an older partial cache.
_IMPLEMENTED_UPTO_ID = 5
_IMPLEMENTED_UPTO_ID = 6
# Bump this when we change card payloads/ordering while keeping the same implemented_upto.
_CACHE_VERSION = 18
_CACHE_VERSION = 23
# "Manifest" is used by the frontend to render the deck quickly, then lazily fetch each card.
@@ -53,6 +54,13 @@ _WRAPPED_CARD_MANIFEST: tuple[dict[str, Any], ...] = (
"category": "C",
"kind": "text/message_chars",
},
{
"id": 6,
"title": "这一年,你把哪些词说了一遍又一遍?",
"scope": "global",
"category": "C",
"kind": "text/keywords_wordcloud",
},
{
"id": 3,
"title": "谁是你「秒回」的置顶关心?",
@@ -105,7 +113,9 @@ def _list_message_tables(conn: sqlite3.Connection) -> list[str]:
for r in rows:
if not r or not r[0]:
continue
name = str(r[0])
name = _decode_sqlite_text(r[0]).strip()
if not name:
continue
ln = name.lower()
if ln.startswith(("msg_", "chat_")):
names.append(name)
@@ -290,7 +300,7 @@ def build_wrapped_annual_response(
) -> dict[str, Any]:
"""Build annual wrapped response for the given account/year.
For now we implement cards up to id=5 (plus a meta overview card id=0).
For now we implement cards up to id=6 (plus a meta overview card id=0).
"""
account_dir = _resolve_account_dir(account)
@@ -315,6 +325,15 @@ def build_wrapped_annual_response(
try:
cached_obj = json.loads(cache_path.read_text(encoding="utf-8"))
if isinstance(cached_obj, dict) and isinstance(cached_obj.get("cards"), list):
# Card#6(关键词词云)要求每次请求返回随机消息批次,不复用旧卡片内容。
for idx, c in enumerate(cached_obj.get("cards") or []):
try:
if int((c or {}).get("id") or -1) != 6:
continue
except Exception:
continue
cached_obj["cards"][idx] = build_card_05_keywords_wordcloud(account_dir=account_dir, year=y)
break
cached_obj["cached"] = True
cached_obj["availableYears"] = available_years
return cached_obj
@@ -331,11 +350,13 @@ def build_wrapped_annual_response(
cards.append(build_card_01_cyber_schedule(account_dir=account_dir, year=y, heatmap=heatmap_sent))
# Page 4: message char counts (sent vs received).
cards.append(build_card_02_message_chars(account_dir=account_dir, year=y))
# Page 5: reply speed / best chat buddy.
# Page 5: annual keywords (bubble storm -> word cloud).
cards.append(build_card_05_keywords_wordcloud(account_dir=account_dir, year=y))
# Page 6: reply speed / best chat buddy.
cards.append(build_card_03_reply_speed(account_dir=account_dir, year=y))
# Page 6: monthly best friends wall (photo wall).
# Page 7: monthly best friends wall (photo wall).
cards.append(build_card_04_monthly_best_friends_wall(account_dir=account_dir, year=y))
# Page 7: annual emoji universe / meme almanac.
# Page 8: annual emoji universe / meme almanac.
cards.append(build_card_04_emoji_universe(account_dir=account_dir, year=y))
obj: dict[str, Any] = {
@@ -505,10 +526,12 @@ def build_wrapped_annual_card(
scope = "global"
cache_path = _wrapped_card_cache_path(account_dir=account_dir, scope=scope, year=y, card_id=cid)
# Card#6 需要每次随机抽样,不使用按卡片缓存。
cacheable = cid != 6
lock = _get_lock(str(cache_path))
with lock:
if (not refresh) and cache_path.exists():
if cacheable and (not refresh) and cache_path.exists():
try:
cached_obj = json.loads(cache_path.read_text(encoding="utf-8"))
if isinstance(cached_obj, dict) and int(cached_obj.get("id") or -1) == cid:
@@ -526,6 +549,8 @@ def build_wrapped_annual_card(
card = build_card_01_cyber_schedule(account_dir=account_dir, year=y, heatmap=heatmap_sent)
elif cid == 2:
card = build_card_02_message_chars(account_dir=account_dir, year=y)
elif cid == 6:
card = build_card_05_keywords_wordcloud(account_dir=account_dir, year=y)
elif cid == 3:
card = build_card_03_reply_speed(account_dir=account_dir, year=y)
elif cid == 4:
@@ -536,9 +561,10 @@ def build_wrapped_annual_card(
# Should be unreachable due to _WRAPPED_CARD_ID_SET check.
raise ValueError(f"Unknown Wrapped card id: {cid}")
try:
cache_path.write_text(json.dumps(card, ensure_ascii=False, indent=2), encoding="utf-8")
except Exception:
logger.exception("Failed to write wrapped card cache: %s", cache_path)
if cacheable:
try:
cache_path.write_text(json.dumps(card, ensure_ascii=False, indent=2), encoding="utf-8")
except Exception:
logger.exception("Failed to write wrapped card cache: %s", cache_path)
return card