mirror of
https://github.com/LifeArchiveProject/WeChatDataAnalysis.git
synced 2026-06-18 15:54:08 +08:00
feat(wrapped): 新增年度关键词词云卡片(Card #6)
- 新增关键词词云后端计算:消息采样、jieba 分词、关键词权重与例句回看数据。 - Wrapped 年度卡片接入 Card #6,并对该卡片改为按请求重算(不走单卡缓存)。 - 前端新增 storm->cloud 词云动效卡片,支持词点击查看例句及动画阶段隐藏 deck 顶部 UI。 - 统一 wrapped 消息表名解码逻辑,兼容 sqlite text_factory=bytes。 - 新增关键词词云测试并加入 jieba 依赖。
This commit is contained in:
@@ -77,7 +77,9 @@ def _list_message_tables(conn: sqlite3.Connection) -> list[str]:
|
||||
for r in rows:
|
||||
if not r or not r[0]:
|
||||
continue
|
||||
name = str(r[0])
|
||||
name = _decode_sqlite_text(r[0]).strip()
|
||||
if not name:
|
||||
continue
|
||||
ln = name.lower()
|
||||
if ln.startswith(("msg_", "chat_")):
|
||||
names.append(name)
|
||||
|
||||
@@ -12,6 +12,7 @@ from typing import Any, Optional
|
||||
from ...chat_search_index import get_chat_search_index_db_path
|
||||
from ...chat_helpers import (
|
||||
_build_avatar_url,
|
||||
_decode_sqlite_text,
|
||||
_iter_message_db_paths,
|
||||
_load_contact_rows,
|
||||
_pick_display_name,
|
||||
@@ -745,7 +746,9 @@ def _list_message_tables(conn: sqlite3.Connection) -> list[str]:
|
||||
for r in rows:
|
||||
if not r or not r[0]:
|
||||
continue
|
||||
name = str(r[0])
|
||||
name = _decode_sqlite_text(r[0]).strip()
|
||||
if not name:
|
||||
continue
|
||||
ln = name.lower()
|
||||
if ln.startswith(("msg_", "chat_")):
|
||||
names.append(name)
|
||||
|
||||
@@ -11,7 +11,7 @@ from typing import Any, Optional
|
||||
|
||||
from pypinyin import lazy_pinyin, Style
|
||||
|
||||
from ...chat_helpers import _decode_message_content, _iter_message_db_paths, _quote_ident
|
||||
from ...chat_helpers import _decode_message_content, _decode_sqlite_text, _iter_message_db_paths, _quote_ident
|
||||
from ...chat_search_index import get_chat_search_index_db_path
|
||||
from ...logging_config import get_logger
|
||||
|
||||
@@ -467,7 +467,9 @@ def _list_message_tables(conn: sqlite3.Connection) -> list[str]:
|
||||
for r in rows:
|
||||
if not r or not r[0]:
|
||||
continue
|
||||
name = str(r[0])
|
||||
name = _decode_sqlite_text(r[0]).strip()
|
||||
if not name:
|
||||
continue
|
||||
ln = name.lower()
|
||||
if ln.startswith(("msg_", "chat_")):
|
||||
names.append(name)
|
||||
|
||||
@@ -0,0 +1,572 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import math
|
||||
import random
|
||||
import re
|
||||
import sqlite3
|
||||
import time
|
||||
from collections import Counter
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import jieba
|
||||
|
||||
from ...chat_helpers import _decode_message_content, _decode_sqlite_text, _iter_message_db_paths, _quote_ident
|
||||
from ...logging_config import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
try:
|
||||
jieba.setLogLevel(logging.ERROR)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
_MD5_HEX_RE = re.compile(r"(?i)\b[0-9a-f]{32}\b")
|
||||
_URL_RE = re.compile(r"(?i)\bhttps?://\S+")
|
||||
_CTRL_RE = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f]")
|
||||
_HAS_CJK_RE = re.compile(r"[\u4e00-\u9fff]")
|
||||
_CJK_SEQ_RE = re.compile(r"[\u4e00-\u9fff]+")
|
||||
_HAS_ALNUM_RE = re.compile(r"[\u4e00-\u9fffA-Za-z0-9]")
|
||||
_EN_WORD_RE = re.compile(r"^[A-Za-z]{3,16}$")
|
||||
_DATEISH_RE = re.compile(
|
||||
r"^(?:"
|
||||
r"\d{4}[-/]\d{1,2}[-/]\d{1,2}"
|
||||
r"|"
|
||||
r"\d{1,2}:\d{2}"
|
||||
r"|"
|
||||
r"\d{1,2}月\d{1,2}日"
|
||||
r")$"
|
||||
)
|
||||
|
||||
# Small but practical stopword list for chat keywords.
|
||||
_STOPWORDS_ZH = {
|
||||
"的",
|
||||
"了",
|
||||
"是",
|
||||
"我",
|
||||
"你",
|
||||
"他",
|
||||
"她",
|
||||
"它",
|
||||
"我们",
|
||||
"你们",
|
||||
"他们",
|
||||
"她们",
|
||||
"它们",
|
||||
"这",
|
||||
"那",
|
||||
"这个",
|
||||
"那个",
|
||||
"这里",
|
||||
"那里",
|
||||
"这样",
|
||||
"那样",
|
||||
"就是",
|
||||
"也是",
|
||||
"还有",
|
||||
"因为",
|
||||
"所以",
|
||||
"但是",
|
||||
"如果",
|
||||
"然后",
|
||||
"已经",
|
||||
"可以",
|
||||
"还是",
|
||||
"可能",
|
||||
"不会",
|
||||
"没有",
|
||||
"不是",
|
||||
"一个",
|
||||
"一下",
|
||||
"一下子",
|
||||
"一下下",
|
||||
"哈哈",
|
||||
"哈哈哈",
|
||||
"嘿嘿",
|
||||
"呜呜",
|
||||
"嗯",
|
||||
"哦",
|
||||
"啊",
|
||||
"呀",
|
||||
"啦",
|
||||
"嘛",
|
||||
"呢",
|
||||
"吧",
|
||||
"额",
|
||||
"诶",
|
||||
"哇",
|
||||
"唉",
|
||||
"好",
|
||||
"行",
|
||||
"可以",
|
||||
"ok",
|
||||
"OK",
|
||||
}
|
||||
|
||||
_STOPWORDS_EN = {
|
||||
"the",
|
||||
"a",
|
||||
"an",
|
||||
"and",
|
||||
"or",
|
||||
"but",
|
||||
"to",
|
||||
"of",
|
||||
"in",
|
||||
"on",
|
||||
"for",
|
||||
"with",
|
||||
"at",
|
||||
"from",
|
||||
"as",
|
||||
"is",
|
||||
"are",
|
||||
"was",
|
||||
"were",
|
||||
"be",
|
||||
"been",
|
||||
"being",
|
||||
"i",
|
||||
"me",
|
||||
"my",
|
||||
"you",
|
||||
"your",
|
||||
"he",
|
||||
"she",
|
||||
"it",
|
||||
"we",
|
||||
"they",
|
||||
"them",
|
||||
"this",
|
||||
"that",
|
||||
"these",
|
||||
"those",
|
||||
"yeah",
|
||||
"haha",
|
||||
"ok",
|
||||
"okay",
|
||||
"pls",
|
||||
"lol",
|
||||
}
|
||||
|
||||
|
||||
def _year_range_epoch_seconds(year: int) -> tuple[int, int]:
|
||||
start = int(datetime(int(year), 1, 1).timestamp())
|
||||
end = int(datetime(int(year) + 1, 1, 1).timestamp())
|
||||
return start, end
|
||||
|
||||
|
||||
def _stable_seed(account_name: str, year: int) -> int:
|
||||
s = f"{str(account_name or '').strip()}|{int(year)}|wrapped_keywords"
|
||||
h = hashlib.sha256(s.encode("utf-8")).hexdigest()
|
||||
return int(h[:8], 16)
|
||||
|
||||
|
||||
def _list_message_tables(conn: sqlite3.Connection) -> list[str]:
|
||||
try:
|
||||
rows = conn.execute("SELECT name FROM sqlite_master WHERE type='table'").fetchall()
|
||||
except Exception:
|
||||
return []
|
||||
names: list[str] = []
|
||||
for r in rows:
|
||||
if not r or not r[0]:
|
||||
continue
|
||||
name = _decode_sqlite_text(r[0]).strip()
|
||||
if not name:
|
||||
continue
|
||||
ln = name.lower()
|
||||
if ln.startswith(("msg_", "chat_")):
|
||||
names.append(name)
|
||||
return names
|
||||
|
||||
|
||||
def _clean_text(text: str) -> str:
|
||||
s = str(text or "")
|
||||
if not s:
|
||||
return ""
|
||||
s = s.replace("\u200b", "").replace("\ufeff", "")
|
||||
s = _CTRL_RE.sub("", s)
|
||||
s = _URL_RE.sub("", s)
|
||||
s = re.sub(r"\s+", " ", s).strip()
|
||||
if not s:
|
||||
return ""
|
||||
# XML-like payloads are rarely useful as bubbles/keywords.
|
||||
if s.startswith("<") or s.startswith('"<'):
|
||||
return ""
|
||||
return s
|
||||
|
||||
|
||||
def _is_good_bubble_text(text: str) -> bool:
|
||||
s = _clean_text(text)
|
||||
if not s:
|
||||
return False
|
||||
# 仅过滤极短噪声,不对消息长度设置上限。
|
||||
if len(s) < 2:
|
||||
return False
|
||||
if _URL_RE.search(s):
|
||||
return False
|
||||
if _MD5_HEX_RE.fullmatch(s.replace(" ", "")):
|
||||
return False
|
||||
# Avoid pure punctuation / emoji / digits.
|
||||
if not re.search(r"[\u4e00-\u9fffA-Za-z]", s):
|
||||
return False
|
||||
if not _HAS_ALNUM_RE.search(s):
|
||||
return False
|
||||
if re.fullmatch(r"[0-9]+", s):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _is_good_example_text(text: str) -> bool:
|
||||
s = _clean_text(text)
|
||||
if not s:
|
||||
return False
|
||||
# 仅过滤极短噪声,不对消息长度设置上限。
|
||||
if len(s) < 4:
|
||||
return False
|
||||
if _URL_RE.search(s):
|
||||
return False
|
||||
if _MD5_HEX_RE.search(s):
|
||||
return False
|
||||
if not re.search(r"[\u4e00-\u9fffA-Za-z]", s):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _normalize_token(tok: str) -> str:
|
||||
s = str(tok or "").strip()
|
||||
if not s:
|
||||
return ""
|
||||
if len(s) > 32:
|
||||
return ""
|
||||
|
||||
# Trim punctuation on both sides.
|
||||
s = re.sub(r"^[^\w\u4e00-\u9fff]+|[^\w\u4e00-\u9fff]+$", "", s, flags=re.UNICODE).strip()
|
||||
if not s:
|
||||
return ""
|
||||
|
||||
if _MD5_HEX_RE.fullmatch(s) or _MD5_HEX_RE.search(s):
|
||||
return ""
|
||||
if _DATEISH_RE.fullmatch(s):
|
||||
return ""
|
||||
|
||||
# Discard if contains obvious long ids (alnum with many digits).
|
||||
if len(s) >= 18 and re.fullmatch(r"[A-Za-z0-9_-]+", s) and sum(ch.isdigit() for ch in s) >= 6:
|
||||
return ""
|
||||
|
||||
# Remove tokens with digits.
|
||||
if any(ch.isdigit() for ch in s):
|
||||
return ""
|
||||
|
||||
has_cjk = bool(_HAS_CJK_RE.search(s))
|
||||
if has_cjk:
|
||||
if not (2 <= len(s) <= 8):
|
||||
return ""
|
||||
if s in _STOPWORDS_ZH:
|
||||
return ""
|
||||
return s
|
||||
|
||||
if _EN_WORD_RE.fullmatch(s):
|
||||
low = s.lower()
|
||||
if low in _STOPWORDS_EN:
|
||||
return ""
|
||||
return low
|
||||
|
||||
return ""
|
||||
|
||||
|
||||
def extract_keywords_jieba(texts: list[str], *, top_n: int = 40) -> list[dict[str, Any]]:
|
||||
counter: Counter[str] = Counter()
|
||||
for raw in texts:
|
||||
s = _clean_text(raw)
|
||||
if not s:
|
||||
continue
|
||||
try:
|
||||
toks = jieba.lcut(s, cut_all=False)
|
||||
except Exception:
|
||||
toks = []
|
||||
had_token = False
|
||||
for tok in toks:
|
||||
w = _normalize_token(tok)
|
||||
if not w:
|
||||
continue
|
||||
counter[w] += 1
|
||||
had_token = True
|
||||
|
||||
# Fallback for short chat phrases that Jieba often splits into single characters
|
||||
# (e.g. "在吗" -> ["在","吗"]) which we intentionally filter out.
|
||||
if not had_token and _HAS_CJK_RE.search(s):
|
||||
for seg in _CJK_SEQ_RE.findall(s):
|
||||
if len(seg) < 2:
|
||||
continue
|
||||
for i in range(0, len(seg) - 1):
|
||||
w = _normalize_token(seg[i : i + 2])
|
||||
if not w:
|
||||
continue
|
||||
counter[w] += 1
|
||||
|
||||
if not counter:
|
||||
return []
|
||||
|
||||
items = [(w, int(c)) for w, c in counter.items() if int(c) > 1]
|
||||
if not items:
|
||||
# If everything is singleton, still provide something.
|
||||
items = [(w, int(c)) for w, c in counter.items() if int(c) > 0]
|
||||
|
||||
items.sort(key=lambda kv: (-kv[1], kv[0]))
|
||||
items = items[: max(0, int(top_n or 0))]
|
||||
if not items:
|
||||
return []
|
||||
|
||||
vals = [math.sqrt(max(0, c)) for _, c in items]
|
||||
minv = min(vals) if vals else 0.0
|
||||
maxv = max(vals) if vals else 0.0
|
||||
|
||||
out: list[dict[str, Any]] = []
|
||||
for (w, c), v in zip(items, vals):
|
||||
if maxv <= minv:
|
||||
weight = 1.0
|
||||
else:
|
||||
weight = 0.2 + 0.8 * ((v - minv) / (maxv - minv))
|
||||
out.append({"word": w, "count": int(c), "weight": round(float(weight), 4)})
|
||||
return out
|
||||
|
||||
|
||||
def pick_examples(
|
||||
keywords: list[dict[str, Any]],
|
||||
message_pool: list[str],
|
||||
*,
|
||||
per_word: int = 3,
|
||||
) -> list[dict[str, Any]]:
|
||||
uniq_msgs = list(dict.fromkeys([_clean_text(x) for x in (message_pool or []) if _clean_text(x)]))
|
||||
out: list[dict[str, Any]] = []
|
||||
|
||||
for kw in keywords:
|
||||
word = str(kw.get("word") or "").strip()
|
||||
if not word:
|
||||
continue
|
||||
count = int(kw.get("count") or 0)
|
||||
|
||||
hits: list[str] = []
|
||||
if _HAS_CJK_RE.search(word):
|
||||
for msg in uniq_msgs:
|
||||
if len(hits) >= int(per_word):
|
||||
break
|
||||
if not _is_good_example_text(msg):
|
||||
continue
|
||||
if word in msg:
|
||||
hits.append(msg)
|
||||
else:
|
||||
wlow = word.lower()
|
||||
for msg in uniq_msgs:
|
||||
if len(hits) >= int(per_word):
|
||||
break
|
||||
if not _is_good_example_text(msg):
|
||||
continue
|
||||
if wlow in msg.lower():
|
||||
hits.append(msg)
|
||||
|
||||
out.append({"word": word, "count": int(count), "messages": hits})
|
||||
|
||||
return out
|
||||
|
||||
|
||||
def build_keywords_payload(
|
||||
*,
|
||||
texts: list[str],
|
||||
seed: int,
|
||||
top_n: int = 40,
|
||||
bubble_limit: int = 180,
|
||||
examples_per_word: int = 3,
|
||||
) -> dict[str, Any]:
|
||||
_ = seed # 保留参数以兼容现有调用/测试;随机采样不再使用固定 seed。
|
||||
keywords = extract_keywords_jieba(list(texts or []), top_n=top_n)
|
||||
|
||||
bubble_candidates = [_clean_text(x) for x in (texts or [])]
|
||||
bubble_candidates = [x for x in bubble_candidates if _is_good_bubble_text(x)]
|
||||
bubble_candidates = list(dict.fromkeys(bubble_candidates))
|
||||
|
||||
rnd = random.SystemRandom()
|
||||
rnd.shuffle(bubble_candidates)
|
||||
bubble_messages = bubble_candidates[: max(0, int(bubble_limit or 0))]
|
||||
|
||||
examples = pick_examples(keywords, texts, per_word=examples_per_word)
|
||||
|
||||
top_kw = None
|
||||
if keywords:
|
||||
top_kw = {"word": str(keywords[0]["word"]), "count": int(keywords[0]["count"])}
|
||||
|
||||
return {
|
||||
"topKeyword": top_kw,
|
||||
"keywords": keywords,
|
||||
"bubbleMessages": bubble_messages,
|
||||
"examples": examples,
|
||||
}
|
||||
|
||||
|
||||
def _scan_message_pool(
|
||||
*,
|
||||
account_dir: Path,
|
||||
year: int,
|
||||
outgoing_only: bool,
|
||||
seed: int,
|
||||
max_pool: int = 3000,
|
||||
max_seen: int = 120_000,
|
||||
) -> tuple[list[str], dict[str, Any]]:
|
||||
start_ts, end_ts = _year_range_epoch_seconds(int(year))
|
||||
_ = seed # 保留参数以兼容现有调用;抽样本身使用非确定性随机。
|
||||
rnd = random.SystemRandom()
|
||||
|
||||
db_paths = _iter_message_db_paths(account_dir)
|
||||
# Prefer chat shards; biz_message often contains service/ads content.
|
||||
db_paths = [p for p in db_paths if not p.name.lower().startswith("biz_message")]
|
||||
rnd.shuffle(db_paths)
|
||||
|
||||
pool: list[str] = []
|
||||
seen = 0
|
||||
|
||||
t0 = time.time()
|
||||
for db_path in db_paths:
|
||||
if not db_path.exists():
|
||||
continue
|
||||
|
||||
conn: sqlite3.Connection | None = None
|
||||
try:
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.text_factory = bytes
|
||||
|
||||
my_rowid: int | None = None
|
||||
if outgoing_only:
|
||||
try:
|
||||
r = conn.execute(
|
||||
"SELECT rowid FROM Name2Id WHERE user_name = ? LIMIT 1",
|
||||
(str(account_dir.name),),
|
||||
).fetchone()
|
||||
if r is not None and r[0] is not None:
|
||||
my_rowid = int(r[0])
|
||||
except Exception:
|
||||
my_rowid = None
|
||||
if my_rowid is None:
|
||||
continue
|
||||
|
||||
tables = _list_message_tables(conn)
|
||||
if not tables:
|
||||
continue
|
||||
rnd.shuffle(tables)
|
||||
|
||||
ts_expr = (
|
||||
"CASE "
|
||||
"WHEN CAST(create_time AS INTEGER) > 1000000000000 "
|
||||
"THEN CAST(CAST(create_time AS INTEGER)/1000 AS INTEGER) "
|
||||
"ELSE CAST(create_time AS INTEGER) "
|
||||
"END"
|
||||
)
|
||||
|
||||
for table in tables:
|
||||
if seen >= int(max_seen):
|
||||
break
|
||||
qt = _quote_ident(table)
|
||||
where_sender = ""
|
||||
params: tuple[Any, ...]
|
||||
if outgoing_only and my_rowid is not None:
|
||||
where_sender = " AND CAST(real_sender_id AS INTEGER) = ?"
|
||||
params = (start_ts, end_ts, int(my_rowid))
|
||||
else:
|
||||
params = (start_ts, end_ts)
|
||||
sql = (
|
||||
"SELECT message_content, compress_content "
|
||||
f"FROM {qt} "
|
||||
"WHERE CAST(local_type AS INTEGER) = 1 "
|
||||
f" AND {ts_expr} >= ? AND {ts_expr} < ?"
|
||||
f"{where_sender}"
|
||||
)
|
||||
|
||||
try:
|
||||
cur = conn.execute(sql, params)
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
for r in cur:
|
||||
if seen >= int(max_seen):
|
||||
break
|
||||
raw_txt = ""
|
||||
try:
|
||||
raw_txt = _decode_message_content(r["compress_content"], r["message_content"]).strip()
|
||||
except Exception:
|
||||
raw_txt = ""
|
||||
cleaned = _clean_text(raw_txt)
|
||||
if not cleaned:
|
||||
continue
|
||||
seen += 1
|
||||
|
||||
if len(pool) < int(max_pool):
|
||||
pool.append(cleaned)
|
||||
continue
|
||||
|
||||
# Reservoir sampling over the accepted stream.
|
||||
j = rnd.randrange(seen)
|
||||
if j < int(max_pool):
|
||||
pool[j] = cleaned
|
||||
finally:
|
||||
if conn is not None:
|
||||
try:
|
||||
conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if seen >= int(max_seen):
|
||||
break
|
||||
|
||||
elapsed = time.time() - t0
|
||||
meta = {
|
||||
"scannedMessages": int(seen),
|
||||
"sampledMessages": int(len(pool)),
|
||||
"sampleRate": round(float(len(pool)) / float(seen), 6) if seen > 0 else 0.0,
|
||||
"elapsedSec": round(float(elapsed), 3),
|
||||
}
|
||||
return pool, meta
|
||||
|
||||
|
||||
def build_card_05_keywords_wordcloud(*, account_dir: Path, year: int) -> dict[str, Any]:
|
||||
title = "这一年,你把哪些词说了一遍又一遍?"
|
||||
seed = _stable_seed(str(account_dir.name or ""), int(year))
|
||||
|
||||
pool, meta = _scan_message_pool(account_dir=account_dir, year=year, outgoing_only=True, seed=seed)
|
||||
if len(pool) < 80:
|
||||
pool, meta = _scan_message_pool(account_dir=account_dir, year=year, outgoing_only=False, seed=seed ^ 0x1234)
|
||||
|
||||
payload = build_keywords_payload(texts=pool, seed=seed)
|
||||
|
||||
logger.info(
|
||||
"Wrapped card#6 keywords computed: account=%s year=%s keywords=%s bubble=%s scanned=%s sampled=%s elapsed=%.2fs",
|
||||
str(account_dir.name or "").strip(),
|
||||
int(year),
|
||||
len(payload.get("keywords") or []),
|
||||
len(payload.get("bubbleMessages") or []),
|
||||
int(meta.get("scannedMessages") or 0),
|
||||
int(meta.get("sampledMessages") or 0),
|
||||
float(meta.get("elapsedSec") or 0.0),
|
||||
)
|
||||
|
||||
return {
|
||||
"id": 6,
|
||||
"title": title,
|
||||
"scope": "global",
|
||||
"category": "C",
|
||||
"status": "ok",
|
||||
"kind": "text/keywords_wordcloud",
|
||||
"narrative": "你的年度关键词词云",
|
||||
"data": {
|
||||
"year": int(year),
|
||||
**payload,
|
||||
"meta": {
|
||||
"scannedMessages": int(meta.get("scannedMessages") or 0),
|
||||
"sampledMessages": int(meta.get("sampledMessages") or 0),
|
||||
"sampleRate": float(meta.get("sampleRate") or 0.0),
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -8,13 +8,14 @@ from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional
|
||||
|
||||
from ..chat_helpers import _iter_message_db_paths, _quote_ident, _resolve_account_dir
|
||||
from ..chat_helpers import _decode_sqlite_text, _iter_message_db_paths, _quote_ident, _resolve_account_dir
|
||||
from ..chat_search_index import get_chat_search_index_db_path
|
||||
from ..logging_config import get_logger
|
||||
from .storage import wrapped_cache_dir, wrapped_cache_path
|
||||
from .cards.card_00_global_overview import build_card_00_global_overview
|
||||
from .cards.card_01_cyber_schedule import WeekdayHourHeatmap, build_card_01_cyber_schedule, compute_weekday_hour_heatmap
|
||||
from .cards.card_02_message_chars import build_card_02_message_chars
|
||||
from .cards.card_05_keywords_wordcloud import build_card_05_keywords_wordcloud
|
||||
from .cards.card_03_reply_speed import build_card_03_reply_speed
|
||||
from .cards.card_04_monthly_best_friends_wall import build_card_04_monthly_best_friends_wall
|
||||
from .cards.card_04_emoji_universe import build_card_04_emoji_universe
|
||||
@@ -24,9 +25,9 @@ logger = get_logger(__name__)
|
||||
|
||||
# We use this number to version the cache filename so adding more cards won't accidentally serve
|
||||
# an older partial cache.
|
||||
_IMPLEMENTED_UPTO_ID = 5
|
||||
_IMPLEMENTED_UPTO_ID = 6
|
||||
# Bump this when we change card payloads/ordering while keeping the same implemented_upto.
|
||||
_CACHE_VERSION = 18
|
||||
_CACHE_VERSION = 23
|
||||
|
||||
|
||||
# "Manifest" is used by the frontend to render the deck quickly, then lazily fetch each card.
|
||||
@@ -53,6 +54,13 @@ _WRAPPED_CARD_MANIFEST: tuple[dict[str, Any], ...] = (
|
||||
"category": "C",
|
||||
"kind": "text/message_chars",
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"title": "这一年,你把哪些词说了一遍又一遍?",
|
||||
"scope": "global",
|
||||
"category": "C",
|
||||
"kind": "text/keywords_wordcloud",
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"title": "谁是你「秒回」的置顶关心?",
|
||||
@@ -105,7 +113,9 @@ def _list_message_tables(conn: sqlite3.Connection) -> list[str]:
|
||||
for r in rows:
|
||||
if not r or not r[0]:
|
||||
continue
|
||||
name = str(r[0])
|
||||
name = _decode_sqlite_text(r[0]).strip()
|
||||
if not name:
|
||||
continue
|
||||
ln = name.lower()
|
||||
if ln.startswith(("msg_", "chat_")):
|
||||
names.append(name)
|
||||
@@ -290,7 +300,7 @@ def build_wrapped_annual_response(
|
||||
) -> dict[str, Any]:
|
||||
"""Build annual wrapped response for the given account/year.
|
||||
|
||||
For now we implement cards up to id=5 (plus a meta overview card id=0).
|
||||
For now we implement cards up to id=6 (plus a meta overview card id=0).
|
||||
"""
|
||||
|
||||
account_dir = _resolve_account_dir(account)
|
||||
@@ -315,6 +325,15 @@ def build_wrapped_annual_response(
|
||||
try:
|
||||
cached_obj = json.loads(cache_path.read_text(encoding="utf-8"))
|
||||
if isinstance(cached_obj, dict) and isinstance(cached_obj.get("cards"), list):
|
||||
# Card#6(关键词词云)要求每次请求返回随机消息批次,不复用旧卡片内容。
|
||||
for idx, c in enumerate(cached_obj.get("cards") or []):
|
||||
try:
|
||||
if int((c or {}).get("id") or -1) != 6:
|
||||
continue
|
||||
except Exception:
|
||||
continue
|
||||
cached_obj["cards"][idx] = build_card_05_keywords_wordcloud(account_dir=account_dir, year=y)
|
||||
break
|
||||
cached_obj["cached"] = True
|
||||
cached_obj["availableYears"] = available_years
|
||||
return cached_obj
|
||||
@@ -331,11 +350,13 @@ def build_wrapped_annual_response(
|
||||
cards.append(build_card_01_cyber_schedule(account_dir=account_dir, year=y, heatmap=heatmap_sent))
|
||||
# Page 4: message char counts (sent vs received).
|
||||
cards.append(build_card_02_message_chars(account_dir=account_dir, year=y))
|
||||
# Page 5: reply speed / best chat buddy.
|
||||
# Page 5: annual keywords (bubble storm -> word cloud).
|
||||
cards.append(build_card_05_keywords_wordcloud(account_dir=account_dir, year=y))
|
||||
# Page 6: reply speed / best chat buddy.
|
||||
cards.append(build_card_03_reply_speed(account_dir=account_dir, year=y))
|
||||
# Page 6: monthly best friends wall (photo wall).
|
||||
# Page 7: monthly best friends wall (photo wall).
|
||||
cards.append(build_card_04_monthly_best_friends_wall(account_dir=account_dir, year=y))
|
||||
# Page 7: annual emoji universe / meme almanac.
|
||||
# Page 8: annual emoji universe / meme almanac.
|
||||
cards.append(build_card_04_emoji_universe(account_dir=account_dir, year=y))
|
||||
|
||||
obj: dict[str, Any] = {
|
||||
@@ -505,10 +526,12 @@ def build_wrapped_annual_card(
|
||||
|
||||
scope = "global"
|
||||
cache_path = _wrapped_card_cache_path(account_dir=account_dir, scope=scope, year=y, card_id=cid)
|
||||
# Card#6 需要每次随机抽样,不使用按卡片缓存。
|
||||
cacheable = cid != 6
|
||||
|
||||
lock = _get_lock(str(cache_path))
|
||||
with lock:
|
||||
if (not refresh) and cache_path.exists():
|
||||
if cacheable and (not refresh) and cache_path.exists():
|
||||
try:
|
||||
cached_obj = json.loads(cache_path.read_text(encoding="utf-8"))
|
||||
if isinstance(cached_obj, dict) and int(cached_obj.get("id") or -1) == cid:
|
||||
@@ -526,6 +549,8 @@ def build_wrapped_annual_card(
|
||||
card = build_card_01_cyber_schedule(account_dir=account_dir, year=y, heatmap=heatmap_sent)
|
||||
elif cid == 2:
|
||||
card = build_card_02_message_chars(account_dir=account_dir, year=y)
|
||||
elif cid == 6:
|
||||
card = build_card_05_keywords_wordcloud(account_dir=account_dir, year=y)
|
||||
elif cid == 3:
|
||||
card = build_card_03_reply_speed(account_dir=account_dir, year=y)
|
||||
elif cid == 4:
|
||||
@@ -536,9 +561,10 @@ def build_wrapped_annual_card(
|
||||
# Should be unreachable due to _WRAPPED_CARD_ID_SET check.
|
||||
raise ValueError(f"Unknown Wrapped card id: {cid}")
|
||||
|
||||
try:
|
||||
cache_path.write_text(json.dumps(card, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
except Exception:
|
||||
logger.exception("Failed to write wrapped card cache: %s", cache_path)
|
||||
if cacheable:
|
||||
try:
|
||||
cache_path.write_text(json.dumps(card, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
except Exception:
|
||||
logger.exception("Failed to write wrapped card cache: %s", cache_path)
|
||||
|
||||
return card
|
||||
|
||||
Reference in New Issue
Block a user