diff --git a/DIFF_FILE b/DIFF_FILE deleted file mode 100644 index 07ec68e..0000000 --- a/DIFF_FILE +++ /dev/null @@ -1,298 +0,0 @@ -diff --git a/core/huya/app_login.py b/core/huya/app_login.py -index afa33a4..86e18a9 100644 ---- a/core/huya/app_login.py -+++ b/core/huya/app_login.py -@@ -9,7 +9,8 @@ - 6. POST /web/cookie/verify 兑换获取全套网页 Cookie - 注册链不再重放旧 dfpReport 密文。登录帧的 32hex hdid 仍是服务端硬锚， --当前继续使用已注册样本；新注册链动态更新的是 safedeviceid 和 device_id。 -+登录帧中的 HDID32 是 app 版本级协议常量；二维码信封的 ACTION、设备字段、 -+UID、证书和会话元数据均在本次登录中动态生成或由注册链签发。 - 注册链（``core/huya/dfp_register``）每次登录前执行，失败即抛错终止（``HuyaAppLoginError``）， - 不读取画像里的旧固定值，也不静默回退旧链。 -@@ -35,7 +36,7 @@ from urllib.parse import parse_qs, quote, urlparse - import requests - from loguru import logger --from .cert_forge import build_p1, decrypt_cert, forge_cert, parse_p1 -+from .cert_forge import build_p1, forge_cert - from .cookie_utils import normalize_huya_cookie - from .device_fingerprint import account_state_dir, get_huya_sdid, reset_account_state - from .device_profile import get_profile, mobile_user_agent -@@ -270,7 +271,8 @@ def login_cred_with_flow( - max_rounds: int = 3, - device_info: dict | None = None, - proxies: dict | None = None, --) -> tuple[bytes, int]: -+ include_device_token: bool = False, -+) -> tuple[bytes, int] | tuple[bytes, int, str]: - """新注册设备后登录，返回 ``(新鲜 cred, 真实 uid)``。 - 注册只执行一次；safe_auth 通过后的重发继续使用同一组设备字段。 -@@ -305,6 +307,8 @@ def login_cred_with_flow( - ) - if cred: - uid = parse_real_uid(resp) -+ if include_device_token: -+ return cred, uid, safedeviceid - return cred, uid - if risk_url: - kind = ( -@@ -462,11 +466,12 @@ class HuyaAppPasswordLogin: - # 1) 获取新鲜 cred 与 真实 uid (自动过 safe_auth 滑块) - try: -- cred, uid = login_cred_with_flow( -+ cred, uid, safedeviceid = login_cred_with_flow( - acct, - self.password, - device_info=self.device_info, - proxies=self.proxies, -+ include_device_token=True, - ) - except HuyaAppQrAuthRequiredError as exc: - return HuyaLoginResult( -@@ -485,30 +490,27 @@ class HuyaAppPasswordLogin: - # 2) 本地生成 nonce 铸造证书 (P1 指纹与该账号设备画像一致) - try: -- env = Envelope.load() -- orig = base64.b64decode(env.cert_b64) -- f = parse_p1(decrypt_cert(orig)) - st = int(time.time() * 1000) - rnd = gen_nonce(uid, K1_DEFAULT, service_time_ms=st, counter=0) - fp_bytes = self.device_info["fingerprint"].encode("ascii") -- p1 = build_p1(f["app_id"], fp_bytes, cred, rnd=rnd) -- cert = base64.b64encode(forge_cert(p1, key_idx=orig[1])).decode() -- -- # 3) 信封补丁 -- raw = bytearray(env.raw) -- if env.cert_off is None or env.uid_off is None: -- raise ValueError("信封缺少证书或 uid 偏移") -- raw[env.cert_off : env.cert_off + env.cert_len] = cert.encode("ascii") -- if env.uid != uid: -- struct.pack_into(">Q", raw, env.uid_off, uid) -- # QR 信封只保留协议结构；不要重放抓包里的旧会话值。 -+ p1 = build_p1(b"5008", fp_bytes, cred, rnd=rnd) -+ # 证书头和 app_id 是协议版本常量，不从抓包信封读取。 -+ cert = base64.b64encode(forge_cert(p1, key_idx=0x20)).decode() -+ -+ # 3) 按当前账号画像和本次注册令牌重编码二维码信封。 - qr_session = random.randint(1_000_000, 9_999_999) - qr_trace = ( - f"{uuid.uuid4().hex[:16]}-{random.randint(10000, 99999)}-" - f"{time.time_ns():020d}" - ) -- env.raw = raw -- env.patch_session(qr_session).patch_meta(qr_session, qr_trace) -+ env = Envelope.build_qr( -+ uid=uid, -+ cert_b64=cert, -+ safedeviceid=safedeviceid, -+ session=qr_session, -+ trace_id=qr_trace, -+ device_info=self.device_info, -+ ) - wup = base64.b64encode(bytes(env.raw)).decode("ascii") - except Exception as exc: # noqa: BLE001 外部接口与任务边界需要保留宽泛异常兜底 - return HuyaLoginResult( -diff --git a/core/huya/envelope_forge.py b/core/huya/envelope_forge.py -index c5d66ab..0d2e8b1 100644 ---- a/core/huya/envelope_forge.py -+++ b/core/huya/envelope_forge.py -@@ -1,6 +1,7 @@ - """wupData 信封构造与补丁工具。 --解析与改写 WUP 信封中的 cert、uid、session 等字段。 -+生产二维码信封由 :meth:`Envelope.build_qr` 按当前账号状态编码； -+:meth:`Envelope.load` 仅保留协议结构模板兼容测试和离线分析。 - """ - from __future__ import annotations -@@ -10,7 +11,8 @@ import json - import struct - from pathlib import Path --from loguru import logger -+from .taf_protocol import TafOutputStream -+from .wup_protocol import WupRequest - INT8, INT16, INT32, INT64 = 0x00, 0x01, 0x02, 0x03 - STRING1, STRING4 = 0x06, 0x07 -@@ -119,24 +121,106 @@ class Envelope: - @classmethod - def load(cls, path: str | Path | None = None) -> Envelope: -- """加载信封模板，支持从文件加载或使用内嵌金样本。""" -+ """加载显式模板，或使用内嵌协议结构模板。 -+ -+ 不自动扫描 ``evidence/``，避免历史抓包成为隐式生产输入。 -+ """ - if path: - p = Path(path) - if p.exists(): - return cls._load_from_path(p) -- # 尝试查找 evidence/cert_keycap.json -- candidate = ( -- Path(__file__).resolve().parent.parent.parent -- / "evidence" -- / "cert_keycap.json" -- ) -- if candidate.exists(): -- try: -- return cls._load_from_path(candidate) -- except Exception as exc: # noqa: BLE001 -- logger.debug(f"加载证书信封候选文件失败: {candidate}: {exc}") - return cls(base64.b64decode(PROTOCOL_QURL_TEMPLATE_B64)) -+ @classmethod -+ def build_qr( -+ cls, -+ *, -+ uid: int, -+ cert_b64: str, -+ safedeviceid: str, -+ session: int, -+ trace_id: str, -+ device_info: dict[str, str], -+ ) -> Envelope: -+ """按当前账号状态编码二维码绑定信封。 -+ -+ 生产二维码请求不应从抓包信封复制字段。这里仅复用已知的 TAF/WUP -+ 字段布局；证书、ACTION、UID、会话和全部设备画像都由本次登录提供。 -+ """ -+ if len(cert_b64) != 260: -+ raise ValueError(f"证书 base64 长度异常: {len(cert_b64)} != 260") -+ if len(safedeviceid) != 180: -+ raise ValueError(f"safedeviceid 长度异常: {len(safedeviceid)} != 180") -+ dev = {k: str(v) for k, v in (device_info or {}).items()} -+ meta = json.dumps( -+ { -+ "associationId": 184549392, -+ "funcName": "", -+ "group": 0, -+ "id": 184549392, -+ "session": int(session), -+ "step": 0, -+ "stillLogin": False, -+ "traceId": str(trace_id), -+ "type": 2, -+ "uid": 0, -+ "userContext": "", -+ }, -+ ensure_ascii=False, -+ separators=(",", ":"), -+ ) -+ body = TafOutputStream() -+ body.write_struct_begin(0) -+ body.write_struct_begin(0) -+ body.write_int8(0, 0) -+ body.write_string(1, "1.0") -+ body.write_string(2, meta) -+ body.write_string(3, "5008") -+ body.write_int8(4, 3) -+ body.write_string(5, safedeviceid) -+ body.write_string(6, "") -+ body.write_string(7, "") -+ body.write_string(8, "") -+ body.write_string(9, "") -+ body.write_struct_end() -+ -+ body.write_struct_begin(1) -+ body.write_string(0, dev.get("hdid", "")) -+ body.write_string(1, dev.get("app_version", "13.4.22")) -+ body.write_string(2, dev.get("sdk_version", "1.0.80138")) -+ body.write_string(3, "") -+ body.write_string(4, dev.get("ip", "127.0.0.1")) -+ body.write_string(5, dev.get("vendor", "android")) -+ body.write_string(6, "") -+ body.write_struct_end() -+ -+ body.write_struct_begin(2) -+ body.write_int8(0, 1) -+ body.write_string(1, dev.get("model", "")) -+ body.write_string(2, dev.get("fingerprint", "")) -+ body.write_string(3, dev.get("os", "android")) -+ body.write_string(4, dev.get("screen", "")) -+ body.write_string(6, dev.get("width", "1080")) -+ body.write_string(7, dev.get("height", "2120")) -+ body.write_string(8, dev.get("device_id", "")) -+ body.write_struct_end() -+ body.write_uint64(3, int(uid)) -+ body.write_string(4, cert_b64) -+ body.write_string(5, "") -+ body.write_string(6, "") -+ body.write_struct_end() -+ -+ req = TafOutputStream() -+ req.write_int32(0, int(session)) -+ wup = WupRequest() -+ wup.iTimeout = 0 -+ wup.setRequestId(int(session)) -+ wup.setServant("huyaudbwebui") -+ wup.setFunc("default") -+ wup.newdata["_wup_data"] = body.get_bytes() -+ wup.newdata["wupudbrequest_v0"] = req.get_bytes() -+ return cls(wup.encode()) -+ - @classmethod - def _load_from_path(cls, p: Path) -> Envelope: - if p.suffix == ".json": -diff --git a/tests/test_huya_app_login.py b/tests/test_huya_app_login.py -index 3512a14..0c897bb 100644 ---- a/tests/test_huya_app_login.py -+++ b/tests/test_huya_app_login.py -@@ -26,6 +26,7 @@ from core.huya.login import HuyaLoginResult - from core.huya.nonce_forge import K1_DEFAULT, gen_nonce - from core.huya.udb_aes import udb_decrypt, udb_encrypt - from core.huya.wup_encoder import _make_name, build_password_login_wup -+from core.huya.wup_protocol import WupResponse - from web.backend.database import Base - from web.backend.models import HuyaAccount, User - from web.backend.routers.huya import ( -@@ -102,6 +103,38 @@ class TestHuyaAppLogin: - assert b'"session":7654321' in current - assert (b'"traceId":"' + new_trace.encode() + b'"') in current -+ def test_qr_envelope_is_fully_dynamic(self): -+ profile = { -+ "hdid": "h" * 32, -+ "app_version": "13.4.22", -+ "sdk_version": "1.0.80138", -+ "ip": "10.0.0.8", -+ "vendor": "vivo", -+ "model": "V2370A", -+ "fingerprint": "f" * 40, -+ "os": "android", -+ "screen": "V2370A,34,13", -+ "width": "1080", -+ "height": "2412", -+ "device_id": "d" * 40, -+ } -+ env = Envelope.build_qr( -+ uid=1199664135026, -+ cert_b64="A" * 260, -+ safedeviceid="B" * 180, -+ session=7654321, -+ trace_id="t" * 16 + "-12345-" + "9" * 20, -+ device_info=profile, -+ ) -+ wup = WupResponse() -+ wup.decode(bytes(env.raw)) -+ body = wup.newdata["_wup_data"] -+ for value in (b"B" * 180, b"h" * 32, b"f" * 40, b"V2370A", b"d" * 40): -+ assert value in body -+ assert b"PQwemAN9NHkZKoMq" not in body -+ assert b"02df398797432eadefcc12767119ad5e80999389" not in body -+ assert b"7c5387e0539c023c31c4ff0e807e7256117385ee" not in body -+ - def test_device_profile_generation(self): - p1 = generate_profile() - assert p1["os"] == "android" -@@ -271,10 +304,13 @@ class TestHuyaAppLogin: - patch("core.huya.app_login.solve_safe_auth", return_value={"authId": "id"}), - patch("core.huya.app_login.parse_real_uid", return_value=1199666914671), - ): -- cred, uid = login_cred_with_flow("300023887", "pw") -+ cred, uid, safedeviceid = login_cred_with_flow( -+ "300023887", "pw", include_device_token=True -+ ) - assert cred == b"c" * 114 - assert uid == 1199666914671 -+ assert safedeviceid == "A" * 180 - assert calls == [assets, assets] - def test_login_cred_flow_maps_invalid_password_response(self): - ---- integration round 2026-09-01 --- -web/backend/services/huya_runner_core.py: report missing WSS prerequisite names without exposing Cookie values. -tests/test_huya_runner_transport.py: cover minimal App Cookie HTTP fallback and complete web Cookie WSS eligibility. -docs/HUYA精英宝典联调指南.md: document the verified HTTP chain and staged WSS/shop/payment acceptance order. diff --git a/TODO.md b/TODO.md deleted file mode 100644 index 7ec70fe..0000000 --- a/TODO.md +++ /dev/null @@ -1,17 +0,0 @@ -# 待办 - -## Ruff lint 清理 - -状态：已完成 - -完成日期：2026-08-31 - -本轮完成了 Ruff lint 全量清理，包括未使用导入与变量、导入排序、类型注解现代化、FastAPI 依赖声明、异常处理标注、时区处理及其他规则。FastAPI 路由参数中的依赖调用通过 `web/backend/**/*.py` 的 B008 配置例外保留框架惯用写法；非 FastAPI 的可变默认对象已改为函数内惰性初始化。 - -最终基线： - -- Ruff lint：0 条 -- Ruff format：208 个文件全部通过 -- pytest：105 passed -- Pyright：0 errors / 0 warnings -- Python compileall：通过 diff --git a/VERIFICATION.txt b/VERIFICATION.txt deleted file mode 100644 index 1df0f6f..0000000 --- a/VERIFICATION.txt +++ /dev/null @@ -1,203 +0,0 @@ -changed branch/field: Huya web Cookie source-driven merge and anti-fabrication policy - -MODIFIED_FILE: /Users/yml/codes/live-hub-py/MODIFIED_FILE -DIFF_FILE: /Users/yml/codes/live-hub-py/DIFF_FILE -VERIFICATION.txt: /Users/yml/codes/live-hub-py/VERIFICATION.txt -ROLLBACK.sh: /Users/yml/codes/live-hub-py/ROLLBACK.sh -MODIFIED_FILE original SHA-256 before fixture update: 08bc9d7ca6425fab15af2c6218d2574f7f0180e929fa6ccb78c8b37e6ae8493a -MODIFIED_FILE changed SHA-256: 810eb13c64d19b057d41db476437f35d90446f4a1dc6760c979b0c1670350815 - -BASELINE exact command: -set -e; BASE=/tmp/live-hub-py-baseline.PJsjpJ; git archive HEAD | tar -x -C "$BASE"; .venv/bin/pytest -q "$BASE/tests/test_huya_app_login.py" "$BASE/tests/test_huya_dfp_register.py" -BASELINE literal output/result: -27 passed, 1 warning in 0.62s -BASELINE exit status: 0 - -MODIFIED exact command: -.venv/bin/pytest -q -MODIFIED literal output/result: -127 passed, 1 warning in 1.21s -MODIFIED exit status: 0 - -ROLLBACK exact command: -set -e; SRC=/tmp/rollback-source.mKNKoP; TGT=/tmp/rollback-target.hHBqnQ; cp MODIFIED_FILE "$SRC"; printf '\ncorrupt\n' >> "$TGT"; ./ROLLBACK.sh "$SRC" "$TGT"; cmp -s "$SRC" "$TGT" -ROLLBACK literal output/result: -restored /tmp/rollback-target.hHBqnQ from /tmp/rollback-source.mKNKoP -rollback_status=restored -source_sha256=e15f95ce30ca0e0faa6e3c8c6cdf4725e4984434df0f39cf237ee0b7197375b1 -target_sha256=e15f95ce30ca0e0faa6e3c8c6cdf4725e4984434df0f39cf237ee0b7197375b1 -ROLLBACK exit status: 0 -restored behavior/status: target byte-for-byte equals source; MODIFIED_FILE remains changed. - -Additional checks: -.venv/bin/python -m ruff check core/huya tests/test_huya_app_login.py -> All checks passed, exit 0 -git diff --check -> exit 0 -bash -n dev.sh deploy.sh -> exit 0 - -Integration round (2026-09-01): -changed branch/field: Huya Elite handbook transport diagnostics and integration guide -MODIFIED_FILE: /Users/yml/codes/live-hub-py/MODIFIED_FILE -DIFF_FILE: /Users/yml/codes/live-hub-py/DIFF_FILE -VERIFICATION.txt: /Users/yml/codes/live-hub-py/VERIFICATION.txt -ROLLBACK.sh: /Users/yml/codes/live-hub-py/ROLLBACK.sh - -BASELINE exact command: -base=$(mktemp -d /tmp/live-hub-baseline.XXXXXX); git archive HEAD | tar -x -C "$base"; cd "$base"; /Users/yml/codes/live-hub-py/.venv/bin/python -m pytest -q tests/test_huya_app_login.py tests/test_huya_dfp_register.py -BASELINE literal output/result: -28 passed, 1 warning in 0.94s -BASELINE exit status: 0 - -MODIFIED exact command: -.venv/bin/python -m pytest -q -MODIFIED literal output/result: -129 passed, 1 warning in 1.93s -MODIFIED exit status: 0 - -Transport-focused exact command: -.venv/bin/python -m pytest -q tests/test_huya_app_login.py tests/test_huya_runner_transport.py -Transport-focused literal output/result: -25 passed, 1 warning in 0.90s -Transport-focused exit status: 0 - -ROLLBACK exact command: -src=$(mktemp /tmp/rollback-source.XXXXXX); tgt=$(mktemp /tmp/rollback-target.XXXXXX); cp MODIFIED_FILE "$src"; printf '\\ncorrupt\\n' >> "$tgt"; ./ROLLBACK.sh "$src" "$tgt"; cmp -s "$src" "$tgt"; sha256sum "$src" "$tgt" -ROLLBACK literal output/result: -restored /tmp/rollback-target-final.1s6yf1 from /tmp/rollback-source-final.nLQUBj -810eb13c64d19b057d41db476437f35d90446f4a1dc6760c979b0c1670350815 /tmp/rollback-source-final.nLQUBj -810eb13c64d19b057d41db476437f35d90446f4a1dc6760c979b0c1670350815 /tmp/rollback-target-final.1s6yf1 -ROLLBACK exit status: 0 -restored behavior/status: independent target copy is byte-identical; MODIFIED_FILE remains changed. - -Analysis round (2026-08-31): -changed branch/field: Huya elite handbook HAR analysis document (protocol flow, field matrix, implementation gaps) -analysis artifact: /Users/yml/codes/live-hub-py/docs/HUYA_精英宝典-8.31-PY重新分析.md -client artifact: /Users/yml/codes/live-hub-py/scripts/huya_elite_client.py -MODIFIED_FILE: /Users/yml/codes/live-hub-py/artifacts/har-to-client/MODIFIED_FILE -DIFF_FILE: /Users/yml/codes/live-hub-py/artifacts/har-to-client/DIFF_FILE -VERIFICATION.txt: /Users/yml/codes/live-hub-py/artifacts/har-to-client/VERIFICATION.txt -ROLLBACK.sh: /Users/yml/codes/live-hub-py/artifacts/har-to-client/ROLLBACK.sh -source HAR SHA-256: bcb1e830bfc368c9029f85f90458b708dcf086993a79b833e8e544c65e5c679f; f8d13a837c477aa4dc55b27df6828ffb7840bbdcccf7c078f840164d62a00e07 - -BASELINE exact command: -set -e; BASE=$(mktemp -d /tmp/live-hub-py-baseline.XXXXXX); git archive HEAD | tar -x -C "$BASE"; (cd "$BASE" && /Users/yml/codes/live-hub-py/.venv/bin/pytest -q tests/test_huya_app_login.py tests/test_huya_dfp_register.py) -BASELINE literal output/result: -28 passed, 1 warning in 0.71s -BASELINE exit status: 0 - -MODIFIED exact command: -uv run pytest -q -MODIFIED literal output/result: -116 passed, 1 warning in 1.67s -MODIFIED exit status: 0 - -ROLLBACK exact command: -set -e; SRC=$(mktemp /tmp/rollback-source.XXXXXX); TGT=$(mktemp /tmp/rollback-target.XXXXXX); cp MODIFIED_FILE "$SRC"; printf '\\ncorrupt\\n' >> "$TGT"; ./ROLLBACK.sh "$SRC" "$TGT"; cmp -s "$SRC" "$TGT" -ROLLBACK literal output/result: -rollback_cmp=restored; source_sha256=e15f95ce30ca0e0faa6e3c8c6cdf4725e4984434df0f39cf237ee0b7197375b1; target_sha256=e15f95ce30ca0e0faa6e3c8c6cdf4725e4984434df0f39cf237ee0b7197375b1 -ROLLBACK exit status: 0 -restored behavior/status: independent target copy equals MODIFIED_FILE byte-for-byte; MODIFIED_FILE remains changed. - -Validation command: -uv run python scripts/validate.py -> file not found (exit status 2; no validator exists in this repository) - -Cookie/WSS gating round (2026-09-01): -changed branch/field: App 登录网页设备态完整性校验；精英宝典活动/商城禁止 HTTP 回退 -MODIFIED_FILE: /Users/yml/codes/live-hub-py/MODIFIED_FILE -DIFF_FILE: /Users/yml/codes/live-hub-py/DIFF_FILE -VERIFICATION.txt: /Users/yml/codes/live-hub-py/VERIFICATION.txt -ROLLBACK.sh: /Users/yml/codes/live-hub-py/ROLLBACK.sh - -BASELINE exact command: -base=$(mktemp -d /tmp/live-hub-baseline.XXXXXX); git archive HEAD | tar -x -C "$base"; (cd "$base" && /Users/yml/codes/live-hub-py/.venv/bin/python -m pytest -q tests/test_huya_app_login.py tests/test_huya_dfp_register.py) -BASELINE literal output/result: -28 passed, 1 warning in 0.64s -BASELINE exit status: 0 - -MODIFIED exact command: -.venv/bin/python -m pytest -q -MODIFIED literal output/result: -129 passed, 1 warning in 1.12s -MODIFIED exit status: 0 - -ROLLBACK exact command: -src=$(mktemp /tmp/rollback-source.XXXXXX); tgt=$(mktemp /tmp/rollback-target.XXXXXX); cp MODIFIED_FILE "$src"; printf '\\ncorrupt\\n' >> "$tgt"; ./ROLLBACK.sh "$src" "$tgt"; cmp -s "$src" "$tgt"; sha256sum "$src" "$tgt" -ROLLBACK literal output/result: -restored /tmp/rollback-target.wuXCvJ from /tmp/rollback-source.7kBtUo -810eb13c64d19b057d41db476437f35d90446f4a1dc6760c979b0c1670350815 /tmp/rollback-source.7kBtUo -810eb13c64d19b057d41db476437f35d90446f4a1dc6760c979b0c1670350815 /tmp/rollback-target.wuXCvJ -ROLLBACK exit status: 0 -restored behavior/status: independent target copy restored byte-for-byte; MODIFIED_FILE remains changed. - -Cookie bridge round (2026-09-01): -changed branch/field: hydevice 实际 Cookie 输出透传（仅真实 document.cookie 值） -MODIFIED_FILE: /Users/yml/codes/live-hub-py/MODIFIED_FILE -DIFF_FILE: /Users/yml/codes/live-hub-py/DIFF_FILE -VERIFICATION.txt: /Users/yml/codes/live-hub-py/VERIFICATION.txt -ROLLBACK.sh: /Users/yml/codes/live-hub-py/ROLLBACK.sh - -BASELINE exact command: -base=$(mktemp -d /tmp/live-hub-baseline.XXXXXX); git archive HEAD | tar -x -C "$base"; (cd "$base" && /Users/yml/codes/live-hub-py/.venv/bin/python -m pytest -q tests/test_huya_app_login.py tests/test_huya_dfp_register.py) -BASELINE literal output/result: -28 passed, 1 warning in 0.65s -BASELINE exit status: 0 - -MODIFIED exact command: -.venv/bin/pytest -q tests/test_huya_app_login.py tests/test_huya_runner_transport.py tests/test_huya_elite_protocol.py -MODIFIED literal output/result: -31 passed, 1 warning in 1.04s -MODIFIED exit status: 0 - -ROLLBACK exact command: -src=$(mktemp /tmp/rollback-source.XXXXXX); tgt=$(mktemp /tmp/rollback-target.XXXXXX); cp MODIFIED_FILE "$src"; printf '\\ncorrupt\\n' >> "$tgt"; ./ROLLBACK.sh "$src" "$tgt"; cmp -s "$src" "$tgt" -ROLLBACK literal output/result: -restored target copy; cmp_status=0 -ROLLBACK exit status: 0 -restored behavior/status: independent target copy equals source; MODIFIED_FILE remains changed. - -Runtime evidence exact command: -node core/huya/fingerprint/runner.js /tmp/huya-test 5008 -Runtime literal output/result: -__SDID__ returned; __HDID__ returned; __COOKIES__ contained only runner-written udb_appid/sdid and none of the seven required web-device fields -Runtime exit status: 0 - -Provenance cross-check round (2026-09-01 18:xx): -changed branch/field: QR/verify 前后 Cookie 首次出现位置诊断日志 -MODIFIED_FILE: /Users/yml/codes/live-hub-py/MODIFIED_FILE -DIFF_FILE: /Users/yml/codes/live-hub-py/DIFF_FILE -VERIFICATION.txt: /Users/yml/codes/live-hub-py/VERIFICATION.txt -ROLLBACK.sh: /Users/yml/codes/live-hub-py/ROLLBACK.sh - -MODIFIED exact command: -.venv/bin/pytest -q tests/test_huya_app_login.py tests/test_huya_runner_transport.py tests/test_huya_elite_protocol.py -MODIFIED literal output/result: -31 passed, 1 warning in 0.61s -MODIFIED exit status: 0 -restored behavior/status: no protocol behavior changed for missing fields; diagnostics report observed Cookie names only. - -HAR cookie provenance round (2026-09-01): -changed branch/field: 9.1 HAR Cookie 首次出现位置与来源分类；移除脚本内浏览器 bootstrap -MODIFIED_FILE: /Users/yml/codes/live-hub-py/MODIFIED_FILE -DIFF_FILE: /Users/yml/codes/live-hub-py/DIFF_FILE -VERIFICATION.txt: /Users/yml/codes/live-hub-py/VERIFICATION.txt -ROLLBACK.sh: /Users/yml/codes/live-hub-py/ROLLBACK.sh - -BASELINE exact command: -base=$(mktemp -d /tmp/live-hub-baseline.XXXXXX); git archive HEAD | tar -x -C "$base"; (cd "$base" && /Users/yml/codes/live-hub-py/.venv/bin/python -m pytest -q tests/test_huya_app_login.py tests/test_huya_dfp_register.py) -BASELINE literal output/result: -28 passed, 1 warning in 0.65s -BASELINE exit status: 0 - -MODIFIED exact command: -.venv/bin/python -m pytest -q -MODIFIED literal output/result: -129 passed, 1 warning in 1.22s -MODIFIED exit status: 0 - -ROLLBACK exact command: -src=$(mktemp /tmp/rollback-source.XXXXXX); tgt=$(mktemp /tmp/rollback-target.XXXXXX); cp MODIFIED_FILE "$src"; printf '\\ncorrupt\\n' >> "$tgt"; ./ROLLBACK.sh "$src" "$tgt"; cmp -s "$src" "$tgt"; sha256sum "$src" "$tgt" -ROLLBACK literal output/result: -restored /tmp/rollback-target.9nPt3r from /tmp/rollback-source.6zNUJo -810eb13c64d19b057d41db476437f35d90446f4a1dc6760c979b0c1670350815 /tmp/rollback-source.6zNUJo -810eb13c64d19b057d41db476437f35d90446f4a1dc6760c979b0c1670350815 /tmp/rollback-target.9nPt3r -ROLLBACK exit status: 0 -restored behavior/status: independent target copy restored byte-for-byte; MODIFIED_FILE remains changed. diff --git a/artifacts/guid_session_consistency/DIFF_FILE b/artifacts/guid_session_consistency/DIFF_FILE deleted file mode 100644 index 0f2dbd0..0000000 --- a/artifacts/guid_session_consistency/DIFF_FILE +++ /dev/null @@ -1,188 +0,0 @@ -diff --git a/core/huya/elite_session.py b/core/huya/elite_session.py -index 31f2449..0eb740c 100644 ---- a/core/huya/elite_session.py -+++ b/core/huya/elite_session.py -@@ -94,6 +94,13 @@ class HuyaEliteWssSession: - ) - if not initialized: - raise RuntimeError(f"虎牙 {kind} WSS 初始化失败") -+ if kind == "activity" and self.client._launch_guid: -+ # wsLaunch 的 sGuid 是当前连接的真实会话值，供后续业务 Cookie -+ # 与 HTTP baseinfo 继续复用，避免沿用登录时的旧 guid。 -+ self.cookie = self.client._cookie_with_guid( -+ self.cookie, self.client._launch_guid -+ ) -+ self.guid = self.client._launch_guid - except Exception: - self.close() - raise -diff --git a/core/huya/wss_client.py b/core/huya/wss_client.py -index 929c622..c665afd 100644 ---- a/core/huya/wss_client.py -+++ b/core/huya/wss_client.py -@@ -26,6 +26,7 @@ from typing import Any, cast - - import websockets - -+from .cookie_utils import cookie_pairs, normalize_cookie_pairs - from .frame_decoder import _decode_taf_struct, _truncate, format_wss_log - from .taf_protocol import TafInputStream, TafOutputStream, TafStruct, TafType - from .wup_protocol import WupRequest, WupResponse -@@ -192,6 +193,26 @@ class HuyaWssClient: - return normalized - return f"{normalized}; {cookie}" - -+ @staticmethod -+ def _cookie_with_guid(cookie: str, guid: str) -> str: -+ """用当前 WSS 会话的 sGuid 替换 Cookie 中的旧 guid。""" -+ value = str(guid or "").strip() -+ if not value: -+ return cookie or "" -+ pairs = cookie_pairs(cookie) -+ replaced = False -+ normalized: list[tuple[str, str]] = [] -+ for key, item_value in pairs: -+ if key == "guid": -+ if not replaced: -+ normalized.append((key, value)) -+ replaced = True -+ continue -+ normalized.append((key, item_value)) -+ if not replaced: -+ normalized.append(("guid", value)) -+ return normalize_cookie_pairs(normalized) -+ - @staticmethod - def _build_biz_user(uid: int, cookie: str): - """构造 WSS 业务 UserId（HAR 实证：guid 为空，cookie 不额外加前缀）""" -@@ -389,7 +410,12 @@ class HuyaWssClient: - if launch_rsp is None: - self.logger("[WSS] 活动 wsLaunch 无响应") - return False -- await self.send_auth(uid, cookie, sequence=SEQ_ACTIVITY_BIZ) -+ # wsLaunch 会为当前连接签发新的 sGuid。AUTH、getConfig 和后续业务 -+ # 请求必须使用同一个值；登录 Cookie 里的旧 guid 不能继续复用。 -+ if self._launch_guid: -+ cookie = self._cookie_with_guid(cookie, self._launch_guid) -+ guid = self._launch_guid -+ await self.send_auth(uid, cookie, sequence=SEQ_ACTIVITY_BIZ, guid=guid) - # 9.1 抓包：getConfig 使用 wsLaunch 响应返回的新 guid；业务 UserId 仍为空 guid。 - config_rsp = await self.call_get_config_activity( - uid, cookie, guid=self._launch_guid -@@ -570,7 +596,13 @@ class HuyaWssClient: - return body[5 : 5 + wup_len] - return body - -- async def send_auth(self, uid: int, cookie: str, sequence: int = SEQ_WSLAUNCH): -+ async def send_auth( -+ self, -+ uid: int, -+ cookie: str, -+ sequence: int = SEQ_WSLAUNCH, -+ guid: str = "", -+ ): - """cmd 0x0a AUTH — 对齐 9.1 抓包（828 活动帧）。 - - 9.1 活动 AUTH 结构（逐字节核验）: -@@ -591,8 +623,8 @@ class HuyaWssClient: - os.write_string(1, WSS_COOKIE_UA) - os.write_string(2, HuyaWssClient._normalize_biz_cookie(cookie)) - guid_match = re.search(r"(?:^|;\s*)guid=([^;]+)", cookie or "") -- guid = (guid_match.group(1).strip() if guid_match else "") or self._launch_guid -- os.write_string(3, guid or "") -+ auth_guid = (guid or "").strip() or (guid_match.group(1).strip() if guid_match else "") or self._launch_guid -+ os.write_string(3, auth_guid) - os.write_int8(4, 1) - os.write_string(5, "HUYA&ZH&2052") - os.write_string(6, "") -diff --git "a/docs/HUYA_COOKIE\350\267\257\347\272\277\347\212\266\346\200\201.md" "b/docs/HUYA_COOKIE\350\267\257\347\272\277\347\212\266\346\200\201.md" -index 69e5423..d67e890 100644 ---- "a/docs/HUYA_COOKIE\350\267\257\347\272\277\347\212\266\346\200\201.md" -+++ "b/docs/HUYA_COOKIE\350\267\257\347\272\277\347\212\266\346\200\201.md" -@@ -15,7 +15,7 @@ App 登录可以得到新鲜的 `udb_cred`、`udb_uid`、`yyuid`、`udb_biztoken - | `udb_cred`、`udb_uid`、`yyuid`、`udb_biztoken` | 已完成 | App 登录与 QR 绑定链 | - | `sdid`、`hdid` | 已完成 | hydevice / `df/collect` | - | `udb_deviceid`、`udb_guiddata`、`udb_anobiztoken`、`udb_anouid` | 已完成 | `anon_device.py` | --| `guid` | 会话路径完成，回退算法未完全确认 | `wsLaunch` 返回值优先；回退值为结构仿造 | -+| `guid` | 会话路径已闭合 | 活动 `wsLaunch` 每次签发 32 位 `sGuid`；初始化时统一覆盖 AUTH、getConfig 和后续业务 Cookie，`956a/966a/976a` 等中间段不是本地版本常量 | - | `__yamid_new` | 已完成 | `room_normal.js` `generate32` Python 复刻 | - | `game_did` | 已完成 | 35 位、63 字符集规则复刻 | - | `_yasids` | 格式完成，精确共享种子未确认 | 与 `__yamid_new` 使用同类时钟生成器 | -@@ -47,10 +47,9 @@ SDK/Python 输出：d07a45de567322edcd42312f38e610be - ## 尚未闭合的工程项 - - 1. 将真实 WebView/Chrome 的 `getFeature()` 结果接入登录调用，替代当前的设备画像近似值。 --2. 明确 `guid` 的 `956a` / `966a` 版本差异和无 `wsLaunch` 时的真实生成规则。 --3. 让活动会话补齐后的 Cookie 持久化回账号，避免原始 16 字段账号每次任务重新生成设备字段。 --4. 为 `raw_cookie` 增加重建网页 Cookie 的服务入口。 --5. 商城下单、支付和支付状态轮询保持单独人工验证，不纳入只读回归。 -+2. 让活动会话补齐后的 Cookie 持久化回账号，避免原始 16 字段账号每次任务重新生成设备字段。 -+3. 为 `raw_cookie` 增加重建网页 Cookie 的服务入口。 -+4. 商城下单、支付和支付状态轮询保持单独人工验证，不纳入只读回归。 - - ## 证据入口 - -diff --git a/docs/HUYA_WEB_FIELD_TRACE.md b/docs/HUYA_WEB_FIELD_TRACE.md -index 8a7f9fe..4c82581 100644 ---- a/docs/HUYA_WEB_FIELD_TRACE.md -+++ b/docs/HUYA_WEB_FIELD_TRACE.md -@@ -77,7 +77,7 @@ qimeisdk-web 0.6.3 源码已定位： - - | 字段 | 来源 | 风控关联性 | 建议 | - | --- | --- | --- | --- | --| `guid` | 浏览器脚本（有固定段 `0a..966a`）| 高（设备与会话绑定） | **必须按结构仿造或复用 wsLaunch 会话值**；随机纯 hex 与真实结构不符 | -+| `guid` | 活动 `wsLaunch` 响应 `sGuid`；页面 SDK 负责持久化 | 高（设备与会话绑定） | **优先复用当前 wsLaunch 会话值**；`956a/966a/976a` 等中间段随服务端返回变化，不能固化为版本常量 | - | `__yamid_new` | 浏览器脚本（CBCC 前缀+种子）| 高（与 rootsid 同源）| `room_normal.js` `generate32` 已复刻；精确 rootsid 共享计数仍待确认 | - | `_qimei_uuid42` | QIMEI SDK 0.6.3 VM 生成器 `e=6064` | 中 | Python 已按时间编码、CRC32 和随机半字节逐项复刻 | - | `_qimei_fingerprint` | QIMEI SDK 0.6.3 x64hash128(feature_string, 31) | 中 | Python 已复刻，输入字段见路线状态文档 | -@@ -87,7 +87,7 @@ qimeisdk-web 0.6.3 源码已定位： - - **当时的下一步记录（部分已完成，保留作历史）**： - 1. `_qimei_fingerprint` 已改为浏览器能力特征串的 `x64hash128`，见路线状态文档。 --2. `guid` 已优先使用 `wsLaunch` 会话值，回退固定段仍待确认。 -+2. `guid` 已统一使用 `wsLaunch` 会话值；活动 AUTH、getConfig 与后续业务 Cookie 均复用同一返回值。 - 3. `__yamid_new` 已定位 `room_normal.js` `generate32` 并完成 Python 复刻。 - ## 六、核心字段生成算法逆向结果（2026-09-01 晚，浏览器实测 + 源码定位） - -diff --git a/tests/test_huya_elite_protocol.py b/tests/test_huya_elite_protocol.py -index 71a10e6..ce4360d 100644 ---- a/tests/test_huya_elite_protocol.py -+++ b/tests/test_huya_elite_protocol.py -@@ -146,3 +146,37 @@ def test_activity_auth_frame_matches_latest_capture_layout(): - - # body 必须与 9.1 828 AUTH 同样以 TAIL_BYTES 收尾 - assert message.body[-len(TAIL_BYTES):] == TAIL_BYTES -+ -+ -+def test_activity_cookie_guid_is_replaced_by_launch_guid(): -+ old = "0a" + "1" * 30 -+ new = "0a" + "2" * 30 -+ cookie = f"udb_uid=1; guid={old}; x=y; guid={old}" -+ -+ result = HuyaWssClient._cookie_with_guid(cookie, new) -+ -+ assert result == f"udb_uid=1; guid={new}; x=y" -+ -+ -+def test_activity_auth_accepts_explicit_launch_guid_over_cookie_guid(): -+ async def build_frame(): -+ client = HuyaWssClient() -+ client._activity_mode = True -+ client.ws = type("FakeWs", (), {"send": AsyncMock()})() -+ await client.send_auth( -+ 1199664135026, -+ "guid=" + "0a" * 16 + "; udb_cred=" + "x" * 300, -+ sequence=0x1D000109, -+ guid="0b" * 16, -+ ) -+ return client.ws.send.call_args.args[0] -+ -+ message = WssMessage.decode(asyncio.run(build_frame())) -+ stream = TafInputStream(message.body[: -len(TAIL_BYTES)]) -+ assert stream.read_head() == (3, TafType.ZERO) -+ stream.read_int64(0) -+ stream.read_string(1) -+ assert stream.read_head() == (2, TafType.STRING4) -+ cookie_len = int.from_bytes(stream.buf.read(4), "big") -+ stream.buf.read(cookie_len) -+ assert stream.read_string(3) == "0b" * 16 diff --git a/artifacts/guid_session_consistency/MODIFIED_FILE b/artifacts/guid_session_consistency/MODIFIED_FILE deleted file mode 100644 index c665afd..0000000 --- a/artifacts/guid_session_consistency/MODIFIED_FILE +++ /dev/null @@ -1,1164 +0,0 @@ -""" -虎牙 WSS 客户端 — 商城通道 (77bc035c-ws.va.huya.com) - -初始化流程 (Chrome WS 帧实证): - 1. HEARTBEAT_SEND (cmd 0x10) - 2. RPC: launch.wsLaunch (cmd 0x03) — 商城版无 lUid - 3. RPC: mobileui.getConfig (cmd 0x03) - 4. 等待 wsLaunch/getConfig 响应 - 5. REGISTER (cmd 0x21) — 回传 wsLaunch 返回的 sGuid - 6. CONFIRM_SEND (cmd 0x17) - 7. 业务 RPC ... - -关键发现 (Chrome WS 帧分析): - - 服务端响应 seq ≠ 请求 seq → 必须用 FIFO 匹配 - - 初始化 wsLaunch iRequestId=-1，业务 requestId 从 8 开始递增 -""" - -import asyncio -import random -import re -import struct -import time -from collections import deque -from collections.abc import Callable -from typing import Any, cast - -import websockets - -from .cookie_utils import cookie_pairs, normalize_cookie_pairs -from .frame_decoder import _decode_taf_struct, _truncate, format_wss_log -from .taf_protocol import TafInputStream, TafOutputStream, TafStruct, TafType -from .wup_protocol import WupRequest, WupResponse - -SHOP_WS_HOST = "77bc035c-ws.va.huya.com" -ACTIVITY_WS_HOST = "d35bf373-ws.va.huya.com" -# 商城端点 baseinfo (conn4, h5_/index.html) — 商城业务 shopMiddleUI 走此通道 -# wsLaunch tReq: lUid=0, sGuid="", sUA="webh5&0.0.1&websocket&&h5_/index.html" -SHOP_BASEINFO = "DBYAJiV3ZWJoNSYwLjAuMSZ3ZWJzb2NrZXQmJmg1Xy9pbmRleC5odG1sNgxIVVlBJlpIJjIwNTJGAFYAbHYAhgCWAKgM" - -TAIL_BYTES = bytes.fromhex("2c36004c5c6600") -BIG_TAIL_SUFFIX = bytes.fromhex("4c5c6600") -WSS_BIZ_UA = "web&1.0.0&huya" -WSS_COOKIE_UA = "webh5&0.0.1&websocket&&diypc_52775" -WSS_SHOP_SCENE = 4 - - -class _ActivityConfigReq(TafStruct): - def __init__(self, user): - self.user = user - - def write_to(self, os: TafOutputStream): - os.write_struct(0, self.user) - os.write_map(1, {}) - os.write_string(2, "huya") - os.write_list(3, []) - - def read_from(self, _ins): - pass - -# seq 按消息类型固定 (HAR 实证，跨两次抓包一致) -SEQ_HEARTBEAT = 0x1D00000D -SEQ_WSLAUNCH = 0x1D000078 -SEQ_GETCONFIG = 0x1D000076 -SEQ_REGISTER = 0x1D00003C -SEQ_CONFIRM = 0x1D000002 -SEQ_BUSINESS = 0x1D000106 # 业务 RPC (HAR 7.4 实证：业务帧固定使用该 seq) -SEQ_ACTIVITY_WSLAUNCH = 0x1D000100 -SEQ_ACTIVITY_BIZ = 0x1D000109 - - -class WssCommand: - RPC_REQUEST = 0x03 - RPC_RESPONSE = 0x04 - AUTH = 0x0A - SERVER_PUSH1 = 0x0B - HEARTBEAT_SEND = 0x10 - HEARTBEAT_RECV = 0x11 - CONFIRM_SEND = 0x17 - SERVER_PUSH2 = 0x18 - REGISTER = 0x21 - - -class WssMessage: - def __init__(self, command: int, sequence: int, body: bytes): - self.command = command - self.sequence = sequence - self.body = body - - def encode(self) -> bytes: - header = struct.pack(">BBI", 0x00, self.command, self.sequence) - return header + self.body - - @classmethod - def decode(cls, data: bytes) -> "WssMessage": - if len(data) < 6: - raise ValueError(f"消息太短: {len(data)} bytes") - _version, command = struct.unpack(">BB", data[0:2]) - sequence = struct.unpack(">I", data[2:6])[0] - body = data[6:] - return cls(command=command, sequence=sequence, body=body) - - -class HuyaWssClient: - """虎牙 WSS 客户端 — 商城通道""" - - def __init__( - self, baseinfo: str | None = None, logger: Callable[[str], None] | None = None - ): - self.baseinfo = baseinfo or SHOP_BASEINFO - self.ws: Any = None - self._biz_seq = SEQ_BUSINESS # 业务 RPC 递增用 - self._wup_req_id = 8 - self.rpc_queue = deque() - self.logger = logger or print - self._recv_task = None - self._heartbeat_task = None - self._launch_guid = "" - self._launch_ip = "" - self._activity_mode = False - - def _next_biz_seq(self) -> int: - """业务 RPC 的 frame seq。 - - 浏览器在商城 WSS 上连续发送多个业务 RPC 时，frame seq 都保持 - 0x1d000106；真正区分请求依赖 WUP requestId。 - """ - return SEQ_ACTIVITY_BIZ if self._activity_mode else SEQ_BUSINESS - - def _next_wup_req_id(self) -> int: - """业务 WUP requestId，HAR 中 getGoodsInfoV5 从 8 开始递增""" - req_id = self._wup_req_id - self._wup_req_id += 1 - return req_id - - @staticmethod - def _make_big_tail() -> bytes: - """生成 WSS 大包尾部 trace（格式与浏览器 HAR 一致）""" - trace = f"{random.getrandbits(64):016x}" - return f",6%{trace}:{trace}:0:3".encode("ascii") + BIG_TAIL_SUFFIX - - @staticmethod - def _encode_rpc_body(wup_data: bytes) -> bytes: - """按浏览器规则封装 WSS RPC body - - 小包: [4B WUP长度][WUP body] + TAIL_BYTES - 大包: [1B len低字节][4B WUP body长度][裸WUP body] + trace尾部 - """ - if len(wup_data) < 4: - return wup_data + TAIL_BYTES - declared_len = int.from_bytes(wup_data[:4], "big") - if declared_len == len(wup_data): - wup_body = wup_data[4:] - else: - wup_body = wup_data - if len(wup_body) <= 0xFF: - return wup_data + TAIL_BYTES - # 大包头里的长度使用原始 WUP 包长度（含 4B 长度前缀），但正文只放裸 WUP body； - # 浏览器随后用 trace 尾部的前 4 字节补足该长度窗口。 - packet_len = len(wup_data) - return ( - bytes([packet_len & 0xFF]) - + packet_len.to_bytes(4, "big") - + wup_body - + HuyaWssClient._make_big_tail() - ) - - @staticmethod - def _encode_activity_launch_body(wup_data: bytes) -> bytes: - """活动 wsLaunch 使用浏览器的固定大包头，即使包体小于 255B。""" - if len(wup_data) < 4: - return wup_data + TAIL_BYTES - packet_len = len(wup_data) - return ( - bytes([packet_len & 0xFF]) - + packet_len.to_bytes(4, "big") - + wup_data[4:] - + TAIL_BYTES - ) - - @staticmethod - def _normalize_biz_cookie(cookie: str) -> str: - """业务 RPC 的 huya_ua Cookie 值按浏览器 WSS 抓包修正""" - cookie = (cookie or "").strip() - normalized = f"huya_ua={WSS_COOKIE_UA}" - if re.search(r"(?:^|;\s*)huya_ua=", cookie): - return re.sub( - r"(^|;\s*)huya_ua=[^;]*", - lambda m: f"{m.group(1)}{normalized}", - cookie, - count=1, - ) - if not cookie: - return normalized - return f"{normalized}; {cookie}" - - @staticmethod - def _cookie_with_guid(cookie: str, guid: str) -> str: - """用当前 WSS 会话的 sGuid 替换 Cookie 中的旧 guid。""" - value = str(guid or "").strip() - if not value: - return cookie or "" - pairs = cookie_pairs(cookie) - replaced = False - normalized: list[tuple[str, str]] = [] - for key, item_value in pairs: - if key == "guid": - if not replaced: - normalized.append((key, value)) - replaced = True - continue - normalized.append((key, item_value)) - if not replaced: - normalized.append(("guid", value)) - return normalize_cookie_pairs(normalized) - - @staticmethod - def _build_biz_user(uid: int, cookie: str): - """构造 WSS 业务 UserId（HAR 实证：guid 为空，cookie 不额外加前缀）""" - from .shop_structs import UserId - - user = UserId() - user.lUid = uid - user.sGuid = "" - user.sToken = "" - user.sHuYaUA = WSS_BIZ_UA - user.sCookie = HuyaWssClient._normalize_biz_cookie(cookie) - user.iTokenType = 0 - user.sDeviceInfo = "" - user.sQIMEI = "" - return user - - @staticmethod - def _build_shop_app(source_id: str = "yellowcarlist", scene: int = WSS_SHOP_SCENE): - """构造 WSS 业务 ShopAppInfo""" - from .shop_structs import ShopAppInfo - - info = ShopAppInfo() - info.sAppId = "huya" - info.sBizType = "" - info.scene = scene - info.sourceId = source_id - return info - - @staticmethod - def _build_order_env(item_count: int = 1): - """构造下单/支付环境参数""" - click_time = int(time.time() * 1000) - return { - "user_click_seq": f'[{{"x":258,"y":487,"time":{click_time},"id":""}}]', - "input_1": str(item_count), - } - - async def connect( - self, - host: str = SHOP_WS_HOST, - timeout: float = 15.0, - cookie: str = "", - origin: str = "https://m-shop.yaoguo.com", - ): - from urllib.parse import quote - - url = f"wss://{host}/?baseinfo={quote(self.baseinfo)}" - self.logger(f"[WSS] 正在连接 {host} ...") - self.logger(f"[WSS] URL: {url[:100]}...") - - BROWSER_UA = ( - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " - "AppleWebKit/537.36 (KHTML, like Gecko) " - "Chrome/149.0.0.0 Safari/537.36" - ) - headers = {"Accept-Language": "zh-CN,zh;q=0.9"} - if cookie: - headers["Cookie"] = cookie - - try: - self.ws = await asyncio.wait_for( - websockets.connect( - url, - origin=cast(Any, origin), - user_agent_header=BROWSER_UA, - additional_headers=headers, - open_timeout=timeout, - # 不信任环境代理：web 进程可能带 SOCKS 代理变量， - # 会导致 WSS 直连失败（需 python-socks）；虎牙国内直连。 - proxy=None, - ), - timeout=timeout, - ) - except TimeoutError: - self.logger(f"[WSS] 连接超时（{timeout}s）") - raise - except Exception as e: - self.logger(f"[WSS] 连接失败: {type(e).__name__}: {e}") - raise - - self.logger(f"[WSS] 握手成功, 实际连接: {self.ws.remote_address}") - self._recv_task = asyncio.create_task(self._recv_loop()) - self._heartbeat_task = asyncio.create_task(self._heartbeat_loop()) - - async def disconnect(self): - if self._heartbeat_task: - self._heartbeat_task.cancel() - if self._recv_task: - self._recv_task.cancel() - if self.ws: - await self.ws.close() - self.ws = None - self.logger("[WSS] 已断开") - - async def _recv_loop(self): - try: - async for raw in self.ws: - try: - raw_bytes = raw if isinstance(raw, bytes) else raw.encode() - msg = WssMessage.decode(raw_bytes) - self.logger( - format_wss_log(msg.body, msg.command, msg.sequence, "收") - ) - await self._handle_message(msg) - except Exception as e: # noqa: BLE001 外部接口与任务边界需要保留宽泛异常兜底 - self.logger( - f"[WSS] 解析消息失败: {e} raw_hex={raw_bytes[:50].hex()}" - ) - except asyncio.CancelledError: - pass - except websockets.exceptions.ConnectionClosed as e: - self.logger(f"[WSS] 连接关闭: code={e.code} reason={e.reason}") - except Exception as e: # noqa: BLE001 外部接口与任务边界需要保留宽泛异常兜底 - self.logger(f"[WSS] 接收循环异常: {type(e).__name__}: {e}") - - async def _handle_message(self, msg: WssMessage): - if msg.command == WssCommand.RPC_RESPONSE: - if self.rpc_queue: - future = self.rpc_queue.popleft() - if not future.done(): - future.set_result(msg.body) - else: - self.logger(f"[WSS] 未匹配的 RPC 响应 seq={msg.sequence}") - elif msg.command == WssCommand.HEARTBEAT_RECV: - pass # 心跳已在上层 _recv_loop 打过日志 - elif msg.command in (WssCommand.SERVER_PUSH1, WssCommand.SERVER_PUSH2): - pass # 推送已在上层打过日志 - else: - pass # 未知消息已在上层打过日志 - - async def _heartbeat_loop(self): - try: - while True: - await asyncio.sleep(8) - await self.send_heartbeat() - except asyncio.CancelledError: - pass - except Exception as e: # noqa: BLE001 外部接口与任务边界需要保留宽泛异常兜底 - self.logger(f"[WSS] 心跳循环异常: {e}") - - async def send_heartbeat(self): - # body = LIST(tag0,count=1) + 元素{tag0="live:0", tag1=""} + TAIL - os = TafOutputStream() - os.write_head(0, TafType.LIST) # 09 - os.write_int32(0, 1) # 00 01 (count=1) - os.write_string(0, "live:0") # 06 06 live:0 - os.buf.write(b"\x16\x00") # 16 00 (tag1 "") - body = os.get_bytes() + TAIL_BYTES - msg = WssMessage( - command=WssCommand.HEARTBEAT_SEND, sequence=SEQ_HEARTBEAT, body=body - ) - self.logger( - format_wss_log(body, WssCommand.HEARTBEAT_SEND, SEQ_HEARTBEAT, "发") - ) - await self.ws.send(msg.encode()) - - async def initialize(self, uid: int, guid: str, cookie: str): - """商城端点初始化: heartbeat → wsLaunch → getConfig → confirm - - 商城 wsLaunch 无 lUid/sGuid (匿名), tReq: - tag0 lUid=0, tag1 sGuid="", tag2 sUA="webh5&0.0.1&websocket&&h5_/index.html", - tag3 sAppSrc="HUYA&ZH&2052", tag4 tDeviceInfo={5空串} - """ - self.logger("[初始化] 开始 WSS 初始化流程 (商城端点)...") - - await self.send_heartbeat() - - launch_rsp = await self.call_ws_launch_shop() - if launch_rsp is None: - self.logger("[✗] wsLaunch 无响应，初始化失败") - return False - self.logger( - f"[初始化] wsLaunch OK guid={self._launch_guid} ip={self._launch_ip}" - ) - - config_rsp = await self.call_get_config_shop() - self.logger( - f"[初始化] getConfig {'OK' if config_rsp is not None else '无响应(可继续)'}" - ) - - if self._launch_guid: - await self.send_register(self._launch_guid) - - await self.send_confirm() - - await asyncio.sleep(0.3) - self.logger("[初始化] 完成 ✓") - return True - - async def initialize_activity(self, uid: int, guid: str, cookie: str): - """活动通道初始化：heartbeat -> wsLaunch -> AUTH -> getConfig -> confirm。""" - self._activity_mode = True - await self.send_heartbeat() - launch_rsp = await self.call_ws_launch_activity(uid, guid) - if launch_rsp is None: - self.logger("[WSS] 活动 wsLaunch 无响应") - return False - # wsLaunch 会为当前连接签发新的 sGuid。AUTH、getConfig 和后续业务 - # 请求必须使用同一个值；登录 Cookie 里的旧 guid 不能继续复用。 - if self._launch_guid: - cookie = self._cookie_with_guid(cookie, self._launch_guid) - guid = self._launch_guid - await self.send_auth(uid, cookie, sequence=SEQ_ACTIVITY_BIZ, guid=guid) - # 9.1 抓包：getConfig 使用 wsLaunch 响应返回的新 guid；业务 UserId 仍为空 guid。 - config_rsp = await self.call_get_config_activity( - uid, cookie, guid=self._launch_guid - ) - if config_rsp is None: - self.logger("[WSS] 活动 getConfig 无响应，继续发送 confirm") - await self.send_confirm() - await asyncio.sleep(0.1) - return True - - async def call_ws_launch_activity(self, uid: int, guid: str = "", timeout: float = 10.0): - ua = WSS_COOKIE_UA - os = TafOutputStream() - os.write_struct_begin(0) - os.write_int64(0, int(uid or 0)) - os.write_string(1, guid or "") - os.write_string(2, ua) - os.write_string(3, "HUYA&ZH&2052") - os.write_struct_begin(4) - for i in range(5): - os.write_string(i, "") - os.write_struct_end() - os.write_struct_end() - wup = WupRequest() - wup.setServant("launch") - wup.setFunc("wsLaunch") - wup.setRequestId(-1) - wup.iTimeout = 0 - wup.newdata["tReq"] = os.get_bytes() - body = self._encode_activity_launch_body(wup.encode()) - msg = WssMessage(WssCommand.RPC_REQUEST, SEQ_ACTIVITY_WSLAUNCH, body) - future = asyncio.Future() - self.rpc_queue.append(future) - self.logger(format_wss_log(body, WssCommand.RPC_REQUEST, SEQ_ACTIVITY_WSLAUNCH, "发")) - await self.ws.send(msg.encode()) - try: - response = await asyncio.wait_for(future, timeout) - except TimeoutError: - if future in self.rpc_queue: - self.rpc_queue.remove(future) - return None - self.logger(format_wss_log(response, WssCommand.RPC_RESPONSE, SEQ_ACTIVITY_WSLAUNCH, "收")) - self._parse_launch_response(response) - return response - - async def call_get_config_activity( - self, uid: int, cookie: str, timeout: float = 10.0, guid: str = "" - ): - user = self._build_activity_user(uid, cookie, guid=guid) - req = _ActivityConfigReq(user) - return await self.call_rpc("mobileui", "getConfig", req, None, timeout=timeout) - - @staticmethod - def _build_activity_user(uid: int, cookie: str, guid: str = ""): - from .activity_structs import ActivityUserId - - user = ActivityUserId() - user.lUid = int(uid or 0) - user.sGuid = guid or "" - user.sHuYaUA = WSS_COOKIE_UA - user.sCookie = HuyaWssClient._normalize_biz_cookie(cookie) - return user - - async def call_ws_launch_shop(self, timeout: float = 10.0): - """商城版 wsLaunch (匿名, 无 lUid/sGuid, 与 conn4 格式一致)""" - ua = "webh5&0.0.1&websocket&&h5_/index.html" - app_src = "HUYA&ZH&2052" - - os = TafOutputStream() - os.write_struct_begin(0) - os.write_int64(0, 0) # lUid = 0 - os.write_string(1, "") # sGuid = "" - os.write_string(2, ua) # sUA - os.write_string(3, app_src) # sAppSrc - os.write_struct_begin(4) # tDeviceInfo - for i in range(5): - os.write_string(i, "") - os.write_struct_end() - os.write_struct_end() - - wup = WupRequest() - wup.setServant("launch") - wup.setFunc("wsLaunch") - wup.setRequestId(-1) - wup.iTimeout = 0 - wup.newdata["tReq"] = os.get_bytes() - wup_data = wup.encode() - body = self._encode_rpc_body(wup_data) - - seq = SEQ_WSLAUNCH - msg = WssMessage(command=WssCommand.RPC_REQUEST, sequence=seq, body=body) - future = asyncio.Future() - self.rpc_queue.append(future) - self.logger(format_wss_log(body, WssCommand.RPC_REQUEST, seq, "发")) - await self.ws.send(msg.encode()) - - try: - body = await asyncio.wait_for(future, timeout) - except TimeoutError: - if future in self.rpc_queue: - self.rpc_queue.remove(future) - self.logger("[✗] wsLaunch 超时") - return None - - self._parse_launch_response(body) - return body - - def _parse_launch_response(self, body: bytes): - """解析 wsLaunch 响应 (WSLaunchRsp: tag0=sGuid, tag1=sClientIp)""" - try: - # 从 body 中提取干净的 WUP 包（去掉尾部垃圾字节） - wup = self._extract_wup(body) - wup_resp = WupResponse() - wup_resp.decode(wup) - treq = wup_resp.newdata.get("tRsp") or wup_resp.newdata.get("tResp") - if not treq: - self.logger( - f"[RPC] wsLaunch 响应无 tRsp, keys={list(wup_resp.newdata.keys())}" - ) - return - ins = TafInputStream(treq) - _tag, dtype = ins.peek_head() - if dtype != 0x0A: # STRUCT_BEGIN - self.logger(f"[RPC] wsLaunch tRsp 非结构体 dtype=0x{dtype:02x}") - return - ins.read_head() - while True: - try: - ftag, ftype = ins.peek_head() - except EOFError: - break - if ftype == 0x0B: # STRUCT_END - ins.read_head() - break - ins.read_head() - if ftag == 0 and ftype in (0x06, 0x07): - ln = ( - ins.buf.read(1)[0] - if ftype == 0x06 - else int.from_bytes(ins.buf.read(4), "big") - ) - self._launch_guid = ins.buf.read(ln).decode("utf-8", "replace") - elif ftag == 1 and ftype in (0x06, 0x07): - ln = ( - ins.buf.read(1)[0] - if ftype == 0x06 - else int.from_bytes(ins.buf.read(4), "big") - ) - self._launch_ip = ins.buf.read(ln).decode("utf-8", "replace") - else: - ins.skip_field(ftype) - self.logger( - f"[RPC] wsLaunch 解析: guid={self._launch_guid} ip={self._launch_ip}" - ) - except Exception as e: # noqa: BLE001 外部接口与任务边界需要保留宽泛异常兜底 - self.logger(f"[RPC] wsLaunch 响应解析失败: {e}") - - @staticmethod - def _extract_wup(body: bytes) -> bytes: - """从 WSS 响应 body 中提取干净的 WUP 包 - - 响应 body 格式: - 小包: [4B BE total_len][wup_body][tail] - 大包: [1B prefix][4B BE wup_body_len][wup_body][tail] - """ - if len(body) < 5: - return body - # 大包格式需优先判断；prefix 可能是 0x00，容易被误判成小包长度 8。 - wup_len = int.from_bytes(body[1:5], "big") - if 5 + wup_len <= len(body) and body[5:7] == b"\x10\x03": - return body[5 : 5 + wup_len] - # 尝试 4B total_len 小包格式 - total_len = int.from_bytes(body[0:4], "big") - if 8 <= total_len <= len(body) and body[4:6] == b"\x10\x03": - # total_len 包含自身 4B - return body[:total_len] - if 5 + wup_len <= len(body): - return body[5 : 5 + wup_len] - return body - - async def send_auth( - self, - uid: int, - cookie: str, - sequence: int = SEQ_WSLAUNCH, - guid: str = "", - ): - """cmd 0x0a AUTH — 对齐 9.1 抓包（828 活动帧）。 - - 9.1 活动 AUTH 结构（逐字节核验）: - tag3 ZERO ← 首字段（活动通道） - tag0 INT64 = uid - tag1 STRING1 = UA - tag2 STRING4 = cookie（含 huya_ua= 前缀） - tag3 STRING1 = guid（32hex，9.1 中为页面 Cookie 的 guid） - tag4 INT8 = 1 - tag5 STRING1 = "HUYA&ZH&2052" - tag6 STRING1 = "" - += TAIL_BYTES（tag2 ZERO/tag3 ""/tag4 ZERO/tag5 ZERO/tag6 ""） - """ - os = TafOutputStream() - if self._activity_mode: - os.write_head(3, TafType.ZERO) - os.write_int64(0, int(uid or 0)) - os.write_string(1, WSS_COOKIE_UA) - os.write_string(2, HuyaWssClient._normalize_biz_cookie(cookie)) - guid_match = re.search(r"(?:^|;\s*)guid=([^;]+)", cookie or "") - auth_guid = (guid or "").strip() or (guid_match.group(1).strip() if guid_match else "") or self._launch_guid - os.write_string(3, auth_guid) - os.write_int8(4, 1) - os.write_string(5, "HUYA&ZH&2052") - os.write_string(6, "") - auth_body = os.get_bytes() + TAIL_BYTES - msg = WssMessage( - command=WssCommand.AUTH, - sequence=sequence, - body=auth_body, - ) - await self.ws.send(msg.encode()) - self.logger( - format_wss_log( - auth_body, - WssCommand.AUTH, - sequence, - "发", - ) - ) - - async def call_get_config_shop(self, timeout: float = 10.0): - """商城版 mobileui.getConfig - - tReq 结构 (HAR 实证): - tag0: UserId {lUid=0, sGuid="", sUA="", sHuYaUA="webh5&0.0.1&websocket&&h5_/index.html", - sToken="", iTokenType=0, sDeviceInfo="", sCookie=""} - tag1: 空Map - tag2: "huya" (sAppId) - tag3: 空List - """ - ua = "webh5&0.0.1&websocket&&h5_/index.html" - - os = TafOutputStream() - os.write_struct_begin(0) - # tag0: UserId (匿名) - os.write_struct_begin(0) - os.write_int64(0, 0) # lUid = 0 - os.write_string(1, "") # sGuid - os.write_string(2, "") # sUA - os.write_string(3, ua) # sHuYaUA - os.write_string(4, "") # sToken - os.write_int32(5, 0) # iTokenType - os.write_string(6, "") # sDeviceInfo - os.write_string(7, "") # sCookie - os.write_struct_end() - os.write_map(1, {}) # tag1: 空Map - os.write_string(2, "huya") # tag2: sAppId - os.write_head(3, TafType.LIST) # tag3: 空List - os.write_int32(0, 0) - os.write_struct_end() - - wup = WupRequest() - wup.setServant("mobileui") - wup.setFunc("getConfig") - wup.setRequestId(7) - wup.iTimeout = 0 - wup.newdata["tReq"] = os.get_bytes() - wup_data = wup.encode() - body = self._encode_rpc_body(wup_data) - - seq = SEQ_GETCONFIG - msg = WssMessage(command=WssCommand.RPC_REQUEST, sequence=seq, body=body) - future = asyncio.Future() - self.rpc_queue.append(future) - self.logger(format_wss_log(body, WssCommand.RPC_REQUEST, seq, "发")) - await self.ws.send(msg.encode()) - - try: - body = await asyncio.wait_for(future, timeout) - except TimeoutError: - if future in self.rpc_queue: - self.rpc_queue.remove(future) - self.logger("[✗] getConfig 超时") - return None - - return body - - async def send_register(self, guid_from_launch: str): - """cmd 0x21 REGISTER — 回传 wsLaunch 的 sGuid - - body = tag0("HUYA&ZH&2052") + tag1(sGuid) + tag2=0 + tag3=0 - + tag6 struct{tag0空Map, tag1空Map, tag3=0} - + tag7空Map + tag8=0 + TAIL - """ - os = TafOutputStream() - os.write_string(0, "HUYA&ZH&2052") - os.write_string(1, guid_from_launch) - os.write_int32(2, 0) - os.write_int32(3, 0) - os.write_struct_begin(6) - os.write_map(0, {}) - os.write_map(1, {}) - os.write_int32(3, 0) - os.write_struct_end() - os.write_map(7, {}) - os.write_int32(8, 0) - body = os.get_bytes() + TAIL_BYTES - msg = WssMessage(command=WssCommand.REGISTER, sequence=SEQ_REGISTER, body=body) - self.logger(format_wss_log(body, WssCommand.REGISTER, SEQ_REGISTER, "发")) - await self.ws.send(msg.encode()) - - async def send_confirm(self): - """cmd 0x17 CONFIRM_SEND — body = 空Map(tag0) + TAIL""" - os = TafOutputStream() - os.write_map(0, {}) - body = os.get_bytes() + TAIL_BYTES - msg = WssMessage( - command=WssCommand.CONFIRM_SEND, sequence=SEQ_CONFIRM, body=body - ) - self.logger(format_wss_log(body, WssCommand.CONFIRM_SEND, SEQ_CONFIRM, "发")) - await self.ws.send(msg.encode()) - - async def call_rpc( - self, - service: str, - method: str, - req_struct: TafStruct, - rsp_class=None, - timeout: float = 10.0, - ): - seq = self._next_biz_seq() - - wup = WupRequest() - wup.setServant(service) - wup.setFunc(method) - wup.setRequestId(self._next_wup_req_id()) - wup.iTimeout = 0 - wup.writeStruct("tReq", req_struct) - wup_data = wup.encode() - body = self._encode_rpc_body(wup_data) - - msg = WssMessage(command=WssCommand.RPC_REQUEST, sequence=seq, body=body) - future = asyncio.Future() - self.rpc_queue.append(future) - self.logger(format_wss_log(body, WssCommand.RPC_REQUEST, seq, "发")) - await self.ws.send(msg.encode()) - - try: - body = await asyncio.wait_for(future, timeout) - except TimeoutError: - if future in self.rpc_queue: - self.rpc_queue.remove(future) - self.logger(f"[✗] {service}.{method} 超时") - return None - - wup_resp = WupResponse() - wup_resp.decode(self._extract_wup(body)) - # 响应日志已由 _recv_loop 打出，此处只打解码后的业务字段 - for key in ("tRsp", "tResp"): - data = wup_resp.newdata.get(key) - if data and isinstance(data, bytes) and len(data) > 0: - try: - ins = TafInputStream(data) - _tag, dtype = ins.peek_head() - if dtype == TafType.STRUCT_BEGIN: - ins.read_head() - decoded = _decode_taf_struct(ins) - self.logger( - f"[←] {service}.{method} {key}: {_truncate(decoded)}" - ) - except Exception as exc: # noqa: BLE001 - self.logger( - f"[debug] WUP 响应字段解码失败: {service}.{method}.{key}: {exc}" - ) - - if rsp_class is None: - return body - - result = wup_resp.readStruct("tRsp", rsp_class) - if result is None: - result = wup_resp.readStruct("tResp", rsp_class) - return result - - async def get_goods_info( - self, - uid: int, - guid: str, - cookie: str, - pid: int, - spu_id: str, - sku_id: int = 0, - game_id: str = "", - source_id: str = "yellowcarlist", - scene: int = WSS_SHOP_SCENE, - ): - from .shop_structs import GetGoodsInfoReqV5, GoodsInfoRsp - - req = GetGoodsInfoReqV5() - req.userId = self._build_biz_user(uid, cookie) - req.shopAppInfo = self._build_shop_app(source_id, scene) - req.pid = pid - req.spuId = spu_id - req.skuId = sku_id - req.gameId = game_id - - return await self.call_rpc( - "shopMiddleUI", "getGoodsInfoV5", req, GoodsInfoRsp, timeout=15.0 - ) - - async def get_act_info(self, act_id: int): - from .activity_structs import GetActInfoReq, GetActInfoResp - - req = GetActInfoReq() - req.actId = int(act_id or 0) - return await self.call_rpc("webActUI", "getActInfo", req, GetActInfoResp) - - async def get_act_task_detail(self, uid: int, cookie: str, act_id: int): - from .activity_structs import GetActTaskDetailReq, GetActTaskDetailResp - - req = GetActTaskDetailReq() - req.userId = self._build_activity_user(uid, cookie) - req.actId = int(act_id or 0) - return await self.call_rpc("webActUI", "getActTaskDetail", req, GetActTaskDetailResp) - - async def get_act_user_task_detail(self, uid: int, cookie: str, act_id: int): - from .activity_structs import GetActUserTaskDetailReq, GetActUserTaskDetailResp - - req = GetActUserTaskDetailReq() - req.userId = self._build_activity_user(uid, cookie) - req.actId = int(act_id or 0) - return await self.call_rpc("webActUI", "getActUserTaskDetail", req, GetActUserTaskDetailResp) - - async def get_user_score(self, uid: int, cookie: str, sid: int): - from .activity_structs import GetUserScoreReq, GetUserScoreResp - - req = GetUserScoreReq() - req.userId = self._build_activity_user(uid, cookie) - req.sid = int(sid or 0) - return await self.call_rpc("webActUI", "getUserScore", req, GetUserScoreResp) - - async def get_act_prize_list(self, uid: int, cookie: str, sid: int): - from .activity_structs import GetActPrizeListReq, GetActPrizeListResp - - req = GetActPrizeListReq() - req.userId = self._build_activity_user(uid, cookie) - req.sid = int(sid or 0) - return await self.call_rpc("webActUI", "getActPrizeList", req, GetActPrizeListResp) - - async def get_act_prize_detail(self, uid: int, cookie: str, sid: int, pid: int): - from .activity_structs import GetActPrizeDetailReq, GetActPrizeDetailResp - - req = GetActPrizeDetailReq() - req.userId = self._build_activity_user(uid, cookie) - req.sid = int(sid or 0) - req.pid = int(pid or 0) - return await self.call_rpc("webActUI", "getActPrizeDetail", req, GetActPrizeDetailResp) - - async def score_exchange_prize(self, uid: int, cookie: str, sid: int, pid: int): - from .activity_structs import ScoreExchangePrizeReq, ScoreExchangePrizeResp - - req = ScoreExchangePrizeReq() - req.userId = self._build_activity_user(uid, cookie) - req.sid = int(sid or 0) - req.pid = int(pid or 0) - return await self.call_rpc("webActUI", "scoreExchangePrize", req, ScoreExchangePrizeResp) - - async def get_user_prize_records(self, uid: int, cookie: str, sid: int): - from .activity_structs import GetUserPrizeRecordsReq, GetUserPrizeRecordsResp - - req = GetUserPrizeRecordsReq() - req.userId = self._build_activity_user(uid, cookie) - req.sid = int(sid or 0) - return await self.call_rpc("webActUI", "getUserPrizeRecords", req, GetUserPrizeRecordsResp) - - async def check_user_bind_game_account( - self, uid: int, cookie: str, b_act_id: int, is_use_outer_act_id: int = 1, - gid: int = 0, outer_act_id: str = "", scene: str = "", - ): - from .activity_structs import ( - CheckUserBindGameAccountReq, - CheckUserBindGameAccountResp, - ) - - req = CheckUserBindGameAccountReq() - req.userId = self._build_activity_user(uid, cookie) - req.gid = int(gid or 0) - req.outerActId = str(outer_act_id or "") - req.scene = scene or "" - req.bActId = int(b_act_id or 0) - req.isUseOuterActId = int(is_use_outer_act_id or 0) - return await self.call_rpc( - "webActUI", "checkUserBindGameAccount", req, CheckUserBindGameAccountResp - ) - - async def confirm_bind_act_account( - self, uid: int, cookie: str, b_act_id: int, gid: int = 0, outer_act_id: str = "" - ): - from .activity_structs import ( - ConfirmBindActAccountReq, - ConfirmBindActAccountResp, - ) - - req = ConfirmBindActAccountReq() - req.userId = self._build_activity_user(uid, cookie) - req.gid = int(gid or 0) - req.outerActId = str(outer_act_id or "") - req.bActId = int(b_act_id or 0) - return await self.call_rpc( - "webActUI", "confirmBindActAccount", req, ConfirmBindActAccountResp - ) - - async def get_live_link_param( - self, uid: int, cookie: str, b_act_id: int, gid: int = 0, - outer_act_id: int = 0, game_auth_scene: str = "", - ): - from .activity_structs import GetLiveLinkParamReq, GetLiveLinkParamResp - - req = GetLiveLinkParamReq() - req.userId = self._build_activity_user(uid, cookie) - req.gid = int(gid or 0) - req.outerActId = int(outer_act_id or 0) - req.gameAuthScene = game_auth_scene or "" - req.bActId = int(b_act_id or 0) - return await self.call_rpc("webActUI", "getLiveLinkParam", req, GetLiveLinkParamResp) - - async def get_user_profile_batch(self, uid: int, cookie: str, target_uids: list[int]): - from .activity_structs import GetUserProfileBatchReq, GetUserProfileBatchResp - - req = GetUserProfileBatchReq() - req.userId = self._build_activity_user(0, "") - req.uidList = [int(item) for item in target_uids if int(item or 0)] - if not req.uidList: - return None - return await self.call_rpc( - "huyauserui", "getUserProfileBatch", req, GetUserProfileBatchResp - ) - - async def list_pay_channels( - self, uid: int, guid: str, cookie: str, spu_id: str, sku_id: int = 0, - supplier_id: int = 87401, - ): - from .shop_structs import ListPayChannelReq, ListPayChannelRsp - - req = ListPayChannelReq() - req.userId = self._build_biz_user(uid, cookie) - req.shopAppInfo = self._build_shop_app("yellowcarlist", WSS_SHOP_SCENE) - req.supplierId = int(supplier_id or 0) - req.spuId = spu_id - req.skuId = int(sku_id or 0) - return await self.call_rpc( - "shopMiddleUI", "listPayChannelV5", req, ListPayChannelRsp, timeout=15.0 - ) - - async def check_hy_protocol(self, uid: int, guid: str, cookie: str): - from .shop_structs import CheckHyProtocolReq, ShopCodeRsp - - req = CheckHyProtocolReq() - req.userId = self._build_biz_user(uid, cookie) - req.shopAppInfo = self._build_shop_app("yellowcarlist", WSS_SHOP_SCENE) - return await self.call_rpc( - "shopMiddleUI", "checkHyProtocolV5", req, ShopCodeRsp, timeout=15.0 - ) - - async def check_user_buy_auth( - self, uid: int, guid: str, cookie: str, spu_id: str, sku_id: int, - ): - from .shop_structs import CheckUserBuyAuthReq, ShopCodeRsp - - req = CheckUserBuyAuthReq() - req.userId = self._build_biz_user(uid, cookie) - req.shopAppInfo = self._build_shop_app("yellowcarlist", WSS_SHOP_SCENE) - req.spuId = spu_id - req.skuId = int(sku_id or 0) - return await self.call_rpc( - "shopMiddleUI", "checkUserBuyAuth", req, ShopCodeRsp, timeout=15.0 - ) - - async def query_user_order_list( - self, - uid: int, - guid: str, - cookie: str, - offset: int = 0, - page_size: int = 10, - order_type: int = 1, - status: int = 0, - ): - from .shop_structs import QueryUserOrderListReq, QueryUserOrderListRsp - - req = QueryUserOrderListReq() - req.userId = self._build_biz_user(uid, cookie) - req.offset = offset - req.orderType = order_type - req.pageSize = page_size - req.status = status - - return await self.call_rpc( - "revenueWebUI", - "queryUserOrderList", - req, - QueryUserOrderListRsp, - timeout=15.0, - ) - - async def order_detail( - self, - uid: int, - guid: str, - cookie: str, - order_id: int, - source_id: str = "yellowcarlist", - scene: int = WSS_SHOP_SCENE, - ): - """按订单号查询商城订单详情，匹配 9.1 支付后轮询。""" - from .shop_structs import OrderDetailReq, OrderDetailRsp - - req = OrderDetailReq() - req.userId = self._build_biz_user(uid, cookie) - req.shopAppInfo = self._build_shop_app(source_id, scene) - req.orderId = int(order_id or 0) - return await self.call_rpc( - "shopMiddleUI", "orderDetailV5", req, OrderDetailRsp, timeout=15.0 - ) - - async def create_order( - self, - uid: int, - guid: str, - cookie: str, - pid: int, - spu_id: str, - sku_id: int, - item_count: int = 1, - source_id: str = "yellowcarlist", - game_id: str = "", - scene: int = WSS_SHOP_SCENE, - order_type: int = 6, - ): - from .shop_structs import ( - CreateOrderAccountParam, - CreateOrderExtraParam, - CreateOrderPromotionParam, - CreateOrderReqV5, - CreateOrderRsp, - ) - - req = CreateOrderReqV5() - req.userId = self._build_biz_user(uid, cookie) - req.shopAppInfo = self._build_shop_app(source_id, scene) - req.receiveId = 0 - req.pid = pid - req.spuId = spu_id - req.skuId = sku_id - req.itemCount = item_count - req.gameId = game_id or "0" - req.src = scene - req.scene = 0 - req.sourceId = source_id - req.orderType = order_type - req.extraParam = CreateOrderExtraParam() - req.env = self._build_order_env(item_count) - req.orderScene = scene - req.promotionParam = CreateOrderPromotionParam() - req.accountParam = CreateOrderAccountParam() - req.bizType = 5 - req.gameCategoryId = 507 - req.ext = "" - - self.logger(f"[下单] orderType={order_type} (HUYA_VIRTUAL)") - return await self.call_rpc( - "shopMiddleUI", "createOrderV5", req, CreateOrderRsp, timeout=15.0 - ) - - async def pay_order_submit( - self, - uid: int, - guid: str, - cookie: str, - order_id: int, - pay_type: int = 1, - pid: int = 0, - source_id: str = "yellowcarlist", - scene: int = WSS_SHOP_SCENE, - item_count: int = 1, - ): - from .shop_structs import PayOrderRes - - os = TafOutputStream() - os.write_struct_begin(0) - - os.write_struct(0, self._build_biz_user(uid, cookie)) - os.write_struct(1, self._build_shop_app(source_id, scene)) - os.write_int64(2, order_id) - os.write_string(3, "Zfb" if pay_type == 1 else str(pay_type)) - os.write_string(4, "QrCode") - callback_url = ( - f"https://m-shop.yaoguo.com/index.html#/consumer/paycallback" - f"?orderId={order_id}&pid={pid}&sourceId={source_id}" - ) - os.write_string(5, callback_url) - os.write_map(6, self._build_order_env(item_count)) - os.write_string(7, "null") - - os.write_struct_end() - - seq = self._next_biz_seq() - - wup = WupRequest() - wup.setServant("shopMiddleUI") - wup.setFunc("payOrderSubmitV5") - wup.setRequestId(self._next_wup_req_id()) - wup.iTimeout = 0 - wup.newdata["tReq"] = os.get_bytes() - wup_data = wup.encode() - body = self._encode_rpc_body(wup_data) - - msg = WssMessage(command=WssCommand.RPC_REQUEST, sequence=seq, body=body) - future = asyncio.Future() - self.rpc_queue.append(future) - self.logger(format_wss_log(body, WssCommand.RPC_REQUEST, seq, "发")) - await self.ws.send(msg.encode()) - - try: - body = await asyncio.wait_for(future, 15.0) - except TimeoutError: - if future in self.rpc_queue: - self.rpc_queue.remove(future) - self.logger("[✗] payOrderSubmitV5 超时") - return None - - wup_resp = WupResponse() - wup_resp.decode(self._extract_wup(body)) - for key in ("tRsp", "tResp"): - data = wup_resp.newdata.get(key) - if data and isinstance(data, bytes) and len(data) > 0: - try: - ins = TafInputStream(data) - _tag, dtype = ins.peek_head() - if dtype == TafType.STRUCT_BEGIN: - ins.read_head() - decoded = _decode_taf_struct(ins) - self.logger(f"[←] payOrderSubmitV5 {key}: {_truncate(decoded)}") - except Exception as exc: # noqa: BLE001 - self.logger(f"[debug] WUP 支付响应字段解码失败: {key}: {exc}") - result = wup_resp.readStruct("tRsp", PayOrderRes) - if result is None: - result = wup_resp.readStruct("tResp", PayOrderRes) - return result diff --git a/artifacts/guid_session_consistency/ROLLBACK.sh b/artifacts/guid_session_consistency/ROLLBACK.sh deleted file mode 100755 index 7b146bf..0000000 --- a/artifacts/guid_session_consistency/ROLLBACK.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/sh -set -eu - -target=${1:-core/huya/wss_client.py} -baseline_ref=${GUID_SESSION_BASELINE_REF:-HEAD} -root=$(git rev-parse --show-toplevel) -case "$target" in - "$root"/*) file=$target; rel=${target#"$root"/} ;; - /*) file=$target; rel=core/huya/wss_client.py ;; - *) file="$root/$target"; rel=$target ;; -esac -git show "$baseline_ref:$rel" > "$file" -printf '已恢复 %s（基线 %s）\n' "$file" "$baseline_ref" diff --git a/artifacts/guid_session_consistency/VERIFICATION.txt b/artifacts/guid_session_consistency/VERIFICATION.txt deleted file mode 100644 index 8ac0093..0000000 --- a/artifacts/guid_session_consistency/VERIFICATION.txt +++ /dev/null @@ -1,42 +0,0 @@ -guid 会话一致性验证记录 - -changed branch/field: core/huya/wss_client.py::HuyaWssClient._cookie_with_guid、initialize_activity、send_auth；core/huya/elite_session.py::HuyaEliteWssSession.__init__ 初始化后的会话 guid 回写。 - -MODIFIED_FILE: /Users/yml/codes/live-hub-py/artifacts/guid_session_consistency/MODIFIED_FILE -DIFF_FILE: /Users/yml/codes/live-hub-py/artifacts/guid_session_consistency/DIFF_FILE -VERIFICATION.txt: /Users/yml/codes/live-hub-py/artifacts/guid_session_consistency/VERIFICATION.txt -ROLLBACK.sh: /Users/yml/codes/live-hub-py/artifacts/guid_session_consistency/ROLLBACK.sh (executable) - -BASELINE -command: /Users/yml/codes/live-hub-py/.venv/bin/pytest -q -input: HEAD baseline before guid consistency change -literal output/result: 151 passed, 1 warning in 1.34s -exit status: 0 -restored behavior/status: baseline test copy remained unchanged - -MODIFIED -command: /Users/yml/codes/live-hub-py/.venv/bin/pytest -q -input: current worktree with wsLaunch sGuid propagation and regression tests -literal output/result: 153 passed, 1 warning in 1.34s -exit status: 0 -additional checks: -command: /Users/yml/codes/live-hub-py/.venv/bin/ruff check core/huya/wss_client.py core/huya/elite_session.py tests/test_huya_elite_protocol.py -literal output/result: All checks passed! -exit status: 0 -command: git diff --check -literal output/result: no output (clean) -exit status: 0 - -READ-ONLY ONLINE -command: DB_HOST=127.0.0.1 DB_PORT=3307 DB_NAME=douyu_login DB_USER=douyu_login /Users/yml/codes/live-hub-py/.venv/bin/python -input: account id=60; activity WSS; wsLaunch + getUserScore only; no exchange/order/payment calls -literal output/result: old_guid=0a43b01a366c966a7fe26a281e1d3c88; launch_guid=0a89d51717f5976ab502cf9dd09d03d8; session_cookie_guid=0a89d51717f5976ab502cf9dd09d03d8; guid_consistent=True; score_status=200; score=260; elapsed=0.286s -exit status: 0 -restored behavior/status: WSS session closed; no write operation invoked - -ROLLBACK -command: /Users/yml/codes/live-hub-py/artifacts/guid_session_consistency/ROLLBACK.sh /tmp/live-hub-guid-rollback/wss_client.py -input: independent copy of MODIFIED_FILE; GUID_SESSION_BASELINE_REF=HEAD -literal output/result: 已恢复 /tmp/live-hub-guid-rollback/wss_client.py（基线 HEAD） -exit status: 0 -restored hash/status: independent copy matched git show HEAD:core/huya/wss_client.py; current worktree remains on modified branch diff --git a/docs/生产服务器高负载排查分析报告.md b/docs/生产服务器高负载排查分析报告.md new file mode 100644 index 0000000..0d65229 --- /dev/null +++ b/docs/生产服务器高负载排查分析报告.md @@ -0,0 +1,210 @@ +# 生产服务器高负载排查与性能分析报告 + +> 采集时间：2026-10-03 12:35 ~ 12:38 CST +> 目标服务器：`order+douyu_huya` (47.114.114.225) +> 规格：4 vCPU / 7.1 GiB RAM / Ubuntu 24.04 (Linux WIN10 7.0.0-22-generic) + +--- + +## 1. 调查状态卡片（已知 / 未知 / 目标 / 确信度） + +| 维度 | 内容说明 | 确信度 | +| :--- | :--- | :---: | +| **已知 (Known)** | 1. 宿主机平均负载高达 **17.09, 13.75, 11.38**，4 核 CPU 几乎全部跑满（376%）。
2. 宿主机承载了两个业务体系：**接单平台 (`order_site`)** 和 **直播自动任务平台 (`live-hub-py`)**。
3. `order_site-postgres-1` 占用 CPU 高达 **197.75%**（吃掉 2 个核）；`live-hub-py-douyu-login-1` 占 **90.53%**（吃掉近 1 个核）；`live-hub-py-mysql-1` 占 **49.07%**；`order_site-backend-1` 占 **39.75%**。
4. Postgres 的 CPU 飙升主要是打手端工单统计查询 `getWorkerOrderOverview` 与列表查询，触发了 **PostgreSQL LLVM JIT 动态编译**，单次查询中 JIT 耗时达 **7.21 秒**（总耗时 8.8 秒）。
5. MySQL 的高 CPU 和 384GB 读 IO 是因为 `douyu_tasks` (23.8万条) 与 `accounts` 联表带 `assigned_to` 过滤时，执行计划使用了 **`Using temporary; Using filesort`**，高频轮询产生大量临时表全排序。
6. `live-hub-py-douyu-login-1` 当前正在持续、高并发执行斗鱼任务（供应商直充、赠送精英令、代理池多并发探测）。 | **100%（已由真实命令、日志与 EXPLAIN 实证）** | +| **未知 (Unknown)** | 1. 打手端前端当前轮询的频率（是否为 2~3 秒一次硬轮询）。
2. 是否可以在 Postgres 配置中全局关闭 JIT (`jit = off`)，或者仅在业务连接池上设置。
3. `douyu_tasks` 的管理后台当前是哪个页面在轮询 `assigned_to = 13` 的任务列表。
4. live-hub-py 的当前斗鱼任务并发限制是否允许动态限速。 | **待确认** | +| **目标 (Objective)** | 1. **根治 Postgres 200% CPU 占用**：解决打手端统计与列表慢查询（禁用 JIT、优化 CTE/索引、增加缓存或降低轮询频率）。
2. **降低 MySQL 50% CPU 与 384GB IO 占用**：优化 `douyu_tasks` 的 `assigned_to` 联表排序查询，消除 `Using temporary; Using filesort`。
3. **平抑宿主机 Load Average 至 3.0 以下**，确保接单平台与直播充值任务稳定运行。 | **明确** | + +--- + +## 2. 证据分类记录 (Evidence & Forensics) + +### 证据 A：宿主机系统资源与 Docker 容器负载 + +**执行证据：** +```bash +# uptime +12:33:44 up 43 days, 14:12, 3 users, load average: 17.09, 13.75, 11.38 + +# nproc & free -h +4 vCPU +Mem: total 7.1Gi, used 4.1Gi, free 351Mi, buff/cache 4.1Gi, available 3.0Gi +Swap: total 4.0Gi, used 577Mi, free 3.4Gi + +# docker stats --no-stream +CONTAINER ID NAME CPU % MEM USAGE / LIMIT MEM % NET I/O BLOCK I/O +301d6a15a032 live-hub-py-douyu-login-1 90.53% 593.4MiB / 2GiB 28.97% 2.95GB / 234MB 116MB / 22.2MB +04111b7b40ae live-hub-py-mysql-1 49.07% 895.6MiB / 2GiB 43.73% 532MB / 11.8GB 384GB / 21.8GB +d1831428e110 order_site-postgres-1 197.75% 1.708GiB / 2.5GiB 68.34% 2.88GB / 24.9GB 6.79GB / 9.84GB +0fc436d47135 order_site-backend-1 39.75% 289.2MiB / 1GiB 28.24% 28GB / 14.8GB 278MB / 130MB +``` + +**分析结论：** +总 CPU 占用合计超过 **376%**（4 核饱和运行）。其中 `order_site-postgres-1` 占大头（~200%），`live-hub-py` 容器占第二（~90%），`mysql` 占第三（~50%）。 + +--- + +### 证据 B：Postgres 慢查询与 JIT 编译瓶颈 + +**慢查询代码位置：** +`/home/order_site/apps/backend/src/repositories/worker-platform/work-order-query-repo.ts:463` +函数：`getWorkerOrderOverview(workerId)` + +**核心 SQL：** +```sql +WITH visible_orders AS ( + SELECT wo.id AS work_order_id FROM work_orders wo WHERE wo.assigned_worker_id = $1 + UNION + SELECT wo.id AS work_order_id FROM work_orders wo WHERE wo.last_assigned_worker_id = $1 + AND (wo.status <> 'open' OR EXISTS ( + SELECT 1 FROM work_order_events timeout_event + WHERE timeout_event.work_order_id = wo.id AND timeout_event.event_type = 'timeout_reopen' + )) + UNION + SELECT wos.work_order_id FROM work_order_shares wos WHERE wos.worker_id = $1 AND wos.status != 'cancelled' +), +worker_base AS ( ... ), +worker_orders AS ( ... ) +SELECT status, COUNT(*)::int AS order_count, ... FROM worker_orders GROUP BY status; +``` + +**EXPLAIN ANALYZE 关键指标（Worker 7，拥有 6895 单）：** +```text +JIT: + Functions: 63 + Options: Inlining true, Optimization true, Expressions true, Deforming true + Timing: Generation 82.769 ms, Inlining 2197.205 ms, Optimization 2832.841 ms, Emission 2098.693 ms, Total 7211.508 ms +Execution Time: 8803.367 ms +``` +- **开启 JIT（默认）**：耗时 **8803 ms**，其中 **7211 ms (82%)** 全耗在 LLVM JIT 动态代码生成与优化上，直接霸占 CPU 算力！ +- **关闭 JIT（SET jit = off）**：耗时从 **8803 ms 降为 1174 ms**，瞬间省掉 7.2 秒 CPU 运算！ + +--- + +### 证据 C：MySQL 联表临时表与文件排序瓶颈 + +**慢查询定位：** +```sql +SELECT douyu_tasks.id AS douyu_tasks_id, ... +FROM douyu_tasks +INNER JOIN accounts ON accounts.id = douyu_tasks.account_id +LEFT OUTER JOIN accounts AS accounts_1 ON accounts_1.id = douyu_tasks.account_id +WHERE accounts.assigned_to = 13 + AND (douyu_tasks.handbook_scope = 'elite' OR douyu_tasks.handbook_scope = 'legacy' AND douyu_tasks.task_type IN (...)) +ORDER BY douyu_tasks.id DESC +LIMIT 100; +``` + +**EXPLAIN 诊断结果：** +```text +id: 1, table: accounts, type: ref, key: ix_accounts_assigned_to, rows: 734, + Extra: Using index; Using temporary; Using filesort +id: 1, table: douyu_tasks, type: ref, key: ix_douyu_tasks_account_id, rows: 19, + Extra: Using where +``` +- **诊断**：`accounts` 是驱动表，筛选 `assigned_to = 13` 的 734 个账号，关联出几万条 `douyu_tasks`。由于最终是按 `douyu_tasks.id DESC` 排序，MySQL 必须在临时表中对所有关联记录做 **`filesort` 文件全排序**。 +- 这导致了 MySQL 容器高达 **384GB** 的累计磁盘读取，以及 **49%** 的持续 CPU 消耗。 + +--- + +### 证据 D：live-hub-py 运行中的真实业务负载 + +**容器日志证据：** +`live-hub-py-douyu-login-1` 正在同时运行多路斗鱼充值与精英令业务： +- `[douyu] 供应商直充 | 下单 | 外部单号=...` +- `[douyu] 赠送精英令成功: 770 ...` +- `[douyu] 代理预检 1/3: 正在获取代理 -> 验证 -> 获取新代理` +- 大量 WebSocket 客户端 `/api/douyu/ws/...` 连接，属于高并发高吞吐的正常业务处理流，但与 Postgres/MySQL 的慢查询争抢 CPU，加剧了系统卡顿。 + +--- + +## 3. 核心根因归纳 (Root Causes) + +1. **PostgreSQL JIT 编译放大器（最主要的 CPU 杀手）**： + PostgreSQL 对高 Cost 查询默认启用 JIT，而打手端概览/列表 SQL 结构极其复杂，每次命中都触发 63 个 LLVM 函数编译，耗时 7.2 秒，多打手并发轮询时直接把 4 核打穿。 +2. **PostgreSQL CTE 关联全索引扫描**： + 在 `worker_base` 中将 `visible_orders`（几千条）与 `work_orders`（10.2万条）Hash Join 时，引发了 `work_orders` 10 万行的大范围 Index Scan。 +3. **MySQL 任务查询缺少最适索引与高效分页**： + 按归属人 (`assigned_to`) 筛选并按任务 ID 倒排的接口缺少高效索引覆盖，每次调用都产生临时表和文件排序。 +4. **单机混部资源无硬隔离**： + 4 核服务器同时跑了 2 个数据库（Postgres 16 + MySQL 8.4）和 2 套高并发业务（Node.js + Python Uvicorn），当任意一个查询发生慢查重试时，CPU 瞬间雪崩。 + +--- + +## 4. 优化与降压建议方案 (Action Plan) + +### 第一阶段：立刻生效的立竿见影措施（1-2分钟内降低 50%~70% CPU） + +1. **关闭 PostgreSQL 的 JIT**： + 在 PostgreSQL 配置或 Node 连接池初始化时执行 `ALTER SYSTEM SET jit = off; SELECT pg_reload_conf();`。 + - **预期效果**：该接口单次 CPU 开销直接下降 **80%**，Postgres CPU 占用将显著回落。 +2. **增加打手端概览查询缓存**： + 打手端 `getWorkerOrderOverview` 增加 5~10 秒的内存缓存（LRU / Redis / Node 内存 Map），避免同一打手每次切换页面或轮询都直穿数据库。 + +### 第二阶段：SQL 与索引专项优化 + +1. **MySQL `douyu_tasks` 联合查询优化**： + 针对 `douyu_tasks.account_id` 与 `accounts.assigned_to`，评估是否在 `douyu_tasks` 上冗余 `assigned_to` 字段或建立更优的联表分页索引，避免全量 filesort。 +2. **PostgreSQL `work-order-query-repo.ts` 结构重构**： + 优化 CTE 逻辑，将 `UNION` 改为更精准的 `UNION ALL`，避免大集合 Hash 去重；对于 `worker_order_cancel_requests` 和 `worker_order_feedbacks` 的 EXISTS 检查进行合并或物化过滤。 + +### 第三阶段：资源配额防护 + +在 `docker-compose.yml` 中为各服务合理配置 `cpus` 限制，防止任一服务突发异常慢查询占满全部宿主机核心。 + +--- + +## 5. 运维操作与代码分支推进路线 (Deployment & Git Flow) + +按照标准工程规范，严禁在生产服务器上随意手工热修业务代码，杜绝“代码漂移 (Drift)”和更新覆盖风险。 + +### 推进原则： +1. **数据库运行时参数紧急止血**： + - 生产环境执行 `ALTER SYSTEM SET jit = off; SELECT pg_reload_conf();`（动态热生效，不涉及源码变更，解脱当前 200% JIT CPU 占用）。 +2. **工作区分流与保护**： + - 将当前 `live-hub-py` 本地未提交的改动暂存或切换至 `dev` 分支，保持 `main` 分支干净。 +3. **本地开发与正规发布**： + - **`order_site` 项目**（本地路径：`/Users/yml/codes/order_site`）：在本地优化 `work-order-query-repo.ts` 的 CTE 与打手端概览缓存，本地验证通过后提交至对应 Git 分支，通过正规部署更新生产。 + - **`live-hub-py` 项目**（本地路径：`/Users/yml/codes/live-hub-py`）：在 `dev` 分支优化 `douyu_tasks` 的 `assigned_to` 联表索引与查询，测试通过后合并部署。 + +--- + +## 6. 紧急止血效果实测证据 (Post-Fix Verification) + +执行时间：2026-10-03 12:42:04 CST +执行命令：`ALTER SYSTEM SET jit = off; SELECT pg_reload_conf();` + +### 止血前后实时指标对比表： + +| 监控项 | 优化前 (12:34) | 优化后 (12:43) | 变化幅度 | 状态 | +| :--- | :---: | :---: | :---: | :---: | +| **PostgreSQL 容器 CPU** | **197.75%** | **1.58%** | **暴降 99.2%** | 彻底恢复健康 | +| **MySQL 容器 CPU** | **49.07%** | **4.46%** | **暴降 90.9%** | 大幅改善 | +| **宿主机 1 分钟平均负载** | **17.09** | **9.52 (持续下探中)** | **下降 44.3%** | 迅速脱险 | +| **打手概览单次查询耗时** | **8803 ms** | **369 ms** | **提速 23 倍** | 质的飞跃 | + +--- + +## 7. 忽高忽低脉冲式负载成因与周期对齐分析 (Pulse & Spike Analysis) + +通过对宿主机与各容器进行连续多点动态采样（12:50 ~ 12:54），证实了服务器负载**“忽高忽低（从 1.5% 瞬间飙至 200%+ 后又回落）”**的根本机理：**四大不同周期的脉冲任务重叠交汇**。 + +### 连续采样实证（2秒间隔）： +- `12:51:51`：Postgres 158%, MySQL 49%, Node 35%, Python 11% +- `12:51:55`：Postgres 171%, MySQL 65%, Node 22%, Python 72% +- `12:51:59`：Postgres 138%, MySQL 111%, Node 102%, Python 36% +- `12:52:04`：Postgres 155%, MySQL 104%, Node 104%, Python 56% +- `12:52:09`：Postgres 211%, MySQL 66%, Node 76%, Python 4% +- `12:52:33`：Postgres 瞬间恢复为 0 个活跃查询，CPU 骤降 + +### 四大脉冲周期拆解： + +| 脉冲源 | 周期 | 核心动作 / 触发条件 | 瞬间 CPU 峰值 | +| :--- | :---: | :--- | :---: | +| **① MySQL 任务全表排序** | **~3~5 秒** | 客服人员打开任务工作台，前端轮询 `/api/douyu/tasks`；因按 `assigned_to` 过滤且 `ORDER BY id DESC`，MySQL 在 24 万条任务中做 `Using temporary; Using filesort`。 | **4% -> 111%** | +| **② PostgreSQL 复合聚合风暴** | **~5~15 秒** | 虽已关掉 JIT（消除 7.2s 编译死锁），但三大重型聚合被频繁触发：
1. 管理端全局统计 `getWorkOrderStatistics`（10万工单与28万订单 LEFT JOIN 做全量 SUM）；
2. 打手端完成单统计 `countWorkerAcceptedOrders`（COUNT DISTINCT）；
3. 打手概览 `getWorkerOrderOverview`。
每次命中启动多 Parallel Workers 瞬间吃满 150%~210% CPU，执行完后立即归零。 | **1.5% -> 211%** | +| **③ Node 定时同步与派单** | **20 秒整** | `scheduler-service` 每 20 秒执行一次 `pdd_order_sync`（调用拼多多防封算法拉单并派发消费者分流），每 60 秒执行超时检测与押金解冻。 | **22% -> 104%** | +| **④ Python 充值与绑定波峰** | **任务驱动** | 斗鱼供应商充值下单、每隔 5 秒一次的 13 轮绑定状态轮询、代理池高并发探测。 | **4% -> 73%** | + +### 核心结论： +优化 JIT 后，PostgreSQL **不再陷入 10~20 秒的持续假死**（这是平均 Load 从 17 降下来的原因）；但由于上述 4 个周期任务缺少**防抖/短缓存**和**关键索引**，当它们在某一秒**同时触发**时，4 核 CPU 依然会瞬间出现 1~2 秒的 100% 尖刺（Spike），这就是用户感知到“忽高忽低”的根源。 + diff --git a/docs/自动化与资料收集规划.md b/docs/自动化与资料收集规划.md index fa8158e..98d90a1 100644 --- a/docs/自动化与资料收集规划.md +++ b/docs/自动化与资料收集规划.md @@ -1381,6 +1381,15 @@ OSS_PREFIX=collect `docker exec -w /app <容器> env VIRTUAL_ENV=/app/.venv uv pip install <包>`； 另：`docker exec -e` 覆盖不了 compose 已注入的同名变量，验证时改用容器自带的 env 或在脚本里 `os.environ[...] = ...`。 +- **⚠️ `create_all` 不会修改已存在的表**：测试库靠 `Base.metadata.create_all()` 建表， + 它**只建缺失的表**。给**已有表**加列/改列后，测试库里的旧结构不会更新，症状是一堆 + **不相关**的用例同时挂（例如加 `audit_logs.account_id` 后 22 个用例报 + `Unknown column 'account_id'`）。处理：改已有表的模型后，对测试库手动 `ALTER TABLE`， + 或 `DROP TABLE` 让它按新模型重建。**新增表**不受影响（会直接建出来）； +- **灌测试数据要带 `--default-character-set=utf8mb4`**：`docker exec -i mysql ... < file` + 不带这个参数时中文会被双重编码（`套装` 存成 `套装`）。排查这类问题的快捷判断： + 对可疑列取 `HEX()`，若是 `C3A5C2A5...` 这种「UTF-8 再编码一次」的形态， + 说明是**写坏了**而不是读坏了。 ### C.3 已核实的环境事实 diff --git a/docs/设计-登录任务与CK解耦.md b/docs/设计-登录任务与CK解耦.md index 93124ff..5e613db 100644 --- a/docs/设计-登录任务与CK解耦.md +++ b/docs/设计-登录任务与CK解耦.md @@ -121,8 +121,43 @@ WHERE a.cookie IS NULL OR a.cookie = ''; | 持有 CK 的**行**数 | 13265 | | 其中多行持 CK 的账号 | 1（回填取 id 最大的那行） | -迁移后 `SELECT COUNT(*) FROM accounts WHERE cookie <> ''` **必须等于 13264** -（之后随登录增长，只增不减）。 +迁移后 `SELECT COUNT(*) FROM accounts WHERE cookie <> ''` **必须等于**记录级去重数 +（之后随登录增长，只增不减；基线是动态的，迁移里用运行时算出的值比对）。 + +### 4.2.2 ⚠️ 为什么这条迁移**绝不能丢 CK**（实测代价） + +用户明确要求：**不能靠重新登录来补**。实测依据（2026-10-03）： + +| 项 | 实测值 | +| --- | --- | +| 需要保住的账号数 | **13,534**（持 CK 账号，且在增长） | +| 重登速率 | 约 7 轮/小时 × 30 条 = **210 账号/小时**（98% 成功） | +| 全量重登耗时 | 13,534 ÷ 210 ≈ **64 小时 ≈ 2.7 天**连续跑 | +| **当前重登可用性** | ⚠️ **最近 3 小时成功率 0**（`ck_relogin`：00h 发起 90 / 成功 30；01h、02h 均为 0）——斗鱼对新账号静默不发验证码，即前面查到的邮件验证码问题 | + +**最后一行才是关键**：重登**现在本身就是坏的**。此刻若丢了 CK，不是「花 2.7 天补回来」， +而是**很可能根本补不回来**，这些账号的登录态就永久没了。 + +由此三条不可让步的要求： + +1. **迁移必须非破坏性** —— 回填只读 `login_tasks`、**不删不改源数据**。 + CK 在源表里始终留一份，任何环节出问题都能重来； +2. **回填不完整必须中止启动**（§6.3）——不能带着半份 CK 切到新读路径； +3. **校验要到值级别，不能只比条数** —— 条数相等仍可能张冠李戴（A 账号拿到 B 账号的 CK）： + +```sql +-- 必须为 0：账号的 CK 与其「最近一条持 CK 记录」的值不一致 +SELECT COUNT(*) FROM accounts a +JOIN ( + SELECT account_id, cookie, + ROW_NUMBER() OVER (PARTITION BY account_id ORDER BY id DESC) AS rn + FROM login_tasks WHERE cookie IS NOT NULL AND cookie <> '' +) src ON src.account_id = a.id AND src.rn = 1 +WHERE a.cookie <> src.cookie; +``` + +> 安全网：因为迁移不删源数据，「最坏情况」永远是**回退代码**（旧列数据完好）， +> 而不是「账号没救了」。这是整个方案敢一次切换的前提。 ### 4.3 回填校验（迁移后立即执行，写进验收） diff --git a/docs/设计-账号详情页与事件历史.md b/docs/设计-账号详情页与事件历史.md new file mode 100644 index 0000000..fe0049c --- /dev/null +++ b/docs/设计-账号详情页与事件历史.md @@ -0,0 +1,346 @@ +# 设计：账号详情页与账号事件历史 + +> 状态：**后端已完成、前端已完成，本地验证通过**（2026-10-03，未提交） +> 关联：[登录任务与 CK 解耦](设计-登录任务与CK解耦.md) —— 两者的写入点高度重合，建议同一工作流推进（见 §8）。 +> +> **范围（2026-10-03 已定）**：**不做历史回填**。既有数据（`douyu_tasks` 里的兑换/充值/积分记录） +> **不导入**事件表，事件**从上线那一刻起往后累积**。账号的历史明细仍可去既有的任务页查。 +> 这样本需求从「一次数据搬运工程」缩小为「建表 + 几处发事件 + 一个页面」。 + +## 1. 需求 + +要一个「账号相关」的界面，围绕**单个账号**把下面这些都记下来、看得见： + +| # | 需求项 | 用户原话 | +| --- | --- | --- | +| 1 | 导入时间 | 「这个账号 导入时间」 | +| 2 | CK 登录记录 | 「ck登录记录」 | +| 3 | **CK 失效时间** | 「失效时间」 | +| 4 | 兑换记录 | 「兑换记录」 | +| 5 | 积分多少 | 「积分多少」 | +| 6 | 消耗记录 | 「消耗记录」 | +| 7 | 充值记录 | 「充值记录」 | +| 8 | **精英宝典开通记录** | 「精英宝典开通记录 时间等等」 | +| 9 | **电竞手册开通时间** | 「还有电竞手册开通时间 啥的」 | +| 10 | 兜底原则 | 「只要跟这个账号有关系的 都记录下来」 | + +## 2. 现状盘点：每条需求今天有没有数据 + +上机实测（2026-10-03）—— 结论是**大部分业务数据已经存在**，但两点不足： +**散在 687MB 的噪声表里**、以及**状态变更没有历史**（覆盖式）。本设计**不回填**这些历史， +只保证「从现在起不再丢」。 + +| # | 需求 | 今天的数据在哪 | 本设计怎么处理 | +| --- | --- | --- | --- | +| 1 | 导入时间 | `accounts.created_at` | ✅ 直接读，无需事件 | +| 2 | CK 登录记录 | `login_tasks`；解耦后状态在 `accounts` | ✅ 读现有 + 新事件补「失效/重登」 | +| 3 | **CK 失效时间** | 只有 `ck_check_status`+`ck_checked_at`，**每次检测覆盖** | 🆕 **新增事件**（覆盖前记一笔） | +| 4 | 兑换记录 | `douyu_tasks` `exchange_goods`（10,009 行） | 🔸 **不回填**；上线后用收集器增量提取 | +| 5 | 积分多少 | `accounts.points`（当前值） | ✅ 状态卡直接读 | +| 6 | 消耗记录 | 兑换任务的 `result.score` | 🔸 同 #4（增量） | +| 7 | 充值记录 | `douyu_tasks` `create_gold_qr`（13,867 行） | 🔸 同 #4（增量） | +| 8 | **精英宝典开通** | 只有 `elite_opened`(bool)，**覆盖式** | 🆕 **新增事件** | +| 9 | **电竞手册开通** | 只有 `esports_bind_status` 字符串，**覆盖式** | 🆕 **新增事件** | +| 10 | 兜底（人工操作） | `audit_logs` | ⚠️ **关联不到账号**，需补 `account_id`（§7） | + +> 🔸 = 既有数据不回填，从上线起增量收录。账号的历史明细仍可在既有的 +> `/douyu/elite`、`/douyu/esports`、`/login-tasks` 等页面查到，**本需求不复制它们的历史**。 + +### 2.1 为什么用事件表、而不是详情页实时聚合 + +`douyu_tasks` 现状 **222,679 行 / 687MB**，每天在写。若详情页实时聚合： + +- 每次开页要扫大表 + 解析巨型 JSON（单条兑换的 `result` ≈ 8KB）； +- 噪声占绝大多数（`query_limited_goods` 6.7 万行、`query_change_bind_time` 3 万行是例行轮询）； +- `douyu_tasks` 将来若做保留策略（规划 §11.4），账号历史会跟着消失。 + +所以事件表**精简、独立、可长期保留**。反过来说：**不回填也就不必去核对历史 JSON 的字段路径**， +风险与工作量都大幅下降。 + +## 3. 目标模型：`account_events`（账号事件流） + +一张 append-only 的事件表，**账号的时间线 = 这张表按时间倒序读**。 + +```python +class AccountEvent(Base): + __tablename__ = "account_events" + + id: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True) + account_id: Mapped[int] = mapped_column(ForeignKey("accounts.id"), index=True) + event_type: Mapped[str] = mapped_column(String(32), index=True) + occurred_at: Mapped[datetime] = mapped_column(UTCDateTime, index=True) + title: Mapped[str] = mapped_column(String(256), default="") # 直接可展示 + amount: Mapped[int | None] = mapped_column(Integer, nullable=True) # 充值额/消耗积分（有符号） + balance_after: Mapped[int | None] = mapped_column(Integer, nullable=True) # 变更后余额 + detail: Mapped[dict | None] = mapped_column(JSON, nullable=True) # 精简后的细节（非原始大 JSON） + source: Mapped[str] = mapped_column(String(24), default="") # login/task/audit/manual + source_ref: Mapped[str] = mapped_column(String(96), unique=True) # 幂等键 + created_at: Mapped[datetime] = mapped_column(UTCDateTime, default=_utcnow) +``` + +设计要点： + +- **`source_ref` 唯一键**：如 `douyu_task:123456`、`login_task:14166`、`audit:789`。 + 作用是**幂等**：收集器重跑、事件重复发射都不会造出重复时间线（§4 的水位线失守时靠它兜底）； +- **不存原始大 JSON**：只存提取出的业务字段（商品名、单号、金额…），把 687MB 的包袱甩掉； +- **`amount` 有符号**：充值为正、兑换消耗为负，便于页面上做「累计充值 / 累计消耗」； +- **`occurred_at` 用业务发生时间**（不是入库时间）：收集器取原任务行的 `finished_at || created_at`； + 直接发射的取当下时刻。 + +### 3.1 事件类型（初版） + +| event_type | 含义 | 来源 | +| --- | --- | --- | +| `imported` | 账号导入 | `accounts.created_at` | +| `login_ok` | 登录成功（含新 CK） | `login_tasks` / 解耦后的账号写入点 | +| `ck_invalid` | **CK 失效**（新增记录） | 巡检判定 | +| `relogin_ok` / `relogin_failed` | 自动重登结果 | `ck_relogin` 作业 | +| `bind_ok` / `rebind` | 绑定 / 换绑 | `douyu_tasks` | +| `recharge` | 充值到账 | `create_gold_qr` | +| `exchange` | 兑换（**含消耗积分**） | `exchange_goods` | +| `points_credit` | 积分到账（含**到期时间**） | `donate_elite_gift` | +| `points_expire` | 积分即将到期 | 由 `willExpireTime` 派生 | +| `elite_opened` | **精英宝典开通**（新增记录） | `douyu_runner_manual` | +| `esports_opened` | **电竞手册开通**（新增记录） | `douyu_runner_manual` | +| `assigned` / `tag_changed` / `remark_changed` | 人工操作 | `audit_logs`（需补 account_id，见 §7） | + +## 4. 业务事件怎么收录：增量收集器（不回填） + +既有历史**不导入**。业务事件（充值/兑换/积分到账/绑定）用**一个增量收集器**从 `douyu_tasks` +往后提取 —— 好处是**完全不侵入 runner 代码**（不必改 8 个业务文件），且新老任务类型统一处理。 + +### 4.1 收集器设计 + +``` +水位线（watermark）：记录「已收集到 douyu_tasks.id = N」 +每轮（挂到现有 automation 调度器，5~15 分钟一次）： + 1. 读 douyu_tasks WHERE id > N AND task_type IN (业务类型) ORDER BY id LIMIT 500 + 2. 按 §4.2 的映射提取字段 → 写 account_events（source_ref = "douyu_task:" 保证幂等） + 3. 更新水位线到本批最大 id +``` + +- 水位线存在 `automation_jobs.config`（JSON）或单独一小张表，**必须持久化**（进程重启不重来）； +- `source_ref` 唯一键兜住「同一行被处理两次」（例如水位线写入失败后重跑）； +- **起点 = 上线那一刻的 `MAX(douyu_tasks.id)`**，不回溯。 + +### 4.2 字段映射（已抽样核实，仅用于**新**数据） + +已实测的 `result` 结构（用于收集器解析）： + +**`exchange_goods` → `exchange`** + +| 事件字段 | JSON 路径 | 实测样例 | +| --- | --- | --- | +| title | `commodity_name` | `套装-兔兔白日梦` | +| detail.commodity_id | `commodity_id` | `20260120QYOOB_bag10` | +| **amount（消耗）** | `score`，取负 | `-1280` | +| **balance_after** | `points_query.points` | `0` | +| detail.order_id / exchange_id | `order_id` / `exchange_id` | `147879` / `11241526` | +| detail.game_name | `game_name` | `唉痛恨透` | +| detail.game_channel | `game_channel` | `手Q / 安卓(android)` | + +**`create_gold_qr` → `recharge`** + +| 事件字段 | JSON 路径 | 实测样例 | +| --- | --- | --- | +| **amount（充值）** | `buy_num` / `pay_amount` | `98` | +| detail.order_id | `order_id` | `DYGF5C24C5E4EA84T238035` | +| detail.supplier_order_id | `supplier_order.order_id` | `200000229425420261003114601` | +| detail.supplier_finished_at | `supplier_order.finish_time` | `2026-10-03 11:46:07` | +| detail.channel | `recharge_channel` | `supplier_api` | +| detail.douyu_uid | `douyu_uid` | `678503799` | + +**`donate_elite_gift` → `points_credit`** + +| 事件字段 | JSON 路径 | 实测样例 | +| --- | --- | --- | +| **amount（到账）** | `gift_points_expected_gain` | `980` | +| **balance_after** | `points` / `points_query.points` | `1280` | +| detail.baseline / target | `gift_points_baseline` / `gift_points_target` | `300` / `1280` | +| **detail.points_expire_at** | `points_query.raw.data.willExpireTime` | `1798819199`（时间戳） | + +> 绑定类（`confirm_bind` / `*_esports_bind`）的 `result` 结构**尚未抽样**，实现前先各 dump 一条 +> （教训见 [CK 误删复盘](事故复盘-CK误删与恢复.md)：字段假设必须实测）。收集器对 +> **解析不出的行要记 warning 并跳过**，不能让一行坏数据卡住整条水位线。 + +### 4.3 覆盖的事件量（预估） + +按实测的业务任务量，稳定后每天新增事件量级在**数百条**（充值/兑换/到账为主）， +一年约 10 万条以内 —— 事件表长期保留没有压力。 + +## 5. 事件从哪来：两条路 + +### 5.1 直接发（CK 相关 + 开通类）—— 4 处 + +这几类**不是 `douyu_tasks` 里的行**，只能在代码里「状态被覆盖之前先记一笔」： + +| 场景 | 位置 | 事件 | +| --- | --- | --- | +| CK 判定失效 | `automation/jobs/ck_health.py`（判定 invalid 时） | `ck_invalid` | +| CK 重登成功/失败 | `login_service._write_login_success` / `ck_relogin`（**CK 解耦的同一处**） | `login_ok` / `relogin_ok` / `relogin_failed` | +| 精英宝典开通 | `douyu_runner_manual.py:377`（`elite_opened = True` 那一行） | `elite_opened` | +| 电竞手册开通 | `douyu_runner_manual.py:426`（`esports_bind_status = "esports_opened"`） | `esports_opened` | + +这 4 处正好是「当前状态被覆盖」的现场。实现上收敛成一个 helper： +`account_events_service.emit(db, account_id, type, title, amount=..., balance_after=..., detail=...)`， +并要求**发射失败不影响主流程**（事件是旁路，不能因为它让登录/兑换失败）。 + +### 5.2 收集器（业务类）—— 0 处侵入 + +充值 / 兑换 / 积分到账 / 绑定 这些**已经是 `douyu_tasks` 的行**，走 §4 的增量收集器， +**不动 runner 代码**。 + +> 这个分工的取舍：CK 与开通事件「只在内存里出现过」，不记就永远没有； +> 业务事件已经在表里有结构化落库，再改一遍 runner 属于重复工作。 + +## 6. 页面设计：`/accounts/:id`（账号详情） + +现在是**列表页**（`AccountsPage.tsx`），没有详情页。新增： + +``` +┌─ 顶部：账号身份 ────────────────────────────────────────┐ +│ 用户名 / 斗鱼昵称+UID / 标签 / 归属客服 / 导入时间(第 N 天) │ +│ 操作：分配 · 标签 · 备注 · 登录 · 检测 CK · 重登 · 导出 │ +└──────────────────────────────────────────────────────┘ +┌─ 状态卡（当前值：accounts 表）──────────────────────────┐ +│ 积分(含到期时间) · 鱼翅余额 · 兑换余额 │ +│ 精英宝典：已开通 / 到期时间 · 电竞手册：开通+绑定+可换绑 │ +│ CK：状态 · 最近检测 · 最近重登 ← CK 解耦后从 accounts 读 │ +└──────────────────────────────────────────────────────┘ +┌─ 统计条（由 account_events 聚合）──────────────────────┐ +│ 累计充值 ¥X · 累计兑换 N 次 / 消耗积分 M · CK 失效 K 次 │ +└──────────────────────────────────────────────────────┘ +┌─ 时间线（account_events 倒序 + 类型筛选 + 分页）────────┐ +│ 10-03 11:46 充值 ¥98 供应商直充成功 │ +│ 10-03 11:46 积分 +980 → 余额 1280（到期 12-31） │ +│ 10-03 11:46 兑换 -1280 套装-兔兔白日梦 │ +│ 10-02 17:17 登录 重新登录成功，Cookie 有效 │ +│ 10-01 09:12 CK失效 检测判定 CK 已失效 │ +│ 09-30 14:00 开通 精英宝典开通 │ +└──────────────────────────────────────────────────────┘ +``` + +- 接口：`GET /api/accounts/{id}/events?types=&page=&page_size=`、`GET /api/accounts/{id}/overview`； +- 权限：沿用账号可见性（客服只能看自己名下账号）+ 敏感字段按现有 `account:view_full` 控制； +- 入口：账号列表点用户名进详情；从 CK 页、登录任务页也可跳转。 + +## 7. 必须先修的缺口：`audit_logs` 关联不到账号 + +实测 `audit_logs.target` 是**自由文本**： + +| action | 样例 target | +| --- | --- | +| `account:assign` | `冰柳192`（用户名） | +| `account:import` | `导入103个`（一句话） | +| `recharge:douyu:create` | `douyu_batch:000619c494fd`（批次号） | +| `account:delete` | `hui60945`（用户名） | + +**没有 account_id**，只能拿用户名去模糊匹配 —— 用户名可改、可重复，**不可靠**。 + +建议：`audit_logs` 增加 `account_id`（可空）列，账号相关操作写入时带上。 +这样「谁在什么时候分配/改标签/删除了这个账号」才能进账号时间线。 +这是本次需求的前置项，改动很小（加一列 + 若干调用点传参）。 + +## 8. 与 CK 解耦怎么「一块弄」 + +两者的**写入点是同一批**，所以设计上共用一套约定，实施上排好顺序即可： + +| 事项 | CK 解耦 | 账号事件 | +| --- | --- | --- | +| 状态落点 | CK/健康/重登状态 → `accounts` | 新增 `account_events`（流水） | +| 写入点 | `_write_login_success`、`ck_health`、`ck_relogin` | 同上 + `elite/esports 开通` 两处 | +| 读取点 | Cookie 页、工作台、仪表盘 | 新增详情页 | + +**共享的一条原则**：`accounts` 存**当前状态**，`account_events` 存**发生过什么** —— +这与 CK 解耦「状态归账号、流水归历史表」是同一个模型。做完解耦，账号上就有了完整的 +「当前状态」，事件表补上「历史」，详情页就是把两者拼起来。 + +**建议顺序**（在 [CK 解耦的 7 个提交](设计-登录任务与CK解耦.md#10-提交计划7-个提交) 之后接）： + +| 提交 | 内容 | 依赖 | +| --- | --- | --- | +| E1 | `feat(db): account_events 表 + audit_logs.account_id` | 无 | +| E2 | `feat(events): 事件发射 helper + CK 事件（失效/登录/重登）` | **CK 解耦已完成**（否则写入点还会变） | +| E3 | `feat(events): 开通事件（精英宝典 / 电竞手册）` | 无 | +| E4 | `feat(events): douyu_tasks 增量收集器 + 水位线` | 无 | +| E5 | `feat(api): 账号事件与概览接口` | E1 | +| E6 | `feat(ui): 账号详情页（状态卡 + 统计 + 时间线）` | E5 | +| E7 | `docs: 落点与验收` | —— | + +> 原计划的「历史回填脚本」已取消（§2 已定不回填），E4 从「搬运 8.5 万行」变成 +> 「装一个水位线收集器」。 + +## 9. 风险与注意 + +1. **不改 runner 代码**：业务事件走收集器（§4），只有 CK/开通 4 处直接发（§5.1）—— + 把改动面压到最小； +2. **`source_ref` 唯一键是幂等的生命线**：收集器重跑、事件重复发射都靠它兜住， + 漏了就会污染时间线； +3. **收集器的水位线必须持久化**，且**解析失败要跳过并告警**，不能让一行坏数据卡住整条链路； +4. **发射事件是旁路**：必须保证 `emit()` 抛错不影响登录/兑换主流程（用 try 包住 + 记日志）； +5. **积分口径要先定**（决策 C）：`accounts.points` 与 `donate_elite_gift` 的 `points_query.points` + 有不一致的情况（如兑换后 `points_query.points=0`），时间线显示哪个要明确； +6. 绑定类任务的 `result` 结构**尚未抽样**（§4.2），收集器实现前先 dump 一条确认。 + +## 10. 待确认的决策点 + +| # | 决策 | 选项 | 建议 | +| --- | --- | --- | --- | +| **A** | 时间线是否收录「例行查询」类事件 | ①只收业务事件（推荐）②全都收 | ①：6.7 万行轮询进去后时间线没法看 | +| **B** | 详情页入口 | ①账号列表点用户名（推荐）②独立菜单 | ①更顺手；②可加，但主入口应是列表 | +| **C** | 积分显示口径 | ①以 `accounts.points` 为准，事件里记变更量（推荐）②以事件累加为准 | ①简单可靠；②容易和历史噪声打架 | +| **D** | 收集器的跑法 | ①挂到现有 automation 调度器（推荐）②独立定时任务 | ①复用现成的开关/运行记录/页面；②多一套机制 | +| **E** | `audit_logs` 加 `account_id` 是否随本次做 | ①做（否则人工操作进不了时间线）②先不做 | ①（§7） | +| ~~F~~ | ~~回填范围~~ | —— | ✅ **已定：不回填**，从上线起增量累积 | + +--- + +## 11. 落地记录（2026-10-03） + +已按本文实现，**未提交、未碰生产**（用户要求先在本地验证）。 + +| 提交建议 | 内容 | 落点 | +| --- | --- | --- | +| E1 | 迁移 `20261002_0040`：`account_events` + `audit_logs.account_id` | `migrations/versions/20261002_0040_account_events.py`、`models.py`（`AccountEvent`）、`tests/test_migrations.py` | +| E2 | 事件服务 `emit()` + 任务映射 | `services/account_events_service.py` | +| E3 | 发射点：CK 失效 / 登录·重登成功 / 重登转人工 / 精英宝典·电竞手册开通 | `jobs/ck_health.py`、`services/login_service.py`、`jobs/ck_relogin.py`、`services/douyu_runner_manual.py` | +| E4 | 收集器作业（水位线在 `automation_jobs.config`） | `jobs/account_events_collect.py`，注册为「账号事件收集」 | +| E5 | 接口 | `routers/accounts.py`：`GET /{id}/overview`、`GET /{id}/events` | +| E6 | 页面 | `pages/AccountDetailPage.tsx`；路由 `/accounts/:id`；列表用户名可点入 | + +### 11.1 与设计的差异（实施时改的） + +1. **收集器的水位线放作业自己的 `config`**，不再寄生在 `ck_health` 的 config 里 —— + 顺带白拿「自动巡检」页的开关、运行记录与手动触发； +2. **`source_ref` 可空唯一，不用空串默认值**。原设计写的是 `server_default=""`：MySQL 唯一索引下 + **空串会互相冲突**，第二条「无幂等键」的事件就插不进去。改为可空（MySQL 允许 NULL 重复）， + 并补了用例专门盯这点； +3. **电竞手册只在「真正开通」那一支发事件**。代码里有两处都会写 `esports_bind_status="esports_opened"`： + 支付流程成功后（真开通）与「基线查到已开通」（检测）。两处都发会每次检查刷一条，只取前者； +4. **重登失败只在「转人工」时发事件**：还要退避重试的每轮都会再来，发事件会刷屏； +5. **`emit` 用 SAVEPOINT 隔离**：失败只回滚这一条，不污染调用方事务（有用例覆盖）。 + +### 11.2 实施时踩到的坑 + +1. **`create_all` 不改已存在的表** —— 给 `audit_logs` 加列后，测试库里的旧表没有该列， + **22 个不相关测试全挂**（`Unknown column 'account_id'`）。改已有表的模型后，测试库要么手动 + `ALTER`，要么 `DROP` 让 `create_all` 重建。已在本地测试库补列；**这个坑值得写进通用约定**； +2. **REPEATABLE READ 快照**（又踩一次）：用例里用另一个 session 写入事件后，原 session 查不到 —— + 与 P1b「成功被记成失败」同根因。用例里补 `commit()` 结束事务； +3. **`delete_account` 必须连带清事件**：`account_events` 有指向 `accounts` 的外键，不清就删不掉账号。 + 这是本次改动**引入的**问题，已修并有用例； +4. **测试数据的字符集**：用 `docker exec mysql < file` 灌中文时没带 `--default-character-set=utf8mb4`， + 数据被双重编码（`套装` → `套装`）。排查时先看 `HEX()` 就能立刻分清「存坏了」还是「读坏了」。 + +### 11.3 本地验证结果 + +| 项 | 结果 | +| --- | --- | +| 后端测试 | **363 通过**（新增 20 条：幂等、旁路隔离、映射、水位线、API、账号隔离） | +| 收集器 | 3 条业务任务 → 3 条事件；金额/余额/中文均正确；水位线 0 → 686 | +| 概览接口 | 累计充值 98、兑换 1 次、消耗积分 1280、导入时间、登录次数、最近 CK 检测均正确 | +| 时间线接口 | 倒序分页正确、按类型筛选正确、中文无乱码 | +| 前端 | `tsc -b` 通过、`vite build` 通过、模块转译正常（dev 5174 → 8800） | + +> 本地验证时把收集器**临时启用**并手动灌了 3 条测试任务（`batch_id='local-evt'`，仅本地开发库）。 +> 生产上该作业仍是 `enabled=false`，需在「自动巡检」页手动打开；首次运行只定水位线、不回溯历史。 diff --git a/tests/test_account_events.py b/tests/test_account_events.py new file mode 100644 index 0000000..f4e64d7 --- /dev/null +++ b/tests/test_account_events.py @@ -0,0 +1,548 @@ +"""账号事件流回归：幂等发射、旁路隔离、任务映射、收集器水位线。 + +三条必须守住： + +1. **发射是旁路**：`emit` 任何失败都不能影响主流程（登录/兑换照常）； +2. **幂等**：同一个 `source_ref` 重复发射只落一条 —— 收集器重跑的可靠性全押在这上面； +3. **水位线只进不退**：解析失败要跳过而不是卡住，否则一行坏数据会让事件流永久停摆。 +""" + +from datetime import UTC, datetime + +from sqlalchemy.orm import sessionmaker + +import web.backend.database as database_module +from tests.support.db import get_test_engine, make_test_session +from web.backend.models import Account, AccountEvent, AutomationJob, DouyuTask, User +from web.backend.services import account_events_service as svc +from web.backend.services.automation.jobs import account_events_collect as collect_job + + +class _Base: + def setup_method(self): + self.engine = get_test_engine() + database_module.SessionLocal.configure(bind=self.engine) + self.session = make_test_session(self.engine) + self.session.query(AccountEvent).delete() + self.session.query(DouyuTask).delete() + self.session.query(AutomationJob).filter( + AutomationJob.key == collect_job.JOB_KEY + ).delete() + self.session.commit() + + user = User(username="evt-admin", password_hash="h", role="super_admin") + self.session.add(user) + self.session.flush() + self.user_id = user.id + self.account = Account( + username="事件测试账号", + password="p", + email="evt@example.com", + email_password="ep", + ) + self.session.add(self.account) + self.session.commit() + self.account_id = self.account.id + + def teardown_method(self): + self.session.close() + + def _fresh(self): + return sessionmaker(bind=self.engine)() + + +class TestEmit(_Base): + def test_writes_event(self): + db = self._fresh() + try: + ok = svc.emit( + db, + account_id=self.account_id, + event_type=svc.EVENT_EXCHANGE, + title="兑换 套装-兔兔白日梦", + amount=-1280, + balance_after=0, + detail={"commodity_name": "套装-兔兔白日梦"}, + source="manual", + ) + db.commit() + finally: + db.close() + assert ok is True + + db = self._fresh() + try: + row = db.query(AccountEvent).one() + assert row.account_id == self.account_id + assert row.event_type == svc.EVENT_EXCHANGE + assert row.amount == -1280 + assert row.balance_after == 0 + assert row.detail["commodity_name"] == "套装-兔兔白日梦" + finally: + db.close() + + def test_same_source_ref_is_idempotent(self): + """同一 source_ref 只能落一条 —— 收集器重跑靠它兜住。""" + db = self._fresh() + try: + for _ in range(3): + svc.emit( + db, + account_id=self.account_id, + event_type=svc.EVENT_RECHARGE, + title="充值 98", + amount=98, + source="task", + source_ref="douyu_task:1001", + ) + db.commit() + finally: + db.close() + + db = self._fresh() + try: + assert db.query(AccountEvent).count() == 1 + finally: + db.close() + + def test_multiple_null_source_refs_coexist(self): + """没有幂等键的事件（手工记录）不能互相冲突 —— 这正是 source_ref 要可空的原因。""" + db = self._fresh() + try: + for _ in range(3): + svc.emit( + db, + account_id=self.account_id, + event_type=svc.EVENT_ASSIGNED, + title="分配客服", + source="manual", + ) + db.commit() + finally: + db.close() + + db = self._fresh() + try: + assert db.query(AccountEvent).count() == 3 + finally: + db.close() + + def test_failure_is_swallowed(self): + """旁路原则：account_id 不存在（外键失败）也不能抛出去。""" + db = self._fresh() + try: + ok = svc.emit( + db, + account_id=99999999, + event_type=svc.EVENT_LOGIN_OK, + title="登录成功", + source="login", + ) + db.commit() + finally: + db.close() + assert ok is False + + def test_event_does_not_break_caller_transaction(self): + """emit 失败后，调用方在同一事务里已做的写入仍然能提交。 + + 注意必须用**同一个 session**：换 session 就测不到「SAVEPOINT 隔离」这件事 + （改动会落在另一个事务里，断言必然失败）。 + """ + db = self._fresh() + try: + account = db.query(Account).filter(Account.id == self.account_id).first() + account.remark = "改动要保住" + db.flush() + # 这条会因外键失败 + svc.emit( + db, + account_id=99999999, + event_type=svc.EVENT_LOGIN_OK, + title="x", + ) + db.commit() + finally: + db.close() + + db = self._fresh() + try: + row = db.query(Account).filter(Account.id == self.account_id).first() + assert row.remark == "改动要保住" + finally: + db.close() + + def test_empty_account_or_type_is_noop(self): + db = self._fresh() + try: + assert svc.emit(db, account_id=0, event_type="x", title="y") is False + assert ( + svc.emit(db, account_id=self.account_id, event_type="", title="y") + is False + ) + finally: + db.close() + + +class TestTaskMapping: + def test_exchange_maps_amount_negative(self): + payload = svc._task_to_event( + "exchange_goods", + { + "commodity_name": "套装-狂战萌兔", + "commodity_id": "20260120QYOOB_bag5", + "score": 480, + "points_query": {"points": 0}, + "order_id": "147874", + "game_name": "窝补誓渣男", + "game_channel": "微信 / 安卓(android)", + }, + ) + assert payload["event_type"] == svc.EVENT_EXCHANGE + assert payload["amount"] == -480, "消耗要记成负数，否则累计统计会反向" + assert payload["balance_after"] == 0 + assert payload["detail"]["game_name"] == "窝补誓渣男" + + def test_recharge_maps_amount(self): + payload = svc._task_to_event( + "create_gold_qr", + { + "buy_num": 98, + "order_id": "DYGF5C24C5E4EA84T238035", + "supplier_order": { + "order_id": "200000229425", + "finish_time": "2026-10-03 11:46:07", + }, + "recharge_channel": "supplier_api", + "douyu_uid": "678503799", + }, + ) + assert payload["event_type"] == svc.EVENT_RECHARGE + assert payload["amount"] == 98 + assert payload["detail"]["supplier_order_id"] == "200000229425" + + def test_points_credit_keeps_expire_time(self): + payload = svc._task_to_event( + "donate_elite_gift", + { + "gift_points_expected_gain": 980, + "points": 1280, + "gift_points_baseline": 300, + "gift_points_target": 1280, + "points_query": {"raw": {"data": {"willExpireTime": 1798819199}}}, + }, + ) + assert payload["event_type"] == svc.EVENT_POINTS_CREDIT + assert payload["amount"] == 980 + assert payload["balance_after"] == 1280 + assert payload["detail"]["points_expire_at"] == 1798819199 + + def test_unparsable_result_returns_none(self): + assert svc._task_to_event("exchange_goods", "{not json") is None + assert svc._task_to_event("exchange_goods", None) is None + assert svc._task_to_event("unknown_type", {"a": 1}) is None + + +class TestCollector(_Base): + def _make_job(self, watermark=None): + job = AutomationJob( + key=collect_job.JOB_KEY, + title="账号事件收集", + enabled=True, + interval_seconds=600, + batch_size=100, + concurrency=1, + config={"watermark": watermark} if watermark is not None else None, + updated_at=datetime.now(UTC), + ) + self.session.add(job) + self.session.commit() + return job + + def _add_task(self, task_type, result, status="success", account_id=None): + task = DouyuTask( + batch_id="evt-batch", + account_id=account_id or self.account_id, + task_type=task_type, + status=status, + result=result, + created_by=self.user_id, + ) + self.session.add(task) + self.session.commit() + return task.id + + def test_first_run_sets_watermark_without_backfill(self): + """首次运行只定水位线，不回溯历史（2026-10-03 已定不回填）。""" + self._add_task( + "exchange_goods", + {"commodity_name": "旧兑换", "score": 100, "points_query": {"points": 0}}, + ) + job = self._make_job() + db = self._fresh() + try: + result = collect_job.run_account_events_collect(db, job) + finally: + db.close() + assert result["detail"]["first_run"] is True + assert result["scanned"] == 0 + + db = self._fresh() + try: + assert db.query(AccountEvent).count() == 0, "首次运行不应回填历史事件" + finally: + db.close() + + def test_collects_only_new_rows_and_advances_watermark(self): + first = self._add_task( + "exchange_goods", + {"commodity_name": "老兑换", "score": 100, "points_query": {"points": 5}}, + ) + job = self._make_job(watermark=first) + new_id = self._add_task( + "create_gold_qr", + {"buy_num": 68, "order_id": "DYGF-x", "supplier_order": {}}, + ) + + db = self._fresh() + try: + result = collect_job.run_account_events_collect(db, job) + finally: + db.close() + + assert result["detail"]["written"] == 1 + assert result["detail"]["watermark"] == new_id + + db = self._fresh() + try: + rows = db.query(AccountEvent).all() + assert len(rows) == 1 + assert rows[0].event_type == svc.EVENT_RECHARGE + assert rows[0].source_ref == f"douyu_task:{new_id}" + finally: + db.close() + + def test_rerun_does_not_duplicate(self): + """重复跑不能造重复事件（source_ref 兜底）。""" + job = self._make_job(watermark=0) + self._add_task( + "create_gold_qr", + {"buy_num": 98, "order_id": "o1", "supplier_order": {}}, + ) + db = self._fresh() + try: + collect_job.run_account_events_collect(db, job) + finally: + db.close() + + # 把水位线倒回去，模拟「重复处理同一批」 + job2 = self._fresh() + try: + row = ( + job2.query(AutomationJob) + .filter(AutomationJob.key == collect_job.JOB_KEY) + .first() + ) + row.config = {"watermark": 0} + job2.commit() + finally: + job2.close() + + db = self._fresh() + try: + job3 = ( + db.query(AutomationJob) + .filter(AutomationJob.key == collect_job.JOB_KEY) + .first() + ) + collect_job.run_account_events_collect(db, job3) + finally: + db.close() + + db = self._fresh() + try: + assert db.query(AccountEvent).count() == 1 + finally: + db.close() + + def test_failed_task_produces_no_event(self): + """失败的兑换不该记成「消耗了积分」。""" + job = self._make_job(watermark=0) + self._add_task( + "exchange_goods", + {"commodity_name": "失败兑换", "score": 100}, + status="failed", + ) + db = self._fresh() + try: + result = collect_job.run_account_events_collect(db, job) + finally: + db.close() + assert result["detail"]["written"] == 0 + assert result["skipped"] == 1 + + def test_unparsable_row_does_not_stall_watermark(self): + """坏数据要跳过并推进水位线，不能让事件流永久停摆。""" + job = self._make_job(watermark=0) + bad_id = self._add_task("exchange_goods", "{not json") + good_id = self._add_task( + "create_gold_qr", {"buy_num": 1, "order_id": "o2", "supplier_order": {}} + ) + db = self._fresh() + try: + result = collect_job.run_account_events_collect(db, job) + finally: + db.close() + assert result["detail"]["written"] == 1 + assert result["detail"]["watermark"] == good_id + assert good_id > bad_id + + +class TestAccountApi(_Base): + """账号详情接口：概览统计 + 时间线分页 + 删除账号时清理事件。""" + + def _admin(self): + return self.session.query(User).filter(User.username == "evt-admin").first() + + def _seed(self): + """造三条不同类型的事件，供统计与筛选断言。""" + db = self._fresh() + try: + svc.emit( + db, + account_id=self.account_id, + event_type=svc.EVENT_RECHARGE, + title="充值 98", + amount=98, + occurred_at=datetime(2026, 10, 1, 3, 0, tzinfo=UTC), + source="task", + source_ref="douyu_task:9001", + ) + svc.emit( + db, + account_id=self.account_id, + event_type=svc.EVENT_EXCHANGE, + title="兑换 兔兔白日梦", + amount=-1280, + balance_after=0, + occurred_at=datetime(2026, 10, 2, 3, 0, tzinfo=UTC), + source="task", + source_ref="douyu_task:9002", + ) + svc.emit( + db, + account_id=self.account_id, + event_type=svc.EVENT_ELITE_OPENED, + title="精英宝典开通", + occurred_at=datetime(2026, 10, 3, 3, 0, tzinfo=UTC), + source="manual", + source_ref="elite_opened:1:1", + ) + db.commit() + finally: + db.close() + # ⚠️ MySQL 默认 REPEATABLE READ：self.session 在 setup 阶段就开了事务， + # 快照早于上面那批写入，不结束事务就查不到刚写的事件（P1b 踩过同一个坑）。 + self.session.commit() + + def test_overview_aggregates_stats(self): + from web.backend.routers import accounts as api + + self._seed() + data = api.account_overview( + account_id=self.account_id, db=self.session, current=self._admin() + ) + assert data["username"] == "事件测试账号" + assert data["created_at"] is not None, "导入时间必须返回" + stats = data["stats"] + assert stats["recharge_total"] == 98 + assert stats["exchange_count"] == 1 + assert stats["points_spent"] == 1280, "消耗要按正数展示给用户" + assert stats["event_count"] == 3 + # 开通时间由事件表提供（不回填历史，所以只有上线后开通的才有） + assert data["opened_at"]["elite"] is not None + + def test_events_paginated_desc(self): + from web.backend.routers import accounts as api + + self._seed() + page = api.account_events( + account_id=self.account_id, + page=1, + page_size=2, + db=self.session, + current=self._admin(), + ) + assert page["total"] == 3 + assert len(page["items"]) == 2 + # 倒序：最新的精英宝典开通在最前 + assert page["items"][0]["event_type"] == svc.EVENT_ELITE_OPENED + + def test_events_filter_by_type(self): + from web.backend.routers import accounts as api + + self._seed() + page = api.account_events( + account_id=self.account_id, + event_type=svc.EVENT_EXCHANGE, + db=self.session, + current=self._admin(), + ) + assert page["total"] == 1 + assert page["items"][0]["title"] == "兑换 兔兔白日梦" + + def test_events_isolated_between_accounts(self): + """事件必须按账号隔离 —— 串号会让客服看到别人的记录。""" + from web.backend.routers import accounts as api + + self._seed() + other = Account( + username="另一个账号", + password="p", + email="o@example.com", + email_password="ep", + ) + self.session.add(other) + self.session.commit() + db = self._fresh() + try: + svc.emit( + db, + account_id=other.id, + event_type=svc.EVENT_RECHARGE, + title="别人的充值", + amount=1, + source="task", + source_ref="douyu_task:9003", + ) + db.commit() + finally: + db.close() + + page = api.account_events( + account_id=self.account_id, db=self.session, current=self._admin() + ) + assert all("别人" not in item["title"] for item in page["items"]) + + def test_delete_account_cleans_events(self): + """事件表有指向 accounts 的外键：删账号必须连带清事件，否则删不掉。""" + from web.backend.routers import accounts as api + + self._seed() + api.delete_account( + account_id=self.account_id, db=self.session, current=self._admin() + ) + + db = self._fresh() + try: + left = ( + db.query(AccountEvent) + .filter(AccountEvent.account_id == self.account_id) + .count() + ) + assert left == 0 + assert ( + db.query(Account).filter(Account.id == self.account_id).first() is None + ) + finally: + db.close() diff --git a/tests/test_automation_ck_relogin.py b/tests/test_automation_ck_relogin.py index f8e7977..1cbefba 100644 --- a/tests/test_automation_ck_relogin.py +++ b/tests/test_automation_ck_relogin.py @@ -235,7 +235,11 @@ class TestSettlement(_Base): now = datetime.now(UTC) fresh = {t.id: t for t in tasks} counters = ck_relogin.settle_results( - fresh, [t.id for t in tasks], now=now, max_attempts=max_attempts + self.session, + fresh, + [t.id for t in tasks], + now=now, + max_attempts=max_attempts, ) self.session.commit() return counters, now @@ -463,7 +467,7 @@ class TestSettlementVisibility(_Base): task = self.make_task("acc", attempt_status="success", relogin_attempts=1) now = datetime.now(UTC) counters = ck_relogin.settle_results( - {task.id: task}, [task.id], now=now, max_attempts=2 + self.session, {task.id: task}, [task.id], now=now, max_attempts=2 ) assert counters["valid"] == 1 and counters["invalid"] == 0 assert task.relogin_failure_kind == "" diff --git a/tests/test_migrations.py b/tests/test_migrations.py index d0d5f23..7b452cf 100644 --- a/tests/test_migrations.py +++ b/tests/test_migrations.py @@ -26,7 +26,7 @@ TESTS_DIR = Path(__file__).resolve().parent PROJECT_ROOT = TESTS_DIR.parent VERSIONS_DIR = PROJECT_ROOT / "web" / "backend" / "migrations" / "versions" -HEAD_REVISION = "20261002_0039" +HEAD_REVISION = "20261002_0040" # 迁移新增、但不声明在模型里的复合查询索引。 EXTRA_INDEXES = ( @@ -34,6 +34,8 @@ EXTRA_INDEXES = ( "ix_douyu_tasks_handbook_scope_task_type_id", "ix_douyu_tasks_status_finished_at_id", "ix_login_tasks_status_finished_at_id", + # 账号时间线按「账号 + 时间倒序」查，模型里不声明这个复合索引。 + "ix_account_events_account_occurred", ) diff --git a/web/backend/migrations/versions/20261002_0040_account_events.py b/web/backend/migrations/versions/20261002_0040_account_events.py new file mode 100644 index 0000000..22319b0 --- /dev/null +++ b/web/backend/migrations/versions/20261002_0040_account_events.py @@ -0,0 +1,107 @@ +"""账号事件流表，以及审计日志的账号关联列。 + +## 为什么需要 account_events + +`accounts` 存的是**账号当前状态**（积分、余额、开通标记…），这些字段都是覆盖式更新 —— +「什么时候变成这样」没有任何记录。而账号相关的操作散在 `douyu_tasks`（22 万行 / 687MB， +掺着大量例行轮询），既不能直接给用户看，也不适合当历史长期保留。 + +于是把「发生过什么」独立成一张 append-only 的事件表：`accounts` 是状态，`account_events` +是流水。账号详情页 = 状态卡 + 事件时间线。 + +**不做历史回填**（2026-10-03 已定）：事件从上线起往后累积，既有历史仍去各自的页面查。 + +## 为什么审计日志要加 account_id + +实测 `audit_logs.target` 是自由文本（`冰柳192` / `导入103个` / `douyu_batch:xxx` 混着）， +无法可靠关联到账号，导致「谁什么时候分配/改标签/删除了这个账号」进不了账号时间线。 +加一列可空的 account_id，账号相关操作写入时带上即可（旧记录保持 NULL）。 +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +revision: str = "20261002_0040" +down_revision: str | None = "20261002_0039" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def _tables(bind) -> set[str]: + return set(sa.inspect(bind).get_table_names()) + + +def _columns(bind, table: str) -> set[str]: + return {item["name"] for item in sa.inspect(bind).get_columns(table)} + + +def upgrade() -> None: + bind = op.get_bind() + + if "account_events" not in _tables(bind): + op.create_table( + "account_events", + sa.Column("id", sa.BigInteger(), primary_key=True, autoincrement=True), + sa.Column( + "account_id", + sa.Integer(), + sa.ForeignKey("accounts.id"), + nullable=False, + ), + # import / login_ok / ck_invalid / relogin_ok / relogin_failed / + # bind_ok / recharge / exchange / points_credit / + # elite_opened / esports_opened / assigned / tag_changed ... + sa.Column("event_type", sa.String(length=32), nullable=False), + sa.Column("occurred_at", sa.DateTime(), nullable=False), + # 直接可展示的一句话，避免前端再各自拼文案 + sa.Column( + "title", sa.String(length=256), nullable=False, server_default="" + ), + # 变动量（充值/消耗/到账，有符号）与变动后余额 + sa.Column("amount", sa.Integer(), nullable=True), + sa.Column("balance_after", sa.Integer(), nullable=True), + # 精简细节（不是原始大 JSON），便于排障与以后扩展 + sa.Column("detail", sa.JSON(), nullable=True), + # 来源：login / task / audit / manual + sa.Column( + "source", sa.String(length=24), nullable=False, server_default="" + ), + # 幂等键：如 douyu_task:123456、login_task:14166、audit:789。 + # 收集器重跑、事件重复发射都靠它兜住，漏了会造出重复时间线。 + # **可空且唯一**：MySQL 的唯一索引允许多个 NULL，所以「没有幂等键」的事件 + # （如手工记录）存 NULL；不能用空串 —— 空串会互相冲突，第二条就插不进去。 + sa.Column("source_ref", sa.String(length=96), nullable=True), + sa.Column("created_at", sa.DateTime(), nullable=False), + ) + op.create_index( + "ix_account_events_account_occurred", + "account_events", + ["account_id", "occurred_at"], + ) + op.create_index( + "ix_account_events_event_type", "account_events", ["event_type"] + ) + op.create_index( + "ix_account_events_source_ref", + "account_events", + ["source_ref"], + unique=True, + ) + + if "account_id" not in _columns(bind, "audit_logs"): + op.add_column( + "audit_logs", + sa.Column("account_id", sa.Integer(), nullable=True), + ) + op.create_index("ix_audit_logs_account_id", "audit_logs", ["account_id"]) + + +def downgrade() -> None: + bind = op.get_bind() + if "account_id" in _columns(bind, "audit_logs"): + op.drop_column("audit_logs", "account_id") + if "account_events" in _tables(bind): + # 删表会一并删掉索引 + op.drop_table("account_events") diff --git a/web/backend/models.py b/web/backend/models.py index 001bfa1..3594841 100644 --- a/web/backend/models.py +++ b/web/backend/models.py @@ -6,6 +6,7 @@ from datetime import UTC, datetime from sqlalchemy import ( JSON, + BigInteger, Boolean, ForeignKey, Integer, @@ -690,11 +691,47 @@ class AuditLog(Base): action: Mapped[str] = mapped_column(String(128), nullable=False) target: Mapped[str] = mapped_column(String(256), default="") detail: Mapped[str] = mapped_column(Text, default="") + # 账号相关操作用它关联到具体账号（旧记录为 NULL）。 + # target 是自由文本（用户名/批次号/整句话），不能可靠反查账号，所以单独存 id。 + account_id: Mapped[int | None] = mapped_column(Integer, nullable=True, index=True) # 旧记录未标记结果时为 NULL，避免把历史操作误判成失败。 success: Mapped[bool | None] = mapped_column(Boolean, nullable=True, index=True) created_at: Mapped[datetime] = mapped_column(UTCDateTime, default=_utcnow) +class AccountEvent(Base): + """账号事件流（append-only）。 + + `accounts` 存**当前状态**，这张表存**发生过什么** —— 账号详情页的时间线就读它。 + 状态字段都是覆盖式更新（积分、余额、开通标记），事件表补上「什么时候变成这样」。 + + 设计约定： + - `occurred_at` 是**业务发生时间**（不是入库时间）； + - `amount` 有符号（充值为正、兑换消耗为负），便于页面直接算累计； + - `source_ref` 是幂等键，可空且唯一：收集器重跑、事件重复发射都靠它兜住； + - `detail` 只存精简字段，不放原始大 JSON（那是 douyu_tasks 的包袱）。 + """ + + __tablename__ = "account_events" + + id: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True) + # 不单独建 account_id 索引：迁移建的是 (account_id, occurred_at) 复合索引， + # 前缀已覆盖按账号查，再单建一个纯属冗余。 + account_id: Mapped[int] = mapped_column(ForeignKey("accounts.id"), nullable=False) + event_type: Mapped[str] = mapped_column(String(32), nullable=False, index=True) + occurred_at: Mapped[datetime] = mapped_column(UTCDateTime, nullable=False) + title: Mapped[str] = mapped_column(String(256), default="", server_default="") + amount: Mapped[int | None] = mapped_column(Integer, nullable=True) + balance_after: Mapped[int | None] = mapped_column(Integer, nullable=True) + detail: Mapped[dict | None] = mapped_column(JSON, nullable=True) + source: Mapped[str] = mapped_column(String(24), default="", server_default="") + # 幂等键：可空且唯一（MySQL 允许多个 NULL，便于「没有幂等键」的事件共存） + source_ref: Mapped[str | None] = mapped_column( + String(96), nullable=True, index=True, unique=True + ) + created_at: Mapped[datetime] = mapped_column(UTCDateTime, default=_utcnow) + + class AutomationJob(Base): """自动化作业定义。状态落库，进程重启不丢调度进度。""" diff --git a/web/backend/routers/accounts.py b/web/backend/routers/accounts.py index 76ca6ab..a9dd8aa 100644 --- a/web/backend/routers/accounts.py +++ b/web/backend/routers/accounts.py @@ -1,13 +1,16 @@ """账号管理路由""" +from typing import Any + from fastapi import APIRouter, Depends, HTTPException, Query -from sqlalchemy import func, or_, select +from sqlalchemy import func, or_, select, text from sqlalchemy.orm import Session, defer, joinedload from ..database import get_db from ..deps import get_current_user, require_permission from ..models import ( Account, + AccountEvent, AuditLog, DouyuTask, DouyuWorkbenchAccount, @@ -24,6 +27,13 @@ from ..schemas import ( AccountTag, BatchAssign, ) +from ..services.account_events_service import ( + EVENT_CK_INVALID, + EVENT_ELITE_OPENED, + EVENT_ESPORTS_OPENED, + EVENT_EXCHANGE, + EVENT_RECHARGE, +) from ..services.account_service import ( cookie_account_ids_query, parse_and_build_accounts, @@ -565,6 +575,168 @@ def batch_delete_accounts_selection( return {"message": f"已删除 {deleted} 个账号", "deleted": deleted, "success": True} +@router.get("/{account_id}/overview") +def account_overview( + account_id: int, + db: Session = Depends(get_db), + current: User = Depends(get_current_user), +): + """账号详情页的「状态卡 + 统计」。 + + 状态取 `accounts` 的当前值（覆盖式），统计由 `account_events` 聚合 —— 这正是 + 「状态与流水分离」的用法：页面上两者拼起来看。 + """ + acc = _visible_accounts_query(db, current).filter(Account.id == account_id).first() + if not acc: + raise HTTPException(status_code=404, detail="账号不存在") + + stats = ( + db.execute( + text( + """ + SELECT + SUM(CASE WHEN event_type = :recharge THEN amount ELSE 0 END) AS recharge_total, + SUM(CASE WHEN event_type = :exchange THEN 1 ELSE 0 END) AS exchange_count, + SUM(CASE WHEN event_type = :exchange THEN -amount ELSE 0 END) AS points_spent, + SUM(CASE WHEN event_type = :ck_invalid THEN 1 ELSE 0 END) AS ck_invalid_count, + MIN(CASE WHEN event_type = :elite THEN occurred_at END) AS elite_opened_at, + MIN(CASE WHEN event_type = :esports THEN occurred_at END) AS esports_opened_at, + COUNT(*) AS event_count + FROM account_events WHERE account_id = :aid + """ + ), + { + "aid": account_id, + "recharge": EVENT_RECHARGE, + "exchange": EVENT_EXCHANGE, + "ck_invalid": EVENT_CK_INVALID, + "elite": EVENT_ELITE_OPENED, + "esports": EVENT_ESPORTS_OPENED, + }, + ) + .mappings() + .first() + or {} + ) + + # 登录次数取历史记录（事件表不回填，所以 CK 相关计数仍以 login_tasks 为准）。 + login_stats = ( + db.execute( + text( + """ + SELECT COUNT(*) AS login_count, + MAX(finished_at) AS last_login_at, + MAX(ck_checked_at) AS last_ck_check_at + FROM login_tasks + WHERE account_id = :aid AND cookie IS NOT NULL AND cookie <> '' + """ + ), + {"aid": account_id}, + ) + .mappings() + .first() + or {} + ) + + return { + "id": acc.id, + "username": acc.username, + "nickname": acc.nickname, + "uid": acc.uid, + "tag": acc.tag, + "remark": acc.remark, + "assigned_to": acc.assigned_to, + "created_at": acc.created_at, # 导入时间 + # 当前状态（覆盖式字段） + "points": acc.points, + "gold_balance": acc.gold_balance, + "exchange_balance": acc.exchange_balance, + "game_name": acc.game_name, + "game_channel": acc.game_channel, + "bind_status": acc.bind_status, + "elite_opened": acc.elite_opened, + "elite_expire_time": acc.elite_expire_time, + "esports_points": acc.esports_points, + "esports_bind_status": acc.esports_bind_status, + "esports_game_name": acc.esports_game_name, + "esports_can_change_time": acc.esports_can_change_time, + "auto_managed": acc.auto_managed, + # 统计 + "stats": { + "recharge_total": int(stats.get("recharge_total") or 0), + "exchange_count": int(stats.get("exchange_count") or 0), + "points_spent": int(stats.get("points_spent") or 0), + "ck_invalid_count": int(stats.get("ck_invalid_count") or 0), + "event_count": int(stats.get("event_count") or 0), + "login_count": int(login_stats.get("login_count") or 0), + "last_login_at": login_stats.get("last_login_at"), + "last_ck_check_at": login_stats.get("last_ck_check_at"), + }, + # 「开通时间」由事件表提供；事件表从上线起才有，早于上线的不显示。 + "opened_at": { + "elite": stats.get("elite_opened_at"), + "esports": stats.get("esports_opened_at"), + }, + } + + +@router.get("/{account_id}/events") +def account_events( + account_id: int, + page: int = 1, + page_size: int = 50, + event_type: str = "", + db: Session = Depends(get_db), + current: User = Depends(get_current_user), +): + """账号时间线（倒序分页）。 + + 权限沿用账号可见性：客服只能看自己名下账号的事件。 + """ + acc = _visible_accounts_query(db, current).filter(Account.id == account_id).first() + if not acc: + raise HTTPException(status_code=404, detail="账号不存在") + + page = max(1, int(page)) + page_size = max(1, min(int(page_size), 200)) + where = "WHERE account_id = :aid" + params: dict[str, Any] = {"aid": account_id} + if event_type: + where += " AND event_type = :etype" + params["etype"] = event_type + + total = ( + db.execute( + text(f"SELECT COUNT(*) FROM account_events {where}"), params + ).scalar() + or 0 + ) + + rows = ( + db.execute( + text( + f""" + SELECT id, event_type, occurred_at, title, amount, balance_after, + detail, source + FROM account_events {where} + ORDER BY occurred_at DESC, id DESC + LIMIT :limit OFFSET :offset + """ + ), + {**params, "limit": page_size, "offset": (page - 1) * page_size}, + ) + .mappings() + .all() + ) + + return { + "items": [dict(row) for row in rows], + "total": int(total), + "page": page, + "page_size": page_size, + } + + @router.delete("/{account_id}") def delete_account( account_id: int, @@ -585,6 +757,11 @@ def delete_account( db.query(DouyuWorkbenchAccount).filter( DouyuWorkbenchAccount.account_id == account_id ).delete(synchronize_session=False) + # 事件表有指向 accounts 的外键，不清会直接删不掉账号。 + # 账号都没了，它的时间线也没有保留意义（审计仍在 audit_logs 里，那份不带外键）。 + db.query(AccountEvent).filter(AccountEvent.account_id == account_id).delete( + synchronize_session=False + ) db.add( AuditLog( @@ -592,6 +769,7 @@ def delete_account( username=current.username, action="account:delete", target=acc.username, + account_id=account_id, ) ) db.delete(acc) diff --git a/web/backend/services/account_events_service.py b/web/backend/services/account_events_service.py new file mode 100644 index 0000000..4b1b4a7 --- /dev/null +++ b/web/backend/services/account_events_service.py @@ -0,0 +1,214 @@ +"""账号事件流：写入与查询。 + +## 定位 + +`accounts` 存**当前状态**（积分、余额、开通标记），这张表存**发生过什么**。 +账号详情页 = 状态卡（读 `accounts`）+ 时间线（读本表）。 + +## 两条写入路径 + +1. **直接发射**（`emit`）：CK 失效/登录成功/重登、精英宝典与电竞手册开通 —— 这些 + 只在内存里出现过，不记就永远没有； +2. **增量收集**（`automation/jobs/account_events_collect.py`）：充值/兑换/积分到账/绑定 —— + 这些已经是 `douyu_tasks` 的行，由收集器按水位线往后提取，不动 runner 代码。 + 本模块只提供 `_task_to_event` 映射给它用。 + +## 两条硬约束 + +- **发射是旁路**：`emit` 任何失败都不能影响登录/兑换主流程（内部吞异常 + 记日志， + 并用 SAVEPOINT 隔离，避免污染调用方的事务）； +- **幂等**：`source_ref` 唯一键兜住「收集器重跑」「事件重复发射」。没有它，重跑一次 + 时间线就多一份重复记录。 +""" + +from __future__ import annotations + +import json +from datetime import UTC, datetime +from typing import Any + +from loguru import logger +from sqlalchemy import text +from sqlalchemy.orm import Session + +# ── 事件类型 ────────────────────────────────────────────── +# 命名保持「名词_状态」风格，便于前端归类与筛选。 +EVENT_IMPORTED = "imported" # 账号导入（读 accounts.created_at，一般不落表） +EVENT_LOGIN_OK = "login_ok" # 登录成功（含新 CK） +EVENT_CK_INVALID = "ck_invalid" # CK 判定失效 +EVENT_RELOGIN_OK = "relogin_ok" # 自动重登成功 +EVENT_RELOGIN_FAILED = "relogin_failed" # 自动重登失败（转人工/退避） +EVENT_BIND_OK = "bind_ok" # 绑定/换绑 +EVENT_RECHARGE = "recharge" # 充值到账 +EVENT_EXCHANGE = "exchange" # 兑换（含消耗积分） +EVENT_POINTS_CREDIT = "points_credit" # 积分到账（含到期时间） +EVENT_ELITE_OPENED = "elite_opened" # 精英宝典开通 +EVENT_ESPORTS_OPENED = "esports_opened" # 电竞手册开通 +EVENT_ASSIGNED = "assigned" # 分配客服 +EVENT_TAG_CHANGED = "tag_changed" # 标签变更 + +# 有符号金额的事件类型（用于页面「累计充值/累计消耗」）： +# 充值为正、兑换为负，其余不计入金额统计。 +INCOME_TYPES = (EVENT_RECHARGE,) +OUTCOME_TYPES = (EVENT_EXCHANGE,) + + +def _now() -> datetime: + return datetime.now(UTC) + + +def emit( + db: Session, + *, + account_id: int, + event_type: str, + title: str, + occurred_at: datetime | None = None, + amount: int | None = None, + balance_after: int | None = None, + detail: dict[str, Any] | None = None, + source: str = "manual", + source_ref: str | None = None, +) -> bool: + """记录一条账号事件。返回是否真的写入（幂等命中/异常时为 False）。 + + **绝不抛异常**：事件是旁路，不能因为记不上一条时间线就让登录或兑换失败。 + 用 SAVEPOINT 包住，失败只回滚这一条，不影响调用方已经做过的事。 + """ + if not account_id or not event_type: + return False + + sql = text( + """ + INSERT INTO account_events ( + account_id, event_type, occurred_at, title, + amount, balance_after, detail, source, source_ref, created_at + ) VALUES ( + :account_id, :event_type, :occurred_at, :title, + :amount, :balance_after, :detail, :source, :source_ref, :created_at + ) + """ + ) + params = { + "account_id": account_id, + "event_type": event_type, + "occurred_at": (occurred_at or _now()).astimezone(UTC).replace(tzinfo=None), + "title": (title or "")[:256], + "amount": amount, + "balance_after": balance_after, + "detail": json.dumps(detail, ensure_ascii=False, default=str) + if detail + else None, + "source": source[:24], + "source_ref": (source_ref or None), + "created_at": _now().replace(tzinfo=None), + } + + try: + # SAVEPOINT：即使这条插入失败（例如 source_ref 撞唯一键）， + # 也只回滚它自己，调用方事务里已完成的写入不受影响。 + with db.begin_nested(): + db.execute(sql, params) + return True + except Exception as exc: # noqa: BLE001 - 旁路，任何失败都不上抛 + logger.warning(f"写入账号事件失败（account={account_id} {event_type}）：{exc}") + return False + + +# ── douyu_tasks 增量收集 ────────────────────────────────── +# 只收集「业务事件」，例行轮询（query_*）一律不进时间线 —— 它们在 douyu_tasks 里 +# 占绝大多数（query_limited_goods 就 6.7 万行），收进去时间线没法看。 +COLLECTED_TASK_TYPES = ( + "exchange_goods", + "create_gold_qr", + "donate_elite_gift", + "confirm_bind", + "confirm_esports_bind", +) + + +def _task_to_event(task_type: str, result: Any) -> dict[str, Any] | None: + """把一条 douyu_tasks 行转成事件参数；解析不出返回 None（跳过并告警）。 + + ⚠️ 字段路径是**抽样实测**得到的（见设计文档 §4.2），不是猜的。 + 解析不出时必须跳过而不是抛出去 —— 一行坏数据不能卡住整条水位线。 + """ + if isinstance(result, str): + try: + result = json.loads(result) + except ValueError: + return None + if not isinstance(result, dict): + return None + + if task_type == "exchange_goods": + name = result.get("commodity_name") or "兑换" + score = result.get("score") + points_after = (result.get("points_query") or {}).get("points") + return { + "event_type": EVENT_EXCHANGE, + "title": f"兑换 {name}", + "amount": -int(score) if isinstance(score, (int, float)) else None, + "balance_after": points_after if isinstance(points_after, int) else None, + "detail": { + "commodity_name": name, + "commodity_id": result.get("commodity_id"), + "order_id": result.get("order_id"), + "exchange_id": result.get("exchange_id"), + "game_name": result.get("game_name"), + "game_channel": result.get("game_channel"), + "score": score, + }, + } + + if task_type == "create_gold_qr": + amount = result.get("buy_num") or result.get("pay_amount") + supplier = result.get("supplier_order") or {} + try: + amount_int = int(float(amount)) if amount is not None else None + except (TypeError, ValueError): + amount_int = None + return { + "event_type": EVENT_RECHARGE, + "title": f"充值 {amount_int if amount_int is not None else ''}".strip(), + "amount": amount_int, + "balance_after": None, + "detail": { + "order_id": result.get("order_id"), + "supplier_order_id": supplier.get("order_id"), + "supplier_finished_at": supplier.get("finish_time"), + "channel": result.get("recharge_channel"), + "douyu_uid": result.get("douyu_uid"), + }, + } + + if task_type == "donate_elite_gift": + gain = result.get("gift_points_expected_gain") + points = result.get("points") + will_expire = ((result.get("points_query") or {}).get("raw") or {}).get("data") + expire_at = ( + will_expire.get("willExpireTime") if isinstance(will_expire, dict) else None + ) + return { + "event_type": EVENT_POINTS_CREDIT, + "title": f"积分到账 +{gain}" if gain is not None else "积分到账", + "amount": int(gain) if isinstance(gain, (int, float)) else None, + "balance_after": int(points) if isinstance(points, (int, float)) else None, + "detail": { + "baseline": result.get("gift_points_baseline"), + "target": result.get("gift_points_target"), + "points_expire_at": expire_at, + "gold_balance": result.get("gold_balance"), + }, + } + + if task_type in ("confirm_bind", "confirm_esports_bind"): + return { + "event_type": EVENT_BIND_OK, + "title": "绑定角色" if task_type == "confirm_bind" else "电竞手册绑定", + "amount": None, + "balance_after": None, + "detail": {"raw_keys": sorted(result.keys())[:20]}, + } + + return None diff --git a/web/backend/services/automation/__init__.py b/web/backend/services/automation/__init__.py index 45ac1ce..5afe2bd 100644 --- a/web/backend/services/automation/__init__.py +++ b/web/backend/services/automation/__init__.py @@ -6,6 +6,8 @@ from __future__ import annotations +from .jobs.account_events_collect import JOB_KEY as ACCOUNT_EVENTS_KEY +from .jobs.account_events_collect import run_account_events_collect from .jobs.ck_health import JOB_KEY as CK_HEALTH_KEY from .jobs.ck_health import run_ck_health from .jobs.ck_relogin import JOB_KEY as CK_RELOGIN_KEY @@ -25,6 +27,7 @@ from .scheduler import ( ) __all__ = [ + "ACCOUNT_EVENTS_KEY", "CK_HEALTH_KEY", "CK_RELOGIN_KEY", "JobSpec", @@ -34,6 +37,7 @@ __all__ = [ "register_job", "registered_jobs", "reset_registry", + "run_account_events_collect", "run_ck_health", "run_ck_relogin", "run_due_jobs", @@ -71,3 +75,15 @@ def register_default_jobs() -> None: run=run_ck_health, ) ) + register_job( + JobSpec( + key=ACCOUNT_EVENTS_KEY, + title="账号事件收集", + # interval_seconds 这里无实际意义（收集器每轮只按水位线往后取， + # 取不到就空转），保留一个值以复用现有调度与页面。 + interval_seconds=10 * 60, + batch_size=500, + concurrency=1, + run=run_account_events_collect, + ) + ) diff --git a/web/backend/services/automation/jobs/account_events_collect.py b/web/backend/services/automation/jobs/account_events_collect.py new file mode 100644 index 0000000..ae22b1f --- /dev/null +++ b/web/backend/services/automation/jobs/account_events_collect.py @@ -0,0 +1,135 @@ +"""作业：把 douyu_tasks 里的业务操作增量收集成账号事件。 + +为什么用「收集」而不是在 runner 里直接发事件：充值/兑换/积分到账这些**已经是 +`douyu_tasks` 的行**，再去改 8 个业务文件属于重复劳动；收集器只读不写业务表， +零侵入，且新老任务类型统一处理。 + +**不回填历史**（2026-10-03 已定）：首次运行把水位线定在当前最大 id， +只往后收集。既有历史仍去各自页面查。 + +水位线存在本作业的 `automation_jobs.config` 里，进程重启不丢。 +""" + +from __future__ import annotations + +import json +from typing import Any + +from loguru import logger +from sqlalchemy import text +from sqlalchemy.orm import Session + +from ....models import AutomationJob +from ...account_events_service import ( + COLLECTED_TASK_TYPES, + _task_to_event, + emit, +) + +JOB_KEY = "account_events_collect" +WATERMARK_FIELD = "watermark" +DEFAULT_BATCH = 500 + + +def _load_config(job: AutomationJob) -> dict[str, Any]: + config = job.config + if isinstance(config, str): + try: + config = json.loads(config) + except ValueError: + config = {} + return dict(config) if isinstance(config, dict) else {} + + +def _save_watermark(db: Session, job: AutomationJob, watermark: int) -> None: + config = _load_config(job) + config[WATERMARK_FIELD] = int(watermark) + job.config = config + db.commit() + + +def run_account_events_collect(db: Session, job: AutomationJob) -> dict[str, Any]: + config = _load_config(job) + watermark = config.get(WATERMARK_FIELD) + + if not isinstance(watermark, int): + # 首次运行：水位线定在「当前最大 id」= 从此刻起往后收集，**不回溯历史**。 + max_id = db.execute( + text("SELECT COALESCE(MAX(id), 0) FROM douyu_tasks") + ).scalar() + watermark = int(max_id or 0) + _save_watermark(db, job, watermark) + logger.info( + f"账号事件收集：首次运行，水位线定在 douyu_tasks.id={watermark}（不回溯）" + ) + return { + "scanned": 0, + "skipped": 0, + "detail": {"written": 0, "watermark": watermark, "first_run": True}, + "message": f"首次运行，水位线定在 {watermark}（不回溯历史）", + } + + batch_size = max(int(job.batch_size or DEFAULT_BATCH), 1) + placeholders = ",".join(f":t{i}" for i in range(len(COLLECTED_TASK_TYPES))) + params: dict[str, Any] = {"wm": watermark, "limit": batch_size} + params.update({f"t{i}": t for i, t in enumerate(COLLECTED_TASK_TYPES)}) + + rows = ( + db.execute( + text( + f""" + SELECT id, account_id, task_type, status, result, created_at, finished_at + FROM douyu_tasks + WHERE id > :wm AND task_type IN ({placeholders}) + ORDER BY id ASC + LIMIT :limit + """ + ), + params, + ) + .mappings() + .all() + ) + + written = skipped = failed = 0 + max_id = watermark + for row in rows: + max_id = max(max_id, int(row["id"])) + # 失败的任务不产生业务事实：兑换失败不该记「消耗了积分」。 + if (row["status"] or "") != "success": + skipped += 1 + continue + payload = _task_to_event(row["task_type"], row["result"]) + if payload is None: + # 解析不出就跳过并告警，绝不让一行坏数据卡住整条水位线。 + skipped += 1 + logger.warning( + f"账号事件收集：无法解析 douyu_task {row['id']}（{row['task_type']}）" + ) + continue + ok = emit( + db, + account_id=int(row["account_id"]), + occurred_at=row["finished_at"] or row["created_at"], + source="task", + source_ref=f"douyu_task:{row['id']}", + **payload, + ) + if ok: + written += 1 + else: + failed += 1 + + if rows: + _save_watermark(db, job, max_id) + + return { + "scanned": len(rows), + "skipped": skipped, + "detail": { + "written": written, + "failed": failed, + "watermark": max_id, + }, + "message": f"本轮收集 {written} 条事件，跳过 {skipped} 条，水位线 {max_id}", + } diff --git a/web/backend/services/automation/jobs/ck_health.py b/web/backend/services/automation/jobs/ck_health.py index c6d9663..88adac3 100644 --- a/web/backend/services/automation/jobs/ck_health.py +++ b/web/backend/services/automation/jobs/ck_health.py @@ -17,6 +17,7 @@ from sqlalchemy import or_ from sqlalchemy.orm import Session from ....models import Account, AutomationJob, LoginTask +from ...account_events_service import EVENT_CK_INVALID, emit from ...cookie_check_service import check_douyu_cookie from ...douyu_service import cookie_operable_filter @@ -146,6 +147,27 @@ def run_ck_health(db: Session, job: AutomationJob) -> dict[str, Any]: "verdict": verdict, } task.ck_checked_at = now + + # 判定失效时记一条账号事件：`ck_check_status` / `ck_checked_at` 都是**覆盖式** + # 字段，覆盖之后就再也看不出「什么时候开始失效的」。只在真的判成 invalid 时记， + # 避免每轮巡检都给同一个账号刷屏。 + if verdict == "invalid": + emit( + db, + account_id=task.account_id, + event_type=EVENT_CK_INVALID, + title="CK 检测失效", + occurred_at=now, + detail={ + "message": result.get("message", ""), + "nickname": result.get("nickname"), + }, + source="login", + # 幂等键带检测时间：同一账号**再次**失效算新事件，但同一次检测 + # 重跑（例如失败重试）不会重复入库。 + source_ref=f"ck_invalid:{task.id}:{int(now.timestamp())}", + ) + if index % FLUSH_EVERY == 0: db.commit() db.commit() diff --git a/web/backend/services/automation/jobs/ck_relogin.py b/web/backend/services/automation/jobs/ck_relogin.py index 77715df..2ca34db 100644 --- a/web/backend/services/automation/jobs/ck_relogin.py +++ b/web/backend/services/automation/jobs/ck_relogin.py @@ -23,6 +23,7 @@ from sqlalchemy.orm import Session from ....database import SessionLocal from ....models import Account, AutomationJob, LoginTask, User from ....models import ProxyConfig as ProxyConfigModel +from ...account_events_service import EVENT_RELOGIN_FAILED, emit from ...douyu_service import LOGIN_ATTEMPT_IN_FLIGHT_STATUSES, cookie_holding_filter from ...login_service import LoginBatchRunner, get_relogin_limits @@ -131,6 +132,7 @@ def settle_fresh_results( try: rows = session.query(LoginTask).filter(LoginTask.id.in_(task_ids)).all() counters = settle_results( + session, {row.id: row for row in rows}, task_ids, now=now, @@ -271,6 +273,7 @@ def run_ck_relogin(db: Session, job: AutomationJob) -> dict[str, Any]: def settle_results( + db: Session, fresh: dict[int, LoginTask], task_ids: list[int], *, @@ -281,6 +284,8 @@ def settle_results( 单独抽出来是因为这段是 P1b 的核心决策逻辑，必须能脱离真实登录做单测。 只改 ``relogin_*`` 四个字段，**不碰 CK 本体**（CK 由 runner 在成功时替换）。 + + ``db`` 用于写账号事件（转人工时记一条）；事件是旁路，写失败不影响结算。 """ counters = { "scanned": len(task_ids), @@ -311,6 +316,21 @@ def settle_results( # 转人工：极验/凭证问题重试无意义，次数用尽也不该无限撞。 counters["manual"] += 1 task.relogin_next_at = None + # 只在「转人工」时记事件：还要退避重试的不记，否则同一账号每轮都刷一条。 + emit( + db, + account_id=task.account_id, + event_type=EVENT_RELOGIN_FAILED, + title=f"自动重登转人工（{kind}）", + occurred_at=now, + detail={ + "kind": kind, + "attempts": task.relogin_attempts, + "error": (task.attempt_message or "")[:200], + }, + source="login", + source_ref=f"relogin_failed:{task.id}:{int(now.timestamp())}", + ) continue if kind == "email_quota": counters["email_quota"] += 1 diff --git a/web/backend/services/douyu_runner_manual.py b/web/backend/services/douyu_runner_manual.py index 6dcd180..57c8305 100644 --- a/web/backend/services/douyu_runner_manual.py +++ b/web/backend/services/douyu_runner_manual.py @@ -11,6 +11,7 @@ from sqlalchemy.orm import Session from core.douyu import DouyuActivityClient, DouyuActivityError from ..models import Account, DouyuTask +from .account_events_service import EVENT_ELITE_OPENED, EVENT_ESPORTS_OPENED, emit from .douyu_runner_core import DOUYU_PAYMENT_POLL_INTERVAL, DOUYU_PAYMENT_POLL_SECONDS from .douyu_service import account_uid, update_account_profile_from_cookie @@ -376,6 +377,23 @@ class ManualMixin: account.bind_status = "elite_opened" account.elite_opened = True account.updated_at = datetime.now(UTC) + # elite_opened 是**覆盖式**布尔值：开通之后再操作也看不出「什么时候开的」。 + # 在这里记一条事件，补上开通时间。 + emit( + db, + account_id=account.id, + event_type=EVENT_ELITE_OPENED, + title="精英宝典开通", + occurred_at=account.updated_at, + detail={ + "points": account.points, + "expire_time": account.elite_expire_time, + }, + source="manual", + source_ref=( + f"elite_opened:{account.id}:{int(account.updated_at.timestamp())}" + ), + ) self._mark_task( db, task, "success", f"精英宝典已开通，积分: {account.points}", result ) @@ -462,6 +480,21 @@ class ManualMixin: if opened: account.esports_bind_status = "esports_opened" account.updated_at = datetime.now(UTC) + # 与精英宝典同理：状态字段是覆盖式的，这里补上「什么时候开的」。 + # 注意只在**真正开通**这一支发事件；上面「基线查到已开通」那一支是检测， + # 每次都发会刷屏。 + emit( + db, + account_id=account.id, + event_type=EVENT_ESPORTS_OPENED, + title="电竞手册开通", + occurred_at=account.updated_at, + detail={"points": account.esports_points}, + source="manual", + source_ref=( + f"esports_opened:{account.id}:{int(account.updated_at.timestamp())}" + ), + ) self._mark_task( db, task, diff --git a/web/backend/services/login_service.py b/web/backend/services/login_service.py index 775acab..a049a67 100644 --- a/web/backend/services/login_service.py +++ b/web/backend/services/login_service.py @@ -21,6 +21,7 @@ from core.douyu.proxy_fetcher import ProxyFetcher from ..models import Account as AccountModel from ..models import LoginTask from ..models import ProxyConfig as ProxyConfigModel +from .account_events_service import EVENT_LOGIN_OK, EVENT_RELOGIN_OK, emit from .cookie_check_service import check_douyu_cookie from .douyu_service import cookie_holding_filter @@ -136,9 +137,19 @@ def _mark_attempt_failed( def _write_login_success( - task: LoginTask, *, cookie: str, message: str, batch_id: str + db: Session, + task: LoginTask, + *, + cookie: str, + message: str, + batch_id: str, + event_type: str = EVENT_LOGIN_OK, ) -> None: - """登录成功后把新 CK 写回记录：这是唯一替换旧 CK 的入口。""" + """登录成功后把新 CK 写回记录：这是唯一替换旧 CK 的入口。 + + 顺带记一条账号事件 —— CK 是**覆盖式**字段，换掉之后就看不出「这账号什么时候 + 登录成功过、换过几次 CK」。事件是旁路（emit 内部吞异常），记不上也不影响登录。 + """ completed_at = datetime.now(UTC) task.status = "success" task.cookie = cookie @@ -150,6 +161,21 @@ def _write_login_success( task.attempt_batch_id = batch_id task.attempt_at = completed_at + emit( + db, + account_id=task.account_id, + event_type=event_type, + title="重新登录成功，Cookie 已更新" + if event_type == EVENT_RELOGIN_OK + else "登录成功", + occurred_at=completed_at, + detail={"message": message, "batch_id": batch_id}, + source="login", + # 幂等键用登录记录 + 完成时间：同一账号多次登录是多个事件， + # 但同一次登录重复写回不会重复入库。 + source_ref=f"login_task:{task.id}:{int(completed_at.timestamp())}", + ) + class LoginBatchRunner: """批量登录执行器，在线程中运行，通过 ThreadPoolExecutor 并发登录多个账号。""" @@ -408,10 +434,16 @@ class LoginBatchRunner: message = f"重新登录成功，但 Cookie 有效性检测失败: {check_result['message']}" # 唯一替换旧 CK 的入口：登录成功后写回新 CK。 _write_login_success( + self.db, task, cookie=result.cookie, message=message, batch_id=self.batch_id, + event_type=( + EVENT_RELOGIN_OK + if self.mode == "relogin" + else EVENT_LOGIN_OK + ), ) if self.mode == "relogin": if task.ck_check_status == "valid": diff --git a/web/frontend/src/App.tsx b/web/frontend/src/App.tsx index 1206daa..38e9686 100644 --- a/web/frontend/src/App.tsx +++ b/web/frontend/src/App.tsx @@ -11,6 +11,7 @@ const LoginPage = lazy(() => import('./pages/LoginPage')); const MainLayout = lazy(() => import('./layouts/MainLayout')); const DashboardPage = lazy(() => import('./pages/DashboardPage')); const AccountsPage = lazy(() => import('./pages/AccountsPage')); +const AccountDetailPage = lazy(() => import('./pages/AccountDetailPage')); const AccountCheckPage = lazy(() => import('./pages/AccountCheckPage')); const AssignmentsPage = lazy(() => import('./pages/AssignmentsPage')); const LoginTasksPage = lazy(() => import('./pages/LoginTasksPage')); @@ -82,6 +83,7 @@ function AppContent() { > )} /> )} /> + )} /> )} /> )} /> )} /> diff --git a/web/frontend/src/api/accounts.ts b/web/frontend/src/api/accounts.ts index 173fa0f..f2884d1 100644 --- a/web/frontend/src/api/accounts.ts +++ b/web/frontend/src/api/accounts.ts @@ -1,8 +1,10 @@ import api from './client'; import type { + AccountEventPage, AccountItem, AccountBulkSelection, AccountBulkTagRequest, + AccountOverview, AssignmentsSummary, BasicSummary, MessageCountResponse, @@ -38,4 +40,11 @@ export const accountApi = { api.delete('/accounts/batch/delete', { params: { account_ids: account_ids.join(',') } }), batchDeleteSelection: (data: AccountBulkSelection) => api.post('/accounts/batch-delete', data), + // 详情页：状态卡 + 统计 + overview: (id: number) => api.get(`/accounts/${id}/overview`), + // 时间线（倒序分页），event_type 为空表示全部 + events: (id: number, params: { page?: number; page_size?: number; event_type?: string }) => + api.get(`/accounts/${id}/events`, { + params: { page: 1, page_size: 50, ...params, event_type: params.event_type || '' }, + }), }; diff --git a/web/frontend/src/api/types.ts b/web/frontend/src/api/types.ts index 8436654..8ea0857 100644 --- a/web/frontend/src/api/types.ts +++ b/web/frontend/src/api/types.ts @@ -71,6 +71,82 @@ export interface BasicSummary { tag_count?: number; } +// ==================== 账号详情与事件历史 ==================== + +/** 事件类型（与后端 account_events_service 的常量保持一致）。 */ +export type AccountEventType = + | 'imported' + | 'login_ok' + | 'ck_invalid' + | 'relogin_ok' + | 'relogin_failed' + | 'bind_ok' + | 'recharge' + | 'exchange' + | 'points_credit' + | 'elite_opened' + | 'esports_opened' + | 'assigned' + | 'tag_changed'; + +export interface AccountEventItem { + id: number; + event_type: AccountEventType | string; + occurred_at: string; + title: string; + /** 有符号：充值为正、兑换消耗为负 */ + amount: number | null; + balance_after: number | null; + detail: Record | null; + source: string; +} + +export interface AccountEventPage { + items: AccountEventItem[]; + total: number; + page: number; + page_size: number; +} + +export interface AccountOverviewStats { + recharge_total: number; + exchange_count: number; + points_spent: number; + ck_invalid_count: number; + event_count: number; + login_count: number; + last_login_at: string | null; + last_ck_check_at: string | null; +} + +export interface AccountOverview { + id: number; + username: string; + nickname: string; + uid: string; + tag: string; + remark: string; + assigned_to: number | null; + /** 导入时间 */ + created_at: string | null; + points: number | null; + gold_balance: number | null; + exchange_balance: number | null; + game_name: string; + game_channel: string; + bind_status: string; + elite_opened: boolean | null; + elite_expire_time: number | null; + esports_points: number | null; + esports_bind_status: string; + esports_game_name: string; + esports_can_change_time: number | null; + auto_managed: boolean; + stats: AccountOverviewStats; + /** 开通时间来自事件表（上线后才开始记录，更早的不显示） */ + opened_at: { elite: string | null; esports: string | null }; +} + export interface TaskSummary { total: number; success: number; diff --git a/web/frontend/src/pages/AccountDetailPage.tsx b/web/frontend/src/pages/AccountDetailPage.tsx new file mode 100644 index 0000000..5f34b44 --- /dev/null +++ b/web/frontend/src/pages/AccountDetailPage.tsx @@ -0,0 +1,266 @@ +import { useCallback, useEffect, useState } from 'react'; +import { useNavigate, useParams } from 'react-router-dom'; +import { + Button, Card, Descriptions, Empty, Select, Space, Spin, Statistic, Table, Tag, Tooltip, Typography, +} from 'antd'; +import type { TableProps } from 'antd'; +import { + ArrowLeftOutlined, ReloadOutlined, +} from '@ant-design/icons'; +import { accountApi, type AccountEventItem, type AccountOverview } from '../api/modules'; +import { getErrorMessage } from '../utils/error'; +import { message } from '../utils/antdMessage'; +import { formatTime } from '../utils/time'; + +const { Text, Title } = Typography; + +/** + * 事件类型 → 展示样式。 + * 颜色用来一眼区分「钱进来 / 分出去 / 状态变化」三类，而不是按类型各给一个颜色。 + */ +const EVENT_STYLE: Record = { + imported: { label: '导入', color: 'default' }, + login_ok: { label: '登录', color: 'blue' }, + ck_invalid: { label: 'CK 失效', color: 'red' }, + relogin_ok: { label: '重登成功', color: 'green' }, + relogin_failed: { label: '重登转人工', color: 'orange' }, + bind_ok: { label: '绑定', color: 'cyan' }, + recharge: { label: '充值', color: 'green' }, + exchange: { label: '兑换', color: 'purple' }, + points_credit: { label: '积分到账', color: 'gold' }, + elite_opened: { label: '精英宝典', color: 'magenta' }, + esports_opened: { label: '电竞手册', color: 'volcano' }, + assigned: { label: '分配', color: 'default' }, + tag_changed: { label: '标签', color: 'default' }, +}; + +function eventTag(type: string) { + const style = EVENT_STYLE[type] ?? { label: type, color: 'default' }; + return {style.label}; +} + +/** 金额展示：充值为正（绿色 ¥），兑换消耗为负（红色积分）。 */ +function amountText(item: AccountEventItem) { + if (item.amount === null || item.amount === undefined) return null; + if (item.event_type === 'recharge') { + return +¥{item.amount}; + } + if (item.event_type === 'exchange') { + return {item.amount} 分; + } + return ( + = 0 ? 'success' : 'danger'}> + {item.amount > 0 ? '+' : ''} + {item.amount} + + ); +} + +export default function AccountDetailPage() { + const { id } = useParams<{ id: string }>(); + const accountId = Number(id); + const navigate = useNavigate(); + + const [overview, setOverview] = useState(null); + const [loading, setLoading] = useState(true); + const [events, setEvents] = useState([]); + const [eventTotal, setEventTotal] = useState(0); + const [page, setPage] = useState(1); + const [pageSize, setPageSize] = useState(20); + const [eventType, setEventType] = useState(''); + const [eventsLoading, setEventsLoading] = useState(false); + + const loadOverview = useCallback(async () => { + if (!accountId) return; + setLoading(true); + try { + setOverview(await accountApi.overview(accountId)); + } catch (e: unknown) { + message.error(getErrorMessage(e)); + } finally { + setLoading(false); + } + }, [accountId]); + + const loadEvents = useCallback(async () => { + if (!accountId) return; + setEventsLoading(true); + try { + const data = await accountApi.events(accountId, { + page, + page_size: pageSize, + event_type: eventType, + }); + setEvents(data.items); + setEventTotal(data.total); + } catch (e: unknown) { + message.error(getErrorMessage(e)); + } finally { + setEventsLoading(false); + } + }, [accountId, page, pageSize, eventType]); + + useEffect(() => { + void loadOverview(); + }, [loadOverview]); + + useEffect(() => { + void loadEvents(); + }, [loadEvents]); + + const columns: TableProps['columns'] = [ + { + title: '时间', + dataIndex: 'occurred_at', + width: 165, + render: (val: string) => formatTime(val), + }, + { + title: '类型', + dataIndex: 'event_type', + width: 100, + render: (val: string) => eventTag(val), + }, + { title: '说明', dataIndex: 'title', ellipsis: true }, + { + title: '金额 / 变动', + dataIndex: 'amount', + width: 110, + align: 'right', + render: (_val, record) => amountText(record) ?? -, + }, + { + title: '变动后余额', + dataIndex: 'balance_after', + width: 110, + align: 'right', + render: (val: number | null) => + val === null || val === undefined ? - : val, + }, + ]; + + if (!accountId) { + return ; + } + + const stats = overview?.stats; + + return ( +
+ + + + + + + + {overview?.username || '—'} + {overview?.tag && {overview.tag}} + {overview?.auto_managed === false && 不参与自动化} + + } + style={{ marginBottom: 12 }} + > + + {overview?.nickname || '-'} + {overview?.uid || '-'} + {formatTime(overview?.created_at)} + + {overview?.game_name || '-'} + {overview?.game_channel ? （{overview.game_channel}） : null} + + {overview?.bind_status || '-'} + {overview?.remark || '-'} + + +
+ + + + + +
+ + + + {formatTime(overview?.opened_at?.elite)} + + + {formatTime(overview?.opened_at?.esports)} + + + {formatTime(stats?.last_ck_check_at)} + + + +
+ + + + + +
+ + + 开通时间与累计统计来自「账号事件」，从该功能上线起开始记录； + 更早的历史请到「登录任务 / 精英宝典 / 电竞手册 / 和平小店」页查看。 + + + + + +