diff --git a/docs/斗鱼/斗鱼账号平台最终实施设计.md b/docs/斗鱼/斗鱼账号平台最终实施设计.md index d543e6c..967b615 100644 --- a/docs/斗鱼/斗鱼账号平台最终实施设计.md +++ b/docs/斗鱼/斗鱼账号平台最终实施设计.md @@ -263,6 +263,12 @@ AND force_refresh=false 已有 pending/running 工作项时复用,不创建第二条。 +成功响应除了写当前投影,还保存官方原始字段: + +- `account_sync_states.raw_payload_json`:账号最近一次完整原始响应; +- `account_commodity_states.raw_json`:商品级原始字段; +- 任务 `result` 只保留摘要,避免把任务表当数据库。 + ## 商品快照 `StorehomeSnapshot` 至少包含: @@ -274,6 +280,8 @@ AND force_refresh=false - `items[]` - `by_commodity` - `limited_ids` +- `available_from / available_until / schedule_json` +- `raw` - `success` - `raw` diff --git a/docs/斗鱼/精英宝典商品资格与全量快照.md b/docs/斗鱼/精英宝典商品资格与全量快照.md index 3625dc8..4a99472 100644 --- a/docs/斗鱼/精英宝典商品资格与全量快照.md +++ b/docs/斗鱼/精英宝典商品资格与全量快照.md @@ -117,7 +117,7 @@ GET /wgapi/ordnc/activity/peace/storehome | `storeStatus=0` | 官方商城当前不可售 | `availability=no_stock` | | `openStatus=0` | 活动尚未开放 | `availability=not_open` | | `count<=0` | 当前库存不足 | `availability=no_stock` | -| `startTime/endTime` | 商品外层开放窗口 | 与本地时间比较 | +| `startTime/endTime` | 商品外层开放窗口 | 与本地时间比较,投影到 `available_from/available_until` | | `timeLimited` | 是否限时开放 | `time_limited` | | `specTimes` | 一天多场或分段开放计划 | `schedule_json` | | `eliteLimited=1` | 商品要求精英手册资格 | 单独保存为门槛,不能写成账号限兑 | @@ -186,6 +186,18 @@ POST /wgapi/ordnc/activity/peace/batchExchangeLimit 只在确定商品 `batchExchange=1` 且准备执行批量兑换时调用。 +限时商品不是“查到 `status=0` 就可以换”。自动选号和最终复核都必须同时满足: + +```text +availability = available +AND (available_from IS NULL OR available_from <= now) +AND (available_until IS NULL OR available_until >= now) +AND valid_until > now +``` + +`next_check_at` 会落在窗口边界或能力 TTL 两者中更早的时间点。跨过边界后必须先重新 +查询官方,不能把上一时段形成的 `available` 继续当作可兑换。 + ## 当前代码路径 ### 单次 HTTP 入口 @@ -199,31 +211,31 @@ POST /wgapi/ordnc/activity/peace/batchExchangeLimit ### 当前只读包装 -`web/backend/services/douyu_activity_query.py` 当前有两个包装: +`web/backend/services/douyu_activity_query.py` 提供: - `query_activity_goods()`:返回完整商品列表; +- `query_activity_goods_result()`:返回完整商品列表和顶层原始响应; - `query_limited_goods_result()`:返回受限商品和原始响应。 -两者底层都是同一个 `storehome` 请求,但调用方可能在不同分支重复请求。 +底层都是同一个 `storehome` 请求。调用方不再逐礼包查询。 -### 当前正确路径 +### 当前实现 -`query_account_overview` 获取完整商品列表后调用 -`sync_account_goods_states()`,可一次写入全部商品状态。 +`storehome_snapshot_from_result()` 把一次官方响应归一成 `StorehomeSnapshot`; +`sync_storehome_snapshot()` 一次写入全部商品能力,并保存: -### 当前缺口 +- `account_commodity_states.raw_json`:每个商品的完整原始字段; +- `account_sync_states.raw_payload_json`:该账号最近一次官方查询的完整原始响应; +- `account_sync_states.payload_hash`:原始响应 SHA-256,用于判断是否真的发生变化; +- `available_from / available_until / schedule_json`:限时窗口; +- `account_sync_runs`:本次同步的输入条数、变更数、耗时和结果。 -1. `query_limited_goods` 任务拿到完整 `storehome` 响应,但目前只处理受限商品和 - 高价值套装,没有把全部商品写入 `account_commodity_states`。 -2. 履约实时检测 `refresh_douyu_account_state()` 请求了完整 `storehome`,但只提取 - `limited_ids`,仅为当前订单商品更新一条能力记录,其余商品状态被丢弃。 -3. `probe_account()` 的 `with_eligibility` 和 `with_goods` 是独立分支。未来同时开启 - 两者时,会各请求一次同一个 `storehome`。 -4. 没有统一的快照对象约束所有调用方共享一次响应。 +`query_account_overview`、`query_limited_goods`、`refresh_goods` 和履约实时检测都 +走这条路径,任务 JSON 只保留结果摘要,不再承担账号数据库职责。 ## 目标快照契约 -建议建立唯一快照入口: +唯一快照入口: ```text fetch_storehome_snapshot(account, manual_id, rid) @@ -243,6 +255,9 @@ goods_query_ok raw ``` +保留原始响应是有意设计:限时窗口、库存、开放状态和账号资格字段由斗鱼控制,后续可能 +新增字段。归一化投影用于快速筛选,原始字段用于回溯、重放和修复解析规则。 + 以下能力全部从同一快照派生: - 完整商品列表; diff --git a/docs/斗鱼/设计-账号能力中心与统一选号.md b/docs/斗鱼/设计-账号能力中心与统一选号.md index ee4445f..0414f7c 100644 --- a/docs/斗鱼/设计-账号能力中心与统一选号.md +++ b/docs/斗鱼/设计-账号能力中心与统一选号.md @@ -108,6 +108,7 @@ CK 生命周期、账号库存目标、覆盖率、查询同步契约和防重 - `availability`:`available / limited / no_stock / not_open / expired / not_offered / unknown`; - `raw_status`:官网原始账号级状态; +- `raw_json`:该商品最近一次官方原始字段; - `observed_at`:最近一次观察时间; - `valid_until`:当前结论的匹配有效期; - `next_check_at`:下次建议检查时间; @@ -117,6 +118,9 @@ CK 生命周期、账号库存目标、覆盖率、查询同步契约和防重 这张表保存的是当前投影,不是只用于 15 分钟缓存的临时数据。过期记录仍然可以被账号 管理查询和展示,只是不能直接作为自动分配依据。 +`storehome` 顶层完整响应保存在 `account_sync_states.raw_payload_json`。一层保存最新 +顶层响应供协议回溯,一层保存商品级原始字段供精确重放;两者都不替代规范化匹配条件。 + ### 事件账本 `account_events` 继续保存不可变事实: diff --git a/tests/test_account_actions.py b/tests/test_account_actions.py index 3ead7bf..2ed5ffc 100644 --- a/tests/test_account_actions.py +++ b/tests/test_account_actions.py @@ -264,26 +264,9 @@ class TestCkInvalidOnlyOnTransition(_Base): self._run() assert len(self.events(account.id, svc.EVENT_CK_INVALID)) == 1 - # 恢复(巡检判 valid) - from web.backend.services.automation.jobs import ck_health - - original = ck_health.check_douyu_cookie - ck_health.check_douyu_cookie = lambda _c: { - "valid": True, - "message": "ok", - "fish_ball": 1, - "nickname": "n", - "level": 1, - "checked_at": NOW, - } - self.session.query(Account).update( - {Account.ck_checked_at: None}, synchronize_session=False - ) + # invalid 不再进入普通 CK 巡检;恢复由重登成功后的 CK 验证写回。 + on_ck_verdict(self.session, account, "valid", detail={}, occurred_at=NOW) self.session.commit() - try: - ck_health.run_ck_health(self.session, self._job()) - finally: - ck_health.check_douyu_cookie = original # 再失效 self._run() diff --git a/tests/test_account_capability_service.py b/tests/test_account_capability_service.py index 68123d6..482a70e 100644 --- a/tests/test_account_capability_service.py +++ b/tests/test_account_capability_service.py @@ -117,8 +117,11 @@ def test_sync_stores_time_window_and_next_boundary(db): assert state.availability == "limited" assert state.time_limited is True assert state.available_until == end_at + assert state.valid_until == end_at assert state.next_check_at == end_at assert state.schedule_json + assert state.raw_json["timeLimited"] == 1 + assert state.raw_json["specTimes"][0]["endTime"] == str(int(end_at.timestamp())) def test_sync_account_goods_states_writes_capabilities_and_exchange_stats(db): diff --git a/tests/test_account_platform_foundation.py b/tests/test_account_platform_foundation.py new file mode 100644 index 0000000..2f76826 --- /dev/null +++ b/tests/test_account_platform_foundation.py @@ -0,0 +1,380 @@ +"""账号平台基础闭环:全量快照、工作队列、CK 周期和登录预算。""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta +from types import SimpleNamespace + +import pytest + +from web.backend.models import ( + Account, + AccountCommodityState, + AccountSyncState, + AccountWorkItem, + LoginQuotaSetting, +) +from web.backend.services import account_sync_service +from web.backend.services.account_lifecycle import on_ck_verdict +from web.backend.services.account_sync_service import ( + DOMAIN_STOREHOME, + sync_storehome_snapshot, +) +from web.backend.services.account_work_queue import ( + complete_account_work, + enqueue_account_work, +) +from web.backend.services.automation.jobs import ck_health +from web.backend.services.douyu_account_state import DouyuLiveAccountState +from web.backend.services.douyu_read_gateway import StorehomeSnapshot +from web.backend.services.login_quota_service import ( + CATEGORY_MANUAL, + finish_login_attempt, + reserve_login_attempt, +) +from web.backend.services.platform_fulfillment.service import _exchange_gate + + +def _account(name: str, **overrides) -> Account: + values = { + "username": name, + "password": "p", + "email": f"{name}@example.com", + "email_password": "ep", + "cookie": "acf_uid=100", + "cookie_updated_at": datetime.now(UTC), + "status": "active", + "auto_managed": True, + } + values.update(overrides) + return Account(**values) + + +def _snapshot(count: int = 60) -> StorehomeSnapshot: + items = tuple( + { + "commodityId": f"goods-{index}", + "commodityName": f"商品 {index}", + "score": index * 10, + "status": 1 if index == 0 else 0, + } + for index in range(count) + ) + return StorehomeSnapshot( + manual_id="manual", + rid="room", + observed_at=datetime.now(UTC), + items=items, + limited_ids=frozenset({"goods-0"}), + raw={"data": {"list": list(items)}}, + ) + + +def test_storehome_snapshot_writes_all_goods_and_sync_state(db): + account = _account("snapshot-account") + db.add(account) + db.commit() + + result = sync_storehome_snapshot( + db, + account, + _snapshot(), + source="test.storehome", + ) + db.commit() + + states = ( + db.query(AccountCommodityState) + .filter( + AccountCommodityState.account_id == account.id, + AccountCommodityState.scope == "elite", + ) + .all() + ) + sync_state = ( + db.query(AccountSyncState) + .filter( + AccountSyncState.account_id == account.id, + AccountSyncState.domain == DOMAIN_STOREHOME, + ) + .one() + ) + assert result.item_count == 60 + assert len(states) == 60 + assert {row.commodity_id for row in states} == { + f"goods-{index}" for index in range(60) + } + assert sync_state.status == "success" + assert sync_state.valid_until is not None + assert sync_state.raw_payload_json["data"]["list"][0]["commodityId"] == "goods-0" + assert len(sync_state.payload_hash) == 64 + limited = next(row for row in states if row.commodity_id == "goods-0") + assert limited.raw_json["status"] == 1 + + +def test_storehome_sync_failure_is_recorded_without_success_state(db, monkeypatch): + account = _account("snapshot-failure") + db.add(account) + db.commit() + + def fail(*_args, **_kwargs): + raise RuntimeError("projection failed") + + monkeypatch.setattr(account_sync_service, "sync_account_goods_states", fail) + + with pytest.raises(RuntimeError, match="projection failed"): + sync_storehome_snapshot( + db, + account, + _snapshot(), + source="test.storehome", + ) + + sync_state = ( + db.query(AccountSyncState) + .filter( + AccountSyncState.account_id == account.id, + AccountSyncState.domain == DOMAIN_STOREHOME, + ) + .one() + ) + assert sync_state.status == "failed" + assert "projection failed" in sync_state.error_message + assert sync_state.next_retry_at is not None + assert sync_state.raw_payload_json["data"]["list"][0]["commodityId"] == "goods-0" + + +def test_account_work_queue_deduplicates_only_active_items(db): + account = _account("queue-account") + db.add(account) + db.commit() + + first = enqueue_account_work( + db, + account_id=account.id, + work_type="storehome_sync", + priority=80, + ) + second = enqueue_account_work( + db, + account_id=account.id, + work_type="storehome_sync", + priority=20, + ) + assert second.id == first.id + assert second.priority == 20 + + complete_account_work(db, first) + third = enqueue_account_work( + db, + account_id=account.id, + work_type="storehome_sync", + priority=80, + ) + assert third.id != first.id + assert db.query(AccountWorkItem).count() == 2 + + +def test_ck_verdict_updates_lifecycle_and_moves_relogin_work(db): + issued_at = datetime.now(UTC) - timedelta(days=7, hours=9) + account = _account("ck-lifecycle", cookie_updated_at=issued_at) + db.add(account) + db.commit() + + invalid_at = datetime.now(UTC) + on_ck_verdict( + db, + account, + "invalid", + detail={"message": "未登录"}, + occurred_at=invalid_at, + ) + db.flush() + + work = ( + db.query(AccountWorkItem) + .filter( + AccountWorkItem.account_id == account.id, + AccountWorkItem.work_type == "login_relogin", + ) + .one() + ) + assert work.status == "pending" + assert account.ck_invalid_detected_at == invalid_at + assert account.ck_observed_lifetime_seconds is not None + assert account.ck_expected_expires_at is None + + account.cookie_updated_at = datetime.now(UTC) + on_ck_verdict( + db, + account, + "valid", + detail={"message": "有效"}, + occurred_at=datetime.now(UTC), + ) + db.flush() + + db.refresh(work) + assert work.status == "success" + assert account.ck_last_valid_at is not None + assert account.ck_expected_expires_at is not None + assert account.ck_expiry_source == "predicted" + + +def test_ck_health_skips_invalid_and_manual_accounts(db): + accounts = [ + _account("ck-empty", ck_check_status=""), + _account("ck-valid", ck_check_status="valid"), + _account("ck-invalid", ck_check_status="invalid"), + _account("ck-manual", ck_check_status="manual"), + ] + db.add_all(accounts) + db.commit() + job = SimpleNamespace(interval_seconds=3600, batch_size=200) + + selected = ck_health._select_accounts(db, job, datetime.now(UTC)) + + assert {account.username for account in selected} == {"ck-empty", "ck-valid"} + + +def test_time_limited_goods_block_fulfillment_outside_window(): + state = DouyuLiveAccountState( + account_id=1, + account_name="time-limited", + checked_at=datetime.now(UTC), + ck_verdict="valid", + ck_message="有效", + elite_opened=True, + points=1000, + limited_ids=frozenset(), + role_change_available_at=None, + commodity_availability={"time-goods": "expired"}, + commodity_blocking_reasons={"time-goods": "限时兑换已结束"}, + ) + + reasons = state.blocking_reasons(commodity_id="time-goods") + gate = _exchange_gate(state, "time-goods") + + assert "限时兑换已结束" in reasons + assert gate == ("not_available", "commodity_expired", "限时兑换已结束") + + +def test_missing_storehome_goods_blocks_fulfillment_as_off_shelf(): + state = DouyuLiveAccountState( + account_id=2, + account_name="off-shelf", + checked_at=datetime.now(UTC), + ck_verdict="valid", + ck_message="有效", + elite_opened=True, + points=1000, + limited_ids=frozenset(), + role_change_available_at=None, + commodity_availability={"another-goods": "available"}, + ) + + reasons = state.blocking_reasons(commodity_id="removed-goods") + gate = _exchange_gate(state, "removed-goods") + + assert any("可能已下架" in reason for reason in reasons) + assert gate[:2] == ("not_available", "commodity_not_offered") + + +def test_login_quota_enforces_per_account_and_hourly_budget(db): + first = _account("quota-first") + second = _account("quota-second") + third = _account("quota-third") + fourth = _account("quota-fourth") + db.add_all([first, second, third, fourth]) + db.add( + LoginQuotaSetting( + id=1, + daily_attempt_limit=20, + hourly_attempt_limit=2, + per_account_daily_limit=2, + manual_daily_limit=10, + ) + ) + db.commit() + now = datetime.now(UTC).replace(microsecond=0) + + first_attempt = reserve_login_attempt( + db, + account=first, + mode="login", + trigger_source="manual", + batch_id="batch-1", + created_by=None, + now=now.replace(hour=1), + ) + assert first_attempt.allowed + assert first_attempt.attempt is not None + assert first_attempt.attempt.category == CATEGORY_MANUAL + finish_login_attempt( + db, + first_attempt.attempt, + status="success", + now=now.replace(hour=1), + ) + + second_attempt = reserve_login_attempt( + db, + account=first, + mode="login", + trigger_source="manual", + batch_id="batch-2", + created_by=None, + now=now.replace(hour=2), + ) + assert second_attempt.allowed + assert second_attempt.attempt is not None + finish_login_attempt( + db, + second_attempt.attempt, + status="failed", + now=now.replace(hour=2), + ) + + denied_same_account = reserve_login_attempt( + db, + account=first, + mode="login", + trigger_source="manual", + batch_id="batch-3", + created_by=None, + now=now.replace(hour=3), + ) + assert not denied_same_account.allowed + assert "上限" in denied_same_account.reason + + first_hour = reserve_login_attempt( + db, + account=second, + mode="login", + trigger_source="manual", + batch_id="batch-4", + created_by=None, + now=now.replace(hour=4), + ) + second_hour = reserve_login_attempt( + db, + account=third, + mode="login", + trigger_source="manual", + batch_id="batch-4", + created_by=None, + now=now.replace(hour=4), + ) + third_hour = reserve_login_attempt( + db, + account=fourth, + mode="login", + trigger_source="manual", + batch_id="batch-4", + created_by=None, + now=now.replace(hour=4), + ) + assert first_hour.allowed + assert second_hour.allowed + assert not third_hour.allowed + assert "每小时" in third_hour.reason diff --git a/tests/test_migrations.py b/tests/test_migrations.py index 5addfc3..84bd430 100644 --- a/tests/test_migrations.py +++ b/tests/test_migrations.py @@ -42,7 +42,7 @@ TESTS_DIR = Path(__file__).resolve().parent PROJECT_ROOT = TESTS_DIR.parent VERSIONS_DIR = PROJECT_ROOT / "web" / "backend" / "migrations" / "versions" -HEAD_REVISION = "20261121_0109" +HEAD_REVISION = "20261121_0110" # 迁移新增、但不声明在模型里的复合查询索引。 EXTRA_INDEXES = ( @@ -58,6 +58,54 @@ EXTRA_INDEXES = ( ) +_ACCOUNT_PLATFORM_COLUMNS = { + "ck_expected_expires_at": "DATETIME NULL", + "ck_last_valid_at": "DATETIME NULL", + "ck_invalid_detected_at": "DATETIME NULL", + "ck_observed_lifetime_seconds": "INTEGER NULL", + "ck_expiry_source": "VARCHAR(24) NOT NULL DEFAULT ''", +} + +_LATER_ORM_COLUMNS = { + "platform_fulfillment_tasks": { + "sale_mode": "VARCHAR(16) NOT NULL DEFAULT 'unknown'", + "campaign_key": "VARCHAR(128) NOT NULL DEFAULT ''", + "campaign_name": "VARCHAR(255) NOT NULL DEFAULT ''", + "offer_available_from": "DATETIME NULL", + "offer_available_until": "DATETIME NULL", + "classification_source": "VARCHAR(24) NOT NULL DEFAULT 'unknown'", + "exchange_result_image_key": "VARCHAR(512) NOT NULL DEFAULT ''", + "exchange_result_image_saved_at": "DATETIME NULL", + }, + "douyu_external_orders": { + "succeeded_at": "DATETIME NULL", + }, +} + + +def _add_account_platform_columns(engine) -> None: + """让“旧 revision + 当前 ORM”测试夹具兼容已新增的账号列。""" + inspector = inspect(engine) + existing = {item["name"] for item in inspector.get_columns("accounts")} + with engine.begin() as conn: + for name, ddl in _ACCOUNT_PLATFORM_COLUMNS.items(): + if name not in existing: + conn.execute(text(f"ALTER TABLE accounts ADD COLUMN `{name}` {ddl}")) + + +def _add_later_orm_columns(engine, tables: tuple[str, ...]) -> None: + """旧 revision 上用当前 ORM 写夹具时,补齐后续纯列迁移。""" + inspector = inspect(engine) + with engine.begin() as conn: + for table in tables: + existing = {item["name"] for item in inspector.get_columns(table)} + for name, ddl in _LATER_ORM_COLUMNS[table].items(): + if name not in existing: + conn.execute( + text(f"ALTER TABLE `{table}` ADD COLUMN `{name}` {ddl}") + ) + + def _migration_config() -> Config: config = Config(str(PROJECT_ROOT / "alembic.ini")) config.set_main_option( @@ -458,6 +506,7 @@ class TestMigrationSmoke: cfg = _migration_config() cfg.set_main_option("sqlalchemy.url", mig_url) command.upgrade(cfg, "20261121_0106") + _add_account_platform_columns(mig_engine) with Session(mig_engine) as session: user = User( username="repair-0107-user", @@ -919,6 +968,7 @@ class TestMigrationSmoke: cfg = _migration_config() cfg.set_main_option("sqlalchemy.url", mig_url) command.upgrade(cfg, "20261118_0089") + _add_account_platform_columns(mig_engine) with Session(mig_engine) as session: user = User( @@ -1059,6 +1109,7 @@ class TestMigrationSmoke: cfg = _migration_config() cfg.set_main_option("sqlalchemy.url", mig_url) command.upgrade(cfg, "20261118_0091") + _add_account_platform_columns(mig_engine) with Session(mig_engine) as session: account = Account( @@ -1121,6 +1172,10 @@ class TestMigrationSmoke: cfg = _migration_config() cfg.set_main_option("sqlalchemy.url", mig_url) command.upgrade(cfg, "20261118_0094") + _add_later_orm_columns( + mig_engine, + ("platform_fulfillment_tasks", "douyu_external_orders"), + ) with Session(mig_engine) as session: repaired_task = PlatformFulfillmentTask( @@ -1194,6 +1249,8 @@ class TestMigrationSmoke: cfg = _migration_config() cfg.set_main_option("sqlalchemy.url", mig_url) command.upgrade(cfg, "20261119_0097") + _add_account_platform_columns(mig_engine) + _add_later_orm_columns(mig_engine, ("douyu_external_orders",)) with Session(mig_engine) as session: owner = User( diff --git a/web/backend/main.py b/web/backend/main.py index 2f1f56f..07e174e 100644 --- a/web/backend/main.py +++ b/web/backend/main.py @@ -60,6 +60,7 @@ async def lifespan(app: FastAPI): from .database import SessionLocal, dispose_engines from .services.douyu_service import cleanup_orphan_douyu_tasks from .services.huya_service import cleanup_orphan_huya_tasks + from .services.login_quota_service import cleanup_orphan_login_attempts from .services.login_service import ( cleanup_orphan_login_tasks, cleanup_orphan_relogin_tasks, @@ -84,6 +85,9 @@ async def lifespan(app: FastAPI): cleaned_login = cleanup_orphan_login_tasks(db) if cleaned_login: logger.info(f"启动清理登录残留任务: {cleaned_login} 条") + cleaned_attempts = cleanup_orphan_login_attempts(db) + if cleaned_attempts: + logger.info(f"启动收敛中断的登录尝试流水: {cleaned_attempts} 条") from .services.yyb_service import cleanup_orphan_yyb_tasks cleaned_yyb = cleanup_orphan_yyb_tasks(db, message="任务已中断(服务重启)") diff --git a/web/backend/migrations/migration_manifest.json b/web/backend/migrations/migration_manifest.json index 5a4b5e2..a3aca73 100644 --- a/web/backend/migrations/migration_manifest.json +++ b/web/backend/migrations/migration_manifest.json @@ -437,6 +437,10 @@ "20261121_0109_account_pool_tier_index.py": { "revision": "20261121_0109", "sha256": "0aa85febbc0f2c99f5768bba2364f8797a748c4de740780e33a2017311f591b0" + }, + "20261121_0110_account_platform_foundation.py": { + "revision": "20261121_0110", + "sha256": "94c0158a061a7670d1c15cfd57172d98eddd271a4d82be0cb7d0ea65013f8712" } } } diff --git a/web/backend/migrations/versions/20261121_0110_account_platform_foundation.py b/web/backend/migrations/versions/20261121_0110_account_platform_foundation.py new file mode 100644 index 0000000..aab2b03 --- /dev/null +++ b/web/backend/migrations/versions/20261121_0110_account_platform_foundation.py @@ -0,0 +1,650 @@ +"""斗鱼账号平台基础:CK 周期、同步状态、工作队列、供给目标和登录配额。""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op + +revision: str = "20261121_0110" +down_revision: str | None = "20261121_0109" +branch_labels: str | None = None +depends_on: str | None = None + + +def _table_exists(table: str) -> bool: + return bool( + op.get_bind() + .execute( + sa.text( + """ + SELECT COUNT(*) + FROM information_schema.tables + WHERE table_schema = DATABASE() + AND table_name = :table + """ + ), + {"table": table}, + ) + .scalar() + ) + + +def _column_exists(table: str, column: str) -> bool: + return bool( + op.get_bind() + .execute( + sa.text( + """ + SELECT COUNT(*) + FROM information_schema.columns + WHERE table_schema = DATABASE() + AND table_name = :table + AND column_name = :column + """ + ), + {"table": table, "column": column}, + ) + .scalar() + ) + + +def upgrade() -> None: + for column in ( + sa.Column("ck_expected_expires_at", sa.DateTime(), nullable=True), + sa.Column("ck_last_valid_at", sa.DateTime(), nullable=True), + sa.Column("ck_invalid_detected_at", sa.DateTime(), nullable=True), + sa.Column("ck_observed_lifetime_seconds", sa.Integer(), nullable=True), + sa.Column( + "ck_expiry_source", + sa.String(length=24), + nullable=False, + server_default="", + ), + ): + if not _column_exists("accounts", column.name): + op.add_column("accounts", column) + + # 商品级原始字段:已有商品能力表,必须只做加法,不能靠 create_table 兜底。 + if not _column_exists("account_commodity_states", "raw_json"): + op.add_column( + "account_commodity_states", + sa.Column("raw_json", sa.JSON(), nullable=True), + ) + + if not _table_exists("account_sync_states"): + op.create_table( + "account_sync_states", + sa.Column("id", sa.BigInteger(), primary_key=True, autoincrement=True), + sa.Column( + "account_id", + sa.Integer(), + sa.ForeignKey("accounts.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column( + "platform", + sa.String(length=16), + nullable=False, + server_default="douyu", + ), + sa.Column("domain", sa.String(length=32), nullable=False), + sa.Column("last_attempt_at", sa.DateTime(), nullable=True), + sa.Column("last_success_at", sa.DateTime(), nullable=True), + sa.Column("valid_until", sa.DateTime(), nullable=True), + sa.Column("next_retry_at", sa.DateTime(), nullable=True), + sa.Column( + "status", + sa.String(length=16), + nullable=False, + server_default="never", + ), + sa.Column( + "error_code", + sa.String(length=64), + nullable=False, + server_default="", + ), + sa.Column("error_message", sa.Text(), nullable=False), + sa.Column( + "source", + sa.String(length=32), + nullable=False, + server_default="", + ), + sa.Column( + "payload_hash", + sa.String(length=64), + nullable=False, + server_default="", + ), + sa.Column("raw_payload_json", sa.JSON(), nullable=True), + sa.Column( + "state_version", + sa.BigInteger(), + nullable=False, + server_default="0", + ), + sa.Column( + "created_at", + sa.DateTime(), + nullable=False, + server_default=sa.text("CURRENT_TIMESTAMP"), + ), + sa.Column( + "updated_at", + sa.DateTime(), + nullable=False, + server_default=sa.text("CURRENT_TIMESTAMP"), + ), + sa.UniqueConstraint( + "account_id", + "platform", + "domain", + name="uq_account_sync_state_domain", + ), + ) + op.create_index( + "ix_account_sync_state_due", + "account_sync_states", + ["platform", "domain", "status", "next_retry_at", "account_id"], + ) + op.create_index( + "ix_account_sync_state_success", + "account_sync_states", + ["platform", "domain", "last_success_at", "account_id"], + ) + elif not _column_exists("account_sync_states", "raw_payload_json"): + op.add_column( + "account_sync_states", + sa.Column("raw_payload_json", sa.JSON(), nullable=True), + ) + + if not _table_exists("account_sync_runs"): + op.create_table( + "account_sync_runs", + sa.Column("id", sa.BigInteger(), primary_key=True, autoincrement=True), + sa.Column( + "account_id", + sa.Integer(), + sa.ForeignKey("accounts.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column( + "platform", + sa.String(length=16), + nullable=False, + server_default="douyu", + ), + sa.Column("domain", sa.String(length=32), nullable=False), + sa.Column("task_id", sa.BigInteger(), nullable=True), + sa.Column("started_at", sa.DateTime(), nullable=False), + sa.Column("finished_at", sa.DateTime(), nullable=True), + sa.Column( + "status", + sa.String(length=16), + nullable=False, + server_default="running", + ), + sa.Column( + "source", + sa.String(length=32), + nullable=False, + server_default="", + ), + sa.Column( + "item_count", + sa.Integer(), + nullable=False, + server_default="0", + ), + sa.Column( + "changed_count", + sa.Integer(), + nullable=False, + server_default="0", + ), + sa.Column( + "duration_ms", + sa.Integer(), + nullable=False, + server_default="0", + ), + sa.Column( + "error_code", + sa.String(length=64), + nullable=False, + server_default="", + ), + sa.Column("error_message", sa.Text(), nullable=False), + sa.Column("official_observed_at", sa.DateTime(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(), + nullable=False, + server_default=sa.text("CURRENT_TIMESTAMP"), + ), + ) + op.create_index( + "ix_account_sync_runs_account_domain_time", + "account_sync_runs", + ["account_id", "domain", "started_at"], + ) + op.create_index( + "ix_account_sync_runs_domain_status_time", + "account_sync_runs", + ["domain", "status", "started_at"], + ) + + if not _table_exists("account_work_items"): + op.create_table( + "account_work_items", + sa.Column("id", sa.BigInteger(), primary_key=True, autoincrement=True), + sa.Column( + "account_id", + sa.Integer(), + sa.ForeignKey("accounts.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column("work_type", sa.String(length=32), nullable=False), + sa.Column( + "priority", + sa.Integer(), + nullable=False, + server_default="100", + ), + sa.Column( + "status", + sa.String(length=16), + nullable=False, + server_default="pending", + ), + sa.Column("due_at", sa.DateTime(), nullable=False), + sa.Column("expires_at", sa.DateTime(), nullable=True), + sa.Column( + "attempt_count", + sa.Integer(), + nullable=False, + server_default="0", + ), + sa.Column( + "max_attempts", + sa.Integer(), + nullable=False, + server_default="3", + ), + sa.Column("dedupe_key", sa.String(length=191), nullable=False), + sa.Column( + "active_dedupe_key", + sa.String(length=191), + sa.Computed( + "CASE WHEN status IN ('pending', 'running') " + "THEN dedupe_key ELSE NULL END", + persisted=False, + ), + nullable=True, + ), + sa.Column( + "reason", + sa.String(length=255), + nullable=False, + server_default="", + ), + sa.Column("payload_json", sa.JSON(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(), + nullable=False, + server_default=sa.text("CURRENT_TIMESTAMP"), + ), + sa.Column("started_at", sa.DateTime(), nullable=True), + sa.Column("finished_at", sa.DateTime(), nullable=True), + sa.Column("last_error", sa.Text(), nullable=False), + sa.Column( + "updated_at", + sa.DateTime(), + nullable=False, + server_default=sa.text("CURRENT_TIMESTAMP"), + ), + ) + op.create_index( + "uq_account_work_active_dedupe", + "account_work_items", + ["active_dedupe_key"], + unique=True, + ) + op.create_index( + "ix_account_work_due", + "account_work_items", + ["status", "work_type", "due_at", "priority", "id"], + ) + op.create_index( + "ix_account_work_account_status", + "account_work_items", + ["account_id", "status", "work_type"], + ) + + if not _table_exists("account_supply_targets"): + op.create_table( + "account_supply_targets", + sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True), + sa.Column( + "platform", + sa.String(length=16), + nullable=False, + server_default="douyu", + ), + sa.Column( + "scope", + sa.String(length=16), + nullable=False, + server_default="elite", + ), + sa.Column( + "commodity_id", + sa.String(length=64), + nullable=False, + server_default="", + ), + sa.Column("value_class", sa.String(length=16), nullable=False), + sa.Column( + "target_covered", + sa.Integer(), + nullable=False, + server_default="0", + ), + sa.Column( + "target_available", + sa.Integer(), + nullable=False, + server_default="0", + ), + sa.Column( + "target_allocatable", + sa.Integer(), + nullable=False, + server_default="0", + ), + sa.Column( + "safety_buffer", + sa.Integer(), + nullable=False, + server_default="0", + ), + sa.Column( + "enabled", + sa.Boolean(), + nullable=False, + server_default="1", + ), + sa.Column( + "updated_by", + sa.Integer(), + sa.ForeignKey("users.id", ondelete="SET NULL"), + nullable=True, + ), + sa.Column( + "created_at", + sa.DateTime(), + nullable=False, + server_default=sa.text("CURRENT_TIMESTAMP"), + ), + sa.Column( + "updated_at", + sa.DateTime(), + nullable=False, + server_default=sa.text("CURRENT_TIMESTAMP"), + ), + sa.UniqueConstraint( + "platform", + "scope", + "commodity_id", + "value_class", + name="uq_account_supply_target", + ), + ) + op.create_index( + "ix_account_supply_target_enabled", + "account_supply_targets", + ["platform", "enabled", "scope"], + ) + + if not _table_exists("login_attempts"): + op.create_table( + "login_attempts", + sa.Column("id", sa.BigInteger(), primary_key=True, autoincrement=True), + sa.Column( + "account_id", + sa.Integer(), + sa.ForeignKey("accounts.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column( + "batch_id", + sa.String(length=64), + nullable=False, + server_default="", + ), + sa.Column("category", sa.String(length=32), nullable=False), + sa.Column( + "trigger_source", + sa.String(length=16), + nullable=False, + server_default="manual", + ), + sa.Column( + "created_by", + sa.Integer(), + sa.ForeignKey("users.id", ondelete="SET NULL"), + nullable=True, + ), + sa.Column( + "started_at", + sa.DateTime(), + nullable=False, + server_default=sa.text("CURRENT_TIMESTAMP"), + ), + sa.Column("finished_at", sa.DateTime(), nullable=True), + sa.Column( + "status", + sa.String(length=16), + nullable=False, + server_default="pending", + ), + sa.Column( + "failure_kind", + sa.String(length=16), + nullable=False, + server_default="", + ), + sa.Column( + "duration_ms", + sa.Integer(), + nullable=False, + server_default="0", + ), + sa.Column( + "proxy_key", + sa.String(length=64), + nullable=False, + server_default="", + ), + sa.Column( + "email_code_requested", + sa.Boolean(), + nullable=False, + server_default="0", + ), + sa.Column("quota_day", sa.Date(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(), + nullable=False, + server_default=sa.text("CURRENT_TIMESTAMP"), + ), + ) + op.create_index( + "ix_login_attempts_account_day", + "login_attempts", + ["account_id", "quota_day", "category"], + ) + op.create_index( + "ix_login_attempts_day_category_status", + "login_attempts", + ["quota_day", "category", "status"], + ) + op.create_index( + "ix_login_attempts_batch", + "login_attempts", + ["batch_id", "account_id"], + ) + + if not _table_exists("login_quota_settings"): + op.create_table( + "login_quota_settings", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column( + "daily_attempt_limit", + sa.Integer(), + nullable=False, + server_default="1500", + ), + sa.Column( + "hourly_attempt_limit", + sa.Integer(), + nullable=False, + server_default="100", + ), + sa.Column( + "per_account_daily_limit", + sa.Integer(), + nullable=False, + server_default="2", + ), + sa.Column( + "onboarding_daily_limit", + sa.Integer(), + nullable=False, + server_default="150", + ), + sa.Column( + "relogin_high_value_daily_limit", + sa.Integer(), + nullable=False, + server_default="300", + ), + sa.Column( + "relogin_ordinary_daily_limit", + sa.Integer(), + nullable=False, + server_default="500", + ), + sa.Column( + "manual_daily_limit", + sa.Integer(), + nullable=False, + server_default="300", + ), + sa.Column( + "login_concurrency", + sa.Integer(), + nullable=False, + server_default="3", + ), + sa.Column( + "failure_pause_ratio", + sa.Integer(), + nullable=False, + server_default="40", + ), + sa.Column( + "enabled", + sa.Boolean(), + nullable=False, + server_default="1", + ), + sa.Column( + "updated_by", + sa.Integer(), + sa.ForeignKey("users.id", ondelete="SET NULL"), + nullable=True, + ), + sa.Column( + "updated_at", + sa.DateTime(), + nullable=False, + server_default=sa.text("CURRENT_TIMESTAMP"), + ), + ) + op.execute( + sa.text("INSERT IGNORE INTO login_quota_settings (id) VALUES (1)") + ) + + if not _table_exists("login_quota_usage"): + op.create_table( + "login_quota_usage", + sa.Column("id", sa.BigInteger(), primary_key=True, autoincrement=True), + sa.Column("quota_day", sa.Date(), nullable=False), + sa.Column("quota_hour", sa.Integer(), nullable=False), + sa.Column("category", sa.String(length=32), nullable=False), + sa.Column( + "attempt_count", + sa.Integer(), + nullable=False, + server_default="0", + ), + sa.Column( + "success_count", + sa.Integer(), + nullable=False, + server_default="0", + ), + sa.Column( + "failure_count", + sa.Integer(), + nullable=False, + server_default="0", + ), + sa.Column( + "updated_at", + sa.DateTime(), + nullable=False, + server_default=sa.text("CURRENT_TIMESTAMP"), + ), + sa.UniqueConstraint( + "quota_day", + "quota_hour", + "category", + name="uq_login_quota_usage_bucket", + ), + ) + op.create_index( + "ix_login_quota_usage_day_category", + "login_quota_usage", + ["quota_day", "category"], + ) + + +def downgrade() -> None: + for table in ( + "login_quota_usage", + "login_quota_settings", + "login_attempts", + "account_supply_targets", + "account_work_items", + "account_sync_runs", + "account_sync_states", + ): + if _table_exists(table): + op.drop_table(table) + + for column in ( + "ck_expiry_source", + "ck_observed_lifetime_seconds", + "ck_invalid_detected_at", + "ck_last_valid_at", + "ck_expected_expires_at", + ): + if _column_exists("accounts", column): + op.drop_column("accounts", column) + if _column_exists("account_commodity_states", "raw_json"): + op.drop_column("account_commodity_states", "raw_json") diff --git a/web/backend/models.py b/web/backend/models.py index 52490ee..21ff470 100644 --- a/web/backend/models.py +++ b/web/backend/models.py @@ -2,7 +2,7 @@ from __future__ import annotations -from datetime import UTC, datetime +from datetime import UTC, date, datetime from sqlalchemy import ( JSON, @@ -10,6 +10,7 @@ from sqlalchemy import ( Boolean, CheckConstraint, Computed, + Date, ForeignKey, Index, Integer, @@ -227,6 +228,23 @@ class Account(Base): ) # valid / invalid / error / 空 = 未检测 ck_check_result: Mapped[dict | None] = mapped_column(JSON, nullable=True) ck_checked_at: Mapped[datetime | None] = mapped_column(UTCDateTime, nullable=True) + # 当前 CK 周期的预测和观测信息。真实签发时间继续使用 cookie_updated_at。 + ck_expected_expires_at: Mapped[datetime | None] = mapped_column( + UTCDateTime, nullable=True + ) + ck_last_valid_at: Mapped[datetime | None] = mapped_column( + UTCDateTime, nullable=True + ) + ck_invalid_detected_at: Mapped[datetime | None] = mapped_column( + UTCDateTime, nullable=True + ) + ck_observed_lifetime_seconds: Mapped[int | None] = mapped_column( + Integer, nullable=True + ) + # observed / predicted / provider_expiry + ck_expiry_source: Mapped[str] = mapped_column( + String(24), nullable=False, default="", server_default="" + ) # ── 登录态:自动重登的退避进度 ───────────────────────────────────────── relogin_attempts: Mapped[int] = mapped_column( @@ -1803,6 +1821,9 @@ class AccountCommodityState(Base): raw_status: Mapped[str] = mapped_column( String(16), nullable=False, default="", server_default="" ) + # 官方商品原始字段。限时窗口、场次、库存、资格字段以后可能增加; + # 归一化状态出问题时必须能回到原始响应,不再只靠猜测字段名。 + raw_json: Mapped[dict | None] = mapped_column(JSON, nullable=True) time_limited: Mapped[bool] = mapped_column( Boolean, nullable=False, default=False, server_default="0" ) @@ -2215,6 +2236,395 @@ class PlatformFulfillmentEvent(Base): ) +class AccountSyncState(Base): + """账号在某个查询域的当前同步状态。""" + + __tablename__ = "account_sync_states" + + id: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True) + account_id: Mapped[int] = mapped_column( + ForeignKey("accounts.id", ondelete="CASCADE"), nullable=False + ) + platform: Mapped[str] = mapped_column( + String(16), nullable=False, default="douyu", server_default="douyu" + ) + domain: Mapped[str] = mapped_column(String(32), nullable=False) + last_attempt_at: Mapped[datetime | None] = mapped_column( + UTCDateTime, nullable=True + ) + last_success_at: Mapped[datetime | None] = mapped_column( + UTCDateTime, nullable=True + ) + valid_until: Mapped[datetime | None] = mapped_column(UTCDateTime, nullable=True) + next_retry_at: Mapped[datetime | None] = mapped_column( + UTCDateTime, nullable=True + ) + status: Mapped[str] = mapped_column( + String(16), nullable=False, default="never", server_default="never" + ) + error_code: Mapped[str] = mapped_column( + String(64), nullable=False, default="", server_default="" + ) + error_message: Mapped[str] = mapped_column(Text, nullable=False, default="") + source: Mapped[str] = mapped_column( + String(32), nullable=False, default="", server_default="" + ) + payload_hash: Mapped[str] = mapped_column( + String(64), nullable=False, default="", server_default="" + ) + # 最近一次官方查询拿到的完整原始响应。商品级原始字段在 + # account_commodity_states.raw_json,这里保留整个响应以支持协议回溯。 + raw_payload_json: Mapped[dict | None] = mapped_column(JSON, nullable=True) + state_version: Mapped[int] = mapped_column( + BigInteger, nullable=False, default=0, server_default="0" + ) + created_at: Mapped[datetime] = mapped_column(UTCDateTime, default=_utcnow) + updated_at: Mapped[datetime] = mapped_column( + UTCDateTime, default=_utcnow, onupdate=_utcnow + ) + + __table_args__ = ( + UniqueConstraint( + "account_id", + "platform", + "domain", + name="uq_account_sync_state_domain", + ), + Index( + "ix_account_sync_state_due", + "platform", + "domain", + "status", + "next_retry_at", + "account_id", + ), + Index( + "ix_account_sync_state_success", + "platform", + "domain", + "last_success_at", + "account_id", + ), + ) + + +class AccountSyncRun(Base): + """一次账号查询同步尝试,只追加。""" + + __tablename__ = "account_sync_runs" + + id: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True) + account_id: Mapped[int] = mapped_column( + ForeignKey("accounts.id", ondelete="CASCADE"), nullable=False + ) + platform: Mapped[str] = mapped_column( + String(16), nullable=False, default="douyu", server_default="douyu" + ) + domain: Mapped[str] = mapped_column(String(32), nullable=False) + task_id: Mapped[int | None] = mapped_column(BigInteger, nullable=True) + started_at: Mapped[datetime] = mapped_column(UTCDateTime, nullable=False) + finished_at: Mapped[datetime | None] = mapped_column(UTCDateTime, nullable=True) + status: Mapped[str] = mapped_column( + String(16), nullable=False, default="running" + ) + source: Mapped[str] = mapped_column( + String(32), nullable=False, default="", server_default="" + ) + item_count: Mapped[int] = mapped_column( + Integer, nullable=False, default=0, server_default="0" + ) + changed_count: Mapped[int] = mapped_column( + Integer, nullable=False, default=0, server_default="0" + ) + duration_ms: Mapped[int] = mapped_column( + Integer, nullable=False, default=0, server_default="0" + ) + error_code: Mapped[str] = mapped_column( + String(64), nullable=False, default="", server_default="" + ) + error_message: Mapped[str] = mapped_column(Text, nullable=False, default="") + official_observed_at: Mapped[datetime | None] = mapped_column( + UTCDateTime, nullable=True + ) + created_at: Mapped[datetime] = mapped_column(UTCDateTime, default=_utcnow) + + __table_args__ = ( + Index( + "ix_account_sync_runs_account_domain_time", + "account_id", + "domain", + "started_at", + ), + Index( + "ix_account_sync_runs_domain_status_time", + "domain", + "status", + "started_at", + ), + ) + + +class AccountWorkItem(Base): + """账号内部维护工作队列,不是用户可见业务任务。""" + + __tablename__ = "account_work_items" + + id: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True) + account_id: Mapped[int] = mapped_column( + ForeignKey("accounts.id", ondelete="CASCADE"), nullable=False + ) + work_type: Mapped[str] = mapped_column(String(32), nullable=False) + priority: Mapped[int] = mapped_column( + Integer, nullable=False, default=100, server_default="100" + ) + status: Mapped[str] = mapped_column( + String(16), nullable=False, default="pending", server_default="pending" + ) + due_at: Mapped[datetime] = mapped_column(UTCDateTime, nullable=False) + expires_at: Mapped[datetime | None] = mapped_column(UTCDateTime, nullable=True) + attempt_count: Mapped[int] = mapped_column( + Integer, nullable=False, default=0, server_default="0" + ) + max_attempts: Mapped[int] = mapped_column( + Integer, nullable=False, default=3, server_default="3" + ) + dedupe_key: Mapped[str] = mapped_column(String(191), nullable=False) + active_dedupe_key: Mapped[str | None] = mapped_column( + String(191), + Computed( + "CASE WHEN status IN ('pending', 'running') THEN dedupe_key ELSE NULL END", + persisted=False, + ), + nullable=True, + ) + reason: Mapped[str] = mapped_column( + String(255), nullable=False, default="", server_default="" + ) + payload_json: Mapped[dict | None] = mapped_column(JSON, nullable=True) + created_at: Mapped[datetime] = mapped_column(UTCDateTime, default=_utcnow) + started_at: Mapped[datetime | None] = mapped_column(UTCDateTime, nullable=True) + finished_at: Mapped[datetime | None] = mapped_column(UTCDateTime, nullable=True) + last_error: Mapped[str] = mapped_column(Text, nullable=False, default="") + updated_at: Mapped[datetime] = mapped_column( + UTCDateTime, default=_utcnow, onupdate=_utcnow + ) + + __table_args__ = ( + Index("uq_account_work_active_dedupe", "active_dedupe_key", unique=True), + Index( + "ix_account_work_due", + "status", + "work_type", + "due_at", + "priority", + "id", + ), + Index( + "ix_account_work_account_status", + "account_id", + "status", + "work_type", + ), + ) + + +class AccountSupplyTarget(Base): + """高价值、普通和商品维度的账号供给目标。""" + + __tablename__ = "account_supply_targets" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + platform: Mapped[str] = mapped_column( + String(16), nullable=False, default="douyu", server_default="douyu" + ) + scope: Mapped[str] = mapped_column( + String(16), nullable=False, default="elite", server_default="elite" + ) + commodity_id: Mapped[str] = mapped_column( + String(64), nullable=False, default="", server_default="" + ) + value_class: Mapped[str] = mapped_column(String(16), nullable=False) + target_covered: Mapped[int] = mapped_column( + Integer, nullable=False, default=0, server_default="0" + ) + target_available: Mapped[int] = mapped_column( + Integer, nullable=False, default=0, server_default="0" + ) + target_allocatable: Mapped[int] = mapped_column( + Integer, nullable=False, default=0, server_default="0" + ) + safety_buffer: Mapped[int] = mapped_column( + Integer, nullable=False, default=0, server_default="0" + ) + enabled: Mapped[bool] = mapped_column( + Boolean, nullable=False, default=True, server_default="1" + ) + updated_by: Mapped[int | None] = mapped_column( + ForeignKey("users.id", ondelete="SET NULL"), nullable=True + ) + created_at: Mapped[datetime] = mapped_column(UTCDateTime, default=_utcnow) + updated_at: Mapped[datetime] = mapped_column( + UTCDateTime, default=_utcnow, onupdate=_utcnow + ) + + __table_args__ = ( + UniqueConstraint( + "platform", + "scope", + "commodity_id", + "value_class", + name="uq_account_supply_target", + ), + Index( + "ix_account_supply_target_enabled", + "platform", + "enabled", + "scope", + ), + ) + + +class LoginAttempt(Base): + """一次完整登录/重登/账号检测尝试,只追加。""" + + __tablename__ = "login_attempts" + + id: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True) + account_id: Mapped[int] = mapped_column( + ForeignKey("accounts.id", ondelete="CASCADE"), nullable=False + ) + batch_id: Mapped[str] = mapped_column( + String(64), nullable=False, default="", server_default="" + ) + category: Mapped[str] = mapped_column(String(32), nullable=False) + trigger_source: Mapped[str] = mapped_column( + String(16), nullable=False, default="manual", server_default="manual" + ) + created_by: Mapped[int | None] = mapped_column( + ForeignKey("users.id", ondelete="SET NULL"), nullable=True + ) + started_at: Mapped[datetime] = mapped_column(UTCDateTime, default=_utcnow) + finished_at: Mapped[datetime | None] = mapped_column(UTCDateTime, nullable=True) + status: Mapped[str] = mapped_column( + String(16), nullable=False, default="pending", server_default="pending" + ) + failure_kind: Mapped[str] = mapped_column( + String(16), nullable=False, default="", server_default="" + ) + duration_ms: Mapped[int] = mapped_column( + Integer, nullable=False, default=0, server_default="0" + ) + proxy_key: Mapped[str] = mapped_column( + String(64), nullable=False, default="", server_default="" + ) + email_code_requested: Mapped[bool] = mapped_column( + Boolean, nullable=False, default=False, server_default="0" + ) + quota_day: Mapped[date] = mapped_column(Date, nullable=False) + created_at: Mapped[datetime] = mapped_column(UTCDateTime, default=_utcnow) + + __table_args__ = ( + Index( + "ix_login_attempts_account_day", + "account_id", + "quota_day", + "category", + ), + Index( + "ix_login_attempts_day_category_status", + "quota_day", + "category", + "status", + ), + Index( + "ix_login_attempts_batch", + "batch_id", + "account_id", + ), + ) + + +class LoginQuotaSetting(Base): + """全局登录预算设置,默认只有 id=1 一行。""" + + __tablename__ = "login_quota_settings" + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + daily_attempt_limit: Mapped[int] = mapped_column( + Integer, nullable=False, default=1500, server_default="1500" + ) + hourly_attempt_limit: Mapped[int] = mapped_column( + Integer, nullable=False, default=100, server_default="100" + ) + per_account_daily_limit: Mapped[int] = mapped_column( + Integer, nullable=False, default=2, server_default="2" + ) + onboarding_daily_limit: Mapped[int] = mapped_column( + Integer, nullable=False, default=150, server_default="150" + ) + relogin_high_value_daily_limit: Mapped[int] = mapped_column( + Integer, nullable=False, default=300, server_default="300" + ) + relogin_ordinary_daily_limit: Mapped[int] = mapped_column( + Integer, nullable=False, default=500, server_default="500" + ) + manual_daily_limit: Mapped[int] = mapped_column( + Integer, nullable=False, default=300, server_default="300" + ) + login_concurrency: Mapped[int] = mapped_column( + Integer, nullable=False, default=3, server_default="3" + ) + failure_pause_ratio: Mapped[int] = mapped_column( + Integer, nullable=False, default=40, server_default="40" + ) + enabled: Mapped[bool] = mapped_column( + Boolean, nullable=False, default=True, server_default="1" + ) + updated_by: Mapped[int | None] = mapped_column( + ForeignKey("users.id", ondelete="SET NULL"), nullable=True + ) + updated_at: Mapped[datetime] = mapped_column( + UTCDateTime, default=_utcnow, onupdate=_utcnow + ) + + +class LoginQuotaUsage(Base): + """按小时和分类累计登录配额用量。""" + + __tablename__ = "login_quota_usage" + + id: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True) + quota_day: Mapped[date] = mapped_column(Date, nullable=False) + quota_hour: Mapped[int] = mapped_column(Integer, nullable=False) + category: Mapped[str] = mapped_column(String(32), nullable=False) + attempt_count: Mapped[int] = mapped_column( + Integer, nullable=False, default=0, server_default="0" + ) + success_count: Mapped[int] = mapped_column( + Integer, nullable=False, default=0, server_default="0" + ) + failure_count: Mapped[int] = mapped_column( + Integer, nullable=False, default=0, server_default="0" + ) + updated_at: Mapped[datetime] = mapped_column( + UTCDateTime, default=_utcnow, onupdate=_utcnow + ) + + __table_args__ = ( + UniqueConstraint( + "quota_day", + "quota_hour", + "category", + name="uq_login_quota_usage_bucket", + ), + Index( + "ix_login_quota_usage_day_category", + "quota_day", + "category", + ), + ) + + class IntegrationOutbox(Base): """跨系统待发送事件;与业务状态在同一数据库事务内提交。""" diff --git a/web/backend/schemas.py b/web/backend/schemas.py index 7dad387..01b4881 100644 --- a/web/backend/schemas.py +++ b/web/backend/schemas.py @@ -364,10 +364,10 @@ class AccountOut(BaseModel): # ---- 登录任务 ---- class LoginBatchRequest(BaseModel): - account_ids: list[int] - max_login_retries: int = 0 # 登录整体重试次数,0=无限重试直到成功(后端兜底 20 次) - max_total_time: float = 300 # 单账号登录总时长上限(秒),0=不限制 - concurrency: int = 3 # 并发数,1-10 + account_ids: list[int] = Field(..., min_length=1, max_length=50) + max_login_retries: int = Field(0, ge=0) + max_total_time: float = Field(300, ge=0) + concurrency: int = Field(3, ge=1, le=10) api_strategy: str = "wgapi" # 接口策略: wgapi(新版)或 iframe(旧版备选) mode: str = Field( "login", pattern="^(login|check)$" @@ -375,7 +375,10 @@ class LoginBatchRequest(BaseModel): class CookieReloginRequest(BaseModel): - ids: list[int] = [] # 需要重新登录的**账号** ID(2026-10-04 CK 解耦前是记录 ID) + ids: list[int] = Field( + default_factory=list, + max_length=50, + ) # 需要重新登录的**账号** ID(2026-10-04 CK 解耦前是记录 ID) class LoginTaskOut(BaseModel): diff --git a/web/backend/services/account_capability_service.py b/web/backend/services/account_capability_service.py index efc4027..c6438d5 100644 --- a/web/backend/services/account_capability_service.py +++ b/web/backend/services/account_capability_service.py @@ -11,7 +11,7 @@ from dataclasses import dataclass from datetime import UTC, datetime, timedelta from typing import Any -from sqlalchemy import case, func +from sqlalchemy import case, func, or_ from sqlalchemy.orm import Session from ..models import Account, AccountCommodityState, AccountEvent @@ -189,6 +189,37 @@ def classify_goods_item( return AVAILABILITY_UNKNOWN, "官方商品列表未返回可判定的账号状态", raw_status +@dataclass(frozen=True) +class GoodsAssessment: + availability: str + blocking_reason: str + raw_status: str + + +def classify_goods_items( + goods_items: list[dict[str, Any]], + *, + now: datetime | None = None, +) -> dict[str, GoodsAssessment]: + """一次性归一完整商品列表,供选号、实时复核和展示共享。""" + assessments: dict[str, GoodsAssessment] = {} + for item in goods_items: + if not isinstance(item, dict): + continue + commodity_id = str( + item.get("commodityId") or item.get("commodity_id") or "" + ).strip() + if not commodity_id: + continue + availability, reason, raw_status = classify_goods_item(item, now=now) + assessments[commodity_id] = GoodsAssessment( + availability=availability, + blocking_reason=reason, + raw_status=raw_status, + ) + return assessments + + def _exchange_stats( db: Session, *, @@ -279,7 +310,7 @@ def sync_account_goods_states( if not goods_items or account.id is None: return 0 moment = observed_at or datetime.now(UTC) - valid_until = moment + timedelta(seconds=capability_ttl_seconds()) + ttl_valid_until = moment + timedelta(seconds=capability_ttl_seconds()) existing_rows = ( db.query(AccountCommodityState) @@ -324,17 +355,18 @@ def sync_account_goods_states( state.availability = availability state.blocking_reason = reason[:255] state.raw_status = raw_status[:16] + state.raw_json = dict(item) state.time_limited = window.time_limited state.available_from = window.start state.available_until = window.end state.schedule_json = window.schedule or None state.observed_at = moment - state.valid_until = valid_until - state.next_check_at = ( - min(valid_until, window.next_boundary) + state.valid_until = ( + min(ttl_valid_until, window.next_boundary) if window.next_boundary is not None - else valid_until + else ttl_valid_until ) + state.next_check_at = state.valid_until state.last_exchange_at = last_exchange_at state.exchange_count_365d = int(exchange_count) state.source = source[:24] @@ -353,9 +385,14 @@ def sync_account_goods_states( continue state.availability = AVAILABILITY_NOT_OFFERED state.blocking_reason = "官方商品列表本次未返回该商品" + state.raw_json = None + state.time_limited = False + state.available_from = None + state.available_until = None + state.schedule_json = None state.observed_at = moment state.valid_until = None - state.next_check_at = valid_until + state.next_check_at = ttl_valid_until state.source = source[:24] state.updated_at = moment changed += 1 @@ -538,6 +575,14 @@ def account_capability_counts( AccountCommodityState.availability == AVAILABILITY_AVAILABLE, AccountCommodityState.valid_until.isnot(None), AccountCommodityState.valid_until > moment, + or_( + AccountCommodityState.available_from.is_(None), + AccountCommodityState.available_from <= moment, + ), + or_( + AccountCommodityState.available_until.is_(None), + AccountCommodityState.available_until >= moment, + ), ) .scalar() or 0 @@ -557,9 +602,11 @@ __all__ = [ "AVAILABILITY_NOT_OPEN", "AVAILABILITY_NO_STOCK", "AVAILABILITY_UNKNOWN", + "GoodsAssessment", "account_capability_counts", "capability_ttl_seconds", "classify_goods_item", + "classify_goods_items", "goods_time_window", "invalidate_account_commodity_states", "mark_commodity_exchanged", diff --git a/web/backend/services/account_lifecycle.py b/web/backend/services/account_lifecycle.py index b7212f5..7a972e5 100644 --- a/web/backend/services/account_lifecycle.py +++ b/web/backend/services/account_lifecycle.py @@ -28,7 +28,7 @@ from __future__ import annotations -from datetime import UTC, datetime +from datetime import UTC, datetime, timedelta from typing import Any from loguru import logger @@ -43,8 +43,12 @@ from .account_events_service import ( EVENT_STATUS_CHANGED, emit, ) +from .account_work_queue import complete_account_work_type, enqueue_account_work from .high_value_service import invalidate_high_value +WORK_LOGIN_RELOGIN = "login_relogin" +DEFAULT_CK_LIFETIME = timedelta(days=7) + # ── 状态取值域(6 个)────────────────────────────────────── # 刻意**不做** `relogin`(退避重试中):它完全等价于 # `invalid + relogin_next_at is not None`,独立成状态就多出一个必须与 @@ -318,6 +322,53 @@ def on_ck_verdict( } account.ck_checked_at = now + if verdict == "valid": + account.ck_last_valid_at = now + account.ck_invalid_detected_at = None + issued_at = _as_utc(account.cookie_updated_at) + if issued_at is not None: + account.ck_expected_expires_at = issued_at + DEFAULT_CK_LIFETIME + account.ck_expiry_source = "predicted" + if account.id is not None: + try: + complete_account_work_type( + db, + account_id=int(account.id), + work_type=WORK_LOGIN_RELOGIN, + now=now, + ) + except Exception as exc: # noqa: BLE001 - 队列是派生索引 + logger.warning( + "完成重登工作项失败 account_id={}: {}", + account.id, + exc, + ) + elif verdict == "invalid": + if previous != "invalid": + account.ck_invalid_detected_at = now + issued_at = _as_utc(account.cookie_updated_at) + if issued_at is not None and issued_at <= now: + lifetime = int((now - issued_at).total_seconds()) + if 0 < lifetime <= int(timedelta(days=90).total_seconds()): + account.ck_observed_lifetime_seconds = lifetime + account.ck_expected_expires_at = None + account.ck_expiry_source = "observed" + if account.id is not None: + try: + enqueue_account_work( + db, + account_id=int(account.id), + work_type=WORK_LOGIN_RELOGIN, + priority=30, + reason="CK 已判定失效,等待受预算控制的重登", + ) + except Exception as exc: # noqa: BLE001 - 生命周期主流程优先 + logger.warning( + "登记自动重登工作项失败 account_id={}: {}", + account.id, + exc, + ) + if verdict == "invalid" and previous != "invalid": emit( db, @@ -357,6 +408,12 @@ def on_login_success( account.relogin_next_at = None account.relogin_failure_kind = "" account.relogin_last_error = "" + complete_account_work_type( + db, + account_id=int(account.id), + work_type=WORK_LOGIN_RELOGIN, + now=occurred_at, + ) sync_status(db, account, occurred_at=occurred_at) @@ -380,6 +437,11 @@ def on_ck_cleared( account.ck_check_status = "" account.ck_check_result = None account.ck_checked_at = None + account.ck_expected_expires_at = None + account.ck_last_valid_at = None + account.ck_invalid_detected_at = None + account.ck_observed_lifetime_seconds = None + account.ck_expiry_source = "" emit( db, @@ -424,3 +486,9 @@ def warn_if_drift(account: Account) -> None: logger.warning( f"账号状态漂移:account={account.id} 落库={account.status} 推导={expected}" ) + + +def _as_utc(value: datetime | None) -> datetime | None: + if value is None: + return None + return value.replace(tzinfo=UTC) if value.tzinfo is None else value.astimezone(UTC) diff --git a/web/backend/services/account_sync_service.py b/web/backend/services/account_sync_service.py new file mode 100644 index 0000000..ac244e2 --- /dev/null +++ b/web/backend/services/account_sync_service.py @@ -0,0 +1,301 @@ +"""账号同步状态、运行记录和官网结果投影。""" + +from __future__ import annotations + +import hashlib +import json +import time +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from typing import Any + +from sqlalchemy.orm import Session + +from ..models import Account, AccountSyncRun, AccountSyncState +from .account_capability_service import ( + capability_ttl_seconds, + sync_account_goods_states, +) +from .douyu_read_gateway import StorehomeSnapshot +from .high_value_service import ( + DEFAULT_HIGH_VALUE_GOODS, + apply_high_value_goods, + extract_high_value_goods, +) + +DOMAIN_STOREHOME = "storehome" +DOMAIN_EXCHANGE_RECORDS = "exchange_records" + + +@dataclass(frozen=True) +class StorehomeSyncResult: + item_count: int + changed_count: int + observed_at: datetime + + +def _payload_hash(raw_payload: dict[str, Any]) -> str: + payload = json.dumps( + raw_payload, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + default=str, + ) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +def get_sync_state( + db: Session, + *, + account_id: int, + domain: str, + platform: str = "douyu", +) -> AccountSyncState | None: + return ( + db.query(AccountSyncState) + .filter( + AccountSyncState.account_id == account_id, + AccountSyncState.platform == platform, + AccountSyncState.domain == domain, + ) + .first() + ) + + +def is_domain_fresh( + db: Session, + *, + account_id: int, + domain: str, + now: datetime | None = None, +) -> bool: + moment = now or datetime.now(UTC) + state = get_sync_state(db, account_id=account_id, domain=domain) + return bool( + state + and state.status == "success" + and state.valid_until + and state.valid_until > moment + ) + + +def _get_or_create_state( + db: Session, + *, + account_id: int, + domain: str, + platform: str = "douyu", +) -> AccountSyncState: + state = get_sync_state( + db, + account_id=account_id, + domain=domain, + platform=platform, + ) + if state is None: + state = AccountSyncState( + account_id=account_id, + platform=platform, + domain=domain, + ) + db.add(state) + db.flush() + return state + + +def mark_domain_dirty( + db: Session, + *, + account_id: int, + domain: str, + reason: str = "", +) -> AccountSyncState: + state = _get_or_create_state(db, account_id=account_id, domain=domain) + state.status = "dirty" + state.valid_until = None + state.next_retry_at = datetime.now(UTC) + state.error_code = "dirty" + state.error_message = reason[:2000] + state.state_version = int(state.state_version or 0) + 1 + db.flush() + return state + + +def start_sync_run( + db: Session, + *, + account_id: int, + domain: str, + source: str, + task_id: int | None = None, +) -> AccountSyncRun: + moment = datetime.now(UTC) + state = _get_or_create_state(db, account_id=account_id, domain=domain) + state.last_attempt_at = moment + state.status = "running" + state.error_code = "" + state.error_message = "" + state.source = source[:32] + run = AccountSyncRun( + account_id=account_id, + platform="douyu", + domain=domain, + task_id=task_id, + started_at=moment, + status="running", + source=source[:32], + ) + db.add(run) + db.flush() + return run + + +def finish_sync_success( + db: Session, + run: AccountSyncRun, + *, + item_count: int, + changed_count: int, + observed_at: datetime, + valid_for_seconds: int | None = None, + raw_payload: dict[str, Any] | None = None, +) -> None: + moment = datetime.now(UTC) + duration_ms = max(0, int((moment - run.started_at).total_seconds() * 1000)) + run.status = "success" + run.finished_at = moment + run.item_count = int(item_count) + run.changed_count = int(changed_count) + run.duration_ms = duration_ms + run.official_observed_at = observed_at + state = _get_or_create_state( + db, + account_id=int(run.account_id), + domain=run.domain, + platform=run.platform, + ) + state.status = "success" + state.last_success_at = moment + state.valid_until = moment + timedelta( + seconds=max( + 1, + int( + valid_for_seconds + if valid_for_seconds is not None + else capability_ttl_seconds() + ), + ) + ) + state.next_retry_at = state.valid_until + state.error_code = "" + state.error_message = "" + if raw_payload is not None: + state.raw_payload_json = raw_payload + state.payload_hash = _payload_hash(raw_payload) + state.state_version = int(state.state_version or 0) + 1 + db.flush() + + +def finish_sync_failure( + db: Session, + run: AccountSyncRun, + *, + error: BaseException, + retry_after_seconds: int = 300, + error_code: str = "sync_failed", + raw_payload: dict[str, Any] | None = None, +) -> None: + moment = datetime.now(UTC) + duration_ms = max(0, int((moment - run.started_at).total_seconds() * 1000)) + message = f"{type(error).__name__}: {error}"[:2000] + run.status = "failed" + run.finished_at = moment + run.duration_ms = duration_ms + run.error_code = error_code[:64] + run.error_message = message + state = _get_or_create_state( + db, + account_id=int(run.account_id), + domain=run.domain, + platform=run.platform, + ) + state.status = "failed" + state.next_retry_at = moment + timedelta(seconds=max(1, retry_after_seconds)) + state.error_code = error_code[:64] + state.error_message = message + if raw_payload is not None: + state.raw_payload_json = raw_payload + state.payload_hash = _payload_hash(raw_payload) + state.state_version = int(state.state_version or 0) + 1 + db.flush() + + +def sync_storehome_snapshot( + db: Session, + account: Account, + snapshot: StorehomeSnapshot, + *, + source: str, + task_id: int | None = None, +) -> StorehomeSyncResult: + """把一次完整 storehome 快照投影成账号全部商品状态。""" + start = time.monotonic() + run = start_sync_run( + db, + account_id=int(account.id), + domain=DOMAIN_STOREHOME, + source=source, + task_id=task_id, + ) + try: + items = list(snapshot.items) + changed = sync_account_goods_states( + db, + account, + items, + scope="elite", + observed_at=snapshot.observed_at, + source="storehome_sync", + ) + high_value = extract_high_value_goods( + items, + DEFAULT_HIGH_VALUE_GOODS, + [], + ) + apply_high_value_goods(account, high_value, checked_at=snapshot.observed_at) + account.updated_at = snapshot.observed_at + finish_sync_success( + db, + run, + item_count=len(items), + changed_count=changed, + observed_at=snapshot.observed_at, + raw_payload=snapshot.raw, + ) + run.duration_ms = max( + int(run.duration_ms or 0), + int((time.monotonic() - start) * 1000), + ) + return StorehomeSyncResult( + item_count=len(items), + changed_count=changed, + observed_at=snapshot.observed_at, + ) + except Exception as exc: + finish_sync_failure(db, run, error=exc, raw_payload=snapshot.raw) + raise + + +__all__ = [ + "DOMAIN_EXCHANGE_RECORDS", + "DOMAIN_STOREHOME", + "StorehomeSyncResult", + "finish_sync_failure", + "finish_sync_success", + "get_sync_state", + "is_domain_fresh", + "mark_domain_dirty", + "start_sync_run", + "sync_storehome_snapshot", +] diff --git a/web/backend/services/account_value.py b/web/backend/services/account_value.py index e801e0f..479e5b6 100644 --- a/web/backend/services/account_value.py +++ b/web/backend/services/account_value.py @@ -157,6 +157,24 @@ def is_high_value_expr(*, now: datetime | None = None): return value_tier_expr(now=now) <= HIGH_VALUE_TIER_MAX +def is_high_value_account( + account: Account, + *, + now: datetime | None = None, +) -> bool: + """Python 侧价值判定,供登录分类和库存队列复用 SQL 口径。""" + moment = now or datetime.now(UTC) + fresh = bool( + account.high_value_checked_at + and account.high_value_checked_at >= high_value_fresh_cutoff(now=moment) + ) + available = fresh and ( + account.high_value_bairimeng_status == HIGH_VALUE_STATUS_AVAILABLE + or account.high_value_mengtu_status == HIGH_VALUE_STATUS_AVAILABLE + ) + return available or int(account.points or 0) >= HIGH_VALUE_POINTS + + def suit_sort_rank_expr(*, now: datetime | None = None): """列表「高价值套装」列的排序,和格子上看到的一致。数字越小越值钱。 diff --git a/web/backend/services/account_work_queue.py b/web/backend/services/account_work_queue.py new file mode 100644 index 0000000..0363837 --- /dev/null +++ b/web/backend/services/account_work_queue.py @@ -0,0 +1,162 @@ +"""账号内部维护工作队列。""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta +from typing import Any + +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Session + +from ..models import AccountWorkItem + + +def enqueue_account_work( + db: Session, + *, + account_id: int, + work_type: str, + priority: int = 100, + due_at: datetime | None = None, + max_attempts: int = 3, + reason: str = "", + payload: dict[str, Any] | None = None, +) -> AccountWorkItem: + dedupe_key = f"{work_type}:{account_id}" + existing = ( + db.query(AccountWorkItem) + .filter( + AccountWorkItem.account_id == account_id, + AccountWorkItem.work_type == work_type, + AccountWorkItem.status.in_(("pending", "running")), + ) + .first() + ) + if existing is not None: + existing.priority = min(existing.priority, priority) + if due_at is not None and due_at < existing.due_at: + existing.due_at = due_at + db.flush() + return existing + item = AccountWorkItem( + account_id=account_id, + work_type=work_type[:32], + priority=int(priority), + due_at=due_at or datetime.now(UTC), + max_attempts=max(1, int(max_attempts)), + dedupe_key=dedupe_key[:191], + reason=reason[:255], + payload_json=payload, + ) + try: + with db.begin_nested(): + db.add(item) + db.flush() + return item + except IntegrityError: + return ( + db.query(AccountWorkItem) + .filter(AccountWorkItem.active_dedupe_key == dedupe_key[:191]) + .one() + ) + + +def claim_account_work( + db: Session, + *, + work_types: tuple[str, ...], + limit: int = 10, + now: datetime | None = None, +) -> list[AccountWorkItem]: + moment = now or datetime.now(UTC) + rows = ( + db.query(AccountWorkItem) + .filter( + AccountWorkItem.work_type.in_(work_types), + AccountWorkItem.status == "pending", + AccountWorkItem.due_at <= moment, + ) + .order_by( + AccountWorkItem.priority.asc(), + AccountWorkItem.due_at.asc(), + AccountWorkItem.id.asc(), + ) + .with_for_update(skip_locked=True) + .limit(max(1, int(limit))) + .all() + ) + for row in rows: + row.status = "running" + row.started_at = moment + row.attempt_count = int(row.attempt_count or 0) + 1 + row.updated_at = moment + if rows: + db.flush() + return rows + + +def complete_account_work( + db: Session, + item: AccountWorkItem, + *, + now: datetime | None = None, +) -> None: + moment = now or datetime.now(UTC) + item.status = "success" + item.finished_at = moment + item.last_error = "" + item.updated_at = moment + db.flush() + + +def complete_account_work_type( + db: Session, + *, + account_id: int, + work_type: str, + now: datetime | None = None, +) -> int: + """完成某个账号指定类型的待办;成功后旧待办不再阻塞后续工作。""" + moment = now or datetime.now(UTC) + items = ( + db.query(AccountWorkItem) + .filter( + AccountWorkItem.account_id == account_id, + AccountWorkItem.work_type == work_type, + AccountWorkItem.status.in_(("pending", "running")), + ) + .all() + ) + for item in items: + complete_account_work(db, item, now=moment) + return len(items) + + +def fail_account_work( + db: Session, + item: AccountWorkItem, + *, + error: BaseException | str, + retry_after_seconds: int = 300, + now: datetime | None = None, +) -> None: + moment = now or datetime.now(UTC) + message = str(error)[:2000] + item.last_error = message + item.updated_at = moment + if int(item.attempt_count or 0) >= int(item.max_attempts or 1): + item.status = "failed" + item.finished_at = moment + else: + item.status = "pending" + item.due_at = moment + timedelta(seconds=max(1, retry_after_seconds)) + db.flush() + + +__all__ = [ + "claim_account_work", + "complete_account_work", + "complete_account_work_type", + "enqueue_account_work", + "fail_account_work", +] diff --git a/web/backend/services/automation/jobs/ck_health.py b/web/backend/services/automation/jobs/ck_health.py index 6d5bdd2..0461701 100644 --- a/web/backend/services/automation/jobs/ck_health.py +++ b/web/backend/services/automation/jobs/ck_health.py @@ -68,6 +68,9 @@ def _candidates_query(db: Session, job: AutomationJob, now: datetime): db.query(Account) .filter(cookie_operable_filter()) .filter(Account.auto_managed.is_(True)) + # invalid 已进入自动重登队列,manual 已停止自动处理;普通 CK 探针 + # 再扫它们只会重复消耗官方请求,并掩盖真正需要发现的新失效。 + .filter(Account.ck_check_status.notin_(("invalid", "manual"))) # 生命周期状态机:停用/归档的账号不参与任何自动化(见 account_lifecycle)。 .filter(Account.status.notin_(MANUAL_STATES)) .filter( diff --git a/web/backend/services/automation/jobs/ck_relogin.py b/web/backend/services/automation/jobs/ck_relogin.py index 60e2773..a69a186 100644 --- a/web/backend/services/automation/jobs/ck_relogin.py +++ b/web/backend/services/automation/jobs/ck_relogin.py @@ -255,6 +255,7 @@ def run_ck_relogin(db: Session, job: AutomationJob) -> dict[str, Any]: loop=None, concurrency=max(min(int(job.concurrency or 3), 8), 1), mode="relogin", + trigger_source="automation", ) batch_id = runner.batch_id try: diff --git a/web/backend/services/douyu_account_probe.py b/web/backend/services/douyu_account_probe.py index 12b44e4..f454b10 100644 --- a/web/backend/services/douyu_account_probe.py +++ b/web/backend/services/douyu_account_probe.py @@ -733,6 +733,8 @@ def probe_account( points: int | None = None points_response: dict[str, Any] = {} limited_goods: list[dict[str, Any]] = [] + storehome_goods: list[dict[str, Any]] = [] + storehome_raw: dict[str, Any] = {} if with_eligibility: uid = str(account.uid or "").strip() if ctn: @@ -754,6 +756,22 @@ def probe_account( for item in limited_result.get("limited_goods") or [] if isinstance(item, dict) ] + raw_storehome = ( + limited_result.get("raw") + if isinstance(limited_result.get("raw"), dict) + else {} + ) + raw_data = ( + raw_storehome.get("data") + if isinstance(raw_storehome.get("data"), dict) + else {} + ) + raw_list = raw_data.get("list") + storehome_goods = [ + dict(item) + for item in (raw_list if isinstance(raw_list, list) else []) + if isinstance(item, dict) + ] except Exception as exc: # noqa: BLE001 errors["limited_goods"] = f"{type(exc).__name__}: {_text(exc)}" @@ -839,6 +857,8 @@ def probe_account( "error": errors.get("elite", ""), }, "eligibility": { + "manual_id": manual_id, + "rid": str(config_info.get("rid") or ""), "points": points, "points_response": points_response, "points_expire_at": ( @@ -854,6 +874,8 @@ def probe_account( ).strip() } ), + "storehome_goods": storehome_goods, + "storehome_raw": storehome_raw, "error": errors.get("points", ""), }, "roles": roles, @@ -979,6 +1001,39 @@ def persist_probe_result( if gold.get("status") == CAPABILITY_OK and _to_int(gold.get("balance")) is not None: account.gold_balance = _to_int(gold.get("balance")) + eligibility = probe.get("eligibility") or {} + storehome_goods = [ + dict(item) + for item in eligibility.get("storehome_goods") or [] + if isinstance(item, dict) + ] + if account.id is not None and storehome_goods: + from .account_sync_service import sync_storehome_snapshot + from .douyu_read_gateway import StorehomeSnapshot + + storehome_raw = ( + eligibility.get("storehome_raw") + if isinstance(eligibility.get("storehome_raw"), dict) + else {} + ) + sync_storehome_snapshot( + db, + account, + StorehomeSnapshot( + manual_id=str(eligibility.get("manual_id") or ""), + rid=str(eligibility.get("rid") or ""), + observed_at=now, + items=tuple(storehome_goods), + limited_ids=frozenset( + str(item) + for item in eligibility.get("limited_ids") or [] + if str(item) + ), + raw=storehome_raw, + ), + source=source[:32], + ) + account.updated_at = now if record_event and account.id is not None: emit( diff --git a/web/backend/services/douyu_account_state.py b/web/backend/services/douyu_account_state.py index cd8e99e..f6601fd 100644 --- a/web/backend/services/douyu_account_state.py +++ b/web/backend/services/douyu_account_state.py @@ -8,7 +8,7 @@ from __future__ import annotations from collections.abc import Callable -from dataclasses import dataclass +from dataclasses import dataclass, field from datetime import UTC, datetime from typing import Any @@ -18,6 +18,10 @@ from core.douyu import DouyuActivityClient from ..models import Account from .account_binding import is_role_change_cooling +from .account_capability_service import ( + AVAILABILITY_AVAILABLE, + classify_goods_items, +) from .cookie_check_service import check_douyu_cookie from .douyu_account_probe import config_mapping, persist_probe_result, probe_account @@ -35,6 +39,8 @@ class DouyuLiveAccountState: role_change_available_at: datetime | None error: str = "" role_change_blocked: bool = False + commodity_availability: dict[str, str] = field(default_factory=dict) + commodity_blocking_reasons: dict[str, str] = field(default_factory=dict) def blocking_reasons( self, @@ -48,6 +54,15 @@ class DouyuLiveAccountState: reasons.append(self.error) if commodity_id in self.limited_ids: reasons.append("该商品已达限兑上限") + elif self.commodity_availability: + availability = self.commodity_availability.get(commodity_id) + if availability is None: + reasons.append("官方商品列表本次未返回该商品,可能已下架") + elif availability != AVAILABILITY_AVAILABLE: + reasons.append( + self.commodity_blocking_reasons.get(commodity_id) + or f"商品当前不可兑换({availability})" + ) if self.role_change_blocked: available_text = ( self.role_change_available_at.isoformat() @@ -150,6 +165,23 @@ def refresh_douyu_account_state( limited_ids = frozenset( str(item) for item in eligibility.get("limited_ids") or [] if str(item) ) + assessments = classify_goods_items( + [ + item + for item in eligibility.get("storehome_goods") or [] + if isinstance(item, dict) + ], + now=now, + ) + commodity_availability = { + commodity_id: assessment.availability + for commodity_id, assessment in assessments.items() + } + commodity_blocking_reasons = { + commodity_id: assessment.blocking_reason + for commodity_id, assessment in assessments.items() + if assessment.blocking_reason + } return DouyuLiveAccountState( account_id=int(account.id), account_name=account.nickname @@ -167,6 +199,8 @@ def refresh_douyu_account_state( role_change_available_at=account.role_change_available_at, error=";".join(errors), role_change_blocked=is_role_change_cooling(bind_info), + commodity_availability=commodity_availability, + commodity_blocking_reasons=commodity_blocking_reasons, ) diff --git a/web/backend/services/douyu_activity_query.py b/web/backend/services/douyu_activity_query.py index ec5ef8d..df89cc5 100644 --- a/web/backend/services/douyu_activity_query.py +++ b/web/backend/services/douyu_activity_query.py @@ -108,10 +108,23 @@ def query_activity_goods( manual_id: str, rid: str, ) -> list[dict[str, Any]]: - result = client.list_goods(manual_id=manual_id, rid=rid) + result = query_activity_goods_result( + client, + manual_id=manual_id, + rid=rid, + ) return [dict(item) for item in result.get("goods") or [] if isinstance(item, dict)] +def query_activity_goods_result( + client: DouyuActivityClient, + *, + manual_id: str, + rid: str, +) -> dict[str, Any]: + return client.list_goods(manual_id=manual_id, rid=rid) + + def query_esports_goods_result( client: DouyuActivityClient, *, @@ -232,6 +245,7 @@ def query_exchange_orders( __all__ = [ "activity_client", "query_activity_goods", + "query_activity_goods_result", "query_bind_info", "query_csrf", "query_elite_user_info", diff --git a/web/backend/services/douyu_collab_service.py b/web/backend/services/douyu_collab_service.py index 48e48e4..a0d6667 100644 --- a/web/backend/services/douyu_collab_service.py +++ b/web/backend/services/douyu_collab_service.py @@ -48,6 +48,10 @@ from .account_binding import ( can_change_role, role_change_available_at, ) +from .account_capability_service import ( + AVAILABILITY_AVAILABLE, + classify_goods_items, +) from .account_events_service import ( CATEGORY_OPS, EVENT_COLLAB_SYNCED, @@ -793,6 +797,20 @@ def recommend_accounts( } if cid and cid in limited_ids: return + if cid: + assessments = classify_goods_items( + [ + item + for item in eligibility.get("storehome_goods") or [] + if isinstance(item, dict) + ] + ) + assessment = assessments.get(str(cid)) + if assessments and ( + assessment is None + or assessment.availability != AVAILABILITY_AVAILABLE + ): + return elite = probe.get("elite") or {} live_points = _to_int(eligibility.get("points")) diff --git a/web/backend/services/douyu_read_gateway.py b/web/backend/services/douyu_read_gateway.py new file mode 100644 index 0000000..22bcb36 --- /dev/null +++ b/web/backend/services/douyu_read_gateway.py @@ -0,0 +1,102 @@ +"""斗鱼只读查询网关。 + +所有账号同步都从这里获取结构化快照,避免页面、Runner 和自动化各自解释原始响应。 +""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import UTC, datetime +from typing import Any + +from core.douyu import DouyuActivityClient + +from .douyu_activity_query import query_limited_goods_result + + +@dataclass(frozen=True) +class StorehomeSnapshot: + """单账号完整商城资格快照。""" + + manual_id: str + rid: str + observed_at: datetime + items: tuple[dict[str, Any], ...] + limited_ids: frozenset[str] + raw: dict[str, Any] + + @property + def item_count(self) -> int: + return len(self.items) + + +def extract_storehome_goods(raw: Any) -> list[dict[str, Any]]: + """从官方 ``storehome`` 响应中提取完整商品列表。""" + if not isinstance(raw, dict): + return [] + data = raw.get("data") + if not isinstance(data, dict): + return [] + rows = data.get("list") + if not isinstance(rows, list): + return [] + return [dict(item) for item in rows if isinstance(item, dict)] + + +def storehome_snapshot_from_result( + result: dict[str, Any], + *, + manual_id: str, + rid: str, + observed_at: datetime | None = None, +) -> StorehomeSnapshot: + """把已经拿到的官方响应归一成可落库的完整快照。""" + raw = result.get("raw") if isinstance(result.get("raw"), dict) else {} + if raw and raw.get("error") not in (None, 0, "0"): + raise RuntimeError(str(raw.get("msg") or "查询商城商品列表失败")) + items = extract_storehome_goods(raw) + if not items and isinstance(result.get("goods"), list): + items = [ + dict(item) for item in result["goods"] if isinstance(item, dict) + ] + limited_ids = frozenset( + str(item.get("commodityId") or item.get("commodity_id") or "") + for item in items + if str(item.get("status") or "") == "1" + and str(item.get("commodityId") or item.get("commodity_id") or "").strip() + ) + return StorehomeSnapshot( + manual_id=str(manual_id), + rid=str(rid or ""), + observed_at=observed_at or datetime.now(UTC), + items=tuple(items), + limited_ids=limited_ids, + raw=raw, + ) + + +def fetch_storehome_snapshot( + client: DouyuActivityClient, + *, + manual_id: str, + rid: str, +) -> StorehomeSnapshot: + """一次请求返回当前账号全部商品资格。""" + result = query_limited_goods_result( + client, + manual_id=manual_id, + rid=str(rid or ""), + ) + return storehome_snapshot_from_result( + result, + manual_id=manual_id, + rid=rid, + ) + + +__all__ = [ + "StorehomeSnapshot", + "extract_storehome_goods", + "fetch_storehome_snapshot", + "storehome_snapshot_from_result", +] diff --git a/web/backend/services/douyu_runner_goods.py b/web/backend/services/douyu_runner_goods.py index 5e8487f..0aaac20 100644 --- a/web/backend/services/douyu_runner_goods.py +++ b/web/backend/services/douyu_runner_goods.py @@ -12,15 +12,26 @@ from sqlalchemy.orm import Session from core.douyu import DouyuActivityClient, DouyuActivityError from ..models import Account, DouyuEsportsGoodsSnapshot, DouyuGoodsSnapshot, DouyuTask +from .account_sync_service import ( + DOMAIN_EXCHANGE_RECORDS, + finish_sync_failure, + finish_sync_success, + start_sync_run, + sync_storehome_snapshot, +) from .business_events import record_exchange from .douyu_activity_query import ( query_activity_goods, + query_activity_goods_result, query_csrf, query_elite_user_info, query_esports_goods_result, query_exchange_records_result, query_limited_goods_result, ) +from .douyu_goods_service import upsert_goods_snapshot +from .douyu_read_gateway import storehome_snapshot_from_result +from .douyu_runner_overview import record_official_exchange_records from .high_value_service import ( DEFAULT_HIGH_VALUE_GOODS, HIGH_VALUE_STATUS_OWNED, @@ -78,12 +89,29 @@ class GoodsMixin: self, db: Session, task: DouyuTask, account: Account, cookie: str, config: dict ): client = self._client(cookie) - goods = query_activity_goods( + result = query_activity_goods_result( client, manual_id=str(config["manual_id"]), rid=str(config["rid"]), ) - self._upsert_goods(db, goods) + goods = [ + dict(item) + for item in result.get("goods") or [] + if isinstance(item, dict) + ] + snapshot = storehome_snapshot_from_result( + result, + manual_id=str(config["manual_id"]), + rid=str(config["rid"]), + ) + upsert_goods_snapshot(db, goods, commit=False) + sync_storehome_snapshot( + db, + account, + snapshot, + source="douyu_task.refresh_goods", + task_id=task.id, + ) high_value_goods = extract_high_value_goods(goods, DEFAULT_HIGH_VALUE_GOODS, []) apply_high_value_goods(account, high_value_goods) account.updated_at = datetime.now(UTC) @@ -723,7 +751,19 @@ class GoodsMixin: client, manual_id=str(config["manual_id"]), rid=str(config["rid"]) ) limited = result["limited_goods"] - goods = (result.get("raw") or {}).get("data", {}).get("list") or [] + snapshot = storehome_snapshot_from_result( + result, + manual_id=str(config["manual_id"]), + rid=str(config["rid"]), + ) + goods = list(snapshot.items) + sync_storehome_snapshot( + db, + account, + snapshot, + source="douyu_task.query_limited_goods", + task_id=task.id, + ) high_value_goods = extract_high_value_goods(goods, DEFAULT_HIGH_VALUE_GOODS, []) apply_high_value_goods(account, high_value_goods) names = [ @@ -760,6 +800,45 @@ class GoodsMixin: offset=0, ) records = result.get("records") or [] + observed_at = datetime.now(UTC) + sync_run = start_sync_run( + db, + account_id=int(account.id), + domain=DOMAIN_EXCHANGE_RECORDS, + source="douyu_task.query_exchange_records", + task_id=task.id, + ) + try: + added = record_official_exchange_records( + db, + account, + [("elite", record) for record in records if isinstance(record, dict)], + ) + finish_sync_success( + db, + sync_run, + item_count=len(records), + changed_count=added, + observed_at=observed_at, + raw_payload=( + result.get("raw") + if isinstance(result.get("raw"), dict) + else None + ), + ) + except Exception as exc: + finish_sync_failure( + db, + sync_run, + error=exc, + raw_payload=( + result.get("raw") + if isinstance(result.get("raw"), dict) + else None + ), + ) + raise + result["exchange_events_added"] = added account.updated_at = datetime.now(UTC) self._mark_task( db, diff --git a/web/backend/services/douyu_runner_overview.py b/web/backend/services/douyu_runner_overview.py index 13962c9..27a14e5 100644 --- a/web/backend/services/douyu_runner_overview.py +++ b/web/backend/services/douyu_runner_overview.py @@ -33,7 +33,7 @@ from sqlalchemy.orm import Session from core.douyu import DouyuActivityClient from ..models import Account, DouyuTask -from .account_capability_service import sync_account_goods_states +from .account_sync_service import sync_storehome_snapshot from .business_events import record_official_exchange from .douyu_account_probe import ( ROLE_BOUND, @@ -48,13 +48,14 @@ from .douyu_account_probe import ( resolve_peace_role, ) from .douyu_activity_query import ( - query_activity_goods, + query_activity_goods_result, query_csrf, query_esports_bind_info, query_exchange_records, query_xpd_bind_info, query_xpd_embed, ) +from .douyu_read_gateway import StorehomeSnapshot, storehome_snapshot_from_result from .douyu_runner_donate import _parse_expire_at, _points_expire_at from .high_value_service import ( DEFAULT_HIGH_VALUE_GOODS, @@ -84,7 +85,7 @@ def _group_by_scope( return grouped -def _record_official_exchanges( +def record_official_exchange_records( db: Session, account: Account, pairs: list[tuple[str, dict[str, Any]]] ) -> int: """把官方兑换记录补成账号事件,返回**新写入**的条数。 @@ -293,6 +294,7 @@ class OverviewMixin: all_records: list[tuple[str, dict[str, Any]]] = [] pending_goods: list[dict[str, Any]] = [] goods_query_ok = False + storehome_snapshot: StorehomeSnapshot | None = None # ---- 鱼翅 + 钱包兑换余额 ---- try: @@ -372,7 +374,18 @@ class OverviewMixin: goods_items: list[dict[str, Any]] = [] try: # 🎯 拉取完整商城商品列表(含每个商品准确的 status 状态:1=已达兑换上限,0=可兑换) - goods_items = query_activity_goods(client, manual_id=manual_id, rid=rid) + goods_result = query_activity_goods_result( + client, + manual_id=manual_id, + rid=rid, + ) + storehome_snapshot = storehome_snapshot_from_result( + goods_result, + manual_id=manual_id, + rid=rid, + observed_at=queried_at, + ) + goods_items = list(storehome_snapshot.items) overview["limited_goods_total"] = len(goods_items) goods_query_ok = True except Exception as exc: # noqa: BLE001 @@ -496,7 +509,7 @@ class OverviewMixin: # 兑换记录开始前结束前一步可能留下的角色快照事务。 db.commit() # 把官方记录**补成账号事件**(带真实兑换时间、按作用域去重)。 - overview["exchange_events_added"] = _record_official_exchanges( + overview["exchange_events_added"] = record_official_exchange_records( db, account, all_records ) overview["exchange_records"] = { @@ -508,14 +521,15 @@ class OverviewMixin: # 只有完整商城列表查询成功时才更新,避免把网络失败写成“不可兑换”。 if goods_query_ok: try: - overview["commodity_states_synced"] = sync_account_goods_states( + if storehome_snapshot is None: + raise RuntimeError("完整商品响应未形成快照") + overview["commodity_states_synced"] = sync_storehome_snapshot( db, account, - pending_goods, - scope="elite", - observed_at=queried_at, - source="overview", - ) + storehome_snapshot, + source="douyu_task.account_overview", + task_id=task.id, + ).changed_count except Exception as exc: # noqa: BLE001 errors["commodity_states"] = _err(exc) diff --git a/web/backend/services/login_quota_service.py b/web/backend/services/login_quota_service.py new file mode 100644 index 0000000..a490278 --- /dev/null +++ b/web/backend/services/login_quota_service.py @@ -0,0 +1,341 @@ +"""登录尝试流水和数据库级登录预算。""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import UTC, date, datetime +from zoneinfo import ZoneInfo + +from sqlalchemy.orm import Session + +from ..models import ( + Account, + LoginAttempt, + LoginQuotaSetting, + LoginQuotaUsage, + LoginTask, +) +from .account_value import is_high_value_account +from .douyu_service import LOGIN_ATTEMPT_IN_FLIGHT_STATUSES + +CATEGORY_ALL = "all" +CATEGORY_MANUAL = "manual_login" +CATEGORY_ONBOARDING = "onboarding" +CATEGORY_RELOGIN_HIGH = "relogin_high_value" +CATEGORY_RELOGIN_ORDINARY = "relogin_ordinary" +CATEGORY_ACCOUNT_CHECK = "account_check" + +BUSINESS_TZ = ZoneInfo("Asia/Shanghai") + + +@dataclass(frozen=True) +class QuotaDecision: + allowed: bool + reason: str = "" + attempt: LoginAttempt | None = None + + +def _bucket(now: datetime | None = None) -> tuple[date, int]: + moment = now or datetime.now(UTC) + aware = moment if moment.tzinfo else moment.replace(tzinfo=UTC) + local = aware.astimezone(BUSINESS_TZ) + return local.date(), local.hour + + +def classify_login_category( + account: Account, + *, + mode: str, + trigger_source: str = "manual", + now: datetime | None = None, +) -> str: + if mode == "check": + return CATEGORY_ACCOUNT_CHECK + if mode == "relogin": + return ( + CATEGORY_RELOGIN_HIGH + if is_high_value_account(account, now=now) + else CATEGORY_RELOGIN_ORDINARY + ) + if ( + trigger_source == "automation" + or account.status == "imported" + or not account.cookie + or account.cookie_updated_at is None + ): + return CATEGORY_ONBOARDING + return CATEGORY_MANUAL + + +def _setting(db: Session) -> LoginQuotaSetting: + row = ( + db.query(LoginQuotaSetting) + .filter(LoginQuotaSetting.id == 1) + .with_for_update() + .first() + ) + if row is None: + row = LoginQuotaSetting(id=1) + db.add(row) + db.flush() + return row + + +def login_concurrency_limit(db: Session) -> int: + """读取数据库配置的登录并发上限;未初始化时使用保守默认值。""" + setting = db.query(LoginQuotaSetting).filter(LoginQuotaSetting.id == 1).first() + if setting is None or not setting.enabled: + return 3 + return max(1, min(int(setting.login_concurrency or 3), 10)) + + +def _usage_row( + db: Session, + *, + quota_day: date, + quota_hour: int, + category: str, +) -> LoginQuotaUsage: + row = ( + db.query(LoginQuotaUsage) + .filter( + LoginQuotaUsage.quota_day == quota_day, + LoginQuotaUsage.quota_hour == quota_hour, + LoginQuotaUsage.category == category, + ) + .first() + ) + if row is None: + row = LoginQuotaUsage( + quota_day=quota_day, + quota_hour=quota_hour, + category=category, + ) + db.add(row) + db.flush() + return row + + +def _daily_total(db: Session, quota_day: date) -> int: + rows = ( + db.query(LoginQuotaUsage.attempt_count) + .filter( + LoginQuotaUsage.quota_day == quota_day, + LoginQuotaUsage.category == CATEGORY_ALL, + ) + .all() + ) + return sum(int(value or 0) for (value,) in rows) + + +def _hourly_total(db: Session, quota_day: date, quota_hour: int) -> int: + value = ( + db.query(LoginQuotaUsage.attempt_count) + .filter( + LoginQuotaUsage.quota_day == quota_day, + LoginQuotaUsage.quota_hour == quota_hour, + LoginQuotaUsage.category == CATEGORY_ALL, + ) + .scalar() + ) + return int(value or 0) + + +def _category_daily_total(db: Session, quota_day: date, category: str) -> int: + rows = ( + db.query(LoginQuotaUsage.attempt_count) + .filter( + LoginQuotaUsage.quota_day == quota_day, + LoginQuotaUsage.category == category, + ) + .all() + ) + return sum(int(value or 0) for (value,) in rows) + + +def _category_limit(setting: LoginQuotaSetting, category: str) -> int: + return { + CATEGORY_ONBOARDING: int(setting.onboarding_daily_limit), + CATEGORY_RELOGIN_HIGH: int(setting.relogin_high_value_daily_limit), + CATEGORY_RELOGIN_ORDINARY: int(setting.relogin_ordinary_daily_limit), + CATEGORY_MANUAL: int(setting.manual_daily_limit), + CATEGORY_ACCOUNT_CHECK: int(setting.manual_daily_limit), + }.get(category, int(setting.daily_attempt_limit)) + + +def reserve_login_attempt( + db: Session, + *, + account: Account, + mode: str, + trigger_source: str, + batch_id: str, + created_by: int | None, + now: datetime | None = None, +) -> QuotaDecision: + moment = now or datetime.now(UTC) + quota_day, quota_hour = _bucket(moment) + setting = _setting(db) + if not setting.enabled: + category = classify_login_category( + account, + mode=mode, + trigger_source=trigger_source, + now=moment, + ) + attempt = LoginAttempt( + account_id=int(account.id), + batch_id=batch_id[:64], + category=category, + trigger_source=trigger_source[:16], + created_by=created_by, + started_at=moment, + status="pending", + quota_day=quota_day, + ) + db.add(attempt) + db.flush() + return QuotaDecision(True, attempt=attempt) + + in_flight = ( + db.query(LoginTask.id) + .filter( + LoginTask.account_id == account.id, + LoginTask.attempt_status.in_(LOGIN_ATTEMPT_IN_FLIGHT_STATUSES), + ) + .first() + ) + in_flight_attempt = ( + db.query(LoginAttempt.id) + .filter( + LoginAttempt.account_id == account.id, + LoginAttempt.status.in_(("pending", "running")), + ) + .first() + ) + if in_flight is not None or in_flight_attempt is not None: + return QuotaDecision(False, reason="该账号已有登录任务在执行") + + account_day_count = ( + db.query(LoginAttempt.id) + .filter( + LoginAttempt.account_id == account.id, + LoginAttempt.quota_day == quota_day, + ) + .count() + ) + if account_day_count >= int(setting.per_account_daily_limit): + return QuotaDecision(False, reason="该账号已达到今日登录尝试上限") + + if _daily_total(db, quota_day) >= int(setting.daily_attempt_limit): + return QuotaDecision(False, reason="全局每日登录尝试预算已用完") + if _hourly_total(db, quota_day, quota_hour) >= int( + setting.hourly_attempt_limit + ): + return QuotaDecision(False, reason="全局每小时登录尝试预算已用完") + + category = classify_login_category( + account, + mode=mode, + trigger_source=trigger_source, + now=moment, + ) + if _category_daily_total(db, quota_day, category) >= _category_limit( + setting, category + ): + return QuotaDecision(False, reason=f"{category} 今日登录预算已用完") + + attempt = LoginAttempt( + account_id=int(account.id), + batch_id=batch_id[:64], + category=category, + trigger_source=trigger_source[:16], + created_by=created_by, + started_at=moment, + status="pending", + quota_day=quota_day, + ) + db.add(attempt) + for usage_category in (CATEGORY_ALL, category): + usage = _usage_row( + db, + quota_day=quota_day, + quota_hour=quota_hour, + category=usage_category, + ) + usage.attempt_count = int(usage.attempt_count or 0) + 1 + db.flush() + return QuotaDecision(True, attempt=attempt) + + +def finish_login_attempt( + db: Session, + attempt: LoginAttempt, + *, + status: str, + failure_kind: str = "", + now: datetime | None = None, +) -> None: + if attempt.status in {"success", "failed", "error", "stopped"}: + return + moment = now or datetime.now(UTC) + attempt.status = status[:16] + attempt.failure_kind = failure_kind[:16] + attempt.finished_at = moment + attempt.duration_ms = max( + 0, + int((moment - attempt.started_at).total_seconds() * 1000), + ) + quota_day, quota_hour = _bucket(attempt.started_at) + for usage_category in (CATEGORY_ALL, attempt.category): + usage = _usage_row( + db, + quota_day=quota_day, + quota_hour=quota_hour, + category=usage_category, + ) + if status == "success": + usage.success_count = int(usage.success_count or 0) + 1 + elif status in {"failed", "error"}: + usage.failure_count = int(usage.failure_count or 0) + 1 + db.flush() + + +def cleanup_orphan_login_attempts( + db: Session, + *, + now: datetime | None = None, +) -> int: + """服务重启后把遗留的尝试流水收敛为 interrupted。""" + moment = now or datetime.now(UTC) + attempts = ( + db.query(LoginAttempt) + .filter(LoginAttempt.status.in_(("pending", "running"))) + .all() + ) + for attempt in attempts: + finish_login_attempt( + db, + attempt, + status="failed", + failure_kind="interrupted", + now=moment, + ) + if attempts: + db.commit() + return len(attempts) + + +__all__ = [ + "CATEGORY_ACCOUNT_CHECK", + "CATEGORY_MANUAL", + "CATEGORY_ONBOARDING", + "CATEGORY_RELOGIN_HIGH", + "CATEGORY_RELOGIN_ORDINARY", + "QuotaDecision", + "classify_login_category", + "cleanup_orphan_login_attempts", + "finish_login_attempt", + "login_concurrency_limit", + "reserve_login_attempt", +] diff --git a/web/backend/services/login_service.py b/web/backend/services/login_service.py index 7ae5618..59a0bea 100644 --- a/web/backend/services/login_service.py +++ b/web/backend/services/login_service.py @@ -18,7 +18,7 @@ from core.douyu import DouyuLogin, create_login_api_strategy from core.douyu.login import AccountLike from ..models import Account as AccountModel -from ..models import LoginTask +from ..models import LoginAttempt, LoginTask from ..models import ProxyConfig as ProxyConfigModel from .account_events_service import ( EVENT_LOGIN_FAILED, @@ -27,10 +27,17 @@ from .account_events_service import ( emit, ) from .account_lifecycle import on_login_success, sync_status +from .account_sync_service import DOMAIN_STOREHOME, mark_domain_dirty +from .account_work_queue import enqueue_account_work from .cookie_check_service import check_douyu_cookie from .douyu_account_probe import persist_probe_result, probe_session from .douyu_service import LOGIN_ATTEMPT_IN_FLIGHT_STATUSES from .login_failure import classify_failure, failure_kind_label, is_account_fault +from .login_quota_service import ( + finish_login_attempt, + login_concurrency_limit, + reserve_login_attempt, +) from .proxy_service import build_proxy_runtime from .task_runtime import TaskLane, submit @@ -240,6 +247,23 @@ def _write_login_success( ) +def _enqueue_post_login_sync(db: Session, account: AccountModel) -> None: + """登录并验证成功后,把商品能力标脏并进入内部同步队列。""" + mark_domain_dirty( + db, + account_id=int(account.id), + domain=DOMAIN_STOREHOME, + reason="登录成功,等待同步完整商城商品能力", + ) + enqueue_account_work( + db, + account_id=int(account.id), + work_type=DOMAIN_STOREHOME + "_sync", + priority=20, + reason="登录成功联动同步", + ) + + def _persist_login_cookie_verification( db: Session, task: LoginTask, @@ -313,6 +337,7 @@ class LoginBatchRunner: concurrency: int = 3, api_strategy: str = "wgapi", mode: str = "login", + trigger_source: str = "manual", ): self.db = db self.account_ids = account_ids @@ -326,9 +351,16 @@ class LoginBatchRunner: self.log_queue = log_queue self.loop = loop self.batch_id = uuid.uuid4().hex[:12] - self.concurrency = max(1, min(concurrency, 10)) # 限制 1-10 + self.concurrency = max( + 1, + min(concurrency, 10, login_concurrency_limit(db)), + ) self.api_strategy = create_login_api_strategy(api_strategy) self.mode = mode if mode in ("login", "check", "relogin") else "login" + self.trigger_source = trigger_source if trigger_source in { + "manual", + "automation", + } else "manual" self._stop = threading.Event() self._counter_lock = threading.Lock() self._completed = 0 @@ -587,6 +619,14 @@ class LoginBatchRunner: batch_id=self.batch_id, mode=self.mode, ) + if account.ck_check_status == "valid": + try: + _enqueue_post_login_sync(worker_db, account) + except Exception as exc: # noqa: BLE001 + self._push_log( + "warning", + f"[{current}] 登录成功,但登记商品同步队列失败: {exc}", + ) if account is not None and account.ck_check_status == "valid": self._push_log( "success", @@ -718,6 +758,21 @@ class LoginBatchRunner: self._push_log("warning", f"跳过无权账号: {acc.username}") continue + quota = reserve_login_attempt( + self.db, + account=acc, + mode=self.mode, + trigger_source=self.trigger_source, + batch_id=batch_id, + created_by=self.created_by, + ) + if not quota.allowed: + self._push_log( + "warning", + f"跳过账号 {acc.username}: {quota.reason}", + ) + continue + # 解耦后不再需要「哪一行持有 CK」这个不变量(CK 在账号上)。 # 这里只需要保证**同一账号不并发两行**: # ① 复用该账号在途的尝试(避免并发两个批次各建一行); @@ -806,6 +861,10 @@ class LoginBatchRunner: self._push_log("info", f"批量{action_name}任务 {batch_id} 完成") self._push_log("result", "") finally: + try: + _settle_login_attempts(batch_id) + except Exception as exc: # noqa: BLE001 + logger.exception(f"[登录批次 {batch_id}] 结算登录尝试流水失败: {exc}") # 标记批次结束:WS 端据此决定何时清理注册表(断线重连可继续订阅日志)。 batch_registry.mark_finished(batch_id) # 确保 DB Session 被关闭,避免连接泄漏 @@ -853,6 +912,61 @@ class BatchRegistry: return self._batches.pop(batch_id, None) +def _settle_login_attempts(batch_id: str) -> None: + """用新的短事务把本批不可变登录流水结转到终态。""" + session = WorkerSessionLocal() + try: + attempts = ( + session.query(LoginAttempt) + .filter( + LoginAttempt.batch_id == batch_id, + LoginAttempt.status.in_(("pending", "running")), + ) + .all() + ) + for attempt in attempts: + task = ( + session.query(LoginTask) + .filter( + LoginTask.account_id == attempt.account_id, + LoginTask.attempt_batch_id == batch_id, + ) + .order_by(LoginTask.id.desc()) + .first() + ) + task_status = (task.attempt_status if task else "") or ( + task.status if task else "" + ) + message = (task.attempt_message if task else "") or ( + task.message if task else "" + ) + if task_status == "success" or task_status == "account_verified": + status = "success" + failure_kind = "" + elif task_status in CHECK_STATUS_MESSAGES: + status = "failed" + failure_kind = ( + "network" + if task_status == "account_auth_unknown" + else "credential" + ) + elif task_status in {"failed", "error"}: + status = task_status + failure_kind = classify_failure(message) + else: + status = "failed" + failure_kind = "interrupted" + finish_login_attempt( + session, + attempt, + status=status, + failure_kind=failure_kind, + ) + session.commit() + finally: + session.close() + + # 模块级单例 batch_registry = BatchRegistry() diff --git a/web/backend/services/platform_fulfillment/service.py b/web/backend/services/platform_fulfillment/service.py index 012e2a6..2e4d122 100644 --- a/web/backend/services/platform_fulfillment/service.py +++ b/web/backend/services/platform_fulfillment/service.py @@ -35,7 +35,13 @@ from ..account_binding import ( ) from ..account_capability_service import ( AVAILABILITY_AVAILABLE, + AVAILABILITY_EXPIRED, AVAILABILITY_LIMITED, + AVAILABILITY_NO_STOCK, + AVAILABILITY_NOT_OFFERED, + AVAILABILITY_NOT_OPEN, + AVAILABILITY_UNKNOWN, + classify_goods_items, record_account_commodity_state, ) from ..business_events import record_account_query, record_douyu_task_started @@ -92,6 +98,47 @@ def _matching_mode() -> str: return mode if mode in {"legacy", "hybrid", "capability"} else "hybrid" +def _exchange_gate( + state: DouyuLiveAccountState, + commodity_id: str, +) -> tuple[str, str, str]: + """把实时商品资格映射为履约可展示的硬门槛。""" + if commodity_id in state.limited_ids: + return ( + "limited", + "commodity_limit_reached", + "该账号已达到此商品兑换上限", + ) + if not state.commodity_availability: + return "", "", "" + availability = state.commodity_availability.get(commodity_id) + reason = state.commodity_blocking_reasons.get(commodity_id, "") + if availability is None: + return ( + "not_available", + "commodity_not_offered", + reason or "官方商品列表本次未返回该商品,可能已下架", + ) + if availability == AVAILABILITY_AVAILABLE: + return "", "", "" + issue_map = { + AVAILABILITY_NOT_OPEN: ("commodity_not_open", "商品限时兑换尚未开始"), + AVAILABILITY_EXPIRED: ("commodity_expired", "商品限时兑换已经结束"), + AVAILABILITY_NO_STOCK: ("commodity_no_stock", "商品当前无库存"), + AVAILABILITY_NOT_OFFERED: ("commodity_not_offered", "商品当前未上架"), + AVAILABILITY_UNKNOWN: ("commodity_status_unknown", "商品资格状态未知"), + AVAILABILITY_LIMITED: ( + "commodity_limit_reached", + "该账号已达到此商品兑换上限", + ), + } + issue_code, fallback = issue_map.get( + availability, + ("commodity_not_exchangeable", "商品当前不可兑换"), + ) + return "not_available", issue_code, reason or fallback + + _SYNC_LOCK = Lock() _AUTO_SYNC_LOCK = Lock() _AUTO_SYNC_LAST_ATTEMPT = float("-inf") @@ -2152,14 +2199,27 @@ def allocate_task(db: Session, *, task_id: int, current: User) -> dict[str, Any] if live_state.ck_verdict == "valid" and not live_state.error: try: + availability = live_state.commodity_availability.get( + task.commodity_id + ) + blocking_reason = live_state.commodity_blocking_reasons.get( + task.commodity_id, + "", + ) + if limited: + availability = AVAILABILITY_LIMITED + blocking_reason = "该账号已达此商品兑换上限" + elif live_state.commodity_availability and availability is None: + availability = AVAILABILITY_NOT_OFFERED + blocking_reason = "官方商品列表本次未返回该商品,可能已下架" + elif availability is None: + availability = AVAILABILITY_AVAILABLE record_account_commodity_state( db, account, commodity_id=task.commodity_id, - availability=( - AVAILABILITY_LIMITED if limited else AVAILABILITY_AVAILABLE - ), - blocking_reason=("该账号已达此商品兑换上限" if limited else ""), + availability=availability, + blocking_reason=blocking_reason, raw_status="1" if limited else "0", observed_at=live_state.checked_at, source="live_check", @@ -2307,10 +2367,13 @@ def allocate_task(db: Session, *, task_id: int, current: User) -> dict[str, Any] soft_reasons = selected_state.soft_requirement_reasons( required_points=required_points ) - if task.commodity_id in selected_state.limited_ids: - task.exchange_status = "limited" - task.issue_code = "commodity_limit_reached" - task.last_error = "该账号已达到此商品兑换上限" + gate_status, gate_issue, gate_message = _exchange_gate( + selected_state, task.commodity_id + ) + if gate_status: + task.exchange_status = gate_status + task.issue_code = gate_issue + task.last_error = gate_message elif selected_state.elite_opened is not True: task.exchange_status = "not_opened" task.issue_code = "elite_not_opened" @@ -2875,10 +2938,47 @@ def refresh_exchange_eligibility( db.commit() return get_task_detail(db, task.id) + eligibility = probe.get("eligibility") or {} + storehome_goods = [ + dict(item) + for item in eligibility.get("storehome_goods") or [] + if isinstance(item, dict) + ] + assessments = classify_goods_items(storehome_goods, now=_now()) + assessment = assessments.get(task.commodity_id) if task.commodity_id in limited_ids: - task.exchange_status = "limited" - task.issue_code = "commodity_limit_reached" - task.last_error = "该账号已达到此商品兑换上限" + availability = AVAILABILITY_LIMITED + commodity_status = "limited" + issue_code = "commodity_limit_reached" + commodity_message = "该账号已达到此商品兑换上限" + elif assessments and assessment is None: + availability = AVAILABILITY_NOT_OFFERED + commodity_status = "not_available" + issue_code = "commodity_not_offered" + commodity_message = "官方商品列表本次未返回该商品,可能已下架" + elif assessment is not None and assessment.availability != AVAILABILITY_AVAILABLE: + availability = assessment.availability + commodity_status = "not_available" + issue_code = { + AVAILABILITY_NOT_OPEN: "commodity_not_open", + AVAILABILITY_EXPIRED: "commodity_expired", + AVAILABILITY_NO_STOCK: "commodity_no_stock", + AVAILABILITY_NOT_OFFERED: "commodity_not_offered", + AVAILABILITY_UNKNOWN: "commodity_status_unknown", + }.get(assessment.availability, "commodity_not_exchangeable") + commodity_message = ( + assessment.blocking_reason or "商品当前不可兑换" + ) + else: + availability = AVAILABILITY_AVAILABLE + commodity_status = "" + issue_code = "" + commodity_message = "" + + if commodity_status: + task.exchange_status = commodity_status + task.issue_code = issue_code + task.last_error = commodity_message elif not account.elite_opened: task.exchange_status = "not_opened" task.issue_code = "elite_not_opened" @@ -2895,15 +2995,15 @@ def refresh_exchange_eligibility( db, account, commodity_id=task.commodity_id, - availability=( - AVAILABILITY_LIMITED + availability=availability, + blocking_reason=commodity_message, + raw_status=( + assessment.raw_status + if assessment is not None + else "1" if task.commodity_id in limited_ids - else AVAILABILITY_AVAILABLE + else "0" ), - blocking_reason=( - "该账号已达此商品兑换上限" if task.commodity_id in limited_ids else "" - ), - raw_status="1" if task.commodity_id in limited_ids else "0", observed_at=_now(), source="eligibility_check", scope=task.scope or SCOPE_ELITE, @@ -2918,6 +3018,8 @@ def refresh_exchange_eligibility( "commodity_id": task.commodity_id, "exchange_status": task.exchange_status, "limited": task.commodity_id in limited_ids, + "availability": availability, + "issue_code": task.issue_code, "points": account.points, "elite_opened": account.elite_opened, }, diff --git a/web/frontend/src/pages/PlatformFulfillmentPage.tsx b/web/frontend/src/pages/PlatformFulfillmentPage.tsx index 6e59f81..477e75b 100644 --- a/web/frontend/src/pages/PlatformFulfillmentPage.tsx +++ b/web/frontend/src/pages/PlatformFulfillmentPage.tsx @@ -72,6 +72,7 @@ const EXCHANGE_STATUS_LABELS: Record = { points_short: '积分不足', not_opened: '精英宝典未开通', limited: '已达限兑上限', + not_available: '商品当前不可兑换', check_failed: '资格查询失败', manual_done: '人工兑换完成', }; @@ -146,6 +147,12 @@ const ISSUE_LABELS: Record = { points_short: '积分不足', elite_not_opened: '精英宝典未开通', commodity_limit_reached: '商品已达限兑上限', + commodity_not_open: '商品限时兑换未开始', + commodity_expired: '商品限时兑换已结束', + commodity_no_stock: '商品当前无库存', + commodity_not_offered: '商品当前未上架', + commodity_status_unknown: '商品资格状态未知', + commodity_not_exchangeable: '商品当前不可兑换', eligibility_check_failed: '资格查询失败', binding_failed: '绑定失败', account_role_cooldown: '斗鱼账号仍在换绑冷却期', @@ -188,6 +195,7 @@ function exchangeStatusHint(status: string) { if (status === 'points_short') return '当前积分不足,账号不会被淘汰。完成充值后点击“刷新兑换资格”即可继续。'; if (status === 'not_opened') return '账号尚未开通精英宝典,不会被淘汰。开通后点击“刷新兑换资格”即可继续。'; if (status === 'limited') return '当前账号已达到该商品限兑上限,可等待限制解除、人工处理或释放后换号。'; + if (status === 'not_available') return '商品当前受时间、库存或上架状态限制,暂不可兑换。请等待开放后重新查询资格。'; if (status === 'check_failed') return '实时查询失败,账号仍保留。请稍后重新刷新兑换资格。'; if (status === 'eligible') return '账号已满足当前商品兑换条件,可以继续人工兑换。'; return '';