From e652d302830848af575078bff0689d04806bd2a4 Mon Sep 17 00:00:00 2001 From: yml Date: Sun, 22 Nov 2026 19:15:44 +0800 Subject: [PATCH] =?UTF-8?q?perf(platform-fulfillment):=20=E4=BC=98?= =?UTF-8?q?=E5=8C=96=E6=9C=AC=E5=9C=B0=E9=80=89=E5=8F=B7=E4=B8=8E=E5=A4=B1?= =?UTF-8?q?=E8=B4=A5=E4=BB=BB=E5=8A=A1=E8=B0=83=E5=BA=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 13 +- docker-compose.yml | 6 + docs/直播账号履约工作台.md | 44 +- tests/test_account_capability_service.py | 5 +- tests/test_platform_fulfillment.py | 135 ++---- ...est_platform_fulfillment_auto_allocator.py | 93 +++- .../services/account_capability_service.py | 47 +- .../platform_fulfillment/account_selector.py | 74 +++- .../platform_fulfillment/auto_allocator.py | 9 +- .../services/platform_fulfillment/service.py | 406 +++++++++++++++++- 10 files changed, 701 insertions(+), 131 deletions(-) diff --git a/.env.example b/.env.example index 3a73cb2..ba89998 100644 --- a/.env.example +++ b/.env.example @@ -202,10 +202,19 @@ INTEGRATION_OUTBOX_MAX_ATTEMPTS=12 PLATFORM_FULFILLMENT_RECONCILE_SECONDS=900 # 可选:指定集成操作人;不填时使用最早创建的有效超级管理员 # PLATFORM_FULFILLMENT_INTEGRATION_ACTOR_ID=1 -# 账号商品能力快照有效期(秒);过期后不再直接参与自动匹配 +# 不稳定能力快照的兜底有效期(秒);库存/未知状态默认使用下面的独立短 TTL ACCOUNT_COMMODITY_STATE_TTL_SECONDS=900 -# legacy=旧实时扫号 / hybrid=能力索引优先、失败回退旧逻辑 / capability=只读能力索引 +# 稳定能力:账号具备活动资格、未开放等,默认 6 小时;兑换前仍会实时复核 +ACCOUNT_COMMODITY_STABLE_TTL_SECONDS=21600 +ACCOUNT_COMMODITY_LIMITED_TTL_SECONDS=86400 +ACCOUNT_COMMODITY_NO_STOCK_TTL_SECONDS=900 +ACCOUNT_COMMODITY_UNKNOWN_TTL_SECONDS=300 +# legacy=旧实时扫号 / hybrid=本地能力索引(默认,零外部请求)/ capability=同 hybrid PLATFORM_FULFILLMENT_MATCH_MODE=hybrid +# 单轮本地选号批量;本地选号不再逐账号请求官网 +PLATFORM_FULFILLMENT_AUTO_ALLOCATE_BATCH=20 +# 能力池缺失时,每次最多向独立账号同步槽补入多少个账号 +PLATFORM_FULFILLMENT_CACHE_REFRESH_SEED_COUNT=20 # 订单 Webhook 连续唤醒时保持最小选号间隔,避免忙轮询压满数据库。 PLATFORM_FULFILLMENT_AUTO_ALLOCATE_MIN_INTERVAL_SECONDS=5 # 客户绑定二维码有效期 diff --git a/docker-compose.yml b/docker-compose.yml index 86c5f96..59f07ca 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -119,7 +119,13 @@ services: - ORDER_SITE_PLATFORM_FULFILLMENT_URL=${ORDER_SITE_PLATFORM_FULFILLMENT_URL:-} - PLATFORM_FULFILLMENT_INTEGRATION_ACTOR_ID=${PLATFORM_FULFILLMENT_INTEGRATION_ACTOR_ID:-} - ACCOUNT_COMMODITY_STATE_TTL_SECONDS=${ACCOUNT_COMMODITY_STATE_TTL_SECONDS:-900} + - ACCOUNT_COMMODITY_STABLE_TTL_SECONDS=${ACCOUNT_COMMODITY_STABLE_TTL_SECONDS:-21600} + - ACCOUNT_COMMODITY_LIMITED_TTL_SECONDS=${ACCOUNT_COMMODITY_LIMITED_TTL_SECONDS:-86400} + - ACCOUNT_COMMODITY_NO_STOCK_TTL_SECONDS=${ACCOUNT_COMMODITY_NO_STOCK_TTL_SECONDS:-900} + - ACCOUNT_COMMODITY_UNKNOWN_TTL_SECONDS=${ACCOUNT_COMMODITY_UNKNOWN_TTL_SECONDS:-300} - PLATFORM_FULFILLMENT_MATCH_MODE=${PLATFORM_FULFILLMENT_MATCH_MODE:-hybrid} + - PLATFORM_FULFILLMENT_AUTO_ALLOCATE_BATCH=${PLATFORM_FULFILLMENT_AUTO_ALLOCATE_BATCH:-20} + - PLATFORM_FULFILLMENT_CACHE_REFRESH_SEED_COUNT=${PLATFORM_FULFILLMENT_CACHE_REFRESH_SEED_COUNT:-20} - PLATFORM_FULFILLMENT_AUTO_ALLOCATE_MIN_INTERVAL_SECONDS=${PLATFORM_FULFILLMENT_AUTO_ALLOCATE_MIN_INTERVAL_SECONDS:-5} - DOUYU_BIND_ROLE_POLL_SECONDS=${DOUYU_BIND_ROLE_POLL_SECONDS:-1200} - PLATFORM_BINDING_QR_TTL_SECONDS=${PLATFORM_BINDING_QR_TTL_SECONDS:-1260} diff --git a/docs/直播账号履约工作台.md b/docs/直播账号履约工作台.md index a22b9e3..d288ad5 100644 --- a/docs/直播账号履约工作台.md +++ b/docs/直播账号履约工作台.md @@ -55,33 +55,39 @@ 完整的斗鱼绑定字段、`actAlias` 多来源规则、pending 判定原因和日志检索方式见 [斗鱼协议总览](斗鱼/DOUYU_PROTOCOL_OVERVIEW.md)。 -选号前会调用 `web/backend/services/douyu_account_state.py` 实时刷新账号: +2026-11-22 起,默认选号是**纯本地数据库操作**: -- 不依赖 `status`、`ck_check_status`、`elite_opened`、`points` 等缓存结论; -- 只要账号有 CK、允许自动管理且未停用/归档,就进入实时检测; -- 每次选号最多检测 5 个账号,结果写回 `ck_check_*`、生命周期状态、 - 精英宝典、积分、限兑、绑定状态、换绑冷却和账号概览快照; -- 下一次点击会优先检测尚未检测或最久未检测的下一批账号; -- 实时刷新完成后才判断是否创建租约,失败事件会记录每个账号的具体原因。 +- `hybrid` / `capability` 只读取新鲜 `account_commodity_states` 和账号主表, + 不再在分配热路径串行请求斗鱼官网; +- 一个账号的 `storehome` 商品列表会一次写入全部商品能力,不需要逐商品查询; +- 分配阶段只做账号硬过滤、候选排序和数据库租约抢占,通常应在毫秒级完成; +- 绑定二维码阶段会校验 CK/角色,兑换前“刷新兑换资格”仍会实时访问官网, + 因此本地选号只负责快速占号,不替代最终业务校验; +- `legacy` 保留旧实时扫号逻辑,仅用于故障回滚和对比,不再是生产默认路径。 -2026-11-18 起增加“能力索引优先”的混合匹配: +能力索引与调度规则: - 一键查询或资料刷新把官方 `storehome` 商品状态写入 `account_commodity_states`; - `storehome` 一次返回账号可见的全部商品,逐商品写状态,但每个账号只需一次列表请求; - 同时保存 `timeLimited`、`specTimes` 和起止时间,未开始/已结束与账号限兑分别处理; -- `PLATFORM_FULFILLMENT_MATCH_MODE=hybrid` 时先查新鲜能力索引,没有结果再回退旧扫号; -- `capability` 模式只读能力索引,不执行全量候选扫描; -- `legacy` 模式保留旧逻辑,用于快速回滚; -- 能力索引命中后仍只对最终候选做一次实时确认,避免缓存过期直接分配; +- 可兑换、未开放等稳定能力默认缓存 6 小时,库存 15 分钟,未知状态 5 分钟; + 否则 3 万账号池不可能在 15 分钟内刷新一遍,能力索引会长期处于过期状态; - 有 pending 履约需求时,`account_capability_refresh` 作业优先刷新对应商品的过期能力; - 作业消息只报告本轮结果,不再每轮执行会扫描全账号的精确剩余量统计。 +- 能力刷新和外部请求由 `account_sync_queue` 独立执行槽消费,不占用选号 worker; +- 本地能力池缺失时,选号失败只向该独立队列补入最多 20 个账号, + 可用 `PLATFORM_FULFILLMENT_CACHE_REFRESH_SEED_COUNT` 调整; +- 自动选号按 `next_allocate_at` 轮转,而不是永久按订单时间排在最前; +- “未开放”任务直接排到商品开放时间,“无库存”至少退避 5 分钟, + “能力快照缺失”退避 2 分钟,避免失败任务每 45 秒占满执行槽; +- 单轮默认处理 20 个本地任务,可用 `PLATFORM_FULFILLMENT_AUTO_ALLOCATE_BATCH` 调整。 -能力快照默认 15 分钟有效,可用 `ACCOUNT_COMMODITY_STATE_TTL_SECONDS` 调整。 -过期能力不会直接参与自动匹配。真正创建租约仍然依赖账号租约唯一约束, -因此并发 worker 不会把同一账号分配给两张订单。 +能力 TTL 可分别用 `ACCOUNT_COMMODITY_STABLE_TTL_SECONDS`、 +`ACCOUNT_COMMODITY_NO_STOCK_TTL_SECONDS` 和 +`ACCOUNT_COMMODITY_UNKNOWN_TTL_SECONDS` 调整。过期能力不会参与自动匹配。 +真正创建租约仍然依赖账号租约唯一约束,因此并发 worker 不会把同一账号分配给两张订单。 -选号硬条件只保留:CK 实时有效、未达到该商品官方限兑上限、角色换绑冷却已结束、 -无有效租约。积分不足和精英宝典未开通属于**软条件**: +选号硬条件只保留:本地 CK 结论有效、未达到该商品官方限兑上限、角色换绑冷却已结束、 +无有效租约。CK 与兑换资格会在绑码/兑换阶段再次实时校验;积分不足和精英宝典未开通属于**软条件**: - 不会淘汰账号,也不会阻止生成绑定二维码; - 优先选择已开通且积分足够的账号,其次选择只需充值或只需开通的账号; @@ -92,7 +98,7 @@ 1. 新订单由 `order_site` 事务 Outbox 自动进入;人工“同步待履约订单”只用于历史补查。 2. 为待映射商品选择斗鱼商品和积分价格。 -3. 点击“自动选号”,系统实时检查 CK、租约、365 天限兑和换绑冷却; +3. 点击“自动选号”,系统从本地能力池检查租约、365 天限兑和换绑冷却; 积分与精英宝典开通状态只影响账号排序和后续处理提示。 4. 点击“生成二维码”,让客户扫码绑定游戏角色。 5. 角色出现后点击“确认绑定”。 diff --git a/tests/test_account_capability_service.py b/tests/test_account_capability_service.py index 482a70e..6c47d57 100644 --- a/tests/test_account_capability_service.py +++ b/tests/test_account_capability_service.py @@ -165,7 +165,10 @@ def test_sync_account_goods_states_writes_capabilities_and_exchange_stats(db): assert states["C"].availability == "no_stock" assert states["D"].availability == "unknown" assert states["A"].valid_until == NOW + timedelta( - seconds=service.capability_ttl_seconds() + seconds=service.capability_ttl_seconds("available") + ) + assert states["C"].valid_until == NOW + timedelta( + seconds=service.capability_ttl_seconds("no_stock") ) diff --git a/tests/test_platform_fulfillment.py b/tests/test_platform_fulfillment.py index 87371ce..b8113a0 100644 --- a/tests/test_platform_fulfillment.py +++ b/tests/test_platform_fulfillment.py @@ -17,6 +17,7 @@ from web.backend.models import ( AccountCommodityState, AccountEvent, AccountLease, + AccountWorkItem, AutomationJob, DouyuExternalOrder, DouyuGoodsSnapshot, @@ -31,6 +32,7 @@ from web.backend.models import ( from web.backend.permissions import ROLE_PERMISSIONS from web.backend.schemas import DouyuCollabOrderOut from web.backend.services import douyu_collab_service as collab +from web.backend.services.account_sync_runner import WORK_STOREHOME_SYNC from web.backend.services.automation.jobs import ( platform_fulfillment_lease_reaper as reaper, ) @@ -94,6 +96,8 @@ class _Base: points: int = 2000, opened: bool = True, role_available_at: datetime | None = None, + capability: bool = True, + availability: str = "available", ) -> Account: row = Account( username=name, @@ -110,6 +114,21 @@ class _Base: role_change_available_at=role_available_at, ) self.session.add(row) + self.session.flush() + if capability: + self.session.add( + AccountCommodityState( + account_id=row.id, + platform="douyu", + scope="elite", + commodity_id=COMMODITY, + commodity_name=INTERNAL_SKU, + availability=availability, + observed_at=NOW, + valid_until=NOW + timedelta(hours=6), + source="test_fixture", + ) + ) self.session.commit() return row @@ -1058,8 +1077,8 @@ class TestAccountSelection(_Base): assert old_retry_at is None or task.next_allocate_at >= old_retry_at self.refresh_mock.assert_not_called() - def test_capability_match_checks_only_selected_candidate(self, monkeypatch): - account = self.account("capability-fast") + def test_capability_match_uses_local_index_without_live_probe(self, monkeypatch): + account = self.account("capability-fast", capability=False) task = self.task() self.session.add( AccountCommodityState( @@ -1085,47 +1104,14 @@ class TestAccountSelection(_Base): assert detail["task"]["account"]["id"] == account.id assert detail["task"]["executionStatus"] == "allocated" - self.refresh_mock.assert_called_once() + self.refresh_mock.assert_not_called() - def test_hybrid_falls_back_to_live_scan_when_capability_candidate_fails( + def test_hybrid_without_capability_does_not_fall_back_to_live_scan( self, monkeypatch, ): - capability_account = self.account("capability-stale") - live_account = self.account("live-fallback") + self.account("live-fallback", capability=False) task = self.task() - self.session.add( - AccountCommodityState( - account_id=capability_account.id, - platform="douyu", - scope="elite", - commodity_id=COMMODITY, - availability="available", - observed_at=NOW, - valid_until=NOW + timedelta(minutes=15), - source="overview", - ) - ) - self.session.commit() - - def live_state(_db, account, **_kwargs): - return DouyuLiveAccountState( - account_id=account.id, - account_name=account.username, - checked_at=NOW, - ck_verdict="valid", - ck_message="有效", - elite_opened=True, - points=2000, - limited_ids=( - frozenset({COMMODITY}) - if account.id == capability_account.id - else frozenset() - ), - role_change_available_at=None, - ) - - self.refresh_mock.side_effect = live_state monkeypatch.setenv("PLATFORM_FULFILLMENT_MATCH_MODE", "hybrid") detail = service.allocate_task( @@ -1134,10 +1120,16 @@ class TestAccountSelection(_Base): current=self.admin, ) - assert detail["task"]["account"]["id"] == live_account.id - assert self.refresh_mock.call_count == 2 + assert detail["task"]["account"] is None + assert detail["task"]["executionStatus"] == "ready" + assert detail["task"]["issueCode"] == "capability_cache_miss" + self.refresh_mock.assert_not_called() + work_item = self.session.query(AccountWorkItem).one() + assert work_item.work_type == WORK_STOREHOME_SYNC + assert work_item.priority == 20 - def test_allocation_refreshes_live_state_instead_of_cached_filters(self): + def test_legacy_mode_keeps_live_scan_fallback(self, monkeypatch): + monkeypatch.setenv("PLATFORM_FULFILLMENT_MATCH_MODE", "legacy") account = Account( username="live-refresh-account", password="p", @@ -1226,19 +1218,6 @@ class TestAccountSelection(_Base): def test_elite_not_opened_is_soft_and_does_not_block_allocation(self): account = self.account("elite-closed", opened=False) - self.refresh_mock.side_effect = lambda _db, row, **_kwargs: ( - DouyuLiveAccountState( - account_id=row.id, - account_name=row.username, - checked_at=NOW, - ck_verdict="valid", - ck_message="有效", - elite_opened=False, - points=2000, - limited_ids=frozenset(), - role_change_available_at=None, - ) - ) task = self.task() detail = service.allocate_task( @@ -1253,27 +1232,10 @@ class TestAccountSelection(_Base): assert detail["task"]["issueCode"] == "elite_not_opened" def test_presale_not_open_allows_allocation_and_advance_binding(self): - account = self.account("presale-future") + account = self.account("presale-future", availability="not_open") task = self.task() task.sale_mode = "presale" self.session.commit() - self.refresh_mock.side_effect = lambda _db, row, **_kwargs: ( - DouyuLiveAccountState( - account_id=row.id, - account_name=row.username, - checked_at=NOW, - ck_verdict="valid", - ck_message="有效", - elite_opened=True, - points=2000, - limited_ids=frozenset(), - role_change_available_at=None, - commodity_availability={COMMODITY: "not_open"}, - commodity_blocking_reasons={ - COMMODITY: "限时兑换未开始,开始于 2026-11-26T10:00:00+00:00" - }, - ) - ) detail = service.allocate_task( self.session, @@ -1287,27 +1249,10 @@ class TestAccountSelection(_Base): assert detail["task"]["issueCode"] == "commodity_not_open" def test_direct_sale_still_blocks_when_commodity_not_open(self): - self.account("direct-future") + self.account("direct-future", availability="not_open") task = self.task() task.sale_mode = "direct" self.session.commit() - self.refresh_mock.side_effect = lambda _db, row, **_kwargs: ( - DouyuLiveAccountState( - account_id=row.id, - account_name=row.username, - checked_at=NOW, - ck_verdict="valid", - ck_message="有效", - elite_opened=True, - points=2000, - limited_ids=frozenset(), - role_change_available_at=None, - commodity_availability={COMMODITY: "not_open"}, - commodity_blocking_reasons={ - COMMODITY: "限时兑换未开始,开始于 2026-11-26T10:00:00+00:00" - }, - ) - ) detail = service.allocate_task( self.session, @@ -1317,10 +1262,10 @@ class TestAccountSelection(_Base): assert detail["task"]["account"] is None assert detail["task"]["executionStatus"] == "ready" - assert detail["task"]["issueCode"] == "no_eligible_account" + assert detail["task"]["issueCode"] == "commodity_not_open" def test_capability_pool_includes_future_not_open_for_presale_only(self): - account = self.account("capability-future") + account = self.account("capability-future", capability=False) self.session.add( AccountCommodityState( account_id=account.id, @@ -1378,7 +1323,11 @@ class TestAccountSelection(_Base): assert [row.username for row in candidates] == ["small", "large"] def test_active_collab_order_blocks_all_automatic_candidate_pools(self): - account = self.account("active-collab-candidate", points=2000) + account = self.account( + "active-collab-candidate", + points=2000, + capability=False, + ) self.session.add( AccountCommodityState( account_id=account.id, diff --git a/tests/test_platform_fulfillment_auto_allocator.py b/tests/test_platform_fulfillment_auto_allocator.py index a03518e..6574884 100644 --- a/tests/test_platform_fulfillment_auto_allocator.py +++ b/tests/test_platform_fulfillment_auto_allocator.py @@ -113,7 +113,7 @@ def test_auto_allocator_schedules_retry_only_after_failed_allocation(db): order_match_started_at=datetime.now(UTC) - timedelta(days=1), ) ) - moment = datetime.now(UTC) + moment = datetime.now(UTC).replace(microsecond=0) task = PlatformFulfillmentTask( source_system="order_site", source_order_id="10", @@ -202,3 +202,94 @@ def test_auto_allocator_prioritizes_oldest_paid_task_after_due_check(db): auto_allocator.run_auto_allocation_once(db, limit=1, now=moment) assert calls == [older.id] + + +def test_failed_task_retry_does_not_block_other_due_tasks(db): + actor = User(username="auto-rotate-actor", password_hash="h", role="super_admin") + moment = datetime.now(UTC) + db.add( + PlatformFulfillmentSetting( + id=1, + lease_duration_minutes=None, + order_match_started_at=moment - timedelta(days=1), + ) + ) + failing = PlatformFulfillmentTask( + source_system="order_site", + source_order_id="30", + source_order_item_id="30", + internal_sku_key="skin", + commodity_id="commodity-1", + execution_status="ready", + binding_status="not_started", + exchange_status="not_started", + paid_at=moment - timedelta(minutes=10), + next_allocate_at=moment, + updated_at=moment, + ) + waiting = PlatformFulfillmentTask( + source_system="order_site", + source_order_id="31", + source_order_item_id="31", + internal_sku_key="skin", + commodity_id="commodity-1", + execution_status="ready", + binding_status="not_started", + exchange_status="not_started", + paid_at=moment - timedelta(minutes=1), + next_allocate_at=moment, + updated_at=moment, + ) + db.add_all([actor, failing, waiting]) + db.commit() + + calls: list[int] = [] + with ( + patch.object(auto_allocator.service, "_integration_actor", return_value=actor), + patch.object( + auto_allocator.service, + "allocate_task", + side_effect=lambda _db, *, task_id, current: ( + calls.append(task_id) or {"task": {"account": None}} + ), + ), + ): + auto_allocator.run_auto_allocation_once(db, limit=1, now=moment) + auto_allocator.run_auto_allocation_once( + db, + limit=1, + now=moment + timedelta(seconds=1), + ) + + assert calls == [failing.id, waiting.id] + + +def test_schedule_retry_preserves_longer_capability_retry(db): + actor = User(username="auto-backoff-actor", password_hash="h", role="super_admin") + moment = datetime.now(UTC).replace(microsecond=0) + retry_at = moment + timedelta(minutes=5) + task = PlatformFulfillmentTask( + source_system="order_site", + source_order_id="40", + source_order_item_id="40", + internal_sku_key="skin", + commodity_id="commodity-1", + execution_status="ready", + binding_status="not_started", + exchange_status="not_started", + paid_at=moment, + next_allocate_at=retry_at, + updated_at=moment, + ) + db.add_all([actor, task]) + db.commit() + + auto_allocator._schedule_retry( + db, + task_id=task.id, + moment=moment, + retry_seconds=45, + ) + + db.refresh(task) + assert task.next_allocate_at == retry_at diff --git a/web/backend/services/account_capability_service.py b/web/backend/services/account_capability_service.py index c6438d5..f3c027d 100644 --- a/web/backend/services/account_capability_service.py +++ b/web/backend/services/account_capability_service.py @@ -25,6 +25,10 @@ AVAILABILITY_EXPIRED = "expired" AVAILABILITY_UNKNOWN = "unknown" DEFAULT_CAPABILITY_TTL_SECONDS = 15 * 60 +DEFAULT_STABLE_CAPABILITY_TTL_SECONDS = 6 * 60 * 60 +DEFAULT_LIMITED_CAPABILITY_TTL_SECONDS = 24 * 60 * 60 +DEFAULT_NO_STOCK_CAPABILITY_TTL_SECONDS = 15 * 60 +DEFAULT_UNKNOWN_CAPABILITY_TTL_SECONDS = 5 * 60 def _env_positive_int(name: str, default: int) -> int: @@ -35,11 +39,39 @@ def _env_positive_int(name: str, default: int) -> int: return value if value > 0 else default -def capability_ttl_seconds() -> int: - return _env_positive_int( +def capability_ttl_seconds(availability: str | None = None) -> int: + """按能力稳定性返回快照 TTL。 + + 30k 级账号池无法在 15 分钟内全量刷新一遍。把稳定的“可兑换/未开放” + 仍按 15 分钟处理,会让本地能力池长期处于过期状态,选号又退化到逐个 + 实时扫描。库存和未知状态变化快,因此保持短 TTL;账号是否具备活动资格、 + 是否达到限兑上限则使用长 TTL,兑换前仍会再做实时复核。 + """ + generic = _env_positive_int( "ACCOUNT_COMMODITY_STATE_TTL_SECONDS", DEFAULT_CAPABILITY_TTL_SECONDS, ) + if availability in {AVAILABILITY_AVAILABLE, AVAILABILITY_NOT_OPEN}: + return _env_positive_int( + "ACCOUNT_COMMODITY_STABLE_TTL_SECONDS", + DEFAULT_STABLE_CAPABILITY_TTL_SECONDS, + ) + if availability == AVAILABILITY_LIMITED: + return _env_positive_int( + "ACCOUNT_COMMODITY_LIMITED_TTL_SECONDS", + DEFAULT_LIMITED_CAPABILITY_TTL_SECONDS, + ) + if availability == AVAILABILITY_NO_STOCK: + return _env_positive_int( + "ACCOUNT_COMMODITY_NO_STOCK_TTL_SECONDS", + DEFAULT_NO_STOCK_CAPABILITY_TTL_SECONDS, + ) + if availability == AVAILABILITY_UNKNOWN: + return _env_positive_int( + "ACCOUNT_COMMODITY_UNKNOWN_TTL_SECONDS", + DEFAULT_UNKNOWN_CAPABILITY_TTL_SECONDS, + ) + return generic def _to_int(value: Any) -> int | None: @@ -310,8 +342,6 @@ def sync_account_goods_states( if not goods_items or account.id is None: return 0 moment = observed_at or datetime.now(UTC) - ttl_valid_until = moment + timedelta(seconds=capability_ttl_seconds()) - existing_rows = ( db.query(AccountCommodityState) .filter( @@ -347,6 +377,9 @@ def sync_account_goods_states( availability, reason, raw_status = classify_goods_item(item, now=moment) window = goods_time_window(item, now=moment) + ttl_valid_until = moment + timedelta( + seconds=capability_ttl_seconds(availability) + ) last_exchange_at, exchange_count = exchange_stats.get( commodity_id, (None, 0), @@ -392,7 +425,9 @@ def sync_account_goods_states( state.schedule_json = None state.observed_at = moment state.valid_until = None - state.next_check_at = ttl_valid_until + state.next_check_at = moment + timedelta( + seconds=capability_ttl_seconds(AVAILABILITY_NOT_OFFERED) + ) state.source = source[:24] state.updated_at = moment changed += 1 @@ -418,7 +453,7 @@ def record_account_commodity_state( """写入单个商品的实时检查结论,供下一轮匹配直接使用。""" moment = observed_at or datetime.now(UTC) ttl = ( - capability_ttl_seconds() + capability_ttl_seconds(availability) if valid_for_seconds is None else max(1, int(valid_for_seconds)) ) diff --git a/web/backend/services/platform_fulfillment/account_selector.py b/web/backend/services/platform_fulfillment/account_selector.py index ed104eb..fa3452c 100644 --- a/web/backend/services/platform_fulfillment/account_selector.py +++ b/web/backend/services/platform_fulfillment/account_selector.py @@ -63,6 +63,14 @@ class AccountCandidate: return self.nickname or self.username or f"#{self.account_id}" +@dataclass(frozen=True) +class CapabilityCandidate: + """一个新鲜能力快照及其对应账号,供本地零外部请求选号。""" + + account: Account + state: AccountCommodityState + + def _now(now: datetime | None = None) -> datetime: return now or datetime.now(UTC) @@ -269,7 +277,7 @@ def live_candidate_accounts( return rows -def capability_candidate_accounts( +def _capability_candidate_query( db: Session, *, commodity_id: str, @@ -278,11 +286,10 @@ def capability_candidate_accounts( limit: int = 20, now: datetime | None = None, allow_not_open: bool = False, -) -> list[Account]: - """从新鲜的商品能力索引读取候选账号,不产生外部请求。""" +): commodity = commodity_id.strip() if not commodity: - return [] + return None moment = _now(now) since = moment - timedelta(days=EXCHANGE_COOLDOWN_DAYS) owner_ids = _pool_owner_ids() @@ -330,7 +337,7 @@ def capability_candidate_accounts( ), ) return ( - db.query(Account) + db.query(Account, AccountCommodityState) .join( AccountCommodityState, AccountCommodityState.account_id == Account.id, @@ -349,6 +356,7 @@ def capability_candidate_accounts( ), Account.cookie.isnot(None), Account.cookie != "", + Account.ck_check_status == "valid", Account.auto_managed.is_(True), Account.status.notin_(MANUAL_STATES), or_( @@ -376,10 +384,62 @@ def capability_candidate_accounts( Account.id.asc(), ) .limit(max(1, min(int(limit), 100))) - .all() ) +def capability_candidate_rows( + db: Session, + *, + commodity_id: str, + required_points: int = 0, + scope: str = "elite", + limit: int = 20, + now: datetime | None = None, + allow_not_open: bool = False, +) -> list[CapabilityCandidate]: + """读取账号与能力快照;只访问本地数据库,不触发官网请求。""" + query = _capability_candidate_query( + db, + commodity_id=commodity_id, + required_points=required_points, + scope=scope, + limit=limit, + now=now, + allow_not_open=allow_not_open, + ) + if query is None: + return [] + return [ + CapabilityCandidate(account=account, state=state) + for account, state in query.all() + ] + + +def capability_candidate_accounts( + db: Session, + *, + commodity_id: str, + required_points: int = 0, + scope: str = "elite", + limit: int = 20, + now: datetime | None = None, + allow_not_open: bool = False, +) -> list[Account]: + """从新鲜的商品能力索引读取候选账号,不产生外部请求。""" + return [ + candidate.account + for candidate in capability_candidate_rows( + db, + commodity_id=commodity_id, + required_points=required_points, + scope=scope, + limit=limit, + now=now, + allow_not_open=allow_not_open, + ) + ] + + def has_recent_exchange( db: Session, *, @@ -411,6 +471,7 @@ def lease_account( commodity_id: str, ttl_minutes: int | None = None, now: datetime | None = None, + commit: bool = True, ) -> AccountLease | None: """尝试领取一个账号;撞唯一索引时返回 None。""" return acquire_account_lease( @@ -423,6 +484,7 @@ def lease_account( batch_id=f"pf:{task_id}", ttl_minutes=ttl_minutes, now=now, + commit=commit, ) diff --git a/web/backend/services/platform_fulfillment/auto_allocator.py b/web/backend/services/platform_fulfillment/auto_allocator.py index 3a4daef..fe809ca 100644 --- a/web/backend/services/platform_fulfillment/auto_allocator.py +++ b/web/backend/services/platform_fulfillment/auto_allocator.py @@ -19,7 +19,7 @@ from . import service DEFAULT_INTERVAL_SECONDS = 20 DEFAULT_MIN_INTERVAL_SECONDS = 5 -DEFAULT_BATCH_SIZE = 2 +DEFAULT_BATCH_SIZE = 20 DEFAULT_RETRY_SECONDS = 45 _stop_event = threading.Event() @@ -103,7 +103,9 @@ def _schedule_retry( task = db.get(PlatformFulfillmentTask, int(task_id)) if task is None or task.execution_status in {"completed", "cancelled"}: return - task.next_allocate_at = moment + timedelta(seconds=retry_seconds) + scheduled_retry = moment + timedelta(seconds=retry_seconds) + if task.next_allocate_at is None or task.next_allocate_at < scheduled_retry: + task.next_allocate_at = scheduled_retry db.commit() @@ -151,6 +153,9 @@ def run_auto_allocation_once( ) candidates = ( query.order_by( + # 失败任务会被推进到未来;按 next_allocate_at 轮转可保证队首 + # 两个永久无库存任务不会每轮都抢走全部执行槽。 + PlatformFulfillmentTask.next_allocate_at.asc(), PlatformFulfillmentTask.paid_at.asc(), PlatformFulfillmentTask.id.asc(), ) diff --git a/web/backend/services/platform_fulfillment/service.py b/web/backend/services/platform_fulfillment/service.py index 15a0b26..923bbbe 100644 --- a/web/backend/services/platform_fulfillment/service.py +++ b/web/backend/services/platform_fulfillment/service.py @@ -18,6 +18,8 @@ from starlette.concurrency import run_in_threadpool from ...models import ( Account, + AccountCommodityState, + AccountEvent, AccountLease, DouyuExternalOrder, DouyuGoodsSnapshot, @@ -45,6 +47,9 @@ from ..account_capability_service import ( classify_goods_items, record_account_commodity_state, ) +from ..account_lifecycle import MANUAL_STATES +from ..account_sync_runner import WORK_STOREHOME_SYNC +from ..account_work_queue import enqueue_account_work from ..business_events import record_account_query, record_douyu_task_started from ..douyu_account_probe import ( CAPABILITY_OK, @@ -68,7 +73,9 @@ from ..file_service import build_file_url from . import classification from .account_selector import ( AccountCandidate, + CapabilityCandidate, capability_candidate_accounts, + capability_candidate_rows, has_recent_exchange, lease_account, live_candidate_accounts, @@ -93,6 +100,9 @@ BINDING_QR_TTL_SECONDS = int( ) AUTO_SYNC_INTERVAL_SECONDS = 900.0 DEFAULT_LIVE_CANDIDATE_LIMIT = 20 +LOCAL_STOCK_RETRY_SECONDS = 5 * 60 +LOCAL_CACHE_MISS_RETRY_SECONDS = 2 * 60 +LOCAL_LEASE_CONFLICT_RETRY_SECONDS = 5 def _matching_mode() -> str: @@ -2296,6 +2306,389 @@ def upsert_mapping( return update_mapping(db, mapping_id=row.id, **values) +def _cached_capability_state( + candidate: CapabilityCandidate, + *, + commodity_id: str, +) -> DouyuLiveAccountState: + """把新鲜本地能力快照转换成统一的履约状态,不访问斗鱼官网。""" + account = candidate.account + state = candidate.state + reason = str(state.blocking_reason or "") + return DouyuLiveAccountState( + account_id=int(account.id), + account_name=account.nickname or account.username or f"#{account.id}", + checked_at=state.observed_at or _now(), + ck_verdict="valid", + ck_message="本地能力快照关联账号 CK 有效", + elite_opened=account.elite_opened, + points=account.points, + limited_ids=frozenset(), + role_change_available_at=account.role_change_available_at, + commodity_availability={commodity_id: state.availability}, + commodity_blocking_reasons=( + {commodity_id: reason} if reason else {} + ), + ) + + +def _capability_availability_counts( + db: Session, + *, + commodity_id: str, + scope: str, + now: datetime, +) -> dict[str, int]: + rows = ( + db.query(AccountCommodityState.availability, func.count(AccountCommodityState.id)) + .filter( + AccountCommodityState.platform == TARGET_PLATFORM, + AccountCommodityState.scope == scope, + AccountCommodityState.commodity_id == commodity_id, + AccountCommodityState.valid_until.isnot(None), + AccountCommodityState.valid_until > now, + ) + .group_by(AccountCommodityState.availability) + .all() + ) + return {str(availability): int(count or 0) for availability, count in rows} + + +def _local_allocation_retry_at( + db: Session, + *, + task: PlatformFulfillmentTask, + now: datetime, + availability_counts: dict[str, int], + lease_conflict: bool, +) -> tuple[datetime, str]: + """按本地状态决定下一轮时间,避免永久无库存任务每 45 秒占槽。""" + if lease_conflict: + return ( + now + timedelta(seconds=LOCAL_LEASE_CONFLICT_RETRY_SECONDS), + "lease_conflict", + ) + next_open_at = ( + db.query(func.min(AccountCommodityState.available_from)) + .filter( + AccountCommodityState.platform == TARGET_PLATFORM, + AccountCommodityState.scope == (task.scope or SCOPE_ELITE), + AccountCommodityState.commodity_id == task.commodity_id, + AccountCommodityState.availability == AVAILABILITY_NOT_OPEN, + AccountCommodityState.available_from.isnot(None), + AccountCommodityState.available_from > now, + ) + .scalar() + ) + if next_open_at is not None: + return next_open_at + timedelta(seconds=5), "commodity_not_open" + if availability_counts.get(AVAILABILITY_NOT_OPEN): + return now + timedelta(seconds=LOCAL_STOCK_RETRY_SECONDS), "commodity_not_open" + if availability_counts.get(AVAILABILITY_NO_STOCK): + return now + timedelta(seconds=LOCAL_STOCK_RETRY_SECONDS), "commodity_no_stock" + if availability_counts: + return now + timedelta(seconds=LOCAL_STOCK_RETRY_SECONDS), "no_eligible_account" + return now + timedelta(seconds=LOCAL_CACHE_MISS_RETRY_SECONDS), "capability_cache_miss" + + +def _seed_capability_refresh( + db: Session, + *, + task: PlatformFulfillmentTask, + now: datetime, +) -> int: + """能力池缺失时只入队独立同步槽,不在分配线程里请求官网。""" + try: + seed_limit = int( + os.getenv("PLATFORM_FULFILLMENT_CACHE_REFRESH_SEED_COUNT", "20") + ) + except ValueError: + seed_limit = 20 + seed_limit = max(1, min(seed_limit, 100)) + account_ids = [ + int(account_id) + for (account_id,) in ( + db.query(AccountCommodityState.account_id) + .join(Account, Account.id == AccountCommodityState.account_id) + .filter( + AccountCommodityState.platform == TARGET_PLATFORM, + AccountCommodityState.scope == (task.scope or SCOPE_ELITE), + AccountCommodityState.commodity_id == task.commodity_id, + Account.cookie.isnot(None), + Account.cookie != "", + Account.ck_check_status == "valid", + Account.auto_managed.is_(True), + Account.status.notin_(MANUAL_STATES), + ~db.query(AccountLease.id) + .filter(AccountLease.active_account_id == Account.id) + .exists(), + ) + .order_by( + AccountCommodityState.observed_at.is_(None).desc(), + AccountCommodityState.observed_at.asc(), + AccountCommodityState.account_id.asc(), + ) + .limit(seed_limit) + .all() + ) + ] + if not account_ids: + account_ids = [ + int(account.id) + for account in live_candidate_accounts( + db, + limit=seed_limit, + required_points=int(task.required_points or 0), + commodity_id=task.commodity_id, + ) + ] + for account_id in dict.fromkeys(account_ids): + enqueue_account_work( + db, + account_id=account_id, + work_type=WORK_STOREHOME_SYNC, + priority=20, + due_at=now, + reason=f"履约任务 #{task.id} 缺少商品能力快照", + payload={ + "commodity_id": task.commodity_id, + "platform_task_id": int(task.id), + }, + ) + return len(set(account_ids)) + + +def _allocate_task_from_capability( + db: Session, + *, + task: PlatformFulfillmentTask, + current: User, + mode: str, +) -> dict[str, Any]: + """只用本地能力索引选号;外部资格查询留给独立同步队列。""" + moment = _now() + required_points = int(task.required_points or 0) + try: + candidate_limit = int( + os.getenv( + "PLATFORM_FULFILLMENT_LIVE_CANDIDATE_LIMIT", + str(DEFAULT_LIVE_CANDIDATE_LIMIT), + ) + ) + except ValueError: + candidate_limit = DEFAULT_LIVE_CANDIDATE_LIMIT + candidate_limit = max(1, min(candidate_limit, 100)) + is_presale = classification.normalize_sale_mode(task.sale_mode) == "presale" + capability_candidates = capability_candidate_rows( + db, + limit=candidate_limit, + required_points=required_points, + commodity_id=task.commodity_id, + scope=task.scope or SCOPE_ELITE, + now=moment, + allow_not_open=is_presale, + ) + recent_exchange_ids: set[int] = set() + if capability_candidates: + candidate_ids = [int(item.account.id) for item in capability_candidates] + since = moment - timedelta(days=365) + recent_exchange_ids = { + int(account_id) + for (account_id,) in ( + db.query(AccountEvent.account_id) + .filter( + AccountEvent.account_id.in_(candidate_ids), + AccountEvent.event_type == "exchange", + AccountEvent.subject_key == task.commodity_id, + AccountEvent.occurred_at >= since, + ) + .distinct() + .all() + ) + } + + diagnostics: list[dict[str, Any]] = [] + qualified: list[tuple[CapabilityCandidate, DouyuLiveAccountState]] = [] + for candidate in capability_candidates: + account_id = int(candidate.account.id) + if account_id in recent_exchange_ids: + diagnostics.append( + { + "accountId": account_id, + "accountName": candidate.account.nickname + or candidate.account.username, + "reasons": ["365 天内已有该商品兑换事件"], + "source": "capability", + } + ) + continue + qualified.append( + ( + candidate, + _cached_capability_state( + candidate, + commodity_id=task.commodity_id, + ), + ) + ) + + qualified.sort( + key=lambda item: ( + item[1].readiness_rank(required_points=required_points), + int(item[1].points or 0), + int(item[0].account.id), + ) + ) + selected: CapabilityCandidate | None = None + selected_state: DouyuLiveAccountState | None = None + lease = None + lease_duration_minutes = _lease_duration_minutes(db) + for candidate, cached_state in qualified: + account = candidate.account + candidate_model = AccountCandidate( + account_id=int(account.id), + username=str(account.username or ""), + nickname=str(account.nickname or ""), + points=int(cached_state.points or 0), + gold_balance=account.gold_balance, + points_expire_at=account.points_expire_at, + role_change_available_at=account.role_change_available_at, + ) + lease = lease_account( + db, + candidate=candidate_model, + holder_user_id=current.id, + task_id=task.id, + commodity_id=task.commodity_id, + ttl_minutes=lease_duration_minutes, + commit=False, + ) + if lease is not None: + selected = candidate + selected_state = cached_state + break + diagnostics.append( + { + "accountId": int(account.id), + "accountName": cached_state.account_name, + "reasons": ["账号刚被其他任务占用"], + "source": "capability", + } + ) + + if selected is None or selected_state is None or lease is None: + availability_counts = _capability_availability_counts( + db, + commodity_id=task.commodity_id, + scope=task.scope or SCOPE_ELITE, + now=moment, + ) + retry_at, retry_reason = _local_allocation_retry_at( + db, + task=task, + now=moment, + availability_counts=availability_counts, + lease_conflict=bool(qualified), + ) + refresh_queued = ( + _seed_capability_refresh(db, task=task, now=moment) + if retry_reason == "capability_cache_miss" + else 0 + ) + if retry_reason == "commodity_not_open": + task.issue_code = "commodity_not_open" + task.last_error = "该商品尚未到开放时间,已按开放时间安排重新选号" + elif retry_reason == "commodity_no_stock": + task.issue_code = "commodity_no_stock" + task.last_error = "本地能力池显示该商品当前无库存,稍后自动重试" + elif retry_reason == "capability_cache_miss": + task.issue_code = "capability_cache_miss" + task.last_error = "暂无该商品的新鲜能力快照,等待后台补货刷新" + elif retry_reason == "lease_conflict": + task.issue_code = "account_lease_conflict" + task.last_error = "已找到候选账号,但刚被其他任务占用" + else: + task.issue_code = "no_eligible_account" + task.last_error = "本地能力池暂无可用账号" + task.execution_status = "ready" + task.next_allocate_at = retry_at + task.updated_at = moment + _append_event( + db, + task.id, + "account_allocation_failed", + detail={ + "candidateCount": len(capability_candidates), + "matchingMode": mode, + "strategy": "local_capability", + "capabilityCandidates": len(capability_candidates), + "availabilityCounts": availability_counts, + "retryAt": _jsonable(retry_at), + "retryReason": retry_reason, + "refreshQueued": refresh_queued, + "diagnostics": diagnostics[:20], + }, + ) + _emit_order_site_binding_state(db, task.id) + db.commit() + return get_task_detail(db, task.id) + + selected_account = selected.account + task.account_id = selected_account.id + task.lease_id = lease.id + task.operator_id = current.id + task.execution_status = "allocated" + task.next_allocate_at = None + task.binding_status = "not_started" + soft_reasons = selected_state.soft_requirement_reasons( + required_points=required_points + ) + gate_status, gate_issue, gate_message = _exchange_gate( + selected_state, + task.commodity_id, + allow_not_open=is_presale, + ) + if gate_status: + task.exchange_status = gate_status + task.issue_code = gate_issue + task.last_error = gate_message + elif selected_state.elite_opened is not True: + task.exchange_status = "not_opened" + task.issue_code = "elite_not_opened" + task.last_error = "账号尚未开通精英宝典,可开通后继续" + elif selected_state.points is None or selected_state.points < required_points: + task.exchange_status = "points_short" + task.issue_code = "points_short" + task.last_error = "账号积分不足,可充值后继续" + else: + task.exchange_status = "eligible" + task.issue_code = "" + task.last_error = "" + task.updated_at = moment + _apply_task_lease_policy(db, task, lease=lease) + _append_event( + db, + task.id, + "account_allocated", + actor_type="user", + actor_id=current.id, + detail={ + "accountId": selected_account.id, + "accountName": selected_state.account_name, + "points": selected_state.points, + "eliteOpened": selected_state.elite_opened, + "softReasons": soft_reasons, + "exchangeStatus": task.exchange_status, + "allocationStrategy": "local_capability", + "capabilityObservedAt": _jsonable(selected.state.observed_at), + "capabilityValidUntil": _jsonable(selected.state.valid_until), + }, + ) + _emit_order_site_binding_state(db, task.id) + db.commit() + return get_task_detail(db, task.id) + + def prepare_manual_allocation( db: Session, *, @@ -2336,6 +2729,18 @@ def allocate_task(db: Session, *, task_id: int, current: User) -> dict[str, Any] if lease and lease.status == "active": return get_task_detail(db, task.id) + mode = _matching_mode() + if mode in {"hybrid", "capability"}: + # 2026-11-22:分配热路径只读本地能力索引。官网资格刷新已经由独立 + # account_sync_queue 执行;绑码和兑换前仍有各自的实时校验,不应让 + # 一次选号串行探测 20 个账号并阻塞其他订单。 + return _allocate_task_from_capability( + db, + task=task, + current=current, + mode=mode, + ) + config = ensure_douyu_config(db) lease_duration_minutes = _lease_duration_minutes(db) manual_id = str(douyu_config_value("manual_id", config.manual_id)) @@ -2351,7 +2756,6 @@ def allocate_task(db: Session, *, task_id: int, current: User) -> dict[str, Any] except ValueError: candidate_limit = DEFAULT_LIVE_CANDIDATE_LIMIT candidate_limit = max(1, min(candidate_limit, 100)) - mode = _matching_mode() is_presale = classification.normalize_sale_mode(task.sale_mode) == "presale" capability_candidates = ( capability_candidate_accounts(