diff --git a/docs/直播账号履约工作台.md b/docs/直播账号履约工作台.md index a6e8673..395dc63 100644 --- a/docs/直播账号履约工作台.md +++ b/docs/直播账号履约工作台.md @@ -1,10 +1,10 @@ # 直播账号履约工作台 -更新时间:2026-11-17 +更新时间:2026-11-18 ## 目标 -从 `order_site` 只读获取人工渠道待履约订单,由系统选择内部斗鱼账号, +消费 `order_site` 事务 Outbox 推送的人工渠道订单项快照,由系统选择内部斗鱼账号, 客户扫码绑定游戏角色,运营人工完成精英宝典兑换。 首期边界: @@ -20,7 +20,8 @@ ## 数据边界 - `order_site` 是订单、支付、退款和人工履约状态的真源。 -- `live-hub` 只通过 `external_order_source.py` 读取订单数据库。 +- 正常建单只消费 `platform_fulfillment.order_snapshot` 事件;只读订单数据库仅用于 + 旧版事件按明确订单项 ID 定向兼容和人工显式补查。 - `live-hub` 的履约作业记录在 `platform_fulfillment_tasks`。 - 内部 SKU 与斗鱼商品必须显式配置 `platform_commodity_mappings`。 - CK 始终保存在原有账号字段中,不进入履约任务接口。 @@ -67,7 +68,7 @@ ## 操作流程 -1. 在“直播账号履约工作台”点击“同步待履约订单”。 +1. 新订单由 `order_site` 事务 Outbox 自动进入;人工“同步待履约订单”只用于历史补查。 2. 为待映射商品选择斗鱼商品和积分价格。 3. 点击“自动选号”,系统实时检查 CK、租约、365 天限兑和换绑冷却; 积分与精英宝典开通状态只影响账号排序和后续处理提示。 @@ -88,7 +89,7 @@ ## 环境 -读取 `order_site` 需要: +旧版定向兼容或人工补查 `order_site` 时才需要: ```dotenv ORDER_SITE_DATABASE_URL=postgresql://... @@ -108,7 +109,14 @@ ORDER_SITE_LOOKUP_ENABLED=true `X-Integration-Timestamp`、`X-Integration-Signature`; - 签名值为 `sha256=HMAC-SHA256(timestamp + "." + rawBody)`; - 接收方通过 `integration_inbox.event_id` 幂等,失败时整笔事务回滚; -- 15 分钟低频对账只作为 dead/漏单兜底,不再由列表请求或人工刷新承担主同步。 +- `order_site → live-hub` 的订单事件携带订单项快照、稳定去重指纹和 `contractVersion=2`; + 正常路径不回查待履约表,不存在“最近 500 条”正确性窗口; +- 旧版事件没有快照时只按明确订单项 ID 定向读取,不得退化为全表窗口; +- live-hub 新任务写入 `next_allocate_at=NOW()` 并主动唤醒选号 worker;首次选号不再等待 + 固定 45 秒,失败后才设置下一次重试时间; +- 回传 `order_site` 的快照携带单调 `stateVersion`,接收方拒绝旧版本覆盖; +- 15 分钟低频对账按任务 ID 游标分批补齐状态 Outbox,只作为 dead/漏单兜底; + 批次限制吞吐,不形成永久扫描窗口。 `order_site` 客户页通过服务端接口调用 live-hub,不允许客户浏览器直接访问 live-hub 管理 API。同步客户会话接口位于: diff --git a/tests/test_migrations.py b/tests/test_migrations.py index bba4e06..97f39db 100644 --- a/tests/test_migrations.py +++ b/tests/test_migrations.py @@ -35,7 +35,7 @@ TESTS_DIR = Path(__file__).resolve().parent PROJECT_ROOT = TESTS_DIR.parent VERSIONS_DIR = PROJECT_ROOT / "web" / "backend" / "migrations" / "versions" -HEAD_REVISION = "20261118_0088" +HEAD_REVISION = "20261118_0089" # 迁移新增、但不声明在模型里的复合查询索引。 EXTRA_INDEXES = ( diff --git a/tests/test_platform_fulfillment.py b/tests/test_platform_fulfillment.py index e7809a4..7a8f9fa 100644 --- a/tests/test_platform_fulfillment.py +++ b/tests/test_platform_fulfillment.py @@ -357,8 +357,8 @@ class TestSyncAndMapping(_Base): ) monkeypatch.setattr( service, - "sync_pending_tasks", - lambda *_args, **_kwargs: {"fetched": 0}, + "lookup_pending_manual_fulfillment", + lambda *_args, **_kwargs: None, ) with pytest.raises(HTTPException) as exc_info: @@ -397,6 +397,104 @@ class TestSyncAndMapping(_Base): assert task.required_points == 1280 +class TestSnapshotSync(_Base): + def snapshot(self, **overrides): + payload = { + "sourceOrderId": "100", + "sourceOrderItemId": "101", + "sourceFulfillmentId": "201", + "platformOrderId": "ORDER-101", + "provider": "91kaquan", + "shopId": "shop-1", + "shopName": "测试店", + "orderStatus": "paid", + "payStatus": "paid", + "skuCode": INTERNAL_SKU, + "skuName": INTERNAL_SKU, + "internalSkuKey": INTERNAL_SKU, + "quantity": 1, + "revenueAmountFen": 9900, + "orderCreatedAt": NOW.isoformat(), + "paidAt": NOW.isoformat(), + "fulfillmentStatus": "assigned", + "fulfillmentChannel": "manual", + "configEnabled": True, + "douyuFulfillmentEnabled": True, + } + payload.update(overrides) + return payload + + def test_order_snapshot_creates_task_without_external_database_read( + self, monkeypatch + ): + self.mapping() + monkeypatch.setattr( + service, + "lookup_pending_manual_fulfillment", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + AssertionError("新快照契约不应回查 order_site") + ), + ) + + result = service.sync_order_snapshot( + self.session, + { + "eventType": "platform_fulfillment.order_snapshot", + "contractVersion": 2, + "data": self.snapshot(), + }, + ) + + task = self.session.query(PlatformFulfillmentTask).one() + assert result["created"] == 1 + assert task.execution_status == "ready" + assert task.next_allocate_at is not None + assert task.state_version == 1 + outbox = self.session.query(IntegrationOutbox).one() + assert outbox.payload_json["items"][0]["stateVersion"] == 1 + + def test_legacy_event_only_reads_the_specified_order_item(self, monkeypatch): + self.mapping() + requested: list[str] = [] + + def lookup(item_id: str): + requested.append(item_id) + return self.snapshot() + + monkeypatch.setattr(service, "lookup_pending_manual_fulfillment", lookup) + + result = service.sync_order_item_ids(self.session, ["101"]) + + assert requested == ["101"] + assert result["created"] == 1 + assert self.session.query(PlatformFulfillmentTask).count() == 1 + + def test_terminal_snapshot_cancels_task_and_increments_state_version(self): + self.mapping() + service.sync_order_snapshot(self.session, {"data": self.snapshot()}) + service.sync_order_snapshot( + self.session, + {"data": self.snapshot(orderStatus="refunded")}, + ) + + task = self.session.query(PlatformFulfillmentTask).one() + assert task.execution_status == "cancelled" + assert task.state_version == 2 + assert task.next_allocate_at is None + + def test_unpaid_snapshot_does_not_enter_live_fulfillment(self): + self.mapping() + + result = service.sync_order_snapshot( + self.session, + {"data": self.snapshot(payStatus="unpaid")}, + ) + + assert result["created"] == 0 + assert result["ignored"] == 1 + assert self.session.query(PlatformFulfillmentTask).count() == 0 + + class TestAccountSelection(_Base): def test_allocation_refreshes_live_state_instead_of_cached_filters(self): account = Account( diff --git a/tests/test_platform_fulfillment_auto_allocator.py b/tests/test_platform_fulfillment_auto_allocator.py index 2ff333a..904df5c 100644 --- a/tests/test_platform_fulfillment_auto_allocator.py +++ b/tests/test_platform_fulfillment_auto_allocator.py @@ -33,7 +33,8 @@ def test_auto_allocator_only_scans_ready_unassigned_tasks(db): binding_status="not_started", exchange_status="not_started", paid_at=datetime.now(UTC), - updated_at=datetime.now(UTC) - timedelta(minutes=2), + updated_at=datetime.now(UTC), + next_allocate_at=datetime.now(UTC), ) mapped_pending = PlatformFulfillmentTask( source_system="order_site", @@ -80,3 +81,46 @@ def test_auto_allocator_only_scans_ready_unassigned_tasks(db): assert calls == [ready.id] assert result == {"checked": 1, "allocated": 1, "failed": 0} + + +def test_auto_allocator_schedules_retry_only_after_failed_allocation(db): + actor = User(username="auto-retry-actor", password_hash="h", role="super_admin") + db.add( + PlatformFulfillmentSetting( + id=1, + lease_duration_minutes=None, + order_match_started_at=datetime.now(UTC) - timedelta(days=1), + ) + ) + moment = datetime.now(UTC) + task = PlatformFulfillmentTask( + source_system="order_site", + source_order_id="10", + source_order_item_id="10", + platform_order_id="AUTO-RETRY", + internal_sku_key="skin", + commodity_id="commodity-1", + execution_status="ready", + binding_status="not_started", + exchange_status="not_started", + paid_at=moment, + next_allocate_at=moment, + updated_at=moment, + ) + db.add_all([actor, task]) + db.commit() + + with ( + patch.object(auto_allocator.service, "_integration_actor", return_value=actor), + patch.object( + auto_allocator.service, + "allocate_task", + return_value={"task": {"account": None}}, + ), + ): + result = auto_allocator.run_auto_allocation_once(db, limit=1, now=moment) + + db.refresh(task) + assert result == {"checked": 1, "allocated": 0, "failed": 1} + assert task.next_allocate_at is not None + assert task.next_allocate_at >= moment + timedelta(seconds=10) diff --git a/web/backend/migrations/versions/20261117_0085_platform_fulfillment_lease_settings.py b/web/backend/migrations/versions/20261117_0085_platform_fulfillment_lease_settings.py index e20212e..4cdc9d8 100644 --- a/web/backend/migrations/versions/20261117_0085_platform_fulfillment_lease_settings.py +++ b/web/backend/migrations/versions/20261117_0085_platform_fulfillment_lease_settings.py @@ -28,6 +28,21 @@ def _table_exists(table: str) -> bool: ) +def _column_exists(table: str, column: str) -> bool: + return bool( + op.get_bind() + .execute( + sa.text( + "SELECT COUNT(*) FROM information_schema.columns " + "WHERE table_schema = DATABASE() " + "AND table_name = :table AND column_name = :column" + ), + {"table": table, "column": column}, + ) + .scalar() + ) + + def upgrade() -> None: op.alter_column( "account_leases", @@ -50,12 +65,19 @@ def upgrade() -> None: ), sa.PrimaryKeyConstraint("id"), ) + order_match_column = ( + ", order_match_started_at" + if _column_exists("platform_fulfillment_settings", "order_match_started_at") + else "" + ) + order_match_value = ", NOW()" if order_match_column else "" op.execute( sa.text( - """ + f""" INSERT INTO platform_fulfillment_settings ( id, lease_duration_minutes, updated_by, updated_at - ) VALUES (1, NULL, NULL, NOW()) + {order_match_column} + ) VALUES (1, NULL, NULL, NOW() {order_match_value}) ON DUPLICATE KEY UPDATE id = id """ ) diff --git a/web/backend/migrations/versions/20261118_0089_platform_fulfillment_immediate_allocation.py b/web/backend/migrations/versions/20261118_0089_platform_fulfillment_immediate_allocation.py new file mode 100644 index 0000000..7cfd657 --- /dev/null +++ b/web/backend/migrations/versions/20261118_0089_platform_fulfillment_immediate_allocation.py @@ -0,0 +1,90 @@ +"""直播账号履约即时选号与状态版本 + +- 新任务通过 ``next_allocate_at`` 立即进入选号 worker; +- 失败任务由 worker 自己设置下一次重试时间,不再统一等待 45 秒; +- ``state_version`` 为回传 order_site 的状态提供单调版本,防止乱序覆盖。 +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op + +revision: str = "20261118_0089" +down_revision: str | None = "20261118_0088" +branch_labels: str | None = None +depends_on: str | None = None + +TABLE = "platform_fulfillment_tasks" +NEXT_ALLOCATE_INDEX = "ix_platform_fulfillment_next_allocate" + + +def _column_exists(column: str) -> bool: + return bool( + op.get_bind() + .execute( + sa.text( + "SELECT COUNT(*) FROM information_schema.columns " + "WHERE table_schema = DATABASE() " + "AND table_name = :table AND column_name = :column" + ), + {"table": TABLE, "column": column}, + ) + .scalar() + ) + + +def _index_exists(index: str) -> bool: + return bool( + op.get_bind() + .execute( + sa.text( + "SELECT COUNT(*) FROM information_schema.statistics " + "WHERE table_schema = DATABASE() " + "AND table_name = :table AND index_name = :index" + ), + {"table": TABLE, "index": index}, + ) + .scalar() + ) + + +def upgrade() -> None: + if not _column_exists("state_version"): + op.add_column( + TABLE, + sa.Column( + "state_version", + sa.BigInteger(), + nullable=False, + server_default=sa.text("0"), + ), + ) + if not _column_exists("next_allocate_at"): + op.add_column( + TABLE, + sa.Column("next_allocate_at", sa.DateTime(), nullable=True), + ) + op.get_bind().execute( + sa.text( + "UPDATE platform_fulfillment_tasks " + "SET next_allocate_at = UTC_TIMESTAMP() " + "WHERE execution_status IN ('ready', 'manual_required') " + "AND account_id IS NULL" + ) + ) + if not _index_exists(NEXT_ALLOCATE_INDEX): + op.create_index( + NEXT_ALLOCATE_INDEX, + TABLE, + ["execution_status", "next_allocate_at", "id"], + ) + + +def downgrade() -> None: + if _index_exists(NEXT_ALLOCATE_INDEX): + op.drop_index(NEXT_ALLOCATE_INDEX, table_name=TABLE) + if _column_exists("next_allocate_at"): + op.drop_column(TABLE, "next_allocate_at") + if _column_exists("state_version"): + op.drop_column(TABLE, "state_version") diff --git a/web/backend/models.py b/web/backend/models.py index 861b938..0d759a4 100644 --- a/web/backend/models.py +++ b/web/backend/models.py @@ -1859,6 +1859,9 @@ class PlatformFulfillmentTask(Base): issue_code: Mapped[str] = mapped_column( String(64), nullable=False, default="", server_default="" ) + state_version: Mapped[int] = mapped_column( + BigInteger, nullable=False, default=0, server_default="0" + ) last_error: Mapped[str] = mapped_column(Text, nullable=False, default="") source_snapshot_json: Mapped[dict | None] = mapped_column(JSON, nullable=True) operator_id: Mapped[int | None] = mapped_column( @@ -1868,6 +1871,9 @@ class PlatformFulfillmentTask(Base): paid_at: Mapped[datetime | None] = mapped_column( UTCDateTime, nullable=True, index=True ) + next_allocate_at: Mapped[datetime | None] = mapped_column( + UTCDateTime, nullable=True, default=_utcnow + ) created_at: Mapped[datetime] = mapped_column(UTCDateTime, default=_utcnow) updated_at: Mapped[datetime] = mapped_column( UTCDateTime, default=_utcnow, onupdate=_utcnow @@ -1889,6 +1895,12 @@ class PlatformFulfillmentTask(Base): "source_order_id", "source_order_item_id", ), + Index( + "ix_platform_fulfillment_next_allocate", + "execution_status", + "next_allocate_at", + "id", + ), ) diff --git a/web/backend/routers/platform_fulfillment.py b/web/backend/routers/platform_fulfillment.py index 016988b..9b4e606 100644 --- a/web/backend/routers/platform_fulfillment.py +++ b/web/backend/routers/platform_fulfillment.py @@ -80,6 +80,10 @@ def sync_tasks( current: User = Depends(require_permission("platform_fulfillment:use")), ) -> dict[str, Any]: result = service.sync_pending_tasks(db, limit=limit) + if result.get("taskIds"): + from ..services.platform_fulfillment.auto_allocator import wake_auto_allocator + + wake_auto_allocator(result["taskIds"]) record_audit( db, current, @@ -98,7 +102,6 @@ def get_task( current: User = Depends(require_permission("platform_fulfillment:view")), ) -> dict[str, Any]: del current - service.refresh_binding(db, task_id=task_id) return service.get_task_detail(db, task_id) diff --git a/web/backend/routers/platform_fulfillment_integration.py b/web/backend/routers/platform_fulfillment_integration.py index b149df6..1201547 100644 --- a/web/backend/routers/platform_fulfillment_integration.py +++ b/web/backend/routers/platform_fulfillment_integration.py @@ -22,7 +22,7 @@ from ..services.integration.security import ( verify_webhook_signature, webhook_tolerance_seconds, ) -from ..services.platform_fulfillment import service +from ..services.platform_fulfillment import auto_allocator, service router = APIRouter( prefix="/api/integration/platform-fulfillment", @@ -81,9 +81,27 @@ async def sync_order_event( received_at=datetime.now(UTC), ) db.add(inbox) - result = service.sync_pending_tasks(db, limit=500, commit=False) + data = payload.get("data") + snapshot = data if isinstance(data, dict) else payload + has_snapshot = bool( + str(snapshot.get("sourceOrderItemId") or "").strip() + and str(snapshot.get("internalSkuKey") or "").strip() + ) + if has_snapshot: + result = service.sync_order_snapshot(db, payload, commit=False) + else: + item_ids = [ + str(value).strip() + for value in ( + snapshot.get("sourceOrderItemId"), + snapshot.get("orderItemId"), + ) + if str(value or "").strip() + ] + result = service.sync_order_item_ids(db, item_ids, commit=False) inbox.processed_at = datetime.now(UTC) db.commit() + auto_allocator.wake_auto_allocator(result.get("taskIds", [])) return {**result, "duplicate": False} diff --git a/web/backend/services/external_order_source.py b/web/backend/services/external_order_source.py index 837604a..8f348c1 100644 --- a/web/backend/services/external_order_source.py +++ b/web/backend/services/external_order_source.py @@ -257,43 +257,98 @@ def list_pending_manual_fulfillments( ) if not sku_keys: return [] - result: list[dict[str, Any]] = [] - for row in _fetch_pending_manual_fulfillments( - limit=limit, - internal_sku_keys=sku_keys, - paid_at_from=paid_at_from, - ): - internal_sku_key = str( - row.get("manual_sku_key") - or row.get("sku_name") - or row.get("sku_code") - or "" - ).strip() - result.append( - { - "source_system": PLATFORM_FULFILLMENT_SOURCE, - "source_order_id": str(row.get("source_order_id") or ""), - "source_order_item_id": str(row.get("source_order_item_id") or ""), - "source_fulfillment_id": str(row.get("source_fulfillment_id") or ""), - "platform_order_id": str(row.get("platform_order_id") or ""), - "provider": str(row.get("provider") or ""), - "platform": str(row.get("platform") or ""), - "shop_id": str(row.get("shop_id") or ""), - "shop_name": str(row.get("shop_name") or ""), - "order_status": str(row.get("order_status") or ""), - "pay_status": str(row.get("pay_status") or ""), - "total_amount": int(row.get("total_amount") or 0), - "revenue_amount_fen": int(row.get("revenue_amount_fen") or 0), - "paid_at": row.get("paid_at"), - "sku_code": str(row.get("sku_code") or ""), - "sku_name": str(row.get("sku_name") or ""), - "internal_sku_key": internal_sku_key, - "quantity": int(row.get("quantity") or 1), - "spec_json": row.get("spec_json"), - "item_updated_at": row.get("item_updated_at"), - } + return [ + _normalize_pending_row(row) + for row in _fetch_pending_manual_fulfillments( + limit=limit, + internal_sku_keys=sku_keys, + paid_at_from=paid_at_from, ) - return result + ] + + +def _normalize_pending_row(row: dict[str, Any]) -> dict[str, Any]: + internal_sku_key = str( + row.get("manual_sku_key") or row.get("sku_name") or row.get("sku_code") or "" + ).strip() + return { + "source_system": PLATFORM_FULFILLMENT_SOURCE, + "source_order_id": str(row.get("source_order_id") or ""), + "source_order_item_id": str(row.get("source_order_item_id") or ""), + "source_fulfillment_id": str(row.get("source_fulfillment_id") or ""), + "platform_order_id": str(row.get("platform_order_id") or ""), + "provider": str(row.get("provider") or ""), + "platform": str(row.get("platform") or ""), + "shop_id": str(row.get("shop_id") or ""), + "shop_name": str(row.get("shop_name") or ""), + "order_status": str(row.get("order_status") or ""), + "pay_status": str(row.get("pay_status") or ""), + "total_amount": int(row.get("total_amount") or 0), + "revenue_amount_fen": int(row.get("revenue_amount_fen") or 0), + "paid_at": row.get("paid_at"), + "sku_code": str(row.get("sku_code") or ""), + "sku_name": str(row.get("sku_name") or ""), + "internal_sku_key": internal_sku_key, + "quantity": int(row.get("quantity") or 1), + "spec_json": row.get("spec_json"), + "item_updated_at": row.get("item_updated_at"), + } + + +def lookup_pending_manual_fulfillment( + source_order_item_id: str, +) -> dict[str, Any] | None: + """按订单项定向读取旧版事件缺失的快照,不再拉取最近 500 条。""" + item_id = str(source_order_item_id or "").strip() + if not item_id: + return None + if not _enabled(): + raise OrderLookupUnavailable("外部订单查询未启用") + + sql = text( + """ + SELECT o.id AS source_order_id, + o.platform_order_id, + o.provider, + o.platform, + o.shop_id, + o.shop_name, + o.order_status, + o.pay_status, + o.total_amount, + o.paid_at, + o.updated_at AS order_updated_at, + oi.id AS source_order_item_id, + oi.sku_code, + oi.sku_name, + oi.quantity, + oi.revenue_amount_fen, + oi.manual_sku_key, + oi.spec_json, + oi.updated_at AS item_updated_at, + f.id AS source_fulfillment_id, + f.channel AS fulfillment_channel, + f.status AS fulfillment_status, + f.issue_code AS fulfillment_issue_code + FROM orders o + JOIN order_items oi ON oi.order_id = o.id + JOIN fulfillments f ON f.order_item_id = oi.id + WHERE oi.id = :source_order_item_id + AND f.channel = 'manual' + LIMIT 1 + """ + ) + try: + with _get_engine().connect() as conn: + row = ( + conn.execute(sql, {"source_order_item_id": int(item_id)}) + .mappings() + .first() + ) + except Exception as exc: + logger.warning(f"定向查询 order_site 订单项失败({item_id}):{exc}") + raise + return _normalize_pending_row(dict(row)) if row is not None else None def _route_of(rows: list[dict[str, Any]]) -> str: diff --git a/web/backend/services/platform_fulfillment/auto_allocator.py b/web/backend/services/platform_fulfillment/auto_allocator.py index f1c2e4f..71eea09 100644 --- a/web/backend/services/platform_fulfillment/auto_allocator.py +++ b/web/backend/services/platform_fulfillment/auto_allocator.py @@ -4,10 +4,12 @@ from __future__ import annotations import os import threading +from collections.abc import Iterable from datetime import UTC, datetime, timedelta from typing import Any from loguru import logger +from sqlalchemy import or_ from sqlalchemy.orm import Session from ...database import SessionLocal @@ -21,6 +23,9 @@ DEFAULT_RETRY_SECONDS = 45 _stop_event = threading.Event() _thread: threading.Thread | None = None _lock = threading.Lock() +_pending_lock = threading.Lock() +_pending_task_ids: set[int] = set() +_wake_event = threading.Event() def _env_int(name: str, default: int, minimum: int, maximum: int) -> int: @@ -31,13 +36,50 @@ def _env_int(name: str, default: int, minimum: int, maximum: int) -> int: return max(minimum, min(maximum, value)) +def wake_auto_allocator(task_ids: Iterable[int] | None = None) -> None: + """立即唤醒选号 worker;传入任务 ID 时优先处理这些新任务。""" + normalized = {int(task_id) for task_id in (task_ids or []) if int(task_id) > 0} + if normalized: + with _pending_lock: + _pending_task_ids.update(normalized) + _wake_event.set() + + +def _take_pending_task_ids(limit: int) -> list[int]: + with _pending_lock: + selected = sorted(_pending_task_ids)[:limit] + for task_id in selected: + _pending_task_ids.discard(task_id) + return selected + + +def _has_pending_task_ids() -> bool: + with _pending_lock: + return bool(_pending_task_ids) + + +def _schedule_retry( + db: Session, + *, + task_id: int, + moment: datetime, + retry_seconds: int, +) -> None: + task = db.get(PlatformFulfillmentTask, int(task_id)) + if task is None or task.execution_status in {"completed", "cancelled"}: + return + task.next_allocate_at = moment + timedelta(seconds=retry_seconds) + db.commit() + + def run_auto_allocation_once( db: Session, *, limit: int | None = None, now: datetime | None = None, + task_ids: list[int] | None = None, ) -> dict[str, int]: - """扫描 ready 且尚未分配账号的任务,逐笔调用统一选号逻辑。""" + """领取已到期的 ready 任务;新任务不再等待固定 45 秒。""" moment = now or datetime.now(UTC) batch_size = limit or _env_int( "PLATFORM_FULFILLMENT_AUTO_ALLOCATE_BATCH", @@ -51,18 +93,34 @@ def run_auto_allocation_once( 10, 3600, ) + # MySQL DATETIME 默认只保存到秒,写入时会按微秒四舍五入; + # 给 1 秒容差,避免刚写入的 next_allocate_at 被 worker 当成“未来任务”。 + due_cutoff = moment + timedelta(seconds=1) query = db.query(PlatformFulfillmentTask).filter( PlatformFulfillmentTask.execution_status == "ready", PlatformFulfillmentTask.account_id.is_(None), PlatformFulfillmentTask.commodity_id != "", - PlatformFulfillmentTask.updated_at <= moment - timedelta(seconds=retry_seconds), + or_( + PlatformFulfillmentTask.next_allocate_at.is_(None), + PlatformFulfillmentTask.next_allocate_at <= due_cutoff, + ), ) + normalized_ids = [ + int(task_id) for task_id in dict.fromkeys(task_ids or []) if int(task_id) > 0 + ] + if normalized_ids: + query = query.filter(PlatformFulfillmentTask.id.in_(normalized_ids)) query = service._apply_order_match_cutoff( query, service._order_match_started_at(db), ) candidates = ( - query.order_by(PlatformFulfillmentTask.updated_at.asc()).limit(batch_size).all() + query.order_by( + PlatformFulfillmentTask.next_allocate_at.asc(), + PlatformFulfillmentTask.id.asc(), + ) + .limit(batch_size) + .all() ) if not candidates: return {"checked": 0, "allocated": 0, "failed": 0} @@ -71,20 +129,41 @@ def run_auto_allocation_once( allocated = 0 failed = 0 for task in candidates: + candidate_task_id = int(task.id) try: detail: dict[str, Any] = service.allocate_task( db, - task_id=int(task.id), + task_id=candidate_task_id, current=actor, ) if detail.get("task", {}).get("account"): allocated += 1 + allocated_task = db.get(PlatformFulfillmentTask, candidate_task_id) + if allocated_task is not None: + allocated_task.next_allocate_at = None + db.commit() else: failed += 1 + _schedule_retry( + db, + task_id=candidate_task_id, + moment=moment, + retry_seconds=retry_seconds, + ) except Exception as exc: # noqa: BLE001 - 单个任务异常不能中断整批 db.rollback() failed += 1 - logger.warning("直播账号履约自动选号失败 task_id={}: {}", task.id, exc) + _schedule_retry( + db, + task_id=candidate_task_id, + moment=moment, + retry_seconds=retry_seconds, + ) + logger.warning( + "直播账号履约自动选号失败 task_id={}: {}", + candidate_task_id, + exc, + ) return { "checked": len(candidates), "allocated": allocated, @@ -99,10 +178,18 @@ def _run() -> None: 5, 3600, ) - while not _stop_event.wait(interval): + while not _stop_event.is_set(): db = SessionLocal() try: - result = run_auto_allocation_once(db) + pending_ids = _take_pending_task_ids( + _env_int( + "PLATFORM_FULFILLMENT_AUTO_ALLOCATE_BATCH", + DEFAULT_BATCH_SIZE, + 1, + 20, + ) + ) + result = run_auto_allocation_once(db, task_ids=pending_ids or None) if result["checked"]: logger.info("直播账号履约自动选号: {}", result) except Exception as exc: # noqa: BLE001 - worker 不能因单轮异常退出 @@ -110,6 +197,12 @@ def _run() -> None: logger.warning("直播账号履约自动选号轮次失败: {}", exc) finally: db.close() + if _stop_event.is_set(): + break + if _has_pending_task_ids(): + continue + _wake_event.wait(interval) + _wake_event.clear() def start_platform_fulfillment_auto_allocator() -> bool: @@ -118,6 +211,9 @@ def start_platform_fulfillment_auto_allocator() -> bool: if _thread is not None and _thread.is_alive(): return False _stop_event.clear() + _wake_event.clear() + with _pending_lock: + _pending_task_ids.clear() _thread = threading.Thread( target=_run, name="platform-fulfillment-auto-allocator", @@ -130,6 +226,7 @@ def start_platform_fulfillment_auto_allocator() -> bool: def stop_platform_fulfillment_auto_allocator() -> None: global _thread _stop_event.set() + _wake_event.set() with _lock: thread = _thread _thread = None diff --git a/web/backend/services/platform_fulfillment/reconciler.py b/web/backend/services/platform_fulfillment/reconciler.py index 2033cc0..fba2f44 100644 --- a/web/backend/services/platform_fulfillment/reconciler.py +++ b/web/backend/services/platform_fulfillment/reconciler.py @@ -1,4 +1,4 @@ -"""低频订单对账兜底;正常同步由 order_site 签名 Webhook 驱动。""" +"""低频状态对账兜底;正常订单同步完全由 order_site 事务 Outbox 驱动。""" from __future__ import annotations @@ -12,6 +12,7 @@ from ...database import SessionLocal _stop_event = threading.Event() _thread: threading.Thread | None = None _lock = threading.Lock() +_LAST_TASK_ID = 0 def _interval_seconds() -> int: @@ -23,6 +24,7 @@ def _interval_seconds() -> int: def _run() -> None: + global _LAST_TASK_ID from . import service while not _stop_event.is_set(): @@ -31,11 +33,20 @@ def _run() -> None: break db = SessionLocal() try: - result = service.sync_pending_tasks(db, limit=500) - queued = service.reconcile_binding_state_outbox(db, limit=500) + queued = 0 + for _page in range(20): + result = service.reconcile_binding_state_outbox( + db, + limit=500, + after_task_id=_LAST_TASK_ID, + ) + queued += int(result["queued"]) + _LAST_TASK_ID = int(result["nextTaskId"]) + if not result["hasMore"]: + _LAST_TASK_ID = 0 + break logger.info( - "直播账号履约低频对账完成: order_sync={} outbox_requeued={}", - result, + "直播账号履约低频对账完成: outbox_requeued={}", queued, ) except Exception as exc: # noqa: BLE001 - 对账失败等待下一轮 diff --git a/web/backend/services/platform_fulfillment/service.py b/web/backend/services/platform_fulfillment/service.py index 5499075..588b731 100644 --- a/web/backend/services/platform_fulfillment/service.py +++ b/web/backend/services/platform_fulfillment/service.py @@ -35,6 +35,7 @@ from ..douyu_service import douyu_config_value, ensure_douyu_config from ..external_order_source import ( OrderLookupUnavailable, list_pending_manual_fulfillments, + lookup_pending_manual_fulfillment, ) from .account_selector import ( AccountCandidate, @@ -97,6 +98,360 @@ def _mapping_for_sku( ) +def _parse_snapshot_datetime(value: Any) -> datetime | None: + if isinstance(value, datetime): + return value if value.tzinfo is not None else value.replace(tzinfo=UTC) + text = str(value or "").strip() + if not text: + return None + try: + parsed = datetime.fromisoformat(text) + except ValueError: + return None + return parsed if parsed.tzinfo is not None else parsed.replace(tzinfo=UTC) + + +def _snapshot_value(data: dict[str, Any], *names: str, default: Any = "") -> Any: + for name in names: + if name in data and data[name] is not None: + return data[name] + return default + + +def _order_snapshot_from_event(payload: dict[str, Any]) -> dict[str, Any]: + data = payload.get("data") + return data if isinstance(data, dict) else payload + + +def _upsert_order_snapshot( + db: Session, + snapshot: dict[str, Any], + *, + mapping: PlatformCommodityMapping | None, +) -> tuple[PlatformFulfillmentTask | None, bool, bool]: + """应用一条订单项快照,返回 (task, created, ignored)。""" + item_id = str( + _snapshot_value(snapshot, "sourceOrderItemId", "source_order_item_id", default="") + ).strip() + if not item_id: + return None, False, True + + paid_at = _parse_snapshot_datetime( + _snapshot_value(snapshot, "paidAt", "paid_at", default=None) + ) + order_created_at = _parse_snapshot_datetime( + _snapshot_value( + snapshot, + "orderCreatedAt", + "order_created_at", + default=None, + ) + ) + order_status = str( + _snapshot_value(snapshot, "orderStatus", "order_status", default="") + ) + pay_status = str( + _snapshot_value(snapshot, "payStatus", "pay_status", default="paid") + ) + terminal_order = order_status in {"refunded", "cancelled", "closed"} + if not terminal_order and pay_status != "paid": + return None, False, True + cutoff = _order_match_started_at(db) + match_basis_at = paid_at or order_created_at + if ( + match_basis_at is not None + and cutoff is not None + and match_basis_at < cutoff + ): + return None, False, True + + task = ( + db.query(PlatformFulfillmentTask) + .filter( + PlatformFulfillmentTask.source_system == "order_site", + PlatformFulfillmentTask.source_order_item_id == item_id, + ) + .first() + ) + created = task is None + if task is None: + task = PlatformFulfillmentTask( + source_system="order_site", + source_order_id=str( + _snapshot_value( + snapshot, + "sourceOrderId", + "source_order_id", + "orderId", + default="", + ) + ), + source_order_item_id=item_id, + source_fulfillment_id=str( + _snapshot_value( + snapshot, + "sourceFulfillmentId", + "source_fulfillment_id", + "fulfillmentId", + default="", + ) + ), + target_platform=TARGET_PLATFORM, + scope=SCOPE_ELITE, + created_at=_now(), + updated_at=_now(), + ) + db.add(task) + db.flush() + _append_event( + db, + task.id, + "task_created", + detail={ + "platformOrderId": _snapshot_value( + snapshot, "platformOrderId", "platform_order_id", default="" + ) + }, + ) + + before = ( + task.platform_order_id, + task.source_order_id, + task.source_fulfillment_id, + task.provider, + task.shop_id, + task.shop_name, + task.sku_code, + task.sku_name, + task.internal_sku_key, + task.quantity, + task.order_amount_fen, + task.paid_at, + task.execution_status, + task.commodity_mapping_id, + task.commodity_id, + task.required_points, + task.issue_code, + ) + if task.execution_status in {"completed", "cancelled"}: + task.source_snapshot_json = _jsonable(snapshot) + return task, created, False + + terminal = ( + terminal_order + or _snapshot_value( + snapshot, + "fulfillmentStatus", + "fulfillment_status", + default="", + ) + == "archived" + or _snapshot_value( + snapshot, + "configEnabled", + "config_enabled", + default=True, + ) + is False + or _snapshot_value( + snapshot, + "douyuFulfillmentEnabled", + "douyu_fulfillment_enabled", + default=True, + ) + is False + ) + + task.platform_order_id = str( + _snapshot_value(snapshot, "platformOrderId", "platform_order_id", default="") + ) + task.source_order_id = str( + _snapshot_value( + snapshot, + "sourceOrderId", + "source_order_id", + "orderId", + default=task.source_order_id, + ) + ) + task.source_fulfillment_id = str( + _snapshot_value( + snapshot, + "sourceFulfillmentId", + "source_fulfillment_id", + "fulfillmentId", + default=task.source_fulfillment_id, + ) + ) + task.provider = str( + _snapshot_value(snapshot, "provider", default=task.provider) + ) + task.shop_id = str( + _snapshot_value(snapshot, "shopId", "shop_id", default="") + ) + task.shop_name = str( + _snapshot_value(snapshot, "shopName", "shop_name", default="") + ) + task.sku_code = str( + _snapshot_value(snapshot, "skuCode", "sku_code", default="") + ) + task.sku_name = str( + _snapshot_value(snapshot, "skuName", "sku_name", default="") + ) + task.internal_sku_key = str( + _snapshot_value( + snapshot, + "internalSkuKey", + "internal_sku_key", + default="", + ) + ).strip() + task.quantity = max( + 1, + int(_snapshot_value(snapshot, "quantity", default=1) or 1), + ) + task.order_amount_fen = int( + _snapshot_value( + snapshot, + "revenueAmountFen", + "revenue_amount_fen", + "totalAmountFen", + "total_amount", + default=0, + ) + or 0 + ) + task.paid_at = paid_at + task.source_snapshot_json = _jsonable(snapshot) + task.updated_at = _now() + + if terminal: + task.execution_status = "cancelled" + task.issue_code = "source_cancelled" + task.last_error = "订单已关闭或直播履约配置已停用" + task.next_allocate_at = None + else: + _apply_mapping(task, mapping) + if task.execution_status == "ready" and task.account_id is None: + task.next_allocate_at = _now() + elif task.account_id is not None: + task.next_allocate_at = None + + after = ( + task.platform_order_id, + task.source_order_id, + task.source_fulfillment_id, + task.provider, + task.shop_id, + task.shop_name, + task.sku_code, + task.sku_name, + task.internal_sku_key, + task.quantity, + task.order_amount_fen, + task.paid_at, + task.execution_status, + task.commodity_mapping_id, + task.commodity_id, + task.required_points, + task.issue_code, + ) + changed = created or before != after + if changed: + _emit_order_site_binding_state(db, task.id) + return task, created, False + + +def _mapping_map(db: Session) -> dict[str, PlatformCommodityMapping]: + rows = ( + db.query(PlatformCommodityMapping) + .filter( + PlatformCommodityMapping.target_platform == TARGET_PLATFORM, + PlatformCommodityMapping.enabled.is_(True), + ) + .all() + ) + return { + str(row.internal_sku_key or "").strip(): row + for row in rows + if str(row.internal_sku_key or "").strip() + } + + +def sync_order_snapshot( + db: Session, + payload: dict[str, Any], + *, + commit: bool = True, +) -> dict[str, Any]: + """消费 order_site 订单项快照;正常路径不再读取 order_site 数据库。""" + snapshot = _order_snapshot_from_event(payload) + mapping_by_sku = _mapping_map(db) + mapping = mapping_by_sku.get( + str( + _snapshot_value( + snapshot, + "internalSkuKey", + "internal_sku_key", + default="", + ) + ).strip() + ) + task, created, ignored = _upsert_order_snapshot(db, snapshot, mapping=mapping) + if commit: + db.commit() + return { + "fetched": 1, + "created": int(created), + "updated": int(task is not None and not created), + "ignored": int(ignored), + "taskIds": [int(task.id)] if task is not None else [], + } + + +def sync_order_item_ids( + db: Session, + source_order_item_ids: list[str], + *, + commit: bool = True, +) -> dict[str, Any]: + """兼容旧版事件:只定向读取明确订单项,不再走最近 500 条窗口。""" + mapping_by_sku = _mapping_map(db) + created = 0 + updated = 0 + ignored = 0 + task_ids: list[int] = [] + for item_id in dict.fromkeys(str(item or "").strip() for item in source_order_item_ids): + if not item_id: + ignored += 1 + continue + snapshot = lookup_pending_manual_fulfillment(item_id) + if snapshot is None: + ignored += 1 + continue + mapping = mapping_by_sku.get(str(snapshot.get("internal_sku_key") or "").strip()) + task, was_created, was_ignored = _upsert_order_snapshot( + db, + snapshot, + mapping=mapping, + ) + if task is None: + ignored += int(was_ignored) + continue + created += int(was_created) + updated += int(not was_created) + task_ids.append(int(task.id)) + if commit: + db.commit() + return { + "fetched": len(task_ids), + "created": created, + "updated": updated, + "ignored": ignored, + "taskIds": task_ids, + } + + def get_settings(db: Session) -> dict[str, Any]: now = _now() row = ( @@ -272,13 +627,25 @@ def _emit_order_site_binding_state( extra: dict[str, Any] | None = None, ) -> bool: """当前事务内写 Outbox;HTTP 投递由独立 worker 完成。""" + task = ( + db.query(PlatformFulfillmentTask) + .filter(PlatformFulfillmentTask.id == int(task_id)) + .with_for_update() + .one() + ) + next_state_version = int(task.state_version or 0) + 1 payload = customer_binding_out(db, task_id) + payload["stateVersion"] = next_state_version if extra: payload.update(extra) - return enqueue_order_site_binding_state( + enqueued = enqueue_order_site_binding_state( db, payload, ) + if enqueued: + task.state_version = next_state_version + db.flush() + return enqueued def _load_task(db: Session, task_id: int) -> PlatformFulfillmentTask: @@ -355,6 +722,7 @@ def _sync_pending_tasks( created = 0 updated = 0 skipped_unmapped = 0 + task_ids: list[int] = [] for row in rows: item_id = str(row.get("source_order_item_id") or "") if not item_id: @@ -414,7 +782,10 @@ def _sync_pending_tasks( task.updated_at = _now() if task.execution_status in {"pending_mapping", "ready"}: _apply_mapping(task, mapping) + if task.execution_status == "ready" and task.account_id is None: + task.next_allocate_at = _now() _emit_order_site_binding_state(db, task.id) + task_ids.append(int(task.id)) if commit: db.commit() @@ -425,6 +796,7 @@ def _sync_pending_tasks( "skippedUnmapped": skipped_unmapped, "configuredSkuCount": len(mappings_by_sku), "orderMatchStartedAt": _jsonable(match_started_at), + "taskIds": task_ids, } @@ -723,6 +1095,7 @@ def customer_binding_out(db: Session, task_id: int) -> dict[str, Any]: "platName": str(bind_result.get("plat_name") or ""), "bindReadyForConfirm": (bool(pending_role) and not binding_failed), "bindConfirmed": bool(bind_result.get("bind_confirmed")) and binding_confirmed, + "stateVersion": int(task.state_version or 0), "completedAt": _jsonable(task.completed_at), "updatedAt": _jsonable(task.updated_at), } @@ -741,7 +1114,11 @@ def ensure_customer_session( platform_order_id=platform_order_id, ) if task is None: - sync_pending_tasks(db, limit=500) + sync_order_item_ids( + db, + [source_order_item_id], + commit=True, + ) task = _find_customer_task( db, source_order_item_id=source_order_item_id, @@ -891,9 +1268,15 @@ def upsert_mapping( ) pending = _apply_order_match_cutoff(pending, _order_match_started_at(db)) pending = pending.all() + task_ids: list[int] = [] for task in pending: _apply_mapping(task, row if row.enabled else None) task.updated_at = _now() + task.next_allocate_at = ( + _now() + if task.execution_status == "ready" and task.account_id is None + else None + ) _append_event( db, task.id, @@ -902,7 +1285,13 @@ def upsert_mapping( actor_id=current.id, detail={"commodityId": row.commodity_id, "enabled": row.enabled}, ) + _emit_order_site_binding_state(db, task.id) + task_ids.append(int(task.id)) db.commit() + if task_ids: + from .auto_allocator import wake_auto_allocator + + wake_auto_allocator(task_ids) return next( item for item in list_mappings(db, keyword=sku_key) if item["id"] == row.id ) @@ -1019,6 +1408,7 @@ def allocate_task(db: Session, *, task_id: int, current: User) -> dict[str, Any] ) if selected is None or selected_state is None or lease is None: task.execution_status = "ready" + task.next_allocate_at = _now() + timedelta(seconds=45) task.issue_code = "no_eligible_account" if not candidates: task.last_error = "没有带 CK、允许自动管理且未停用的斗鱼账号" @@ -1049,6 +1439,7 @@ def allocate_task(db: Session, *, task_id: int, current: User) -> dict[str, Any] task.lease_id = lease.id task.operator_id = current.id task.execution_status = "allocated" + task.next_allocate_at = None task.binding_status = "not_started" soft_reasons = selected_state.soft_requirement_reasons( required_points=required_points @@ -1390,19 +1781,27 @@ def sync_task_from_binding_runner(db: Session, *, douyu_task_id: int) -> int: return len(task_ids) -def reconcile_binding_state_outbox(db: Session, *, limit: int = 500) -> int: - """低频补齐 active 任务当前快照;只会在缺失或 dead 时重新入队。""" +def reconcile_binding_state_outbox( + db: Session, + *, + limit: int = 500, + after_task_id: int = 0, +) -> dict[str, Any]: + """按 ID 游标补齐 active 快照;批次只限制吞吐,不形成永久扫描窗口。""" + batch_size = max(1, min(limit, 500)) + normalized_after = max(0, int(after_task_id or 0)) task_ids = [ int(row.id) for row in ( db.query(PlatformFulfillmentTask.id) .filter( + PlatformFulfillmentTask.id > normalized_after, PlatformFulfillmentTask.execution_status.notin_( ["completed", "cancelled"] ) ) - .order_by(PlatformFulfillmentTask.updated_at.asc()) - .limit(max(1, min(limit, 500))) + .order_by(PlatformFulfillmentTask.id.asc()) + .limit(batch_size) .all() ) ] @@ -1410,7 +1809,11 @@ def reconcile_binding_state_outbox(db: Session, *, limit: int = 500) -> int: 1 for task_id in task_ids if _emit_order_site_binding_state(db, task_id) ) db.commit() - return queued + return { + "queued": queued, + "nextTaskId": max(task_ids, default=normalized_after), + "hasMore": len(task_ids) == batch_size, + } async def confirm_binding( @@ -1705,6 +2108,8 @@ __all__ = [ "release_task_account", "reset_task_binding", "start_binding", + "sync_order_item_ids", + "sync_order_snapshot", "sync_pending_tasks", "sync_pending_tasks_if_due", "sync_task_from_binding_runner",