diff --git a/.env.example b/.env.example index 444d5d0..2e6b764 100644 --- a/.env.example +++ b/.env.example @@ -150,6 +150,13 @@ ORDER_SITE_LOOKUP_TIMEOUT_SECONDS=3 PLATFORM_FULFILLMENT_INTEGRATION_KEY= # live-hub -> order_site 状态回调地址;生产示例 http://172.23.3.80 ORDER_SITE_PLATFORM_FULFILLMENT_URL= +# 签名 Webhook 允许的时钟偏差;Outbox 投递 worker 的轮询与重试参数 +PLATFORM_FULFILLMENT_WEBHOOK_TOLERANCE_SECONDS=300 +INTEGRATION_OUTBOX_INTERVAL_SECONDS=1 +INTEGRATION_OUTBOX_BATCH_SIZE=20 +INTEGRATION_OUTBOX_MAX_ATTEMPTS=12 +# 低频对账兜底;正常同步由签名 Webhook 驱动(默认 15 分钟) +PLATFORM_FULFILLMENT_RECONCILE_SECONDS=900 # 可选:指定集成操作人;不填时使用最早创建的有效超级管理员 # PLATFORM_FULFILLMENT_INTEGRATION_ACTOR_ID=1 # 客户绑定二维码有效期 diff --git a/docs/直播账号履约工作台.md b/docs/直播账号履约工作台.md index 4592db1..a6e8673 100644 --- a/docs/直播账号履约工作台.md +++ b/docs/直播账号履约工作台.md @@ -100,14 +100,25 @@ ORDER_SITE_LOOKUP_ENABLED=true ## order_site 客户领取集成 +跨项目异步状态统一采用“事务 Outbox + 签名 Webhook”: + +- 业务状态与待发送事件在同一个数据库事务内写入 `integration_outbox`; +- 独立 worker 使用 `FOR UPDATE SKIP LOCKED` 领取事件并指数退避重试; +- 请求携带 `X-Integration-Key`、`X-Integration-Event-Id`、 + `X-Integration-Timestamp`、`X-Integration-Signature`; +- 签名值为 `sha256=HMAC-SHA256(timestamp + "." + rawBody)`; +- 接收方通过 `integration_inbox.event_id` 幂等,失败时整笔事务回滚; +- 15 分钟低频对账只作为 dead/漏单兜底,不再由列表请求或人工刷新承担主同步。 + `order_site` 客户页通过服务端接口调用 live-hub,不允许客户浏览器直接访问 -live-hub 管理 API。内部接口统一位于: +live-hub 管理 API。同步客户会话接口位于: ```text /api/integration/platform-fulfillment/customer-sessions/* ``` -接口使用 `X-Integration-Key` 鉴权,密钥来自 +请求响应接口使用 `X-Integration-Key` 鉴权,异步事件在密钥之外还必须通过 +时间窗与 HMAC 验签,密钥来自 `PLATFORM_FULFILLMENT_INTEGRATION_KEY`。返回给 `order_site` 的客户会话只包含 二维码、二维码有效期、绑定阶段、平台识别角色和状态提示,不包含账号、CK、 积分、租约或限兑信息。 @@ -124,7 +135,6 @@ live-hub 创建新的绑定任务。 ## 暂不实现 -- `order_site` 状态回写接口 - 自动充值鱼翅 - 自动兑换 - 多数量拆分 diff --git a/tests/test_integration_outbox.py b/tests/test_integration_outbox.py new file mode 100644 index 0000000..a94ba0a --- /dev/null +++ b/tests/test_integration_outbox.py @@ -0,0 +1,87 @@ +"""事务 Outbox 与签名 Webhook 基础行为。""" + +from __future__ import annotations + +from web.backend.models import IntegrationOutbox +from web.backend.services.integration.outbox import ( + enqueue_integration_event, + outbox_backoff_seconds, +) +from web.backend.services.integration.security import ( + build_webhook_signature, + verify_webhook_signature, +) + + +def test_webhook_signature_binds_timestamp_and_body(): + raw_body = b'{"eventId":"evt-1","items":[]}' + timestamp = "1783394219" + signature = build_webhook_signature("secret-key", timestamp, raw_body) + + assert verify_webhook_signature( + secret="secret-key", + timestamp=timestamp, + raw_body=raw_body, + signature=signature, + now_seconds=int(timestamp), + ) + assert not verify_webhook_signature( + secret="secret-key", + timestamp=timestamp, + raw_body=raw_body + b" ", + signature=signature, + now_seconds=int(timestamp), + ) + + +def test_webhook_signature_rejects_expired_timestamp(): + timestamp = "1783394219" + raw_body = b"{}" + signature = build_webhook_signature("secret-key", timestamp, raw_body) + + assert not verify_webhook_signature( + secret="secret-key", + timestamp=timestamp, + raw_body=raw_body, + signature=signature, + tolerance_seconds=300, + now_seconds=int(timestamp) + 301, + ) + + +def test_outbox_enqueue_is_deduplicated_and_backoff_is_bounded(db): + assert enqueue_integration_event( + db, + destination="order_site", + event_type="platform_fulfillment.binding_state_changed", + payload={"items": [{"sessionId": 1}]}, + dedupe_key="binding-state:1:same", + ) + assert not enqueue_integration_event( + db, + destination="order_site", + event_type="platform_fulfillment.binding_state_changed", + payload={"items": [{"sessionId": 1}]}, + dedupe_key="binding-state:1:same", + ) + db.commit() + + row = db.query(IntegrationOutbox).one() + assert row.status == "pending" + assert row.payload_json["eventId"] == row.event_id + + row.status = "dead" + db.commit() + assert enqueue_integration_event( + db, + destination="order_site", + event_type="platform_fulfillment.binding_state_changed", + payload={"items": [{"sessionId": 1}]}, + dedupe_key="binding-state:1:same", + ) + db.commit() + db.refresh(row) + assert row.status == "pending" + assert row.attempts == 0 + assert outbox_backoff_seconds(1) == 5 + assert outbox_backoff_seconds(99) == 3600 diff --git a/tests/test_platform_fulfillment.py b/tests/test_platform_fulfillment.py index 88f779a..943c0ad 100644 --- a/tests/test_platform_fulfillment.py +++ b/tests/test_platform_fulfillment.py @@ -468,8 +468,8 @@ class TestBindingAndCompletion(_Base): notified: list[int] = [] monkeypatch.setattr( service, - "notify_order_site_binding_state_async", - lambda task_id: notified.append(task_id), + "_emit_order_site_binding_state", + lambda _db, task_id: notified.append(task_id) or True, ) task = self._allocated() notified.clear() diff --git a/tests/test_platform_fulfillment_integration.py b/tests/test_platform_fulfillment_integration.py index 5edba73..973804f 100644 --- a/tests/test_platform_fulfillment_integration.py +++ b/tests/test_platform_fulfillment_integration.py @@ -135,7 +135,8 @@ def test_auto_order_sync_is_throttled(monkeypatch): "fetched": 1 } assert service.sync_pending_tasks_if_due(object(), limit=100, now=110.0) is None - assert service.sync_pending_tasks_if_due(object(), limit=100, now=161.0) == { + assert service.sync_pending_tasks_if_due(object(), limit=100, now=999.0) is None + assert service.sync_pending_tasks_if_due(object(), limit=100, now=1000.0) == { "fetched": 1 } assert calls == [100, 100] diff --git a/web/backend/main.py b/web/backend/main.py index 1b88ddc..4739f27 100644 --- a/web/backend/main.py +++ b/web/backend/main.py @@ -101,9 +101,22 @@ async def lifespan(app: FastAPI): register_default_jobs() start_scheduler() + from .services.integration import ( + start_integration_outbox_worker, + stop_integration_outbox_worker, + ) + from .services.platform_fulfillment.reconciler import ( + start_platform_fulfillment_reconciler, + stop_platform_fulfillment_reconciler, + ) + + start_integration_outbox_worker() + start_platform_fulfillment_reconciler() try: yield finally: + stop_platform_fulfillment_reconciler() + stop_integration_outbox_worker() stop_scheduler() diff --git a/web/backend/migrations/versions/20261117_0086_integration_outbox.py b/web/backend/migrations/versions/20261117_0086_integration_outbox.py new file mode 100644 index 0000000..ccd901c --- /dev/null +++ b/web/backend/migrations/versions/20261117_0086_integration_outbox.py @@ -0,0 +1,110 @@ +"""事务 Outbox 与入站 Webhook 幂等表""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op + +revision: str = "20261117_0086" +down_revision: str | None = "20261117_0085" +branch_labels: str | None = None +depends_on: str | None = None + + +def _table_exists(table: str) -> bool: + return bool( + op.get_bind() + .execute( + sa.text( + "SELECT COUNT(*) FROM information_schema.tables " + "WHERE table_schema = DATABASE() AND table_name = :table" + ), + {"table": table}, + ) + .scalar() + ) + + +def _index_exists(index: str) -> bool: + return bool( + op.get_bind() + .execute( + sa.text( + "SELECT COUNT(*) FROM information_schema.statistics " + "WHERE table_schema = DATABASE() AND index_name = :index" + ), + {"index": index}, + ) + .scalar() + ) + + +def upgrade() -> None: + if not _table_exists("integration_outbox"): + op.create_table( + "integration_outbox", + sa.Column("id", sa.BigInteger(), autoincrement=True, nullable=False), + sa.Column("event_id", sa.String(length=36), nullable=False), + sa.Column( + "destination", + sa.String(length=32), + nullable=False, + server_default="order_site", + ), + sa.Column("event_type", sa.String(length=128), nullable=False), + sa.Column("dedupe_key", sa.String(length=255), nullable=True), + sa.Column("payload_json", sa.JSON(), nullable=False), + sa.Column( + "status", + sa.String(length=16), + nullable=False, + server_default="pending", + ), + sa.Column("attempts", sa.Integer(), nullable=False, server_default="0"), + sa.Column("next_attempt_at", sa.DateTime(), nullable=False), + sa.Column("locked_at", sa.DateTime(), nullable=True), + sa.Column("sent_at", sa.DateTime(), nullable=True), + sa.Column("last_error", sa.Text(), nullable=False), + sa.Column("created_at", sa.DateTime(), nullable=False), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("event_id", name="uq_integration_outbox_event_id"), + sa.UniqueConstraint("dedupe_key", name="uq_integration_outbox_dedupe_key"), + ) + if not _index_exists("ix_integration_outbox_due"): + op.create_index( + "ix_integration_outbox_due", + "integration_outbox", + ["status", "next_attempt_at", "id"], + ) + if not _index_exists("ix_integration_outbox_destination"): + op.create_index( + "ix_integration_outbox_destination", + "integration_outbox", + ["destination", "status", "id"], + ) + + if not _table_exists("integration_inbox"): + op.create_table( + "integration_inbox", + sa.Column("event_id", sa.String(length=36), nullable=False), + sa.Column( + "source", sa.String(length=32), nullable=False, server_default="" + ), + sa.Column( + "event_type", sa.String(length=128), nullable=False, server_default="" + ), + sa.Column("received_at", sa.DateTime(), nullable=False), + sa.Column("processed_at", sa.DateTime(), nullable=True), + sa.PrimaryKeyConstraint("event_id"), + ) + + +def downgrade() -> None: + if _index_exists("ix_integration_outbox_destination"): + op.drop_index("ix_integration_outbox_destination", table_name="integration_outbox") + if _index_exists("ix_integration_outbox_due"): + op.drop_index("ix_integration_outbox_due", table_name="integration_outbox") + if _table_exists("integration_inbox"): + op.drop_table("integration_inbox") + if _table_exists("integration_outbox"): + op.drop_table("integration_outbox") diff --git a/web/backend/models.py b/web/backend/models.py index 491e933..63a5883 100644 --- a/web/backend/models.py +++ b/web/backend/models.py @@ -1914,6 +1914,73 @@ class PlatformFulfillmentEvent(Base): ) +class IntegrationOutbox(Base): + """跨系统待发送事件;与业务状态在同一数据库事务内提交。""" + + __tablename__ = "integration_outbox" + + id: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True) + event_id: Mapped[str] = mapped_column(String(36), nullable=False, unique=True) + destination: Mapped[str] = mapped_column( + String(32), nullable=False, default="order_site", server_default="order_site" + ) + event_type: Mapped[str] = mapped_column(String(128), nullable=False) + dedupe_key: Mapped[str | None] = mapped_column( + String(255), nullable=True, unique=True + ) + payload_json: Mapped[dict] = mapped_column(JSON, nullable=False) + status: Mapped[str] = mapped_column( + String(16), nullable=False, default="pending", server_default="pending" + ) + attempts: Mapped[int] = mapped_column( + Integer, nullable=False, default=0, server_default="0" + ) + next_attempt_at: Mapped[datetime] = mapped_column( + UTCDateTime, nullable=False, default=_utcnow + ) + locked_at: Mapped[datetime | None] = mapped_column(UTCDateTime, nullable=True) + sent_at: Mapped[datetime | None] = mapped_column(UTCDateTime, nullable=True) + last_error: Mapped[str] = mapped_column( + Text, nullable=False, default="" + ) + created_at: Mapped[datetime] = mapped_column( + UTCDateTime, nullable=False, default=_utcnow + ) + + __table_args__ = ( + Index( + "ix_integration_outbox_due", + "status", + "next_attempt_at", + "id", + ), + Index( + "ix_integration_outbox_destination", + "destination", + "status", + "id", + ), + ) + + +class IntegrationInbox(Base): + """入站 Webhook 幂等表,event_id 全局唯一。""" + + __tablename__ = "integration_inbox" + + event_id: Mapped[str] = mapped_column(String(36), primary_key=True) + source: Mapped[str] = mapped_column( + String(32), nullable=False, default="", server_default="" + ) + event_type: Mapped[str] = mapped_column( + String(128), nullable=False, default="", server_default="" + ) + received_at: Mapped[datetime] = mapped_column( + UTCDateTime, nullable=False, default=_utcnow + ) + processed_at: Mapped[datetime | None] = mapped_column(UTCDateTime, nullable=True) + + class AutomationJob(Base): """自动化作业定义。状态落库,进程重启不丢调度进度。""" diff --git a/web/backend/routers/platform_fulfillment.py b/web/backend/routers/platform_fulfillment.py index 4346853..16755e8 100644 --- a/web/backend/routers/platform_fulfillment.py +++ b/web/backend/routers/platform_fulfillment.py @@ -2,7 +2,6 @@ from __future__ import annotations -import logging from typing import Any from fastapi import APIRouter, Depends, Query @@ -21,7 +20,6 @@ from ..services.audit_service import record_audit from ..services.platform_fulfillment import service router = APIRouter(prefix="/api/platform-fulfillment", tags=["直播账号履约"]) -logger = logging.getLogger(__name__) @router.get("/settings") @@ -65,10 +63,6 @@ def list_tasks( current: User = Depends(require_permission("platform_fulfillment:view")), ) -> dict[str, Any]: del current - try: - service.sync_pending_tasks_if_due(db, limit=100) - except Exception as exc: # noqa: BLE001 - logger.warning("自动同步待履约订单失败,继续返回本地任务: %s", exc) return service.list_tasks( db, status=status, diff --git a/web/backend/routers/platform_fulfillment_integration.py b/web/backend/routers/platform_fulfillment_integration.py index a90397f..b149df6 100644 --- a/web/backend/routers/platform_fulfillment_integration.py +++ b/web/backend/routers/platform_fulfillment_integration.py @@ -2,19 +2,26 @@ from __future__ import annotations +import json import os +from datetime import UTC, datetime from hmac import compare_digest from typing import Any -from fastapi import APIRouter, Depends, Header, HTTPException +from fastapi import APIRouter, Depends, Header, HTTPException, Request from sqlalchemy.orm import Session from ..database import get_db +from ..models import IntegrationInbox from ..schemas import ( PlatformFulfillmentCustomerBindingAction, PlatformFulfillmentCustomerSessionBatchRequest, PlatformFulfillmentCustomerSessionRequest, ) +from ..services.integration.security import ( + verify_webhook_signature, + webhook_tolerance_seconds, +) from ..services.platform_fulfillment import service router = APIRouter( @@ -33,16 +40,51 @@ def require_integration_key( raise HTTPException(status_code=401, detail="直播账号履约集成鉴权失败") -@router.post( - "/events/order-sync", - dependencies=[Depends(require_integration_key)], -) -def sync_order_event( - payload: dict[str, Any], +@router.post("/events/order-sync") +async def sync_order_event( + request: Request, db: Session = Depends(get_db), + x_integration_key: str = Header("", alias="X-Integration-Key"), + x_integration_event_id: str = Header("", alias="X-Integration-Event-Id"), + x_integration_timestamp: str = Header("", alias="X-Integration-Timestamp"), + x_integration_signature: str = Header("", alias="X-Integration-Signature"), ) -> dict[str, Any]: - del payload - return service.sync_pending_tasks(db, limit=500) + require_integration_key(x_integration_key) + raw_body = await request.body() + if not verify_webhook_signature( + secret=x_integration_key, + timestamp=x_integration_timestamp, + raw_body=raw_body, + signature=x_integration_signature, + tolerance_seconds=webhook_tolerance_seconds(), + ): + raise HTTPException(status_code=401, detail="直播账号履约 Webhook 签名无效") + + try: + payload = json.loads(raw_body) + except json.JSONDecodeError as exc: + raise HTTPException(status_code=400, detail="Webhook JSON 无效") from exc + if not isinstance(payload, dict): + raise HTTPException(status_code=400, detail="Webhook 载荷必须是对象") + + event_id = str(payload.get("eventId") or "").strip() + event_type = str(payload.get("eventType") or "").strip() + if not event_id or event_id != x_integration_event_id.strip(): + raise HTTPException(status_code=400, detail="Webhook 事件编号不一致") + if db.get(IntegrationInbox, event_id) is not None: + return {"fetched": 0, "created": 0, "updated": 0, "duplicate": True} + + inbox = IntegrationInbox( + event_id=event_id, + source="order_site", + event_type=event_type, + received_at=datetime.now(UTC), + ) + db.add(inbox) + result = service.sync_pending_tasks(db, limit=500, commit=False) + inbox.processed_at = datetime.now(UTC) + db.commit() + return {**result, "duplicate": False} @router.post( diff --git a/web/backend/services/douyu_runner_core.py b/web/backend/services/douyu_runner_core.py index 69f0d0a..8884be5 100644 --- a/web/backend/services/douyu_runner_core.py +++ b/web/backend/services/douyu_runner_core.py @@ -188,6 +188,17 @@ class DouyuBatchRunnerCore: } asyncio.run_coroutine_threadsafe(self.log_queue.put(event), self.loop) + def _sync_platform_fulfillment(self, db: Session, task: DouyuTask) -> None: + """绑定/确认任务完成后立即刷新履约投影并写 Outbox。""" + try: + from .platform_fulfillment.service import sync_task_from_binding_runner + + sync_task_from_binding_runner(db, douyu_task_id=int(task.id)) + except Exception: # noqa: BLE001 - 履约投影失败不能覆盖任务执行结果 + logger.exception( + "[douyu] 同步直播账号履约状态失败: task_id={}", task.id + ) + def _mark_task( self, db: Session, @@ -203,6 +214,7 @@ class DouyuBatchRunnerCore: task.finished_at = datetime.now(UTC) db.commit() self._push_task_event(task) + self._sync_platform_fulfillment(db, task) def _update_task_progress( self, @@ -218,6 +230,7 @@ class DouyuBatchRunnerCore: task.result = result db.commit() self._push_task_event(task) + self._sync_platform_fulfillment(db, task) def _upsert_goods(self, db: Session, goods: list[dict]) -> None: upsert_goods_snapshot(db, goods) diff --git a/web/backend/services/integration/__init__.py b/web/backend/services/integration/__init__.py new file mode 100644 index 0000000..788d03f --- /dev/null +++ b/web/backend/services/integration/__init__.py @@ -0,0 +1,16 @@ +"""跨系统集成基础能力。""" + +from .outbox import ( + enqueue_integration_event, + start_integration_outbox_worker, + stop_integration_outbox_worker, +) +from .security import build_webhook_signature, verify_webhook_signature + +__all__ = [ + "build_webhook_signature", + "enqueue_integration_event", + "start_integration_outbox_worker", + "stop_integration_outbox_worker", + "verify_webhook_signature", +] diff --git a/web/backend/services/integration/outbox.py b/web/backend/services/integration/outbox.py new file mode 100644 index 0000000..b473037 --- /dev/null +++ b/web/backend/services/integration/outbox.py @@ -0,0 +1,322 @@ +"""数据库事务 Outbox 与签名 Webhook 投递 worker。""" + +from __future__ import annotations + +import json +import os +import threading +import time +import uuid +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from typing import Any +from urllib.error import HTTPError, URLError +from urllib.request import Request, urlopen + +from loguru import logger +from sqlalchemy.orm import Session + +from ...database import SessionLocal +from ...models import IntegrationOutbox +from .security import build_webhook_signature + +DEFAULT_INTERVAL_SECONDS = 1.0 +DEFAULT_BATCH_SIZE = 20 +DEFAULT_MAX_ATTEMPTS = 12 +REQUEST_TIMEOUT_SECONDS = 5 +PROCESSING_STALE_SECONDS = 120 + +_stop_event = threading.Event() +_worker_thread: threading.Thread | None = None +_lock = threading.Lock() + + +@dataclass(frozen=True) +class _ClaimedEvent: + event_id: str + payload_json: str + attempts: int + + +def enqueue_integration_event( + db: Session, + *, + destination: str, + event_type: str, + payload: dict[str, Any], + dedupe_key: str = "", +) -> bool: + """把事件加入当前事务;调用方必须在业务状态同一事务内 commit。""" + event_id = str(uuid.uuid4()) + body = { + "eventId": event_id, + "eventType": event_type, + "occurredAt": datetime.now(UTC).isoformat(), + **payload, + } + normalized_dedupe = dedupe_key.strip() or None + if normalized_dedupe: + # SessionLocal 关闭了 autoflush;先落当前事务内的业务变更和先前事件, + # 避免同一事务重复调用时看不到刚 add 的 outbox 行。 + db.flush() + existing = ( + db.query(IntegrationOutbox) + .filter(IntegrationOutbox.dedupe_key == normalized_dedupe) + .first() + ) + if existing is not None and existing.status == "dead": + existing.status = "pending" + existing.attempts = 0 + existing.next_attempt_at = datetime.now(UTC) + existing.locked_at = None + existing.last_error = "" + return True + if existing is not None: + return False + + db.add( + IntegrationOutbox( + event_id=event_id, + destination=destination, + event_type=event_type, + dedupe_key=normalized_dedupe, + payload_json=body, + status="pending", + attempts=0, + next_attempt_at=datetime.now(UTC), + created_at=datetime.now(UTC), + ) + ) + return True + + +def outbox_backoff_seconds(attempts: int) -> int: + normalized = max(1, int(attempts or 1)) + return min(3600, 5 * 2 ** min(normalized - 1, 10)) + + +def _configured() -> tuple[str, str]: + base_url = os.getenv("ORDER_SITE_PLATFORM_FULFILLMENT_URL", "").strip().rstrip("/") + integration_key = os.getenv("PLATFORM_FULFILLMENT_INTEGRATION_KEY", "").strip() + return base_url, integration_key + + +def _env_float(name: str, default: float, minimum: float, maximum: float) -> float: + try: + value = float(os.getenv(name, str(default))) + except ValueError: + return default + return max(minimum, min(maximum, value)) + + +def _env_int(name: str, default: int, minimum: int, maximum: int) -> int: + try: + value = int(os.getenv(name, str(default))) + except ValueError: + return default + return max(minimum, min(maximum, value)) + + +def _claim_due_events(limit: int) -> list[_ClaimedEvent]: + db = SessionLocal() + now = datetime.now(UTC) + try: + ( + db.query(IntegrationOutbox) + .filter( + IntegrationOutbox.status == "processing", + IntegrationOutbox.locked_at + < now - timedelta(seconds=PROCESSING_STALE_SECONDS), + ) + .update( + { + IntegrationOutbox.status: "pending", + IntegrationOutbox.locked_at: None, + IntegrationOutbox.next_attempt_at: now, + }, + synchronize_session=False, + ) + ) + rows = ( + db.query(IntegrationOutbox) + .filter( + IntegrationOutbox.status == "pending", + IntegrationOutbox.next_attempt_at <= now, + ) + .order_by(IntegrationOutbox.id.asc()) + .with_for_update(skip_locked=True) + .limit(limit) + .all() + ) + claimed: list[_ClaimedEvent] = [] + for row in rows: + row.status = "processing" + row.locked_at = now + claimed.append( + _ClaimedEvent( + event_id=row.event_id, + payload_json=json.dumps( + row.payload_json or {}, + ensure_ascii=False, + separators=(",", ":"), + ), + attempts=int(row.attempts or 0), + ) + ) + db.commit() + return claimed + except Exception: + db.rollback() + raise + finally: + db.close() + + +def _deliver(event: _ClaimedEvent, base_url: str, integration_key: str) -> None: + timestamp = str(int(time.time())) + signature = build_webhook_signature( + integration_key, + timestamp, + event.payload_json.encode("utf-8"), + ) + request = Request( + f"{base_url}/api/integration/platform-fulfillment/binding-state", + data=event.payload_json.encode("utf-8"), + method="POST", + headers={ + "Content-Type": "application/json", + "X-Integration-Key": integration_key, + "X-Integration-Event-Id": event.event_id, + "X-Integration-Timestamp": timestamp, + "X-Integration-Signature": signature, + }, + ) + try: + with urlopen(request, timeout=REQUEST_TIMEOUT_SECONDS) as response: + if not 200 <= response.status < 300: + raise RuntimeError(f"order_site 返回 HTTP {response.status}") + except HTTPError as exc: + raise RuntimeError(f"order_site 返回 HTTP {exc.code}") from exc + except (URLError, OSError) as exc: + raise RuntimeError(str(exc)) from exc + + +def _mark_sent(event_id: str) -> None: + db = SessionLocal() + try: + ( + db.query(IntegrationOutbox) + .filter(IntegrationOutbox.event_id == event_id) + .update( + { + IntegrationOutbox.status: "sent", + IntegrationOutbox.sent_at: datetime.now(UTC), + IntegrationOutbox.locked_at: None, + IntegrationOutbox.last_error: "", + }, + synchronize_session=False, + ) + ) + db.commit() + finally: + db.close() + + +def _mark_failed(event: _ClaimedEvent, error: Exception) -> None: + attempts = event.attempts + 1 + max_attempts = _env_int( + "INTEGRATION_OUTBOX_MAX_ATTEMPTS", DEFAULT_MAX_ATTEMPTS, 1, 100 + ) + dead = attempts >= max_attempts + message = str(error)[:2000] + db = SessionLocal() + try: + ( + db.query(IntegrationOutbox) + .filter(IntegrationOutbox.event_id == event.event_id) + .update( + { + IntegrationOutbox.status: "dead" if dead else "pending", + IntegrationOutbox.attempts: attempts, + IntegrationOutbox.locked_at: None, + IntegrationOutbox.next_attempt_at: datetime.now(UTC) + + ( + timedelta(seconds=0) + if dead + else timedelta(seconds=outbox_backoff_seconds(attempts)) + ), + IntegrationOutbox.last_error: message, + }, + synchronize_session=False, + ) + ) + db.commit() + finally: + db.close() + if dead: + logger.error( + "集成 Outbox 事件超过最大重试次数 event_id={} attempts={} error={}", + event.event_id, + attempts, + message, + ) + + +def _publish_due_events() -> None: + base_url, integration_key = _configured() + if not base_url or not integration_key: + return + batch_size = _env_int( + "INTEGRATION_OUTBOX_BATCH_SIZE", DEFAULT_BATCH_SIZE, 1, 100 + ) + for event in _claim_due_events(batch_size): + try: + _deliver(event, base_url, integration_key) + _mark_sent(event.event_id) + except Exception as exc: # noqa: BLE001 - 投递失败必须回落数据库重试 + _mark_failed(event, exc) + + +def _run() -> None: + interval = _env_float( + "INTEGRATION_OUTBOX_INTERVAL_SECONDS", + DEFAULT_INTERVAL_SECONDS, + 0.25, + 60, + ) + while not _stop_event.is_set(): + try: + _publish_due_events() + except Exception as exc: # noqa: BLE001 - worker 不能因单轮异常退出 + logger.warning("集成 Outbox 投递轮次失败: {}", exc) + _stop_event.wait(interval) + + +def start_integration_outbox_worker() -> bool: + global _worker_thread + with _lock: + if _worker_thread is not None and _worker_thread.is_alive(): + return False + _stop_event.clear() + _worker_thread = threading.Thread( + target=_run, + name="integration-outbox-worker", + daemon=True, + ) + _worker_thread.start() + base_url, integration_key = _configured() + if base_url and integration_key: + logger.info("事务 Outbox worker 已启动") + else: + logger.warning("未配置 order_site 地址或集成密钥,Outbox worker 仅保持待命") + return True + + +def stop_integration_outbox_worker() -> None: + global _worker_thread + _stop_event.set() + with _lock: + thread = _worker_thread + _worker_thread = None + if thread is not None and thread.is_alive(): + thread.join(timeout=3) diff --git a/web/backend/services/integration/security.py b/web/backend/services/integration/security.py new file mode 100644 index 0000000..2804533 --- /dev/null +++ b/web/backend/services/integration/security.py @@ -0,0 +1,72 @@ +"""签名 Webhook 的 HMAC 与时间窗校验。""" + +from __future__ import annotations + +import hashlib +import hmac +import time + +DEFAULT_TOLERANCE_SECONDS = 300 + + +def build_webhook_signature(secret: str, timestamp: str, raw_body: bytes) -> str: + content = timestamp.encode("utf-8") + b"." + raw_body + digest = hmac.new(secret.encode("utf-8"), content, hashlib.sha256).hexdigest() + return f"sha256={digest}" + + +def verify_webhook_signature( + *, + secret: str, + timestamp: str, + raw_body: bytes, + signature: str, + tolerance_seconds: int = DEFAULT_TOLERANCE_SECONDS, + now_seconds: int | None = None, +) -> bool: + return verify_webhook_signature_at( + secret=secret, + timestamp=timestamp, + raw_body=raw_body, + signature=signature, + tolerance_seconds=tolerance_seconds, + now_seconds=now_seconds, + ) + + +def verify_webhook_signature_at( + *, + secret: str, + timestamp: str, + raw_body: bytes, + signature: str, + tolerance_seconds: int = DEFAULT_TOLERANCE_SECONDS, + now_seconds: int | None = None, +) -> bool: + secret = secret.strip() + timestamp = timestamp.strip() + signature = signature.strip() + if not secret or not timestamp or not signature: + return False + try: + timestamp_seconds = int(timestamp) + except ValueError: + return False + if timestamp_seconds <= 0: + return False + tolerance = max(1, int(tolerance_seconds)) + current = int(now_seconds if now_seconds is not None else time.time()) + if abs(current - timestamp_seconds) > tolerance: + return False + expected = build_webhook_signature(secret, timestamp, raw_body) + return hmac.compare_digest(expected, signature) + + +def webhook_tolerance_seconds() -> int: + import os + + try: + value = int(os.getenv("PLATFORM_FULFILLMENT_WEBHOOK_TOLERANCE_SECONDS", "300")) + except ValueError: + return DEFAULT_TOLERANCE_SECONDS + return value if value > 0 else DEFAULT_TOLERANCE_SECONDS diff --git a/web/backend/services/platform_fulfillment/order_site_notifier.py b/web/backend/services/platform_fulfillment/order_site_notifier.py index 7316802..71e55e9 100644 --- a/web/backend/services/platform_fulfillment/order_site_notifier.py +++ b/web/backend/services/platform_fulfillment/order_site_notifier.py @@ -1,86 +1,53 @@ -"""把直播账号履约状态变化主动推送给 order_site。""" +"""把直播账号履约状态变化写入 order_site 的事务 Outbox。""" from __future__ import annotations +import hashlib import json -import os -import time -import uuid -from datetime import UTC, datetime -from threading import Thread -from urllib.error import HTTPError, URLError -from urllib.request import Request, urlopen +from typing import Any -from loguru import logger +from sqlalchemy.orm import Session -from ...database import SessionLocal +from ..integration.outbox import enqueue_integration_event -PUBLISH_ATTEMPTS = 3 -PUBLISH_TIMEOUT_SECONDS = 5 +EVENT_TYPE = "platform_fulfillment.binding_state_changed" -def notify_order_site_binding_state_async(task_id: int) -> None: - """状态提交后异步推送;失败由工作台低频对账补齐。""" - Thread( - target=_publish_binding_state, - args=(int(task_id),), - name=f"order-site-fulfillment-{task_id}", - daemon=True, - ).start() +def _state_dedupe_key(payload: dict[str, Any]) -> str: + state = { + "sessionId": payload.get("sessionId"), + "executionStatus": payload.get("executionStatus"), + "bindingStatus": payload.get("bindingStatus"), + "exchangeStatus": payload.get("exchangeStatus"), + "issueCode": payload.get("issueCode"), + "accountReady": payload.get("accountReady"), + "bindTaskStatus": payload.get("bindTaskStatus"), + "bindPhase": payload.get("bindPhase"), + "qrUrl": payload.get("qrUrl"), + "roleName": payload.get("roleName"), + "areaName": payload.get("areaName"), + "platName": payload.get("platName"), + "bindReadyForConfirm": payload.get("bindReadyForConfirm"), + "bindConfirmed": payload.get("bindConfirmed"), + } + digest = hashlib.sha256( + json.dumps(state, ensure_ascii=False, sort_keys=True).encode("utf-8") + ).hexdigest()[:32] + return f"binding-state:{payload.get('sessionId')}:{digest}" -def _publish_binding_state(task_id: int) -> None: - base_url = os.getenv("ORDER_SITE_PLATFORM_FULFILLMENT_URL", "").strip().rstrip("/") - integration_key = os.getenv("PLATFORM_FULFILLMENT_INTEGRATION_KEY", "").strip() - if not base_url or not integration_key: - return - - db = SessionLocal() - try: - from . import service - - payload = service.customer_binding_out(db, task_id) - except Exception as exc: # noqa: BLE001 - logger.warning("构造 order_site 履约状态事件失败 task_id={}: {}", task_id, exc) - return - finally: - db.close() - - body = json.dumps( - { - "eventId": str(uuid.uuid4()), - "occurredAt": datetime.now(UTC).isoformat(), - "items": [payload], - }, - ensure_ascii=False, - ).encode("utf-8") - endpoint = f"{base_url}/api/integration/platform-fulfillment/binding-state" - - for attempt in range(1, PUBLISH_ATTEMPTS + 1): - request = Request( - endpoint, - data=body, - method="POST", - headers={ - "Content-Type": "application/json", - "X-Integration-Key": integration_key, - }, - ) - try: - with urlopen(request, timeout=PUBLISH_TIMEOUT_SECONDS) as response: - if 200 <= response.status < 300: - return - raise RuntimeError(f"order_site 返回 HTTP {response.status}") - except (HTTPError, URLError, OSError, RuntimeError) as exc: - if attempt >= PUBLISH_ATTEMPTS: - logger.warning( - "推送 order_site 履约状态失败 task_id={} attempts={}: {}", - task_id, - attempt, - exc, - ) - return - time.sleep(float(attempt)) +def enqueue_order_site_binding_state( + db: Session, + payload: dict[str, Any], +) -> bool: + """必须在任务状态变更的同一事务内调用。""" + return enqueue_integration_event( + db, + destination="order_site", + event_type=EVENT_TYPE, + payload={"items": [payload]}, + dedupe_key=_state_dedupe_key(payload), + ) -__all__ = ["notify_order_site_binding_state_async"] +__all__ = ["enqueue_order_site_binding_state"] diff --git a/web/backend/services/platform_fulfillment/reconciler.py b/web/backend/services/platform_fulfillment/reconciler.py new file mode 100644 index 0000000..2033cc0 --- /dev/null +++ b/web/backend/services/platform_fulfillment/reconciler.py @@ -0,0 +1,72 @@ +"""低频订单对账兜底;正常同步由 order_site 签名 Webhook 驱动。""" + +from __future__ import annotations + +import os +import threading + +from loguru import logger + +from ...database import SessionLocal + +_stop_event = threading.Event() +_thread: threading.Thread | None = None +_lock = threading.Lock() + + +def _interval_seconds() -> int: + try: + value = int(os.getenv("PLATFORM_FULFILLMENT_RECONCILE_SECONDS", "900")) + except ValueError: + return 900 + return max(60, value) + + +def _run() -> None: + from . import service + + while not _stop_event.is_set(): + # 不抢占启动时的 Webhook;首轮延后一分钟。 + if _stop_event.wait(min(60, _interval_seconds())): + break + db = SessionLocal() + try: + result = service.sync_pending_tasks(db, limit=500) + queued = service.reconcile_binding_state_outbox(db, limit=500) + logger.info( + "直播账号履约低频对账完成: order_sync={} outbox_requeued={}", + result, + queued, + ) + except Exception as exc: # noqa: BLE001 - 对账失败等待下一轮 + db.rollback() + logger.warning("直播账号履约低频对账失败: {}", exc) + finally: + db.close() + if _stop_event.wait(_interval_seconds()): + break + + +def start_platform_fulfillment_reconciler() -> bool: + global _thread + with _lock: + if _thread is not None and _thread.is_alive(): + return False + _stop_event.clear() + _thread = threading.Thread( + target=_run, + name="platform-fulfillment-reconciler", + daemon=True, + ) + _thread.start() + return True + + +def stop_platform_fulfillment_reconciler() -> None: + global _thread + _stop_event.set() + with _lock: + thread = _thread + _thread = None + if thread is not None and thread.is_alive(): + thread.join(timeout=3) diff --git a/web/backend/services/platform_fulfillment/service.py b/web/backend/services/platform_fulfillment/service.py index 252ff3b..326c05d 100644 --- a/web/backend/services/platform_fulfillment/service.py +++ b/web/backend/services/platform_fulfillment/service.py @@ -40,7 +40,7 @@ from .account_selector import ( release_lease, renew_lease, ) -from .order_site_notifier import notify_order_site_binding_state_async +from .order_site_notifier import enqueue_order_site_binding_state TARGET_PLATFORM = "douyu" SCOPE_ELITE = "elite" @@ -55,7 +55,7 @@ ACTIVE_STATUSES = { } BINDING_QR_TTL_SECONDS = int(os.getenv("PLATFORM_BINDING_QR_TTL_SECONDS", "600")) -AUTO_SYNC_INTERVAL_SECONDS = 60.0 +AUTO_SYNC_INTERVAL_SECONDS = 900.0 _SYNC_LOCK = Lock() _AUTO_SYNC_LOCK = Lock() @@ -228,6 +228,14 @@ def _append_event( ) +def _emit_order_site_binding_state(db: Session, task_id: int) -> bool: + """当前事务内写 Outbox;HTTP 投递由独立 worker 完成。""" + return enqueue_order_site_binding_state( + db, + customer_binding_out(db, task_id), + ) + + def _load_task(db: Session, task_id: int) -> PlatformFulfillmentTask: task = ( db.query(PlatformFulfillmentTask) @@ -259,7 +267,12 @@ def _apply_mapping( task.last_error = "" -def _sync_pending_tasks(db: Session, *, limit: int = 100) -> dict[str, Any]: +def _sync_pending_tasks( + db: Session, + *, + limit: int = 100, + commit: bool = True, +) -> dict[str, Any]: """从 order_site 拉取人工待履约订单项并幂等生成本地作业。""" try: rows = list_pending_manual_fulfillments(limit=limit) @@ -320,15 +333,22 @@ def _sync_pending_tasks(db: Session, *, limit: int = 100) -> dict[str, Any]: task.updated_at = _now() if task.execution_status in {"pending_mapping", "ready"}: _apply_mapping(task, _mapping_for_sku(db, task.internal_sku_key)) + _emit_order_site_binding_state(db, task.id) - db.commit() + if commit: + db.commit() return {"fetched": len(rows), "created": created, "updated": updated} -def sync_pending_tasks(db: Session, *, limit: int = 100) -> dict[str, Any]: +def sync_pending_tasks( + db: Session, + *, + limit: int = 100, + commit: bool = True, +) -> dict[str, Any]: """串行同步订单,避免自动同步与人工同步并发创建重复任务。""" with _SYNC_LOCK: - return _sync_pending_tasks(db, limit=limit) + return _sync_pending_tasks(db, limit=limit, commit=commit) def sync_pending_tasks_if_due( @@ -539,7 +559,7 @@ def _customer_task_message(task: PlatformFulfillmentTask) -> str: "elite_not_opened": "账号正在准备精英宝典,请稍后重试", "points_short": "账号正在补充积分,请稍后重试", "commodity_limit_reached": "当前商品暂时无法兑换,请联系客服", - "binding_failed": "绑定失败,请刷新二维码后重试", + "binding_failed": "暂未识别到绑定角色,请继续使用当前二维码完成扫码", "eligibility_check_failed": "状态查询异常,请稍后刷新", "manual_required": "当前需要人工处理,请联系客服", } @@ -892,8 +912,8 @@ def allocate_task(db: Session, *, task_id: int, current: User) -> dict[str, Any] "diagnostics": diagnostics, }, ) + _emit_order_site_binding_state(db, task.id) db.commit() - notify_order_site_binding_state_async(task.id) return get_task_detail(db, task.id) task.account_id = selected.account_id @@ -936,8 +956,8 @@ def allocate_task(db: Session, *, task_id: int, current: User) -> dict[str, Any] "exchangeStatus": task.exchange_status, }, ) + _emit_order_site_binding_state(db, task.id) db.commit() - notify_order_site_binding_state_async(task.id) return get_task_detail(db, task.id) @@ -972,8 +992,8 @@ def release_task_account( actor_id=current.id, detail={"reason": reason}, ) + _emit_order_site_binding_state(db, task.id) db.commit() - notify_order_site_binding_state_async(task.id) return get_task_detail(db, task.id) @@ -1044,8 +1064,8 @@ async def start_binding( actor_id=current.id, detail={"bindTaskId": bind_task.id}, ) + _emit_order_site_binding_state(db, task.id) db.commit() - notify_order_site_binding_state_async(task.id) start_douyu_batch( batch_id=batch_id, task_type="get_bind_qr", @@ -1090,8 +1110,6 @@ def refresh_binding(db: Session, *, task_id: int) -> dict[str, Any]: task.execution_status = "waiting_bind" task.issue_code = "" task.last_error = "" - task.updated_at = _now() - db.commit() after = ( task.binding_status, task.execution_status, @@ -1100,10 +1118,50 @@ def refresh_binding(db: Session, *, task_id: int) -> dict[str, Any]: task.last_error, ) if after != before: - notify_order_site_binding_state_async(task.id) + task.updated_at = _now() + _emit_order_site_binding_state(db, task.id) + db.commit() return get_task_detail(db, task.id) +def sync_task_from_binding_runner(db: Session, *, douyu_task_id: int) -> int: + """斗鱼绑定任务落库后,把关联履约任务和 Outbox 一并刷新。""" + task_ids = [ + int(row.id) + for row in ( + db.query(PlatformFulfillmentTask.id) + .filter(PlatformFulfillmentTask.bind_task_id == int(douyu_task_id)) + .all() + ) + ] + for task_id in task_ids: + refresh_binding(db, task_id=task_id) + return len(task_ids) + + +def reconcile_binding_state_outbox(db: Session, *, limit: int = 500) -> int: + """低频补齐 active 任务当前快照;只会在缺失或 dead 时重新入队。""" + task_ids = [ + int(row.id) + for row in ( + db.query(PlatformFulfillmentTask.id) + .filter( + PlatformFulfillmentTask.execution_status.notin_( + ["completed", "cancelled"] + ) + ) + .order_by(PlatformFulfillmentTask.updated_at.asc()) + .limit(max(1, min(limit, 500))) + .all() + ) + ] + queued = sum( + 1 for task_id in task_ids if _emit_order_site_binding_state(db, task_id) + ) + db.commit() + return queued + + async def confirm_binding( db: Session, *, task_id: int, current: User ) -> dict[str, Any]: @@ -1145,8 +1203,8 @@ async def confirm_binding( actor_id=current.id, detail={"confirmTaskId": confirm_task.id}, ) + _emit_order_site_binding_state(db, task.id) db.commit() - notify_order_site_binding_state_async(task.id) start_douyu_batch( batch_id=batch_id, task_type="confirm_bind", @@ -1192,8 +1250,8 @@ def refresh_exchange_eligibility( actor_id=current.id, detail={"error": str(exc)}, ) + _emit_order_site_binding_state(db, task.id) db.commit() - notify_order_site_binding_state_async(task.id) return get_task_detail(db, task.id) if task.commodity_id in eligibility.limited_ids: @@ -1228,8 +1286,8 @@ def refresh_exchange_eligibility( "limitedCount": len(eligibility.limited_ids), }, ) + _emit_order_site_binding_state(db, task.id) db.commit() - notify_order_site_binding_state_async(task.id) return get_task_detail(db, task.id) @@ -1269,8 +1327,8 @@ def complete_manual_exchange( actor_id=current.id, detail={"note": note[:1000]}, ) + _emit_order_site_binding_state(db, task.id) db.commit() - notify_order_site_binding_state_async(task.id) return get_task_detail(db, task.id) @@ -1295,8 +1353,8 @@ def mark_manual_required( actor_id=current.id, detail={"reason": task.last_error}, ) + _emit_order_site_binding_state(db, task.id) db.commit() - notify_order_site_binding_state_async(task.id) return get_task_detail(db, task.id) @@ -1326,8 +1384,8 @@ def cancel_task( actor_id=current.id, detail={"reason": task.last_error}, ) + _emit_order_site_binding_state(db, task.id) db.commit() - notify_order_site_binding_state_async(task.id) return get_task_detail(db, task.id) @@ -1341,11 +1399,13 @@ __all__ = [ "list_mappings", "list_tasks", "mark_manual_required", + "reconcile_binding_state_outbox", "refresh_binding", "refresh_exchange_eligibility", "release_task_account", "start_binding", "sync_pending_tasks", "sync_pending_tasks_if_due", + "sync_task_from_binding_runner", "upsert_mapping", ]