feat(fulfillment): 增加签名 Webhook 事务 Outbox

- 新增直播履约状态事务 Outbox 与签名投递 worker
- 增加 order_site 状态回传、重试和低频对账兜底
- 补充 Outbox、签名校验与集成接口测试
This commit is contained in:
yml
2026-11-18 10:59:20 +08:00
parent 00c4a268c2
commit bba3d9dac9
17 changed files with 967 additions and 114 deletions
+7
View File
@@ -150,6 +150,13 @@ ORDER_SITE_LOOKUP_TIMEOUT_SECONDS=3
PLATFORM_FULFILLMENT_INTEGRATION_KEY=
# live-hub -> order_site 状态回调地址;生产示例 http://172.23.3.80
ORDER_SITE_PLATFORM_FULFILLMENT_URL=
# 签名 Webhook 允许的时钟偏差;Outbox 投递 worker 的轮询与重试参数
PLATFORM_FULFILLMENT_WEBHOOK_TOLERANCE_SECONDS=300
INTEGRATION_OUTBOX_INTERVAL_SECONDS=1
INTEGRATION_OUTBOX_BATCH_SIZE=20
INTEGRATION_OUTBOX_MAX_ATTEMPTS=12
# 低频对账兜底;正常同步由签名 Webhook 驱动(默认 15 分钟)
PLATFORM_FULFILLMENT_RECONCILE_SECONDS=900
# 可选:指定集成操作人;不填时使用最早创建的有效超级管理员
# PLATFORM_FULFILLMENT_INTEGRATION_ACTOR_ID=1
# 客户绑定二维码有效期
+13 -3
View File
@@ -100,14 +100,25 @@ ORDER_SITE_LOOKUP_ENABLED=true
## order_site 客户领取集成
跨项目异步状态统一采用“事务 Outbox + 签名 Webhook”:
- 业务状态与待发送事件在同一个数据库事务内写入 `integration_outbox`;
- 独立 worker 使用 `FOR UPDATE SKIP LOCKED` 领取事件并指数退避重试;
- 请求携带 `X-Integration-Key`、`X-Integration-Event-Id`、
`X-Integration-Timestamp`、`X-Integration-Signature`;
- 签名值为 `sha256=HMAC-SHA256(timestamp + "." + rawBody)`;
- 接收方通过 `integration_inbox.event_id` 幂等,失败时整笔事务回滚;
- 15 分钟低频对账只作为 dead/漏单兜底,不再由列表请求或人工刷新承担主同步。
`order_site` 客户页通过服务端接口调用 live-hub,不允许客户浏览器直接访问
live-hub 管理 API。内部接口统一位于:
live-hub 管理 API。同步客户会话接口位于:
```text
/api/integration/platform-fulfillment/customer-sessions/*
```
接口使用 `X-Integration-Key` 鉴权,密钥来自
请求响应接口使用 `X-Integration-Key` 鉴权,异步事件在密钥之外还必须通过
时间窗与 HMAC 验签,密钥来自
`PLATFORM_FULFILLMENT_INTEGRATION_KEY`。返回给 `order_site` 的客户会话只包含
二维码、二维码有效期、绑定阶段、平台识别角色和状态提示,不包含账号、CK、
积分、租约或限兑信息。
@@ -124,7 +135,6 @@ live-hub 创建新的绑定任务。
## 暂不实现
- `order_site` 状态回写接口
- 自动充值鱼翅
- 自动兑换
- 多数量拆分
+87
View File
@@ -0,0 +1,87 @@
"""事务 Outbox 与签名 Webhook 基础行为。"""
from __future__ import annotations
from web.backend.models import IntegrationOutbox
from web.backend.services.integration.outbox import (
enqueue_integration_event,
outbox_backoff_seconds,
)
from web.backend.services.integration.security import (
build_webhook_signature,
verify_webhook_signature,
)
def test_webhook_signature_binds_timestamp_and_body():
raw_body = b'{"eventId":"evt-1","items":[]}'
timestamp = "1783394219"
signature = build_webhook_signature("secret-key", timestamp, raw_body)
assert verify_webhook_signature(
secret="secret-key",
timestamp=timestamp,
raw_body=raw_body,
signature=signature,
now_seconds=int(timestamp),
)
assert not verify_webhook_signature(
secret="secret-key",
timestamp=timestamp,
raw_body=raw_body + b" ",
signature=signature,
now_seconds=int(timestamp),
)
def test_webhook_signature_rejects_expired_timestamp():
timestamp = "1783394219"
raw_body = b"{}"
signature = build_webhook_signature("secret-key", timestamp, raw_body)
assert not verify_webhook_signature(
secret="secret-key",
timestamp=timestamp,
raw_body=raw_body,
signature=signature,
tolerance_seconds=300,
now_seconds=int(timestamp) + 301,
)
def test_outbox_enqueue_is_deduplicated_and_backoff_is_bounded(db):
assert enqueue_integration_event(
db,
destination="order_site",
event_type="platform_fulfillment.binding_state_changed",
payload={"items": [{"sessionId": 1}]},
dedupe_key="binding-state:1:same",
)
assert not enqueue_integration_event(
db,
destination="order_site",
event_type="platform_fulfillment.binding_state_changed",
payload={"items": [{"sessionId": 1}]},
dedupe_key="binding-state:1:same",
)
db.commit()
row = db.query(IntegrationOutbox).one()
assert row.status == "pending"
assert row.payload_json["eventId"] == row.event_id
row.status = "dead"
db.commit()
assert enqueue_integration_event(
db,
destination="order_site",
event_type="platform_fulfillment.binding_state_changed",
payload={"items": [{"sessionId": 1}]},
dedupe_key="binding-state:1:same",
)
db.commit()
db.refresh(row)
assert row.status == "pending"
assert row.attempts == 0
assert outbox_backoff_seconds(1) == 5
assert outbox_backoff_seconds(99) == 3600
+2 -2
View File
@@ -468,8 +468,8 @@ class TestBindingAndCompletion(_Base):
notified: list[int] = []
monkeypatch.setattr(
service,
"notify_order_site_binding_state_async",
lambda task_id: notified.append(task_id),
"_emit_order_site_binding_state",
lambda _db, task_id: notified.append(task_id) or True,
)
task = self._allocated()
notified.clear()
@@ -135,7 +135,8 @@ def test_auto_order_sync_is_throttled(monkeypatch):
"fetched": 1
}
assert service.sync_pending_tasks_if_due(object(), limit=100, now=110.0) is None
assert service.sync_pending_tasks_if_due(object(), limit=100, now=161.0) == {
assert service.sync_pending_tasks_if_due(object(), limit=100, now=999.0) is None
assert service.sync_pending_tasks_if_due(object(), limit=100, now=1000.0) == {
"fetched": 1
}
assert calls == [100, 100]
+13
View File
@@ -101,9 +101,22 @@ async def lifespan(app: FastAPI):
register_default_jobs()
start_scheduler()
from .services.integration import (
start_integration_outbox_worker,
stop_integration_outbox_worker,
)
from .services.platform_fulfillment.reconciler import (
start_platform_fulfillment_reconciler,
stop_platform_fulfillment_reconciler,
)
start_integration_outbox_worker()
start_platform_fulfillment_reconciler()
try:
yield
finally:
stop_platform_fulfillment_reconciler()
stop_integration_outbox_worker()
stop_scheduler()
@@ -0,0 +1,110 @@
"""事务 Outbox 与入站 Webhook 幂等表"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
revision: str = "20261117_0086"
down_revision: str | None = "20261117_0085"
branch_labels: str | None = None
depends_on: str | None = None
def _table_exists(table: str) -> bool:
return bool(
op.get_bind()
.execute(
sa.text(
"SELECT COUNT(*) FROM information_schema.tables "
"WHERE table_schema = DATABASE() AND table_name = :table"
),
{"table": table},
)
.scalar()
)
def _index_exists(index: str) -> bool:
return bool(
op.get_bind()
.execute(
sa.text(
"SELECT COUNT(*) FROM information_schema.statistics "
"WHERE table_schema = DATABASE() AND index_name = :index"
),
{"index": index},
)
.scalar()
)
def upgrade() -> None:
if not _table_exists("integration_outbox"):
op.create_table(
"integration_outbox",
sa.Column("id", sa.BigInteger(), autoincrement=True, nullable=False),
sa.Column("event_id", sa.String(length=36), nullable=False),
sa.Column(
"destination",
sa.String(length=32),
nullable=False,
server_default="order_site",
),
sa.Column("event_type", sa.String(length=128), nullable=False),
sa.Column("dedupe_key", sa.String(length=255), nullable=True),
sa.Column("payload_json", sa.JSON(), nullable=False),
sa.Column(
"status",
sa.String(length=16),
nullable=False,
server_default="pending",
),
sa.Column("attempts", sa.Integer(), nullable=False, server_default="0"),
sa.Column("next_attempt_at", sa.DateTime(), nullable=False),
sa.Column("locked_at", sa.DateTime(), nullable=True),
sa.Column("sent_at", sa.DateTime(), nullable=True),
sa.Column("last_error", sa.Text(), nullable=False),
sa.Column("created_at", sa.DateTime(), nullable=False),
sa.PrimaryKeyConstraint("id"),
sa.UniqueConstraint("event_id", name="uq_integration_outbox_event_id"),
sa.UniqueConstraint("dedupe_key", name="uq_integration_outbox_dedupe_key"),
)
if not _index_exists("ix_integration_outbox_due"):
op.create_index(
"ix_integration_outbox_due",
"integration_outbox",
["status", "next_attempt_at", "id"],
)
if not _index_exists("ix_integration_outbox_destination"):
op.create_index(
"ix_integration_outbox_destination",
"integration_outbox",
["destination", "status", "id"],
)
if not _table_exists("integration_inbox"):
op.create_table(
"integration_inbox",
sa.Column("event_id", sa.String(length=36), nullable=False),
sa.Column(
"source", sa.String(length=32), nullable=False, server_default=""
),
sa.Column(
"event_type", sa.String(length=128), nullable=False, server_default=""
),
sa.Column("received_at", sa.DateTime(), nullable=False),
sa.Column("processed_at", sa.DateTime(), nullable=True),
sa.PrimaryKeyConstraint("event_id"),
)
def downgrade() -> None:
if _index_exists("ix_integration_outbox_destination"):
op.drop_index("ix_integration_outbox_destination", table_name="integration_outbox")
if _index_exists("ix_integration_outbox_due"):
op.drop_index("ix_integration_outbox_due", table_name="integration_outbox")
if _table_exists("integration_inbox"):
op.drop_table("integration_inbox")
if _table_exists("integration_outbox"):
op.drop_table("integration_outbox")
+67
View File
@@ -1914,6 +1914,73 @@ class PlatformFulfillmentEvent(Base):
)
class IntegrationOutbox(Base):
"""跨系统待发送事件;与业务状态在同一数据库事务内提交。"""
__tablename__ = "integration_outbox"
id: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True)
event_id: Mapped[str] = mapped_column(String(36), nullable=False, unique=True)
destination: Mapped[str] = mapped_column(
String(32), nullable=False, default="order_site", server_default="order_site"
)
event_type: Mapped[str] = mapped_column(String(128), nullable=False)
dedupe_key: Mapped[str | None] = mapped_column(
String(255), nullable=True, unique=True
)
payload_json: Mapped[dict] = mapped_column(JSON, nullable=False)
status: Mapped[str] = mapped_column(
String(16), nullable=False, default="pending", server_default="pending"
)
attempts: Mapped[int] = mapped_column(
Integer, nullable=False, default=0, server_default="0"
)
next_attempt_at: Mapped[datetime] = mapped_column(
UTCDateTime, nullable=False, default=_utcnow
)
locked_at: Mapped[datetime | None] = mapped_column(UTCDateTime, nullable=True)
sent_at: Mapped[datetime | None] = mapped_column(UTCDateTime, nullable=True)
last_error: Mapped[str] = mapped_column(
Text, nullable=False, default=""
)
created_at: Mapped[datetime] = mapped_column(
UTCDateTime, nullable=False, default=_utcnow
)
__table_args__ = (
Index(
"ix_integration_outbox_due",
"status",
"next_attempt_at",
"id",
),
Index(
"ix_integration_outbox_destination",
"destination",
"status",
"id",
),
)
class IntegrationInbox(Base):
"""入站 Webhook 幂等表,event_id 全局唯一。"""
__tablename__ = "integration_inbox"
event_id: Mapped[str] = mapped_column(String(36), primary_key=True)
source: Mapped[str] = mapped_column(
String(32), nullable=False, default="", server_default=""
)
event_type: Mapped[str] = mapped_column(
String(128), nullable=False, default="", server_default=""
)
received_at: Mapped[datetime] = mapped_column(
UTCDateTime, nullable=False, default=_utcnow
)
processed_at: Mapped[datetime | None] = mapped_column(UTCDateTime, nullable=True)
class AutomationJob(Base):
"""自动化作业定义。状态落库,进程重启不丢调度进度。"""
@@ -2,7 +2,6 @@
from __future__ import annotations
import logging
from typing import Any
from fastapi import APIRouter, Depends, Query
@@ -21,7 +20,6 @@ from ..services.audit_service import record_audit
from ..services.platform_fulfillment import service
router = APIRouter(prefix="/api/platform-fulfillment", tags=["直播账号履约"])
logger = logging.getLogger(__name__)
@router.get("/settings")
@@ -65,10 +63,6 @@ def list_tasks(
current: User = Depends(require_permission("platform_fulfillment:view")),
) -> dict[str, Any]:
del current
try:
service.sync_pending_tasks_if_due(db, limit=100)
except Exception as exc: # noqa: BLE001
logger.warning("自动同步待履约订单失败,继续返回本地任务: %s", exc)
return service.list_tasks(
db,
status=status,
@@ -2,19 +2,26 @@
from __future__ import annotations
import json
import os
from datetime import UTC, datetime
from hmac import compare_digest
from typing import Any
from fastapi import APIRouter, Depends, Header, HTTPException
from fastapi import APIRouter, Depends, Header, HTTPException, Request
from sqlalchemy.orm import Session
from ..database import get_db
from ..models import IntegrationInbox
from ..schemas import (
PlatformFulfillmentCustomerBindingAction,
PlatformFulfillmentCustomerSessionBatchRequest,
PlatformFulfillmentCustomerSessionRequest,
)
from ..services.integration.security import (
verify_webhook_signature,
webhook_tolerance_seconds,
)
from ..services.platform_fulfillment import service
router = APIRouter(
@@ -33,16 +40,51 @@ def require_integration_key(
raise HTTPException(status_code=401, detail="直播账号履约集成鉴权失败")
@router.post(
"/events/order-sync",
dependencies=[Depends(require_integration_key)],
)
def sync_order_event(
payload: dict[str, Any],
@router.post("/events/order-sync")
async def sync_order_event(
request: Request,
db: Session = Depends(get_db),
x_integration_key: str = Header("", alias="X-Integration-Key"),
x_integration_event_id: str = Header("", alias="X-Integration-Event-Id"),
x_integration_timestamp: str = Header("", alias="X-Integration-Timestamp"),
x_integration_signature: str = Header("", alias="X-Integration-Signature"),
) -> dict[str, Any]:
del payload
return service.sync_pending_tasks(db, limit=500)
require_integration_key(x_integration_key)
raw_body = await request.body()
if not verify_webhook_signature(
secret=x_integration_key,
timestamp=x_integration_timestamp,
raw_body=raw_body,
signature=x_integration_signature,
tolerance_seconds=webhook_tolerance_seconds(),
):
raise HTTPException(status_code=401, detail="直播账号履约 Webhook 签名无效")
try:
payload = json.loads(raw_body)
except json.JSONDecodeError as exc:
raise HTTPException(status_code=400, detail="Webhook JSON 无效") from exc
if not isinstance(payload, dict):
raise HTTPException(status_code=400, detail="Webhook 载荷必须是对象")
event_id = str(payload.get("eventId") or "").strip()
event_type = str(payload.get("eventType") or "").strip()
if not event_id or event_id != x_integration_event_id.strip():
raise HTTPException(status_code=400, detail="Webhook 事件编号不一致")
if db.get(IntegrationInbox, event_id) is not None:
return {"fetched": 0, "created": 0, "updated": 0, "duplicate": True}
inbox = IntegrationInbox(
event_id=event_id,
source="order_site",
event_type=event_type,
received_at=datetime.now(UTC),
)
db.add(inbox)
result = service.sync_pending_tasks(db, limit=500, commit=False)
inbox.processed_at = datetime.now(UTC)
db.commit()
return {**result, "duplicate": False}
@router.post(
+13
View File
@@ -188,6 +188,17 @@ class DouyuBatchRunnerCore:
}
asyncio.run_coroutine_threadsafe(self.log_queue.put(event), self.loop)
def _sync_platform_fulfillment(self, db: Session, task: DouyuTask) -> None:
"""绑定/确认任务完成后立即刷新履约投影并写 Outbox。"""
try:
from .platform_fulfillment.service import sync_task_from_binding_runner
sync_task_from_binding_runner(db, douyu_task_id=int(task.id))
except Exception: # noqa: BLE001 - 履约投影失败不能覆盖任务执行结果
logger.exception(
"[douyu] 同步直播账号履约状态失败: task_id={}", task.id
)
def _mark_task(
self,
db: Session,
@@ -203,6 +214,7 @@ class DouyuBatchRunnerCore:
task.finished_at = datetime.now(UTC)
db.commit()
self._push_task_event(task)
self._sync_platform_fulfillment(db, task)
def _update_task_progress(
self,
@@ -218,6 +230,7 @@ class DouyuBatchRunnerCore:
task.result = result
db.commit()
self._push_task_event(task)
self._sync_platform_fulfillment(db, task)
def _upsert_goods(self, db: Session, goods: list[dict]) -> None:
upsert_goods_snapshot(db, goods)
@@ -0,0 +1,16 @@
"""跨系统集成基础能力。"""
from .outbox import (
enqueue_integration_event,
start_integration_outbox_worker,
stop_integration_outbox_worker,
)
from .security import build_webhook_signature, verify_webhook_signature
__all__ = [
"build_webhook_signature",
"enqueue_integration_event",
"start_integration_outbox_worker",
"stop_integration_outbox_worker",
"verify_webhook_signature",
]
+322
View File
@@ -0,0 +1,322 @@
"""数据库事务 Outbox 与签名 Webhook 投递 worker。"""
from __future__ import annotations
import json
import os
import threading
import time
import uuid
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
from typing import Any
from urllib.error import HTTPError, URLError
from urllib.request import Request, urlopen
from loguru import logger
from sqlalchemy.orm import Session
from ...database import SessionLocal
from ...models import IntegrationOutbox
from .security import build_webhook_signature
DEFAULT_INTERVAL_SECONDS = 1.0
DEFAULT_BATCH_SIZE = 20
DEFAULT_MAX_ATTEMPTS = 12
REQUEST_TIMEOUT_SECONDS = 5
PROCESSING_STALE_SECONDS = 120
_stop_event = threading.Event()
_worker_thread: threading.Thread | None = None
_lock = threading.Lock()
@dataclass(frozen=True)
class _ClaimedEvent:
event_id: str
payload_json: str
attempts: int
def enqueue_integration_event(
db: Session,
*,
destination: str,
event_type: str,
payload: dict[str, Any],
dedupe_key: str = "",
) -> bool:
"""把事件加入当前事务;调用方必须在业务状态同一事务内 commit。"""
event_id = str(uuid.uuid4())
body = {
"eventId": event_id,
"eventType": event_type,
"occurredAt": datetime.now(UTC).isoformat(),
**payload,
}
normalized_dedupe = dedupe_key.strip() or None
if normalized_dedupe:
# SessionLocal 关闭了 autoflush;先落当前事务内的业务变更和先前事件,
# 避免同一事务重复调用时看不到刚 add 的 outbox 行。
db.flush()
existing = (
db.query(IntegrationOutbox)
.filter(IntegrationOutbox.dedupe_key == normalized_dedupe)
.first()
)
if existing is not None and existing.status == "dead":
existing.status = "pending"
existing.attempts = 0
existing.next_attempt_at = datetime.now(UTC)
existing.locked_at = None
existing.last_error = ""
return True
if existing is not None:
return False
db.add(
IntegrationOutbox(
event_id=event_id,
destination=destination,
event_type=event_type,
dedupe_key=normalized_dedupe,
payload_json=body,
status="pending",
attempts=0,
next_attempt_at=datetime.now(UTC),
created_at=datetime.now(UTC),
)
)
return True
def outbox_backoff_seconds(attempts: int) -> int:
normalized = max(1, int(attempts or 1))
return min(3600, 5 * 2 ** min(normalized - 1, 10))
def _configured() -> tuple[str, str]:
base_url = os.getenv("ORDER_SITE_PLATFORM_FULFILLMENT_URL", "").strip().rstrip("/")
integration_key = os.getenv("PLATFORM_FULFILLMENT_INTEGRATION_KEY", "").strip()
return base_url, integration_key
def _env_float(name: str, default: float, minimum: float, maximum: float) -> float:
try:
value = float(os.getenv(name, str(default)))
except ValueError:
return default
return max(minimum, min(maximum, value))
def _env_int(name: str, default: int, minimum: int, maximum: int) -> int:
try:
value = int(os.getenv(name, str(default)))
except ValueError:
return default
return max(minimum, min(maximum, value))
def _claim_due_events(limit: int) -> list[_ClaimedEvent]:
db = SessionLocal()
now = datetime.now(UTC)
try:
(
db.query(IntegrationOutbox)
.filter(
IntegrationOutbox.status == "processing",
IntegrationOutbox.locked_at
< now - timedelta(seconds=PROCESSING_STALE_SECONDS),
)
.update(
{
IntegrationOutbox.status: "pending",
IntegrationOutbox.locked_at: None,
IntegrationOutbox.next_attempt_at: now,
},
synchronize_session=False,
)
)
rows = (
db.query(IntegrationOutbox)
.filter(
IntegrationOutbox.status == "pending",
IntegrationOutbox.next_attempt_at <= now,
)
.order_by(IntegrationOutbox.id.asc())
.with_for_update(skip_locked=True)
.limit(limit)
.all()
)
claimed: list[_ClaimedEvent] = []
for row in rows:
row.status = "processing"
row.locked_at = now
claimed.append(
_ClaimedEvent(
event_id=row.event_id,
payload_json=json.dumps(
row.payload_json or {},
ensure_ascii=False,
separators=(",", ":"),
),
attempts=int(row.attempts or 0),
)
)
db.commit()
return claimed
except Exception:
db.rollback()
raise
finally:
db.close()
def _deliver(event: _ClaimedEvent, base_url: str, integration_key: str) -> None:
timestamp = str(int(time.time()))
signature = build_webhook_signature(
integration_key,
timestamp,
event.payload_json.encode("utf-8"),
)
request = Request(
f"{base_url}/api/integration/platform-fulfillment/binding-state",
data=event.payload_json.encode("utf-8"),
method="POST",
headers={
"Content-Type": "application/json",
"X-Integration-Key": integration_key,
"X-Integration-Event-Id": event.event_id,
"X-Integration-Timestamp": timestamp,
"X-Integration-Signature": signature,
},
)
try:
with urlopen(request, timeout=REQUEST_TIMEOUT_SECONDS) as response:
if not 200 <= response.status < 300:
raise RuntimeError(f"order_site 返回 HTTP {response.status}")
except HTTPError as exc:
raise RuntimeError(f"order_site 返回 HTTP {exc.code}") from exc
except (URLError, OSError) as exc:
raise RuntimeError(str(exc)) from exc
def _mark_sent(event_id: str) -> None:
db = SessionLocal()
try:
(
db.query(IntegrationOutbox)
.filter(IntegrationOutbox.event_id == event_id)
.update(
{
IntegrationOutbox.status: "sent",
IntegrationOutbox.sent_at: datetime.now(UTC),
IntegrationOutbox.locked_at: None,
IntegrationOutbox.last_error: "",
},
synchronize_session=False,
)
)
db.commit()
finally:
db.close()
def _mark_failed(event: _ClaimedEvent, error: Exception) -> None:
attempts = event.attempts + 1
max_attempts = _env_int(
"INTEGRATION_OUTBOX_MAX_ATTEMPTS", DEFAULT_MAX_ATTEMPTS, 1, 100
)
dead = attempts >= max_attempts
message = str(error)[:2000]
db = SessionLocal()
try:
(
db.query(IntegrationOutbox)
.filter(IntegrationOutbox.event_id == event.event_id)
.update(
{
IntegrationOutbox.status: "dead" if dead else "pending",
IntegrationOutbox.attempts: attempts,
IntegrationOutbox.locked_at: None,
IntegrationOutbox.next_attempt_at: datetime.now(UTC)
+ (
timedelta(seconds=0)
if dead
else timedelta(seconds=outbox_backoff_seconds(attempts))
),
IntegrationOutbox.last_error: message,
},
synchronize_session=False,
)
)
db.commit()
finally:
db.close()
if dead:
logger.error(
"集成 Outbox 事件超过最大重试次数 event_id={} attempts={} error={}",
event.event_id,
attempts,
message,
)
def _publish_due_events() -> None:
base_url, integration_key = _configured()
if not base_url or not integration_key:
return
batch_size = _env_int(
"INTEGRATION_OUTBOX_BATCH_SIZE", DEFAULT_BATCH_SIZE, 1, 100
)
for event in _claim_due_events(batch_size):
try:
_deliver(event, base_url, integration_key)
_mark_sent(event.event_id)
except Exception as exc: # noqa: BLE001 - 投递失败必须回落数据库重试
_mark_failed(event, exc)
def _run() -> None:
interval = _env_float(
"INTEGRATION_OUTBOX_INTERVAL_SECONDS",
DEFAULT_INTERVAL_SECONDS,
0.25,
60,
)
while not _stop_event.is_set():
try:
_publish_due_events()
except Exception as exc: # noqa: BLE001 - worker 不能因单轮异常退出
logger.warning("集成 Outbox 投递轮次失败: {}", exc)
_stop_event.wait(interval)
def start_integration_outbox_worker() -> bool:
global _worker_thread
with _lock:
if _worker_thread is not None and _worker_thread.is_alive():
return False
_stop_event.clear()
_worker_thread = threading.Thread(
target=_run,
name="integration-outbox-worker",
daemon=True,
)
_worker_thread.start()
base_url, integration_key = _configured()
if base_url and integration_key:
logger.info("事务 Outbox worker 已启动")
else:
logger.warning("未配置 order_site 地址或集成密钥,Outbox worker 仅保持待命")
return True
def stop_integration_outbox_worker() -> None:
global _worker_thread
_stop_event.set()
with _lock:
thread = _worker_thread
_worker_thread = None
if thread is not None and thread.is_alive():
thread.join(timeout=3)
@@ -0,0 +1,72 @@
"""签名 Webhook 的 HMAC 与时间窗校验。"""
from __future__ import annotations
import hashlib
import hmac
import time
DEFAULT_TOLERANCE_SECONDS = 300
def build_webhook_signature(secret: str, timestamp: str, raw_body: bytes) -> str:
content = timestamp.encode("utf-8") + b"." + raw_body
digest = hmac.new(secret.encode("utf-8"), content, hashlib.sha256).hexdigest()
return f"sha256={digest}"
def verify_webhook_signature(
*,
secret: str,
timestamp: str,
raw_body: bytes,
signature: str,
tolerance_seconds: int = DEFAULT_TOLERANCE_SECONDS,
now_seconds: int | None = None,
) -> bool:
return verify_webhook_signature_at(
secret=secret,
timestamp=timestamp,
raw_body=raw_body,
signature=signature,
tolerance_seconds=tolerance_seconds,
now_seconds=now_seconds,
)
def verify_webhook_signature_at(
*,
secret: str,
timestamp: str,
raw_body: bytes,
signature: str,
tolerance_seconds: int = DEFAULT_TOLERANCE_SECONDS,
now_seconds: int | None = None,
) -> bool:
secret = secret.strip()
timestamp = timestamp.strip()
signature = signature.strip()
if not secret or not timestamp or not signature:
return False
try:
timestamp_seconds = int(timestamp)
except ValueError:
return False
if timestamp_seconds <= 0:
return False
tolerance = max(1, int(tolerance_seconds))
current = int(now_seconds if now_seconds is not None else time.time())
if abs(current - timestamp_seconds) > tolerance:
return False
expected = build_webhook_signature(secret, timestamp, raw_body)
return hmac.compare_digest(expected, signature)
def webhook_tolerance_seconds() -> int:
import os
try:
value = int(os.getenv("PLATFORM_FULFILLMENT_WEBHOOK_TOLERANCE_SECONDS", "300"))
except ValueError:
return DEFAULT_TOLERANCE_SECONDS
return value if value > 0 else DEFAULT_TOLERANCE_SECONDS
@@ -1,86 +1,53 @@
"""把直播账号履约状态变化主动推送给 order_site。"""
"""把直播账号履约状态变化写入 order_site 的事务 Outbox。"""
from __future__ import annotations
import hashlib
import json
import os
import time
import uuid
from datetime import UTC, datetime
from threading import Thread
from urllib.error import HTTPError, URLError
from urllib.request import Request, urlopen
from typing import Any
from loguru import logger
from sqlalchemy.orm import Session
from ...database import SessionLocal
from ..integration.outbox import enqueue_integration_event
PUBLISH_ATTEMPTS = 3
PUBLISH_TIMEOUT_SECONDS = 5
EVENT_TYPE = "platform_fulfillment.binding_state_changed"
def notify_order_site_binding_state_async(task_id: int) -> None:
"""状态提交后异步推送;失败由工作台低频对账补齐。"""
Thread(
target=_publish_binding_state,
args=(int(task_id),),
name=f"order-site-fulfillment-{task_id}",
daemon=True,
).start()
def _state_dedupe_key(payload: dict[str, Any]) -> str:
state = {
"sessionId": payload.get("sessionId"),
"executionStatus": payload.get("executionStatus"),
"bindingStatus": payload.get("bindingStatus"),
"exchangeStatus": payload.get("exchangeStatus"),
"issueCode": payload.get("issueCode"),
"accountReady": payload.get("accountReady"),
"bindTaskStatus": payload.get("bindTaskStatus"),
"bindPhase": payload.get("bindPhase"),
"qrUrl": payload.get("qrUrl"),
"roleName": payload.get("roleName"),
"areaName": payload.get("areaName"),
"platName": payload.get("platName"),
"bindReadyForConfirm": payload.get("bindReadyForConfirm"),
"bindConfirmed": payload.get("bindConfirmed"),
}
digest = hashlib.sha256(
json.dumps(state, ensure_ascii=False, sort_keys=True).encode("utf-8")
).hexdigest()[:32]
return f"binding-state:{payload.get('sessionId')}:{digest}"
def _publish_binding_state(task_id: int) -> None:
base_url = os.getenv("ORDER_SITE_PLATFORM_FULFILLMENT_URL", "").strip().rstrip("/")
integration_key = os.getenv("PLATFORM_FULFILLMENT_INTEGRATION_KEY", "").strip()
if not base_url or not integration_key:
return
db = SessionLocal()
try:
from . import service
payload = service.customer_binding_out(db, task_id)
except Exception as exc: # noqa: BLE001
logger.warning("构造 order_site 履约状态事件失败 task_id={}: {}", task_id, exc)
return
finally:
db.close()
body = json.dumps(
{
"eventId": str(uuid.uuid4()),
"occurredAt": datetime.now(UTC).isoformat(),
"items": [payload],
},
ensure_ascii=False,
).encode("utf-8")
endpoint = f"{base_url}/api/integration/platform-fulfillment/binding-state"
for attempt in range(1, PUBLISH_ATTEMPTS + 1):
request = Request(
endpoint,
data=body,
method="POST",
headers={
"Content-Type": "application/json",
"X-Integration-Key": integration_key,
},
)
try:
with urlopen(request, timeout=PUBLISH_TIMEOUT_SECONDS) as response:
if 200 <= response.status < 300:
return
raise RuntimeError(f"order_site 返回 HTTP {response.status}")
except (HTTPError, URLError, OSError, RuntimeError) as exc:
if attempt >= PUBLISH_ATTEMPTS:
logger.warning(
"推送 order_site 履约状态失败 task_id={} attempts={}: {}",
task_id,
attempt,
exc,
)
return
time.sleep(float(attempt))
def enqueue_order_site_binding_state(
db: Session,
payload: dict[str, Any],
) -> bool:
"""必须在任务状态变更的同一事务内调用。"""
return enqueue_integration_event(
db,
destination="order_site",
event_type=EVENT_TYPE,
payload={"items": [payload]},
dedupe_key=_state_dedupe_key(payload),
)
__all__ = ["notify_order_site_binding_state_async"]
__all__ = ["enqueue_order_site_binding_state"]
@@ -0,0 +1,72 @@
"""低频订单对账兜底;正常同步由 order_site 签名 Webhook 驱动。"""
from __future__ import annotations
import os
import threading
from loguru import logger
from ...database import SessionLocal
_stop_event = threading.Event()
_thread: threading.Thread | None = None
_lock = threading.Lock()
def _interval_seconds() -> int:
try:
value = int(os.getenv("PLATFORM_FULFILLMENT_RECONCILE_SECONDS", "900"))
except ValueError:
return 900
return max(60, value)
def _run() -> None:
from . import service
while not _stop_event.is_set():
# 不抢占启动时的 Webhook;首轮延后一分钟。
if _stop_event.wait(min(60, _interval_seconds())):
break
db = SessionLocal()
try:
result = service.sync_pending_tasks(db, limit=500)
queued = service.reconcile_binding_state_outbox(db, limit=500)
logger.info(
"直播账号履约低频对账完成: order_sync={} outbox_requeued={}",
result,
queued,
)
except Exception as exc: # noqa: BLE001 - 对账失败等待下一轮
db.rollback()
logger.warning("直播账号履约低频对账失败: {}", exc)
finally:
db.close()
if _stop_event.wait(_interval_seconds()):
break
def start_platform_fulfillment_reconciler() -> bool:
global _thread
with _lock:
if _thread is not None and _thread.is_alive():
return False
_stop_event.clear()
_thread = threading.Thread(
target=_run,
name="platform-fulfillment-reconciler",
daemon=True,
)
_thread.start()
return True
def stop_platform_fulfillment_reconciler() -> None:
global _thread
_stop_event.set()
with _lock:
thread = _thread
_thread = None
if thread is not None and thread.is_alive():
thread.join(timeout=3)
@@ -40,7 +40,7 @@ from .account_selector import (
release_lease,
renew_lease,
)
from .order_site_notifier import notify_order_site_binding_state_async
from .order_site_notifier import enqueue_order_site_binding_state
TARGET_PLATFORM = "douyu"
SCOPE_ELITE = "elite"
@@ -55,7 +55,7 @@ ACTIVE_STATUSES = {
}
BINDING_QR_TTL_SECONDS = int(os.getenv("PLATFORM_BINDING_QR_TTL_SECONDS", "600"))
AUTO_SYNC_INTERVAL_SECONDS = 60.0
AUTO_SYNC_INTERVAL_SECONDS = 900.0
_SYNC_LOCK = Lock()
_AUTO_SYNC_LOCK = Lock()
@@ -228,6 +228,14 @@ def _append_event(
)
def _emit_order_site_binding_state(db: Session, task_id: int) -> bool:
"""当前事务内写 Outbox;HTTP 投递由独立 worker 完成。"""
return enqueue_order_site_binding_state(
db,
customer_binding_out(db, task_id),
)
def _load_task(db: Session, task_id: int) -> PlatformFulfillmentTask:
task = (
db.query(PlatformFulfillmentTask)
@@ -259,7 +267,12 @@ def _apply_mapping(
task.last_error = ""
def _sync_pending_tasks(db: Session, *, limit: int = 100) -> dict[str, Any]:
def _sync_pending_tasks(
db: Session,
*,
limit: int = 100,
commit: bool = True,
) -> dict[str, Any]:
"""从 order_site 拉取人工待履约订单项并幂等生成本地作业。"""
try:
rows = list_pending_manual_fulfillments(limit=limit)
@@ -320,15 +333,22 @@ def _sync_pending_tasks(db: Session, *, limit: int = 100) -> dict[str, Any]:
task.updated_at = _now()
if task.execution_status in {"pending_mapping", "ready"}:
_apply_mapping(task, _mapping_for_sku(db, task.internal_sku_key))
_emit_order_site_binding_state(db, task.id)
db.commit()
if commit:
db.commit()
return {"fetched": len(rows), "created": created, "updated": updated}
def sync_pending_tasks(db: Session, *, limit: int = 100) -> dict[str, Any]:
def sync_pending_tasks(
db: Session,
*,
limit: int = 100,
commit: bool = True,
) -> dict[str, Any]:
"""串行同步订单,避免自动同步与人工同步并发创建重复任务。"""
with _SYNC_LOCK:
return _sync_pending_tasks(db, limit=limit)
return _sync_pending_tasks(db, limit=limit, commit=commit)
def sync_pending_tasks_if_due(
@@ -539,7 +559,7 @@ def _customer_task_message(task: PlatformFulfillmentTask) -> str:
"elite_not_opened": "账号正在准备精英宝典,请稍后重试",
"points_short": "账号正在补充积分,请稍后重试",
"commodity_limit_reached": "当前商品暂时无法兑换,请联系客服",
"binding_failed": "绑定失败,请刷新二维码后重试",
"binding_failed": "暂未识别到绑定角色,请继续使用当前二维码完成扫码",
"eligibility_check_failed": "状态查询异常,请稍后刷新",
"manual_required": "当前需要人工处理,请联系客服",
}
@@ -892,8 +912,8 @@ def allocate_task(db: Session, *, task_id: int, current: User) -> dict[str, Any]
"diagnostics": diagnostics,
},
)
_emit_order_site_binding_state(db, task.id)
db.commit()
notify_order_site_binding_state_async(task.id)
return get_task_detail(db, task.id)
task.account_id = selected.account_id
@@ -936,8 +956,8 @@ def allocate_task(db: Session, *, task_id: int, current: User) -> dict[str, Any]
"exchangeStatus": task.exchange_status,
},
)
_emit_order_site_binding_state(db, task.id)
db.commit()
notify_order_site_binding_state_async(task.id)
return get_task_detail(db, task.id)
@@ -972,8 +992,8 @@ def release_task_account(
actor_id=current.id,
detail={"reason": reason},
)
_emit_order_site_binding_state(db, task.id)
db.commit()
notify_order_site_binding_state_async(task.id)
return get_task_detail(db, task.id)
@@ -1044,8 +1064,8 @@ async def start_binding(
actor_id=current.id,
detail={"bindTaskId": bind_task.id},
)
_emit_order_site_binding_state(db, task.id)
db.commit()
notify_order_site_binding_state_async(task.id)
start_douyu_batch(
batch_id=batch_id,
task_type="get_bind_qr",
@@ -1090,8 +1110,6 @@ def refresh_binding(db: Session, *, task_id: int) -> dict[str, Any]:
task.execution_status = "waiting_bind"
task.issue_code = ""
task.last_error = ""
task.updated_at = _now()
db.commit()
after = (
task.binding_status,
task.execution_status,
@@ -1100,10 +1118,50 @@ def refresh_binding(db: Session, *, task_id: int) -> dict[str, Any]:
task.last_error,
)
if after != before:
notify_order_site_binding_state_async(task.id)
task.updated_at = _now()
_emit_order_site_binding_state(db, task.id)
db.commit()
return get_task_detail(db, task.id)
def sync_task_from_binding_runner(db: Session, *, douyu_task_id: int) -> int:
"""斗鱼绑定任务落库后,把关联履约任务和 Outbox 一并刷新。"""
task_ids = [
int(row.id)
for row in (
db.query(PlatformFulfillmentTask.id)
.filter(PlatformFulfillmentTask.bind_task_id == int(douyu_task_id))
.all()
)
]
for task_id in task_ids:
refresh_binding(db, task_id=task_id)
return len(task_ids)
def reconcile_binding_state_outbox(db: Session, *, limit: int = 500) -> int:
"""低频补齐 active 任务当前快照;只会在缺失或 dead 时重新入队。"""
task_ids = [
int(row.id)
for row in (
db.query(PlatformFulfillmentTask.id)
.filter(
PlatformFulfillmentTask.execution_status.notin_(
["completed", "cancelled"]
)
)
.order_by(PlatformFulfillmentTask.updated_at.asc())
.limit(max(1, min(limit, 500)))
.all()
)
]
queued = sum(
1 for task_id in task_ids if _emit_order_site_binding_state(db, task_id)
)
db.commit()
return queued
async def confirm_binding(
db: Session, *, task_id: int, current: User
) -> dict[str, Any]:
@@ -1145,8 +1203,8 @@ async def confirm_binding(
actor_id=current.id,
detail={"confirmTaskId": confirm_task.id},
)
_emit_order_site_binding_state(db, task.id)
db.commit()
notify_order_site_binding_state_async(task.id)
start_douyu_batch(
batch_id=batch_id,
task_type="confirm_bind",
@@ -1192,8 +1250,8 @@ def refresh_exchange_eligibility(
actor_id=current.id,
detail={"error": str(exc)},
)
_emit_order_site_binding_state(db, task.id)
db.commit()
notify_order_site_binding_state_async(task.id)
return get_task_detail(db, task.id)
if task.commodity_id in eligibility.limited_ids:
@@ -1228,8 +1286,8 @@ def refresh_exchange_eligibility(
"limitedCount": len(eligibility.limited_ids),
},
)
_emit_order_site_binding_state(db, task.id)
db.commit()
notify_order_site_binding_state_async(task.id)
return get_task_detail(db, task.id)
@@ -1269,8 +1327,8 @@ def complete_manual_exchange(
actor_id=current.id,
detail={"note": note[:1000]},
)
_emit_order_site_binding_state(db, task.id)
db.commit()
notify_order_site_binding_state_async(task.id)
return get_task_detail(db, task.id)
@@ -1295,8 +1353,8 @@ def mark_manual_required(
actor_id=current.id,
detail={"reason": task.last_error},
)
_emit_order_site_binding_state(db, task.id)
db.commit()
notify_order_site_binding_state_async(task.id)
return get_task_detail(db, task.id)
@@ -1326,8 +1384,8 @@ def cancel_task(
actor_id=current.id,
detail={"reason": task.last_error},
)
_emit_order_site_binding_state(db, task.id)
db.commit()
notify_order_site_binding_state_async(task.id)
return get_task_detail(db, task.id)
@@ -1341,11 +1399,13 @@ __all__ = [
"list_mappings",
"list_tasks",
"mark_manual_required",
"reconcile_binding_state_outbox",
"refresh_binding",
"refresh_exchange_eligibility",
"release_task_account",
"start_binding",
"sync_pending_tasks",
"sync_pending_tasks_if_due",
"sync_task_from_binding_runner",
"upsert_mapping",
]