- 新增直播履约状态事务 Outbox 与签名投递 worker - 增加 order_site 状态回传、重试和低频对账兜底 - 补充 Outbox、签名校验与集成接口测试
323 lines
9.8 KiB
Python
323 lines
9.8 KiB
Python
"""数据库事务 Outbox 与签名 Webhook 投递 worker。"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from dataclasses import dataclass
|
|
from datetime import UTC, datetime, timedelta
|
|
from typing import Any
|
|
from urllib.error import HTTPError, URLError
|
|
from urllib.request import Request, urlopen
|
|
|
|
from loguru import logger
|
|
from sqlalchemy.orm import Session
|
|
|
|
from ...database import SessionLocal
|
|
from ...models import IntegrationOutbox
|
|
from .security import build_webhook_signature
|
|
|
|
DEFAULT_INTERVAL_SECONDS = 1.0
|
|
DEFAULT_BATCH_SIZE = 20
|
|
DEFAULT_MAX_ATTEMPTS = 12
|
|
REQUEST_TIMEOUT_SECONDS = 5
|
|
PROCESSING_STALE_SECONDS = 120
|
|
|
|
_stop_event = threading.Event()
|
|
_worker_thread: threading.Thread | None = None
|
|
_lock = threading.Lock()
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class _ClaimedEvent:
|
|
event_id: str
|
|
payload_json: str
|
|
attempts: int
|
|
|
|
|
|
def enqueue_integration_event(
|
|
db: Session,
|
|
*,
|
|
destination: str,
|
|
event_type: str,
|
|
payload: dict[str, Any],
|
|
dedupe_key: str = "",
|
|
) -> bool:
|
|
"""把事件加入当前事务;调用方必须在业务状态同一事务内 commit。"""
|
|
event_id = str(uuid.uuid4())
|
|
body = {
|
|
"eventId": event_id,
|
|
"eventType": event_type,
|
|
"occurredAt": datetime.now(UTC).isoformat(),
|
|
**payload,
|
|
}
|
|
normalized_dedupe = dedupe_key.strip() or None
|
|
if normalized_dedupe:
|
|
# SessionLocal 关闭了 autoflush;先落当前事务内的业务变更和先前事件,
|
|
# 避免同一事务重复调用时看不到刚 add 的 outbox 行。
|
|
db.flush()
|
|
existing = (
|
|
db.query(IntegrationOutbox)
|
|
.filter(IntegrationOutbox.dedupe_key == normalized_dedupe)
|
|
.first()
|
|
)
|
|
if existing is not None and existing.status == "dead":
|
|
existing.status = "pending"
|
|
existing.attempts = 0
|
|
existing.next_attempt_at = datetime.now(UTC)
|
|
existing.locked_at = None
|
|
existing.last_error = ""
|
|
return True
|
|
if existing is not None:
|
|
return False
|
|
|
|
db.add(
|
|
IntegrationOutbox(
|
|
event_id=event_id,
|
|
destination=destination,
|
|
event_type=event_type,
|
|
dedupe_key=normalized_dedupe,
|
|
payload_json=body,
|
|
status="pending",
|
|
attempts=0,
|
|
next_attempt_at=datetime.now(UTC),
|
|
created_at=datetime.now(UTC),
|
|
)
|
|
)
|
|
return True
|
|
|
|
|
|
def outbox_backoff_seconds(attempts: int) -> int:
|
|
normalized = max(1, int(attempts or 1))
|
|
return min(3600, 5 * 2 ** min(normalized - 1, 10))
|
|
|
|
|
|
def _configured() -> tuple[str, str]:
|
|
base_url = os.getenv("ORDER_SITE_PLATFORM_FULFILLMENT_URL", "").strip().rstrip("/")
|
|
integration_key = os.getenv("PLATFORM_FULFILLMENT_INTEGRATION_KEY", "").strip()
|
|
return base_url, integration_key
|
|
|
|
|
|
def _env_float(name: str, default: float, minimum: float, maximum: float) -> float:
|
|
try:
|
|
value = float(os.getenv(name, str(default)))
|
|
except ValueError:
|
|
return default
|
|
return max(minimum, min(maximum, value))
|
|
|
|
|
|
def _env_int(name: str, default: int, minimum: int, maximum: int) -> int:
|
|
try:
|
|
value = int(os.getenv(name, str(default)))
|
|
except ValueError:
|
|
return default
|
|
return max(minimum, min(maximum, value))
|
|
|
|
|
|
def _claim_due_events(limit: int) -> list[_ClaimedEvent]:
|
|
db = SessionLocal()
|
|
now = datetime.now(UTC)
|
|
try:
|
|
(
|
|
db.query(IntegrationOutbox)
|
|
.filter(
|
|
IntegrationOutbox.status == "processing",
|
|
IntegrationOutbox.locked_at
|
|
< now - timedelta(seconds=PROCESSING_STALE_SECONDS),
|
|
)
|
|
.update(
|
|
{
|
|
IntegrationOutbox.status: "pending",
|
|
IntegrationOutbox.locked_at: None,
|
|
IntegrationOutbox.next_attempt_at: now,
|
|
},
|
|
synchronize_session=False,
|
|
)
|
|
)
|
|
rows = (
|
|
db.query(IntegrationOutbox)
|
|
.filter(
|
|
IntegrationOutbox.status == "pending",
|
|
IntegrationOutbox.next_attempt_at <= now,
|
|
)
|
|
.order_by(IntegrationOutbox.id.asc())
|
|
.with_for_update(skip_locked=True)
|
|
.limit(limit)
|
|
.all()
|
|
)
|
|
claimed: list[_ClaimedEvent] = []
|
|
for row in rows:
|
|
row.status = "processing"
|
|
row.locked_at = now
|
|
claimed.append(
|
|
_ClaimedEvent(
|
|
event_id=row.event_id,
|
|
payload_json=json.dumps(
|
|
row.payload_json or {},
|
|
ensure_ascii=False,
|
|
separators=(",", ":"),
|
|
),
|
|
attempts=int(row.attempts or 0),
|
|
)
|
|
)
|
|
db.commit()
|
|
return claimed
|
|
except Exception:
|
|
db.rollback()
|
|
raise
|
|
finally:
|
|
db.close()
|
|
|
|
|
|
def _deliver(event: _ClaimedEvent, base_url: str, integration_key: str) -> None:
|
|
timestamp = str(int(time.time()))
|
|
signature = build_webhook_signature(
|
|
integration_key,
|
|
timestamp,
|
|
event.payload_json.encode("utf-8"),
|
|
)
|
|
request = Request(
|
|
f"{base_url}/api/integration/platform-fulfillment/binding-state",
|
|
data=event.payload_json.encode("utf-8"),
|
|
method="POST",
|
|
headers={
|
|
"Content-Type": "application/json",
|
|
"X-Integration-Key": integration_key,
|
|
"X-Integration-Event-Id": event.event_id,
|
|
"X-Integration-Timestamp": timestamp,
|
|
"X-Integration-Signature": signature,
|
|
},
|
|
)
|
|
try:
|
|
with urlopen(request, timeout=REQUEST_TIMEOUT_SECONDS) as response:
|
|
if not 200 <= response.status < 300:
|
|
raise RuntimeError(f"order_site 返回 HTTP {response.status}")
|
|
except HTTPError as exc:
|
|
raise RuntimeError(f"order_site 返回 HTTP {exc.code}") from exc
|
|
except (URLError, OSError) as exc:
|
|
raise RuntimeError(str(exc)) from exc
|
|
|
|
|
|
def _mark_sent(event_id: str) -> None:
|
|
db = SessionLocal()
|
|
try:
|
|
(
|
|
db.query(IntegrationOutbox)
|
|
.filter(IntegrationOutbox.event_id == event_id)
|
|
.update(
|
|
{
|
|
IntegrationOutbox.status: "sent",
|
|
IntegrationOutbox.sent_at: datetime.now(UTC),
|
|
IntegrationOutbox.locked_at: None,
|
|
IntegrationOutbox.last_error: "",
|
|
},
|
|
synchronize_session=False,
|
|
)
|
|
)
|
|
db.commit()
|
|
finally:
|
|
db.close()
|
|
|
|
|
|
def _mark_failed(event: _ClaimedEvent, error: Exception) -> None:
|
|
attempts = event.attempts + 1
|
|
max_attempts = _env_int(
|
|
"INTEGRATION_OUTBOX_MAX_ATTEMPTS", DEFAULT_MAX_ATTEMPTS, 1, 100
|
|
)
|
|
dead = attempts >= max_attempts
|
|
message = str(error)[:2000]
|
|
db = SessionLocal()
|
|
try:
|
|
(
|
|
db.query(IntegrationOutbox)
|
|
.filter(IntegrationOutbox.event_id == event.event_id)
|
|
.update(
|
|
{
|
|
IntegrationOutbox.status: "dead" if dead else "pending",
|
|
IntegrationOutbox.attempts: attempts,
|
|
IntegrationOutbox.locked_at: None,
|
|
IntegrationOutbox.next_attempt_at: datetime.now(UTC)
|
|
+ (
|
|
timedelta(seconds=0)
|
|
if dead
|
|
else timedelta(seconds=outbox_backoff_seconds(attempts))
|
|
),
|
|
IntegrationOutbox.last_error: message,
|
|
},
|
|
synchronize_session=False,
|
|
)
|
|
)
|
|
db.commit()
|
|
finally:
|
|
db.close()
|
|
if dead:
|
|
logger.error(
|
|
"集成 Outbox 事件超过最大重试次数 event_id={} attempts={} error={}",
|
|
event.event_id,
|
|
attempts,
|
|
message,
|
|
)
|
|
|
|
|
|
def _publish_due_events() -> None:
|
|
base_url, integration_key = _configured()
|
|
if not base_url or not integration_key:
|
|
return
|
|
batch_size = _env_int(
|
|
"INTEGRATION_OUTBOX_BATCH_SIZE", DEFAULT_BATCH_SIZE, 1, 100
|
|
)
|
|
for event in _claim_due_events(batch_size):
|
|
try:
|
|
_deliver(event, base_url, integration_key)
|
|
_mark_sent(event.event_id)
|
|
except Exception as exc: # noqa: BLE001 - 投递失败必须回落数据库重试
|
|
_mark_failed(event, exc)
|
|
|
|
|
|
def _run() -> None:
|
|
interval = _env_float(
|
|
"INTEGRATION_OUTBOX_INTERVAL_SECONDS",
|
|
DEFAULT_INTERVAL_SECONDS,
|
|
0.25,
|
|
60,
|
|
)
|
|
while not _stop_event.is_set():
|
|
try:
|
|
_publish_due_events()
|
|
except Exception as exc: # noqa: BLE001 - worker 不能因单轮异常退出
|
|
logger.warning("集成 Outbox 投递轮次失败: {}", exc)
|
|
_stop_event.wait(interval)
|
|
|
|
|
|
def start_integration_outbox_worker() -> bool:
|
|
global _worker_thread
|
|
with _lock:
|
|
if _worker_thread is not None and _worker_thread.is_alive():
|
|
return False
|
|
_stop_event.clear()
|
|
_worker_thread = threading.Thread(
|
|
target=_run,
|
|
name="integration-outbox-worker",
|
|
daemon=True,
|
|
)
|
|
_worker_thread.start()
|
|
base_url, integration_key = _configured()
|
|
if base_url and integration_key:
|
|
logger.info("事务 Outbox worker 已启动")
|
|
else:
|
|
logger.warning("未配置 order_site 地址或集成密钥,Outbox worker 仅保持待命")
|
|
return True
|
|
|
|
|
|
def stop_integration_outbox_worker() -> None:
|
|
global _worker_thread
|
|
_stop_event.set()
|
|
with _lock:
|
|
thread = _worker_thread
|
|
_worker_thread = None
|
|
if thread is not None and thread.is_alive():
|
|
thread.join(timeout=3)
|