feat(fulfillment): 增加签名 Webhook 事务 Outbox
- 新增直播履约状态事务 Outbox 与签名投递 worker - 增加 order_site 状态回传、重试和低频对账兜底 - 补充 Outbox、签名校验与集成接口测试
This commit is contained in:
@@ -0,0 +1,16 @@
|
||||
"""跨系统集成基础能力。"""
|
||||
|
||||
from .outbox import (
|
||||
enqueue_integration_event,
|
||||
start_integration_outbox_worker,
|
||||
stop_integration_outbox_worker,
|
||||
)
|
||||
from .security import build_webhook_signature, verify_webhook_signature
|
||||
|
||||
__all__ = [
|
||||
"build_webhook_signature",
|
||||
"enqueue_integration_event",
|
||||
"start_integration_outbox_worker",
|
||||
"stop_integration_outbox_worker",
|
||||
"verify_webhook_signature",
|
||||
]
|
||||
@@ -0,0 +1,322 @@
|
||||
"""数据库事务 Outbox 与签名 Webhook 投递 worker。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
from loguru import logger
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ...database import SessionLocal
|
||||
from ...models import IntegrationOutbox
|
||||
from .security import build_webhook_signature
|
||||
|
||||
DEFAULT_INTERVAL_SECONDS = 1.0
|
||||
DEFAULT_BATCH_SIZE = 20
|
||||
DEFAULT_MAX_ATTEMPTS = 12
|
||||
REQUEST_TIMEOUT_SECONDS = 5
|
||||
PROCESSING_STALE_SECONDS = 120
|
||||
|
||||
_stop_event = threading.Event()
|
||||
_worker_thread: threading.Thread | None = None
|
||||
_lock = threading.Lock()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _ClaimedEvent:
|
||||
event_id: str
|
||||
payload_json: str
|
||||
attempts: int
|
||||
|
||||
|
||||
def enqueue_integration_event(
|
||||
db: Session,
|
||||
*,
|
||||
destination: str,
|
||||
event_type: str,
|
||||
payload: dict[str, Any],
|
||||
dedupe_key: str = "",
|
||||
) -> bool:
|
||||
"""把事件加入当前事务;调用方必须在业务状态同一事务内 commit。"""
|
||||
event_id = str(uuid.uuid4())
|
||||
body = {
|
||||
"eventId": event_id,
|
||||
"eventType": event_type,
|
||||
"occurredAt": datetime.now(UTC).isoformat(),
|
||||
**payload,
|
||||
}
|
||||
normalized_dedupe = dedupe_key.strip() or None
|
||||
if normalized_dedupe:
|
||||
# SessionLocal 关闭了 autoflush;先落当前事务内的业务变更和先前事件,
|
||||
# 避免同一事务重复调用时看不到刚 add 的 outbox 行。
|
||||
db.flush()
|
||||
existing = (
|
||||
db.query(IntegrationOutbox)
|
||||
.filter(IntegrationOutbox.dedupe_key == normalized_dedupe)
|
||||
.first()
|
||||
)
|
||||
if existing is not None and existing.status == "dead":
|
||||
existing.status = "pending"
|
||||
existing.attempts = 0
|
||||
existing.next_attempt_at = datetime.now(UTC)
|
||||
existing.locked_at = None
|
||||
existing.last_error = ""
|
||||
return True
|
||||
if existing is not None:
|
||||
return False
|
||||
|
||||
db.add(
|
||||
IntegrationOutbox(
|
||||
event_id=event_id,
|
||||
destination=destination,
|
||||
event_type=event_type,
|
||||
dedupe_key=normalized_dedupe,
|
||||
payload_json=body,
|
||||
status="pending",
|
||||
attempts=0,
|
||||
next_attempt_at=datetime.now(UTC),
|
||||
created_at=datetime.now(UTC),
|
||||
)
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def outbox_backoff_seconds(attempts: int) -> int:
|
||||
normalized = max(1, int(attempts or 1))
|
||||
return min(3600, 5 * 2 ** min(normalized - 1, 10))
|
||||
|
||||
|
||||
def _configured() -> tuple[str, str]:
|
||||
base_url = os.getenv("ORDER_SITE_PLATFORM_FULFILLMENT_URL", "").strip().rstrip("/")
|
||||
integration_key = os.getenv("PLATFORM_FULFILLMENT_INTEGRATION_KEY", "").strip()
|
||||
return base_url, integration_key
|
||||
|
||||
|
||||
def _env_float(name: str, default: float, minimum: float, maximum: float) -> float:
|
||||
try:
|
||||
value = float(os.getenv(name, str(default)))
|
||||
except ValueError:
|
||||
return default
|
||||
return max(minimum, min(maximum, value))
|
||||
|
||||
|
||||
def _env_int(name: str, default: int, minimum: int, maximum: int) -> int:
|
||||
try:
|
||||
value = int(os.getenv(name, str(default)))
|
||||
except ValueError:
|
||||
return default
|
||||
return max(minimum, min(maximum, value))
|
||||
|
||||
|
||||
def _claim_due_events(limit: int) -> list[_ClaimedEvent]:
|
||||
db = SessionLocal()
|
||||
now = datetime.now(UTC)
|
||||
try:
|
||||
(
|
||||
db.query(IntegrationOutbox)
|
||||
.filter(
|
||||
IntegrationOutbox.status == "processing",
|
||||
IntegrationOutbox.locked_at
|
||||
< now - timedelta(seconds=PROCESSING_STALE_SECONDS),
|
||||
)
|
||||
.update(
|
||||
{
|
||||
IntegrationOutbox.status: "pending",
|
||||
IntegrationOutbox.locked_at: None,
|
||||
IntegrationOutbox.next_attempt_at: now,
|
||||
},
|
||||
synchronize_session=False,
|
||||
)
|
||||
)
|
||||
rows = (
|
||||
db.query(IntegrationOutbox)
|
||||
.filter(
|
||||
IntegrationOutbox.status == "pending",
|
||||
IntegrationOutbox.next_attempt_at <= now,
|
||||
)
|
||||
.order_by(IntegrationOutbox.id.asc())
|
||||
.with_for_update(skip_locked=True)
|
||||
.limit(limit)
|
||||
.all()
|
||||
)
|
||||
claimed: list[_ClaimedEvent] = []
|
||||
for row in rows:
|
||||
row.status = "processing"
|
||||
row.locked_at = now
|
||||
claimed.append(
|
||||
_ClaimedEvent(
|
||||
event_id=row.event_id,
|
||||
payload_json=json.dumps(
|
||||
row.payload_json or {},
|
||||
ensure_ascii=False,
|
||||
separators=(",", ":"),
|
||||
),
|
||||
attempts=int(row.attempts or 0),
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
return claimed
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def _deliver(event: _ClaimedEvent, base_url: str, integration_key: str) -> None:
|
||||
timestamp = str(int(time.time()))
|
||||
signature = build_webhook_signature(
|
||||
integration_key,
|
||||
timestamp,
|
||||
event.payload_json.encode("utf-8"),
|
||||
)
|
||||
request = Request(
|
||||
f"{base_url}/api/integration/platform-fulfillment/binding-state",
|
||||
data=event.payload_json.encode("utf-8"),
|
||||
method="POST",
|
||||
headers={
|
||||
"Content-Type": "application/json",
|
||||
"X-Integration-Key": integration_key,
|
||||
"X-Integration-Event-Id": event.event_id,
|
||||
"X-Integration-Timestamp": timestamp,
|
||||
"X-Integration-Signature": signature,
|
||||
},
|
||||
)
|
||||
try:
|
||||
with urlopen(request, timeout=REQUEST_TIMEOUT_SECONDS) as response:
|
||||
if not 200 <= response.status < 300:
|
||||
raise RuntimeError(f"order_site 返回 HTTP {response.status}")
|
||||
except HTTPError as exc:
|
||||
raise RuntimeError(f"order_site 返回 HTTP {exc.code}") from exc
|
||||
except (URLError, OSError) as exc:
|
||||
raise RuntimeError(str(exc)) from exc
|
||||
|
||||
|
||||
def _mark_sent(event_id: str) -> None:
|
||||
db = SessionLocal()
|
||||
try:
|
||||
(
|
||||
db.query(IntegrationOutbox)
|
||||
.filter(IntegrationOutbox.event_id == event_id)
|
||||
.update(
|
||||
{
|
||||
IntegrationOutbox.status: "sent",
|
||||
IntegrationOutbox.sent_at: datetime.now(UTC),
|
||||
IntegrationOutbox.locked_at: None,
|
||||
IntegrationOutbox.last_error: "",
|
||||
},
|
||||
synchronize_session=False,
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def _mark_failed(event: _ClaimedEvent, error: Exception) -> None:
|
||||
attempts = event.attempts + 1
|
||||
max_attempts = _env_int(
|
||||
"INTEGRATION_OUTBOX_MAX_ATTEMPTS", DEFAULT_MAX_ATTEMPTS, 1, 100
|
||||
)
|
||||
dead = attempts >= max_attempts
|
||||
message = str(error)[:2000]
|
||||
db = SessionLocal()
|
||||
try:
|
||||
(
|
||||
db.query(IntegrationOutbox)
|
||||
.filter(IntegrationOutbox.event_id == event.event_id)
|
||||
.update(
|
||||
{
|
||||
IntegrationOutbox.status: "dead" if dead else "pending",
|
||||
IntegrationOutbox.attempts: attempts,
|
||||
IntegrationOutbox.locked_at: None,
|
||||
IntegrationOutbox.next_attempt_at: datetime.now(UTC)
|
||||
+ (
|
||||
timedelta(seconds=0)
|
||||
if dead
|
||||
else timedelta(seconds=outbox_backoff_seconds(attempts))
|
||||
),
|
||||
IntegrationOutbox.last_error: message,
|
||||
},
|
||||
synchronize_session=False,
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
finally:
|
||||
db.close()
|
||||
if dead:
|
||||
logger.error(
|
||||
"集成 Outbox 事件超过最大重试次数 event_id={} attempts={} error={}",
|
||||
event.event_id,
|
||||
attempts,
|
||||
message,
|
||||
)
|
||||
|
||||
|
||||
def _publish_due_events() -> None:
|
||||
base_url, integration_key = _configured()
|
||||
if not base_url or not integration_key:
|
||||
return
|
||||
batch_size = _env_int(
|
||||
"INTEGRATION_OUTBOX_BATCH_SIZE", DEFAULT_BATCH_SIZE, 1, 100
|
||||
)
|
||||
for event in _claim_due_events(batch_size):
|
||||
try:
|
||||
_deliver(event, base_url, integration_key)
|
||||
_mark_sent(event.event_id)
|
||||
except Exception as exc: # noqa: BLE001 - 投递失败必须回落数据库重试
|
||||
_mark_failed(event, exc)
|
||||
|
||||
|
||||
def _run() -> None:
|
||||
interval = _env_float(
|
||||
"INTEGRATION_OUTBOX_INTERVAL_SECONDS",
|
||||
DEFAULT_INTERVAL_SECONDS,
|
||||
0.25,
|
||||
60,
|
||||
)
|
||||
while not _stop_event.is_set():
|
||||
try:
|
||||
_publish_due_events()
|
||||
except Exception as exc: # noqa: BLE001 - worker 不能因单轮异常退出
|
||||
logger.warning("集成 Outbox 投递轮次失败: {}", exc)
|
||||
_stop_event.wait(interval)
|
||||
|
||||
|
||||
def start_integration_outbox_worker() -> bool:
|
||||
global _worker_thread
|
||||
with _lock:
|
||||
if _worker_thread is not None and _worker_thread.is_alive():
|
||||
return False
|
||||
_stop_event.clear()
|
||||
_worker_thread = threading.Thread(
|
||||
target=_run,
|
||||
name="integration-outbox-worker",
|
||||
daemon=True,
|
||||
)
|
||||
_worker_thread.start()
|
||||
base_url, integration_key = _configured()
|
||||
if base_url and integration_key:
|
||||
logger.info("事务 Outbox worker 已启动")
|
||||
else:
|
||||
logger.warning("未配置 order_site 地址或集成密钥,Outbox worker 仅保持待命")
|
||||
return True
|
||||
|
||||
|
||||
def stop_integration_outbox_worker() -> None:
|
||||
global _worker_thread
|
||||
_stop_event.set()
|
||||
with _lock:
|
||||
thread = _worker_thread
|
||||
_worker_thread = None
|
||||
if thread is not None and thread.is_alive():
|
||||
thread.join(timeout=3)
|
||||
@@ -0,0 +1,72 @@
|
||||
"""签名 Webhook 的 HMAC 与时间窗校验。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
import time
|
||||
|
||||
DEFAULT_TOLERANCE_SECONDS = 300
|
||||
|
||||
|
||||
def build_webhook_signature(secret: str, timestamp: str, raw_body: bytes) -> str:
|
||||
content = timestamp.encode("utf-8") + b"." + raw_body
|
||||
digest = hmac.new(secret.encode("utf-8"), content, hashlib.sha256).hexdigest()
|
||||
return f"sha256={digest}"
|
||||
|
||||
|
||||
def verify_webhook_signature(
|
||||
*,
|
||||
secret: str,
|
||||
timestamp: str,
|
||||
raw_body: bytes,
|
||||
signature: str,
|
||||
tolerance_seconds: int = DEFAULT_TOLERANCE_SECONDS,
|
||||
now_seconds: int | None = None,
|
||||
) -> bool:
|
||||
return verify_webhook_signature_at(
|
||||
secret=secret,
|
||||
timestamp=timestamp,
|
||||
raw_body=raw_body,
|
||||
signature=signature,
|
||||
tolerance_seconds=tolerance_seconds,
|
||||
now_seconds=now_seconds,
|
||||
)
|
||||
|
||||
|
||||
def verify_webhook_signature_at(
|
||||
*,
|
||||
secret: str,
|
||||
timestamp: str,
|
||||
raw_body: bytes,
|
||||
signature: str,
|
||||
tolerance_seconds: int = DEFAULT_TOLERANCE_SECONDS,
|
||||
now_seconds: int | None = None,
|
||||
) -> bool:
|
||||
secret = secret.strip()
|
||||
timestamp = timestamp.strip()
|
||||
signature = signature.strip()
|
||||
if not secret or not timestamp or not signature:
|
||||
return False
|
||||
try:
|
||||
timestamp_seconds = int(timestamp)
|
||||
except ValueError:
|
||||
return False
|
||||
if timestamp_seconds <= 0:
|
||||
return False
|
||||
tolerance = max(1, int(tolerance_seconds))
|
||||
current = int(now_seconds if now_seconds is not None else time.time())
|
||||
if abs(current - timestamp_seconds) > tolerance:
|
||||
return False
|
||||
expected = build_webhook_signature(secret, timestamp, raw_body)
|
||||
return hmac.compare_digest(expected, signature)
|
||||
|
||||
|
||||
def webhook_tolerance_seconds() -> int:
|
||||
import os
|
||||
|
||||
try:
|
||||
value = int(os.getenv("PLATFORM_FULFILLMENT_WEBHOOK_TOLERANCE_SECONDS", "300"))
|
||||
except ValueError:
|
||||
return DEFAULT_TOLERANCE_SECONDS
|
||||
return value if value > 0 else DEFAULT_TOLERANCE_SECONDS
|
||||
Reference in New Issue
Block a user