"""数据库事务 Outbox 与签名 Webhook 投递 worker。""" from __future__ import annotations import json import os import threading import time import uuid from dataclasses import dataclass from datetime import UTC, datetime, timedelta from typing import Any from urllib.error import HTTPError, URLError from urllib.request import Request, urlopen from loguru import logger from sqlalchemy.orm import Session from ...database import SessionLocal from ...models import IntegrationOutbox from .security import build_webhook_signature DEFAULT_INTERVAL_SECONDS = 1.0 DEFAULT_BATCH_SIZE = 20 DEFAULT_MAX_ATTEMPTS = 12 REQUEST_TIMEOUT_SECONDS = 5 PROCESSING_STALE_SECONDS = 120 _stop_event = threading.Event() _worker_thread: threading.Thread | None = None _lock = threading.Lock() @dataclass(frozen=True) class _ClaimedEvent: event_id: str payload_json: str attempts: int def enqueue_integration_event( db: Session, *, destination: str, event_type: str, payload: dict[str, Any], dedupe_key: str = "", ) -> bool: """把事件加入当前事务;调用方必须在业务状态同一事务内 commit。""" event_id = str(uuid.uuid4()) body = { "eventId": event_id, "eventType": event_type, "occurredAt": datetime.now(UTC).isoformat(), **payload, } normalized_dedupe = dedupe_key.strip() or None if normalized_dedupe: # SessionLocal 关闭了 autoflush;先落当前事务内的业务变更和先前事件, # 避免同一事务重复调用时看不到刚 add 的 outbox 行。 db.flush() existing = ( db.query(IntegrationOutbox) .filter(IntegrationOutbox.dedupe_key == normalized_dedupe) .first() ) if existing is not None and existing.status == "dead": existing.status = "pending" existing.attempts = 0 existing.next_attempt_at = datetime.now(UTC) existing.locked_at = None existing.last_error = "" return True if existing is not None: return False db.add( IntegrationOutbox( event_id=event_id, destination=destination, event_type=event_type, dedupe_key=normalized_dedupe, payload_json=body, status="pending", attempts=0, next_attempt_at=datetime.now(UTC), created_at=datetime.now(UTC), ) ) return True def outbox_backoff_seconds(attempts: int) -> int: normalized = max(1, int(attempts or 1)) return min(3600, 5 * 2 ** min(normalized - 1, 10)) def _configured() -> tuple[str, str]: base_url = os.getenv("ORDER_SITE_PLATFORM_FULFILLMENT_URL", "").strip().rstrip("/") integration_key = os.getenv("PLATFORM_FULFILLMENT_INTEGRATION_KEY", "").strip() return base_url, integration_key def _env_float(name: str, default: float, minimum: float, maximum: float) -> float: try: value = float(os.getenv(name, str(default))) except ValueError: return default return max(minimum, min(maximum, value)) def _env_int(name: str, default: int, minimum: int, maximum: int) -> int: try: value = int(os.getenv(name, str(default))) except ValueError: return default return max(minimum, min(maximum, value)) def _claim_due_events(limit: int) -> list[_ClaimedEvent]: db = SessionLocal() now = datetime.now(UTC) try: ( db.query(IntegrationOutbox) .filter( IntegrationOutbox.status == "processing", IntegrationOutbox.locked_at < now - timedelta(seconds=PROCESSING_STALE_SECONDS), ) .update( { IntegrationOutbox.status: "pending", IntegrationOutbox.locked_at: None, IntegrationOutbox.next_attempt_at: now, }, synchronize_session=False, ) ) rows = ( db.query(IntegrationOutbox) .filter( IntegrationOutbox.status == "pending", IntegrationOutbox.next_attempt_at <= now, ) .order_by(IntegrationOutbox.id.asc()) .with_for_update(skip_locked=True) .limit(limit) .all() ) claimed: list[_ClaimedEvent] = [] for row in rows: row.status = "processing" row.locked_at = now claimed.append( _ClaimedEvent( event_id=row.event_id, payload_json=json.dumps( row.payload_json or {}, ensure_ascii=False, separators=(",", ":"), ), attempts=int(row.attempts or 0), ) ) db.commit() return claimed except Exception: db.rollback() raise finally: db.close() def _deliver(event: _ClaimedEvent, base_url: str, integration_key: str) -> None: timestamp = str(int(time.time())) signature = build_webhook_signature( integration_key, timestamp, event.payload_json.encode("utf-8"), ) request = Request( f"{base_url}/api/integration/platform-fulfillment/binding-state", data=event.payload_json.encode("utf-8"), method="POST", headers={ "Content-Type": "application/json", "X-Integration-Key": integration_key, "X-Integration-Event-Id": event.event_id, "X-Integration-Timestamp": timestamp, "X-Integration-Signature": signature, }, ) try: with urlopen(request, timeout=REQUEST_TIMEOUT_SECONDS) as response: if not 200 <= response.status < 300: raise RuntimeError(f"order_site 返回 HTTP {response.status}") except HTTPError as exc: raise RuntimeError(f"order_site 返回 HTTP {exc.code}") from exc except (URLError, OSError) as exc: raise RuntimeError(str(exc)) from exc def _mark_sent(event_id: str) -> None: db = SessionLocal() try: ( db.query(IntegrationOutbox) .filter(IntegrationOutbox.event_id == event_id) .update( { IntegrationOutbox.status: "sent", IntegrationOutbox.sent_at: datetime.now(UTC), IntegrationOutbox.locked_at: None, IntegrationOutbox.last_error: "", }, synchronize_session=False, ) ) db.commit() finally: db.close() def _mark_failed(event: _ClaimedEvent, error: Exception) -> None: attempts = event.attempts + 1 max_attempts = _env_int( "INTEGRATION_OUTBOX_MAX_ATTEMPTS", DEFAULT_MAX_ATTEMPTS, 1, 100 ) dead = attempts >= max_attempts message = str(error)[:2000] db = SessionLocal() try: ( db.query(IntegrationOutbox) .filter(IntegrationOutbox.event_id == event.event_id) .update( { IntegrationOutbox.status: "dead" if dead else "pending", IntegrationOutbox.attempts: attempts, IntegrationOutbox.locked_at: None, IntegrationOutbox.next_attempt_at: datetime.now(UTC) + ( timedelta(seconds=0) if dead else timedelta(seconds=outbox_backoff_seconds(attempts)) ), IntegrationOutbox.last_error: message, }, synchronize_session=False, ) ) db.commit() finally: db.close() if dead: logger.error( "集成 Outbox 事件超过最大重试次数 event_id={} attempts={} error={}", event.event_id, attempts, message, ) def _publish_due_events() -> None: base_url, integration_key = _configured() if not base_url or not integration_key: return batch_size = _env_int( "INTEGRATION_OUTBOX_BATCH_SIZE", DEFAULT_BATCH_SIZE, 1, 100 ) for event in _claim_due_events(batch_size): try: _deliver(event, base_url, integration_key) _mark_sent(event.event_id) except Exception as exc: # noqa: BLE001 - 投递失败必须回落数据库重试 _mark_failed(event, exc) def _run() -> None: interval = _env_float( "INTEGRATION_OUTBOX_INTERVAL_SECONDS", DEFAULT_INTERVAL_SECONDS, 0.25, 60, ) while not _stop_event.is_set(): try: _publish_due_events() except Exception as exc: # noqa: BLE001 - worker 不能因单轮异常退出 logger.warning("集成 Outbox 投递轮次失败: {}", exc) _stop_event.wait(interval) def start_integration_outbox_worker() -> bool: global _worker_thread with _lock: if _worker_thread is not None and _worker_thread.is_alive(): return False _stop_event.clear() _worker_thread = threading.Thread( target=_run, name="integration-outbox-worker", daemon=True, ) _worker_thread.start() base_url, integration_key = _configured() if base_url and integration_key: logger.info("事务 Outbox worker 已启动") else: logger.warning("未配置 order_site 地址或集成密钥,Outbox worker 仅保持待命") return True def stop_integration_outbox_worker() -> None: global _worker_thread _stop_event.set() with _lock: thread = _worker_thread _worker_thread = None if thread is not None and thread.is_alive(): thread.join(timeout=3)