perf(douyu): 拆分任务槽并改造绑定扫码调度

This commit is contained in:
yml
2026-11-20 22:03:20 +08:00
parent 8869e427d2
commit 071d45ad1d
40 changed files with 1635 additions and 245 deletions
@@ -0,0 +1,145 @@
"""add durable scheduling fields to douyu_tasks
Long waits such as QR scan polling must survive a process restart without
holding a worker thread. The scheduler claims due rows by lane and next_run_at.
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
revision: str = "20261120_0101"
down_revision: str | None = "20261120_0100"
branch_labels: str | None = None
depends_on: str | None = None
INDEX_NAME = "ix_douyu_tasks_lane_due"
def _columns() -> set[str]:
inspector = sa.inspect(op.get_bind())
if not inspector.has_table("douyu_tasks"):
return set()
return {column["name"] for column in inspector.get_columns("douyu_tasks")}
def _indexes() -> set[str]:
inspector = sa.inspect(op.get_bind())
if not inspector.has_table("douyu_tasks"):
return set()
return {index["name"] for index in inspector.get_indexes("douyu_tasks")}
def upgrade() -> None:
columns = _columns()
if "queue_lane" not in columns:
op.add_column(
"douyu_tasks",
sa.Column(
"queue_lane",
sa.String(length=32),
nullable=False,
server_default="",
),
)
if "next_run_at" not in columns:
op.add_column(
"douyu_tasks",
sa.Column("next_run_at", sa.DateTime(), nullable=True),
)
if "run_deadline_at" not in columns:
op.add_column(
"douyu_tasks",
sa.Column("run_deadline_at", sa.DateTime(), nullable=True),
)
if "last_run_at" not in columns:
op.add_column(
"douyu_tasks",
sa.Column("last_run_at", sa.DateTime(), nullable=True),
)
if "attempt_count" not in columns:
op.add_column(
"douyu_tasks",
sa.Column(
"attempt_count",
sa.Integer(),
nullable=False,
server_default="0",
),
)
if "active_binding_key" not in columns:
op.add_column(
"douyu_tasks",
sa.Column(
"active_binding_key",
sa.String(length=64),
sa.Computed(
"CASE WHEN task_type = 'get_bind_qr' "
"AND platform_task_id IS NOT NULL "
"AND status IN ('planned', 'pending', 'running') "
"THEN CONCAT('bind:', platform_task_id) ELSE NULL END",
persisted=True,
),
nullable=True,
),
)
# Keep the newest active QR task if legacy concurrency produced duplicates.
# Older runtime rows are superseded; historical executed tasks are untouched.
op.execute(
sa.text(
"""
UPDATE douyu_tasks AS task
JOIN (
SELECT platform_task_id, MAX(id) AS keep_id
FROM douyu_tasks
WHERE task_type = 'get_bind_qr'
AND platform_task_id IS NOT NULL
AND status IN ('planned', 'pending', 'running')
GROUP BY platform_task_id
HAVING COUNT(*) > 1
) AS duplicate
ON duplicate.platform_task_id = task.platform_task_id
SET task.status = 'stopped',
task.message = '已被新的绑定二维码任务取代(迁移收口)',
task.finished_at = UTC_TIMESTAMP(),
task.next_run_at = NULL
WHERE task.task_type = 'get_bind_qr'
AND task.status IN ('planned', 'pending', 'running')
AND task.id <> duplicate.keep_id
"""
)
)
if INDEX_NAME not in _indexes():
op.create_index(
INDEX_NAME,
"douyu_tasks",
["queue_lane", "status", "next_run_at", "id"],
unique=False,
)
indexes = _indexes()
if "uq_douyu_tasks_active_binding" not in indexes:
op.create_index(
"uq_douyu_tasks_active_binding",
"douyu_tasks",
["active_binding_key"],
unique=True,
)
def downgrade() -> None:
if "uq_douyu_tasks_active_binding" in _indexes():
op.drop_index("uq_douyu_tasks_active_binding", table_name="douyu_tasks")
if INDEX_NAME in _indexes():
op.drop_index(INDEX_NAME, table_name="douyu_tasks")
columns = _columns()
for name in (
"active_binding_key",
"attempt_count",
"last_run_at",
"run_deadline_at",
"next_run_at",
"queue_lane",
):
if name in columns:
op.drop_column("douyu_tasks", name)