chenshoulin
58850314cf
用「WCF + 通用存储过程网关」接口替代 Selenium（D://Learning//Python//工单自动化）导入工单： 单 Sprint 12 个项目 175s 跑完（旧方案 25~35 分钟），失败可断点续跑且不重复建单。 - 分层：transport（HTTP/协议）/ api（存储过程语义）/ domain（纯规则）/ workflow（阶段编排）/ store（SQLite 台账）/ report / CLI - 幂等：台账驱动断点续跑；dry-run 与跳过行不污染进度；--rollback-on-fail 清理半成品 - 命令：plan / apply / report / status，真实写入需 --yes - tools：只读探针、存储过程源码导出（sp_helptext）、导入诊断 - docs：调研与设计文档（含联调踩到的 5 个坑）+ 6 个关键存储过程源码 实测：Sprint 2026 Sprint 10 的 12 个项目全部完成建单→导入→确认→推送 TFS， 工单 225134–225145，35 个功能模块，重跑 plan 全部 skip-pushed。69 个单测通过。
417 lines
18 KiB
Python
417 lines
18 KiB
Python
"""workflow 层：单项目编排、阶段推进、补偿与并发。
|
||
|
||
阶段顺序固定，台账里记到哪一步，下次就从下一步开始 —— 这是断点续跑的唯一依据。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
import threading
|
||
import time
|
||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||
from dataclasses import dataclass, field
|
||
|
||
from . import domain
|
||
from .api import QuickTeamApi
|
||
from .config import Config
|
||
from .domain import Action, PlanRow
|
||
from .errors import Category, QtError
|
||
from .models import Project, User, WorkSheet, WorkSheetVersion
|
||
from .report import RunLog
|
||
from .store import Store
|
||
|
||
STAGES = ["WS_CREATED", "WS_FILLED", "EXCEL_IMPORTED", "WS_VERIFIED", "TFS_PUSHED"]
|
||
STAGE_INDEX = {stage: index for index, stage in enumerate(STAGES)}
|
||
|
||
|
||
@dataclass
|
||
class RunOptions:
|
||
adopt: bool = False # 接管无台账记录的既有工单
|
||
skip_tfs: bool = False # 只建单+导入+确认，不推送
|
||
rollback_on_fail: bool = False
|
||
limit: int | None = None
|
||
projects: list[str] = field(default_factory=list)
|
||
force: bool = False # 跳过 Excel 模板预检
|
||
dry_run: bool = False # 只演练：不产生任何服务端写入
|
||
|
||
|
||
@dataclass
|
||
class ProjectProgress:
|
||
row: PlanRow
|
||
stage: str
|
||
worksheet_id: int | None = None
|
||
version_id: int | None = None
|
||
elapsed_ms: int = 0
|
||
|
||
|
||
class Importer:
|
||
def __init__(self, cfg: Config, store: Store, run_id: str, log: RunLog, options: RunOptions) -> None:
|
||
self.cfg = cfg
|
||
self.store = store
|
||
self.run_id = run_id
|
||
self.log = log
|
||
self.options = options
|
||
|
||
self._thread_state = threading.local()
|
||
self.api = QuickTeamApi(
|
||
self._build_transport(),
|
||
dry_run=options.dry_run,
|
||
log=lambda message: self.log.info(message, level="dry"),
|
||
)
|
||
self._tfs_lock = threading.Lock()
|
||
self._tfs_context: tuple[str, str] | None = None
|
||
self.user: User | None = None
|
||
self.team_name = ""
|
||
|
||
# ------------------------------------------------------------------ 装配
|
||
|
||
def _build_transport(self):
|
||
from .transport import Transport
|
||
|
||
def on_call(op: str, elapsed: int, ok: bool, detail: str) -> None:
|
||
project = getattr(self._thread_state, "project", "")
|
||
self.store.add_event(
|
||
self.run_id,
|
||
stage="API",
|
||
action="ok" if ok else "fail",
|
||
project_id=getattr(self._thread_state, "project_id", ""),
|
||
detail=f"{op} {detail}".strip(),
|
||
elapsed_ms=elapsed,
|
||
)
|
||
self.log.event("API", "ok" if ok else "fail", project=project, detail=op, elapsed_ms=elapsed)
|
||
|
||
return Transport(
|
||
self.cfg.base_url, timeout=self.cfg.timeout, retries=2, on_call=on_call
|
||
)
|
||
|
||
# ------------------------------------------------------------------ plan
|
||
|
||
def connect(self) -> User:
|
||
if not self.user:
|
||
self.user = self.api.login(self.cfg.username, self.cfg.password)
|
||
teams = self.api.teams(self.user.user_id)
|
||
team = self.api.resolve_team(teams, self.cfg.team_name)
|
||
self.team_name = team.team_name
|
||
self.team_id = team.team_id
|
||
self.log.info(f"已登录：{self.user.real_name}（{self.user.user_name}），团队：{team.team_name}")
|
||
return self.user
|
||
|
||
def plan(self) -> list[PlanRow]:
|
||
self.connect()
|
||
projects = self.api.projects(self.team_id)
|
||
files = domain.list_xlsx(self.cfg.work_dir)
|
||
rows: list[PlanRow] = []
|
||
|
||
for path in files:
|
||
stem = os.path.splitext(os.path.basename(path))[0]
|
||
project, reason = domain.match_project(stem, projects)
|
||
if project is None:
|
||
rows.append(PlanRow(project_name=stem, file_path=path, action=Action.ERROR, reason=reason))
|
||
continue
|
||
|
||
if not self.options.force:
|
||
ok, why = domain.precheck_xlsx(path)
|
||
if not ok:
|
||
rows.append(
|
||
PlanRow(project_name=project.project_name, file_path=path, action=Action.ERROR,
|
||
project=project, reason=f"Excel 预检失败：{why}")
|
||
)
|
||
continue
|
||
|
||
ledger = self.store.latest_job(project.project_id, self.cfg.sprint)
|
||
existing = [
|
||
sheet for sheet in self.api.work_sheets(project.project_id)
|
||
if sheet.system_name == self.cfg.sprint
|
||
]
|
||
|
||
action, why, worksheet = domain.decide(
|
||
existing, ledger_stage=ledger["stage"] if ledger else None, adopt=self.options.adopt
|
||
)
|
||
rows.append(
|
||
PlanRow(
|
||
project_name=project.project_name,
|
||
file_path=path,
|
||
action=action,
|
||
reason=why,
|
||
project=project,
|
||
worksheet=worksheet,
|
||
existing_count=len(existing),
|
||
)
|
||
)
|
||
return rows
|
||
|
||
# ------------------------------------------------------------------ apply
|
||
|
||
def apply(self) -> list[ProjectProgress]:
|
||
rows = self.plan()
|
||
write_rows = [row for row in rows if row.action.writes]
|
||
if self.options.projects:
|
||
wanted = {domain.normalize_name(name) for name in self.options.projects}
|
||
write_rows = [row for row in write_rows if domain.normalize_name(row.project_name) in wanted]
|
||
if self.options.limit:
|
||
write_rows = write_rows[: self.options.limit]
|
||
|
||
# 非执行行也落台账，报告里才能看到「为什么跳过」
|
||
for row in rows:
|
||
if row.action.writes:
|
||
continue
|
||
project_id = row.project.project_id if row.project else row.project_name
|
||
if self.store.latest_job(project_id, self.cfg.sprint):
|
||
# 已经有真实进度记录，别用 SKIPPED 覆盖它（否则下次会把「已推送」当成「无进度」）
|
||
self.log.event(str(row.action), "skip", project=row.project_name, detail=row.reason)
|
||
continue
|
||
self.store.upsert_job(
|
||
self.run_id,
|
||
project_id=project_id,
|
||
project_name=row.project_name,
|
||
system_name=self.cfg.sprint,
|
||
source_file=row.file_path,
|
||
action=str(row.action),
|
||
stage="SKIPPED" if row.action != Action.ERROR else "FAILED",
|
||
worksheet_id=row.worksheet.worksheet_id if row.worksheet else None,
|
||
reason=row.reason,
|
||
error=row.reason if row.action == Action.ERROR else "",
|
||
)
|
||
self.log.event(str(row.action), "skip", project=row.project_name, detail=row.reason)
|
||
|
||
if not write_rows:
|
||
self.log.info("没有需要执行的项目")
|
||
return []
|
||
|
||
self.log.info(f"开始执行 {len(write_rows)} 个项目（并发 {self.cfg.workers}）")
|
||
results: list[ProjectProgress] = []
|
||
with ThreadPoolExecutor(max_workers=max(1, self.cfg.workers)) as pool:
|
||
futures = {pool.submit(self._run_project, row): row for row in write_rows}
|
||
for future in as_completed(futures):
|
||
row = futures[future]
|
||
try:
|
||
results.append(future.result())
|
||
except QtError as error:
|
||
if error.category in {Category.AUTH, Category.SP_MISSING}:
|
||
self.log.warn(f"中止：{error}")
|
||
raise
|
||
results.append(ProjectProgress(row=row, stage="FAILED"))
|
||
self.log.warn(f"{row.project_name} 失败：{error}")
|
||
except Exception as error: # 兜底，避免一个项目的异常带崩整批
|
||
results.append(ProjectProgress(row=row, stage="FAILED"))
|
||
self.log.warn(f"{row.project_name} 未预期异常：{error!r}")
|
||
return results
|
||
|
||
def _run_project(self, row: PlanRow) -> ProjectProgress:
|
||
assert row.project is not None
|
||
started = time.time()
|
||
self._thread_state.project = row.project_name
|
||
self._thread_state.project_id = row.project.project_id
|
||
|
||
ledger = self.store.latest_job(row.project.project_id, self.cfg.sprint)
|
||
if row.action == Action.CREATE:
|
||
# plan 已经确认服务端没有该 Sprint 的工单，台账里的旧记录可能来自已被回滚的
|
||
# 半成品（工单已删除）。此时必须忽略台账，否则会去填一张不存在的工单。
|
||
ledger = None
|
||
done_upto = self._completed_index(ledger["stage"] if ledger else "")
|
||
worksheet_id = (ledger or {}).get("worksheet_id") or (
|
||
row.worksheet.worksheet_id if row.worksheet else None
|
||
)
|
||
version_id: int | None = (ledger or {}).get("version_id")
|
||
|
||
def need(stage: str) -> bool:
|
||
return STAGE_INDEX[stage] > done_upto
|
||
|
||
stage = "PLANNED"
|
||
created_here = row.action == Action.CREATE
|
||
try:
|
||
if need("WS_CREATED"):
|
||
stage = "WS_CREATED"
|
||
worksheet_id = self.api.create_worksheet()
|
||
self._mark(row, stage, worksheet_id, None)
|
||
|
||
if need("WS_FILLED"):
|
||
stage = "WS_FILLED"
|
||
assert worksheet_id is not None
|
||
evaluator, technical, customer = self.cfg.resolved_roles(self.user.real_name)
|
||
self.api.fill_worksheet(
|
||
worksheet_id,
|
||
row.project,
|
||
system_name=self.cfg.sprint,
|
||
start_time=self.cfg.start_time,
|
||
end_time=self.cfg.end_time,
|
||
evaluator=evaluator,
|
||
technical_leader=technical,
|
||
customer_leader=customer,
|
||
belong_team=self.cfg.belong_team,
|
||
content=self.cfg.content_desc,
|
||
operator=self.user.real_name,
|
||
service_type=self.cfg.service_type,
|
||
)
|
||
self._mark(row, stage, worksheet_id, None)
|
||
|
||
if need("EXCEL_IMPORTED"):
|
||
stage = "EXCEL_IMPORTED"
|
||
assert worksheet_id is not None
|
||
version = self._version(row.project.project_id, worksheet_id)
|
||
version_id = version.version_id
|
||
remote = self.api.upload_excel(row.file_path)
|
||
self.api.import_excel(worksheet_id, version_id, remote)
|
||
self._mark(row, stage, worksheet_id, version_id)
|
||
|
||
if need("WS_VERIFIED"):
|
||
stage = "WS_VERIFIED"
|
||
assert worksheet_id is not None
|
||
detail = self._detail(worksheet_id)
|
||
version = self._version(row.project.project_id, worksheet_id)
|
||
if detail.done:
|
||
self.log.event(stage, "skip", project=row.project_name, detail="工单已完成，无需确认")
|
||
elif version is None:
|
||
raise QtError(f"工单 {worksheet_id} 没有版本记录，无法确认", Category.PARAM)
|
||
elif domain.VERSION_TRANSITIONS.get(version.version_code) is None:
|
||
self.log.event(stage, "skip", project=row.project_name, detail="已是流程终点，无需确认")
|
||
else:
|
||
try:
|
||
next_operator, next_code, percent = domain.next_transition(version.version_code)
|
||
except ValueError as error:
|
||
raise QtError(str(error), Category.PARAM) from None
|
||
self.api.verify_worksheet(
|
||
worksheet_id,
|
||
version.version_id,
|
||
next_operator=next_operator,
|
||
next_version_code=next_code,
|
||
operator=self.user.real_name,
|
||
percent=percent,
|
||
)
|
||
self._mark(row, stage, worksheet_id, version.version_id if version else version_id)
|
||
|
||
if need("TFS_PUSHED"):
|
||
assert worksheet_id is not None
|
||
if self.options.skip_tfs:
|
||
self.log.event("TFS_PUSHED", "skip", project=row.project_name, detail="--skip-tfs")
|
||
else:
|
||
stage = "TFS_PUSHED"
|
||
area, iteration = self._resolve_tfs()
|
||
features = [] if self.api.dry_run else [
|
||
item.feature_id for item in self.api.features(worksheet_id)
|
||
]
|
||
if not features and not self.api.dry_run:
|
||
raise QtError("工单没有功能模块，跳过推送（请检查 Excel 是否为空表）", Category.EXCEL)
|
||
created = self.api.push_tfs(features, self.cfg.tfs_project, area, iteration)
|
||
self.log.event(
|
||
"TFS_PUSHED", "ok", project=row.project_name,
|
||
detail=f"features={len(features)} 创建={created}",
|
||
)
|
||
self._mark(row, stage, worksheet_id, version_id)
|
||
|
||
elapsed = int((time.time() - started) * 1000)
|
||
self.log.info(f" ✓ {row.project_name} → {stage} {elapsed / 1000:.1f}s")
|
||
return ProjectProgress(row=row, stage=stage, worksheet_id=worksheet_id,
|
||
version_id=version_id, elapsed_ms=elapsed)
|
||
|
||
except QtError as error:
|
||
elapsed = int((time.time() - started) * 1000)
|
||
self.store.upsert_job(
|
||
self.run_id,
|
||
project_id=row.project.project_id,
|
||
project_name=row.project_name,
|
||
system_name=self.cfg.sprint,
|
||
source_file=row.file_path,
|
||
action=str(row.action),
|
||
stage=f"FAILED:{stage}",
|
||
worksheet_id=worksheet_id,
|
||
version_id=version_id,
|
||
reason=row.reason,
|
||
error=str(error),
|
||
)
|
||
self.store.add_event(
|
||
self.run_id, stage=stage, action="fail", project_id=row.project.project_id,
|
||
detail=str(error), elapsed_ms=elapsed,
|
||
)
|
||
self.log.warn(f"{row.project_name} 在 {stage} 失败：{error}")
|
||
|
||
if self.options.rollback_on_fail and created_here and STAGE_INDEX[stage] < STAGE_INDEX["WS_VERIFIED"]:
|
||
self._rollback(row, worksheet_id, stage)
|
||
if error.category in {Category.AUTH, Category.SP_MISSING}:
|
||
raise
|
||
return ProjectProgress(row=row, stage=f"FAILED:{stage}", worksheet_id=worksheet_id,
|
||
version_id=version_id, elapsed_ms=elapsed)
|
||
finally:
|
||
self._thread_state.project = ""
|
||
self._thread_state.project_id = ""
|
||
|
||
# ------------------------------------------------------------------ 辅助
|
||
|
||
@staticmethod
|
||
def _completed_index(stage: str) -> int:
|
||
"""台账阶段 → 已完成的阶段下标。
|
||
|
||
`FAILED:EXCEL_IMPORTED` 表示「导入这一步失败了」，所以它之前的都算完成，
|
||
重跑要从 EXCEL_IMPORTED 本身重试，而不是从头再来（否则会重复建单）。
|
||
"""
|
||
if not stage:
|
||
return -1
|
||
if stage.startswith("FAILED:"):
|
||
failed_at = stage.split(":", 1)[1]
|
||
return STAGE_INDEX.get(failed_at, 0) - 1
|
||
return STAGE_INDEX.get(stage, -1)
|
||
|
||
def _detail(self, worksheet_id: int) -> WorkSheet:
|
||
if self.api.dry_run:
|
||
return WorkSheet(worksheet_id=worksheet_id, version_id=0, version_code="")
|
||
return self.api.worksheet(worksheet_id)
|
||
|
||
def _version(self, project_id: str, worksheet_id: int) -> WorkSheetVersion | None:
|
||
"""当前待确认版本。导入明细与确认工单都挂在它上面。
|
||
|
||
`WorkSheet.CurrentVersionId` 在第一次保存后仍为 NULL（要等确认才回填），
|
||
所以不能拿它当 VersionId 用 —— 那会把 WorkSheetTask 插到不存在的版本上，
|
||
报 FK_WorkSheetTask_WorkSheetVersion 冲突。
|
||
"""
|
||
if self.api.dry_run:
|
||
return WorkSheetVersion(version_id=0, worksheet_id=worksheet_id,
|
||
version_code="技术责任人确认版", version_status="N")
|
||
return self.api.pending_version(project_id, worksheet_id)
|
||
|
||
def _resolve_tfs(self) -> tuple[str, str]:
|
||
if self.cfg.area_path and self.cfg.iteration_path:
|
||
return self.cfg.area_path, self.cfg.iteration_path
|
||
with self._tfs_lock:
|
||
if self._tfs_context:
|
||
return self._tfs_context
|
||
projects = self.api.tfs_projects()
|
||
target = next((p for p in projects if p.get("Name") == self.cfg.tfs_project), None)
|
||
if target is None:
|
||
raise QtError(f"TFS 项目「{self.cfg.tfs_project}」不存在", Category.TFS)
|
||
teams = self.api.tfs_teams(self.cfg.tfs_project, target["ProjectId"])
|
||
area, iteration = QuickTeamApi.resolve_tfs_context(teams, self.cfg.tfs_team, self.cfg.sprint)
|
||
self._tfs_context = (area, iteration)
|
||
self.log.info(f"TFS 上下文：area={area} iteration={iteration}")
|
||
return self._tfs_context
|
||
|
||
def _mark(
|
||
self, row: PlanRow, stage: str, worksheet_id: int | None, version_id: int | None
|
||
) -> None:
|
||
assert row.project is not None
|
||
self.store.upsert_job(
|
||
self.run_id,
|
||
project_id=row.project.project_id,
|
||
project_name=row.project_name,
|
||
system_name=self.cfg.sprint,
|
||
source_file=row.file_path,
|
||
action=str(row.action),
|
||
stage=stage,
|
||
worksheet_id=worksheet_id,
|
||
version_id=version_id,
|
||
reason=row.reason,
|
||
)
|
||
self.log.event(stage, "ok", project=row.project_name, detail=f"ws={worksheet_id}")
|
||
|
||
def _rollback(self, row: PlanRow, worksheet_id: int | None, stage: str) -> None:
|
||
if not worksheet_id:
|
||
return
|
||
try:
|
||
self.api.delete_worksheet(worksheet_id)
|
||
self.store.add_event(
|
||
self.run_id, stage=stage, action="rollback", project_id=row.project.project_id,
|
||
detail=f"已删除半成品工单 {worksheet_id}",
|
||
)
|
||
self.log.warn(f"{row.project_name} 已回滚（删除工单 {worksheet_id}）")
|
||
except QtError as error:
|
||
self.log.warn(f"{row.project_name} 回滚失败：{error}")
|