"""workflow 层:单项目编排、阶段推进、补偿与并发。 阶段顺序固定,台账里记到哪一步,下次就从下一步开始 —— 这是断点续跑的唯一依据。 """ from __future__ import annotations import os import threading import time from concurrent.futures import ThreadPoolExecutor, as_completed from dataclasses import dataclass, field from . import domain from .api import QuickTeamApi from .config import Config from .domain import Action, PlanRow from .errors import Category, QtError from .models import Project, User, WorkSheet, WorkSheetVersion from .report import RunLog from .store import Store STAGES = ["WS_CREATED", "WS_FILLED", "EXCEL_IMPORTED", "WS_VERIFIED", "TFS_PUSHED"] STAGE_INDEX = {stage: index for index, stage in enumerate(STAGES)} @dataclass class RunOptions: adopt: bool = False # 接管无台账记录的既有工单 skip_tfs: bool = False # 只建单+导入+确认,不推送 rollback_on_fail: bool = False limit: int | None = None projects: list[str] = field(default_factory=list) force: bool = False # 跳过 Excel 模板预检 dry_run: bool = False # 只演练:不产生任何服务端写入 @dataclass class ProjectProgress: row: PlanRow stage: str worksheet_id: int | None = None version_id: int | None = None elapsed_ms: int = 0 class Importer: def __init__(self, cfg: Config, store: Store, run_id: str, log: RunLog, options: RunOptions) -> None: self.cfg = cfg self.store = store self.run_id = run_id self.log = log self.options = options self._thread_state = threading.local() self.api = QuickTeamApi( self._build_transport(), dry_run=options.dry_run, log=lambda message: self.log.info(message, level="dry"), ) self._tfs_lock = threading.Lock() self._tfs_context: tuple[str, str] | None = None self.user: User | None = None self.team_name = "" # ------------------------------------------------------------------ 装配 def _build_transport(self): from .transport import Transport def on_call(op: str, elapsed: int, ok: bool, detail: str) -> None: project = getattr(self._thread_state, "project", "") self.store.add_event( self.run_id, stage="API", action="ok" if ok else "fail", project_id=getattr(self._thread_state, "project_id", ""), detail=f"{op} {detail}".strip(), elapsed_ms=elapsed, ) self.log.event("API", "ok" if ok else "fail", project=project, detail=op, elapsed_ms=elapsed) return Transport( self.cfg.base_url, timeout=self.cfg.timeout, retries=2, on_call=on_call ) # ------------------------------------------------------------------ plan def connect(self) -> User: if not self.user: self.user = self.api.login(self.cfg.username, self.cfg.password) teams = self.api.teams(self.user.user_id) team = self.api.resolve_team(teams, self.cfg.team_name) self.team_name = team.team_name self.team_id = team.team_id self.log.info(f"已登录:{self.user.real_name}({self.user.user_name}),团队:{team.team_name}") return self.user def plan(self) -> list[PlanRow]: self.connect() projects = self.api.projects(self.team_id) files = domain.list_xlsx(self.cfg.work_dir) rows: list[PlanRow] = [] for path in files: stem = os.path.splitext(os.path.basename(path))[0] project, reason = domain.match_project(stem, projects) if project is None: rows.append(PlanRow(project_name=stem, file_path=path, action=Action.ERROR, reason=reason)) continue if not self.options.force: ok, why = domain.precheck_xlsx(path) if not ok: rows.append( PlanRow(project_name=project.project_name, file_path=path, action=Action.ERROR, project=project, reason=f"Excel 预检失败:{why}") ) continue ledger = self.store.latest_job(project.project_id, self.cfg.sprint) existing = [ sheet for sheet in self.api.work_sheets(project.project_id) if sheet.system_name == self.cfg.sprint ] action, why, worksheet = domain.decide( existing, ledger_stage=ledger["stage"] if ledger else None, adopt=self.options.adopt ) rows.append( PlanRow( project_name=project.project_name, file_path=path, action=action, reason=why, project=project, worksheet=worksheet, existing_count=len(existing), ) ) return rows # ------------------------------------------------------------------ apply def apply(self) -> list[ProjectProgress]: rows = self.plan() write_rows = [row for row in rows if row.action.writes] if self.options.projects: wanted = {domain.normalize_name(name) for name in self.options.projects} write_rows = [row for row in write_rows if domain.normalize_name(row.project_name) in wanted] if self.options.limit: write_rows = write_rows[: self.options.limit] # 非执行行也落台账,报告里才能看到「为什么跳过」 for row in rows: if row.action.writes: continue project_id = row.project.project_id if row.project else row.project_name if self.store.latest_job(project_id, self.cfg.sprint): # 已经有真实进度记录,别用 SKIPPED 覆盖它(否则下次会把「已推送」当成「无进度」) self.log.event(str(row.action), "skip", project=row.project_name, detail=row.reason) continue self.store.upsert_job( self.run_id, project_id=project_id, project_name=row.project_name, system_name=self.cfg.sprint, source_file=row.file_path, action=str(row.action), stage="SKIPPED" if row.action != Action.ERROR else "FAILED", worksheet_id=row.worksheet.worksheet_id if row.worksheet else None, reason=row.reason, error=row.reason if row.action == Action.ERROR else "", ) self.log.event(str(row.action), "skip", project=row.project_name, detail=row.reason) if not write_rows: self.log.info("没有需要执行的项目") return [] self.log.info(f"开始执行 {len(write_rows)} 个项目(并发 {self.cfg.workers})") results: list[ProjectProgress] = [] with ThreadPoolExecutor(max_workers=max(1, self.cfg.workers)) as pool: futures = {pool.submit(self._run_project, row): row for row in write_rows} for future in as_completed(futures): row = futures[future] try: results.append(future.result()) except QtError as error: if error.category in {Category.AUTH, Category.SP_MISSING}: self.log.warn(f"中止:{error}") raise results.append(ProjectProgress(row=row, stage="FAILED")) self.log.warn(f"{row.project_name} 失败:{error}") except Exception as error: # 兜底,避免一个项目的异常带崩整批 results.append(ProjectProgress(row=row, stage="FAILED")) self.log.warn(f"{row.project_name} 未预期异常:{error!r}") return results def _run_project(self, row: PlanRow) -> ProjectProgress: assert row.project is not None started = time.time() self._thread_state.project = row.project_name self._thread_state.project_id = row.project.project_id ledger = self.store.latest_job(row.project.project_id, self.cfg.sprint) if row.action == Action.CREATE: # plan 已经确认服务端没有该 Sprint 的工单,台账里的旧记录可能来自已被回滚的 # 半成品(工单已删除)。此时必须忽略台账,否则会去填一张不存在的工单。 ledger = None done_upto = self._completed_index(ledger["stage"] if ledger else "") worksheet_id = (ledger or {}).get("worksheet_id") or ( row.worksheet.worksheet_id if row.worksheet else None ) version_id: int | None = (ledger or {}).get("version_id") def need(stage: str) -> bool: return STAGE_INDEX[stage] > done_upto stage = "PLANNED" created_here = row.action == Action.CREATE try: if need("WS_CREATED"): stage = "WS_CREATED" worksheet_id = self.api.create_worksheet() self._mark(row, stage, worksheet_id, None) if need("WS_FILLED"): stage = "WS_FILLED" assert worksheet_id is not None evaluator, technical, customer = self.cfg.resolved_roles(self.user.real_name) self.api.fill_worksheet( worksheet_id, row.project, system_name=self.cfg.sprint, start_time=self.cfg.start_time, end_time=self.cfg.end_time, evaluator=evaluator, technical_leader=technical, customer_leader=customer, belong_team=self.cfg.belong_team, content=self.cfg.content_desc, operator=self.user.real_name, service_type=self.cfg.service_type, ) self._mark(row, stage, worksheet_id, None) if need("EXCEL_IMPORTED"): stage = "EXCEL_IMPORTED" assert worksheet_id is not None version = self._version(row.project.project_id, worksheet_id) version_id = version.version_id remote = self.api.upload_excel(row.file_path) self.api.import_excel(worksheet_id, version_id, remote) self._mark(row, stage, worksheet_id, version_id) if need("WS_VERIFIED"): stage = "WS_VERIFIED" assert worksheet_id is not None detail = self._detail(worksheet_id) version = self._version(row.project.project_id, worksheet_id) if detail.done: self.log.event(stage, "skip", project=row.project_name, detail="工单已完成,无需确认") elif version is None: raise QtError(f"工单 {worksheet_id} 没有版本记录,无法确认", Category.PARAM) elif domain.VERSION_TRANSITIONS.get(version.version_code) is None: self.log.event(stage, "skip", project=row.project_name, detail="已是流程终点,无需确认") else: try: next_operator, next_code, percent = domain.next_transition(version.version_code) except ValueError as error: raise QtError(str(error), Category.PARAM) from None self.api.verify_worksheet( worksheet_id, version.version_id, next_operator=next_operator, next_version_code=next_code, operator=self.user.real_name, percent=percent, ) self._mark(row, stage, worksheet_id, version.version_id if version else version_id) if need("TFS_PUSHED"): assert worksheet_id is not None if self.options.skip_tfs: self.log.event("TFS_PUSHED", "skip", project=row.project_name, detail="--skip-tfs") else: stage = "TFS_PUSHED" area, iteration = self._resolve_tfs() features = [] if self.api.dry_run else [ item.feature_id for item in self.api.features(worksheet_id) ] if not features and not self.api.dry_run: raise QtError("工单没有功能模块,跳过推送(请检查 Excel 是否为空表)", Category.EXCEL) created = self.api.push_tfs(features, self.cfg.tfs_project, area, iteration) self.log.event( "TFS_PUSHED", "ok", project=row.project_name, detail=f"features={len(features)} 创建={created}", ) self._mark(row, stage, worksheet_id, version_id) elapsed = int((time.time() - started) * 1000) self.log.info(f" ✓ {row.project_name} → {stage} {elapsed / 1000:.1f}s") return ProjectProgress(row=row, stage=stage, worksheet_id=worksheet_id, version_id=version_id, elapsed_ms=elapsed) except QtError as error: elapsed = int((time.time() - started) * 1000) self.store.upsert_job( self.run_id, project_id=row.project.project_id, project_name=row.project_name, system_name=self.cfg.sprint, source_file=row.file_path, action=str(row.action), stage=f"FAILED:{stage}", worksheet_id=worksheet_id, version_id=version_id, reason=row.reason, error=str(error), ) self.store.add_event( self.run_id, stage=stage, action="fail", project_id=row.project.project_id, detail=str(error), elapsed_ms=elapsed, ) self.log.warn(f"{row.project_name} 在 {stage} 失败:{error}") if self.options.rollback_on_fail and created_here and STAGE_INDEX[stage] < STAGE_INDEX["WS_VERIFIED"]: self._rollback(row, worksheet_id, stage) if error.category in {Category.AUTH, Category.SP_MISSING}: raise return ProjectProgress(row=row, stage=f"FAILED:{stage}", worksheet_id=worksheet_id, version_id=version_id, elapsed_ms=elapsed) finally: self._thread_state.project = "" self._thread_state.project_id = "" # ------------------------------------------------------------------ 辅助 @staticmethod def _completed_index(stage: str) -> int: """台账阶段 → 已完成的阶段下标。 `FAILED:EXCEL_IMPORTED` 表示「导入这一步失败了」,所以它之前的都算完成, 重跑要从 EXCEL_IMPORTED 本身重试,而不是从头再来(否则会重复建单)。 """ if not stage: return -1 if stage.startswith("FAILED:"): failed_at = stage.split(":", 1)[1] return STAGE_INDEX.get(failed_at, 0) - 1 return STAGE_INDEX.get(stage, -1) def _detail(self, worksheet_id: int) -> WorkSheet: if self.api.dry_run: return WorkSheet(worksheet_id=worksheet_id, version_id=0, version_code="") return self.api.worksheet(worksheet_id) def _version(self, project_id: str, worksheet_id: int) -> WorkSheetVersion | None: """当前待确认版本。导入明细与确认工单都挂在它上面。 `WorkSheet.CurrentVersionId` 在第一次保存后仍为 NULL(要等确认才回填), 所以不能拿它当 VersionId 用 —— 那会把 WorkSheetTask 插到不存在的版本上, 报 FK_WorkSheetTask_WorkSheetVersion 冲突。 """ if self.api.dry_run: return WorkSheetVersion(version_id=0, worksheet_id=worksheet_id, version_code="技术责任人确认版", version_status="N") return self.api.pending_version(project_id, worksheet_id) def _resolve_tfs(self) -> tuple[str, str]: if self.cfg.area_path and self.cfg.iteration_path: return self.cfg.area_path, self.cfg.iteration_path with self._tfs_lock: if self._tfs_context: return self._tfs_context projects = self.api.tfs_projects() target = next((p for p in projects if p.get("Name") == self.cfg.tfs_project), None) if target is None: raise QtError(f"TFS 项目「{self.cfg.tfs_project}」不存在", Category.TFS) teams = self.api.tfs_teams(self.cfg.tfs_project, target["ProjectId"]) area, iteration = QuickTeamApi.resolve_tfs_context(teams, self.cfg.tfs_team, self.cfg.sprint) self._tfs_context = (area, iteration) self.log.info(f"TFS 上下文:area={area} iteration={iteration}") return self._tfs_context def _mark( self, row: PlanRow, stage: str, worksheet_id: int | None, version_id: int | None ) -> None: assert row.project is not None self.store.upsert_job( self.run_id, project_id=row.project.project_id, project_name=row.project_name, system_name=self.cfg.sprint, source_file=row.file_path, action=str(row.action), stage=stage, worksheet_id=worksheet_id, version_id=version_id, reason=row.reason, ) self.log.event(stage, "ok", project=row.project_name, detail=f"ws={worksheet_id}") def _rollback(self, row: PlanRow, worksheet_id: int | None, stage: str) -> None: if not worksheet_id: return try: self.api.delete_worksheet(worksheet_id) self.store.add_event( self.run_id, stage=stage, action="rollback", project_id=row.project.project_id, detail=f"已删除半成品工单 {worksheet_id}", ) self.log.warn(f"{row.project_name} 已回滚(删除工单 {worksheet_id})") except QtError as error: self.log.warn(f"{row.project_name} 回滚失败:{error}")