eval_ph1 — 코드 동작 총정리

flowchart TB
classDef cli fill:#eaf0fd,stroke:#2f5bd0,color:#16202c;
classDef orch fill:#efeaff,stroke:#6b4fc4,color:#16202c;
classDef cap fill:#fdf4e3,stroke:#b07400,color:#16202c;
classDef db fill:#e9f6ee,stroke:#1f8a4c,color:#16202c;
classDef viz fill:#eaf0fd,stroke:#2f5bd0,color:#16202c;
CLI["🧑‍🔬 cli.py · run / report / compare"]:::cli
subgraph ORCH["② 오케스트레이션 · runner.py"]
  RB["runner.run_batch<br/>async · Semaphore · asyncpg pool"]:::orch
  REG["teams.registry<br/>build + 재현성 가드"]:::orch
  TEAM["SelectorGroupChat · run_stream<br/>Planner → Executor → Critic · Selector"]:::orch
end
subgraph CAP["③ 기록 · 채점"]
  LOG["TraceLogger<br/>event→message_type · 마스킹"]:::cap
  SC["scorer<br/>success · format_valid"]:::cap
end
subgraph DBP["④ PostgreSQL · schema eval_ph1"]
  T1[(tasks)]:::db
  T2[(teams)]:::db
  RUNS[(runs ★)]:::db
  EV[(events ★)]:::db
  MV["mv_batch_summary · mv_passk · mv_tool_summary"]:::db
  VW["v_research / v_analysis / v_cppx grounding"]:::db
end
GRAF["📊 Grafana ①–⑥ · MAST"]:::viz
CLI -->|"run_batch(TaskFilter)"| RB
RB --> REG --> TEAM
RB -.->|select| T1
REG -.->|upsert| T2
TEAM -->|AutoGen events| LOG
TEAM -->|final_answer| SC
LOG -->|INSERT seq| EV
SC -->|UPDATE success| RUNS
RB -.->|refresh_views| MV
RUNS --> MV
EV --> MV
MV -->|postgres SQL| GRAF
VW -->|live view| GRAF
    

flowchart LR
classDef sel fill:#efeaff,stroke:#6b4fc4;
classDef p fill:#eaf0fd,stroke:#2f5bd0;
classDef e fill:#e9f6ee,stroke:#1f8a4c;
classDef c fill:#fdf4e3,stroke:#b07400;
classDef fin fill:#e9f6ee,stroke:#1f8a4c;
SEL{{"🚦 Selector<br/>next + reason_code (JSON)"}}:::sel
P["🤖 Planner<br/>계획 · FINAL ANSWER 초안"]:::p
E["🛠 Executor<br/>read_attached_file · python_calc"]:::e
C["🧐 Critic<br/>형식 · 정답 검증"]:::c
F["🏁 종료<br/>APPROVE | max_messages"]:::fin
SEL -->|NEEDS_PLANNING| P
P -.->|발화| SEL
SEL -->|NEEDS_EXECUTION| E
E -.->|결과| SEL
SEL -->|NEEDS_VERIFICATION| C
C -.->|NEEDS_REVISION| SEL
C -->|APPROVE| F
    

flowchart LR
classDef ok fill:#e9f6ee,stroke:#1f8a4c;
classDef bad fill:#fbecea,stroke:#c0392b;
B["팀 빌드"] --> H["prompt_bundle_hash<br/>sha256(planner+executor+critic+selector)"]
H --> Q{"기존 team_id 행과<br/>해시 불일치?"}
Q -->|YES| X["raise TeamConfigMismatchError<br/>즉시 중단"]:::bad
Q -->|NO| K["ensure_team_record<br/>runs 진행"]:::ok
    

flowchart LR
classDef ev fill:#eef2f8,stroke:#9aa7b6;
classDef map fill:#efeaff,stroke:#6b4fc4;
classDef db fill:#e9f6ee,stroke:#1f8a4c;
subgraph S["AutoGen 이벤트 스트림"]
  A1["TextMessage / ToolCallSummary"]:::ev
  A2["ToolCallRequestEvent"]:::ev
  A3["ToolCallExecutionEvent"]:::ev
  A4["SelectorEvent"]:::ev
  A5["TaskResult"]:::ev
end
M["mapping → message_type<br/>seq+1 · 비밀 *** 마스킹<br/>content/result 절단+해시<br/>INSERT 실패→경고 후 계속"]:::map
subgraph EVS["events 행 (run_id, seq 순)"]
  R1["agent_message"]:::db
  R2["tool_call (id·name)"]:::db
  R3["tool_result / tool_error"]:::db
  R4["selector_decision"]:::db
  R5["final_answer"]:::db
end
A1 --> M
A2 --> M
A3 --> M
A4 --> M
A5 --> M
M --> R1
M --> R2
M --> R3
M --> R4
M --> R5
R2 -.->|"응답 짝 필수 (CHECK)"| R3
    

flowchart LR
classDef io fill:#eef2f8,stroke:#9aa7b6;
classDef sc fill:#fdf4e3,stroke:#b07400;
classDef out fill:#e9f6ee,stroke:#1f8a4c;
IN["final_answer<br/>(open-ended: _full)"]:::io
RT{"benchmark?"}
IN --> RT
RT -->|gaia| G["gaia.score<br/>정규화 · 리스트 순서보존 · 1e-6"]:::sc
RT -->|sql| SQ["sql.score<br/>실행결과 멀티셋"]:::sc
RT -->|research| RE["research.grounding<br/>인용 F1"]:::sc
RT -->|analysis| AN["analysis.score<br/>형식만 (정답 비교 없음)"]:::sc
RT -->|extract| EX["extract.score<br/>field_accuracy"]:::sc
RT -->|cppx · roles| CP["fact-ratio · coverage · CER"]:::sc
G --> O1["runs.success / format_valid"]:::out
SQ --> O1
AN --> O1
EX --> O1
RE --> O2["도메인 테이블<br/>research_ / cppx_grounding"]:::out
CP --> O2
    

erDiagram
  tasks ||--o{ runs : "task_id"
  teams ||--o{ runs : "team_id"
  runs ||--o{ events : "run_id · CASCADE"
  runs ||--o| research_grounding : "run_id"
  runs ||--o| cppx_grounding : "run_id"
  teams ||--o{ experiments : "before/after"
  tasks {
    text task_id PK
    text benchmark
    text gold_answer
    bool requires_web
  }
  runs {
    uuid run_id PK
    text batch_id
    real temperature
    bool success
    text status
  }
  events {
    uuid run_id FK
    int seq PK
    text message_type
    jsonb tool_args
  }
    

flowchart LR
classDef src fill:#e9f6ee,stroke:#1f8a4c;
classDef dash fill:#eaf0fd,stroke:#2f5bd0;
MBS["mv_batch_summary"]:::src
MPK["mv_passk"]:::src
MTS["mv_tool_summary"]:::src
VAG["v_analysis_grounded"]:::src
VRG["v_research_grounding"]:::src
VTC["v_team_config / graph"]:::src
RUN["runs (failure_stage)"]:::src
EXP["experiments"]:::src
D1["① 총괄 개요"]:::dash
D2["② 실험 메트릭"]:::dash
D3["③ 분석 grounding"]:::dash
D4["④ 리서치 grounding"]:::dash
D6["⑥ 팀 구성"]:::dash
DM["MAST 실패분류"]:::dash
MBS --> D2
MPK --> D2
MTS --> D2
EXP --> D2
EXP --> D1
VAG --> D3
VRG --> D4
VTC --> D6
RUN --> DM
RUN --> D1