Async Team Post-Hook Agent as Judge

Automatic async evaluation of team outputs using post-hooks

This example demonstrates using Agent as Judge as an async post-hook on a Team to automatically evaluate team responses.

Add the following code to your Python file

1import asyncio
2
3from kern.agent import Agent
4from kern.db.sqlite import AsyncSqliteDb
5from kern.eval.agent_as_judge import AgentAsJudgeEval
6from kern.models.openai import OpenAIResponses
7from kern.team.team import Team
8
9
10async def main():
11    # Setup database to persist eval results
12    db = AsyncSqliteDb(db_file="tmp/agent_as_judge_team_post_hook_async.db")
13
14    # Eval runs as post-hook, results saved to database
15    agent_as_judge_eval = AgentAsJudgeEval(
16        name="Team Response Quality",
17        model=OpenAIResponses(id="gpt-5.2"),
18        criteria="Response should be well-researched, clear, comprehensive, and show good collaboration between team members",
19        scoring_strategy="numeric",
20        threshold=7,
21        db=db,
22    )
23
24    # Setup a team with researcher and writer
25    researcher = Agent(
26        name="Researcher",
27        role="Research and gather information",
28        model=OpenAIResponses(id="gpt-5.2"),
29    )
30
31    writer = Agent(
32        name="Writer",
33        role="Write clear and concise summaries",
34        model=OpenAIResponses(id="gpt-5.2"),
35    )
36
37    research_team = Team(
38        name="Research Team",
39        model=OpenAIResponses(id="gpt-5.2"),
40        members=[researcher, writer],
41        instructions=["First research the topic thoroughly, then write a clear summary."],
42        post_hooks=[agent_as_judge_eval],
43        db=db,
44    )
45
46    response = await research_team.arun("Explain quantum computing")
47    print(response.content)
48
49    # Query database for eval results
50    print("Evaluation Results:")
51    eval_runs = await db.get_eval_runs()
52    if eval_runs:
53        latest = eval_runs[-1]
54        if latest.eval_data and "results" in latest.eval_data:
55            result = latest.eval_data["results"][0]
56            print(f"Score: {result.get('score', 'N/A')}/10")
57            print(f"Status: {'PASSED' if result.get('passed') else 'FAILED'}")
58            print(f"Reason: {result.get('reason', 'N/A')[:200]}...")
59
60
61if __name__ == "__main__":
62    asyncio.run(main())

Set up your virtual environment

1uv venv --python 3.12
2source .venv/bin/activate

1uv venv --python 3.12
2.venv\Scripts\activate

Install dependencies

1uv pip install -U kern-ai openai

Export your OpenAI API key

1export OPENAI_API_KEY="your_openai_api_key_here"

1$Env:OPENAI_API_KEY="your_openai_api_key_here"

Run the example

1python agent_as_judge_team_post_hook_async.py