Evaluating Llama Stack Models with Built-in Evaluation Framework

pip install llama-stack-client==0.2.8

cat << 'EOF' > eval_basic.py
from llama_stack_client import LlamaStackClient
import pprint

# Connect to Llama Stack
client = LlamaStackClient(
    base_url="http://localhost:8321",
    timeout=600.0
)

# Create evaluation examples
handmade_eval_rows = [
    {
        "input_query": "What is the capital of France?",
        "generated_answer": "The capital of France is Paris.",
        "expected_answer": "Paris",
    },
    {
        "input_query": "Who wrote Romeo and Juliet?",
        "generated_answer": "William Shakespeare wrote Romeo and Juliet.",
        "expected_answer": "shakespeare",  # lowercase - will fail!
    },
    {
        "input_query": "What is 2 + 2?",
        "generated_answer": "The answer is 4.",
        "expected_answer": "4",
    }
]

print("📝 Testing subset_of evaluation:")
pprint.pprint(handmade_eval_rows)

# Run subset_of evaluation
scoring_response = client.scoring.score(
    input_rows=handmade_eval_rows,
    scoring_functions={"basic::subset_of": None}
)

print("\n📊 Results:")
pprint.pprint(scoring_response)

# Show accuracy
results = scoring_response.results['basic::subset_of']
accuracy = results.aggregated_results['accuracy']['accuracy']
print(f"\n📈 Accuracy: {accuracy:.1%}")
EOF

python eval_basic.py

cat << 'EOF' > eval_judge.py
from llama_stack_client import LlamaStackClient
import pprint

# Connect to Llama Stack
client = LlamaStackClient(
    base_url="http://localhost:8321",
    timeout=600.0
)

# Get available model for judging
available_models = [
    model.identifier for model in client.models.list() if model.model_type == "llm"
]
judge_model = available_models[0]

# Same evaluation examples
handmade_eval_rows = [
    {
        "input_query": "What is the capital of France?",
        "generated_answer": "The capital of France is Paris.",
        "expected_answer": "Paris",
    },
    {
        "input_query": "Who wrote Romeo and Juliet?",
        "generated_answer": "William Shakespeare wrote Romeo and Juliet.",
        "expected_answer": "shakespeare",
    },
    {
        "input_query": "What is 2 + 2?",
        "generated_answer": "The answer is 4.",
        "expected_answer": "4",
    }
]

# Judge prompt
JUDGE_PROMPT = """
Given a QUESTION and GENERATED_RESPONSE and EXPECTED_RESPONSE.

Compare the factual content. Ignore differences in style, grammar, or punctuation.
Answer by selecting one option:
(A) The GENERATED_RESPONSE is a subset of the EXPECTED_RESPONSE and is fully consistent.
(B) The GENERATED_RESPONSE is a superset of the EXPECTED_RESPONSE and is fully consistent.
(C) The GENERATED_RESPONSE contains all the same details as the EXPECTED_RESPONSE.
(D) There is a disagreement between the responses.
(E) The answers differ, but these differences don't matter factually.

Format: "Answer: One of ABCDE, Explanation: "

QUESTION: {input_query}
GENERATED_RESPONSE: {generated_answer}
EXPECTED_RESPONSE: {expected_answer}
"""

print(f"🤖 Testing LLM-as-judge with {judge_model}:")

# Run LLM-as-judge evaluation
scoring_response = client.scoring.score(
    input_rows=handmade_eval_rows,
    scoring_functions={
        "llm-as-judge::base": {
            "judge_model": judge_model,
            "prompt_template": JUDGE_PROMPT,
            "type": "llm_as_judge",
            "judge_score_regexes": ["Answer: (A|B|C|D|E)"],
        }
    }
)

print("\n📊 Results:")
for i, score_row in enumerate(scoring_response.results['llm-as-judge::base'].score_rows):
    print(f"\n{i+1}. {handmade_eval_rows[i]['input_query']}")
    print(f"   Score: {score_row['score']}")
    print(f"   Reasoning: {score_row['judge_feedback']}")
EOF

python eval_judge.py

cat << 'EOF' > eval_dataset.py
from llama_stack_client import LlamaStackClient
import pprint

# Connect to Llama Stack
client = LlamaStackClient(
    base_url="http://localhost:8321",
    timeout=600.0
)

# Get model
available_models = [
    model.identifier for model in client.models.list() if model.model_type == "llm"
]
model_id = available_models[0]

# Register SimpleQA dataset
print("📚 Registering SimpleQA dataset...")
client.datasets.register(
    purpose="eval/messages-answer",
    source={
        "type": "uri",
        "uri": "huggingface://datasets/llamastack/simpleqa?split=train",
    },
    dataset_id="huggingface::simpleqa",
)

# Get sample questions
eval_rows = client.datasets.iterrows(
    dataset_id="huggingface::simpleqa",
    limit=3,
)

print("\n📋 Sample questions:")
for i, row in enumerate(eval_rows.data):
    print(f"{i+1}. {row['input_query']}")
    print(f"   Expected: {row['expected_answer']}")

# Register benchmark
client.benchmarks.register(
    benchmark_id="meta-reference::simpleqa",
    dataset_id="huggingface::simpleqa",
    scoring_functions=["llm-as-judge::base"],
)

# Evaluate model
print(f"\n🤖 Evaluating {model_id} on knowledge questions...")
response = client.eval.evaluate_rows(
    benchmark_id="meta-reference::simpleqa",
    input_rows=eval_rows.data,
    scoring_functions=["llm-as-judge::base"],
    benchmark_config={
        "eval_candidate": {
            "type": "model",
            "model": model_id,
            "sampling_params": {
                "strategy": {"type": "greedy"},
                "max_tokens": 512,
            },
        },
    },
)

print("\n📊 Results:")
for i, gen in enumerate(response.generations):
    score = response.scores['llm-as-judge::base'].score_rows[i]
    print(f"\n{i+1}. Question: {eval_rows.data[i]['input_query']}")
    print(f"   Expected: {eval_rows.data[i]['expected_answer']}")
    print(f"   Generated: {gen['generated_answer']}")
    print(f"   Score: {score['score']}")
EOF

python eval_dataset.py

cat << 'EOF' > eval_mmlu.py
from llama_stack_client import LlamaStackClient
import pprint

# Connect to Llama Stack
client = LlamaStackClient(
    base_url="http://localhost:8321",
    timeout=600.0
)

# Get model
available_models = [
    model.identifier for model in client.models.list() if model.model_type == "llm"
]
model_id = available_models[0]

# Define system prompt for multiple choice questions
SYSTEM_PROMPT_TEMPLATE = """
You are an expert in {subject} whose job is to answer multiple choice questions.

First, reason about the correct answer.

Then write the answer in the following format where X is exactly one of A,B,C,D:

Answer: X

Make sure X is one of A,B,C,D.

If you are uncertain of the correct answer, guess the most likely one.
"""

# Sample MMLU-style questions (normally you'd load from the actual dataset)
mmlu_sample_rows = [
    {
        "input_query": "What is the capital of France?\nA) London\nB) Berlin\nC) Paris\nD) Madrid",
        "expected_answer": "C",
        "chat_completion_input": '[{"role": "user", "content": "What is the capital of France?\\nA) London\\nB) Berlin\\nC) Paris\\nD) Madrid"}]'
    },
    {
        "input_query": "Which of the following is a prime number?\nA) 4\nB) 6\nC) 8\nD) 7",
        "expected_answer": "D",
        "chat_completion_input": '[{"role": "user", "content": "Which of the following is a prime number?\\nA) 4\\nB) 6\\nC) 8\\nD) 7"}]'
    },
    {
        "input_query": "Who wrote 'Romeo and Juliet'?\nA) Charles Dickens\nB) William Shakespeare\nC) Mark Twain\nD) Jane Austen",
        "expected_answer": "B",
        "chat_completion_input": '[{"role": "user", "content": "Who wrote \'Romeo and Juliet\'?\\nA) Charles Dickens\\nB) William Shakespeare\\nC) Mark Twain\\nD) Jane Austen"}]'
    }
]

print("📚 MMLU-style multiple choice evaluation:")
for i, row in enumerate(mmlu_sample_rows):
    print(f"{i+1}. {row['input_query']}")
    print(f"   Expected: {row['expected_answer']}")

# Create system message being an expert in Academic Subjects
system_message = {
    "role": "system",
    "content": SYSTEM_PROMPT_TEMPLATE.format(subject="academic subjects"),
}

# Register benchmark
client.benchmarks.register(
    benchmark_id="meta-reference::mmlu-sample",
    dataset_id="mmlu-sample",
    scoring_functions=[],
)

# Evaluate with regex parser for multiple choice
print(f"\n🎯 Evaluating {model_id} on MMLU-style questions...")
response = client.eval.evaluate_rows(
    benchmark_id="meta-reference::mmlu-sample",
    input_rows=mmlu_sample_rows,
    scoring_functions=["basic::regex_parser_multiple_choice_answer"],
    benchmark_config={
        "eval_candidate": {
            "type": "model",
            "model": model_id,
            "sampling_params": {
                "strategy": {
                    "type": "top_p",
                    "temperature": 0.1,
                    "top_p": 0.95,
                },
                "max_tokens": 512,
            },
            "system_message": system_message,
        },
    },
)

print("\n📊 MMLU Results:")
for i, gen in enumerate(response.generations):
    score = response.scores['basic::regex_parser_multiple_choice_answer'].score_rows[i]
    print(f"\n{i+1}. Question: {mmlu_sample_rows[i]['input_query'].split('?')[0]}?")
    print(f"   Expected: {mmlu_sample_rows[i]['expected_answer']}")
    print(f"   Generated: {gen['generated_answer']}")
    print(f"   Score: {score['score']}")

# Calculate accuracy
results = response.scores['basic::regex_parser_multiple_choice_answer']
if 'accuracy' in results.aggregated_results:
    accuracy = results.aggregated_results['accuracy']['accuracy']
    print(f"\n📈 MMLU Accuracy: {accuracy:.1%}")
EOF

python eval_mmlu.py