Skip to content

test_hle_eval

ipw.tests.evaluation.test_hle_eval

Tests for evaluation/hle.py — HLEHandler LLM judge.

TestParseGrade

Test _parse_grade response parsing.

Source code in intelligence-per-watt/src/ipw/tests/evaluation/test_hle_eval.py 
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
class TestParseGrade:
    """Test _parse_grade response parsing."""

    def test_correct_yes(self) -> None:
        assert _parse_grade("correct: yes") == "CORRECT"

    def test_correct_no(self) -> None:
        assert _parse_grade("correct: no") == "INCORRECT"

    def test_not_attempted(self) -> None:
        assert _parse_grade("correct: not_attempted") == "NOT_ATTEMPTED"

    def test_fallback_correct(self) -> None:
        assert _parse_grade("The answer is CORRECT") == "CORRECT"

    def test_fallback_incorrect(self) -> None:
        assert _parse_grade("INCORRECT") == "INCORRECT"

    def test_unparseable(self) -> None:
        assert _parse_grade("random text") == "NOT_ATTEMPTED"

TestHLEHandler

Test HLEHandler with mocked LLM client.

Source code in intelligence-per-watt/src/ipw/tests/evaluation/test_hle_eval.py 
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
class TestHLEHandler:
    """Test HLEHandler with mocked LLM client."""

    def _make_handler(self, chat_return: str = "") -> HLEHandler:
        mock_client = MagicMock()
        mock_client.chat.return_value = chat_return
        return HLEHandler(client=mock_client)

    def test_empty_response(self) -> None:
        handler = self._make_handler()
        is_correct, meta = handler.evaluate(
            problem="Q", reference="A", model_answer="", metadata={}
        )
        assert is_correct is False
        assert meta["grade"] == "NOT_ATTEMPTED"

    def test_no_ground_truth(self) -> None:
        handler = self._make_handler()
        is_correct, meta = handler.evaluate(
            problem="Q", reference="", model_answer="ans", metadata={}
        )
        assert is_correct is None

    def test_correct_grading(self) -> None:
        handler = self._make_handler(
            chat_return="extracted_final_answer: 2.224 MeV\nreasoning: correct\ncorrect: yes"
        )
        is_correct, meta = handler.evaluate(
            problem="Binding energy?",
            reference="2.224 MeV",
            model_answer="2.224 MeV",
            metadata={},
        )
        assert is_correct is True
        assert meta["grade"] == "CORRECT"

    def test_incorrect_grading(self) -> None:
        handler = self._make_handler(
            chat_return="extracted_final_answer: 1.0 MeV\nreasoning: wrong\ncorrect: no"
        )
        is_correct, meta = handler.evaluate(
            problem="Q", reference="A", model_answer="wrong", metadata={}
        )
        assert is_correct is False
        assert meta["grade"] == "INCORRECT"

    def test_requires_chat_method(self) -> None:
        mock_client = MagicMock(spec=[])
        handler = HLEHandler(client=mock_client)

        with pytest.raises(RuntimeError, match="requires a client"):
            handler.evaluate(
                problem="Q", reference="A", model_answer="ans", metadata={}
            )

    def test_api_error_returns_none(self) -> None:
        mock_client = MagicMock()
        mock_client.chat.side_effect = RuntimeError("API error")
        handler = HLEHandler(client=mock_client)

        is_correct, meta = handler.evaluate(
            problem="Q", reference="A", model_answer="ans", metadata={}
        )
        assert is_correct is None
        assert "error" in meta