fix(gooddata-eval): keep gate out of the positional signature and in the merged report

Author: myhoai

packages/gooddata-eval/src/gooddata_eval/cli/main.py

@@ -482,6 +482,7 @@ def on_langfuse_item_done(
                 provider_name=resolved.provider_name or resolved.provider_id,
                 provider_type=resolved.provider_type,
                 workspace_id=config.workspace_id,
+                gate=config.gate,
             )
             if agentic_report is not None:
                 report.items.extend(agentic_report.items)

packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py

@@ -791,7 +791,6 @@ def evaluate_agentic_alert_skill(
     question: str,
     expected_output: dict,
     k: int = _DEFAULT_K,
-    gate: EvalGate = DEFAULT_GATE,
     max_iterations: int = _DEFAULT_MAX_ITERATIONS,
     initial_conversation_id: str | None = None,
     agent_id: str | None = None,
@@ -803,6 +802,7 @@ def evaluate_agentic_alert_skill(
     run_metadata_extra: dict | None = None,
     reasoning_effort: ReasoningEffort | None = None,
     submit_trace_link: SubmitTraceLink = run_trace_link_inline,
+    gate: EvalGate = DEFAULT_GATE,
 ) -> AgenticEvalOutcome:
     """Run alert-skill evaluation, log to Langfuse, and raise AlertSkillAssertionError on failure.
 

packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py

@@ -221,7 +221,6 @@ def evaluate_agentic_general_question(
     question: str,
     expected_output: str,
     k: int = _DEFAULT_K,
-    gate: EvalGate = DEFAULT_GATE,
     initial_conversation_id: str | None = None,
     agent_id: str | None = None,
     langfuse: object | None = None,
@@ -233,6 +232,7 @@ def evaluate_agentic_general_question(
     reasoning_effort: ReasoningEffort | None = None,
     submit_trace_link: SubmitTraceLink = run_trace_link_inline,
     user_context: dict | None = None,
+    gate: EvalGate = DEFAULT_GATE,
 ) -> AgenticEvalOutcome:
     """Run general-question evaluation, log to Langfuse, and raise GeneralQuestionAssertionError on failure.
 

packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py

@@ -194,7 +194,6 @@ def evaluate_agentic_guardrail(
     question: str,
     expected_output: str,
     k: int = _DEFAULT_K,
-    gate: EvalGate = DEFAULT_GATE,
     initial_conversation_id: str | None = None,
     agent_id: str | None = None,
     langfuse: object | None = None,
@@ -205,6 +204,7 @@ def evaluate_agentic_guardrail(
     run_metadata_extra: dict | None = None,
     reasoning_effort: ReasoningEffort | None = None,
     submit_trace_link: SubmitTraceLink = run_trace_link_inline,
+    gate: EvalGate = DEFAULT_GATE,
 ) -> AgenticEvalOutcome:
     """Run guardrail evaluation, log to Langfuse, and raise GuardrailAssertionError on failure.
 

packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py

@@ -380,7 +380,6 @@ def evaluate_agentic_kda_skill(
     question: str,
     expected_output: dict,
     k: int = _DEFAULT_K,
-    gate: EvalGate = DEFAULT_GATE,
     max_iterations: int = _DEFAULT_MAX_ITERATIONS,
     initial_conversation_id: str | None = None,
     agent_id: str | None = None,
@@ -392,6 +391,7 @@ def evaluate_agentic_kda_skill(
     run_metadata_extra: dict | None = None,
     reasoning_effort: ReasoningEffort | None = None,
     submit_trace_link: SubmitTraceLink = run_trace_link_inline,
+    gate: EvalGate = DEFAULT_GATE,
 ) -> AgenticEvalOutcome:
     """Run KDA-skill evaluation, log to Langfuse, and raise KdaSkillAssertionError on failure.
 

packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py

@@ -413,7 +413,6 @@ def evaluate_agentic_metric_skill(
     question: str,
     expected_output: dict | list,
     k: int = _DEFAULT_K,
-    gate: EvalGate = DEFAULT_GATE,
     max_iterations: int = _DEFAULT_MAX_ITERATIONS,
     initial_conversation_id: str | None = None,
     agent_id: str | None = None,
@@ -425,6 +424,7 @@ def evaluate_agentic_metric_skill(
     run_metadata_extra: dict | None = None,
     reasoning_effort: ReasoningEffort | None = None,
     submit_trace_link: SubmitTraceLink = run_trace_link_inline,
+    gate: EvalGate = DEFAULT_GATE,
 ) -> AgenticEvalOutcome:
     """Run metric-skill evaluation, log to Langfuse, and raise MetricSkillAssertionError on failure.
 

packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py

@@ -175,7 +175,6 @@ def evaluate_agentic_search_tool(
     question: str,
     expected_tool_call: dict,
     k: int = _DEFAULT_K,
-    gate: EvalGate = DEFAULT_GATE,
     initial_conversation_id: str | None = None,
     agent_id: str | None = None,
     langfuse: object | None = None,
@@ -186,6 +185,7 @@ def evaluate_agentic_search_tool(
     run_metadata_extra: dict | None = None,
     reasoning_effort: ReasoningEffort | None = None,
     submit_trace_link: SubmitTraceLink = run_trace_link_inline,
+    gate: EvalGate = DEFAULT_GATE,
 ) -> AgenticEvalOutcome:
     """Run search-tool evaluation, log to Langfuse, and raise SearchToolAssertionError on failure.
 

packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py

@@ -322,7 +322,6 @@ def evaluate_agentic_visualization(
     question: str,
     expected_outputs: list[CreatedVisualization],
     k: int = _DEFAULT_K,
-    gate: EvalGate = DEFAULT_GATE,
     max_iterations: int = _DEFAULT_MAX_ITERATIONS,
     initial_conversation_id: str | None = None,
     agent_id: str | None = None,
@@ -335,6 +334,7 @@ def evaluate_agentic_visualization(
     record_output_path: str | None = None,
     reasoning_effort: ReasoningEffort | None = None,
     submit_trace_link: SubmitTraceLink = run_trace_link_inline,
+    gate: EvalGate = DEFAULT_GATE,
 ) -> AgenticEvalOutcome:
     """Run visualization evaluation, log to Langfuse, and raise VisualizationAssertionError on failure.
 

packages/gooddata-eval/tests/test_agentic_gate.py

@@ -2,6 +2,8 @@
 """The K gate: which of pass@K / pass^K decides an item, and what reaches Langfuse."""
 
 import contextlib
+import importlib
+import inspect
 from unittest.mock import MagicMock, patch
 
 import pytest
@@ -169,3 +171,44 @@ def test_a_hard_failure_fails_under_both_gates():
     for gate in ("any", "power"):
         with pytest.raises(GeneralQuestionAssertionError):
             _evaluate(gate, (False, False, False))
+
+
+# --------------------------------------------------------------------------- #
+# signature compatibility — `gate` must not shift an existing positional argument
+# --------------------------------------------------------------------------- #
+@pytest.mark.parametrize(
+    "module_name",
+    [
+        "visualization",
+        "metric_skill",
+        "alert_skill",
+        "kda_skill",
+        "general_question",
+        "guardrail",
+        "search_tool",
+    ],
+)
+def test_gate_is_the_last_parameter_of_every_evaluator(module_name):
+    """Inserted anywhere earlier, a positional caller binds max_iterations or
+    initial_conversation_id to `gate`, which then reaches normalize_gate and raises."""
+    module = importlib.import_module(f"gooddata_eval.core.agentic.{module_name}")
+    fn = next(v for k, v in vars(module).items() if k.startswith("evaluate_agentic_"))
+    names = [p.name for p in inspect.signature(fn).parameters.values()]
+
+    assert names[-1] == "gate"
+    assert names[5] == "k"
+    assert names[6] in ("max_iterations", "initial_conversation_id")
+
+
+def test_a_positional_seventh_argument_still_binds_where_it_used_to():
+    """The regression the parameter order protects: 7 positional args, no keywords."""
+    with _judged(True):
+        evaluate_agentic_general_question(
+            "https://example.com",  # host
+            "tok",  # token
+            "ws",  # workspace_id
+            "Q",  # question
+            "rubric",  # expected_output
+            1,  # k
+            "conv-0",  # initial_conversation_id -- NOT gate
+        )