diff --git a/packages/gooddata-eval/scripts/verify_guardrail_refusal_criteria.py b/packages/gooddata-eval/scripts/verify_guardrail_refusal_criteria.py index c246fc05c..4a5320632 100644 --- a/packages/gooddata-eval/scripts/verify_guardrail_refusal_criteria.py +++ b/packages/gooddata-eval/scripts/verify_guardrail_refusal_criteria.py @@ -39,7 +39,9 @@ from dotenv import load_dotenv -load_dotenv("/Users/petertomko/gdc-mic-ai-evaluation/.env") +# Whatever .env the caller points at, defaulting to the working directory. It used to be an +# absolute path, which made the script runnable on exactly one machine. +load_dotenv(os.environ.get("GD_EVAL_ENV_FILE", ".env")) from gooddata_eval.core.agentic.guardrail import _GUARDRAIL_EVALUATION_STEPS # noqa: E402 from gooddata_eval.core.evaluators._guardrail_criteria import ( # noqa: E402 diff --git a/packages/gooddata-eval/src/gooddata_eval/core/models.py b/packages/gooddata-eval/src/gooddata_eval/core/models.py index 10b0438af..a3459fb01 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/models.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/models.py @@ -134,8 +134,8 @@ class ReasoningStepEvent(BaseModel): # Reasoning summaries are full paragraphs, e.g. "**Identifying analytics needs**\n\nI'm # analyzing..." -- using the whole thing as a latency_breakdown label would make every -# entry an unreadable wall of text. Same bolded-title convention this repo's own reasoning -# tooling already keys off of (see gdc-mic-ai-evaluation's generate_dashboard_summary.py). +# entry an unreadable wall of text. The bolded title is the summary's own heading, and +# downstream reporting keys off it for the same reason. _REASONING_TITLE_RE = re.compile(r"^\*\*(.+?)\*\*") _REASONING_LABEL_MAX_LEN = 60 diff --git a/packages/gooddata-eval/tests/test_from_insights.py b/packages/gooddata-eval/tests/test_from_insights.py index bab65d190..eda614439 100644 --- a/packages/gooddata-eval/tests/test_from_insights.py +++ b/packages/gooddata-eval/tests/test_from_insights.py @@ -382,7 +382,7 @@ def test_built_envelope_is_loadable_as_a_dataset_item(tmp_path): ], ), ) - envelope = build(spec, "How did spend trend by month?", "micai_diagnose_master", set()) + envelope = build(spec, "How did spend trend by month?", "demo_workspace", set()) assert "_shape" not in envelope["expected_output"]["visualization"] assert _validation_errors(envelope) is None @@ -390,7 +390,7 @@ def test_built_envelope_is_loadable_as_a_dataset_item(tmp_path): items = load_local_dataset(tmp_path) assert [i.id for i in items] == [envelope["id"]] assert items[0].test_kind == "visualization" - assert items[0].dataset_name == "micai_diagnose_master" + assert items[0].dataset_name == "demo_workspace" def test_mint_id_is_stable_and_collision_safe(): diff --git a/packages/gooddata-eval/tests/test_scoring.py b/packages/gooddata-eval/tests/test_scoring.py index baed8900c..3a1e87133 100644 --- a/packages/gooddata-eval/tests/test_scoring.py +++ b/packages/gooddata-eval/tests/test_scoring.py @@ -79,10 +79,10 @@ def test_check_filters_exact_attribute_match(): # state, type) and never the LIST under state["include"], so two filters selecting the # same elements in a different order compare unequal. # -# Found from a real eval run (gdc-mic-ai-evaluation, micai_diagnose_master, 2026-09-10): -# a question filtering cross-border traffic scored metrics_correct=True, -# dimensions_correct=True, filters_correct=False, because the fixture listed -# ["Inter-region", "Intra-region"] and the agent emitted ["Intra-region", "Inter-region"]. +# Found from a real eval run: a question filtering on a two-value attribute scored +# metrics_correct=True, dimensions_correct=True, filters_correct=False, because the fixture +# listed ["Inter-region", "Intra-region"] and the agent emitted the same two the other way +# round. # Element order is not something an agent has any reason to keep stable between runs, so # every question needing a multi-value attribute filter passes or fails partly at random. @@ -95,7 +95,7 @@ def viz(values): "filter_by": { "f_a": { "type": "attribute_filter", - "using": "label/cross_border_name", + "using": "label/region_name", "state": {"include": values}, } }, @@ -286,10 +286,10 @@ def test_normalized_filters_is_empty_per_category_when_unfiltered(): assert normalized_filters(viz) == {"date": [], "ranking": [], "attribute": []} -def _attr_viz(values, key="include", using="label/cross_border_name"): +def _attr_viz(values, key="include", using="label/region_name"): return _viz( query={ - "fields": {"m": {"using": "metric/approval_rate"}}, + "fields": {"m": {"using": "metric/conversion_rate"}}, "filter_by": {"f": {"type": "attribute_filter", "using": using, "state": {key: values}}}, }, metrics=["m"], @@ -337,8 +337,8 @@ def test_include_and_exclude_of_the_same_elements_still_differ(): def test_the_same_elements_on_a_different_label_still_differ(): - expected = _attr_viz(["A", "B"], using="label/cross_border_name") - assert check_filters(expected, _attr_viz(["B", "A"], using="label/region_name")).attribute_ok is False + expected = _attr_viz(["A", "B"], using="label/region_name") + assert check_filters(expected, _attr_viz(["B", "A"], using="label/channel_name")).attribute_ok is False def test_a_mixed_type_element_list_does_not_crash_scoring():