Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,9 @@

from dotenv import load_dotenv

load_dotenv("/Users/petertomko/gdc-mic-ai-evaluation/.env")
# Whatever .env the caller points at, defaulting to the working directory. It used to be an
# absolute path, which made the script runnable on exactly one machine.
load_dotenv(os.environ.get("GD_EVAL_ENV_FILE", ".env"))

from gooddata_eval.core.agentic.guardrail import _GUARDRAIL_EVALUATION_STEPS # noqa: E402
from gooddata_eval.core.evaluators._guardrail_criteria import ( # noqa: E402
Expand Down
4 changes: 2 additions & 2 deletions packages/gooddata-eval/src/gooddata_eval/core/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -134,8 +134,8 @@ class ReasoningStepEvent(BaseModel):

# Reasoning summaries are full paragraphs, e.g. "**Identifying analytics needs**\n\nI'm
# analyzing..." -- using the whole thing as a latency_breakdown label would make every
# entry an unreadable wall of text. Same bolded-title convention this repo's own reasoning
# tooling already keys off of (see gdc-mic-ai-evaluation's generate_dashboard_summary.py).
# entry an unreadable wall of text. The bolded title is the summary's own heading, and
# downstream reporting keys off it for the same reason.
_REASONING_TITLE_RE = re.compile(r"^\*\*(.+?)\*\*")
_REASONING_LABEL_MAX_LEN = 60

Expand Down
4 changes: 2 additions & 2 deletions packages/gooddata-eval/tests/test_from_insights.py
Original file line number Diff line number Diff line change
Expand Up @@ -382,15 +382,15 @@ def test_built_envelope_is_loadable_as_a_dataset_item(tmp_path):
],
),
)
envelope = build(spec, "How did spend trend by month?", "micai_diagnose_master", set())
envelope = build(spec, "How did spend trend by month?", "demo_workspace", set())
assert "_shape" not in envelope["expected_output"]["visualization"]
assert _validation_errors(envelope) is None

(tmp_path / f"{envelope['id']}.json").write_text(json.dumps(envelope, indent=2))
items = load_local_dataset(tmp_path)
assert [i.id for i in items] == [envelope["id"]]
assert items[0].test_kind == "visualization"
assert items[0].dataset_name == "micai_diagnose_master"
assert items[0].dataset_name == "demo_workspace"


def test_mint_id_is_stable_and_collision_safe():
Expand Down
18 changes: 9 additions & 9 deletions packages/gooddata-eval/tests/test_scoring.py
Original file line number Diff line number Diff line change
Expand Up @@ -79,10 +79,10 @@ def test_check_filters_exact_attribute_match():
# state, type) and never the LIST under state["include"], so two filters selecting the
# same elements in a different order compare unequal.
#
# Found from a real eval run (gdc-mic-ai-evaluation, micai_diagnose_master, 2026-09-10):
# a question filtering cross-border traffic scored metrics_correct=True,
# dimensions_correct=True, filters_correct=False, because the fixture listed
# ["Inter-region", "Intra-region"] and the agent emitted ["Intra-region", "Inter-region"].
# Found from a real eval run: a question filtering on a two-value attribute scored
# metrics_correct=True, dimensions_correct=True, filters_correct=False, because the fixture
# listed ["Inter-region", "Intra-region"] and the agent emitted the same two the other way
# round.
# Element order is not something an agent has any reason to keep stable between runs, so
# every question needing a multi-value attribute filter passes or fails partly at random.

Expand All @@ -95,7 +95,7 @@ def viz(values):
"filter_by": {
"f_a": {
"type": "attribute_filter",
"using": "label/cross_border_name",
"using": "label/region_name",
"state": {"include": values},
}
},
Expand Down Expand Up @@ -286,10 +286,10 @@ def test_normalized_filters_is_empty_per_category_when_unfiltered():
assert normalized_filters(viz) == {"date": [], "ranking": [], "attribute": []}


def _attr_viz(values, key="include", using="label/cross_border_name"):
def _attr_viz(values, key="include", using="label/region_name"):
return _viz(
query={
"fields": {"m": {"using": "metric/approval_rate"}},
"fields": {"m": {"using": "metric/conversion_rate"}},
"filter_by": {"f": {"type": "attribute_filter", "using": using, "state": {key: values}}},
},
metrics=["m"],
Expand Down Expand Up @@ -337,8 +337,8 @@ def test_include_and_exclude_of_the_same_elements_still_differ():


def test_the_same_elements_on_a_different_label_still_differ():
expected = _attr_viz(["A", "B"], using="label/cross_border_name")
assert check_filters(expected, _attr_viz(["B", "A"], using="label/region_name")).attribute_ok is False
expected = _attr_viz(["A", "B"], using="label/region_name")
assert check_filters(expected, _attr_viz(["B", "A"], using="label/channel_name")).attribute_ok is False


def test_a_mixed_type_element_list_does_not_crash_scoring():
Expand Down
Loading