{"version":1,"extractor":"496c9d83e6eb3668","files":{"evaluation/execution_match.py":{"size":17584,"mtimeMs":1789189526392.8706,"hash":"e4774685c66e46f63c5494d672aed0de311f5525f0ca11f67d2636d3f0ea8f67","nodes":[{"id":"evaluation/execution_match.py","name":"execution_match.py","kind":"file","path":"evaluation/execution_match.py","span":"L1-L508","signature":null,"exported":true,"origin":"ast","body_hash":"e4774685c66e46f63c5494d672aed0de311f5525f0ca11f67d2636d3f0ea8f67","chars":17578,"summary_state":"pending","summary":null,"crux":null,"body_text":"\"\"\"Standalone execution-match scoring for S2SQL tasks. The headline entry point is :func:`compare_execution_results`. Inputs are column-major dicts (``{\"col_name\": [values, ...]}``) — that's the shape the SQLite executor returns once we transpose its row-major output. Behavior preserved: - 2-digit float rounding (``NUMBER_OF_DIGITS_CONSIDERED``). - String cells compare **case-insensitively** after strip (``\"No\"`` == ``\"no\"``); JSON leaf strings use the same rule. - JSON array cells compare by **key-agnostic primitive-value coverage**: the smaller object's leaf values must each match a distinct value in the larger object (so compact gold summaries match verbose agent JSON without hand-written field aliases). - Hungarian-algorithm column matching via ``scipy.optimize.linear_sum_assignment``. - Loose mode keeps the 75 % precision/recall threshold boost and the one-vs-many row tolerance for ``ORDER BY`` queries. - Strict mode disables both. - ``is_sql_ordered`` uses sqlglot and tolerates parse failures by falling back to a substring scan for ``order by``. \"\"\" from __future__ import annotations import json from copy import deepcopy from typing import Any, Dict, List, Tuple import numpy as np import sqlglot from sqlglot.optimizer.scope import traverse_scope NUMBER_OF_DIGITS_CONSIDERED = 2 S2SQL_NUMBER_OF_DIGITS_CONSIDERED = 3 MAX_NUMBER_OF_EXTRA_COLUMNS = 3 MAX_NUMBER_OF_MISSING_COLUMNS = 1 _MISSING_COLUMNS_SCORE = 0.5 _WRONG_ORDER_SCORE = 0.5 PERCENTAGE_ATOL = 0.1"},{"id":"evaluation/execution_match.py#_try_parse_json_value","name":"_try_parse_json_value","kind":"function","path":"evaluation/execution_match.py","span":"L46-L57","signature":"def _try_parse_json_value(val: Any) -> Any | None","exported":false,"origin":"ast","body_hash":"83772c23c69bba36f4182ce194793b0127166d3a0ccb0cc67c14c54a3fcc363b","body_text":"def _try_parse_json_value(val: Any) -> Any | None: if isinstance(val, str): text = val.strip() if not text or text[0] not in \"[{\": return None try: return json.loads(text) except json.JSONDecodeError: return None if isinstance(val, (list, dict)): return val return None","summary_state":"pending","summary":null,"crux":null},{"id":"evaluation/execution_match.py#_normalize_primitive","name":"_normalize_primitive","kind":"function","path":"evaluation/execution_match.py","span":"L60-L66","signature":"def _normalize_primitive(val: Any, *, float_digits: int = NUMBER_OF_DIGITS_CONSIDERED) -> Any","exported":false,"origin":"ast","body_hash":"34996418332bf2b41a6764d5f23f5ebb2bed9853883168288741c4ef63462422","body_text":"def _normalize_primitive(val: Any, *, float_digits: int = NUMBER_OF_DIGITS_CONSIDERED) -> Any: if isinstance(val, float): return round(val, float_digits) if isinstance(val, str): # Case-insensitive cell match: \"No\" and \"no\" count as equal. return val.strip().casefold() return val","summary_state":"pending","summary":null,"crux":null},{"id":"evaluation/execution_match.py#_values_equal","name":"_values_equal","kind":"function","path":"evaluation/execution_match.py","span":"L69-L81","signature":"def _values_equal( a: Any, b: Any, *, float_digits: int = NUMBER_OF_DIGITS_CONSIDERED, ) -> bool","exported":false,"origin":"ast","body_hash":"9e4129f180a46955f378debb770fe4e59e3de4a1d881790b866b16fba2f5841f","body_text":"def _values_equal( a: Any, b: Any, *, float_digits: int = NUMBER_OF_DIGITS_CONSIDERED, ) -> bool: a = _normalize_primitive(a, float_digits=float_digits) b = _normalize_primitive(b, float_digits=float_digits) if isinstance(a, float) and isinstance(b, (int, float)): return bool(np.isclose(float(a), float(b), rtol=1e-5, atol=1e-8, equal_nan=True)) if isinstance(b, float) and isinstance(a, (int, float)): return bool(np.isclose(float(a), float(b), rtol=1e-5, atol=1e-8, equal_nan=True)) return a == b","summary_state":"pending","summary":null,"crux":null},{"id":"evaluation/execution_match.py#_primitive_values","name":"_primitive_values","kind":"function","path":"evaluation/execution_match.py","span":"L84-L102","signature":"def _primitive_values( obj: Any, *, float_digits: int = NUMBER_OF_DIGITS_CONSIDERED, ) -> List[Any]","exported":false,"origin":"ast","body_hash":"d4c4ed1a1d7af1b03467836800ee09b448882f6043a2db23950e424b05a8344d","body_text":"def _primitive_values( obj: Any, *, float_digits: int = NUMBER_OF_DIGITS_CONSIDERED, ) -> List[Any]: \"\"\"Collect leaf primitives from a parsed JSON structure.\"\"\" if isinstance(obj, dict): out: List[Any] = [] for v in obj.values(): out.extend(_primitive_values(v, float_digits=float_digits)) return out if isinstance(obj, list): out: List[Any] = [] for item in obj: out.extend(_primitive_values(item, float_digits=float_digits)) return out if obj is None or isinstance(obj, (bool, int, float, str)): return [_normalize_primitive(obj, float_digits=float_digits)] return [obj]","summary_state":"pending","summary":null,"crux":null},{"id":"evaluation/execution_match.py#_primitive_values_cover","name":"_primitive_values_cover","kind":"function","path":"evaluation/execution_match.py","span":"L105-L126","signature":"def _primitive_values_cover( smaller: List[Any], larger: List[Any], *, float_digits: int = NUMBER_OF_DIGITS_CONSIDERED, ) -> bool","exported":false,"origin":"ast","body_hash":"b317831e1e77beb23fe234148027d20575a8bbf6f9d9b81366c9a93e339dd1c5","body_text":"def _primitive_values_cover( smaller: List[Any], larger: List[Any], *, float_digits: int = NUMBER_OF_DIGITS_CONSIDERED, ) -> bool: \"\"\"Return True when every value in *smaller* matches a distinct value in *larger*.\"\"\" if not smaller: return True used = [False] * len(larger) for sv in smaller: matched = False for i, lv in enumerate(larger): if used[i]: continue if _values_equal(sv, lv, float_digits=float_digits): used[i] = True matched = True break if not matched: return False return True","summary_state":"pending","summary":null,"crux":null},{"id":"evaluation/execution_match.py#_json_objects_compatible","name":"_json_objects_compatible","kind":"function","path":"evaluation/execution_match.py","span":"L129-L149","signature":"def _json_objects_compatible( left: Any, right: Any, *, float_digits: int = NUMBER_OF_DIGITS_CONSIDERED, ) -> bool","exported":false,"origin":"ast","body_hash":"07aec8653e5c4ab11b33f88e14f38036f001db8f78c6a75644c828661f12262f","body_text":"def _json_objects_compatible( left: Any, right: Any, *, float_digits: int = NUMBER_OF_DIGITS_CONSIDERED, ) -> bool: \"\"\"Key-agnostic JSON object/array compatibility via value coverage.\"\"\" if isinstance(left, dict) and isinstance(right, dict): v_left = _primitive_values(left, float_digits=float_digits) v_right = _primitive_values(right, float_digits=float_digits) if len(v_left) <= len(v_right): return _primitive_values_cover(v_left, v_right, float_digits=float_digits) return _primitive_values_cover(v_right, v_left, float_digits=float_digits) if isinstance(left, list) and isinstance(right, list): if len(left) != len(right): return False return all( _json_objects_compatible(a, b, float_digits=float_digits) for a, b in zip(left, right) ) return _values_equal(left, right, float_digits=float_digits)","summary_state":"pending","summary":null,"crux":null},{"id":"evaluation/execution_match.py#_json_parsed_equal","name":"_json_parsed_equal","kind":"function","path":"evaluation/execution_match.py","span":"L152-L171","signature":"def _json_parsed_equal( left: Any, right: Any, *, float_digits: int = NUMBER_OF_DIGITS_CONSIDERED, ) -> bool","exported":false,"origin":"ast","body_hash":"149524cf47500d411dba11be240918cd0874a91922c2e289612b1172ecbb4378","body_text":"def _json_parsed_equal( left: Any, right: Any, *, float_digits: int = NUMBER_OF_DIGITS_CONSIDERED, ) -> bool: if isinstance(left, dict) and not isinstance(right, dict): right = [right] left = [left] if isinstance(right, dict) and not isinstance(left, dict): left = [left] right = [right] if isinstance(left, list) and isinstance(right, list): if len(left) != len(right): return False return all( _json_objects_compatible(a, b, float_digits=float_digits) for a, b in zip(left, right) ) return _json_objects_compatible(left, right, float_digits=float_digits)","summary_state":"pending","summary":null,"crux":null},{"id":"evaluation/execution_match.py#_cells_equal","name":"_cells_equal","kind":"function","path":"evaluation/execution_match.py","span":"L174-L184","signature":"def _cells_equal( left: Any, right: Any, *, float_digits: int = NUMBER_OF_DIGITS_CONSIDERED, ) -> bool","exported":false,"origin":"ast","body_hash":"cc1ea9ff32a81449d93b96adcf2aa9703f550af6aa8bf54ebea94217fa7894eb","body_text":"def _cells_equal( left: Any, right: Any, *, float_digits: int = NUMBER_OF_DIGITS_CONSIDERED, ) -> bool: left_json = _try_parse_json_value(left) right_json = _try_parse_json_value(right) if left_json is not None and right_json is not None: return _json_parsed_equal(left_json, right_json, float_digits=float_digits) return _values_equal(left, right, float_digits=float_digits)","summary_state":"pending","summary":null,"crux":null},{"id":"evaluation/execution_match.py#_columns_equal","name":"_columns_equal","kind":"function","path":"evaluation/execution_match.py","span":"L187-L208","signature":"def _columns_equal( col_a: List[Any], col_b: List[Any], *, order_matters: bool, float_digits: int = NUMBER_OF_DIGITS_CONSIDERED, ) -> bool","exported":false,"origin":"ast","body_hash":"19844fc2c7bbdcfe51cfa04545d58fefc973c0f2f047e7d56953deed6b07aa7a","body_text":"def _columns_equal( col_a: List[Any], col_b: List[Any], *, order_matters: bool, float_digits: int = NUMBER_OF_DIGITS_CONSIDERED, ) -> bool: if len(col_a) != len(col_b): return False if order_matters: return all( _cells_equal(a, b, float_digits=float_digits) for a, b in zip(col_a, col_b) ) unmatched = list(col_b) for a in col_a: for i, b in enumerate(unmatched): if _cells_equal(a, b, float_digits=float_digits): unmatched.pop(i) break else: return False return True","summary_state":"pending","summary":null,"crux":null},{"id":"evaluation/execution_match.py#_round_floats","name":"_round_floats","kind":"function","path":"evaluation/execution_match.py","span":"L211-L220","signature":"def _round_floats( denotation: Dict[str, List[Any]], *, float_digits: int = NUMBER_OF_DIGITS_CONSIDERED, ) -> None","exported":false,"origin":"ast","body_hash":"1a9688ef8d525369f72e78fa5f9e498b397833951c971b1d97109c78bd9dcde5","body_text":"def _round_floats( denotation: Dict[str, List[Any]], *, float_digits: int = NUMBER_OF_DIGITS_CONSIDERED, ) -> None: \"\"\"Round floats in place to *float_digits* decimal places.\"\"\" for column_values in denotation.values(): for ix, value in enumerate(column_values): if isinstance(value, float): column_values[ix] = round(value, float_digits)","summary_state":"pending","summary":null,"crux":null},{"id":"evaluation/execution_match.py#_sequences_equal_ignore_order","name":"_sequences_equal_ignore_order","kind":"function","path":"evaluation/execution_match.py","span":"L223-L230","signature":"def _sequences_equal_ignore_order(seq1: List[Any], seq2: List[Any]) -> bool","exported":false,"origin":"ast","body_hash":"728b76f4f3a42b6aec9d59fcadf970123959dac2f03fbd0cd16b161a6614be07","body_text":"def _sequences_equal_ignore_order(seq1: List[Any], seq2: List[Any]) -> bool: unmatched = deepcopy(seq2) for el in seq1: try: unmatched.remove(el) except ValueError: return False return not unmatched","summary_state":"pending","summary":null,"crux":null},{"id":"evaluation/execution_match.py#_rows_for_dataframe","name":"_rows_for_dataframe","kind":"function","path":"evaluation/execution_match.py","span":"L233-L235","signature":"def _rows_for_dataframe(df: Dict[str, List[Any]]) -> int","exported":false,"origin":"ast","body_hash":"ab45c7e7ebdbdacbeeef09046b3765471eba1ee162934000a0d213eae6bbc2db","body_text":"def _rows_for_dataframe(df: Dict[str, List[Any]]) -> int: vals = list(df.values()) return len(vals[0]) if vals else 0","summary_state":"pending","summary":null,"crux":null},{"id":"evaluation/execution_match.py#_dataframe_similarity","name":"_dataframe_similarity","kind":"function","path":"evaluation/execution_match.py","span":"L238-L284","signature":"def _dataframe_similarity( pred: Dict[str, List[Any]], gold: Dict[str, List[Any]], order_matters: bool, *, float_digits: int = NUMBER_OF_DIGITS_CONSIDERED, ) -> float","exported":false,"origin":"ast","body_hash":"4e5ad9f8833fb946ed4bedca65fc9e97e3e1f6381aeaa83b545b3cd03358d79e","body_text":"def _dataframe_similarity( pred: Dict[str, List[Any]], gold: Dict[str, List[Any]], order_matters: bool, *, float_digits: int = NUMBER_OF_DIGITS_CONSIDERED, ) -> float: \"\"\"Hungarian-algorithm column match; return [0, 1] similarity.\"\"\" from scipy.optimize import linear_sum_assignment if not pred: return 0.0 extra = len(pred) - len(gold) if extra > MAX_NUMBER_OF_EXTRA_COLUMNS or -extra > MAX_NUMBER_OF_MISSING_COLUMNS: return 0.0 if order_matters: def metric(a: List, b: List) -> float: return ( 0.0 if _columns_equal(a, b, order_matters=True, float_digits=float_digits) else 1.0 ) else: def metric(a: List, b: List) -> float: return ( 0.0 if _columns_equal(a, b, order_matters=False, float_digits=float_digits) else 1.0 ) pred_values = list(pred.values()) gold_values = list(gold.values()) distances = np.empty((len(pred_values), len(gold_values)), dtype=np.double) for i, pv in enumerate(pred_values): for j, gv in enumerate(gold_values): distances[i, j] = metric(pv, gv) row_ind, col_ind = linear_sum_assignment(cost_matrix=distances) min_cost = round(sum(distances[i, j] for i, j in zip(row_ind, col_ind))) if extra < 0 and min_cost == 0: return _MISSING_COLUMNS_SCORE if min_cost == 0: return 1.0 if order_matters and _rows_for_dataframe(pred) == _rows_for_dataframe(gold): return _dataframe_similarity(pred, gold, False, float_digits=float_digits) * _WRONG_ORDER_SCORE return 0.0","summary_state":"pending","summary":null,"crux":null},{"id":"evaluation/execution_match.py#_dataframe_similarity.metric","name":"metric","kind":"function","path":"evaluation/execution_match.py","span":"L255-L260","signature":"def metric(a: List, b: List) -> float","exported":true,"origin":"ast","body_hash":"837e1a7a82af160447a9cc2b2cd221ac4193ef3c6a6e8700eb6513d50ac1d0e9","body_text":"def metric(a: List, b: List) -> float: return ( 0.0 if _columns_equal(a, b, order_matters=True, float_digits=float_digits) else 1.0 )","summary_state":"pending","summary":null,"crux":null},{"id":"evaluation/execution_match.py#_dataframe_similarity.metric~2","name":"metric","kind":"function","path":"evaluation/execution_match.py","span":"L262-L267","signature":"def metric(a: List, b: List) -> float","exported":true,"origin":"ast","body_hash":"444c262e69a5d2a3b95a4c4018104e36b79ed499bbc8947071e9199ab2db8669","body_text":"def metric(a: List, b: List) -> float: return ( 0.0 if _columns_equal(a, b, order_matters=False, float_digits=float_digits) else 1.0 )","summary_state":"pending","summary":null,"crux":null},{"id":"evaluation/execution_match.py#_check_one_vs_many","name":"_check_one_vs_many","kind":"function","path":"evaluation/execution_match.py","span":"L287-L297","signature":"def _check_one_vs_many( top: Dict[str, List[Any]], all_rows: Dict[str, List[Any]], *, float_digits: int = NUMBER_OF_DIGITS_CONSIDERED, ) -> bool","exported":false,"origin":"ast","body_hash":"000731628ec509aee4ba651ee6b9c16608200542171da06270c8985227dc0c38","body_text":"def _check_one_vs_many( top: Dict[str, List[Any]], all_rows: Dict[str, List[Any]], *, float_digits: int = NUMBER_OF_DIGITS_CONSIDERED, ) -> bool: \"\"\"Match ``top`` (1 row) against the first row of ``all_rows`` (>1 rows).\"\"\" if not (_rows_for_dataframe(top) == 1 and _rows_for_dataframe(all_rows) > 1): return False truncated = {col: [vals[0]] for col, vals in all_rows.items()} return _dataframe_similarity(top, truncated, order_matters=True, float_digits=float_digits) > 0.5","summary_state":"pending","summary":null,"crux":null},{"id":"evaluation/execution_match.py#_compute_eval_score","name":"_compute_eval_score","kind":"function","path":"evaluation/execution_match.py","span":"L300-L320","signature":"def _compute_eval_score( pred: Dict[str, List[Any]], gold: Dict[str, List[Any]], order_matters: bool, max_extra: int, *, float_digits: int = NUMBER_OF_DIGITS_CONSIDERED, ) -> float","exported":false,"origin":"ast","body_hash":"8fb1d31bb41a97792752769e5c2d5a6191198451369f31cd5bf418f88bcf716b","body_text":"def _compute_eval_score( pred: Dict[str, List[Any]], gold: Dict[str, List[Any]], order_matters: bool, max_extra: int, *, float_digits: int = NUMBER_OF_DIGITS_CONSIDERED, ) -> float: if order_matters: if ( _check_one_vs_many(pred, gold, float_digits=float_digits) or _check_one_vs_many(gold, pred, float_digits=float_digits) ): return 1.0 if not pred and not gold: return 1.0 if len(pred) + MAX_NUMBER_OF_MISSING_COLUMNS < len(gold): return 0.0 if len(pred) > len(gold) + max_extra: return 0.0 return _dataframe_similarity(pred, gold, order_matters, float_digits=float_digits)","summary_state":"pending","summary":null,"crux":null},{"id":"evaluation/execution_match.py#_get_overlap_columns","name":"_get_overlap_columns","kind":"function","path":"evaluation/execution_match.py","span":"L323-L407","signature":"def _get_overlap_columns( denotation_pred: Dict[str, List[Any]], denotation_gold: Dict[str, List[Any]], *, ignore_extra_columns_for_recall: bool = False, float_digits: int = NUMBER_OF_DIGITS_CONSIDERED, ) -> Tuple[int, int, float, float]","exported":false,"origin":"ast","body_hash":"9483ac944ab9b315caa531267a025af49152877703fd0becdb230ec04203c31e","body_text":"def _get_overlap_columns( denotation_pred: Dict[str, List[Any]], denotation_gold: Dict[str, List[Any]], *, ignore_extra_columns_for_recall: bool = False, float_digits: int = NUMBER_OF_DIGITS_CONSIDERED, ) -> Tuple[int, int, float, float]: \"\"\"Per-column matching via ``pandas.testing.assert_series_equal`` (loose). Returns ``(exact, percent, precision, recall)`` where (matching the upstream \"swapped\" convention): - precision = matches / |gold cols| - recall = matches / |pred cols|, or matches / |gold cols| when *ignore_extra_columns_for_recall* is set (extra agent columns no longer penalize recall — used by the s2sql judge). \"\"\" import pandas as pd from pandas.testing import assert_series_equal df_gold = pd.DataFrame(denotation_gold) df_pred = pd.DataFrame(denotation_pred) if df_pred.empty or len(df_pred) != len(df_gold): return 0, 0, 0.0, 0.0 df_gold_temp = df_gold.copy(deep=True) n_exact = 0 n_percent = 0 for col_pred_name in df_pred.columns: for col_gold_name in list(df_gold_temp.columns): col_pred = df_pred[col_pred_name].sort_values().reset_index(drop=True) col_gold = df_gold_temp[col_gold_name].sort_values().reset_index(drop=True) if col_pred.dtype == \"object\": col_pred = col_pred.where(col_pred.notna(), None) if col_gold.dtype == \"object\": col_gold = col_gold.where(col_gold.notna(), None) if pd.api.types.is_datetime64_any_dtype(col_pred) or pd.api.types.is_datetime64_any_dtype(col_gold): try: col_pred = col_pred.astype(str) col_gold = col_gold.astype(str) except Exception: pass try: assert_series_equal(col_pred, col_gold, check_dtype=False, check_names=False) n_exact += 1 df_gold_temp = df_gold_temp.drop(columns=[col_gold_name]) break except (AssertionError, TypeError, AttributeError): if _columns_equal( col_pred.tolist(), col_gold.tolist(), order_matters=False, float_digits=float_digits, ): n_exact += 1 df_gold_temp = df_gold_temp.drop(columns=[col_gold_name]) break col_pred_num = pd.to_numeric(col_pred, errors=\"coerce\") col_gold_num = pd.to_numeric(col_gold, errors=\"coerce\") if col_pred_num.isnull().all() or col_gold_num.isnull().all(): continue if np.isclose(col_pred_num, col_gold_num, rtol=1e-5, atol=1e-8, equal_nan=True).all(): n_exact += 1 df_gold_temp = df_gold_temp.drop(columns=[col_gold_name]) break if ( (abs(col_pred_num.dropna()) <= 100).all() and (abs(col_gold_num.dropna()) <= 100).all() and ( np.isclose(col_pred_num, col_gold_num * 100, atol=PERCENTAGE_ATOL, equal_nan=True).all() or np.isclose(col_pred_num * 100, col_gold_num, atol=PERCENTAGE_ATOL, equal_nan=True).all() ) ): n_percent += 1 df_gold_temp = df_gold_temp.drop(columns=[col_gold_name]) break total = n_exact + n_percent precision = total / len(df_gold.columns) if len(df_gold.columns) > 0 else 0.0 recall_denom = ( len(df_gold.columns) if ignore_extra_columns_for_recall else len(df_pred.columns) ) recall = total / recall_denom if recall_denom > 0 else 0.0 return n_exact, n_percent, precision, recall","summary_state":"pending","summary":null,"crux":null},{"id":"evaluation/execution_match.py#is_sql_ordered","name":"is_sql_ordered","kind":"function","path":"evaluation/execution_match.py","span":"L410-L423","signature":"def is_sql_ordered(sql: str) -> bool","exported":true,"origin":"ast","body_hash":"dc1d73260368c475653cb553723a0cc92f879a7552feb3c3769fe74f98cd3784","body_text":"def is_sql_ordered(sql: str) -> bool: \"\"\"Return ``True`` if *sql*'s outermost SELECT carries an ``ORDER BY``.\"\"\" if not sql: return False try: scopes = traverse_scope(sqlglot.parse_one(sql)) if not scopes: return \"order by\" in sql.lower() for order_by in scopes[-1].find_all(sqlglot.exp.Order): if isinstance(order_by.parent, sqlglot.exp.Select): return True except Exception: return \"order by\" in sql.lower() return False","summary_state":"pending","summary":null,"crux":null},{"id":"evaluation/execution_match.py#compare_execution_results","name":"compare_execution_results","kind":"function","path":"evaluation/execution_match.py","span":"L426-L507","signature":"def compare_execution_results( pred_result: Dict[str, List[Any]], gold_result: Dict[str, List[Any]], *, order_matters: bool = False, max_number_of_extra_columns: int = MAX_NUMBER_OF_EXTRA_COLUMNS, strict: bool = False, ignore_extra_columns_for_recall: bool = False, float_digits: int = NUMBER_OF_DIGITS_CONSIDERED, ) -> Tuple[float, float, float]","exported":true,"origin":"ast","body_hash":"0c722a0dcd512870456c731ac8fb92f90c76114a7ea188f909b295f656d3ebe3","body_text":"def compare_execution_results( pred_result: Dict[str, List[Any]], gold_result: Dict[str, List[Any]], *, order_matters: bool = False, max_number_of_extra_columns: int = MAX_NUMBER_OF_EXTRA_COLUMNS, strict: bool = False, ignore_extra_columns_for_recall: bool = False, float_digits: int = NUMBER_OF_DIGITS_CONSIDERED, ) -> Tuple[float, float, float]: \"\"\"Return ``(eval_score_v2, column_precision, column_recall)``. ``eval_score_v2`` mirrors the ``aggregate_eval_results.py`` formula:: GREATEST( eval_score::int, CASE WHEN (col_p>=0.75 AND col_r>=0.75) OR col_p>=1 THEN 1 ELSE 0 END ) Strict mode (``strict=True``) disables both the one-vs-many row tolerance and the precision/recall threshold boost — useful when a downstream metric needs the raw similarity. When *ignore_extra_columns_for_recall* is ``True``, recall is ``matches / |gold cols|`` instead of ``matches / |pred cols|`` so bonus agent columns (e.g. ``seshkey``) do not block the 75 % threshold boost. *float_digits* controls how many decimal places floats are rounded to before comparison (default 2; s2sql uses 3). \"\"\" pred_copy = deepcopy(pred_result) gold_copy = deepcopy(gold_result) _round_floats(pred_copy, float_digits=float_digits) _round_floats(gold_copy, float_digits=float_digits) if not pred_copy and not gold_copy: return 1.0, 1.0, 1.0 if not pred_copy or not gold_copy: return 0.0, 0.0, 0.0 pred_rows = _rows_for_dataframe(pred_copy) gold_rows = _rows_for_dataframe(gold_copy) if pred_rows != gold_rows: if ( not strict and order_matters and ( _check_one_vs_many(pred_copy, gold_copy, float_digits=float_digits) or _check_one_vs_many(gold_copy, pred_copy, float_digits=float_digits) ) ): return 1.0, 1.0, 1.0 return 0.0, 0.0, 0.0 eval_score = _compute_eval_score( pred_copy, gold_copy, order_matters, max_number_of_extra_columns, float_digits=float_digits, ) _, _, col_p, col_r = _get_overlap_columns( pred_copy, gold_copy, ignore_extra_columns_for_recall=ignore_extra_columns_for_recall, float_digits=float_digits, ) threshold_met = bool( col_p and col_r and ((col_p >= 0.75 and col_r >= 0.75) or col_p >= 1.0) ) if strict: eval_score_v2 = eval_score elif threshold_met: eval_score_v2 = 1.0 else: eval_score_v2 = eval_score return eval_score_v2, col_p, col_r","summary_state":"pending","summary":null,"crux":null}],"rawEdges":[{"source":"evaluation/execution_match.py","relation":"imports","specifier":"json","file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py","relation":"imports","specifier":"copy","file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py","relation":"imports","specifier":"typing","file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py","relation":"imports","specifier":"sqlglot","file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py","relation":"imports","specifier":"sqlglot.optimizer.scope","file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py","relation":"contains","targetId":"evaluation/execution_match.py#_try_parse_json_value","file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_try_parse_json_value","relation":"calls","name":"isinstance","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_try_parse_json_value","relation":"calls","name":"strip","viaMember":true,"file":"evaluation/execution_match.py","recvType":"Any"},{"source":"evaluation/execution_match.py#_try_parse_json_value","relation":"calls","name":"loads","viaMember":true,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_try_parse_json_value","relation":"calls","name":"isinstance","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py","relation":"contains","targetId":"evaluation/execution_match.py#_normalize_primitive","file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_normalize_primitive","relation":"calls","name":"isinstance","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_normalize_primitive","relation":"calls","name":"round","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_normalize_primitive","relation":"calls","name":"isinstance","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_normalize_primitive","relation":"calls","name":"casefold","viaMember":true,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_normalize_primitive","relation":"calls","name":"strip","viaMember":true,"file":"evaluation/execution_match.py","recvType":"Any"},{"source":"evaluation/execution_match.py","relation":"contains","targetId":"evaluation/execution_match.py#_values_equal","file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_values_equal","relation":"calls","name":"_normalize_primitive","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_values_equal","relation":"calls","name":"_normalize_primitive","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_values_equal","relation":"calls","name":"isinstance","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_values_equal","relation":"calls","name":"isinstance","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_values_equal","relation":"calls","name":"bool","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_values_equal","relation":"calls","name":"isclose","viaMember":true,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_values_equal","relation":"calls","name":"float","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_values_equal","relation":"calls","name":"float","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_values_equal","relation":"calls","name":"isinstance","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_values_equal","relation":"calls","name":"isinstance","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_values_equal","relation":"calls","name":"bool","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_values_equal","relation":"calls","name":"isclose","viaMember":true,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_values_equal","relation":"calls","name":"float","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_values_equal","relation":"calls","name":"float","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py","relation":"contains","targetId":"evaluation/execution_match.py#_primitive_values","file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_primitive_values","relation":"calls","name":"isinstance","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_primitive_values","relation":"calls","name":"values","viaMember":true,"file":"evaluation/execution_match.py","recvType":"Any"},{"source":"evaluation/execution_match.py#_primitive_values","relation":"calls","name":"extend","viaMember":true,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_primitive_values","relation":"calls","name":"_primitive_values","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_primitive_values","relation":"calls","name":"isinstance","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_primitive_values","relation":"calls","name":"extend","viaMember":true,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_primitive_values","relation":"calls","name":"_primitive_values","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_primitive_values","relation":"calls","name":"isinstance","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_primitive_values","relation":"calls","name":"_normalize_primitive","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py","relation":"contains","targetId":"evaluation/execution_match.py#_primitive_values_cover","file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_primitive_values_cover","relation":"calls","name":"len","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_primitive_values_cover","relation":"calls","name":"enumerate","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_primitive_values_cover","relation":"calls","name":"_values_equal","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py","relation":"contains","targetId":"evaluation/execution_match.py#_json_objects_compatible","file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_json_objects_compatible","relation":"calls","name":"isinstance","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_json_objects_compatible","relation":"calls","name":"isinstance","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_json_objects_compatible","relation":"calls","name":"_primitive_values","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_json_objects_compatible","relation":"calls","name":"_primitive_values","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_json_objects_compatible","relation":"calls","name":"len","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_json_objects_compatible","relation":"calls","name":"len","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_json_objects_compatible","relation":"calls","name":"_primitive_values_cover","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_json_objects_compatible","relation":"calls","name":"_primitive_values_cover","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_json_objects_compatible","relation":"calls","name":"isinstance","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_json_objects_compatible","relation":"calls","name":"isinstance","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_json_objects_compatible","relation":"calls","name":"len","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_json_objects_compatible","relation":"calls","name":"len","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_json_objects_compatible","relation":"calls","name":"all","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_json_objects_compatible","relation":"calls","name":"_json_objects_compatible","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_json_objects_compatible","relation":"calls","name":"zip","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_json_objects_compatible","relation":"calls","name":"_values_equal","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py","relation":"contains","targetId":"evaluation/execution_match.py#_json_parsed_equal","file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_json_parsed_equal","relation":"calls","name":"isinstance","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_json_parsed_equal","relation":"calls","name":"isinstance","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_json_parsed_equal","relation":"calls","name":"isinstance","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_json_parsed_equal","relation":"calls","name":"isinstance","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_json_parsed_equal","relation":"calls","name":"isinstance","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_json_parsed_equal","relation":"calls","name":"isinstance","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_json_parsed_equal","relation":"calls","name":"len","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_json_parsed_equal","relation":"calls","name":"len","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_json_parsed_equal","relation":"calls","name":"all","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_json_parsed_equal","relation":"calls","name":"_json_objects_compatible","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_json_parsed_equal","relation":"calls","name":"zip","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_json_parsed_equal","relation":"calls","name":"_json_objects_compatible","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py","relation":"contains","targetId":"evaluation/execution_match.py#_cells_equal","file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_cells_equal","relation":"calls","name":"_try_parse_json_value","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_cells_equal","relation":"calls","name":"_try_parse_json_value","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_cells_equal","relation":"calls","name":"_json_parsed_equal","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_cells_equal","relation":"calls","name":"_values_equal","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py","relation":"contains","targetId":"evaluation/execution_match.py#_columns_equal","file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_columns_equal","relation":"calls","name":"len","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_columns_equal","relation":"calls","name":"len","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_columns_equal","relation":"calls","name":"all","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_columns_equal","relation":"calls","name":"_cells_equal","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_columns_equal","relation":"calls","name":"zip","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_columns_equal","relation":"calls","name":"list","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_columns_equal","relation":"calls","name":"enumerate","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_columns_equal","relation":"calls","name":"_cells_equal","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_columns_equal","relation":"calls","name":"pop","viaMember":true,"file":"evaluation/execution_match.py","recvType":"list"},{"source":"evaluation/execution_match.py","relation":"contains","targetId":"evaluation/execution_match.py#_round_floats","file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_round_floats","relation":"calls","name":"values","viaMember":true,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_round_floats","relation":"calls","name":"enumerate","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_round_floats","relation":"calls","name":"isinstance","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_round_floats","relation":"calls","name":"round","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py","relation":"contains","targetId":"evaluation/execution_match.py#_sequences_equal_ignore_order","file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_sequences_equal_ignore_order","relation":"calls","name":"deepcopy","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_sequences_equal_ignore_order","relation":"calls","name":"remove","viaMember":true,"file":"evaluation/execution_match.py","recvType":"deepcopy"},{"source":"evaluation/execution_match.py","relation":"contains","targetId":"evaluation/execution_match.py#_rows_for_dataframe","file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_rows_for_dataframe","relation":"calls","name":"list","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_rows_for_dataframe","relation":"calls","name":"values","viaMember":true,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_rows_for_dataframe","relation":"calls","name":"len","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py","relation":"contains","targetId":"evaluation/execution_match.py#_dataframe_similarity","file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py","relation":"imports","specifier":"scipy.optimize","file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_dataframe_similarity","relation":"calls","name":"len","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_dataframe_similarity","relation":"calls","name":"len","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_dataframe_similarity","relation":"contains","targetId":"evaluation/execution_match.py#_dataframe_similarity.metric","file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_dataframe_similarity.metric","relation":"calls","name":"_columns_equal","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_dataframe_similarity","relation":"contains","targetId":"evaluation/execution_match.py#_dataframe_similarity.metric~2","file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_dataframe_similarity.metric~2","relation":"calls","name":"_columns_equal","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_dataframe_similarity","relation":"calls","name":"list","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_dataframe_similarity","relation":"calls","name":"values","viaMember":true,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_dataframe_similarity","relation":"calls","name":"list","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_dataframe_similarity","relation":"calls","name":"values","viaMember":true,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_dataframe_similarity","relation":"calls","name":"empty","viaMember":true,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_dataframe_similarity","relation":"calls","name":"len","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_dataframe_similarity","relation":"calls","name":"len","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_dataframe_similarity","relation":"calls","name":"enumerate","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_dataframe_similarity","relation":"calls","name":"enumerate","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_dataframe_similarity","relation":"calls","name":"metric","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_dataframe_similarity","relation":"calls","name":"linear_sum_assignment","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_dataframe_similarity","relation":"calls","name":"round","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_dataframe_similarity","relation":"calls","name":"sum","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_dataframe_similarity","relation":"calls","name":"zip","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_dataframe_similarity","relation":"calls","name":"_rows_for_dataframe","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_dataframe_similarity","relation":"calls","name":"_rows_for_dataframe","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_dataframe_similarity","relation":"calls","name":"_dataframe_similarity","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py","relation":"contains","targetId":"evaluation/execution_match.py#_check_one_vs_many","file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_check_one_vs_many","relation":"calls","name":"_rows_for_dataframe","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_check_one_vs_many","relation":"calls","name":"_rows_for_dataframe","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_check_one_vs_many","relation":"calls","name":"items","viaMember":true,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_check_one_vs_many","relation":"calls","name":"_dataframe_similarity","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py","relation":"contains","targetId":"evaluation/execution_match.py#_compute_eval_score","file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_compute_eval_score","relation":"calls","name":"_check_one_vs_many","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_compute_eval_score","relation":"calls","name":"_check_one_vs_many","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_compute_eval_score","relation":"calls","name":"len","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_compute_eval_score","relation":"calls","name":"len","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_compute_eval_score","relation":"calls","name":"len","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_compute_eval_score","relation":"calls","name":"len","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_compute_eval_score","relation":"calls","name":"_dataframe_similarity","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py","relation":"contains","targetId":"evaluation/execution_match.py#_get_overlap_columns","file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py","relation":"imports","specifier":"pandas.testing","file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_get_overlap_columns","relation":"calls","name":"DataFrame","viaMember":true,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_get_overlap_columns","relation":"calls","name":"DataFrame","viaMember":true,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_get_overlap_columns","relation":"calls","name":"len","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_get_overlap_columns","relation":"calls","name":"len","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_get_overlap_columns","relation":"calls","name":"copy","viaMember":true,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_get_overlap_columns","relation":"calls","name":"list","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_get_overlap_columns","relation":"calls","name":"reset_index","viaMember":true,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_get_overlap_columns","relation":"calls","name":"sort_values","viaMember":true,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_get_overlap_columns","relation":"calls","name":"reset_index","viaMember":true,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_get_overlap_columns","relation":"calls","name":"sort_values","viaMember":true,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_get_overlap_columns","relation":"calls","name":"where","viaMember":true,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_get_overlap_columns","relation":"calls","name":"notna","viaMember":true,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_get_overlap_columns","relation":"calls","name":"where","viaMember":true,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_get_overlap_columns","relation":"calls","name":"notna","viaMember":true,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_get_overlap_columns","relation":"calls","name":"is_datetime64_any_dtype","viaMember":true,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_get_overlap_columns","relation":"calls","name":"is_datetime64_any_dtype","viaMember":true,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_get_overlap_columns","relation":"calls","name":"astype","viaMember":true,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_get_overlap_columns","relation":"calls","name":"astype","viaMember":true,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_get_overlap_columns","relation":"calls","name":"assert_series_equal","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_get_overlap_columns","relation":"calls","name":"drop","viaMember":true,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_get_overlap_columns","relation":"calls","name":"_columns_equal","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_get_overlap_columns","relation":"calls","name":"tolist","viaMember":true,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_get_overlap_columns","relation":"calls","name":"tolist","viaMember":true,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_get_overlap_columns","relation":"calls","name":"drop","viaMember":true,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_get_overlap_columns","relation":"calls","name":"to_numeric","viaMember":true,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_get_overlap_columns","relation":"calls","name":"to_numeric","viaMember":true,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_get_overlap_columns","relation":"calls","name":"all","viaMember":true,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_get_overlap_columns","relation":"calls","name":"isnull","viaMember":true,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_get_overlap_columns","relation":"calls","name":"all","viaMember":true,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_get_overlap_columns","relation":"calls","name":"isnull","viaMember":true,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_get_overlap_columns","relation":"calls","name":"all","viaMember":true,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_get_overlap_columns","relation":"calls","name":"isclose","viaMember":true,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_get_overlap_columns","relation":"calls","name":"drop","viaMember":true,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_get_overlap_columns","relation":"calls","name":"all","viaMember":true,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_get_overlap_columns","relation":"calls","name":"abs","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_get_overlap_columns","relation":"calls","name":"dropna","viaMember":true,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_get_overlap_columns","relation":"calls","name":"all","viaMember":true,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_get_overlap_columns","relation":"calls","name":"abs","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_get_overlap_columns","relation":"calls","name":"dropna","viaMember":true,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_get_overlap_columns","relation":"calls","name":"all","viaMember":true,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_get_overlap_columns","relation":"calls","name":"isclose","viaMember":true,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_get_overlap_columns","relation":"calls","name":"all","viaMember":true,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_get_overlap_columns","relation":"calls","name":"isclose","viaMember":true,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_get_overlap_columns","relation":"calls","name":"drop","viaMember":true,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_get_overlap_columns","relation":"calls","name":"len","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_get_overlap_columns","relation":"calls","name":"len","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_get_overlap_columns","relation":"calls","name":"len","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#_get_overlap_columns","relation":"calls","name":"len","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py","relation":"contains","targetId":"evaluation/execution_match.py#is_sql_ordered","file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#is_sql_ordered","relation":"calls","name":"traverse_scope","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#is_sql_ordered","relation":"calls","name":"parse_one","viaMember":true,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#is_sql_ordered","relation":"calls","name":"lower","viaMember":true,"file":"evaluation/execution_match.py","recvType":"str"},{"source":"evaluation/execution_match.py#is_sql_ordered","relation":"calls","name":"find_all","viaMember":true,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#is_sql_ordered","relation":"calls","name":"isinstance","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#is_sql_ordered","relation":"calls","name":"lower","viaMember":true,"file":"evaluation/execution_match.py","recvType":"str"},{"source":"evaluation/execution_match.py","relation":"contains","targetId":"evaluation/execution_match.py#compare_execution_results","file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#compare_execution_results","relation":"calls","name":"deepcopy","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#compare_execution_results","relation":"calls","name":"deepcopy","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#compare_execution_results","relation":"calls","name":"_round_floats","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#compare_execution_results","relation":"calls","name":"_round_floats","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#compare_execution_results","relation":"calls","name":"_rows_for_dataframe","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#compare_execution_results","relation":"calls","name":"_rows_for_dataframe","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#compare_execution_results","relation":"calls","name":"_check_one_vs_many","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#compare_execution_results","relation":"calls","name":"_check_one_vs_many","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#compare_execution_results","relation":"calls","name":"_compute_eval_score","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#compare_execution_results","relation":"calls","name":"_get_overlap_columns","viaMember":false,"file":"evaluation/execution_match.py"},{"source":"evaluation/execution_match.py#compare_execution_results","relation":"calls","name":"bool","viaMember":false,"file":"evaluation/execution_match.py"}]},"evaluation/llm_judge.py":{"size":4341,"mtimeMs":1789189526392.8706,"hash":"4b22678c410010004c6caebcbbcd3a375c62f59cd32aaa55f6710cf77806b01a","nodes":[{"id":"evaluation/llm_judge.py","name":"llm_judge.py","kind":"file","path":"evaluation/llm_judge.py","span":"L1-L137","signature":null,"exported":true,"origin":"ast","body_hash":"4b22678c410010004c6caebcbbcd3a375c62f59cd32aaa55f6710cf77806b01a","chars":4341,"summary_state":"pending","summary":null,"crux":null,"body_text":"#!/usr/bin/env python3 \"\"\"Run the public text-answer judge over a JSONL file.\"\"\" from __future__ import annotations import argparse import json import os import re from pathlib import Path from typing import Any, Iterator from openai import OpenAI PROMPT_PATH = Path(__file__).resolve().parent / \"prompts\" / \"answer_judge.txt\" PROMPT = PROMPT_PATH.read_text(encoding=\"utf-8\") if __name__ == \"__main__\": main()"},{"id":"evaluation/llm_judge.py#parse_verdict","name":"parse_verdict","kind":"function","path":"evaluation/llm_judge.py","span":"L20-L52","signature":"def parse_verdict(raw: str) -> dict[str, Any]","exported":true,"origin":"ast","body_hash":"2eae0b3cffc7ca4f124d5bc44184781d12d3e56965193253e43f5914351ace3e","body_text":"def parse_verdict(raw: str) -> dict[str, Any]: \"\"\"Parse the auditable fields emitted by the judge prompt.\"\"\" text = raw.replace(\"**\", \"\") correct_match = re.search(r\"correct:\\s*(yes|no)\", text, re.IGNORECASE) reason_match = re.search( r\"reasoning:\\s*(.+?)(?=\\ncorrect:|\\nconfidence:|$)\", text, re.DOTALL | re.IGNORECASE, ) answer_match = re.search( r\"extracted_final_answer:\\s*(.+?)(?=\\n\\[correct_answer\\]|\\nreasoning:|$)\", text, re.DOTALL | re.IGNORECASE, ) confidence_match = re.search( r\"confidence:\\s*(\\d+(?:\\.\\d+)?)%?\", text, re.IGNORECASE, ) return { \"correct\": bool( correct_match and correct_match.group(1).lower() == \"yes\" ), \"parse_ok\": correct_match is not None, \"extracted_final_answer\": ( answer_match.group(1).strip() if answer_match else None ), \"reasoning\": reason_match.group(1).strip() if reason_match else raw.strip(), \"confidence\": ( float(confidence_match.group(1)) if confidence_match else None ), \"raw_judge_output\": raw, }","summary_state":"pending","summary":null,"crux":null},{"id":"evaluation/llm_judge.py#iter_jsonl","name":"iter_jsonl","kind":"function","path":"evaluation/llm_judge.py","span":"L55-L63","signature":"def iter_jsonl(path: Path) -> Iterator[dict[str, Any]]","exported":true,"origin":"ast","body_hash":"ddab8364559eab58cb87885fa07d48149a94acd8bf2c093a960647cee842747d","body_text":"def iter_jsonl(path: Path) -> Iterator[dict[str, Any]]: with path.open(encoding=\"utf-8\") as handle: for line_number, line in enumerate(handle, 1): if not line.strip(): continue value = json.loads(line) if not isinstance(value, dict): raise ValueError(f\"{path}:{line_number}: expected a JSON object\") yield value","summary_state":"pending","summary":null,"crux":null},{"id":"evaluation/llm_judge.py#judge_one","name":"judge_one","kind":"function","path":"evaluation/llm_judge.py","span":"L66-L90","signature":"def judge_one( *, client: OpenAI, model: str, record: dict[str, Any], ) -> dict[str, Any]","exported":true,"origin":"ast","body_hash":"2e84f6113b5da5739529634f32b841331b94eac3744e141f62a2f121ae32a69b","body_text":"def judge_one( *, client: OpenAI, model: str, record: dict[str, Any], ) -> dict[str, Any]: question = str(record.get(\"question\") or \"\").strip() answer = str(record.get(\"response\") or \"\").strip() expected = str(record.get(\"correct_answer\") or \"\").strip() if not question or not answer or not expected: raise ValueError(\"each record needs question, response, and correct_answer\") prompt = PROMPT.format( question=question, answer=answer, expected_answer=expected, ) response = client.chat.completions.create( model=model, messages=[{\"role\": \"user\", \"content\": prompt}], temperature=0.0, max_tokens=2048, ) raw = (response.choices[0].message.content or \"\").strip() return {**record, **parse_verdict(raw)}","summary_state":"pending","summary":null,"crux":null},{"id":"evaluation/llm_judge.py#parse_args","name":"parse_args","kind":"function","path":"evaluation/llm_judge.py","span":"L93-L103","signature":"def parse_args() -> argparse.Namespace","exported":true,"origin":"ast","body_hash":"4310de9009e1b508d7fc4805cfda8ae182b2c176693c8b52345498dc46e4ab4c","body_text":"def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument(\"--input\", required=True, type=Path) parser.add_argument(\"--output\", required=True, type=Path) parser.add_argument(\"--model\", default=os.environ.get(\"JUDGE_MODEL\", \"\")) parser.add_argument( \"--base-url\", default=os.environ.get(\"JUDGE_BASE_URL\", \"\"), help=\"Optional OpenAI-compatible API base URL.\", ) return parser.parse_args()","summary_state":"pending","summary":null,"crux":null},{"id":"evaluation/llm_judge.py#main","name":"main","kind":"function","path":"evaluation/llm_judge.py","span":"L106-L132","signature":"def main() -> None","exported":true,"origin":"ast","body_hash":"b275dd3291a1e794b3cf979d322889494a12276fbc06ed8baced8e445852be36","body_text":"def main() -> None: args = parse_args() model = args.model.strip() if not model: raise SystemExit(\"Set JUDGE_MODEL or pass --model.\") client_kwargs: dict[str, Any] = { \"api_key\": os.environ.get(\"JUDGE_API_KEY\", \"EMPTY\"), } if args.base_url.strip(): client_kwargs[\"base_url\"] = args.base_url.strip() client = OpenAI(**client_kwargs) output = args.output.expanduser().resolve() output.parent.mkdir(parents=True, exist_ok=True) total = 0 correct = 0 with output.open(\"w\", encoding=\"utf-8\") as handle: for record in iter_jsonl(args.input.expanduser().resolve()): judged = judge_one(client=client, model=model, record=record) total += 1 correct += int(judged[\"correct\"]) handle.write(json.dumps(judged, ensure_ascii=False) + \"\\n\") accuracy = correct / total if total else 0.0 print(f\"Judged {total} records | correct={correct} | accuracy={accuracy:.3f}\") print(f\"Wrote {output}\")","summary_state":"pending","summary":null,"crux":null}],"rawEdges":[{"source":"evaluation/llm_judge.py","relation":"imports","specifier":"argparse","file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py","relation":"imports","specifier":"json","file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py","relation":"imports","specifier":"os","file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py","relation":"imports","specifier":"re","file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py","relation":"imports","specifier":"pathlib","file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py","relation":"imports","specifier":"typing","file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py","relation":"imports","specifier":"openai","file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py","relation":"calls","name":"resolve","viaMember":true,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py","relation":"calls","name":"Path","viaMember":false,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py","relation":"calls","name":"read_text","viaMember":true,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py","relation":"contains","targetId":"evaluation/llm_judge.py#parse_verdict","file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py#parse_verdict","relation":"calls","name":"replace","viaMember":true,"file":"evaluation/llm_judge.py","recvType":"str"},{"source":"evaluation/llm_judge.py#parse_verdict","relation":"calls","name":"search","viaMember":true,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py#parse_verdict","relation":"calls","name":"search","viaMember":true,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py#parse_verdict","relation":"calls","name":"search","viaMember":true,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py#parse_verdict","relation":"calls","name":"search","viaMember":true,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py#parse_verdict","relation":"calls","name":"bool","viaMember":false,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py#parse_verdict","relation":"calls","name":"lower","viaMember":true,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py#parse_verdict","relation":"calls","name":"group","viaMember":true,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py#parse_verdict","relation":"calls","name":"strip","viaMember":true,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py#parse_verdict","relation":"calls","name":"group","viaMember":true,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py#parse_verdict","relation":"calls","name":"strip","viaMember":true,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py#parse_verdict","relation":"calls","name":"group","viaMember":true,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py#parse_verdict","relation":"calls","name":"strip","viaMember":true,"file":"evaluation/llm_judge.py","recvType":"str"},{"source":"evaluation/llm_judge.py#parse_verdict","relation":"calls","name":"float","viaMember":false,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py#parse_verdict","relation":"calls","name":"group","viaMember":true,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py","relation":"contains","targetId":"evaluation/llm_judge.py#iter_jsonl","file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py#iter_jsonl","relation":"calls","name":"open","viaMember":true,"file":"evaluation/llm_judge.py","recvType":"Path"},{"source":"evaluation/llm_judge.py#iter_jsonl","relation":"calls","name":"enumerate","viaMember":false,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py#iter_jsonl","relation":"calls","name":"strip","viaMember":true,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py#iter_jsonl","relation":"calls","name":"loads","viaMember":true,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py#iter_jsonl","relation":"calls","name":"isinstance","viaMember":false,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py#iter_jsonl","relation":"calls","name":"ValueError","viaMember":false,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py","relation":"contains","targetId":"evaluation/llm_judge.py#judge_one","file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py#judge_one","relation":"calls","name":"strip","viaMember":true,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py#judge_one","relation":"calls","name":"str","viaMember":false,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py#judge_one","relation":"calls","name":"get","viaMember":true,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py#judge_one","relation":"calls","name":"strip","viaMember":true,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py#judge_one","relation":"calls","name":"str","viaMember":false,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py#judge_one","relation":"calls","name":"get","viaMember":true,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py#judge_one","relation":"calls","name":"strip","viaMember":true,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py#judge_one","relation":"calls","name":"str","viaMember":false,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py#judge_one","relation":"calls","name":"get","viaMember":true,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py#judge_one","relation":"calls","name":"ValueError","viaMember":false,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py#judge_one","relation":"calls","name":"format","viaMember":true,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py#judge_one","relation":"calls","name":"create","viaMember":true,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py#judge_one","relation":"calls","name":"strip","viaMember":true,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py#judge_one","relation":"calls","name":"parse_verdict","viaMember":false,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py","relation":"contains","targetId":"evaluation/llm_judge.py#parse_args","file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py#parse_args","relation":"calls","name":"ArgumentParser","viaMember":true,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py#parse_args","relation":"calls","name":"add_argument","viaMember":true,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py#parse_args","relation":"calls","name":"add_argument","viaMember":true,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py#parse_args","relation":"calls","name":"add_argument","viaMember":true,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py#parse_args","relation":"calls","name":"get","viaMember":true,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py#parse_args","relation":"calls","name":"add_argument","viaMember":true,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py#parse_args","relation":"calls","name":"get","viaMember":true,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py#parse_args","relation":"calls","name":"parse_args","viaMember":true,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py","relation":"contains","targetId":"evaluation/llm_judge.py#main","file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py#main","relation":"calls","name":"parse_args","viaMember":false,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py#main","relation":"calls","name":"strip","viaMember":true,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py#main","relation":"calls","name":"SystemExit","viaMember":false,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py#main","relation":"calls","name":"get","viaMember":true,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py#main","relation":"calls","name":"strip","viaMember":true,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py#main","relation":"calls","name":"strip","viaMember":true,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py#main","relation":"calls","name":"OpenAI","viaMember":false,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py#main","relation":"calls","name":"resolve","viaMember":true,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py#main","relation":"calls","name":"expanduser","viaMember":true,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py#main","relation":"calls","name":"mkdir","viaMember":true,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py#main","relation":"calls","name":"open","viaMember":true,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py#main","relation":"calls","name":"iter_jsonl","viaMember":false,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py#main","relation":"calls","name":"resolve","viaMember":true,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py#main","relation":"calls","name":"expanduser","viaMember":true,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py#main","relation":"calls","name":"judge_one","viaMember":false,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py#main","relation":"calls","name":"int","viaMember":false,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py#main","relation":"calls","name":"write","viaMember":true,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py#main","relation":"calls","name":"dumps","viaMember":true,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py#main","relation":"calls","name":"print","viaMember":false,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py#main","relation":"calls","name":"print","viaMember":false,"file":"evaluation/llm_judge.py"},{"source":"evaluation/llm_judge.py","relation":"calls","name":"main","viaMember":false,"file":"evaluation/llm_judge.py"}]},"evaluation/s2sql_eval.py":{"size":6991,"mtimeMs":1789189526392.8706,"hash":"ebafafecd2729348b69204c80873d65dce0883ddbce303d4f8cc6e09e6b098c1","nodes":[{"id":"evaluation/s2sql_eval.py","name":"s2sql_eval.py","kind":"file","path":"evaluation/s2sql_eval.py","span":"L1-L220","signature":null,"exported":true,"origin":"ast","body_hash":"ebafafecd2729348b69204c80873d65dce0883ddbce303d4f8cc6e09e6b098c1","chars":6991,"summary_state":"pending","summary":null,"crux":null,"body_text":"#!/usr/bin/env python3 \"\"\"Evaluate S2SQL predictions by read-only SQL execution match.\"\"\" from __future__ import annotations import argparse import json from pathlib import Path from typing import Any, Iterator from execution_match import ( S2SQL_NUMBER_OF_DIGITS_CONSIDERED, compare_execution_results, is_sql_ordered, ) from sql_extract import SOURCE_NONE, extract_agent_sql from sqlite_executor import execute_readonly_rows if __name__ == \"__main__\": main()"},{"id":"evaluation/s2sql_eval.py#iter_jsonl","name":"iter_jsonl","kind":"function","path":"evaluation/s2sql_eval.py","span":"L20-L28","signature":"def iter_jsonl(path: Path) -> Iterator[dict[str, Any]]","exported":true,"origin":"ast","body_hash":"ddab8364559eab58cb87885fa07d48149a94acd8bf2c093a960647cee842747d","body_text":"def iter_jsonl(path: Path) -> Iterator[dict[str, Any]]: with path.open(encoding=\"utf-8\") as handle: for line_number, line in enumerate(handle, 1): if not line.strip(): continue value = json.loads(line) if not isinstance(value, dict): raise ValueError(f\"{path}:{line_number}: expected a JSON object\") yield value","summary_state":"pending","summary":null,"crux":null},{"id":"evaluation/s2sql_eval.py#rows_to_columnar","name":"rows_to_columnar","kind":"function","path":"evaluation/s2sql_eval.py","span":"L31-L39","signature":"def rows_to_columnar( rows: list[list[Any]], columns: list[str], ) -> dict[str, list[Any]]","exported":true,"origin":"ast","body_hash":"5e6521b18d7031d4fe2b1a042ef5d02da74fd4f94d4a942cdcd6454d092b6e4b","body_text":"def rows_to_columnar( rows: list[list[Any]], columns: list[str], ) -> dict[str, list[Any]]: result = {column: [] for column in columns} for row in rows: for index, column in enumerate(columns): result[column].append(row[index] if index < len(row) else None) return result","summary_state":"pending","summary":null,"crux":null},{"id":"evaluation/s2sql_eval.py#resolve_database","name":"resolve_database","kind":"function","path":"evaluation/s2sql_eval.py","span":"L42-L54","signature":"def resolve_database( record: dict[str, Any], *, db_root: Path | None, db_template: str, ) -> Path","exported":true,"origin":"ast","body_hash":"90c165914ec1f4b95b73cc2a4e52c8ebb05ff81335de505bba050979f60445cf","body_text":"def resolve_database( record: dict[str, Any], *, db_root: Path | None, db_template: str, ) -> Path: explicit = str(record.get(\"db_path\") or \"\").strip() if explicit: return Path(explicit).expanduser().resolve() db_name = str(record.get(\"db\") or \"\").strip() if not db_name or db_root is None: raise ValueError(\"record needs db_path, or db together with --db-root\") return db_root / db_template.format(db=db_name)","summary_state":"pending","summary":null,"crux":null},{"id":"evaluation/s2sql_eval.py#evaluate_record","name":"evaluate_record","kind":"function","path":"evaluation/s2sql_eval.py","span":"L57-L172","signature":"def evaluate_record( record: dict[str, Any], *, db_root: Path | None, db_template: str, max_rows: int, timeout: float | None, strict: bool, ) -> dict[str, Any]","exported":true,"origin":"ast","body_hash":"8c3dd05e95866b445d5779d81e5e8ed2d3365759e720bd4a762301634fd900ff","body_text":"def evaluate_record( record: dict[str, Any], *, db_root: Path | None, db_template: str, max_rows: int, timeout: float | None, strict: bool, ) -> dict[str, Any]: final_text = str( record.get(\"response\") or record.get(\"final_answer\") or \"\" ).strip() gold_sql = str(record.get(\"gold_sql\") or record.get(\"sql\") or \"\").strip() messages = record.get(\"messages\") if not isinstance(messages, list): messages = None message_format = str(record.get(\"message_format\") or \"openai\") verdict: dict[str, Any] = { \"instance_id\": record.get(\"instance_id\") or record.get(\"id\"), \"judge_kind\": \"s2sql_exec\", \"match\": False, \"score\": 0.0, \"reason\": \"\", \"agent_sql\": \"\", \"agent_sql_source\": SOURCE_NONE, \"column_precision\": 0.0, \"column_recall\": 0.0, \"order_sensitive\": False, \"agent_exec_error\": None, \"gold_exec_error\": None, \"agent_table\": None, \"gold_table\": None, } if not gold_sql: verdict[\"reason\"] = \"missing_gold_sql\" return verdict agent_sql, source = extract_agent_sql( final_text=final_text, messages=messages, message_format=message_format, ) verdict[\"agent_sql\"] = agent_sql verdict[\"agent_sql_source\"] = source if not agent_sql: verdict[\"reason\"] = \"agent_sql_extraction_failed\" return verdict try: db_path = resolve_database( record, db_root=db_root, db_template=db_template, ) except ValueError as exc: verdict[\"reason\"] = str(exc) return verdict if not db_path.is_file(): verdict[\"reason\"] = f\"sqlite_db_not_found: {db_path}\" return verdict gold_rows, gold_columns, gold_meta = execute_readonly_rows( db_path, gold_sql, max_rows=max_rows, timeout_s=timeout, ) if not gold_meta.get(\"ok\"): verdict[\"gold_exec_error\"] = gold_meta.get(\"error\") verdict[\"reason\"] = f\"gold_sql_exec_error: {gold_meta.get('error')}\" return verdict agent_rows, agent_columns, agent_meta = execute_readonly_rows( db_path, agent_sql, max_rows=max_rows, timeout_s=timeout, ) if not agent_meta.get(\"ok\"): verdict[\"agent_exec_error\"] = agent_meta.get(\"error\") verdict[\"reason\"] = f\"agent_sql_exec_error: {agent_meta.get('error')}\" return verdict gold_table = rows_to_columnar(gold_rows, gold_columns) agent_table = rows_to_columnar(agent_rows, agent_columns) order_sensitive = is_sql_ordered(gold_sql) score, precision, recall = compare_execution_results( agent_table, gold_table, order_matters=order_sensitive, strict=strict, ignore_extra_columns_for_recall=True, float_digits=S2SQL_NUMBER_OF_DIGITS_CONSIDERED, ) matched = score >= 1.0 verdict.update( { \"match\": matched, \"score\": 1.0 if matched else 0.0, \"reason\": ( \"exec_match\" if matched else ( f\"exec_mismatch: raw_score={score:.3f} \" f\"col_precision={precision:.3f} col_recall={recall:.3f}\" ) ), \"column_precision\": float(precision), \"column_recall\": float(recall), \"order_sensitive\": order_sensitive, \"agent_table\": {\"columns\": agent_columns, \"rows\": agent_rows}, \"gold_table\": {\"columns\": gold_columns, \"rows\": gold_rows}, } ) return verdict","summary_state":"pending","summary":null,"crux":null},{"id":"evaluation/s2sql_eval.py#parse_args","name":"parse_args","kind":"function","path":"evaluation/s2sql_eval.py","span":"L175-L188","signature":"def parse_args() -> argparse.Namespace","exported":true,"origin":"ast","body_hash":"0aa5c8e8a9073d23efdc751eeaf3d19bf29a5830449767d971ff15c205907d2d","body_text":"def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument(\"--input\", required=True, type=Path) parser.add_argument(\"--output\", required=True, type=Path) parser.add_argument(\"--db-root\", type=Path) parser.add_argument( \"--db-template\", default=\"{db}/{db}_template.sqlite\", help=\"Path below --db-root; {db} is replaced by the record's db field.\", ) parser.add_argument(\"--max-rows\", type=int, default=10_000) parser.add_argument(\"--timeout\", type=float, default=120.0) parser.add_argument(\"--strict\", action=\"store_true\") return parser.parse_args()","summary_state":"pending","summary":null,"crux":null},{"id":"evaluation/s2sql_eval.py#main","name":"main","kind":"function","path":"evaluation/s2sql_eval.py","span":"L191-L215","signature":"def main() -> None","exported":true,"origin":"ast","body_hash":"94fed6b3d02a4b75cd1ea529ba3707da89a37fc0accc1d29ad1e4b7102098749","body_text":"def main() -> None: args = parse_args() db_root = args.db_root.expanduser().resolve() if args.db_root else None output = args.output.expanduser().resolve() output.parent.mkdir(parents=True, exist_ok=True) total = 0 matched = 0 with output.open(\"w\", encoding=\"utf-8\") as handle: for record in iter_jsonl(args.input.expanduser().resolve()): verdict = evaluate_record( record, db_root=db_root, db_template=args.db_template, max_rows=max(1, args.max_rows), timeout=args.timeout, strict=args.strict, ) total += 1 matched += int(verdict[\"match\"]) handle.write(json.dumps(verdict, ensure_ascii=False) + \"\\n\") accuracy = matched / total if total else 0.0 print(f\"Evaluated {total} S2SQL records | matched={matched} | accuracy={accuracy:.3f}\") print(f\"Wrote {output}\")","summary_state":"pending","summary":null,"crux":null}],"rawEdges":[{"source":"evaluation/s2sql_eval.py","relation":"imports","specifier":"argparse","file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py","relation":"imports","specifier":"json","file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py","relation":"imports","specifier":"pathlib","file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py","relation":"imports","specifier":"typing","file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py","relation":"imports","specifier":"execution_match","file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py","relation":"imports","specifier":"sql_extract","file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py","relation":"imports","specifier":"sqlite_executor","file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py","relation":"contains","targetId":"evaluation/s2sql_eval.py#iter_jsonl","file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#iter_jsonl","relation":"calls","name":"open","viaMember":true,"file":"evaluation/s2sql_eval.py","recvType":"Path"},{"source":"evaluation/s2sql_eval.py#iter_jsonl","relation":"calls","name":"enumerate","viaMember":false,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#iter_jsonl","relation":"calls","name":"strip","viaMember":true,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#iter_jsonl","relation":"calls","name":"loads","viaMember":true,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#iter_jsonl","relation":"calls","name":"isinstance","viaMember":false,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#iter_jsonl","relation":"calls","name":"ValueError","viaMember":false,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py","relation":"contains","targetId":"evaluation/s2sql_eval.py#rows_to_columnar","file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#rows_to_columnar","relation":"calls","name":"enumerate","viaMember":false,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#rows_to_columnar","relation":"calls","name":"append","viaMember":true,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#rows_to_columnar","relation":"calls","name":"len","viaMember":false,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py","relation":"contains","targetId":"evaluation/s2sql_eval.py#resolve_database","file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#resolve_database","relation":"calls","name":"strip","viaMember":true,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#resolve_database","relation":"calls","name":"str","viaMember":false,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#resolve_database","relation":"calls","name":"get","viaMember":true,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#resolve_database","relation":"calls","name":"resolve","viaMember":true,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#resolve_database","relation":"calls","name":"expanduser","viaMember":true,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#resolve_database","relation":"calls","name":"Path","viaMember":false,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#resolve_database","relation":"calls","name":"strip","viaMember":true,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#resolve_database","relation":"calls","name":"str","viaMember":false,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#resolve_database","relation":"calls","name":"get","viaMember":true,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#resolve_database","relation":"calls","name":"ValueError","viaMember":false,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#resolve_database","relation":"calls","name":"format","viaMember":true,"file":"evaluation/s2sql_eval.py","recvType":"str"},{"source":"evaluation/s2sql_eval.py","relation":"contains","targetId":"evaluation/s2sql_eval.py#evaluate_record","file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#evaluate_record","relation":"calls","name":"strip","viaMember":true,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#evaluate_record","relation":"calls","name":"str","viaMember":false,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#evaluate_record","relation":"calls","name":"get","viaMember":true,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#evaluate_record","relation":"calls","name":"get","viaMember":true,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#evaluate_record","relation":"calls","name":"strip","viaMember":true,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#evaluate_record","relation":"calls","name":"str","viaMember":false,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#evaluate_record","relation":"calls","name":"get","viaMember":true,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#evaluate_record","relation":"calls","name":"get","viaMember":true,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#evaluate_record","relation":"calls","name":"get","viaMember":true,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#evaluate_record","relation":"calls","name":"isinstance","viaMember":false,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#evaluate_record","relation":"calls","name":"str","viaMember":false,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#evaluate_record","relation":"calls","name":"get","viaMember":true,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#evaluate_record","relation":"calls","name":"get","viaMember":true,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#evaluate_record","relation":"calls","name":"get","viaMember":true,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#evaluate_record","relation":"calls","name":"extract_agent_sql","viaMember":false,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#evaluate_record","relation":"calls","name":"resolve_database","viaMember":false,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#evaluate_record","relation":"calls","name":"str","viaMember":false,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#evaluate_record","relation":"calls","name":"is_file","viaMember":true,"file":"evaluation/s2sql_eval.py","recvType":"resolve_database"},{"source":"evaluation/s2sql_eval.py#evaluate_record","relation":"calls","name":"execute_readonly_rows","viaMember":false,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#evaluate_record","relation":"calls","name":"get","viaMember":true,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#evaluate_record","relation":"calls","name":"get","viaMember":true,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#evaluate_record","relation":"calls","name":"get","viaMember":true,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#evaluate_record","relation":"calls","name":"execute_readonly_rows","viaMember":false,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#evaluate_record","relation":"calls","name":"get","viaMember":true,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#evaluate_record","relation":"calls","name":"get","viaMember":true,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#evaluate_record","relation":"calls","name":"get","viaMember":true,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#evaluate_record","relation":"calls","name":"rows_to_columnar","viaMember":false,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#evaluate_record","relation":"calls","name":"rows_to_columnar","viaMember":false,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#evaluate_record","relation":"calls","name":"is_sql_ordered","viaMember":false,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#evaluate_record","relation":"calls","name":"compare_execution_results","viaMember":false,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#evaluate_record","relation":"calls","name":"update","viaMember":true,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#evaluate_record","relation":"calls","name":"float","viaMember":false,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#evaluate_record","relation":"calls","name":"float","viaMember":false,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py","relation":"contains","targetId":"evaluation/s2sql_eval.py#parse_args","file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#parse_args","relation":"calls","name":"ArgumentParser","viaMember":true,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#parse_args","relation":"calls","name":"add_argument","viaMember":true,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#parse_args","relation":"calls","name":"add_argument","viaMember":true,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#parse_args","relation":"calls","name":"add_argument","viaMember":true,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#parse_args","relation":"calls","name":"add_argument","viaMember":true,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#parse_args","relation":"calls","name":"add_argument","viaMember":true,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#parse_args","relation":"calls","name":"add_argument","viaMember":true,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#parse_args","relation":"calls","name":"add_argument","viaMember":true,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#parse_args","relation":"calls","name":"parse_args","viaMember":true,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py","relation":"contains","targetId":"evaluation/s2sql_eval.py#main","file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#main","relation":"calls","name":"parse_args","viaMember":false,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#main","relation":"calls","name":"resolve","viaMember":true,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#main","relation":"calls","name":"expanduser","viaMember":true,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#main","relation":"calls","name":"resolve","viaMember":true,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#main","relation":"calls","name":"expanduser","viaMember":true,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#main","relation":"calls","name":"mkdir","viaMember":true,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#main","relation":"calls","name":"open","viaMember":true,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#main","relation":"calls","name":"iter_jsonl","viaMember":false,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#main","relation":"calls","name":"resolve","viaMember":true,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#main","relation":"calls","name":"expanduser","viaMember":true,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#main","relation":"calls","name":"evaluate_record","viaMember":false,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#main","relation":"calls","name":"max","viaMember":false,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#main","relation":"calls","name":"int","viaMember":false,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#main","relation":"calls","name":"write","viaMember":true,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#main","relation":"calls","name":"dumps","viaMember":true,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#main","relation":"calls","name":"print","viaMember":false,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py#main","relation":"calls","name":"print","viaMember":false,"file":"evaluation/s2sql_eval.py"},{"source":"evaluation/s2sql_eval.py","relation":"calls","name":"main","viaMember":false,"file":"evaluation/s2sql_eval.py"}]},"evaluation/sql_extract.py":{"size":6828,"mtimeMs":1789189526392.8706,"hash":"37cb6727de23fb7346cb26523bf3260d1489106c2e718c85122fdc2b9a09358e","nodes":[{"id":"evaluation/sql_extract.py","name":"sql_extract.py","kind":"file","path":"evaluation/sql_extract.py","span":"L1-L191","signature":null,"exported":true,"origin":"ast","body_hash":"37cb6727de23fb7346cb26523bf3260d1489106c2e718c85122fdc2b9a09358e","chars":6822,"summary_state":"pending","summary":null,"crux":null,"body_text":"\"\"\"Extract the canonical SQL the agent produced for a ``search_to_sql`` trial. Tries, in order: 1. The last fenced ``` ```sql ... ``` `` block in ``final_text``. 2. The last fenced generic ``` ``` ... ``` `` block in ``final_text``. 3. The ``sql`` argument of the last *successful* ``run_sql`` tool call in the native trajectory (skips calls whose paired tool result is missing or marked as error). Two common message formats are supported: - ``\"openai\"``: assistant ``tool_calls[*].function.{name,arguments}`` paired with ``role=\"tool\"`` results keyed by ``tool_call_id``. - ``\"anthropic\"``: assistant content blocks of ``type=\"tool_use\"`` paired with ``role=\"user\"`` content blocks of ``type=\"tool_result\"`` keyed by ``tool_use_id`` (carrying ``is_error``). \"\"\" from __future__ import annotations import json import re from typing import Any, Iterable _RUN_SQL_TOOL_NAME = \"run_sql\" # Source label vocabulary returned alongside the extracted SQL. SOURCE_FENCED_SQL = \"final_text_fenced_sql\" SOURCE_FENCED_GENERIC = \"final_text_fenced_generic\" SOURCE_LAST_TOOL_CALL = \"last_tool_call\" SOURCE_NONE = \"none\""},{"id":"evaluation/sql_extract.py#_last_fenced","name":"_last_fenced","kind":"function","path":"evaluation/sql_extract.py","span":"L35-L40","signature":"def _last_fenced(text: str, label_pattern: str) -> str | None","exported":false,"origin":"ast","body_hash":"aa7b0ab5ed959ab620170cb010ffde79c629d854069a94e5c7e6b99891821c81","body_text":"def _last_fenced(text: str, label_pattern: str) -> str | None: pattern = re.compile(rf\"```(?:{label_pattern})\\s*\\n(.*?)```\", re.DOTALL) matches = list(pattern.finditer(text or \"\")) if not matches: return None return (matches[-1].group(1).strip() or None)","summary_state":"pending","summary":null,"crux":null},{"id":"evaluation/sql_extract.py#_openai_tool_results_by_id","name":"_openai_tool_results_by_id","kind":"function","path":"evaluation/sql_extract.py","span":"L43-L52","signature":"def _openai_tool_results_by_id(messages: list[dict[str, Any]]) -> dict[str, dict[str, Any]]","exported":false,"origin":"ast","body_hash":"f696f1938a41e0d309ee8214ddf6c2a96a78b17902d1546d8c03faa66ed08b41","body_text":"def _openai_tool_results_by_id(messages: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: \"\"\"Build ``{tool_call_id: tool_message}`` from OpenAI-shaped messages.\"\"\" by_id: dict[str, dict[str, Any]] = {} for m in messages: if not isinstance(m, dict) or m.get(\"role\") != \"tool\": continue cid = str(m.get(\"tool_call_id\") or \"\") if cid: by_id[cid] = m return by_id","summary_state":"pending","summary":null,"crux":null},{"id":"evaluation/sql_extract.py#_openai_tool_result_is_error","name":"_openai_tool_result_is_error","kind":"function","path":"evaluation/sql_extract.py","span":"L55-L73","signature":"def _openai_tool_result_is_error(tool_msg: dict[str, Any] | None) -> bool","exported":false,"origin":"ast","body_hash":"ceaa1c1e4e71e781fb749e6106e9aebaf8014e8446612cab56cf91fba9862ae3","body_text":"def _openai_tool_result_is_error(tool_msg: dict[str, Any] | None) -> bool: \"\"\"OpenAI doesn't mark errors structurally — we treat ``Error:`` / ``Rejected:`` content prefixes as failures, matching how the agent loop encodes :class:`ToolError`.\"\"\" if not isinstance(tool_msg, dict): return True # missing pair → treat as failed content = tool_msg.get(\"content\") if isinstance(content, list): # Defensive — OpenAI tool messages are usually plain strings, but # be robust if a future loop version emits structured content. text = \"\".join( str(c.get(\"text\") or c.get(\"content\") or \"\") for c in content if isinstance(c, dict) ) else: text = str(content or \"\") head = text.lstrip() return head.startswith(\"Error:\") or head.startswith(\"Rejected:\")","summary_state":"pending","summary":null,"crux":null},{"id":"evaluation/sql_extract.py#_last_run_sql_openai","name":"_last_run_sql_openai","kind":"function","path":"evaluation/sql_extract.py","span":"L76-L108","signature":"def _last_run_sql_openai(messages: list[dict[str, Any]]) -> str | None","exported":false,"origin":"ast","body_hash":"b8dc65c3bd1d4125ce79e71ea421f38f4464679f3b7cb2baf62b666698fe2fe5","body_text":"def _last_run_sql_openai(messages: list[dict[str, Any]]) -> str | None: by_id = _openai_tool_results_by_id(messages) for m in reversed(messages): if not isinstance(m, dict) or m.get(\"role\") != \"assistant\": continue calls = m.get(\"tool_calls\") if not isinstance(calls, list): continue for tc in reversed(calls): if not isinstance(tc, dict): continue fn = tc.get(\"function\") if isinstance(tc.get(\"function\"), dict) else None name = (fn or {}).get(\"name\") or tc.get(\"name\") or \"\" if name != _RUN_SQL_TOOL_NAME: continue raw_args = (fn or {}).get(\"arguments\") if isinstance(raw_args, str): try: args = json.loads(raw_args) except json.JSONDecodeError: args = {} elif isinstance(raw_args, dict): args = raw_args else: args = {} sql = str((args or {}).get(\"sql\") or \"\").strip() if not sql: continue cid = str(tc.get(\"id\") or \"\") if cid and _openai_tool_result_is_error(by_id.get(cid)): continue return sql return None","summary_state":"pending","summary":null,"crux":null},{"id":"evaluation/sql_extract.py#_anthropic_tool_results_by_id","name":"_anthropic_tool_results_by_id","kind":"function","path":"evaluation/sql_extract.py","span":"L111-L124","signature":"def _anthropic_tool_results_by_id(messages: list[dict[str, Any]]) -> dict[str, dict[str, Any]]","exported":false,"origin":"ast","body_hash":"159042d5c428d9e5a1ca55b88da475f80738423bf42d5b1855217c2df19afcc0","body_text":"def _anthropic_tool_results_by_id(messages: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: by_id: dict[str, dict[str, Any]] = {} for m in messages: if not isinstance(m, dict) or m.get(\"role\") != \"user\": continue content = m.get(\"content\") if not isinstance(content, list): continue for block in content: if isinstance(block, dict) and block.get(\"type\") == \"tool_result\": tid = str(block.get(\"tool_use_id\") or \"\") if tid: by_id[tid] = block return by_id","summary_state":"pending","summary":null,"crux":null},{"id":"evaluation/sql_extract.py#_last_run_sql_anthropic","name":"_last_run_sql_anthropic","kind":"function","path":"evaluation/sql_extract.py","span":"L127-L151","signature":"def _last_run_sql_anthropic(messages: list[dict[str, Any]]) -> str | None","exported":false,"origin":"ast","body_hash":"0bae6134b754a81bfe948c64173babf34ec7328b4efba1d10809f9fec53c7821","body_text":"def _last_run_sql_anthropic(messages: list[dict[str, Any]]) -> str | None: by_id = _anthropic_tool_results_by_id(messages) for m in reversed(messages): if not isinstance(m, dict) or m.get(\"role\") != \"assistant\": continue content = m.get(\"content\") if not isinstance(content, list): continue for block in reversed(content): if not isinstance(block, dict) or block.get(\"type\") != \"tool_use\": continue if (block.get(\"name\") or \"\") != _RUN_SQL_TOOL_NAME: continue inp = block.get(\"input\") if not isinstance(inp, dict): continue sql = str(inp.get(\"sql\") or \"\").strip() if not sql: continue tid = str(block.get(\"id\") or \"\") paired = by_id.get(tid) if tid else None if paired is not None and bool(paired.get(\"is_error\")): continue return sql return None","summary_state":"pending","summary":null,"crux":null},{"id":"evaluation/sql_extract.py#_walk_messages_for_run_sql","name":"_walk_messages_for_run_sql","kind":"function","path":"evaluation/sql_extract.py","span":"L154-L164","signature":"def _walk_messages_for_run_sql( messages: Iterable[dict[str, Any]] | None, *, message_format: str, ) -> str | None","exported":false,"origin":"ast","body_hash":"985c68a2316b2c0881dcebf9e0913777dbec6eca4dff8258bb7866d5ed9dd0ba","body_text":"def _walk_messages_for_run_sql( messages: Iterable[dict[str, Any]] | None, *, message_format: str, ) -> str | None: if not messages: return None msgs = list(messages) if message_format == \"anthropic\": return _last_run_sql_anthropic(msgs) return _last_run_sql_openai(msgs)","summary_state":"pending","summary":null,"crux":null},{"id":"evaluation/sql_extract.py#extract_agent_sql","name":"extract_agent_sql","kind":"function","path":"evaluation/sql_extract.py","span":"L167-L190","signature":"def extract_agent_sql( *, final_text: str, messages: list[dict[str, Any]] | None, message_format: str = \"openai\", ) -> tuple[str, str]","exported":true,"origin":"ast","body_hash":"8f2f3790e3ea94b537866135c10efa1e44bc1a2d8aef376ccfba07a0ebdf76f4","body_text":"def extract_agent_sql( *, final_text: str, messages: list[dict[str, Any]] | None, message_format: str = \"openai\", ) -> tuple[str, str]: \"\"\"Return ``(sql, source)`` where source explains where the SQL came from. ``source`` is one of ``final_text_fenced_sql`` / ``final_text_fenced_generic`` / ``last_tool_call`` / ``none``. Empty SQL means nothing was extractable; the caller should report ``agent_sql_extraction_failed``. \"\"\" text = final_text or \"\" fenced_sql = _last_fenced(text, \"sql|SQL\") if fenced_sql: return fenced_sql, SOURCE_FENCED_SQL fenced_generic = _last_fenced(text, \"\") if fenced_generic: return fenced_generic, SOURCE_FENCED_GENERIC last = _walk_messages_for_run_sql(messages, message_format=message_format) if last: return last, SOURCE_LAST_TOOL_CALL return \"\", SOURCE_NONE","summary_state":"pending","summary":null,"crux":null}],"rawEdges":[{"source":"evaluation/sql_extract.py","relation":"imports","specifier":"json","file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py","relation":"imports","specifier":"re","file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py","relation":"imports","specifier":"typing","file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py","relation":"contains","targetId":"evaluation/sql_extract.py#_last_fenced","file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_last_fenced","relation":"calls","name":"compile","viaMember":true,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_last_fenced","relation":"calls","name":"list","viaMember":false,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_last_fenced","relation":"calls","name":"finditer","viaMember":true,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_last_fenced","relation":"calls","name":"strip","viaMember":true,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_last_fenced","relation":"calls","name":"group","viaMember":true,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py","relation":"contains","targetId":"evaluation/sql_extract.py#_openai_tool_results_by_id","file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_openai_tool_results_by_id","relation":"calls","name":"isinstance","viaMember":false,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_openai_tool_results_by_id","relation":"calls","name":"get","viaMember":true,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_openai_tool_results_by_id","relation":"calls","name":"str","viaMember":false,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_openai_tool_results_by_id","relation":"calls","name":"get","viaMember":true,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py","relation":"contains","targetId":"evaluation/sql_extract.py#_openai_tool_result_is_error","file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_openai_tool_result_is_error","relation":"calls","name":"isinstance","viaMember":false,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_openai_tool_result_is_error","relation":"calls","name":"get","viaMember":true,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_openai_tool_result_is_error","relation":"calls","name":"isinstance","viaMember":false,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_openai_tool_result_is_error","relation":"calls","name":"join","viaMember":true,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_openai_tool_result_is_error","relation":"calls","name":"str","viaMember":false,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_openai_tool_result_is_error","relation":"calls","name":"get","viaMember":true,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_openai_tool_result_is_error","relation":"calls","name":"get","viaMember":true,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_openai_tool_result_is_error","relation":"calls","name":"isinstance","viaMember":false,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_openai_tool_result_is_error","relation":"calls","name":"str","viaMember":false,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_openai_tool_result_is_error","relation":"calls","name":"lstrip","viaMember":true,"file":"evaluation/sql_extract.py","recvType":"str"},{"source":"evaluation/sql_extract.py#_openai_tool_result_is_error","relation":"calls","name":"startswith","viaMember":true,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_openai_tool_result_is_error","relation":"calls","name":"startswith","viaMember":true,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py","relation":"contains","targetId":"evaluation/sql_extract.py#_last_run_sql_openai","file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_last_run_sql_openai","relation":"calls","name":"_openai_tool_results_by_id","viaMember":false,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_last_run_sql_openai","relation":"calls","name":"reversed","viaMember":false,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_last_run_sql_openai","relation":"calls","name":"isinstance","viaMember":false,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_last_run_sql_openai","relation":"calls","name":"get","viaMember":true,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_last_run_sql_openai","relation":"calls","name":"get","viaMember":true,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_last_run_sql_openai","relation":"calls","name":"isinstance","viaMember":false,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_last_run_sql_openai","relation":"calls","name":"reversed","viaMember":false,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_last_run_sql_openai","relation":"calls","name":"isinstance","viaMember":false,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_last_run_sql_openai","relation":"calls","name":"get","viaMember":true,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_last_run_sql_openai","relation":"calls","name":"isinstance","viaMember":false,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_last_run_sql_openai","relation":"calls","name":"get","viaMember":true,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_last_run_sql_openai","relation":"calls","name":"get","viaMember":true,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_last_run_sql_openai","relation":"calls","name":"get","viaMember":true,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_last_run_sql_openai","relation":"calls","name":"get","viaMember":true,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_last_run_sql_openai","relation":"calls","name":"isinstance","viaMember":false,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_last_run_sql_openai","relation":"calls","name":"loads","viaMember":true,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_last_run_sql_openai","relation":"calls","name":"isinstance","viaMember":false,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_last_run_sql_openai","relation":"calls","name":"strip","viaMember":true,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_last_run_sql_openai","relation":"calls","name":"str","viaMember":false,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_last_run_sql_openai","relation":"calls","name":"get","viaMember":true,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_last_run_sql_openai","relation":"calls","name":"str","viaMember":false,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_last_run_sql_openai","relation":"calls","name":"get","viaMember":true,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_last_run_sql_openai","relation":"calls","name":"_openai_tool_result_is_error","viaMember":false,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_last_run_sql_openai","relation":"calls","name":"get","viaMember":true,"file":"evaluation/sql_extract.py","recvType":"_openai_tool_results_by_id"},{"source":"evaluation/sql_extract.py","relation":"contains","targetId":"evaluation/sql_extract.py#_anthropic_tool_results_by_id","file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_anthropic_tool_results_by_id","relation":"calls","name":"isinstance","viaMember":false,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_anthropic_tool_results_by_id","relation":"calls","name":"get","viaMember":true,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_anthropic_tool_results_by_id","relation":"calls","name":"get","viaMember":true,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_anthropic_tool_results_by_id","relation":"calls","name":"isinstance","viaMember":false,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_anthropic_tool_results_by_id","relation":"calls","name":"isinstance","viaMember":false,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_anthropic_tool_results_by_id","relation":"calls","name":"get","viaMember":true,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_anthropic_tool_results_by_id","relation":"calls","name":"str","viaMember":false,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_anthropic_tool_results_by_id","relation":"calls","name":"get","viaMember":true,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py","relation":"contains","targetId":"evaluation/sql_extract.py#_last_run_sql_anthropic","file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_last_run_sql_anthropic","relation":"calls","name":"_anthropic_tool_results_by_id","viaMember":false,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_last_run_sql_anthropic","relation":"calls","name":"reversed","viaMember":false,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_last_run_sql_anthropic","relation":"calls","name":"isinstance","viaMember":false,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_last_run_sql_anthropic","relation":"calls","name":"get","viaMember":true,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_last_run_sql_anthropic","relation":"calls","name":"get","viaMember":true,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_last_run_sql_anthropic","relation":"calls","name":"isinstance","viaMember":false,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_last_run_sql_anthropic","relation":"calls","name":"reversed","viaMember":false,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_last_run_sql_anthropic","relation":"calls","name":"isinstance","viaMember":false,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_last_run_sql_anthropic","relation":"calls","name":"get","viaMember":true,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_last_run_sql_anthropic","relation":"calls","name":"get","viaMember":true,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_last_run_sql_anthropic","relation":"calls","name":"get","viaMember":true,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_last_run_sql_anthropic","relation":"calls","name":"isinstance","viaMember":false,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_last_run_sql_anthropic","relation":"calls","name":"strip","viaMember":true,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_last_run_sql_anthropic","relation":"calls","name":"str","viaMember":false,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_last_run_sql_anthropic","relation":"calls","name":"get","viaMember":true,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_last_run_sql_anthropic","relation":"calls","name":"str","viaMember":false,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_last_run_sql_anthropic","relation":"calls","name":"get","viaMember":true,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_last_run_sql_anthropic","relation":"calls","name":"get","viaMember":true,"file":"evaluation/sql_extract.py","recvType":"_anthropic_tool_results_by_id"},{"source":"evaluation/sql_extract.py#_last_run_sql_anthropic","relation":"calls","name":"bool","viaMember":false,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_last_run_sql_anthropic","relation":"calls","name":"get","viaMember":true,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py","relation":"contains","targetId":"evaluation/sql_extract.py#_walk_messages_for_run_sql","file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_walk_messages_for_run_sql","relation":"calls","name":"list","viaMember":false,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_walk_messages_for_run_sql","relation":"calls","name":"_last_run_sql_anthropic","viaMember":false,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#_walk_messages_for_run_sql","relation":"calls","name":"_last_run_sql_openai","viaMember":false,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py","relation":"contains","targetId":"evaluation/sql_extract.py#extract_agent_sql","file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#extract_agent_sql","relation":"calls","name":"_last_fenced","viaMember":false,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#extract_agent_sql","relation":"calls","name":"_last_fenced","viaMember":false,"file":"evaluation/sql_extract.py"},{"source":"evaluation/sql_extract.py#extract_agent_sql","relation":"calls","name":"_walk_messages_for_run_sql","viaMember":false,"file":"evaluation/sql_extract.py"}]},"evaluation/sqlite_executor.py":{"size":6729,"mtimeMs":1789189526393.8708,"hash":"0f6ab2b179bdc4ea89fc7042940216c1b3147013aa39bf77a27743852413e51e","nodes":[{"id":"evaluation/sqlite_executor.py","name":"sqlite_executor.py","kind":"file","path":"evaluation/sqlite_executor.py","span":"L1-L217","signature":null,"exported":true,"origin":"ast","body_hash":"0f6ab2b179bdc4ea89fc7042940216c1b3147013aa39bf77a27743852413e51e","chars":6725,"summary_state":"pending","summary":null,"crux":null,"body_text":"\"\"\"Read-only SQLite query execution with a write-statement guard. Two layers: - :func:`execute_readonly_rows` returns ``(rows, columns, meta)`` — used by judges that need the table values directly. - :func:`execute_sqlite_query` wraps that in a pretty-printed text table for the agent's tool observation. Connections are opened in URI ``mode=ro`` so even if the write-token guard misses an exotic write path, SQLite refuses to mutate the file. \"\"\" from __future__ import annotations import re import sqlite3 import time from pathlib import Path from typing import Any _READ_ONLY_SQL = re.compile( r\"^\\s*(WITH|SELECT|PRAGMA|EXPLAIN)\\b\", re.IGNORECASE | re.DOTALL, ) _LEADING_LINE_COMMENT = re.compile(r\"^\\s*--[^\\n]*\\n?\") _LEADING_BLOCK_COMMENT = re.compile(r\"^\\s*/\\*.*?\\*/\", re.DOTALL)"},{"id":"evaluation/sqlite_executor.py#_strip_leading_comments","name":"_strip_leading_comments","kind":"function","path":"evaluation/sqlite_executor.py","span":"L31-L44","signature":"def _strip_leading_comments(sql: str) -> str","exported":false,"origin":"ast","body_hash":"f4c04d7cfeafb166fc7f6f2b2c731ab8b7e604014bb85544d498a159f7b43a07","body_text":"def _strip_leading_comments(sql: str) -> str: \"\"\"Iteratively strip whitespace + leading SQL comments (``--`` / ``/* */``).\"\"\" s = sql or \"\" while True: s2 = s.lstrip() m = _LEADING_LINE_COMMENT.match(s2) if m: s = s2[m.end():] continue m = _LEADING_BLOCK_COMMENT.match(s2) if m: s = s2[m.end():] continue return s2","summary_state":"pending","summary":null,"crux":null},{"id":"evaluation/sqlite_executor.py#_reject_write","name":"_reject_write","kind":"function","path":"evaluation/sqlite_executor.py","span":"L47-L75","signature":"def _reject_write(sql: str) -> str | None","exported":false,"origin":"ast","body_hash":"83cd2890ccf9a861c4adce0e7350b580ff75e02597b3687b3e9a03928ed83008","body_text":"def _reject_write(sql: str) -> str | None: \"\"\"Two-pass guard: token blacklist plus opener-keyword whitelist. Returns an error message when the SQL should be rejected, or ``None`` when it's safe to execute. The token list catches the common write paths (``INSERT`` etc.) and the regex enforces the opening keyword after stripping any leading comments. ``mode=ro`` on the connection is the final guarantee. \"\"\" s = sql.strip().upper() banned = ( \"INSERT \", \"UPDATE \", \"DELETE \", \"DROP \", \"ALTER \", \"CREATE \", \"ATTACH \", \"DETACH \", \"VACUUM\", \"REPLACE \", ) for b in banned: if b in s: return f\"Rejected: disallowed token in query ({b.strip()}).\" head = _strip_leading_comments(sql or \"\") if not _READ_ONLY_SQL.match(head): return \"Rejected: only SELECT / WITH / PRAGMA / EXPLAIN queries are allowed.\" return None","summary_state":"pending","summary":null,"crux":null},{"id":"evaluation/sqlite_executor.py#_install_query_timeout","name":"_install_query_timeout","kind":"function","path":"evaluation/sqlite_executor.py","span":"L78-L93","signature":"def _install_query_timeout( conn: sqlite3.Connection, timeout_s: float | None, ) -> float | None","exported":false,"origin":"ast","body_hash":"928dedaf2a608f360ea0ba8d5272ed281a457a51e958223fa13806b3d5bc22b8","body_text":"def _install_query_timeout( conn: sqlite3.Connection, timeout_s: float | None, ) -> float | None: \"\"\"Abort long-running queries via SQLite's progress handler.\"\"\" if timeout_s is None or timeout_s <= 0: conn.set_progress_handler(None, 0) return None deadline = time.monotonic() + timeout_s def _handler() -> int: return 1 if time.monotonic() >= deadline else 0 conn.set_progress_handler(_handler, 10_000) return deadline","summary_state":"pending","summary":null,"crux":null},{"id":"evaluation/sqlite_executor.py#_install_query_timeout._handler","name":"_handler","kind":"function","path":"evaluation/sqlite_executor.py","span":"L89-L90","signature":"def _handler() -> int","exported":false,"origin":"ast","body_hash":"be31aa4f765d50e1b3c481ce3c16bf579b97d0ca974c45362f752543bb98b60b","body_text":"def _handler() -> int: return 1 if time.monotonic() >= deadline else 0","summary_state":"pending","summary":null,"crux":null},{"id":"evaluation/sqlite_executor.py#_clear_query_timeout","name":"_clear_query_timeout","kind":"function","path":"evaluation/sqlite_executor.py","span":"L96-L97","signature":"def _clear_query_timeout(conn: sqlite3.Connection) -> None","exported":false,"origin":"ast","body_hash":"702918acf311435e82f141cbdc64fe70b41ee4e73a4811414a30c72ed5481bed","body_text":"def _clear_query_timeout(conn: sqlite3.Connection) -> None: conn.set_progress_handler(None, 0)","summary_state":"pending","summary":null,"crux":null},{"id":"evaluation/sqlite_executor.py#execute_readonly_rows","name":"execute_readonly_rows","kind":"function","path":"evaluation/sqlite_executor.py","span":"L100-L156","signature":"def execute_readonly_rows( db_path: Path, sql: str, *, max_rows: int = 500, timeout_s: float | None = None, ) -> tuple[list[list[Any]], list[str], dict[str, Any]]","exported":true,"origin":"ast","body_hash":"328e7e1a50b41f11afe549a6fa5e6487b2d8391c9e2b43eb3e5feae6e7966397","body_text":"def execute_readonly_rows( db_path: Path, sql: str, *, max_rows: int = 500, timeout_s: float | None = None, ) -> tuple[list[list[Any]], list[str], dict[str, Any]]: \"\"\"Run a read-only query; return ``(rows-as-lists, column-names, meta)``. ``meta`` carries ``ok`` (bool), ``error`` (when not ok), and ``truncated`` (bool — set when the result set exceeded *max_rows*). \"\"\" err = _reject_write(sql) if err: return [], [], {\"ok\": False, \"error\": err} db_path = db_path.expanduser().resolve() if not db_path.is_file(): msg = f\"Database file not found: {db_path}\" return [], [], {\"ok\": False, \"error\": msg} try: uri_path = db_path.resolve().as_posix() conn = sqlite3.connect(f\"file:{uri_path}?mode=ro\", uri=True) conn.row_factory = sqlite3.Row except sqlite3.Error as exc: return [], [], {\"ok\": False, \"error\": f\"SQLite connect failed: {exc}\"} deadline = _install_query_timeout(conn, timeout_s) try: cur = conn.execute(sql) # Fetch one extra row to detect truncation cleanly. fetched = cur.fetchmany(max_rows + 1) except sqlite3.Error as exc: if deadline is not None and \"interrupted\" in str(exc).lower(): limit = int(timeout_s) if timeout_s is not None else 0 return [], [], { \"ok\": False, \"error\": f\"Query timed out after {limit}s.\", } return [], [], {\"ok\": False, \"error\": f\"SQL error: {exc}\"} finally: try: _clear_query_timeout(conn) conn.close() except sqlite3.Error: pass truncated = len(fetched) > max_rows fetched = fetched[:max_rows] if not fetched: return [], [], {\"ok\": True, \"truncated\": truncated} cols = list(fetched[0].keys()) rows = [[r[c] for c in cols] for r in fetched] return rows, cols, {\"ok\": True, \"truncated\": truncated}","summary_state":"pending","summary":null,"crux":null},{"id":"evaluation/sqlite_executor.py#execute_sqlite_query","name":"execute_sqlite_query","kind":"function","path":"evaluation/sqlite_executor.py","span":"L159-L216","signature":"def execute_sqlite_query( db_path: Path, sql: str, *, max_rows: int = 500, max_cell_chars: int = 2000, timeout_s: float | None = None, ) -> tuple[str, dict[str, Any]]","exported":true,"origin":"ast","body_hash":"afe77078eac7a7f878d16328a28b2f45ed377057626edea336687ab31ea96da5","body_text":"def execute_sqlite_query( db_path: Path, sql: str, *, max_rows: int = 500, max_cell_chars: int = 2000, timeout_s: float | None = None, ) -> tuple[str, dict[str, Any]]: \"\"\"Run a read-only query; return ``(formatted_text, metadata)``. Output is a fixed-width text table with a ``Rows: N`` header so the agent can quickly see how big the result set is. Cells over *max_cell_chars* are clipped with ``...``. \"\"\" rows, cols, base_meta = execute_readonly_rows( db_path, sql, max_rows=max_rows, timeout_s=timeout_s ) if not base_meta.get(\"ok\"): return base_meta[\"error\"], {\"ok\": False, \"error\": base_meta[\"error\"]} truncated = bool(base_meta.get(\"truncated\")) meta: dict[str, Any] = { \"ok\": True, \"sql\": sql, \"db_path\": str(db_path.expanduser().resolve()), \"truncated\": truncated, } if not rows: return \"(empty result set)\", {**meta, \"row_count\": 0} table_rows: list[list[str]] = [] for r in rows: cells = [] for v in r: s = \"\" if v is None else str(v) if len(s) > max_cell_chars: s = s[: max_cell_chars - 3] + \"...\" cells.append(s) table_rows.append(cells) colnames = cols widths = [len(c) for c in colnames] for tr in table_rows: for i, cell in enumerate(tr): widths[i] = min(60, max(widths[i], len(cell))) def fmt_row(vals: list[str]) -> str: return \" | \".join(v.ljust(widths[i]) for i, v in enumerate(vals)) header = fmt_row(colnames) sep = \"-+-\".join(\"-\" * w for w in widths) body = \"\\n\".join(fmt_row(tr) for tr in table_rows) summary = f\"Rows: {len(table_rows)}\" if truncated: summary += f\" (truncated to first {max_rows}; more rows existed)\" text = f\"{summary}\\n\\n{header}\\n{sep}\\n{body}\" return text, {**meta, \"row_count\": len(table_rows), \"truncated\": truncated}","summary_state":"pending","summary":null,"crux":null},{"id":"evaluation/sqlite_executor.py#execute_sqlite_query.fmt_row","name":"fmt_row","kind":"function","path":"evaluation/sqlite_executor.py","span":"L206-L207","signature":"def fmt_row(vals: list[str]) -> str","exported":true,"origin":"ast","body_hash":"a6aafad81c6795c5bc88e58e633e9de03ce00f2b92c4c5944bd9bede01667a61","body_text":"def fmt_row(vals: list[str]) -> str: return \" | \".join(v.ljust(widths[i]) for i, v in enumerate(vals))","summary_state":"pending","summary":null,"crux":null}],"rawEdges":[{"source":"evaluation/sqlite_executor.py","relation":"imports","specifier":"re","file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py","relation":"imports","specifier":"sqlite3","file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py","relation":"imports","specifier":"time","file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py","relation":"imports","specifier":"pathlib","file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py","relation":"imports","specifier":"typing","file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py","relation":"calls","name":"compile","viaMember":true,"file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py","relation":"calls","name":"compile","viaMember":true,"file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py","relation":"calls","name":"compile","viaMember":true,"file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py","relation":"contains","targetId":"evaluation/sqlite_executor.py#_strip_leading_comments","file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py#_strip_leading_comments","relation":"calls","name":"lstrip","viaMember":true,"file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py#_strip_leading_comments","relation":"calls","name":"match","viaMember":true,"file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py#_strip_leading_comments","relation":"calls","name":"end","viaMember":true,"file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py#_strip_leading_comments","relation":"calls","name":"match","viaMember":true,"file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py#_strip_leading_comments","relation":"calls","name":"end","viaMember":true,"file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py","relation":"contains","targetId":"evaluation/sqlite_executor.py#_reject_write","file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py#_reject_write","relation":"calls","name":"upper","viaMember":true,"file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py#_reject_write","relation":"calls","name":"strip","viaMember":true,"file":"evaluation/sqlite_executor.py","recvType":"str"},{"source":"evaluation/sqlite_executor.py#_reject_write","relation":"calls","name":"strip","viaMember":true,"file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py#_reject_write","relation":"calls","name":"_strip_leading_comments","viaMember":false,"file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py#_reject_write","relation":"calls","name":"match","viaMember":true,"file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py","relation":"contains","targetId":"evaluation/sqlite_executor.py#_install_query_timeout","file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py#_install_query_timeout","relation":"calls","name":"set_progress_handler","viaMember":true,"file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py#_install_query_timeout","relation":"calls","name":"monotonic","viaMember":true,"file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py#_install_query_timeout","relation":"contains","targetId":"evaluation/sqlite_executor.py#_install_query_timeout._handler","file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py#_install_query_timeout._handler","relation":"calls","name":"monotonic","viaMember":true,"file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py#_install_query_timeout","relation":"calls","name":"set_progress_handler","viaMember":true,"file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py","relation":"contains","targetId":"evaluation/sqlite_executor.py#_clear_query_timeout","file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py#_clear_query_timeout","relation":"calls","name":"set_progress_handler","viaMember":true,"file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py","relation":"contains","targetId":"evaluation/sqlite_executor.py#execute_readonly_rows","file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py#execute_readonly_rows","relation":"calls","name":"_reject_write","viaMember":false,"file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py#execute_readonly_rows","relation":"calls","name":"resolve","viaMember":true,"file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py#execute_readonly_rows","relation":"calls","name":"expanduser","viaMember":true,"file":"evaluation/sqlite_executor.py","recvType":"Path"},{"source":"evaluation/sqlite_executor.py#execute_readonly_rows","relation":"calls","name":"is_file","viaMember":true,"file":"evaluation/sqlite_executor.py","recvType":"Path"},{"source":"evaluation/sqlite_executor.py#execute_readonly_rows","relation":"calls","name":"as_posix","viaMember":true,"file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py#execute_readonly_rows","relation":"calls","name":"resolve","viaMember":true,"file":"evaluation/sqlite_executor.py","recvType":"Path"},{"source":"evaluation/sqlite_executor.py#execute_readonly_rows","relation":"calls","name":"connect","viaMember":true,"file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py#execute_readonly_rows","relation":"calls","name":"_install_query_timeout","viaMember":false,"file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py#execute_readonly_rows","relation":"calls","name":"execute","viaMember":true,"file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py#execute_readonly_rows","relation":"calls","name":"fetchmany","viaMember":true,"file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py#execute_readonly_rows","relation":"calls","name":"lower","viaMember":true,"file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py#execute_readonly_rows","relation":"calls","name":"str","viaMember":false,"file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py#execute_readonly_rows","relation":"calls","name":"int","viaMember":false,"file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py#execute_readonly_rows","relation":"calls","name":"_clear_query_timeout","viaMember":false,"file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py#execute_readonly_rows","relation":"calls","name":"close","viaMember":true,"file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py#execute_readonly_rows","relation":"calls","name":"len","viaMember":false,"file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py#execute_readonly_rows","relation":"calls","name":"list","viaMember":false,"file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py#execute_readonly_rows","relation":"calls","name":"keys","viaMember":true,"file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py","relation":"contains","targetId":"evaluation/sqlite_executor.py#execute_sqlite_query","file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py#execute_sqlite_query","relation":"calls","name":"execute_readonly_rows","viaMember":false,"file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py#execute_sqlite_query","relation":"calls","name":"get","viaMember":true,"file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py#execute_sqlite_query","relation":"calls","name":"bool","viaMember":false,"file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py#execute_sqlite_query","relation":"calls","name":"get","viaMember":true,"file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py#execute_sqlite_query","relation":"calls","name":"str","viaMember":false,"file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py#execute_sqlite_query","relation":"calls","name":"resolve","viaMember":true,"file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py#execute_sqlite_query","relation":"calls","name":"expanduser","viaMember":true,"file":"evaluation/sqlite_executor.py","recvType":"Path"},{"source":"evaluation/sqlite_executor.py#execute_sqlite_query","relation":"calls","name":"str","viaMember":false,"file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py#execute_sqlite_query","relation":"calls","name":"len","viaMember":false,"file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py#execute_sqlite_query","relation":"calls","name":"append","viaMember":true,"file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py#execute_sqlite_query","relation":"calls","name":"append","viaMember":true,"file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py#execute_sqlite_query","relation":"calls","name":"len","viaMember":false,"file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py#execute_sqlite_query","relation":"calls","name":"enumerate","viaMember":false,"file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py#execute_sqlite_query","relation":"calls","name":"min","viaMember":false,"file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py#execute_sqlite_query","relation":"calls","name":"max","viaMember":false,"file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py#execute_sqlite_query","relation":"calls","name":"len","viaMember":false,"file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py#execute_sqlite_query","relation":"contains","targetId":"evaluation/sqlite_executor.py#execute_sqlite_query.fmt_row","file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py#execute_sqlite_query.fmt_row","relation":"calls","name":"join","viaMember":true,"file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py#execute_sqlite_query.fmt_row","relation":"calls","name":"ljust","viaMember":true,"file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py#execute_sqlite_query.fmt_row","relation":"calls","name":"enumerate","viaMember":false,"file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py#execute_sqlite_query","relation":"calls","name":"fmt_row","viaMember":false,"file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py#execute_sqlite_query","relation":"calls","name":"join","viaMember":true,"file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py#execute_sqlite_query","relation":"calls","name":"join","viaMember":true,"file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py#execute_sqlite_query","relation":"calls","name":"fmt_row","viaMember":false,"file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py#execute_sqlite_query","relation":"calls","name":"len","viaMember":false,"file":"evaluation/sqlite_executor.py"},{"source":"evaluation/sqlite_executor.py#execute_sqlite_query","relation":"calls","name":"len","viaMember":false,"file":"evaluation/sqlite_executor.py"}]},"preprocess/build_dataset_releases.py":{"size":7046,"mtimeMs":1789189526393.8708,"hash":"65542f97481ba12dab828293d7666b73eece9b9cb22b7fc672cce728aec7fd80","nodes":[{"id":"preprocess/build_dataset_releases.py","name":"build_dataset_releases.py","kind":"file","path":"preprocess/build_dataset_releases.py","span":"L1-L224","signature":null,"exported":true,"origin":"ast","body_hash":"65542f97481ba12dab828293d7666b73eece9b9cb22b7fc672cce728aec7fd80","chars":7046,"summary_state":"pending","summary":null,"crux":null,"body_text":"#!/usr/bin/env python3 \"\"\"Build complete preview and release JSONL files from v1 plus sparse v2 patches.\"\"\" from __future__ import annotations import argparse import hashlib import json from pathlib import Path from typing import Any TASK_FILES = { \"search_to_sql\": Path( \"type_1_search_to_sql/search_to_sql_examples.jsonl\" ), \"sql_to_search\": Path( \"type_1_sql_to_search/sql_to_search_examples.jsonl\" ), \"parallel\": Path( \"type_2_parallel/search_sql_parallel_examples.jsonl\" ), } EXPECTED_PREFIXES = { \"search_to_sql\": \"s2sql_\", \"sql_to_search\": \"sql2s_\", \"parallel\": \"parallel_\", } REQUIRED_FIELDS = (\"id\", \"final_question\", \"hint\", \"db\", \"final_answer\") if __name__ == \"__main__\": main()"},{"id":"preprocess/build_dataset_releases.py#load_jsonl","name":"load_jsonl","kind":"function","path":"preprocess/build_dataset_releases.py","span":"L32-L55","signature":"def load_jsonl(path: Path, *, task_type: str) -> list[dict[str, Any]]","exported":true,"origin":"ast","body_hash":"1811065796c4e528e439e8025b57654d6ad953ec8f054a984ec5b49a74d8d561","body_text":"def load_jsonl(path: Path, *, task_type: str) -> list[dict[str, Any]]: rows: list[dict[str, Any]] = [] seen: set[str] = set() with path.open(encoding=\"utf-8\") as handle: for line_number, line in enumerate(handle, 1): if not line.strip(): continue row = json.loads(line) if not isinstance(row, dict): raise ValueError(f\"{path}:{line_number}: expected a JSON object\") record_id = str(row.get(\"id\") or \"\").strip() if not record_id: raise ValueError(f\"{path}:{line_number}: missing id\") if record_id in seen: raise ValueError(f\"{path}:{line_number}: duplicate id {record_id!r}\") expected_prefix = EXPECTED_PREFIXES[task_type] if not record_id.startswith(expected_prefix): raise ValueError( f\"{path}:{line_number}: id {record_id!r} does not match \" f\"{task_type!r}\" ) seen.add(record_id) rows.append(row) return rows","summary_state":"pending","summary":null,"crux":null},{"id":"preprocess/build_dataset_releases.py#with_task_type","name":"with_task_type","kind":"function","path":"preprocess/build_dataset_releases.py","span":"L58-L73","signature":"def with_task_type( row: dict[str, Any], *, task_type: str, ) -> dict[str, Any]","exported":true,"origin":"ast","body_hash":"a4031fbe4495fec18ebbcbb537bdba14dc6cec6883c7dfe17390691afd57b513","body_text":"def with_task_type( row: dict[str, Any], *, task_type: str, ) -> dict[str, Any]: missing = [field for field in REQUIRED_FIELDS if field not in row] if missing: raise ValueError( f\"{row.get('id', '<unknown>')}: missing required fields \" f\"{', '.join(missing)}\" ) result = {\"id\": row[\"id\"], \"hybrid_type\": task_type} result.update( (key, value) for key, value in row.items() if key != \"id\" ) return result","summary_state":"pending","summary":null,"crux":null},{"id":"preprocess/build_dataset_releases.py#overlay_rows","name":"overlay_rows","kind":"function","path":"preprocess/build_dataset_releases.py","span":"L76-L110","signature":"def overlay_rows( base_rows: list[dict[str, Any]], patch_rows: list[dict[str, Any]], *, task_type: str, ) -> tuple[list[dict[str, Any]], int]","exported":true,"origin":"ast","body_hash":"35f8df679b13c1900688e2b70dfe3f9799768e2fbd292f417f13811a1a8e17df","body_text":"def overlay_rows( base_rows: list[dict[str, Any]], patch_rows: list[dict[str, Any]], *, task_type: str, ) -> tuple[list[dict[str, Any]], int]: base_ids = [str(row[\"id\"]) for row in base_rows] base_set = set(base_ids) base_by_id = {str(row[\"id\"]): row for row in base_rows} patch_by_id = {str(row[\"id\"]): row for row in patch_rows} unknown = sorted(set(patch_by_id) - base_set) if unknown: raise ValueError( f\"{task_type}: v2 contains IDs absent from v1: {', '.join(unknown)}\" ) identical = [ record_id for record_id, patch in patch_by_id.items() if base_by_id[record_id] == patch ] if identical: raise ValueError( f\"{task_type}: v2 patch contains unchanged records: \" f\"{', '.join(sorted(identical))}\" ) merged = [ with_task_type( patch_by_id.get(str(base[\"id\"]), base), task_type=task_type, ) for base in base_rows ] return merged, len(patch_by_id)","summary_state":"pending","summary":null,"crux":null},{"id":"preprocess/build_dataset_releases.py#sha256","name":"sha256","kind":"function","path":"preprocess/build_dataset_releases.py","span":"L113-L118","signature":"def sha256(path: Path) -> str","exported":true,"origin":"ast","body_hash":"f6f842fd5ce43f4f374872c22b5e2898b1f6992d792665e4dceb1cbc88920efd","body_text":"def sha256(path: Path) -> str: digest = hashlib.sha256() with path.open(\"rb\") as handle: for chunk in iter(lambda: handle.read(1024 * 1024), b\"\"): digest.update(chunk) return digest.hexdigest()","summary_state":"pending","summary":null,"crux":null},{"id":"preprocess/build_dataset_releases.py#write_jsonl","name":"write_jsonl","kind":"function","path":"preprocess/build_dataset_releases.py","span":"L121-L125","signature":"def write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None","exported":true,"origin":"ast","body_hash":"e255b9fc63138af8ac45f64975f6b4aa9d174ab61aa78c7e4a361181af6064b7","body_text":"def write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None: path.parent.mkdir(parents=True, exist_ok=True) with path.open(\"w\", encoding=\"utf-8\") as handle: for row in rows: handle.write(json.dumps(row, ensure_ascii=False) + \"\\n\")","summary_state":"pending","summary":null,"crux":null},{"id":"preprocess/build_dataset_releases.py#build_releases","name":"build_releases","kind":"function","path":"preprocess/build_dataset_releases.py","span":"L128-L201","signature":"def build_releases( *, v1_root: Path, v2_root: Path, output_root: Path, ) -> dict[str, Any]","exported":true,"origin":"ast","body_hash":"eee48242b0f00f0d1abe84f2a7d8c479b132e739d92371cd76061c9aea5dd7a5","body_text":"def build_releases( *, v1_root: Path, v2_root: Path, output_root: Path, ) -> dict[str, Any]: preview_rows: list[dict[str, Any]] = [] release_rows: list[dict[str, Any]] = [] counts: dict[str, dict[str, int]] = {} for task_type, relative_path in TASK_FILES.items(): v1_rows = load_jsonl(v1_root / relative_path, task_type=task_type) v2_rows = load_jsonl(v2_root / relative_path, task_type=task_type) merged_rows, patched = overlay_rows( v1_rows, v2_rows, task_type=task_type, ) preview_rows.extend( with_task_type(row, task_type=task_type) for row in v1_rows ) release_rows.extend(merged_rows) counts[task_type] = { \"preview_records\": len(v1_rows), \"release_records\": len(merged_rows), \"v2_overrides\": patched, } preview_path = ( output_root / \"hybrid_ds_preview\" / \"hybrid_ds_preview.jsonl\" ) release_path = ( output_root / \"hybrid_ds_release\" / \"hybrid_ds_release.jsonl\" ) write_jsonl(preview_path, preview_rows) write_jsonl(release_path, release_rows) preview_ids = [str(row[\"id\"]) for row in preview_rows] release_ids = [str(row[\"id\"]) for row in release_rows] if len(preview_ids) != len(set(preview_ids)): raise ValueError(\"preview output contains duplicate IDs\") if len(release_ids) != len(set(release_ids)): raise ValueError(\"release output contains duplicate IDs\") if preview_ids != release_ids: raise ValueError(\"preview and release ID/order mismatch\") manifest = { \"preview\": { \"source\": \"hybrid_ds_v1_full\", \"records\": len(preview_rows), \"path\": str(preview_path.relative_to(output_root)), \"sha256\": sha256(preview_path), }, \"release\": { \"source\": \"hybrid_ds_v1_full overlaid by hybrid_ds_v2_full\", \"records\": len(release_rows), \"overrides\": sum( item[\"v2_overrides\"] for item in counts.values() ), \"path\": str(release_path.relative_to(output_root)), \"sha256\": sha256(release_path), }, \"by_task_type\": counts, } manifest_path = output_root / \"manifest.json\" manifest_path.write_text( json.dumps(manifest, ensure_ascii=False, indent=2) + \"\\n\", encoding=\"utf-8\", ) return manifest","summary_state":"pending","summary":null,"crux":null},{"id":"preprocess/build_dataset_releases.py#parse_args","name":"parse_args","kind":"function","path":"preprocess/build_dataset_releases.py","span":"L204-L209","signature":"def parse_args() -> argparse.Namespace","exported":true,"origin":"ast","body_hash":"0e042f7cb9f59c7a6f4bfa00b2e394b1fcf52b47a5a52eb4efc7771e9aa48a6c","body_text":"def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument(\"--v1-root\", required=True, type=Path) parser.add_argument(\"--v2-root\", required=True, type=Path) parser.add_argument(\"--output-root\", required=True, type=Path) return parser.parse_args()","summary_state":"pending","summary":null,"crux":null},{"id":"preprocess/build_dataset_releases.py#main","name":"main","kind":"function","path":"preprocess/build_dataset_releases.py","span":"L212-L219","signature":"def main() -> None","exported":true,"origin":"ast","body_hash":"da974761debf4f0ee639d898ff776967e99d46d0f1a0071e3082351fb0f83aa6","body_text":"def main() -> None: args = parse_args() manifest = build_releases( v1_root=args.v1_root.expanduser().resolve(), v2_root=args.v2_root.expanduser().resolve(), output_root=args.output_root.expanduser().resolve(), ) print(json.dumps(manifest, ensure_ascii=False, indent=2))","summary_state":"pending","summary":null,"crux":null}],"rawEdges":[{"source":"preprocess/build_dataset_releases.py","relation":"imports","specifier":"argparse","file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py","relation":"imports","specifier":"hashlib","file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py","relation":"imports","specifier":"json","file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py","relation":"imports","specifier":"pathlib","file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py","relation":"imports","specifier":"typing","file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py","relation":"calls","name":"Path","viaMember":false,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py","relation":"calls","name":"Path","viaMember":false,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py","relation":"calls","name":"Path","viaMember":false,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py","relation":"contains","targetId":"preprocess/build_dataset_releases.py#load_jsonl","file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#load_jsonl","relation":"calls","name":"set","viaMember":false,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#load_jsonl","relation":"calls","name":"open","viaMember":true,"file":"preprocess/build_dataset_releases.py","recvType":"Path"},{"source":"preprocess/build_dataset_releases.py#load_jsonl","relation":"calls","name":"enumerate","viaMember":false,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#load_jsonl","relation":"calls","name":"strip","viaMember":true,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#load_jsonl","relation":"calls","name":"loads","viaMember":true,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#load_jsonl","relation":"calls","name":"isinstance","viaMember":false,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#load_jsonl","relation":"calls","name":"ValueError","viaMember":false,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#load_jsonl","relation":"calls","name":"strip","viaMember":true,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#load_jsonl","relation":"calls","name":"str","viaMember":false,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#load_jsonl","relation":"calls","name":"get","viaMember":true,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#load_jsonl","relation":"calls","name":"ValueError","viaMember":false,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#load_jsonl","relation":"calls","name":"ValueError","viaMember":false,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#load_jsonl","relation":"calls","name":"startswith","viaMember":true,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#load_jsonl","relation":"calls","name":"ValueError","viaMember":false,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#load_jsonl","relation":"calls","name":"add","viaMember":true,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#load_jsonl","relation":"calls","name":"append","viaMember":true,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py","relation":"contains","targetId":"preprocess/build_dataset_releases.py#with_task_type","file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#with_task_type","relation":"calls","name":"ValueError","viaMember":false,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#with_task_type","relation":"calls","name":"get","viaMember":true,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#with_task_type","relation":"calls","name":"join","viaMember":true,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#with_task_type","relation":"calls","name":"update","viaMember":true,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#with_task_type","relation":"calls","name":"items","viaMember":true,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py","relation":"contains","targetId":"preprocess/build_dataset_releases.py#overlay_rows","file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#overlay_rows","relation":"calls","name":"str","viaMember":false,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#overlay_rows","relation":"calls","name":"set","viaMember":false,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#overlay_rows","relation":"calls","name":"str","viaMember":false,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#overlay_rows","relation":"calls","name":"str","viaMember":false,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#overlay_rows","relation":"calls","name":"sorted","viaMember":false,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#overlay_rows","relation":"calls","name":"set","viaMember":false,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#overlay_rows","relation":"calls","name":"ValueError","viaMember":false,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#overlay_rows","relation":"calls","name":"join","viaMember":true,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#overlay_rows","relation":"calls","name":"items","viaMember":true,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#overlay_rows","relation":"calls","name":"ValueError","viaMember":false,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#overlay_rows","relation":"calls","name":"join","viaMember":true,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#overlay_rows","relation":"calls","name":"sorted","viaMember":false,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#overlay_rows","relation":"calls","name":"with_task_type","viaMember":false,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#overlay_rows","relation":"calls","name":"get","viaMember":true,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#overlay_rows","relation":"calls","name":"str","viaMember":false,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#overlay_rows","relation":"calls","name":"len","viaMember":false,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py","relation":"contains","targetId":"preprocess/build_dataset_releases.py#sha256","file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#sha256","relation":"calls","name":"sha256","viaMember":true,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#sha256","relation":"calls","name":"open","viaMember":true,"file":"preprocess/build_dataset_releases.py","recvType":"Path"},{"source":"preprocess/build_dataset_releases.py#sha256","relation":"calls","name":"iter","viaMember":false,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#sha256","relation":"calls","name":"read","viaMember":true,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#sha256","relation":"calls","name":"update","viaMember":true,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#sha256","relation":"calls","name":"hexdigest","viaMember":true,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py","relation":"contains","targetId":"preprocess/build_dataset_releases.py#write_jsonl","file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#write_jsonl","relation":"calls","name":"mkdir","viaMember":true,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#write_jsonl","relation":"calls","name":"open","viaMember":true,"file":"preprocess/build_dataset_releases.py","recvType":"Path"},{"source":"preprocess/build_dataset_releases.py#write_jsonl","relation":"calls","name":"write","viaMember":true,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#write_jsonl","relation":"calls","name":"dumps","viaMember":true,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py","relation":"contains","targetId":"preprocess/build_dataset_releases.py#build_releases","file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#build_releases","relation":"calls","name":"items","viaMember":true,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#build_releases","relation":"calls","name":"load_jsonl","viaMember":false,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#build_releases","relation":"calls","name":"load_jsonl","viaMember":false,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#build_releases","relation":"calls","name":"overlay_rows","viaMember":false,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#build_releases","relation":"calls","name":"extend","viaMember":true,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#build_releases","relation":"calls","name":"with_task_type","viaMember":false,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#build_releases","relation":"calls","name":"extend","viaMember":true,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#build_releases","relation":"calls","name":"len","viaMember":false,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#build_releases","relation":"calls","name":"len","viaMember":false,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#build_releases","relation":"calls","name":"write_jsonl","viaMember":false,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#build_releases","relation":"calls","name":"write_jsonl","viaMember":false,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#build_releases","relation":"calls","name":"str","viaMember":false,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#build_releases","relation":"calls","name":"str","viaMember":false,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#build_releases","relation":"calls","name":"len","viaMember":false,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#build_releases","relation":"calls","name":"len","viaMember":false,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#build_releases","relation":"calls","name":"set","viaMember":false,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#build_releases","relation":"calls","name":"ValueError","viaMember":false,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#build_releases","relation":"calls","name":"len","viaMember":false,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#build_releases","relation":"calls","name":"len","viaMember":false,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#build_releases","relation":"calls","name":"set","viaMember":false,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#build_releases","relation":"calls","name":"ValueError","viaMember":false,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#build_releases","relation":"calls","name":"ValueError","viaMember":false,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#build_releases","relation":"calls","name":"len","viaMember":false,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#build_releases","relation":"calls","name":"str","viaMember":false,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#build_releases","relation":"calls","name":"relative_to","viaMember":true,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#build_releases","relation":"calls","name":"sha256","viaMember":false,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#build_releases","relation":"calls","name":"len","viaMember":false,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#build_releases","relation":"calls","name":"sum","viaMember":false,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#build_releases","relation":"calls","name":"values","viaMember":true,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#build_releases","relation":"calls","name":"str","viaMember":false,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#build_releases","relation":"calls","name":"relative_to","viaMember":true,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#build_releases","relation":"calls","name":"sha256","viaMember":false,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#build_releases","relation":"calls","name":"write_text","viaMember":true,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#build_releases","relation":"calls","name":"dumps","viaMember":true,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py","relation":"contains","targetId":"preprocess/build_dataset_releases.py#parse_args","file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#parse_args","relation":"calls","name":"ArgumentParser","viaMember":true,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#parse_args","relation":"calls","name":"add_argument","viaMember":true,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#parse_args","relation":"calls","name":"add_argument","viaMember":true,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#parse_args","relation":"calls","name":"add_argument","viaMember":true,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#parse_args","relation":"calls","name":"parse_args","viaMember":true,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py","relation":"contains","targetId":"preprocess/build_dataset_releases.py#main","file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#main","relation":"calls","name":"parse_args","viaMember":false,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#main","relation":"calls","name":"build_releases","viaMember":false,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#main","relation":"calls","name":"resolve","viaMember":true,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#main","relation":"calls","name":"expanduser","viaMember":true,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#main","relation":"calls","name":"resolve","viaMember":true,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#main","relation":"calls","name":"expanduser","viaMember":true,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#main","relation":"calls","name":"resolve","viaMember":true,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#main","relation":"calls","name":"expanduser","viaMember":true,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#main","relation":"calls","name":"print","viaMember":false,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py#main","relation":"calls","name":"dumps","viaMember":true,"file":"preprocess/build_dataset_releases.py"},{"source":"preprocess/build_dataset_releases.py","relation":"calls","name":"main","viaMember":false,"file":"preprocess/build_dataset_releases.py"}]},"preprocess/prepare_public_tasks.py":{"size":2392,"mtimeMs":1789189526393.8708,"hash":"21f0f5db1138842017771c8e1a2072dbc9ba85df10049e814bed735eda8f910a","nodes":[{"id":"preprocess/prepare_public_tasks.py","name":"prepare_public_tasks.py","kind":"file","path":"preprocess/prepare_public_tasks.py","span":"L1-L69","signature":null,"exported":true,"origin":"ast","body_hash":"21f0f5db1138842017771c8e1a2072dbc9ba85df10049e814bed735eda8f910a","chars":2392,"summary_state":"pending","summary":null,"crux":null,"body_text":"#!/usr/bin/env python3 \"\"\"Create public task records by removing construction and grading fields.\"\"\" from __future__ import annotations import argparse import json from pathlib import Path from typing import Any, Iterator PUBLIC_FIELDS = (\"id\", \"hybrid_type\", \"final_question\", \"hint\", \"db\") REQUIRED_FIELDS = (\"id\", \"hybrid_type\", \"final_question\", \"db\") VALID_TYPES = {\"sql_to_search\", \"search_to_sql\", \"parallel\"} if __name__ == \"__main__\": main()"},{"id":"preprocess/prepare_public_tasks.py#iter_jsonl","name":"iter_jsonl","kind":"function","path":"preprocess/prepare_public_tasks.py","span":"L17-L25","signature":"def iter_jsonl(path: Path) -> Iterator[tuple[int, dict[str, Any]]]","exported":true,"origin":"ast","body_hash":"d080d649e884073081813a4ade1d7b2311722c4c0304ecc01a53d20210615df2","body_text":"def iter_jsonl(path: Path) -> Iterator[tuple[int, dict[str, Any]]]: with path.open(encoding=\"utf-8\") as handle: for line_number, line in enumerate(handle, 1): if not line.strip(): continue value = json.loads(line) if not isinstance(value, dict): raise ValueError(f\"{path}:{line_number}: expected a JSON object\") yield line_number, value","summary_state":"pending","summary":null,"crux":null},{"id":"preprocess/prepare_public_tasks.py#prepare_record","name":"prepare_record","kind":"function","path":"preprocess/prepare_public_tasks.py","span":"L28-L36","signature":"def prepare_record(record: dict[str, Any], *, location: str) -> dict[str, Any]","exported":true,"origin":"ast","body_hash":"b5db31b8912a1911d001c2020bd514d29c7c6a5028443390a2600199fb509c1e","body_text":"def prepare_record(record: dict[str, Any], *, location: str) -> dict[str, Any]: missing = [key for key in REQUIRED_FIELDS if not record.get(key)] if missing: raise ValueError(f\"{location}: missing {', '.join(missing)}\") if record[\"hybrid_type\"] not in VALID_TYPES: raise ValueError( f\"{location}: invalid hybrid_type {record['hybrid_type']!r}\" ) return {key: record[key] for key in PUBLIC_FIELDS if key in record}","summary_state":"pending","summary":null,"crux":null},{"id":"preprocess/prepare_public_tasks.py#parse_args","name":"parse_args","kind":"function","path":"preprocess/prepare_public_tasks.py","span":"L39-L43","signature":"def parse_args() -> argparse.Namespace","exported":true,"origin":"ast","body_hash":"433fb6f24cc78dbb9970ff2e8826c7546429f76df4d3339b0bbaa4b8fff4eb58","body_text":"def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument(\"--input\", required=True, type=Path) parser.add_argument(\"--output\", required=True, type=Path) return parser.parse_args()","summary_state":"pending","summary":null,"crux":null},{"id":"preprocess/prepare_public_tasks.py#main","name":"main","kind":"function","path":"preprocess/prepare_public_tasks.py","span":"L46-L64","signature":"def main() -> None","exported":true,"origin":"ast","body_hash":"37a6bac03dc923ca43c496ce1dbab8ce8c7b7109f549e52f494d55fc57a7ce5c","body_text":"def main() -> None: args = parse_args() source = args.input.expanduser().resolve() output = args.output.expanduser().resolve() output.parent.mkdir(parents=True, exist_ok=True) seen: set[str] = set() count = 0 with output.open(\"w\", encoding=\"utf-8\") as handle: for line_number, record in iter_jsonl(source): public = prepare_record(record, location=f\"{source}:{line_number}\") record_id = str(public[\"id\"]) if record_id in seen: raise ValueError(f\"{source}:{line_number}: duplicate id {record_id!r}\") seen.add(record_id) handle.write(json.dumps(public, ensure_ascii=False) + \"\\n\") count += 1 print(f\"Wrote {count} public task records to {output}\")","summary_state":"pending","summary":null,"crux":null}],"rawEdges":[{"source":"preprocess/prepare_public_tasks.py","relation":"imports","specifier":"argparse","file":"preprocess/prepare_public_tasks.py"},{"source":"preprocess/prepare_public_tasks.py","relation":"imports","specifier":"json","file":"preprocess/prepare_public_tasks.py"},{"source":"preprocess/prepare_public_tasks.py","relation":"imports","specifier":"pathlib","file":"preprocess/prepare_public_tasks.py"},{"source":"preprocess/prepare_public_tasks.py","relation":"imports","specifier":"typing","file":"preprocess/prepare_public_tasks.py"},{"source":"preprocess/prepare_public_tasks.py","relation":"contains","targetId":"preprocess/prepare_public_tasks.py#iter_jsonl","file":"preprocess/prepare_public_tasks.py"},{"source":"preprocess/prepare_public_tasks.py#iter_jsonl","relation":"calls","name":"open","viaMember":true,"file":"preprocess/prepare_public_tasks.py","recvType":"Path"},{"source":"preprocess/prepare_public_tasks.py#iter_jsonl","relation":"calls","name":"enumerate","viaMember":false,"file":"preprocess/prepare_public_tasks.py"},{"source":"preprocess/prepare_public_tasks.py#iter_jsonl","relation":"calls","name":"strip","viaMember":true,"file":"preprocess/prepare_public_tasks.py"},{"source":"preprocess/prepare_public_tasks.py#iter_jsonl","relation":"calls","name":"loads","viaMember":true,"file":"preprocess/prepare_public_tasks.py"},{"source":"preprocess/prepare_public_tasks.py#iter_jsonl","relation":"calls","name":"isinstance","viaMember":false,"file":"preprocess/prepare_public_tasks.py"},{"source":"preprocess/prepare_public_tasks.py#iter_jsonl","relation":"calls","name":"ValueError","viaMember":false,"file":"preprocess/prepare_public_tasks.py"},{"source":"preprocess/prepare_public_tasks.py","relation":"contains","targetId":"preprocess/prepare_public_tasks.py#prepare_record","file":"preprocess/prepare_public_tasks.py"},{"source":"preprocess/prepare_public_tasks.py#prepare_record","relation":"calls","name":"get","viaMember":true,"file":"preprocess/prepare_public_tasks.py"},{"source":"preprocess/prepare_public_tasks.py#prepare_record","relation":"calls","name":"ValueError","viaMember":false,"file":"preprocess/prepare_public_tasks.py"},{"source":"preprocess/prepare_public_tasks.py#prepare_record","relation":"calls","name":"join","viaMember":true,"file":"preprocess/prepare_public_tasks.py"},{"source":"preprocess/prepare_public_tasks.py#prepare_record","relation":"calls","name":"ValueError","viaMember":false,"file":"preprocess/prepare_public_tasks.py"},{"source":"preprocess/prepare_public_tasks.py","relation":"contains","targetId":"preprocess/prepare_public_tasks.py#parse_args","file":"preprocess/prepare_public_tasks.py"},{"source":"preprocess/prepare_public_tasks.py#parse_args","relation":"calls","name":"ArgumentParser","viaMember":true,"file":"preprocess/prepare_public_tasks.py"},{"source":"preprocess/prepare_public_tasks.py#parse_args","relation":"calls","name":"add_argument","viaMember":true,"file":"preprocess/prepare_public_tasks.py"},{"source":"preprocess/prepare_public_tasks.py#parse_args","relation":"calls","name":"add_argument","viaMember":true,"file":"preprocess/prepare_public_tasks.py"},{"source":"preprocess/prepare_public_tasks.py#parse_args","relation":"calls","name":"parse_args","viaMember":true,"file":"preprocess/prepare_public_tasks.py"},{"source":"preprocess/prepare_public_tasks.py","relation":"contains","targetId":"preprocess/prepare_public_tasks.py#main","file":"preprocess/prepare_public_tasks.py"},{"source":"preprocess/prepare_public_tasks.py#main","relation":"calls","name":"parse_args","viaMember":false,"file":"preprocess/prepare_public_tasks.py"},{"source":"preprocess/prepare_public_tasks.py#main","relation":"calls","name":"resolve","viaMember":true,"file":"preprocess/prepare_public_tasks.py"},{"source":"preprocess/prepare_public_tasks.py#main","relation":"calls","name":"expanduser","viaMember":true,"file":"preprocess/prepare_public_tasks.py"},{"source":"preprocess/prepare_public_tasks.py#main","relation":"calls","name":"resolve","viaMember":true,"file":"preprocess/prepare_public_tasks.py"},{"source":"preprocess/prepare_public_tasks.py#main","relation":"calls","name":"expanduser","viaMember":true,"file":"preprocess/prepare_public_tasks.py"},{"source":"preprocess/prepare_public_tasks.py#main","relation":"calls","name":"mkdir","viaMember":true,"file":"preprocess/prepare_public_tasks.py"},{"source":"preprocess/prepare_public_tasks.py#main","relation":"calls","name":"set","viaMember":false,"file":"preprocess/prepare_public_tasks.py"},{"source":"preprocess/prepare_public_tasks.py#main","relation":"calls","name":"open","viaMember":true,"file":"preprocess/prepare_public_tasks.py"},{"source":"preprocess/prepare_public_tasks.py#main","relation":"calls","name":"iter_jsonl","viaMember":false,"file":"preprocess/prepare_public_tasks.py"},{"source":"preprocess/prepare_public_tasks.py#main","relation":"calls","name":"prepare_record","viaMember":false,"file":"preprocess/prepare_public_tasks.py"},{"source":"preprocess/prepare_public_tasks.py#main","relation":"calls","name":"str","viaMember":false,"file":"preprocess/prepare_public_tasks.py"},{"source":"preprocess/prepare_public_tasks.py#main","relation":"calls","name":"ValueError","viaMember":false,"file":"preprocess/prepare_public_tasks.py"},{"source":"preprocess/prepare_public_tasks.py#main","relation":"calls","name":"add","viaMember":true,"file":"preprocess/prepare_public_tasks.py"},{"source":"preprocess/prepare_public_tasks.py#main","relation":"calls","name":"write","viaMember":true,"file":"preprocess/prepare_public_tasks.py"},{"source":"preprocess/prepare_public_tasks.py#main","relation":"calls","name":"dumps","viaMember":true,"file":"preprocess/prepare_public_tasks.py"},{"source":"preprocess/prepare_public_tasks.py#main","relation":"calls","name":"print","viaMember":false,"file":"preprocess/prepare_public_tasks.py"},{"source":"preprocess/prepare_public_tasks.py","relation":"calls","name":"main","viaMember":false,"file":"preprocess/prepare_public_tasks.py"}]},"preprocess/validate_predictions.py":{"size":2174,"mtimeMs":1789189526393.8708,"hash":"24c6d2de1c857a9978e49dec7cb62e821054b0f755e13cac6fc160dc20c48b85","nodes":[{"id":"preprocess/validate_predictions.py","name":"validate_predictions.py","kind":"file","path":"preprocess/validate_predictions.py","span":"L1-L62","signature":null,"exported":true,"origin":"ast","body_hash":"24c6d2de1c857a9978e49dec7cb62e821054b0f755e13cac6fc160dc20c48b85","chars":2174,"summary_state":"pending","summary":null,"crux":null,"body_text":"#!/usr/bin/env python3 \"\"\"Validate the shape and IDs of a HybridDeepResearch predictions file.\"\"\" from __future__ import annotations import argparse import json from pathlib import Path from typing import Any if __name__ == \"__main__\": main()"},{"id":"preprocess/validate_predictions.py#validate_answer","name":"validate_answer","kind":"function","path":"preprocess/validate_predictions.py","span":"L12-L17","signature":"def validate_answer(value: Any, *, location: str) -> None","exported":true,"origin":"ast","body_hash":"557b768066e7f449ae9217d8f7dee11d3eda04e5ac95b415d879065a40157396","body_text":"def validate_answer(value: Any, *, location: str) -> None: if isinstance(value, str): if not value.strip(): raise ValueError(f\"{location}: final_answer must not be empty\") return raise ValueError(f\"{location}: final_answer must be a string\")","summary_state":"pending","summary":null,"crux":null},{"id":"preprocess/validate_predictions.py#validate","name":"validate","kind":"function","path":"preprocess/validate_predictions.py","span":"L20-L48","signature":"def validate(path: Path) -> int","exported":true,"origin":"ast","body_hash":"4bf11b9538632e75b9fd83ad6353f3f11f0b6a474a0d4bab884b975d2afafc08","body_text":"def validate(path: Path) -> int: seen: set[str] = set() count = 0 with path.open(encoding=\"utf-8\") as handle: for line_number, line in enumerate(handle, 1): if not line.strip(): continue location = f\"{path}:{line_number}\" record = json.loads(line) if not isinstance(record, dict): raise ValueError(f\"{location}: expected a JSON object\") extra = set(record) - {\"instance_id\", \"final_answer\"} if extra: raise ValueError( f\"{location}: unexpected fields: {', '.join(sorted(extra))}\" ) instance_id = record.get(\"instance_id\") if not isinstance(instance_id, str) or not instance_id.strip(): raise ValueError(f\"{location}: instance_id must be a non-empty string\") if instance_id in seen: raise ValueError(f\"{location}: duplicate instance_id {instance_id!r}\") if \"final_answer\" not in record: raise ValueError(f\"{location}: missing final_answer\") validate_answer(record[\"final_answer\"], location=location) seen.add(instance_id) count += 1 if not count: raise ValueError(f\"{path}: no prediction records found\") return count","summary_state":"pending","summary":null,"crux":null},{"id":"preprocess/validate_predictions.py#main","name":"main","kind":"function","path":"preprocess/validate_predictions.py","span":"L51-L57","signature":"def main() -> None","exported":true,"origin":"ast","body_hash":"f245f537df5d1dcba000ddc42749fd58a9a8d7aadd584a1a09b351efc4254727","body_text":"def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument(\"predictions\", type=Path) args = parser.parse_args() path = args.predictions.expanduser().resolve() count = validate(path) print(f\"OK: {count} unique prediction records in {path}\")","summary_state":"pending","summary":null,"crux":null}],"rawEdges":[{"source":"preprocess/validate_predictions.py","relation":"imports","specifier":"argparse","file":"preprocess/validate_predictions.py"},{"source":"preprocess/validate_predictions.py","relation":"imports","specifier":"json","file":"preprocess/validate_predictions.py"},{"source":"preprocess/validate_predictions.py","relation":"imports","specifier":"pathlib","file":"preprocess/validate_predictions.py"},{"source":"preprocess/validate_predictions.py","relation":"imports","specifier":"typing","file":"preprocess/validate_predictions.py"},{"source":"preprocess/validate_predictions.py","relation":"contains","targetId":"preprocess/validate_predictions.py#validate_answer","file":"preprocess/validate_predictions.py"},{"source":"preprocess/validate_predictions.py#validate_answer","relation":"calls","name":"isinstance","viaMember":false,"file":"preprocess/validate_predictions.py"},{"source":"preprocess/validate_predictions.py#validate_answer","relation":"calls","name":"strip","viaMember":true,"file":"preprocess/validate_predictions.py","recvType":"Any"},{"source":"preprocess/validate_predictions.py#validate_answer","relation":"calls","name":"ValueError","viaMember":false,"file":"preprocess/validate_predictions.py"},{"source":"preprocess/validate_predictions.py#validate_answer","relation":"calls","name":"ValueError","viaMember":false,"file":"preprocess/validate_predictions.py"},{"source":"preprocess/validate_predictions.py","relation":"contains","targetId":"preprocess/validate_predictions.py#validate","file":"preprocess/validate_predictions.py"},{"source":"preprocess/validate_predictions.py#validate","relation":"calls","name":"set","viaMember":false,"file":"preprocess/validate_predictions.py"},{"source":"preprocess/validate_predictions.py#validate","relation":"calls","name":"open","viaMember":true,"file":"preprocess/validate_predictions.py","recvType":"Path"},{"source":"preprocess/validate_predictions.py#validate","relation":"calls","name":"enumerate","viaMember":false,"file":"preprocess/validate_predictions.py"},{"source":"preprocess/validate_predictions.py#validate","relation":"calls","name":"strip","viaMember":true,"file":"preprocess/validate_predictions.py"},{"source":"preprocess/validate_predictions.py#validate","relation":"calls","name":"loads","viaMember":true,"file":"preprocess/validate_predictions.py"},{"source":"preprocess/validate_predictions.py#validate","relation":"calls","name":"isinstance","viaMember":false,"file":"preprocess/validate_predictions.py"},{"source":"preprocess/validate_predictions.py#validate","relation":"calls","name":"ValueError","viaMember":false,"file":"preprocess/validate_predictions.py"},{"source":"preprocess/validate_predictions.py#validate","relation":"calls","name":"set","viaMember":false,"file":"preprocess/validate_predictions.py"},{"source":"preprocess/validate_predictions.py#validate","relation":"calls","name":"ValueError","viaMember":false,"file":"preprocess/validate_predictions.py"},{"source":"preprocess/validate_predictions.py#validate","relation":"calls","name":"join","viaMember":true,"file":"preprocess/validate_predictions.py"},{"source":"preprocess/validate_predictions.py#validate","relation":"calls","name":"sorted","viaMember":false,"file":"preprocess/validate_predictions.py"},{"source":"preprocess/validate_predictions.py#validate","relation":"calls","name":"get","viaMember":true,"file":"preprocess/validate_predictions.py"},{"source":"preprocess/validate_predictions.py#validate","relation":"calls","name":"isinstance","viaMember":false,"file":"preprocess/validate_predictions.py"},{"source":"preprocess/validate_predictions.py#validate","relation":"calls","name":"strip","viaMember":true,"file":"preprocess/validate_predictions.py"},{"source":"preprocess/validate_predictions.py#validate","relation":"calls","name":"ValueError","viaMember":false,"file":"preprocess/validate_predictions.py"},{"source":"preprocess/validate_predictions.py#validate","relation":"calls","name":"ValueError","viaMember":false,"file":"preprocess/validate_predictions.py"},{"source":"preprocess/validate_predictions.py#validate","relation":"calls","name":"ValueError","viaMember":false,"file":"preprocess/validate_predictions.py"},{"source":"preprocess/validate_predictions.py#validate","relation":"calls","name":"validate_answer","viaMember":false,"file":"preprocess/validate_predictions.py"},{"source":"preprocess/validate_predictions.py#validate","relation":"calls","name":"add","viaMember":true,"file":"preprocess/validate_predictions.py"},{"source":"preprocess/validate_predictions.py#validate","relation":"calls","name":"ValueError","viaMember":false,"file":"preprocess/validate_predictions.py"},{"source":"preprocess/validate_predictions.py","relation":"contains","targetId":"preprocess/validate_predictions.py#main","file":"preprocess/validate_predictions.py"},{"source":"preprocess/validate_predictions.py#main","relation":"calls","name":"ArgumentParser","viaMember":true,"file":"preprocess/validate_predictions.py"},{"source":"preprocess/validate_predictions.py#main","relation":"calls","name":"add_argument","viaMember":true,"file":"preprocess/validate_predictions.py"},{"source":"preprocess/validate_predictions.py#main","relation":"calls","name":"parse_args","viaMember":true,"file":"preprocess/validate_predictions.py"},{"source":"preprocess/validate_predictions.py#main","relation":"calls","name":"resolve","viaMember":true,"file":"preprocess/validate_predictions.py"},{"source":"preprocess/validate_predictions.py#main","relation":"calls","name":"expanduser","viaMember":true,"file":"preprocess/validate_predictions.py"},{"source":"preprocess/validate_predictions.py#main","relation":"calls","name":"validate","viaMember":false,"file":"preprocess/validate_predictions.py"},{"source":"preprocess/validate_predictions.py#main","relation":"calls","name":"print","viaMember":false,"file":"preprocess/validate_predictions.py"},{"source":"preprocess/validate_predictions.py","relation":"calls","name":"main","viaMember":false,"file":"preprocess/validate_predictions.py"}]},"tools/__init__.py":{"size":1858,"mtimeMs":1789189526394.8708,"hash":"1060a5e001b3caac1a9e506ed523b5d9554ea82f55d02aaa26f6cfa3e136d078","nodes":[{"id":"tools/__init__.py","name":"__init__.py","kind":"file","path":"tools/__init__.py","span":"L1-L63","signature":null,"exported":true,"origin":"ast","body_hash":"1060a5e001b3caac1a9e506ed523b5d9554ea82f55d02aaa26f6cfa3e136d078","chars":1858,"summary_state":"pending","summary":null,"crux":null,"body_text":"\"\"\"Public HybridDeepResearch tool contract. These are the capability names and argument shapes an agent should expose. They are not a production agent loop: wire them into your own framework and search provider. \"\"\" from __future__ import annotations import json from pathlib import Path from typing import Any, Callable from .final_answer import FINAL_ANSWER_TOOL_NAME, make_final_answer from .knowledge import make_knowledge_tools from .run_sql import make_run_sql from .web_fetch import make_web_fetch from .web_search import make_web_search SCHEMA_PATH = Path(__file__).with_name(\"schemas.json\")"},{"id":"tools/__init__.py#load_schemas","name":"load_schemas","kind":"function","path":"tools/__init__.py","span":"L23-L24","signature":"def load_schemas() -> list[dict[str, Any]]","exported":true,"origin":"ast","body_hash":"2e8bc720b3076a3b98a08a6144848d958f9736184b0efa9e6c7ebb488f5e1e60","body_text":"def load_schemas() -> list[dict[str, Any]]: return json.loads(SCHEMA_PATH.read_text(encoding=\"utf-8\"))","summary_state":"pending","summary":null,"crux":null},{"id":"tools/__init__.py#openai_tools","name":"openai_tools","kind":"function","path":"tools/__init__.py","span":"L27-L39","signature":"def openai_tools() -> list[dict[str, Any]]","exported":true,"origin":"ast","body_hash":"3dcd5323ab8e152c178a019cbdc83d50607c36dd953e841cc893c388b55765a8","body_text":"def openai_tools() -> list[dict[str, Any]]: \"\"\"Wrap :func:`load_schemas` as OpenAI Chat Completions ``tools``.\"\"\" return [ { \"type\": \"function\", \"function\": { \"name\": schema[\"name\"], \"description\": schema[\"description\"], \"parameters\": schema[\"parameters\"], }, } for schema in load_schemas() ]","summary_state":"pending","summary":null,"crux":null},{"id":"tools/__init__.py#build_toolset","name":"build_toolset","kind":"function","path":"tools/__init__.py","span":"L42-L62","signature":"def build_toolset( *, sqlite_db_path: str | Path | None = None, sqlite_root: str | Path | None = None, db: str | None = None, search_fn=None, fetch_fn=None, ) -> dict[str, Callable[..., str]]","exported":true,"origin":"ast","body_hash":"ed803f24a801b34f715f9176bcaf35976ebdd35ec0ea3aa8462c4055e12aa6fe","body_text":"def build_toolset( *, sqlite_db_path: str | Path | None = None, sqlite_root: str | Path | None = None, db: str | None = None, search_fn=None, fetch_fn=None, ) -> dict[str, Callable[..., str]]: \"\"\"Assemble the public tool callables for one task. Knowledge tools are included when both ``sqlite_root`` and ``db`` are set. \"\"\" tools: dict[str, Callable[..., str]] = { \"web_search\": make_web_search(search_fn), \"web_fetch\": make_web_fetch(fetch_fn), \"run_sql\": make_run_sql(sqlite_db_path), FINAL_ANSWER_TOOL_NAME: make_final_answer(), } if sqlite_root is not None and db: tools.update(make_knowledge_tools(sqlite_root, db)) return tools","summary_state":"pending","summary":null,"crux":null}],"rawEdges":[{"source":"tools/__init__.py","relation":"imports","specifier":"json","file":"tools/__init__.py"},{"source":"tools/__init__.py","relation":"imports","specifier":"pathlib","file":"tools/__init__.py"},{"source":"tools/__init__.py","relation":"imports","specifier":"typing","file":"tools/__init__.py"},{"source":"tools/__init__.py","relation":"imports","specifier":".final_answer","file":"tools/__init__.py"},{"source":"tools/__init__.py","relation":"imports","specifier":".knowledge","file":"tools/__init__.py"},{"source":"tools/__init__.py","relation":"imports","specifier":".run_sql","file":"tools/__init__.py"},{"source":"tools/__init__.py","relation":"imports","specifier":".web_fetch","file":"tools/__init__.py"},{"source":"tools/__init__.py","relation":"imports","specifier":".web_search","file":"tools/__init__.py"},{"source":"tools/__init__.py","relation":"calls","name":"with_name","viaMember":true,"file":"tools/__init__.py"},{"source":"tools/__init__.py","relation":"calls","name":"Path","viaMember":false,"file":"tools/__init__.py"},{"source":"tools/__init__.py","relation":"contains","targetId":"tools/__init__.py#load_schemas","file":"tools/__init__.py"},{"source":"tools/__init__.py#load_schemas","relation":"calls","name":"loads","viaMember":true,"file":"tools/__init__.py"},{"source":"tools/__init__.py#load_schemas","relation":"calls","name":"read_text","viaMember":true,"file":"tools/__init__.py"},{"source":"tools/__init__.py","relation":"contains","targetId":"tools/__init__.py#openai_tools","file":"tools/__init__.py"},{"source":"tools/__init__.py#openai_tools","relation":"calls","name":"load_schemas","viaMember":false,"file":"tools/__init__.py"},{"source":"tools/__init__.py","relation":"contains","targetId":"tools/__init__.py#build_toolset","file":"tools/__init__.py"},{"source":"tools/__init__.py#build_toolset","relation":"calls","name":"make_web_search","viaMember":false,"file":"tools/__init__.py"},{"source":"tools/__init__.py#build_toolset","relation":"calls","name":"make_web_fetch","viaMember":false,"file":"tools/__init__.py"},{"source":"tools/__init__.py#build_toolset","relation":"calls","name":"make_run_sql","viaMember":false,"file":"tools/__init__.py"},{"source":"tools/__init__.py#build_toolset","relation":"calls","name":"make_final_answer","viaMember":false,"file":"tools/__init__.py"},{"source":"tools/__init__.py#build_toolset","relation":"calls","name":"update","viaMember":true,"file":"tools/__init__.py"},{"source":"tools/__init__.py#build_toolset","relation":"calls","name":"make_knowledge_tools","viaMember":false,"file":"tools/__init__.py"}]},"tools/final_answer.py":{"size":334,"mtimeMs":1789189526394.8708,"hash":"c657920345dd9d4167f8954462484be9ff2bea42b1b02d2cf278ce5c11239c38","nodes":[{"id":"tools/final_answer.py","name":"final_answer.py","kind":"file","path":"tools/final_answer.py","span":"L1-L15","signature":null,"exported":true,"origin":"ast","body_hash":"c657920345dd9d4167f8954462484be9ff2bea42b1b02d2cf278ce5c11239c38","chars":334,"summary_state":"pending","summary":null,"crux":null,"body_text":"\"\"\"``final_answer`` tool. The agent loop should treat a successful call as stop.\"\"\" from __future__ import annotations from typing import Callable FINAL_ANSWER_TOOL_NAME = \"final_answer\""},{"id":"tools/final_answer.py#make_final_answer","name":"make_final_answer","kind":"function","path":"tools/final_answer.py","span":"L10-L14","signature":"def make_final_answer() -> Callable[..., str]","exported":true,"origin":"ast","body_hash":"b86cf1a61090006409aaf75dd92aa7936e456e06738d79e77e592e40f0d8b32b","body_text":"def make_final_answer() -> Callable[..., str]: def final_answer(answer: str) -> str: return answer or \"\" return final_answer","summary_state":"pending","summary":null,"crux":null},{"id":"tools/final_answer.py#make_final_answer.final_answer","name":"final_answer","kind":"function","path":"tools/final_answer.py","span":"L11-L12","signature":"def final_answer(answer: str) -> str","exported":true,"origin":"ast","body_hash":"0f7d02f6cc5f31ecc8bd3b99ffac95be36a9eceae1613ec355df1449acf6945c","body_text":"def final_answer(answer: str) -> str: return answer or \"\"","summary_state":"pending","summary":null,"crux":null}],"rawEdges":[{"source":"tools/final_answer.py","relation":"imports","specifier":"typing","file":"tools/final_answer.py"},{"source":"tools/final_answer.py","relation":"contains","targetId":"tools/final_answer.py#make_final_answer","file":"tools/final_answer.py"},{"source":"tools/final_answer.py#make_final_answer","relation":"contains","targetId":"tools/final_answer.py#make_final_answer.final_answer","file":"tools/final_answer.py"}]},"tools/knowledge.py":{"size":3783,"mtimeMs":1789189526394.8708,"hash":"39ac6096649aa70a1e98983b27f06ed0edef92765c330682b61e503c9470b4b6","nodes":[{"id":"tools/knowledge.py","name":"knowledge.py","kind":"file","path":"tools/knowledge.py","span":"L1-L115","signature":null,"exported":true,"origin":"ast","body_hash":"39ac6096649aa70a1e98983b27f06ed0edef92765c330682b61e503c9470b4b6","chars":3783,"summary_state":"pending","summary":null,"crux":null,"body_text":"\"\"\"External-knowledge tools backed by ``<sqlite_root>/<db>/<db>_kb.jsonl``.\"\"\" from __future__ import annotations import json from functools import lru_cache from pathlib import Path from typing import Any, Callable _MAX_FULL_CATALOG_CHARS = 12_000 @lru_cache(maxsize=64)"},{"id":"tools/knowledge.py#_kb_path","name":"_kb_path","kind":"function","path":"tools/knowledge.py","span":"L13-L17","signature":"def _kb_path(sqlite_root: Path, db: str) -> Path","exported":false,"origin":"ast","body_hash":"44f9a539c3db434267556b4b4bd3dae4eb4e631e20abd2d3d36cfd6273527bab","body_text":"def _kb_path(sqlite_root: Path, db: str) -> Path: name = (db or \"\").strip().lower() if not name.replace(\"_\", \"\").isalnum(): raise ValueError(f\"invalid database name: {db!r}\") return sqlite_root.expanduser().resolve() / name / f\"{name}_kb.jsonl\"","summary_state":"pending","summary":null,"crux":null},{"id":"tools/knowledge.py#_load_entries","name":"_load_entries","kind":"function","path":"tools/knowledge.py","span":"L21-L30","signature":"def _load_entries(sqlite_root_str: str, db: str) -> tuple[dict[str, Any], ...]","exported":false,"origin":"ast","body_hash":"5a6c11a68864640df259af12eb57e2ca70d98983a5d54ea3131e07f3733ef6de","body_text":"def _load_entries(sqlite_root_str: str, db: str) -> tuple[dict[str, Any], ...]: path = _kb_path(Path(sqlite_root_str), db) if not path.is_file(): raise FileNotFoundError(f\"Knowledge file not found: {path}\") entries: list[dict[str, Any]] = [] with path.open(encoding=\"utf-8\") as handle: for line in handle: if line.strip(): entries.append(json.loads(line)) return tuple(entries)","summary_state":"pending","summary":null,"crux":null},{"id":"tools/knowledge.py#_numeric_id","name":"_numeric_id","kind":"function","path":"tools/knowledge.py","span":"L33-L42","signature":"def _numeric_id(key: str, db: str) -> int | None","exported":false,"origin":"ast","body_hash":"097682954d6737f0a4530f21a4c2433a80a8f5e071904e1a7dd7c04cc2b20a42","body_text":"def _numeric_id(key: str, db: str) -> int | None: token = (key or \"\").strip() if token.isdigit(): return int(token) prefix = f\"{db.lower()}_\" if token.lower().startswith(prefix): suffix = token[len(prefix) :] if suffix.isdigit(): return int(suffix) return None","summary_state":"pending","summary":null,"crux":null},{"id":"tools/knowledge.py#_matches","name":"_matches","kind":"function","path":"tools/knowledge.py","span":"L45-L58","signature":"def _matches(entry: dict[str, Any], key: str, db: str) -> bool","exported":false,"origin":"ast","body_hash":"320e539a7a28e69c273e9c5a0c7ecaa5d987a7b6a4f730b16612b0743534eea1","body_text":"def _matches(entry: dict[str, Any], key: str, db: str) -> bool: token = (key or \"\").strip() if not token: return False eid = entry.get(\"id\") if eid is not None and str(eid) == token: return True num = _numeric_id(token, db) if num is not None and eid == num: return True if eid is not None and token.lower() == f\"{db.lower()}_{eid}\".lower(): return True title = str(entry.get(\"knowledge\", \"\")).strip() return bool(title) and title.lower() == token.lower()","summary_state":"pending","summary":null,"crux":null},{"id":"tools/knowledge.py#make_knowledge_tools","name":"make_knowledge_tools","kind":"function","path":"tools/knowledge.py","span":"L61-L114","signature":"def make_knowledge_tools( sqlite_root: str | Path, db: str, ) -> dict[str, Callable[..., str]]","exported":true,"origin":"ast","body_hash":"903d2c5323598ecb4f8dab3f85551a949c66c696e3012beb604d7e9705961be0","body_text":"def make_knowledge_tools( sqlite_root: str | Path, db: str, ) -> dict[str, Callable[..., str]]: \"\"\"Return the three knowledge-catalog callables for one database.\"\"\" root = Path(sqlite_root).expanduser().resolve() entries = _load_entries(str(root), db) def get_all_external_knowledge_names() -> str: names = [ str(entry.get(\"knowledge\", \"\")).strip() for entry in entries if entry.get(\"knowledge\") ] return json.dumps(names, ensure_ascii=False) def get_knowledge_definition(knowledge_name: str) -> str: key = (knowledge_name or \"\").strip() if not key: return \"[ERROR]: knowledge_name is required.\" for entry in entries: if _matches(entry, key, db): return json.dumps(entry, ensure_ascii=False, indent=2) return f\"Knowledge not found for {key!r} in database {db!r}.\" def get_all_knowledge_definitions() -> str: full = json.dumps(list(entries), ensure_ascii=False, indent=2) if len(full) <= _MAX_FULL_CATALOG_CHARS: return full names = [ str(entry.get(\"knowledge\", \"\")).strip() for entry in entries if entry.get(\"knowledge\") ] return json.dumps( { \"_notice\": ( f\"Full catalog is {len(full)} chars; returning titles only. \" \"Call get_knowledge_definition for each metric you need.\" ), \"database\": db, \"entry_count\": len(entries), \"knowledge_names\": names, }, ensure_ascii=False, indent=2, ) return { \"get_all_external_knowledge_names\": get_all_external_knowledge_names, \"get_knowledge_definition\": get_knowledge_definition, \"get_all_knowledge_definitions\": get_all_knowledge_definitions, }","summary_state":"pending","summary":null,"crux":null},{"id":"tools/knowledge.py#make_knowledge_tools.get_all_external_knowledge_names","name":"get_all_external_knowledge_names","kind":"function","path":"tools/knowledge.py","span":"L70-L76","signature":"def get_all_external_knowledge_names() -> str","exported":true,"origin":"ast","body_hash":"f07daccfd1762da555fe982ae8b4dc428f670dad7341ef3dd04d9c7e227d6439","body_text":"def get_all_external_knowledge_names() -> str: names = [ str(entry.get(\"knowledge\", \"\")).strip() for entry in entries if entry.get(\"knowledge\") ] return json.dumps(names, ensure_ascii=False)","summary_state":"pending","summary":null,"crux":null},{"id":"tools/knowledge.py#make_knowledge_tools.get_knowledge_definition","name":"get_knowledge_definition","kind":"function","path":"tools/knowledge.py","span":"L78-L85","signature":"def get_knowledge_definition(knowledge_name: str) -> str","exported":true,"origin":"ast","body_hash":"861110c2cf7d829a7d3bbecb8b90c28fedb244dbe7ce3d2709718ecb6a344854","body_text":"def get_knowledge_definition(knowledge_name: str) -> str: key = (knowledge_name or \"\").strip() if not key: return \"[ERROR]: knowledge_name is required.\" for entry in entries: if _matches(entry, key, db): return json.dumps(entry, ensure_ascii=False, indent=2) return f\"Knowledge not found for {key!r} in database {db!r}.\"","summary_state":"pending","summary":null,"crux":null},{"id":"tools/knowledge.py#make_knowledge_tools.get_all_knowledge_definitions","name":"get_all_knowledge_definitions","kind":"function","path":"tools/knowledge.py","span":"L87-L108","signature":"def get_all_knowledge_definitions() -> str","exported":true,"origin":"ast","body_hash":"bc3c33c2a3cb0480f514b69f63963f16dcd5d24a116dc66d456bd0d05704b896","body_text":"def get_all_knowledge_definitions() -> str: full = json.dumps(list(entries), ensure_ascii=False, indent=2) if len(full) <= _MAX_FULL_CATALOG_CHARS: return full names = [ str(entry.get(\"knowledge\", \"\")).strip() for entry in entries if entry.get(\"knowledge\") ] return json.dumps( { \"_notice\": ( f\"Full catalog is {len(full)} chars; returning titles only. \" \"Call get_knowledge_definition for each metric you need.\" ), \"database\": db, \"entry_count\": len(entries), \"knowledge_names\": names, }, ensure_ascii=False, indent=2, )","summary_state":"pending","summary":null,"crux":null}],"rawEdges":[{"source":"tools/knowledge.py","relation":"imports","specifier":"json","file":"tools/knowledge.py"},{"source":"tools/knowledge.py","relation":"imports","specifier":"functools","file":"tools/knowledge.py"},{"source":"tools/knowledge.py","relation":"imports","specifier":"pathlib","file":"tools/knowledge.py"},{"source":"tools/knowledge.py","relation":"imports","specifier":"typing","file":"tools/knowledge.py"},{"source":"tools/knowledge.py","relation":"contains","targetId":"tools/knowledge.py#_kb_path","file":"tools/knowledge.py"},{"source":"tools/knowledge.py#_kb_path","relation":"calls","name":"lower","viaMember":true,"file":"tools/knowledge.py"},{"source":"tools/knowledge.py#_kb_path","relation":"calls","name":"strip","viaMember":true,"file":"tools/knowledge.py"},{"source":"tools/knowledge.py#_kb_path","relation":"calls","name":"isalnum","viaMember":true,"file":"tools/knowledge.py"},{"source":"tools/knowledge.py#_kb_path","relation":"calls","name":"replace","viaMember":true,"file":"tools/knowledge.py"},{"source":"tools/knowledge.py#_kb_path","relation":"calls","name":"ValueError","viaMember":false,"file":"tools/knowledge.py"},{"source":"tools/knowledge.py#_kb_path","relation":"calls","name":"resolve","viaMember":true,"file":"tools/knowledge.py"},{"source":"tools/knowledge.py#_kb_path","relation":"calls","name":"expanduser","viaMember":true,"file":"tools/knowledge.py","recvType":"Path"},{"source":"tools/knowledge.py","relation":"calls","name":"lru_cache","viaMember":false,"file":"tools/knowledge.py"},{"source":"tools/knowledge.py","relation":"contains","targetId":"tools/knowledge.py#_load_entries","file":"tools/knowledge.py"},{"source":"tools/knowledge.py#_load_entries","relation":"calls","name":"_kb_path","viaMember":false,"file":"tools/knowledge.py"},{"source":"tools/knowledge.py#_load_entries","relation":"calls","name":"Path","viaMember":false,"file":"tools/knowledge.py"},{"source":"tools/knowledge.py#_load_entries","relation":"calls","name":"is_file","viaMember":true,"file":"tools/knowledge.py","recvType":"_kb_path"},{"source":"tools/knowledge.py#_load_entries","relation":"calls","name":"FileNotFoundError","viaMember":false,"file":"tools/knowledge.py"},{"source":"tools/knowledge.py#_load_entries","relation":"calls","name":"open","viaMember":true,"file":"tools/knowledge.py","recvType":"_kb_path"},{"source":"tools/knowledge.py#_load_entries","relation":"calls","name":"strip","viaMember":true,"file":"tools/knowledge.py"},{"source":"tools/knowledge.py#_load_entries","relation":"calls","name":"append","viaMember":true,"file":"tools/knowledge.py"},{"source":"tools/knowledge.py#_load_entries","relation":"calls","name":"loads","viaMember":true,"file":"tools/knowledge.py"},{"source":"tools/knowledge.py#_load_entries","relation":"calls","name":"tuple","viaMember":false,"file":"tools/knowledge.py"},{"source":"tools/knowledge.py","relation":"contains","targetId":"tools/knowledge.py#_numeric_id","file":"tools/knowledge.py"},{"source":"tools/knowledge.py#_numeric_id","relation":"calls","name":"strip","viaMember":true,"file":"tools/knowledge.py"},{"source":"tools/knowledge.py#_numeric_id","relation":"calls","name":"isdigit","viaMember":true,"file":"tools/knowledge.py"},{"source":"tools/knowledge.py#_numeric_id","relation":"calls","name":"int","viaMember":false,"file":"tools/knowledge.py"},{"source":"tools/knowledge.py#_numeric_id","relation":"calls","name":"lower","viaMember":true,"file":"tools/knowledge.py","recvType":"str"},{"source":"tools/knowledge.py#_numeric_id","relation":"calls","name":"startswith","viaMember":true,"file":"tools/knowledge.py"},{"source":"tools/knowledge.py#_numeric_id","relation":"calls","name":"lower","viaMember":true,"file":"tools/knowledge.py"},{"source":"tools/knowledge.py#_numeric_id","relation":"calls","name":"len","viaMember":false,"file":"tools/knowledge.py"},{"source":"tools/knowledge.py#_numeric_id","relation":"calls","name":"isdigit","viaMember":true,"file":"tools/knowledge.py"},{"source":"tools/knowledge.py#_numeric_id","relation":"calls","name":"int","viaMember":false,"file":"tools/knowledge.py"},{"source":"tools/knowledge.py","relation":"contains","targetId":"tools/knowledge.py#_matches","file":"tools/knowledge.py"},{"source":"tools/knowledge.py#_matches","relation":"calls","name":"strip","viaMember":true,"file":"tools/knowledge.py"},{"source":"tools/knowledge.py#_matches","relation":"calls","name":"get","viaMember":true,"file":"tools/knowledge.py"},{"source":"tools/knowledge.py#_matches","relation":"calls","name":"str","viaMember":false,"file":"tools/knowledge.py"},{"source":"tools/knowledge.py#_matches","relation":"calls","name":"_numeric_id","viaMember":false,"file":"tools/knowledge.py"},{"source":"tools/knowledge.py#_matches","relation":"calls","name":"lower","viaMember":true,"file":"tools/knowledge.py"},{"source":"tools/knowledge.py#_matches","relation":"calls","name":"lower","viaMember":true,"file":"tools/knowledge.py"},{"source":"tools/knowledge.py#_matches","relation":"calls","name":"lower","viaMember":true,"file":"tools/knowledge.py","recvType":"str"},{"source":"tools/knowledge.py#_matches","relation":"calls","name":"strip","viaMember":true,"file":"tools/knowledge.py"},{"source":"tools/knowledge.py#_matches","relation":"calls","name":"str","viaMember":false,"file":"tools/knowledge.py"},{"source":"tools/knowledge.py#_matches","relation":"calls","name":"get","viaMember":true,"file":"tools/knowledge.py"},{"source":"tools/knowledge.py#_matches","relation":"calls","name":"bool","viaMember":false,"file":"tools/knowledge.py"},{"source":"tools/knowledge.py#_matches","relation":"calls","name":"lower","viaMember":true,"file":"tools/knowledge.py"},{"source":"tools/knowledge.py#_matches","relation":"calls","name":"lower","viaMember":true,"file":"tools/knowledge.py"},{"source":"tools/knowledge.py","relation":"contains","targetId":"tools/knowledge.py#make_knowledge_tools","file":"tools/knowledge.py"},{"source":"tools/knowledge.py#make_knowledge_tools","relation":"calls","name":"resolve","viaMember":true,"file":"tools/knowledge.py"},{"source":"tools/knowledge.py#make_knowledge_tools","relation":"calls","name":"expanduser","viaMember":true,"file":"tools/knowledge.py"},{"source":"tools/knowledge.py#make_knowledge_tools","relation":"calls","name":"Path","viaMember":false,"file":"tools/knowledge.py"},{"source":"tools/knowledge.py#make_knowledge_tools","relation":"calls","name":"_load_entries","viaMember":false,"file":"tools/knowledge.py"},{"source":"tools/knowledge.py#make_knowledge_tools","relation":"calls","name":"str","viaMember":false,"file":"tools/knowledge.py"},{"source":"tools/knowledge.py#make_knowledge_tools","relation":"contains","targetId":"tools/knowledge.py#make_knowledge_tools.get_all_external_knowledge_names","file":"tools/knowledge.py"},{"source":"tools/knowledge.py#make_knowledge_tools.get_all_external_knowledge_names","relation":"calls","name":"strip","viaMember":true,"file":"tools/knowledge.py"},{"source":"tools/knowledge.py#make_knowledge_tools.get_all_external_knowledge_names","relation":"calls","name":"str","viaMember":false,"file":"tools/knowledge.py"},{"source":"tools/knowledge.py#make_knowledge_tools.get_all_external_knowledge_names","relation":"calls","name":"get","viaMember":true,"file":"tools/knowledge.py"},{"source":"tools/knowledge.py#make_knowledge_tools.get_all_external_knowledge_names","relation":"calls","name":"get","viaMember":true,"file":"tools/knowledge.py"},{"source":"tools/knowledge.py#make_knowledge_tools.get_all_external_knowledge_names","relation":"calls","name":"dumps","viaMember":true,"file":"tools/knowledge.py"},{"source":"tools/knowledge.py#make_knowledge_tools","relation":"contains","targetId":"tools/knowledge.py#make_knowledge_tools.get_knowledge_definition","file":"tools/knowledge.py"},{"source":"tools/knowledge.py#make_knowledge_tools.get_knowledge_definition","relation":"calls","name":"strip","viaMember":true,"file":"tools/knowledge.py"},{"source":"tools/knowledge.py#make_knowledge_tools.get_knowledge_definition","relation":"calls","name":"_matches","viaMember":false,"file":"tools/knowledge.py"},{"source":"tools/knowledge.py#make_knowledge_tools.get_knowledge_definition","relation":"calls","name":"dumps","viaMember":true,"file":"tools/knowledge.py"},{"source":"tools/knowledge.py#make_knowledge_tools","relation":"contains","targetId":"tools/knowledge.py#make_knowledge_tools.get_all_knowledge_definitions","file":"tools/knowledge.py"},{"source":"tools/knowledge.py#make_knowledge_tools.get_all_knowledge_definitions","relation":"calls","name":"dumps","viaMember":true,"file":"tools/knowledge.py"},{"source":"tools/knowledge.py#make_knowledge_tools.get_all_knowledge_definitions","relation":"calls","name":"list","viaMember":false,"file":"tools/knowledge.py"},{"source":"tools/knowledge.py#make_knowledge_tools.get_all_knowledge_definitions","relation":"calls","name":"len","viaMember":false,"file":"tools/knowledge.py"},{"source":"tools/knowledge.py#make_knowledge_tools.get_all_knowledge_definitions","relation":"calls","name":"strip","viaMember":true,"file":"tools/knowledge.py"},{"source":"tools/knowledge.py#make_knowledge_tools.get_all_knowledge_definitions","relation":"calls","name":"str","viaMember":false,"file":"tools/knowledge.py"},{"source":"tools/knowledge.py#make_knowledge_tools.get_all_knowledge_definitions","relation":"calls","name":"get","viaMember":true,"file":"tools/knowledge.py"},{"source":"tools/knowledge.py#make_knowledge_tools.get_all_knowledge_definitions","relation":"calls","name":"get","viaMember":true,"file":"tools/knowledge.py"},{"source":"tools/knowledge.py#make_knowledge_tools.get_all_knowledge_definitions","relation":"calls","name":"dumps","viaMember":true,"file":"tools/knowledge.py"},{"source":"tools/knowledge.py#make_knowledge_tools.get_all_knowledge_definitions","relation":"calls","name":"len","viaMember":false,"file":"tools/knowledge.py"},{"source":"tools/knowledge.py#make_knowledge_tools.get_all_knowledge_definitions","relation":"calls","name":"len","viaMember":false,"file":"tools/knowledge.py"}]},"tools/run_sql.py":{"size":1294,"mtimeMs":1789189526394.8708,"hash":"2bbcce87115cfbd63722d5496ec96c545fec84443ce7872b5f1aef3d3619db5e","nodes":[{"id":"tools/run_sql.py","name":"run_sql.py","kind":"file","path":"tools/run_sql.py","span":"L1-L45","signature":null,"exported":true,"origin":"ast","body_hash":"2bbcce87115cfbd63722d5496ec96c545fec84443ce7872b5f1aef3d3619db5e","chars":1294,"summary_state":"pending","summary":null,"crux":null,"body_text":"\"\"\"Read-only ``run_sql`` tool over a per-task SQLite file.\"\"\" from __future__ import annotations import sys from pathlib import Path from typing import Any, Callable _ROOT = Path(__file__).resolve().parents[1] if str(_ROOT) not in sys.path: sys.path.insert(0, str(_ROOT)) from evaluation.sqlite_executor import execute_sqlite_query"},{"id":"tools/run_sql.py#make_run_sql","name":"make_run_sql","kind":"function","path":"tools/run_sql.py","span":"L16-L44","signature":"def make_run_sql( sqlite_db_path: str | Path | None, *, max_rows: int = 500, timeout_s: float | None = 120.0, ) -> Callable[..., str]","exported":true,"origin":"ast","body_hash":"3e11aa7b1076fa227232e3f3aa2f7be95c179eb20850c3c8907bc03420cbb4a6","body_text":"def make_run_sql( sqlite_db_path: str | Path | None, *, max_rows: int = 500, timeout_s: float | None = 120.0, ) -> Callable[..., str]: \"\"\"Return a ``run_sql(sql, description=\"\")`` callable.\"\"\" db_path = ( Path(sqlite_db_path).expanduser().resolve() if sqlite_db_path is not None else None ) def run_sql(sql: str, description: str = \"\") -> str: sql_text = (sql or \"\").strip() if not sql_text: return \"Error: empty sql query.\" if db_path is None: return \"Error: run_sql is not configured (no sqlite database for this run).\" text, meta = execute_sqlite_query( db_path, sql_text, max_rows=max_rows, timeout_s=timeout_s ) if not meta.get(\"ok\"): return text header = f\"{(description or '').strip()}\\n\\n\" if (description or \"\").strip() else \"\" return f\"{header}{text}\" return run_sql","summary_state":"pending","summary":null,"crux":null},{"id":"tools/run_sql.py#make_run_sql.run_sql","name":"run_sql","kind":"function","path":"tools/run_sql.py","span":"L30-L42","signature":"def run_sql(sql: str, description: str = \"\") -> str","exported":true,"origin":"ast","body_hash":"cb4950b5f71b1afe74e8d33a9fd91722e7f9e385f5e6358b63b260707f5cafa6","body_text":"def run_sql(sql: str, description: str = \"\") -> str: sql_text = (sql or \"\").strip() if not sql_text: return \"Error: empty sql query.\" if db_path is None: return \"Error: run_sql is not configured (no sqlite database for this run).\" text, meta = execute_sqlite_query( db_path, sql_text, max_rows=max_rows, timeout_s=timeout_s ) if not meta.get(\"ok\"): return text header = f\"{(description or '').strip()}\\n\\n\" if (description or \"\").strip() else \"\" return f\"{header}{text}\"","summary_state":"pending","summary":null,"crux":null}],"rawEdges":[{"source":"tools/run_sql.py","relation":"imports","specifier":"sys","file":"tools/run_sql.py"},{"source":"tools/run_sql.py","relation":"imports","specifier":"pathlib","file":"tools/run_sql.py"},{"source":"tools/run_sql.py","relation":"imports","specifier":"typing","file":"tools/run_sql.py"},{"source":"tools/run_sql.py","relation":"calls","name":"resolve","viaMember":true,"file":"tools/run_sql.py"},{"source":"tools/run_sql.py","relation":"calls","name":"Path","viaMember":false,"file":"tools/run_sql.py"},{"source":"tools/run_sql.py","relation":"calls","name":"str","viaMember":false,"file":"tools/run_sql.py"},{"source":"tools/run_sql.py","relation":"calls","name":"insert","viaMember":true,"file":"tools/run_sql.py"},{"source":"tools/run_sql.py","relation":"calls","name":"str","viaMember":false,"file":"tools/run_sql.py"},{"source":"tools/run_sql.py","relation":"imports","specifier":"evaluation.sqlite_executor","file":"tools/run_sql.py"},{"source":"tools/run_sql.py","relation":"contains","targetId":"tools/run_sql.py#make_run_sql","file":"tools/run_sql.py"},{"source":"tools/run_sql.py#make_run_sql","relation":"calls","name":"resolve","viaMember":true,"file":"tools/run_sql.py"},{"source":"tools/run_sql.py#make_run_sql","relation":"calls","name":"expanduser","viaMember":true,"file":"tools/run_sql.py"},{"source":"tools/run_sql.py#make_run_sql","relation":"calls","name":"Path","viaMember":false,"file":"tools/run_sql.py"},{"source":"tools/run_sql.py#make_run_sql","relation":"contains","targetId":"tools/run_sql.py#make_run_sql.run_sql","file":"tools/run_sql.py"},{"source":"tools/run_sql.py#make_run_sql.run_sql","relation":"calls","name":"strip","viaMember":true,"file":"tools/run_sql.py"},{"source":"tools/run_sql.py#make_run_sql.run_sql","relation":"calls","name":"execute_sqlite_query","viaMember":false,"file":"tools/run_sql.py"},{"source":"tools/run_sql.py#make_run_sql.run_sql","relation":"calls","name":"get","viaMember":true,"file":"tools/run_sql.py"},{"source":"tools/run_sql.py#make_run_sql.run_sql","relation":"calls","name":"strip","viaMember":true,"file":"tools/run_sql.py"},{"source":"tools/run_sql.py#make_run_sql.run_sql","relation":"calls","name":"strip","viaMember":true,"file":"tools/run_sql.py"}]},"tools/web_fetch.py":{"size":1401,"mtimeMs":1789189526395.8708,"hash":"14d3d6c430e9b014ad9eb54f95a0756d9db981ae423415314feafec73aeb83fb","nodes":[{"id":"tools/web_fetch.py","name":"web_fetch.py","kind":"file","path":"tools/web_fetch.py","span":"L1-L48","signature":null,"exported":true,"origin":"ast","body_hash":"14d3d6c430e9b014ad9eb54f95a0756d9db981ae423415314feafec73aeb83fb","chars":1401,"summary_state":"pending","summary":null,"crux":null,"body_text":"\"\"\"``web_fetch`` tool. Reference fetch; swap in a renderer if you need one.\"\"\" from __future__ import annotations import html import re import urllib.request from typing import Callable _TAG = re.compile(r\"<script[\\s\\S]*?</script>|<style[\\s\\S]*?</style>|<[^>]+>\", re.I) _WS = re.compile(r\"\\n{3,}\") MAX_CHARS = 20_000"},{"id":"tools/web_fetch.py#fetch_url","name":"fetch_url","kind":"function","path":"tools/web_fetch.py","span":"L15-L33","signature":"def fetch_url(url: str) -> str","exported":true,"origin":"ast","body_hash":"87931388092a9a3eb7f7376fb616698fe570909e4df1017daf423703b09408c7","body_text":"def fetch_url(url: str) -> str: address = (url or \"\").strip() if not address: return \"Error: empty url.\" request = urllib.request.Request( address, headers={\"User-Agent\": \"HybridDeepResearch-public-fetch/1.0\"}, method=\"GET\", ) with urllib.request.urlopen(request, timeout=30) as response: raw = response.read() content_type = response.headers.get(\"Content-Type\", \"\") text = raw.decode(\"utf-8\", errors=\"replace\") if \"html\" in content_type.lower() or \"<html\" in text[:400].lower(): text = html.unescape(_TAG.sub(\" \", text)) text = _WS.sub(\"\\n\\n\", text).strip() if len(text) > MAX_CHARS: text = text[: MAX_CHARS - 3] + \"...\" return text or \"(empty page)\"","summary_state":"pending","summary":null,"crux":null},{"id":"tools/web_fetch.py#make_web_fetch","name":"make_web_fetch","kind":"function","path":"tools/web_fetch.py","span":"L36-L47","signature":"def make_web_fetch( fetch_fn: Callable[[str], str] | None = None, ) -> Callable[..., str]","exported":true,"origin":"ast","body_hash":"9a8960ff85cb8ef25b29759017af716c2b7bfe1cb5fc5a4a7c8031a3604508d0","body_text":"def make_web_fetch( fetch_fn: Callable[[str], str] | None = None, ) -> Callable[..., str]: backend = fetch_fn or fetch_url def web_fetch(url: str) -> str: try: return backend(url) except Exception as exc: return f\"Error: web_fetch failed: {exc}\" return web_fetch","summary_state":"pending","summary":null,"crux":null},{"id":"tools/web_fetch.py#make_web_fetch.web_fetch","name":"web_fetch","kind":"function","path":"tools/web_fetch.py","span":"L41-L45","signature":"def web_fetch(url: str) -> str","exported":true,"origin":"ast","body_hash":"1135be70124e36938aafac8c7fb74000ec391ef08fd4cc3b38ff888f5b602baf","body_text":"def web_fetch(url: str) -> str: try: return backend(url) except Exception as exc: return f\"Error: web_fetch failed: {exc}\"","summary_state":"pending","summary":null,"crux":null}],"rawEdges":[{"source":"tools/web_fetch.py","relation":"imports","specifier":"html","file":"tools/web_fetch.py"},{"source":"tools/web_fetch.py","relation":"imports","specifier":"re","file":"tools/web_fetch.py"},{"source":"tools/web_fetch.py","relation":"imports","specifier":"urllib.request","file":"tools/web_fetch.py"},{"source":"tools/web_fetch.py","relation":"imports","specifier":"typing","file":"tools/web_fetch.py"},{"source":"tools/web_fetch.py","relation":"calls","name":"compile","viaMember":true,"file":"tools/web_fetch.py"},{"source":"tools/web_fetch.py","relation":"calls","name":"compile","viaMember":true,"file":"tools/web_fetch.py"},{"source":"tools/web_fetch.py","relation":"contains","targetId":"tools/web_fetch.py#fetch_url","file":"tools/web_fetch.py"},{"source":"tools/web_fetch.py#fetch_url","relation":"calls","name":"strip","viaMember":true,"file":"tools/web_fetch.py"},{"source":"tools/web_fetch.py#fetch_url","relation":"calls","name":"Request","viaMember":true,"file":"tools/web_fetch.py"},{"source":"tools/web_fetch.py#fetch_url","relation":"calls","name":"urlopen","viaMember":true,"file":"tools/web_fetch.py"},{"source":"tools/web_fetch.py#fetch_url","relation":"calls","name":"read","viaMember":true,"file":"tools/web_fetch.py"},{"source":"tools/web_fetch.py#fetch_url","relation":"calls","name":"get","viaMember":true,"file":"tools/web_fetch.py"},{"source":"tools/web_fetch.py#fetch_url","relation":"calls","name":"decode","viaMember":true,"file":"tools/web_fetch.py"},{"source":"tools/web_fetch.py#fetch_url","relation":"calls","name":"lower","viaMember":true,"file":"tools/web_fetch.py"},{"source":"tools/web_fetch.py#fetch_url","relation":"calls","name":"lower","viaMember":true,"file":"tools/web_fetch.py"},{"source":"tools/web_fetch.py#fetch_url","relation":"calls","name":"unescape","viaMember":true,"file":"tools/web_fetch.py"},{"source":"tools/web_fetch.py#fetch_url","relation":"calls","name":"sub","viaMember":true,"file":"tools/web_fetch.py"},{"source":"tools/web_fetch.py#fetch_url","relation":"calls","name":"strip","viaMember":true,"file":"tools/web_fetch.py"},{"source":"tools/web_fetch.py#fetch_url","relation":"calls","name":"sub","viaMember":true,"file":"tools/web_fetch.py"},{"source":"tools/web_fetch.py#fetch_url","relation":"calls","name":"len","viaMember":false,"file":"tools/web_fetch.py"},{"source":"tools/web_fetch.py","relation":"contains","targetId":"tools/web_fetch.py#make_web_fetch","file":"tools/web_fetch.py"},{"source":"tools/web_fetch.py#make_web_fetch","relation":"contains","targetId":"tools/web_fetch.py#make_web_fetch.web_fetch","file":"tools/web_fetch.py"},{"source":"tools/web_fetch.py#make_web_fetch.web_fetch","relation":"calls","name":"backend","viaMember":false,"file":"tools/web_fetch.py"}]},"tools/web_search.py":{"size":2580,"mtimeMs":1789189526395.8708,"hash":"f1d5ff6fb008d06a3fe75e6fd0344cfe57bbc2850271486ba29c264dcad13170","nodes":[{"id":"tools/web_search.py","name":"web_search.py","kind":"file","path":"tools/web_search.py","span":"L1-L84","signature":null,"exported":true,"origin":"ast","body_hash":"f1d5ff6fb008d06a3fe75e6fd0344cfe57bbc2850271486ba29c264dcad13170","chars":2580,"summary_state":"pending","summary":null,"crux":null,"body_text":"\"\"\"``web_search`` tool. Attach your own search backend; names and arguments are fixed.\"\"\" from __future__ import annotations import json import os import urllib.parse import urllib.request from typing import Any, Callable DEFAULT_COUNT = 10 MAX_COUNT = 20"},{"id":"tools/web_search.py#format_hits","name":"format_hits","kind":"function","path":"tools/web_search.py","span":"L15-L25","signature":"def format_hits(hits: list[dict[str, str]], source: str) -> str","exported":true,"origin":"ast","body_hash":"6a9b98866a14206d43271368260c86e56d9a2afd721b4afba35d0bcfe1417fd6","body_text":"def format_hits(hits: list[dict[str, str]], source: str) -> str: if not hits: return f\"source={source}\\n\\n(no results)\" lines = [f\"source={source}\", \"\"] for index, hit in enumerate(hits, 1): lines.append( f\"{index}. {hit.get('title', '')}\\n\" f\" {hit.get('url', '')}\\n\" f\" {hit.get('description', '')}\\n\" ) return \"\\n\".join(lines)","summary_state":"pending","summary":null,"crux":null},{"id":"tools/web_search.py#tavily_search","name":"tavily_search","kind":"function","path":"tools/web_search.py","span":"L28-L57","signature":"def tavily_search(query: str, count: int) -> tuple[list[dict[str, str]], str]","exported":true,"origin":"ast","body_hash":"36cc3f42a799d2c2e00938e9491e0c1f67cc92556d77997ced791945def1413a","body_text":"def tavily_search(query: str, count: int) -> tuple[list[dict[str, str]], str]: \"\"\"Optional Tavily backend. Requires ``TAVILY_API_KEY``.\"\"\" api_key = os.environ.get(\"TAVILY_API_KEY\", \"\").strip() if not api_key: raise RuntimeError(\"TAVILY_API_KEY is not set.\") payload = json.dumps( { \"api_key\": api_key, \"query\": query, \"max_results\": count, \"include_answer\": False, } ).encode(\"utf-8\") request = urllib.request.Request( \"https://api.tavily.com/search\", data=payload, headers={\"Content-Type\": \"application/json\"}, method=\"POST\", ) with urllib.request.urlopen(request, timeout=30) as response: body = json.loads(response.read().decode(\"utf-8\")) hits = [ { \"title\": str(item.get(\"title\") or \"\"), \"url\": str(item.get(\"url\") or \"\"), \"description\": str(item.get(\"content\") or item.get(\"snippet\") or \"\"), } for item in body.get(\"results\") or [] ] return hits, \"tavily\"","summary_state":"pending","summary":null,"crux":null},{"id":"tools/web_search.py#make_web_search","name":"make_web_search","kind":"function","path":"tools/web_search.py","span":"L60-L83","signature":"def make_web_search( search_fn: Callable[[str, int], tuple[list[dict[str, str]], str]] | None = None, ) -> Callable[..., str]","exported":true,"origin":"ast","body_hash":"aacf27472f5be6fb3a25512c60035648ab43fe54f2a004fd2b30a7ef9fec4d01","body_text":"def make_web_search( search_fn: Callable[[str, int], tuple[list[dict[str, str]], str]] | None = None, ) -> Callable[..., str]: \"\"\"Return ``web_search(query, count=10)``. ``search_fn`` should return ``(hits, source_label)``. Each hit needs ``title``, ``url``, and ``description``. If omitted, Tavily is used when ``TAVILY_API_KEY`` is present. \"\"\" backend = search_fn or tavily_search def web_search(query: str, count: int = DEFAULT_COUNT) -> str: q = (query or \"\").strip() if not q: return \"Error: empty query.\" n = max(1, min(MAX_COUNT, int(count))) try: hits, source = backend(q, n) except Exception as exc: return f\"Error: web search failed: {exc}\" return format_hits(hits, source) return web_search","summary_state":"pending","summary":null,"crux":null},{"id":"tools/web_search.py#make_web_search.web_search","name":"web_search","kind":"function","path":"tools/web_search.py","span":"L72-L81","signature":"def web_search(query: str, count: int = DEFAULT_COUNT) -> str","exported":true,"origin":"ast","body_hash":"8bed28eda089a4242d1215d6ba8273b7727db290501be59adfaa59488fe8dd49","body_text":"def web_search(query: str, count: int = DEFAULT_COUNT) -> str: q = (query or \"\").strip() if not q: return \"Error: empty query.\" n = max(1, min(MAX_COUNT, int(count))) try: hits, source = backend(q, n) except Exception as exc: return f\"Error: web search failed: {exc}\" return format_hits(hits, source)","summary_state":"pending","summary":null,"crux":null}],"rawEdges":[{"source":"tools/web_search.py","relation":"imports","specifier":"json","file":"tools/web_search.py"},{"source":"tools/web_search.py","relation":"imports","specifier":"os","file":"tools/web_search.py"},{"source":"tools/web_search.py","relation":"imports","specifier":"urllib.parse","file":"tools/web_search.py"},{"source":"tools/web_search.py","relation":"imports","specifier":"urllib.request","file":"tools/web_search.py"},{"source":"tools/web_search.py","relation":"imports","specifier":"typing","file":"tools/web_search.py"},{"source":"tools/web_search.py","relation":"contains","targetId":"tools/web_search.py#format_hits","file":"tools/web_search.py"},{"source":"tools/web_search.py#format_hits","relation":"calls","name":"enumerate","viaMember":false,"file":"tools/web_search.py"},{"source":"tools/web_search.py#format_hits","relation":"calls","name":"append","viaMember":true,"file":"tools/web_search.py"},{"source":"tools/web_search.py#format_hits","relation":"calls","name":"get","viaMember":true,"file":"tools/web_search.py"},{"source":"tools/web_search.py#format_hits","relation":"calls","name":"get","viaMember":true,"file":"tools/web_search.py"},{"source":"tools/web_search.py#format_hits","relation":"calls","name":"get","viaMember":true,"file":"tools/web_search.py"},{"source":"tools/web_search.py#format_hits","relation":"calls","name":"join","viaMember":true,"file":"tools/web_search.py"},{"source":"tools/web_search.py","relation":"contains","targetId":"tools/web_search.py#tavily_search","file":"tools/web_search.py"},{"source":"tools/web_search.py#tavily_search","relation":"calls","name":"strip","viaMember":true,"file":"tools/web_search.py"},{"source":"tools/web_search.py#tavily_search","relation":"calls","name":"get","viaMember":true,"file":"tools/web_search.py"},{"source":"tools/web_search.py#tavily_search","relation":"calls","name":"RuntimeError","viaMember":false,"file":"tools/web_search.py"},{"source":"tools/web_search.py#tavily_search","relation":"calls","name":"encode","viaMember":true,"file":"tools/web_search.py"},{"source":"tools/web_search.py#tavily_search","relation":"calls","name":"dumps","viaMember":true,"file":"tools/web_search.py"},{"source":"tools/web_search.py#tavily_search","relation":"calls","name":"Request","viaMember":true,"file":"tools/web_search.py"},{"source":"tools/web_search.py#tavily_search","relation":"calls","name":"urlopen","viaMember":true,"file":"tools/web_search.py"},{"source":"tools/web_search.py#tavily_search","relation":"calls","name":"loads","viaMember":true,"file":"tools/web_search.py"},{"source":"tools/web_search.py#tavily_search","relation":"calls","name":"decode","viaMember":true,"file":"tools/web_search.py"},{"source":"tools/web_search.py#tavily_search","relation":"calls","name":"read","viaMember":true,"file":"tools/web_search.py"},{"source":"tools/web_search.py#tavily_search","relation":"calls","name":"str","viaMember":false,"file":"tools/web_search.py"},{"source":"tools/web_search.py#tavily_search","relation":"calls","name":"get","viaMember":true,"file":"tools/web_search.py"},{"source":"tools/web_search.py#tavily_search","relation":"calls","name":"str","viaMember":false,"file":"tools/web_search.py"},{"source":"tools/web_search.py#tavily_search","relation":"calls","name":"get","viaMember":true,"file":"tools/web_search.py"},{"source":"tools/web_search.py#tavily_search","relation":"calls","name":"str","viaMember":false,"file":"tools/web_search.py"},{"source":"tools/web_search.py#tavily_search","relation":"calls","name":"get","viaMember":true,"file":"tools/web_search.py"},{"source":"tools/web_search.py#tavily_search","relation":"calls","name":"get","viaMember":true,"file":"tools/web_search.py"},{"source":"tools/web_search.py#tavily_search","relation":"calls","name":"get","viaMember":true,"file":"tools/web_search.py"},{"source":"tools/web_search.py","relation":"contains","targetId":"tools/web_search.py#make_web_search","file":"tools/web_search.py"},{"source":"tools/web_search.py#make_web_search","relation":"contains","targetId":"tools/web_search.py#make_web_search.web_search","file":"tools/web_search.py"},{"source":"tools/web_search.py#make_web_search.web_search","relation":"calls","name":"strip","viaMember":true,"file":"tools/web_search.py"},{"source":"tools/web_search.py#make_web_search.web_search","relation":"calls","name":"max","viaMember":false,"file":"tools/web_search.py"},{"source":"tools/web_search.py#make_web_search.web_search","relation":"calls","name":"min","viaMember":false,"file":"tools/web_search.py"},{"source":"tools/web_search.py#make_web_search.web_search","relation":"calls","name":"int","viaMember":false,"file":"tools/web_search.py"},{"source":"tools/web_search.py#make_web_search.web_search","relation":"calls","name":"backend","viaMember":false,"file":"tools/web_search.py"},{"source":"tools/web_search.py#make_web_search.web_search","relation":"calls","name":"format_hits","viaMember":false,"file":"tools/web_search.py"}]}}}