import argparse import json import math import os import re import shlex import sys import tempfile import types import unittest from pathlib import Path from tokensand_observe.workflow import compare_local def app_workflow(): # Return the compare_local arguments for your own cases, callbacks and checks. # Optional scorer_controls: give a case a fixed output and independently labeled checks. # Example for example_workflow(): [{"caseId": "small", "output": 5, "expected": {"correct": False}}]. # Use trusted offline evaluators. Controls call only the evaluator, on both variants. # With SDK 0.6.16+, use one lambda per independent check so a scorer error preserves sibling scores. return None def example_workflow(): return dict( cases=[{"id": "small", "input": 2}, {"id": "large", "input": 5}], checks=[{"id": "correct", "label": "Correct result"}], baseline=lambda value: value * 2, candidate=lambda value: value + value, evaluate=lambda value, output, variant: {"correct": output == value * 2}, ) def preview(value): # Only inspect small, plain JSON values. Never call an application's repr. if type(value) is str and len(value) > 2048: return json.dumps(value[:2048], ensure_ascii=True)[:2048] + "... (truncated)" pending = [value] for _ in range(96): if not pending: text = json.dumps(value, ensure_ascii=True) return text if len(text) <= 2048 else text[:2048] + "... (truncated)" item = pending.pop() kind = type(item) if kind is dict and len(item) <= 32 and all(type(k) is str and len(k) <= 256 for k in item): pending.extend(item.values()) elif kind is list and len(item) <= 32: pending.extend(item) elif not (item is None or kind is bool or (kind is str and len(item) <= 2048) or (kind is int and item.bit_length() <= 256) or (kind is float and math.isfinite(item))): break return "" def output_preview(output, reference): full = preview(output) if type(output) is not dict or type(reference) is not dict or not 1 <= len(reference) <= 16: return full if full != "" and not full.endswith("... (truncated)"): return full # Looking up strings in an arbitrary dict can invoke a custom key's equality method. if len(output) > 512 or any(type(k) is not str for k in output): return full if any(type(k) is not str or len(k) > 256 for k in reference): return full fields = [] for key in reference: value = preview(output[key]) if key in output else "" fields.append(json.dumps(key) + "=" + value) text = "fields named in input.expected: " + ", ".join(fields) return text if len(text) <= 2048 else text[:2048] + "... (truncated)" def project_location(filename, line, root): # Only print a real source file under the working directory. No stack, source text or locals. if filename.startswith("<") or not filename.endswith(".py") or root == root.parent: return None try: path = Path(filename).resolve(strict=True) relative = path.relative_to(root) if not path.is_file() or any(p in ("site-packages", "dist-packages", "__pypackages__") for p in relative.parts): return None name = relative.as_posix() if name.isprintable() and len(name) <= 512: return f"{name}:{line}" except (OSError, ValueError, RuntimeError): pass return None def callback_definition(callback, root): if type(callback) is types.MethodType: callback = callback.__func__ if type(callback) is types.FunctionType: code = callback.__code__ return project_location(code.co_filename, code.co_firstlineno, root) return None def timeout_diagnosis(run): source = {"workflow_error": ("workflowError", "workflow callback"), "check_error": ("evaluatorError", "evaluator")}.get(run["status"]) if source and run.get("diagnostic", {}).get(source[0]) == "timeout_exception": return source[1] + " raised a timeout" return "" def save_original_failures(runs, failures, outputs, inputs, exception_sites): pairs = {(case, trial) for (case, _, _, _), trials in failures.items() for trial in trials} rows = {(r["caseId"], r["repetition"] + 1, r["variant"]): r for r in runs} lines = ["Tokens& · Original failed trials"] for case, trial in sorted(pairs): lines.extend(["", f"{case} · trial {trial}"]) value = inputs[case] if type(value) is dict and "expected" in value: lines.append("input.expected: " + preview(value["expected"])) for variant in ("baseline", "candidate"): row = rows.get((case, trial, variant)) if row is None: lines.append(f"{variant}: did not run") continue lines.append(f'{variant}: {row["status"]} · checks: {json.dumps(row["checks"])}') diagnosis = timeout_diagnosis(row) if diagnosis: lines.append(" " + diagnosis) source = exception_sites.get((case, variant, trial)) if source: role, site = source lines.append(f" {role} exception · last project frame {site}") previews = outputs.get((case, variant), []) if trial <= len(previews): lines.append(" output: " + previews[trial - 1]) # Exclusive creation in the system temp directory, owner-only on POSIX. report = tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", prefix="tokensand-failures-", suffix=".txt", delete=False) try: with report: report.write("\n".join(lines) + "\n") except BaseException: os.unlink(report.name) raise return report.name def freeze_json(value): # Preserve the SDK's input types. Serializing a tuple or an integer key # directly would silently change what the application receives. if type(value) is list: return [freeze_json(item) for item in value] if type(value) is dict and all(type(key) is str for key in value): return {key: freeze_json(item) for key, item in value.items()} if value is None or type(value) in (str, bool, int) or (type(value) is float and math.isfinite(value)): return value raise ValueError("Workflow data must contain only JSON values and string keys") def prepare_controls(workflow, controls): if type(controls) is not list: raise ValueError("scorer_controls must be a list") if not controls: return [] inputs = {case["id"]: case["input"] for case in workflow["cases"]} check_ids = {check["id"] for check in workflow["checks"]} prepared = [] for index, control in enumerate(controls, 1): if (type(control) is not dict or set(control) != {"caseId", "output", "expected"} or type(control["caseId"]) is not str or control["caseId"] not in inputs): raise ValueError(f"Scorer control {index} needs a configured caseId, output and expected check scores") expected = control["expected"] if (type(expected) is not dict or not expected or not expected.keys() <= check_ids or any(type(value) is not bool for value in expected.values())): raise ValueError(f"Scorer control {index} needs boolean labels for configured checks") prepared.append((control["caseId"], freeze_json(inputs[control["caseId"]]), freeze_json(control["output"]), dict(expected))) return prepared def check_scorer(controls, checks, evaluate): passed = 0 for index, (case_id, value, output, expected) in enumerate(controls, 1): # The ordinary SDK resolves boolean and deferred checks and isolates JSON # inputs per trial. No application task is run for a calibration example. def fixed_output(_): return freeze_json(output) result = compare_local(cases=[{"id": case_id, "input": value}], checks=checks, baseline=fixed_output, candidate=fixed_output, evaluate=evaluate) problems = set() for run in result["runs"]: for check, label in expected.items(): actual = run["checks"][check] if actual is not label: observed = "unscored" if actual is None else ("pass" if actual else "fail") problems.add(f'{run["variant"]} {check} expected {"pass" if label else "fail"}, got {observed}') if result["state"] != "complete": problems.add("scorer run interrupted") if problems: print(f"Scorer control {index} ({case_id}): " + "; ".join(sorted(problems)), file=sys.stderr) else: passed += 1 return passed def run_check(*, example=False, inspect_case=None, save_failures=False): workflow = example_workflow() if example else app_workflow() if workflow is None: print("Set app_workflow() to your app's cases, callbacks and checks.", file=sys.stderr) return 2 repetitions = workflow.get("repetitions", 3) if type(repetitions) is not int or not 3 <= repetitions <= 10: raise ValueError("Use 3 to 10 repetitions") if not isinstance(workflow["cases"], list) or not workflow["cases"]: raise ValueError("Provide a non-empty list of cases") batch_size = 30 // repetitions if len(workflow["cases"]) > batch_size: # Freeze the whole suite before a callback can mutate a later batch. frozen = freeze_json(workflow["cases"]) ids = set() for case in frozen: if (type(case) is not dict or set(case) != {"id", "input"} or type(case["id"]) is not str or not re.fullmatch(r"[A-Za-z0-9_-]{1,80}", case["id"]) or case["id"] in ids): raise ValueError("Use unique case IDs across the suite and provide an input for every case") ids.add(case["id"]) workflow = {**workflow, "cases": frozen, "checks": freeze_json(workflow["checks"])} # The SDK alternates which variant goes first on each successive trial. if repetitions % 2 and batch_size % 2: batch_size -= 1 workflow = dict(workflow) control_specs = workflow.pop("scorer_controls", []) # --inspect replays only the requested application prefix. controls = [] if inspect_case is None: try: controls = prepare_controls(workflow, control_specs) except ValueError as error: print(str(error), file=sys.stderr) return 2 if controls: control_checks, control_evaluate = freeze_json(workflow["checks"]), workflow["evaluate"] if inspect_case is not None or save_failures: cases = workflow["cases"] inputs = {case["id"]: case["input"] for case in cases} if inspect_case is not None: positions = [i for i, case in enumerate(cases) if case["id"] == inspect_case] if len(positions) != 1: print("Choose one case ID from your configured workflow.", file=sys.stderr) return 2 cases = cases[:positions[0] + 1] value = inputs[inspect_case] if type(value) is dict and "expected" in value: print("input.expected:", preview(value["expected"]), file=sys.stderr) baseline, candidate, evaluate = workflow["baseline"], workflow["candidate"], workflow["evaluate"] outputs = {} root = Path.cwd() callbacks = {"baseline": baseline, "candidate": candidate, "evaluator": evaluate} exception_sites = {} trials_by_arm = {} def invoke(callback, role, row, variant, *args): if not save_failures and row["id"] != inspect_case: return callback(*args) key = (row["id"], variant) if role != "evaluator": trials_by_arm[key] = trials_by_arm.get(key, 0) + 1 try: return callback(*args) except Exception as error: # Skip this wrapper. A dependency can raise below the last application frame. frame = BaseException.__traceback__.__get__(error).tb_next while frame is not None: site = project_location(frame.tb_frame.f_code.co_filename, frame.tb_lineno, root) if site: exception_sites[(*key, trials_by_arm[key])] = (role, site) frame = frame.tb_next raise def inspect_output(row, output, variant): if save_failures or row["id"] == inspect_case: previews = outputs.setdefault((row["id"], variant), []) value = row["input"] reference = value.get("expected") if type(value) is dict else None previews.append(output_preview(output, reference)) if inspect_case is not None: print(f"trial {len(previews)} · {variant} output: {previews[-1]}", file=sys.stderr) return invoke(evaluate, "evaluator", row, variant, row["input"], output, variant) # Earlier cases can prime caches. Carry the ID through the runner, but give # the original callbacks the same inputs in the same order as a suite run. workflow = {**workflow, "cases": [{**case, "input": case} for case in cases], "baseline": lambda row: invoke(baseline, "baseline", row, "baseline", row["input"]), "candidate": lambda row: invoke(candidate, "candidate", row, "candidate", row["input"]), "evaluate": inspect_output} suite = workflow["cases"] runs = [] for start in range(0, len(suite), batch_size): result = compare_local(**{**workflow, "cases": suite[start:start + batch_size]}) runs.extend(result["runs"]) if result["state"] != "complete": break # Compare the same case, check and repetition on both versions. passed = {"baseline": 0, "candidate": 0} def partial_scores(run): values = run["checks"].values() return run["status"] == "check_error" and None in values and any(type(value) is bool for value in values) baselines = {(r["caseId"], r["repetition"]): r["checks"] for r in runs if r["variant"] == "baseline" and (r["status"] == "ok" or partial_scores(r))} failures = {} recovered = {} for run in runs: case, variant, trial = run["caseId"], run["variant"], run["repetition"] + 1 if run["status"] != "ok": if partial_scores(run): for check, score in run["checks"].items(): if score is None: reason = timeout_diagnosis(run) or "unscored (evaluator error)" failures.setdefault((case, variant, check, reason), []).append(trial) else: reason = timeout_diagnosis(run) or "execution error" failures.setdefault((case, variant, run["status"], reason), []).append(trial) continue if all(run["checks"].values()): passed[variant] += 1 if variant == "candidate": baseline = baselines.get((case, trial - 1), {}) for check, matched in run["checks"].items(): if matched is True and baseline.get(check) is False: recovered.setdefault((case, check), []).append(trial) elif matched is False: change = {True: "new failure", False: "also failed on baseline", None: "baseline unscored"}[baseline.get(check)] failures.setdefault((case, variant, check, change), []).append(trial) scope = {"scope": "example" if example else "application"} if inspect_case is not None: scope = {"scope": "case", "caseId": inspect_case} if len(cases) > 1: scope = {"scope": "prefix", "throughCaseId": inspect_case, "plannedCases": len(cases)} if not any(r["caseId"] == inspect_case for r in runs): print("The run stopped before reaching the requested case.", file=sys.stderr) controls_passed = check_scorer(controls, control_checks, control_evaluate) if controls else 0 if controls: scope["scorerControls"] = {"passed": controls_passed, "total": len(controls)} if controls_passed != len(controls): print("Review the evaluator against the labeled outputs before trusting the candidate scores.", file=sys.stderr) print(json.dumps({**scope, "state": result["state"], "runs": len(runs), "passed": sum(passed.values()), "baselinePassed": passed["baseline"], "candidatePassed": passed["candidate"]})) ordered = sorted(failures, key=lambda key: (key[3] != "new failure", key)) for case, variant, check, change in ordered: trials = ", ".join(map(str, failures[case, variant, check, change])) print(f"{case} · {variant}: {check} · {change} (trials: {trials})", file=sys.stderr) for (case, check), trials in sorted(recovered.items()): print(f"{case} · candidate: {check} · recovered (trials: {', '.join(map(str, trials))})", file=sys.stderr) if inspect_case is not None: roles = set() for run in runs: if run["caseId"] != inspect_case: continue if run["status"] == "workflow_error": roles.add(run["variant"]) if run["status"] == "check_error": roles.add("evaluator") if run["variant"] == "candidate" and any(value is False for value in run["checks"].values()): roles.update(("candidate", "evaluator")) sites = {role: site for role, site in exception_sites.values()} for role in sorted(roles | sites.keys()): if role in sites: print(f"{role} exception · last project frame {sites[role]}", file=sys.stderr) else: site = callback_definition(callbacks[role], root) if site: print(f"{role} callback defined at {site}", file=sys.stderr) if ordered and save_failures: try: report = save_original_failures(runs, failures, outputs, inputs, exception_sites) print(f"Original failed trials (private file): {report}", file=sys.stderr) except OSError as error: print(f"Could not save the original failed trials: {error.strerror}", file=sys.stderr) elif ordered and inspect_case is None and not example: command = shlex.join([sys.executable, os.path.relpath(__file__), "--inspect", ordered[0][0]]) print(f"Replay through the failed case and show its values locally: {command}", file=sys.stderr) return int(result["state"] != "complete" or bool(failures) or controls_passed != len(controls)) class TestTokensAndWorkflow(unittest.TestCase): def test_candidate(self): self.assertEqual(run_check(), 0, "See the workflow check output.") if __name__ == "__main__": parser = argparse.ArgumentParser(description="Check the configured application workflow.", allow_abbrev=False) mode = parser.add_mutually_exclusive_group() mode.add_argument("--example", action="store_true", help="run the bundled demo") mode.add_argument("--inspect", metavar="CASE_ID", help="replay through this case and show local values and relevant project source locations") mode.add_argument("--save-failures", action="store_true", help="save bounded original output previews for failed trials to a private temporary file") args = parser.parse_args() sys.exit(run_check(example=args.example, inspect_case=args.inspect, save_failures=args.save_failures))