{ "schema_version": "1.1", "id": "atlas-evals", "slug": "evals", "title": "Evals", "url": "https://feed7.dev/atlas/evals", "current_answer": "Small, fast test sets that measure whether an agent or model change made behavior better or worse.", "implementation_consequence": "Evals turn \"the agent feels worse\" into a CI failure. They are the only reliable gate for increasing agent autonomy.", "agent_context": "Keep evals fast (<30s) and behavior-anchored so they survive model swaps. Wire into CI; fail builds on regression. Start from your 5 most common failure cases.", "confidence": "editorial", "last_verified": null, "last_updated": "2026-07-01T00:00:00.000Z", "evidence": [ { "schema_version": "1.1", "id": "p5", "slug": "eval-harness-post", "url": "https://feed7.dev/p/eval-harness-post", "title": "A minimal eval harness you can run in CI", "why_included": "Shows how to gate agent changes behind a tiny, fast eval set in CI.", "summary": "A 20-case eval wired into CI that fails the build on regression. Full code in the post; runs in under 30 seconds.", "practical_implication": "Makes agent reliability a CI concern, not a vibe. Start with your 5 most common failure cases.", "agent_context": "Wire a 20-case eval into CI; fail the build on regression. Keep it fast (<30s) so agents get feedback each PR.", "source": { "name": "Vercel", "url": "https://vercel.com/blog/minimal-eval-harness-ci", "published_at": "2026-06-26T00:00:00.000Z" }, "source_class": "blog_post", "content_type": "Engineering Post", "layer": "benchmark", "domains": [ "coding" ], "topics": [ "agent-evals" ], "verification": { "status": "source_linked", "label": "Source Linked", "method": "source_feed", "verified_at": null }, "uncertainty": [], "connected_context": null, "lifecycle": "New", "published_at": "2026-06-26T00:00:00.000Z", "modified_at": "2026-06-26T00:00:00.000Z", "supersedes": [], "expires_at": null, "formats": { "html": "https://feed7.dev/p/eval-harness-post", "json": "https://feed7.dev/p/eval-harness-post.json", "markdown": "https://feed7.dev/p/eval-harness-post.md" } }, { "schema_version": "1.1", "id": "p10", "slug": "evals-workshop-aie", "url": "https://feed7.dev/p/evals-workshop-aie", "title": "Building eval sets that survive model swaps — AI Engineer workshop", "why_included": "Eval sets usually die when you change models. This workshop shows how to write ones that transfer.", "summary": "Behavior-anchored evals: assert on user-visible outcomes, not model phrasing. Includes a template repo and a live migration from GPT to Claude.", "practical_implication": "Rewrite phrasing-based assertions as outcome assertions now — before your next model swap forces it.", "agent_context": "Write evals against user-visible outcomes, not model phrasing. Outcome-anchored evals survive model swaps. Template: given/when/then on behavior, never on wording.", "source": { "name": "AI Engineer", "url": "https://www.youtube.com/watch?v=ai-eng-evals-2026", "published_at": "2026-06-21T00:00:00.000Z" }, "source_class": "video", "content_type": "Workshop", "layer": "benchmark", "domains": [ "coding" ], "topics": [ "agent-evals", "model-selection" ], "verification": { "status": "transcript_verified", "label": "Transcript Verified", "method": "transcript_review", "verified_at": null }, "uncertainty": [], "connected_context": null, "lifecycle": "Evergreen", "published_at": "2026-06-21T00:00:00.000Z", "modified_at": "2026-06-21T00:00:00.000Z", "supersedes": [], "expires_at": null, "formats": { "html": "https://feed7.dev/p/evals-workshop-aie", "json": "https://feed7.dev/p/evals-workshop-aie.json", "markdown": "https://feed7.dev/p/evals-workshop-aie.md" } }, { "schema_version": "1.1", "id": "p14", "slug": "linkedin-eval-rollout", "url": "https://feed7.dev/p/linkedin-eval-rollout", "title": "Rolling out agents behind evals — an operator’s playbook", "why_included": "Concrete staged-rollout playbook with numbers — but the claimed win rates are not yet source-linked.", "summary": "Operator describes gating an internal agent behind a 40-case eval, canarying to 10% of tasks, then expanding. Claims 30% fewer escalations.", "practical_implication": "The staging pattern is reusable today; treat the win-rate numbers as unverified until the promised write-up lands.", "agent_context": "Staged agent rollout: gate behind eval set, canary 10% of tasks, expand on pass. Pattern is sound; the 30% improvement claim is unverified.", "source": { "name": "LinkedIn", "url": "https://www.linkedin.com/posts/operator-evals-rollout", "published_at": "2026-07-01T00:00:00.000Z" }, "source_class": "social_media", "content_type": "Social Thread", "layer": "benchmark", "domains": [ "coding" ], "topics": [ "agent-evals", "agent-reliability" ], "verification": { "status": "needs_review", "label": "Needs Review", "method": "unverified", "verified_at": null }, "uncertainty": [ "Win-rate numbers not source-linked; write-up promised but not published." ], "connected_context": null, "lifecycle": "New", "published_at": "2026-07-01T00:00:00.000Z", "modified_at": "2026-07-01T00:00:00.000Z", "supersedes": [], "expires_at": null, "formats": { "html": "https://feed7.dev/p/linkedin-eval-rollout", "json": "https://feed7.dev/p/linkedin-eval-rollout.json", "markdown": "https://feed7.dev/p/linkedin-eval-rollout.md" } }, { "schema_version": "1.1", "id": "p2", "slug": "context-engineering-talk", "url": "https://feed7.dev/p/context-engineering-talk", "title": "Context engineering for coding agents — AI Engineer World’s Fair", "why_included": "A reusable framework for deciding what belongs in an agent’s context window and what to leave out.", "summary": "Introduces a \"context budget\": rank material by decision-relevance, evict anything that does not change the next action. Includes a worked example on a real repo.", "practical_implication": "Practical, tool-agnostic, testable. Apply the budget per session instead of dumping everything into the window.", "agent_context": "Treat context as a budget, not a dump. Rank material by decision-relevance; evict anything that does not change the next action. Works across Cursor, Claude Code, Codex.", "source": { "name": "AI Engineer", "url": "https://www.youtube.com/watch?v=ai-eng-context-2026", "published_at": "2026-06-28T00:00:00.000Z" }, "source_class": "video", "content_type": "AI Engineer Talk", "layer": "context", "domains": [ "coding" ], "topics": [ "context-engineering" ], "verification": { "status": "transcript_verified", "label": "Transcript Verified", "method": "transcript_review", "verified_at": null }, "uncertainty": [ "Framework is presenter’s own; not yet independently benchmarked." ], "connected_context": null, "lifecycle": "Evergreen", "published_at": "2026-06-28T00:00:00.000Z", "modified_at": "2026-06-28T00:00:00.000Z", "supersedes": [], "expires_at": null, "formats": { "html": "https://feed7.dev/p/context-engineering-talk", "json": "https://feed7.dev/p/context-engineering-talk.json", "markdown": "https://feed7.dev/p/context-engineering-talk.md" } }, { "schema_version": "1.1", "id": "auto-3518047f01", "slug": "how-evals-and-prompts-shape-agent-behavior-preetika-bhat-3518047f01", "url": "https://feed7.dev/p/how-evals-and-prompts-shape-agent-behavior-preetika-bhat-3518047f01", "title": "How Evals and Prompts Shape Agent Behavior — Preetika Bhateja & Daniel Bump, YouTube Ads", "why_included": "Start with a few core tasks, inspect traces, change one component at a time, and protect a separate test set.", "summary": "Production agent behavior should be improved through a loop of tool design, small eval sets, trace review, and calibrated judges—not prompt tweaks based on one bad run.", "practical_implication": "Build evals early, inspect traces, and change one prompt, tool, rubric, or model component at a time. Compare human and LLM ratings, maintain a high-agreement golden set, and use ablations to establish whether a change actually improves the agent.", "agent_context": "The YouTube Ads team recommends optimizing a focused set of agent tools before larger evals, then starting with **a few core tasks** and testing both desired behavior and prohibited actions. Explanations help diagnose multi-dimensional outcomes such as accuracy and brand safety.\n\nBuild evals early, inspect traces, and change one prompt, tool, rubric, or model component at a time. Compare human and LLM ratings, maintain a high-agreement golden set, and use ablations to establish whether a change actually improves the agent.\n\nDo not tune against isolated nondeterministic failures. Keep a sparingly used **test set**, refresh it with production data, monitor judge disagreement, and define launch gates before iteration makes the rubric a moving target.", "source": { "name": "AI Engineer", "url": "https://www.youtube.com/watch?v=xyL2Ltkh-SA", "published_at": "2026-07-24T00:00:00.000Z" }, "source_class": "video", "content_type": "Video", "layer": "benchmark", "domains": [ "coding" ], "topics": [ "agent-evals", "agent-reliability", "prompting" ], "verification": { "status": "source_linked", "label": "Source Linked", "method": "source_feed", "verified_at": null }, "uncertainty": [], "connected_context": null, "lifecycle": "New", "published_at": "2026-07-24T00:00:00.000Z", "modified_at": "2026-07-24T00:00:00.000Z", "supersedes": [], "expires_at": null, "formats": { "html": "https://feed7.dev/p/how-evals-and-prompts-shape-agent-behavior-preetika-bhat-3518047f01", "json": "https://feed7.dev/p/how-evals-and-prompts-shape-agent-behavior-preetika-bhat-3518047f01.json", "markdown": "https://feed7.dev/p/how-evals-and-prompts-shape-agent-behavior-preetika-bhat-3518047f01.md" } }, { "schema_version": "1.1", "id": "auto-db6b4984d3", "slug": "harness-engineering-is-not-enough-why-software-factories-db6b4984d3", "url": "https://feed7.dev/p/harness-engineering-is-not-enough-why-software-factories-db6b4984d3", "title": "Harness Engineering is not Enough: Why Software Factories Fail — Dex Horthy, HumanLayer", "why_included": "Agree on design before implementation and build vertical slices so generated changes remain readable and testable.", "summary": "Coding-agent loops can raise throughput without preserving maintainability. Keep human ownership of code, and use upfront alignment to make review affordable instead of trying to automate it away.", "practical_implication": "Keep humans responsible for the resulting code. Use model-assisted planning, agree on design before implementation, and build in vertical slices so every generated change remains practical to read and test.", "agent_context": "The talk argues that coding models are rewarded mainly when **code runs and tests pass**, not when architecture remains easy to change. Review agents and extra loops can raise the floor, but cannot supply a missing maintainability signal.\n\nKeep humans responsible for the resulting code. Use **model-assisted planning**, agree on design before implementation, and build in vertical slices so every generated change remains practical to read and test.\n\nThere is no established benchmark here that proves how well current models preserve codebase quality. Longer-task evaluations such as **SWE Marathon**, DeepSuite, and FrontierCode may help, but model-based quality judges have their own ceiling.", "source": { "name": "AI Engineer", "url": "https://www.youtube.com/watch?v=Ib5GBkD555M", "published_at": "2026-07-23T00:00:00.000Z" }, "source_class": "video", "content_type": "Video", "layer": "agent", "domains": [ "coding" ], "topics": [ "harness-engineering", "agent-reliability", "agent-evals" ], "verification": { "status": "source_linked", "label": "Source Linked", "method": "source_feed", "verified_at": null }, "uncertainty": [], "connected_context": null, "lifecycle": "New", "published_at": "2026-07-23T00:00:00.000Z", "modified_at": "2026-07-23T00:00:00.000Z", "supersedes": [], "expires_at": null, "formats": { "html": "https://feed7.dev/p/harness-engineering-is-not-enough-why-software-factories-db6b4984d3", "json": "https://feed7.dev/p/harness-engineering-is-not-enough-why-software-factories-db6b4984d3.json", "markdown": "https://feed7.dev/p/harness-engineering-is-not-enough-why-software-factories-db6b4984d3.md" } }, { "schema_version": "1.1", "id": "auto-7bc753b969", "slug": "change2task-from-repository-changes-to-executable-coding-7bc753b969", "url": "https://feed7.dev/p/change2task-from-repository-changes-to-executable-coding-7bc753b969", "title": "Change2Task: From Repository Changes to Executable Coding Agent Tasks and Environments", "why_included": "Turn merged PR history into renewable eval tasks while preserving healthy, reconstructed, and restored repository states.", "summary": "Change2Task converts merged PR history into executable coding-agent tasks on current repository states. It could expand continuous eval sets while reducing stale environments and repeated setup.", "practical_implication": "Use repository history as renewable eval data, but preserve the full lifecycle: healthy base, reconstructed task state, and restored state. The five tested families include bug fixes, features, tests, API migrations, and security repairs.", "agent_context": "Change2Task turns merged pull requests into verified tasks on modern repository revisions using **Patch Reversal, Code Mapping, or Agent Reconstruction**. Across **1,130 eligible changes**, it constructed verified tasks at a 79.6% rate.\n\nUse repository history as renewable eval data, but preserve the full lifecycle: healthy base, reconstructed task state, and restored state. The five tested families include bug fixes, features, tests, API migrations, and security repairs.\n\nIt recovered **29.2% more verified tasks** than a PR-based baseline and reached up to 98.0% matched outcome agreement. These are construction results from the evaluated repositories, not proof that generated tasks represent every real maintenance workload.", "source": { "name": "arXiv", "url": "https://arxiv.org/abs/2607.28591v1", "published_at": "2026-07-30T00:00:00.000Z" }, "source_class": "blog_post", "content_type": "Paper", "layer": "benchmark", "domains": [ "coding", "security" ], "topics": [ "coding-agents", "agent-evals", "benchmark-integrity" ], "verification": { "status": "needs_review", "label": "Needs Review", "method": "unverified", "verified_at": null }, "uncertainty": [ "Automatically selected from source material; feed7 has not independently tested the claim." ], "connected_context": null, "lifecycle": "New", "published_at": "2026-07-30T00:00:00.000Z", "modified_at": "2026-07-30T00:00:00.000Z", "supersedes": [], "expires_at": null, "formats": { "html": "https://feed7.dev/p/change2task-from-repository-changes-to-executable-coding-7bc753b969", "json": "https://feed7.dev/p/change2task-from-repository-changes-to-executable-coding-7bc753b969.json", "markdown": "https://feed7.dev/p/change2task-from-repository-changes-to-executable-coding-7bc753b969.md" } }, { "schema_version": "1.1", "id": "auto-cc8e132c61", "slug": "how-enabling-two-settings-tripled-our-scores-on-the-arc--cc8e132c61", "url": "https://feed7.dev/p/how-enabling-two-settings-tripled-our-scores-on-the-arc--cc8e132c61", "title": "How enabling two settings tripled our scores on the ARC-AGI-3 benchmark", "why_included": "Record reasoning retention and compaction with the model name because runtime settings can materially alter agent eval results.", "summary": "Two API settings—reasoning retention and compaction—reportedly tripled GPT-5.6’s ARC-AGI-3 score. Agent evals should treat runtime configuration as part of the tested system.", "practical_implication": "Record these settings alongside the model name in agent evaluations. Configuration can materially affect results, so defaults and explicit settings should not be compared as equivalent systems.", "agent_context": "OpenAI says enabling **reasoning retention** and **compaction** produced **3× ARC-AGI-3 scores** for GPT-5.6 while also improving efficiency.\n\nRecord these settings alongside the model name in agent evaluations. Configuration can materially affect results, so defaults and explicit settings should not be compared as equivalent systems.\n\nThe supplied material provides no absolute scores, token usage, latency, or experimental detail, leaving the size and generality of the efficiency gain unclear.", "source": { "name": "OpenAI", "url": "https://openai.com/index/how-two-settings-tripled-our-arc-agi-3-scores", "published_at": "2026-07-29T00:00:00.000Z" }, "source_class": "blog_post", "content_type": "Official Release", "layer": "benchmark", "domains": [], "topics": [ "agent-evals", "benchmark-integrity", "context-caching" ], "verification": { "status": "official_source", "label": "Official Source", "method": "source_feed", "verified_at": null }, "uncertainty": [], "connected_context": null, "lifecycle": "New", "published_at": "2026-07-29T00:00:00.000Z", "modified_at": "2026-07-29T00:00:00.000Z", "supersedes": [], "expires_at": null, "formats": { "html": "https://feed7.dev/p/how-enabling-two-settings-tripled-our-scores-on-the-arc--cc8e132c61", "json": "https://feed7.dev/p/how-enabling-two-settings-tripled-our-scores-on-the-arc--cc8e132c61.json", "markdown": "https://feed7.dev/p/how-enabling-two-settings-tripled-our-scores-on-the-arc--cc8e132c61.md" } } ], "conflicting_sources": [], "superseded_claims": [], "corpus_evidence": [ { "title": "When Will The Benchmaxxing Plague End? — Nick Heiner, Surge AI", "url": "https://www.youtube.com/watch?v=-npY6XjM8CQ", "source_name": "AI Engineer", "published_at": "2026-08-02T16:30:06+00:00", "summary": "Nick Heiner argues that leaderboard gains can diverge from useful agent behavior through contamination, weak verifiers, reward hacking, and test conditions that users cannot inspect." }, { "title": "Teaching AI to Find Real Vulnerabilities — David Brumley, Bugcrowd", "url": "https://www.youtube.com/watch?v=ZFxh7sqbUZo", "source_name": "AI Engineer", "published_at": "2026-08-01T00:30:06+00:00", "summary": "Cybersecurity agents need deterministic exploit oracles and tasks that reward finding distinct vulnerabilities, not self-reported wins or one crash. A capability ladder makes partial progress measurable." }, { "title": "Rethinking Environments for Long-Horizon Work — Rayan Garg, Theta Software", "url": "https://www.youtube.com/watch?v=2aS7aKoXn64", "source_name": "AI Engineer", "published_at": "2026-08-01T00:00:06+00:00", "summary": "Human task duration is a weak proxy for agent difficulty when agents use different shortcuts and tools. Long-horizon evals need environment complexity, queryable trajectories, and judges that inspect final state." }, { "title": "Verifiable Environments for AI in Biology — Kenny Workman, LatchBio", "url": "https://www.youtube.com/watch?v=3ZMUiFaQ3qg", "source_name": "AI Engineer", "published_at": "2026-07-31T20:00:35+00:00", "summary": "Biology agents need evaluators that verify analysis of large experimental datasets, not recall. LatchBio found human review essential because valid scientific paths can defeat brittle graders." }, { "title": "Ending AI Slop — Thais Castello Branco, Taste Labs", "url": "https://www.youtube.com/watch?v=lCBf9slCanI", "source_name": "AI Engineer", "published_at": "2026-07-31T19:26:20+00:00", "summary": "For subjective agent output, replace vague requests for quality with decomposed brand constraints, then reserve human preference data for style and creativity that resist deterministic checks." }, { "title": "AgentHPOBench: A Benchmark For Evaluating LLM Agents as Sequential Hyperparameter Optimizers", "url": "https://arxiv.org/abs/2607.29626v1", "source_name": "arXiv", "published_at": "2026-07-31T16:58:00+00:00", "summary": "AgentHPOBench tests whether agents can learn from experiment history, not merely produce code. Its results expose weaknesses in sustained refinement and log diagnosis across sequential ML runs." }, { "title": "Benchmarks: The Good, the Bad, and the Ugly — Ali Khial, G2i", "url": "https://www.youtube.com/watch?v=jWq-aZIU0kM", "source_name": "AI Engineer", "published_at": "2026-07-31T16:00:06+00:00", "summary": "Coding benchmark scores can mislead when prompts leak implementation details or tests reject valid solutions. Inspect tasks and verifiers before using a leaderboard to choose an agent." }, { "title": "Reinforcement Learning without Verifiable Rewards — Will Brown, Prime Intellect", "url": "https://www.youtube.com/watch?v=AQv3qRCG6Gw", "source_name": "AI Engineer", "published_at": "2026-07-31T15:00:06+00:00", "summary": "For agent work without deterministic answers, mine production traces into tasks, build controllable simulators, and use scaled judges to derive rubrics while watching for reward hacking." }, { "title": "DeepSeek V4 Flash now runs updated weights on AI Gateway", "url": "https://vercel.com/changelog/deepseek-v4-flash-now-runs-updated-weights-on-ai-gateway", "source_name": "Vercel", "published_at": "2026-07-31T07:00:00+00:00", "summary": "DeepSeek V4 Flash’s updated weights replace the preview behind the existing model ID, raising its reported Terminal-Bench score from 56.9 to 82.7 without code changes." }, { "title": "fighting slop with slop — Vaibhav Gupta, Boundary", "url": "https://www.youtube.com/watch?v=AMiyLItEtLA", "source_name": "AI Engineer", "published_at": "2026-07-31T01:11:30+00:00", "summary": "Boundary replaces blanket code review with stable architecture rules, agent-generated tests, transcript inspection, execution traces, and human triage—a concrete harness pattern, not a license to skip verification." }, { "title": "OSReward: Instituting Standardized Evaluation for Cross-Platform Computer-Use Reward Models", "url": "https://arxiv.org/abs/2607.28609v1", "source_name": "arXiv", "published_at": "2026-07-30T17:57:41+00:00", "summary": "OSReward finds that VLM judges often approve failed computer-use runs. Its benchmark and open reward models offer a more grounded way to evaluate trajectories without paying frontier-model costs." }, { "title": "PAIChecker: Uncovering and Checking PR-Issue Misalignment in SWE-Bench-Like Benchmarks", "url": "https://arxiv.org/abs/2607.28587v1", "source_name": "arXiv", "published_at": "2026-07-30T17:42:44+00:00", "summary": "PAIChecker finds PR–issue mismatch in 13.6% of SWE-bench Verified instances. Benchmark maintainers should validate that the issue, patch, and tests describe the same task before trusting agent scores." }, { "title": "Sample More, Reflect Less: Self-Refine and Reflexion Lose to Repeated Sampling at Equal Token Cost, from 1.5B to 7B", "url": "https://arxiv.org/abs/2607.28576v1", "source_name": "arXiv", "published_at": "2026-07-30T17:38:23+00:00", "summary": "At equal generated-token cost, repeated sampling matched or beat seven reflection, critique, selection, and debate methods. Agent evals should budget every generated token, not compare against one-shot baselines." }, { "title": "First Steps Toward Automated AI Research — Richard Socher, CEO Recursive AI", "url": "https://www.youtube.com/watch?v=pWXUkLP9uWM", "source_name": "AI Engineer", "published_at": "2026-07-30T16:59:37+00:00", "summary": "Socher’s automated-research design combines prior knowledge, measurement data, simulation, physical experiments, and agent orchestration, with early demonstrations in training and CUDA optimization." }, { "title": "Persona Engineering: A Field Guide to AI Synthetic Personas — Ishan Anand, InsightSciences.ai", "url": "https://www.youtube.com/watch?v=YnNF55QV0zs", "source_name": "AI Engineer", "published_at": "2026-07-29T20:15:35+00:00", "summary": "Synthetic personas can extend existing research, but they are forecasts, not extra respondents. Ground prompts richly and validate each setup against human data before using it." }, { "title": "SimulationMaxxing: How we ship agents 20× faster — Aman Gupta (Nubank) + Shreya Rajpal (Snowglobe)", "url": "https://www.youtube.com/watch?v=KMR_RBoCa4M", "source_name": "AI Engineer", "published_at": "2026-07-29T19:00:06+00:00", "summary": "Nubank uses simulated multi-turn traces to evaluate agent changes before production, shortening release cycles while checking simulation results against real data and human review." }, { "title": "MindForge: Teaching Small Language Models Whole-Life-Cycle Software Engineering via Source-Free Program Synthesis", "url": "https://arxiv.org/abs/2607.27146v1", "source_name": "arXiv", "published_at": "2026-07-29T17:23:02+00:00", "summary": "MindForge trains a 27B coding model on source-free program synthesis, raising ProgramBench pass rate from 37.98% to 49.51% and improving across seven unseen benchmarks." }, { "title": "How Forward Deployed Engineering is done at Cognition — Jia Wu", "url": "https://www.youtube.com/watch?v=RVxym6mmIns", "source_name": "AI Engineer", "published_at": "2026-07-28T20:00:06+00:00", "summary": "Cognition measures coding-agent deployments by delivery outcomes, not sessions or tokens: engineering capacity, shorter timelines, and accepted PRs tied to customer work." }, { "title": "How Forward Deployed Engineering is done at Ramp — Leo Mehr", "url": "https://www.youtube.com/watch?v=ITMXwI6QL6A", "source_name": "AI Engineer", "published_at": "2026-07-28T19:00:06+00:00", "summary": "Ramp pairs strict request scoping with agents across the delivery pipeline. Its intake agent reportedly saves about 20% of scoping time, while humans retain final judgment." }, { "title": "Desktop-Delta Bench: Do Computer-Use Models Understand Desktop GUI Transitions?", "url": "https://arxiv.org/abs/2607.26041v1", "source_name": "arXiv", "published_at": "2026-07-28T17:49:51+00:00", "summary": "Desktop-Delta Bench tests whether computer-use models can verify what changed after an action, exposing a reliability gap hidden by end-task and single-frame scores." }, { "title": "Efficient LLM-Generated Shuttling Compilers for Complex Trapped-Ion Architectures", "url": "https://arxiv.org/abs/2607.24714v1", "source_name": "arXiv", "published_at": "2026-07-27T17:51:18+00:00", "summary": "Frontier models iteratively produced trapped-ion compiler code that beat hand-built baselines in some architectures, showing a specification-to-evaluation loop on a hard domain." }, { "title": "ERUnderstand: Evaluating Vision-Language Models on Structured ER Diagrams", "url": "https://arxiv.org/abs/2607.24707v1", "source_name": "arXiv", "published_at": "2026-07-27T17:46:43+00:00", "summary": "ERUnderstand shows vision-language models can recover common ERD elements but often miss rarer schema constructs, so image-to-schema agent workflows still need structural validation." }, { "title": "DeepSWE: A Contamination-Resistant Coding Benchmark — James Shi, Datacurve", "url": "https://www.youtube.com/watch?v=Yk87oUPVaxU", "source_name": "AI Engineer", "published_at": "2026-07-26T18:10:56+00:00", "summary": "DeepSWE uses original long-horizon tasks to reduce contamination and expose coding-agent behaviors hidden by saturated PR-mined suites. Its current task mix still underrepresents some everyday work." }, { "title": "State of Data — Sean Cai, Independent / State of Data", "url": "https://www.youtube.com/watch?v=ZyIoTOAbRfs", "source_name": "AI Engineer", "published_at": "2026-07-26T17:00:06+00:00", "summary": "Real workflow traces may teach agents more than manufactured tasks, while benchmark scores can shift with the harness. Build pipelines around live work and test across scaffolds." }, { "title": "Evals-Driven Development for a Mental Health AI Coach — Akele Reed & Dave Revere, SonderMind", "url": "https://www.youtube.com/watch?v=O72p-rBb2bA", "source_name": "AI Engineer", "published_at": "2026-07-25T23:00:36+00:00", "summary": "SonderMind turns clinician-reviewed failures into release-gating evals, keeping mental-health guardrails modular and testing false positives, false negatives, category, and timing." }, { "title": "From Agent Traces to Agent Simulations — Rustem Feyzkhanov, Snorkel AI", "url": "https://www.youtube.com/watch?v=Ib5t2RLtxvM", "source_name": "AI Engineer", "published_at": "2026-07-25T01:00:02+00:00", "summary": "Reconstructing production traces as replayable environments lets teams compare agent stacks under fixed conditions and gate releases on task outcomes, cost, latency, and retries." }, { "title": "Evaling Video Slop — Maor Bril, Character.ai", "url": "https://www.youtube.com/watch?v=b_PmGocP4rc", "source_name": "AI Engineer", "published_at": "2026-07-25T00:00:02+00:00", "summary": "Video evaluators can reward polish while missing frozen action, broken physics, or failed storytelling. Builders need time-aware criteria and human-calibrated data, not frame quality alone." }, { "title": "Building Closed-Loop Evals for a Multimodal Agent at Scale — Soumya Gupta & Jai Chopra, Uber", "url": "https://www.youtube.com/watch?v=31GUkCBD-Uc", "source_name": "AI Engineer", "published_at": "2026-07-24T22:00:25+00:00", "summary": "Uber’s image-editing agent uses routing, iterative QA, golden-set gates, and production feedback to avoid costly edits, hallucinated food, and quality regressions." }, { "title": "The Future of Evals: From LLM as a Judge to Agent as a Judge — Aparna Dhinakaran, Arize AI", "url": "https://www.youtube.com/watch?v=q2JrUKBMf0w", "source_name": "AI Engineer", "published_at": "2026-07-24T20:00:06+00:00", "summary": "Fixed-rubric judges miss failures that emerge across long, variable agent trajectories. Arize argues for adding agent-based analysis while retaining deterministic and LLM-judge evals." }, { "title": "The Regression Tax: Decomposing Why Skills Help and Hurt LLM Agents", "url": "https://arxiv.org/abs/2607.22520v1", "source_name": "arXiv", "published_at": "2026-07-24T17:50:03+00:00", "summary": "Procedural skills can make an agent fail tasks it previously solved. Evaluate gains and regressions separately, and design skills to preserve input grounding and output verification." }, { "title": "Everything Is a Rollout — Alex Shaw + Ryan Marten, Terminal-Bench, Harbor, Laude Institute", "url": "https://www.youtube.com/watch?v=jRCpXUjz4CI", "source_name": "AI Engineer", "published_at": "2026-07-24T16:00:06+00:00", "summary": "Harbor frames agent development as an empirical loop: run agents in reproducible sandboxes, verify outcomes, inspect trajectories, and evaluate every harness or model change." }, { "title": "Vending-Bench: Long-Horizon Agent Evals — Lukas Petersson, Andon Labs", "url": "https://www.youtube.com/watch?v=cO8qC6HBuBg", "source_name": "AI Engineer", "published_at": "2026-07-24T15:00:06+00:00", "summary": "Vending-Bench shows why long-horizon agents need both repeatable simulations and real-world tests: models drift, exploit incentives, and behave differently when they detect an eval." }, { "title": "Training Frontier Models to Out-Think Hackers — Uri Rolls, Arithmetic & Thom Wolf, Hugging Face", "url": "https://www.youtube.com/watch?v=O-CBZ3JtRvo", "source_name": "AI Engineer", "published_at": "2026-07-24T05:19:15+00:00", "summary": "This security eval tests whether agents can discover and exploit logic flaws across live chained services, using hidden zero-days and deterministic grading instead of source-code pattern matching." }, { "title": "The Unreasonable Effectiveness of Separating the Task from the Model — Maxime Rivest & Isaac Miller", "url": "https://www.youtube.com/watch?v=GgLQ02aO-hs", "source_name": "AI Engineer", "published_at": "2026-07-23T17:45:06+00:00", "summary": "DSPy separates an AI task’s contract from its model, prompt, tools, and harness, letting builders swap or optimize implementations while holding behavior against the same evals." }, { "title": "Beyond Sycophancy: Structured Resistance and Compliance in LLM Moral Reasoning", "url": "https://arxiv.org/abs/2607.21558v1", "source_name": "arXiv", "published_at": "2026-07-23T17:40:07+00:00", "summary": "Three studies suggest LLM compliance varies with opinion distance, claimed source, and coalition structure, so agent tests should probe how framing changes judgment rather than score sycophancy once." }, { "title": "Don't Ship Skills Without Evals — Philipp Schmid, Google DeepMind", "url": "https://www.youtube.com/watch?v=0vphxNt4wyk", "source_name": "AI Engineer", "published_at": "2026-07-14T22:00:06+00:00", "summary": "Agent skills need regression tests, not manual spot checks. Test triggering and output with and without each skill, across repeated trials and the harnesses your team actually uses." } ] }