{"name":"AgentValCon Verification Method","version":"v1.5","candidates":{"grade_minimum":"A","grade_thresholds":{"A":85,"B":70,"C":55,"D":40,"F":0},"also_required":["active","healthy in the last 7 days","not quarantined or under security review"]},"role_test":{"calls_per_test":1,"read_only_tools_only":true,"no_credentials_or_payment":true,"windows_per_day":8,"tests_per_window":120,"tests_per_window_when_refusals_pass_5_percent":60,"tests_per_day":960,"new_candidates_per_day":800,"retests_per_day":160,"per_registrable_domain_per_day":5,"stop_domain_for_the_day_on":429,"answer_judged_up_to_bytes":20000,"excerpt_kept_chars":1000,"answer_hash":"sha256","task_input_must_reach_the_tool":"when none of a task’s input reaches a parameter of the tool, the call is skipped as cannot fill; it is never made with other values and judged against that task’s answers"},"score":{"total":100,"data_quality":{"accuracy":25,"freshness":10,"completeness":10,"clean":10,"consistency":5},"evidence":{"arguments":10,"repeat":10,"latency":10},"report_text":10,"latency_points":[{"up_to_ms":1000,"points":10},{"up_to_ms":2000,"points":8},{"up_to_ms":5000,"points":5},{"up_to_ms":10000,"points":2}],"accuracy_rule":"points only when the answer matches an answer set or a public reference at the same time; no answer source means 0; expected words match as whole words, words the input already gives are not asked for, and a task that asks for nothing beyond its input gives 0","clean_answer_failures":["broken encoding","demo or placeholder","error text","payment demand"],"clean_answer_review":["cut answer","error text inside the answer","placeholder word"],"completeness_rule":"the share of the role's required output present, times the share of non-blank values, times the items returned against the items asked for; a text answer by the share of its fact checks passed; an empty search with no error is complete, an empty or error answer is not","freshness_rule":"only the role's own kind of time counts (published, observed, valid, or a clock in the named zone); retrieval times and unlabelled dates do not","evaluator_version":"dq-2026-11-16a"},"verified_pearl":{"min_score":85,"successes_on_different_days":1,"days_count_when":"accurate and clean on a KST day, under the same tool, endpoint, tool definition, evaluator version and answer set","when_the_evaluator_changes":"a result whose whole answer was kept at test time is judged again by the new evaluator and counts under it; a result without its whole answer keeps its earlier verdict and label and never counts together with the new evaluator’s successes"},"final_pearl":{"rule":"the top verified pearl of a role","one_place_per_registrable_domain_in_a_role":true,"ranked_by":"the pass rate of the role tasks given over the last 7 days, then the latest score","min_recent_pass_rate":0.5,"stale_copy_rule":"a role with a weekly rotating task (a CVE added to KEV in the last 30 days) asks it every other day; a member whose newest answer to it is not accurate is no pearl","reranked":"daily"},"necklace_gate":{"necklace":{"gate":"connection test","stage_1":["steps joined one at a time, each run for real before the next is added","the whole chain runs within its time budget","each required step with its representative member left out: a fallback answers"],"stage_2_connection_runs":3,"stage_2":"the whole chain on that many different task inputs, all passing; one failure sends the step back to stage 1 and stage 2 starts over","every_required_step":"a representative pearl and at least one outside fallback","step_without_an_outside_fallback_in_the_market":"our own agent, approved by its own connection runs; then outside pearls lead 2 or more steps","terms":"allowed or silent","opens":"when both stages pass and operations approve","accuracy_regraded_at_this_stage":false},"after_opening":{"connection_runs_a_day":1,"held_when":"the daily run fails or is older than 48 hours","when_a_fallback_answers_a_public_run":"a task opens to build our own agent for that role"},"substitute":{"gate":"connection test","connection_runs":3},"report_v2":{"score":85,"musts":["accuracy_check","clean_answer"]},"answer_sets":"withheld"},"roles":[{"role":"airport.lookup.v1","field":"travel","example_input":{"code":"ICN"},"answer_source":"answer set (withheld)","checks":["expected facts appear in the answer"],"freshness":"fixed facts (not time-bound)"},{"role":"book.search.v1","field":"reference","example_input":{"query":"Pride and Prejudice","title":"Pride and Prejudice"},"answer_source":"answer set (withheld)","checks":["expected facts appear in the answer"],"freshness":"fixed facts (not time-bound)"},{"role":"company.profile.v1","field":"other","example_input":{"company":"CIK 1045810"},"answer_source":"answer set (withheld)","checks":["expected facts appear in the answer"],"freshness":"fixed facts (not time-bound)"},{"role":"company.registry.v1","field":"business","example_input":{"country":"US","domain":"apple.com","name":"Apple Inc"},"answer_source":"answer set (withheld)","checks":["expected facts appear in the answer"],"freshness":"fixed facts (not time-bound)"},{"role":"country.indicator.v1","field":"economics","example_input":{"country":"KOR","indicator":"population","year":2023},"answer_source":"answer set (withheld)","checks":["expected facts appear in the answer"],"freshness":"fixed facts (not time-bound)"},{"role":"court.case.v1","field":"other","example_input":{"citation":"384 U.S. 436","query":"Miranda v. Arizona"},"answer_source":"answer set (withheld)","checks":["expected facts appear in the answer"],"freshness":"fixed facts (not time-bound)"},{"role":"crypto.price.v1","field":"finance","example_input":{"symbol":"BTC","coin":"bitcoin","id":"bitcoin","query":"bitcoin"},"answer_source":"none (public reference suspended until its terms are confirmed)","checks":["same-time public reference within 2%"],"freshness":"data time within 2 days"},{"role":"cve.epss.v1","field":"other","example_input":{"cve":"CVE-2021-44228","cve_id":"CVE-2021-44228"},"answer_source":"answer set (withheld)","checks":["expected facts appear in the answer"],"freshness":"fixed facts (not time-bound)"},{"role":"cve.exploited.v1","field":"security","example_input":{"cve_id":"CVE-2021-44228"},"answer_source":"answer set (withheld)","checks":["numeric fields match","KEV listing stated correctly"],"freshness":"fixed facts (not time-bound)"},{"role":"cve.record.v1","field":"security","example_input":{"cve_id":"CVE-2021-44228"},"answer_source":"answer set (withheld)","checks":["expected facts appear in the answer","numeric fields match"],"freshness":"fixed facts (not time-bound)"},{"role":"dictionary.define.v1","field":"reference","example_input":{"word":"serendipity"},"answer_source":"answer set (withheld)","checks":["expected facts appear in the answer"],"freshness":"fixed facts (not time-bound)"},{"role":"dns.mail_auth.v1","field":"internet","example_input":{"domain":"python.org","record_type":"TXT","type":"TXT"},"answer_source":"answer set (withheld)","checks":["expected facts appear in the answer"],"freshness":"fixed facts (not time-bound)"},{"role":"domain.lookup.v1","field":"internet","example_input":{"domain":"python.org"},"answer_source":"answer set (withheld)","checks":["expected facts appear in the answer"],"freshness":"fixed facts (not time-bound)"},{"role":"earthquake.feed.v1","field":"science","example_input":{"min_magnitude":5},"answer_source":"answer set (withheld)","checks":["expected facts appear in the answer","same-time public reference: usgs_places"],"freshness":"data time within 30 days"},{"role":"email.deliverability.v1","field":"other","example_input":{"domain":"gmail.com","host":"gmail.com"},"answer_source":"answer set (withheld)","checks":["expected facts appear in the answer"],"freshness":"fixed facts (not time-bound)"},{"role":"encyclopedia.summary.v1","field":"reference","example_input":{"title":"Seoul"},"answer_source":"answer set (withheld)","checks":["expected facts appear in the answer"],"freshness":"fixed facts (not time-bound)"},{"role":"events.search.v1","field":"events","example_input":{"limit":5,"place":"San Francisco","query":"concert"},"answer_source":"answer set (withheld)","checks":["expected facts appear in the answer","published recently"],"freshness":"dates today or later"},{"role":"fx.rate.v1","field":"finance","example_input":{"amount":100,"from":"USD","to":"EUR"},"answer_source":"answer set (withheld)","checks":["expected facts appear in the answer","same-time public reference: rate"],"freshness":"data time within 2 days"},{"role":"geocode.address.v1","field":"maps","example_input":{"address":"1600 Pennsylvania Avenue NW, Washington, DC"},"answer_source":"answer set (withheld)","checks":["expected facts appear in the answer"],"freshness":"fixed facts (not time-bound)"},{"role":"github.releases.v1","field":"developer","example_input":{"repo":"psf/requests","repository":"https://github.com/psf/requests"},"answer_source":"answer set (withheld)","checks":["expected facts appear in the answer"],"freshness":"fixed facts (not time-bound)"},{"role":"github.repo.v1","field":"developer","example_input":{"repo":"facebook/react"},"answer_source":"answer set (withheld)","checks":["expected facts appear in the answer"],"freshness":"fixed facts (not time-bound)"},{"role":"holiday.calendar.v1","field":"reference","example_input":{"country":"US","year":2026},"answer_source":"answer set (withheld)","checks":["expected facts appear in the answer"],"freshness":"fixed facts (not time-bound)"},{"role":"http.headers.v1","field":"other","example_input":{"domain":"github.com","host":"github.com","url":"https://github.com"},"answer_source":"answer set (withheld)","checks":["expected facts appear in the answer"],"freshness":"fixed facts (not time-bound)"},{"role":"ip.lookup.v1","field":"internet","example_input":{"ip":"8.8.8.8"},"answer_source":"answer set (withheld)","checks":["expected facts appear in the answer"],"freshness":"fixed facts (not time-bound)"},{"role":"mcp.registry.lookup.v1","field":"developer","example_input":{"query":"agent concierge"},"answer_source":"answer set (withheld)","checks":["expected facts appear in the answer"],"freshness":"fixed facts (not time-bound)"},{"role":"news.search.v1","field":"news","example_input":{"limit":5,"query":"artificial intelligence"},"answer_source":"answer set (withheld)","checks":["published recently"],"freshness":"data time within 7 days"},{"role":"package.advisories.v1","field":"security","example_input":{"ecosystem":"npm","package":"lodash","version":"4.17.20"},"answer_source":"answer set (withheld)","checks":["expected facts appear in the answer"],"freshness":"fixed facts (not time-bound)"},{"role":"package.health.v1","field":"developer","example_input":{"ecosystem":"npm","package":"express"},"answer_source":"answer set (withheld)","checks":["expected facts appear in the answer","same-time public reference: latest_version"],"freshness":"fixed facts (not time-bound)"},{"role":"package.releases.v1","field":"developer","example_input":{"ecosystem":"npm","package":"lodash"},"answer_source":"answer set (withheld)","checks":["expected facts appear in the answer"],"freshness":"fixed facts (not time-bound)"},{"role":"package.risk.v1","field":"security","example_input":{"ecosystem":"npm","package":"react","repo":"github.com/facebook/react"},"answer_source":"answer set (withheld)","checks":["expected facts appear in the answer","same-time public reference: scorecard"],"freshness":"fixed facts (not time-bound)"},{"role":"papers.search.v1","field":"research","example_input":{"query":"attention is all you need transformer"},"answer_source":"answer set (withheld)","checks":["expected facts appear in the answer"],"freshness":"fixed facts (not time-bound)"},{"role":"place.search.v1","field":"travel","example_input":{"query":"Eiffel Tower"},"answer_source":"answer set (withheld)","checks":["expected facts appear in the answer"],"freshness":"fixed facts (not time-bound)"},{"role":"sanctions.screen.v1","field":"other","example_input":{"entity":"Lazarus Group","name":"Lazarus Group","query":"Lazarus Group"},"answer_source":"answer set (withheld)","checks":["expected facts appear in the answer"],"freshness":"fixed facts (not time-bound)"},{"role":"sec.filings.v1","field":"business","example_input":{"ticker":"AAPL"},"answer_source":"answer set (withheld)","checks":["expected facts appear in the answer"],"freshness":"fixed facts (not time-bound)"},{"role":"software.incident.v1","field":"security","example_input":{"query":"xz-utils"},"answer_source":"answer set (withheld)","checks":["expected facts appear in the answer"],"freshness":"data time within 7 days"},{"role":"stock.quote.v1","field":"finance","example_input":{"symbol":"AAPL","ticker":"AAPL"},"answer_source":"none (public reference suspended until its terms are confirmed)","checks":["same-time public reference within 2%"],"freshness":"data time within 2 days"},{"role":"timezone.time.v1","field":"reference","example_input":{"city":"New York","lat":40.7128,"latitude":40.7128,"lng":-74.006,"lon":-74.006,"longitude":-74.006,"timezone":"America/New_York"},"answer_source":"answer set (withheld)","checks":["same-time public reference: clock_tz"],"freshness":"clock time within 5 minutes of now"},{"role":"tls.certificate.v1","field":"internet","example_input":{"domain":"python.org","host":"python.org","hostname":"python.org","url":"https://python.org"},"answer_source":"answer set (withheld)","checks":["expected facts appear in the answer"],"freshness":"fixed facts (not time-bound)"},{"role":"translate.text.v1","field":"language","example_input":{"target":"fr","text":"Good morning"},"answer_source":"answer set (withheld)","checks":["expected facts appear in the answer"],"freshness":"fixed facts (not time-bound)"},{"role":"unit.convert.v1","field":"reference","example_input":{"from":"mile","to":"kilometer","value":10},"answer_source":"answer set (withheld)","checks":["expected facts appear in the answer"],"freshness":"fixed facts (not time-bound)"},{"role":"weather.forecast.v1","field":"weather","example_input":{"city":"Seoul","latitude":37.5665,"longitude":126.978},"answer_source":"answer set (withheld)","checks":["expected facts appear in the answer","same-time public reference: temperature_c"],"freshness":"dates today or later"},{"role":"web.grounding.v1","field":"reference","example_input":{"question":"What is the capital city of Australia?"},"answer_source":"answer set (withheld)","checks":["expected facts appear in the answer"],"freshness":"fixed facts (not time-bound)"},{"role":"website.ai_readiness.v1","field":"internet","example_input":{"domain":"wikipedia.org"},"answer_source":"answer set (withheld)","checks":["expected facts appear in the answer","same-time public reference: site_files"],"freshness":"fixed facts (not time-bound)"},{"role":"wikidata.entity.v1","field":"other","example_input":{"entity":"Q42","id":"Q42","query":"Douglas Adams"},"answer_source":"answer set (withheld)","checks":["expected facts appear in the answer"],"freshness":"fixed facts (not time-bound)"}],"limits":["One call per test with one input; a role is judged on a small number of tasks.","Answer sets are withheld; examples show inputs only.","Live checks depend on the public reference being available at test time; when it is not, accuracy is not awarded."]}