{ "_id": "6aae736e3d719c1f8407d078", "Package": "evaluatellm", "Title": "Statistical Inference for Language Model Evaluations", "Version": "0.1.0", "Authors@R": "person(\"Charles\", \"Coverdale\", email = \"charlesfcoverdale@gmail.com\",\nrole = c(\"aut\", \"cre\", \"cph\"))", "Description": "Treats language model evaluations as statistical\nexperiments and supplies the inference they require. Provides\ncentral limit theorem and cluster-robust standard errors for\nevaluation scores, paired and unpaired model comparisons,\nvariance decomposition when several responses are drawn per\nquestion, control-variate variance reduction, multiplicity\nadjustment across benchmark suites, and power and minimum\ndetectable effect calculations for planning evaluations,\nfollowing Miller (2024) . For\nevaluations scored by a model judge, implements agreement\nstatistics against a human gold standard and prediction-powered\ninference (Angelopoulos et al. 2023)\n with the power-tuned estimator of\nAngelopoulos, Bates and Jordan (2023)\n, so a small set of human labels\ndebiases a large set of judge scores. Leaderboards are\nsupported through bootstrap rank intervals and Bradley-Terry\nratings (Bradley and Terry 1952) . Accepts\nscores from any evaluation harness.", "License": "MIT + file LICENSE", "Encoding": "UTF-8", "Language": "en-GB", "URL": "https://charlescoverdale.github.io/evaluatellm/,\nhttps://github.com/charlescoverdale/evaluatellm", "BugReports": "https://github.com/charlescoverdale/evaluatellm/issues", "Roxygen": "list(markdown = TRUE)", "RoxygenNote": "7.3.3", "VignetteBuilder": "knitr", "Config/testthat/edition": "3", "Repository": "https://charlescoverdale.r-universe.dev", "Date/Publication": "2026-09-19 10:23:35 UTC", "RemoteUrl": "https://github.com/charlescoverdale/evaluatellm", "RemoteRef": "HEAD", "RemoteSha": "3ef61edaee981b31b84843678a96f6f6ddd27702", "NeedsCompilation": "no", "Packaged": { "Date": "2026-09-19 11:24:55 UTC", "User": "root" }, "Author": "Charles Coverdale [aut, cre, cph]", "Maintainer": "Charles Coverdale ", "_user": "charlescoverdale", "_type": "src", "_file": "evaluatellm_0.1.0.tar.gz", "_fileid": "https://r2.ropensci.org/8bcb784799320da1d052e95deebba5f8b56729d8c3ad8be674094bcdf43ab5aa", "_filesize": 279342, "_sha256": "8bcb784799320da1d052e95deebba5f8b56729d8c3ad8be674094bcdf43ab5aa", "_expires": "2026-12-28T11:35:09.000Z", "_created": "2026-09-19T11:24:55.000Z", "_published": "2026-09-19T11:35:10.827Z", "_jobs": [ { "job": 105889204437, "time": 153, "config": "linux-devel-x86_64", "r": "4.7.0", "check": "OK", "artifact": "10583078365" }, { "job": 105889204472, "time": 111, "config": "linux-release-x86_64", "r": "4.6.1", "check": "OK", "artifact": "10583825700" }, { "job": 105889204420, "time": 110, "config": "macos-oldrel-arm64", "r": "4.5.3", "check": "OK", "artifact": "10583920827" }, { "job": 105889204484, "time": 109, "config": "macos-release-arm64", "r": "4.6.1", "check": "OK", "artifact": "10583596360" }, { "job": 105888795388, "time": 205, "config": "source", "r": "4.6.1", "check": "OK", "artifact": "10583710575" }, { "job": 105889204418, "time": 102, "config": "wasm-release", "r": "4.6.0", "check": "OK", "artifact": "10583318143" }, { "job": 105889204417, "time": 68, "config": "windows-devel-x86_64", "r": "4.7.0", "check": "OK", "artifact": "10583745695" }, { "job": 105889204442, "time": 72, "config": "windows-oldrel-x86_64", "r": "4.5.3", "check": "OK", "artifact": "10583283113" }, { "job": 105889204460, "time": 73, "config": "windows-release-x86_64", "r": "4.6.1", "check": "OK", "artifact": "10584110195" } ], "_host": "GitHub-Actions", "_buildurl": "https://github.com/r-universe/charlescoverdale/actions/runs/35439889843", "_status": "success", "_upstream": "https://github.com/charlescoverdale/evaluatellm", "_commit": { "id": "3ef61edaee981b31b84843678a96f6f6ddd27702", "author": "Charles Coverdale ", "committer": "Charles Coverdale ", "message": "Add llms-full.txt, the agent-facing API reference\n\ncharlescoverdale.github.io/llms.txt points agents at this file for every\npackage it lists, and this package did not have one. Generated from the\nRd files by generate_llms_full.R: every export with its signature,\narguments, return value and example.\n", "time": 1789813415 }, "_maintainer": { "name": "Charles Coverdale", "email": "charlesfcoverdale@gmail.com", "login": "charlescoverdale", "linkedin": "in/charlescoverdale", "twitter": "@cfcoverdale", "description": "economics | climate science | R", "uuid": 21993492 }, "_distro": "resolute", "_registered": true, "_dependencies": [ { "package": "R", "version": ">= 4.1.0", "role": "Depends" }, { "package": "cli", "version": ">= 3.6.0", "role": "Imports" }, { "package": "graphics", "role": "Imports" }, { "package": "grDevices", "role": "Imports" }, { "package": "stats", "role": "Imports" }, { "package": "utils", "role": "Imports" }, { "package": "testthat", "version": ">= 3.0.0", "role": "Suggests" }, { "package": "knitr", "role": "Suggests" }, { "package": "rmarkdown", "role": "Suggests" }, { "package": "sandwich", "role": "Suggests" } ], "_owner": "charlescoverdale", "_selfowned": true, "_usedby": 0, "_updates": [ { "week": "2026-34", "n": 9 }, { "week": "2026-37", "n": 1 }, { "week": "2026-38", "n": 2 } ], "_tags": [ { "name": "v0.1.0", "date": "2026-09-08" } ], "_stars": 0, "_contributors": [ { "user": "charlescoverdale", "count": 12, "uuid": 21993492 } ], "_userbio": { "uuid": 21993492, "type": "user", "name": "Charles Coverdale", "followers": 14, "description": "economics | climate science | R" }, "_downloads": { "count": 48, "source": "https://cranlogs.r-pkg.org/downloads/total/last-month/evaluatellm" }, "_devurl": "https://github.com/charlescoverdale/evaluatellm", "_pkgdown": "https://charlescoverdale.github.io/evaluatellm/", "_searchresults": 0, "_rbuild": "4.6.1", "_assets": [ "extra/citation.cff", "extra/citation.html", "extra/citation.json", "extra/citation.txt", "extra/contents.json", "extra/evaluatellm.html", "extra/manual.pdf", "extra/NEWS.html", "extra/NEWS.txt", "extra/readme.html", "extra/readme.md", "LICENSE" ], "_homeurl": "https://github.com/charlescoverdale/evaluatellm", "_realowner": "charlescoverdale", "_cranurl": true, "_releases": [ { "version": "0.1.0", "date": "2026-09-15" } ], "_exports": [ "as_eval", "ev_bootstrap", "ev_cluster", "ev_elo", "ev_icc", "ev_judge_agreement", "ev_judge_debias", "ev_judge_power", "ev_mde", "ev_multi", "ev_paired", "ev_plot", "ev_power", "ev_rank", "ev_resample", "ev_score", "ev_table", "ev_unpaired", "ev_variance_reduction" ], "_help": [ { "page": "as_eval", "title": "Build an Evaluation Object", "concept": [ "core" ], "topics": [ "as_eval" ] }, { "page": "ev_bootstrap", "title": "Cluster Bootstrap for an Arbitrary Statistic", "concept": [ "single model" ], "topics": [ "ev_bootstrap" ] }, { "page": "ev_cluster", "title": "Cluster-Robust Standard Error for an Evaluation", "concept": [ "single model" ], "topics": [ "ev_cluster" ] }, { "page": "ev_elo", "title": "Bradley-Terry Ratings from Pairwise Preferences", "concept": [ "leaderboard" ], "topics": [ "ev_elo" ] }, { "page": "ev_icc", "title": "Intra-Cluster Correlation", "concept": [ "single model" ], "topics": [ "ev_icc" ] }, { "page": "ev_judge_agreement", "title": "Agreement Between a Model Judge and a Human Gold Standard", "concept": [ "judge" ], "topics": [ "ev_judge_agreement" ] }, { "page": "ev_judge_debias", "title": "Debias a Model Judge with a Small Human Sample", "concept": [ "judge" ], "topics": [ "ev_judge_debias" ] }, { "page": "ev_judge_power", "title": "How Many Human Labels a Debiased Evaluation Needs", "concept": [ "judge" ], "topics": [ "ev_judge_power" ] }, { "page": "ev_mde", "title": "Smallest Difference an Evaluation Can Detect", "concept": [ "planning" ], "topics": [ "ev_mde" ] }, { "page": "ev_multi", "title": "Multiplicity Adjustment Across a Benchmark Suite", "concept": [ "comparison" ], "topics": [ "ev_multi" ] }, { "page": "ev_paired", "title": "Paired Comparison of Two Models", "concept": [ "comparison" ], "topics": [ "ev_paired" ] }, { "page": "ev_plot", "title": "Plot Results with Error Bars", "concept": [ "reporting" ], "topics": [ "ev_plot" ] }, { "page": "ev_power", "title": "Number of Questions Needed to Detect a Difference", "concept": [ "planning" ], "topics": [ "ev_power" ] }, { "page": "ev_rank", "title": "Bootstrap Rank Intervals for a Leaderboard", "concept": [ "leaderboard" ], "topics": [ "ev_rank" ] }, { "page": "ev_resample", "title": "Variance Decomposition for Repeated Sampling", "concept": [ "single model" ], "topics": [ "ev_resample" ] }, { "page": "ev_score", "title": "Evaluation Score with a Standard Error", "concept": [ "single model" ], "topics": [ "ev_score" ] }, { "page": "ev_table", "title": "Collect Results into a Table", "concept": [ "reporting" ], "topics": [ "ev_table" ] }, { "page": "ev_unpaired", "title": "Unpaired Comparison of Two Models", "concept": [ "comparison" ], "topics": [ "ev_unpaired" ] }, { "page": "ev_variance_reduction", "title": "Variance Reduction with a Reference Model", "concept": [ "comparison" ], "topics": [ "ev_variance_reduction" ] } ], "_readme": "https://github.com/charlescoverdale/evaluatellm/raw/HEAD/README.md", "_rundeps": [ "cli" ], "_vignettes": [ { "source": "evaluatellm.Rmd", "filename": "evaluatellm.html", "title": "Getting started with evaluatellm", "engine": "knitr::rmarkdown", "headings": [ "Building the evaluation object", "Scoring one model", "Comparing two models", "Sizing an evaluation before running it", "Several responses per question", "Evaluations graded by a model judge", "Suites and leaderboards", "Reporting" ], "created": "2026-08-23 17:41:28", "modified": "2026-08-23 17:41:28", "commits": 1 } ], "_score": 4.176091259055681, "_indexed": true, "_nocasepkg": "evaluatellm", "_universes": [ "charlescoverdale" ], "_binaries": [ { "r": "4.7.0", "os": "linux", "version": "0.1.0", "date": "2026-09-19T11:27:31.000Z", "distro": "resolute", "commit": "3ef61edaee981b31b84843678a96f6f6ddd27702", "fileid": "https://r2.ropensci.org/78b06028762d2fccc79ad53b92995ab0998a8d2497dbb50c00609bf47ce0f305", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/charlescoverdale/actions/runs/35439889843" }, { "r": "4.6.1", "os": "linux", "version": "0.1.0", "date": "2026-09-19T11:26:53.000Z", "distro": "resolute", "commit": "3ef61edaee981b31b84843678a96f6f6ddd27702", "fileid": "https://r2.ropensci.org/02f1d490531d332c639e80e43e4a50490f259fcc50afec12ebd63ecbc3959114", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/charlescoverdale/actions/runs/35439889843" }, { "r": "4.5.3", "os": "mac", "version": "0.1.0", "date": "2026-09-19T11:32:28.000Z", "commit": "3ef61edaee981b31b84843678a96f6f6ddd27702", "fileid": "https://r2.ropensci.org/1374852ba2d61c2b9a1c45eb7e88f941c44b1412b3be48c7a6fb0f5ab6a0c65d", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/charlescoverdale/actions/runs/35439889843" }, { "r": "4.6.1", "os": "mac", "version": "0.1.0", "date": "2026-09-19T11:34:31.000Z", "commit": "3ef61edaee981b31b84843678a96f6f6ddd27702", "fileid": "https://r2.ropensci.org/a1cf910153f6375419b6b768f61dd1f620cca067461c5cf45d2ae4aaf24e6931", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/charlescoverdale/actions/runs/35439889843" }, { "r": "4.6.0", "os": "wasm", "version": "0.1.0", "date": "2026-09-19T11:26:57.000Z", "commit": "3ef61edaee981b31b84843678a96f6f6ddd27702", "fileid": "https://r2.ropensci.org/c93c0a5e5cc5b9340dce11293d3598efc262984d226b23eaf8a396c030865427", "status": "success", "buildurl": "https://github.com/r-universe/charlescoverdale/actions/runs/35439889843" }, { "r": "4.7.0", "os": "win", "version": "0.1.0", "date": "2026-09-19T11:25:56.000Z", "commit": "3ef61edaee981b31b84843678a96f6f6ddd27702", "fileid": "https://r2.ropensci.org/c406fd606e2566b758a03a1621d33e3bb4dbd4b02027d04bbfd7cce7112a7d91", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/charlescoverdale/actions/runs/35439889843" }, { "r": "4.5.3", "os": "win", "version": "0.1.0", "date": "2026-09-19T11:26:00.000Z", "commit": "3ef61edaee981b31b84843678a96f6f6ddd27702", "fileid": "https://r2.ropensci.org/98225dc50e03c38bb7383e16de9fc19fbdd36d0bdcb68219c58d85962ecbf3c0", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/charlescoverdale/actions/runs/35439889843" }, { "r": "4.6.1", "os": "win", "version": "0.1.0", "date": "2026-09-19T11:26:04.000Z", "commit": "3ef61edaee981b31b84843678a96f6f6ddd27702", "fileid": "https://r2.ropensci.org/7d89379c61d8874bc572935eb23a4b823fc543abb4a3104c9d7d8b959568d649", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/charlescoverdale/actions/runs/35439889843" } ] }