{ "_id": "6a1036d0acfb0bcc41c9a2af", "Package": "pairwiseLLM", "Title": "Pairwise Comparison Tools for Large Language Model-Based Writing\nEvaluation", "Version": "1.3.0", "Authors@R": "person(\"Sterett H.\", \"Mercer\", , \"sterett.mercer@ubc.ca\", role = c(\"aut\", \"cre\"),\ncomment = c(ORCID = \"0000-0002-7940-4221\"))", "Description": "Provides a unified framework for generating, submitting,\nand analyzing pairwise comparisons of writing quality using\nlarge language models (LLMs). The package supports live and/or\nbatch evaluation workflows across multiple providers ('OpenAI',\n'Anthropic', 'Google Gemini', 'Together AI', and locally-hosted\n'Ollama' models), includes bias-tested prompt templates and a\nflexible template registry, and offers tools for constructing\nforward and reversed comparison sets to analyze consistency and\npositional bias. The package additionally supports adaptive\npairing workflows that iteratively select comparisons based on\nmodel uncertainty to improve ranking efficiency. Results can be\nmodeled using frequentist or Bayesian Bradley–Terry–Luce models\n(Bradley & Terry, 1952 ; see also Caron &\nDoucet, 2012 ) or Elo rating\nmethods (see Clark et al., 2018\n) to derive writing quality\nscores. For information on pairwise comparisons and comparitive\njudgement, see Thurstone (1927) and\nHeldsinger & Humphry (2010) .", "License": "MIT + file LICENSE", "Encoding": "UTF-8", "Roxygen": "list(markdown = TRUE)", "RoxygenNote": "7.3.3", "Remotes": [ "stan-dev/cmdstanr" ], "Config/testthat/edition": "3", "URL": "https://github.com/shmercer/pairwiseLLM,\nhttps://shmercer.github.io/pairwiseLLM/", "BugReports": "https://github.com/shmercer/pairwiseLLM/issues", "VignetteBuilder": "knitr", "Collate": "'adaptive_benchmark_metrics.R' 'adaptive_btl_refit.R'\n'adaptive_candidates.R' 'adaptive_constraints.R'\n'adaptive_logs.R' 'adaptive_persist.R' 'adaptive_print.R'\n'adaptive_rank.R' 'adaptive_round_candidates.R'\n'adaptive_run.R' 'adaptive_schemas.R' 'adaptive_select.R'\n'adaptive_simulation_harness.R' 'adaptive_state.R'\n'adaptive_step.R' 'adaptive_trueskill.R' 'adaptive_utility.R'\n'anthropic_batch_api.R' 'anthropic_live.R' 'api_keys.R'\n'bayes_btl_mcmc.R' 'bayes_btl_mcmc_adaptive.R'\n'bayes_btl_summarize.R' 'bt_helpers.R' 'bt_model.R'\n'btl_mcmc_constraints.R' 'btl_mcmc_contracts.R'\n'btl_mcmc_fit_contracts.R' 'btl_mcmc_ingest.R'\n'btl_mcmc_model_variant.R' 'btl_mcmc_state.R'\n'btl_mcmc_stopping.R' 'btl_mcmc_summaries.R'\n'btl_mcmc_theta_summary_stubs.R' 'core_budget.R'\n'cost_estimator.R' 'custom_id.R' 'data-example_writing.R'\n'data_import.R' 'draws_sanitize.R' 'elo_model.R'\n'gemini_batch_api.R' 'gemini_live.R' 'httr2_retry.R'\n'llm_backends.R' 'llm_batch.R' 'llm_multi_batch.R'\n'normalize_results.R' 'ollama_live.R' 'openai_batch_api.R'\n'openai_live.R' 'openai_params.R' 'pairing.R'\n'pairwiseLLM-package.R' 'prompt_template.R'\n'reverse_consistency.R' 'seed_helpers.R' 'together_live.R'\n'traits.R' 'utils-null-coalesce.R'", "Config/pak/sysreqs": "libssl-dev", "Repository": "https://shmercer.r-universe.dev", "Date/Publication": "2026-02-11 04:37:32 UTC", "RemoteUrl": "https://github.com/shmercer/pairwisellm", "RemoteRef": "HEAD", "RemoteSha": "385b4d2a7c5c6a0e3b07e5c03c81c98110ad43f9", "NeedsCompilation": "no", "Packaged": { "Date": "2026-05-12 06:39:12 UTC", "User": "root" }, "Author": "Sterett H. Mercer [aut, cre] (ORCID:\n)", "Maintainer": "Sterett H. Mercer ", "MD5sum": "2af25507dc1b75990d49c7c321d8bbc2", "_user": "shmercer", "_type": "src", "_file": "pairwiseLLM_1.3.0.tar.gz", "_fileid": "cbb13d67c6a2d020fcaffd4673f45d9f62fb4e01662241ec386ceddb083e46b4", "_filesize": 1344285, "_sha256": "cbb13d67c6a2d020fcaffd4673f45d9f62fb4e01662241ec386ceddb083e46b4", "_created": "2026-05-12T06:39:12.000Z", "_published": "2026-05-22T10:58:24.225Z", "_distro": "noble", "_jobs": [ { "job": 77366027781, "time": 223, "config": "linux-devel-x86_64", "r": "4.7.0", "check": "OK", "artifact": "6937402902" }, { "job": 77366027857, "time": 250, "config": "linux-release-x86_64", "r": "4.6.0", "check": "OK", "artifact": "6937410663" }, { "job": 77366028025, "time": 148, "config": "macos-oldrel-arm64", "r": "4.5.3", "check": "OK", "artifact": "6937384127" }, { "job": 77366027589, "time": 171, "config": "macos-release-arm64", "r": "4.6.0", "check": "OK", "artifact": "6937389434" }, { "job": 77366027090, "time": 271, "config": "source", "r": "4.6.0", "check": "OK", "artifact": "6937346476" }, { "job": 77366027193, "time": 127, "config": "wasm-release", "r": "4.6.0", "check": "OK", "artifact": "7158859277" }, { "job": 77366027763, "time": 198, "config": "windows-devel", "r": "4.7.0", "check": "OK", "artifact": "6937396461" }, { "job": 77366027817, "time": 195, "config": "windows-oldrel", "r": "4.5.3", "check": "OK", "artifact": "6937395998" }, { "job": 77366027510, "time": 227, "config": "windows-release", "r": "4.6.0", "check": "OK", "artifact": "6937404865" } ], "_buildurl": "https://github.com/r-universe/shmercer/actions/runs/25717803229", "_status": "success", "_host": "GitHub-Actions", "_upstream": "https://github.com/shmercer/pairwisellm", "_commit": { "id": "385b4d2a7c5c6a0e3b07e5c03c81c98110ad43f9", "author": "Sterett Mercer ", "committer": "GitHub ", "message": "fix equations (#168)\n\n", "time": 1770784652 }, "_maintainer": { "name": "Sterett H. Mercer", "email": "sterett.mercer@ubc.ca", "login": "shmercer", "uuid": 64615368, "orcid": "0000-0002-7940-4221" }, "_registered": true, "_dependencies": [ { "package": "R", "version": ">= 4.1", "role": "Depends" }, { "package": "cli", "role": "Imports" }, { "package": "curl", "role": "Imports" }, { "package": "dplyr", "role": "Imports" }, { "package": "httr2", "role": "Imports" }, { "package": "jsonlite", "role": "Imports" }, { "package": "rlang", "role": "Imports" }, { "package": "stats", "role": "Imports" }, { "package": "tibble", "role": "Imports" }, { "package": "tidyselect", "role": "Imports" }, { "package": "tools", "role": "Imports" }, { "package": "utils", "role": "Imports" }, { "package": "BradleyTerry2", "role": "Suggests" }, { "package": "cmdstanr", "role": "Suggests" }, { "package": "EloChoice", "role": "Suggests" }, { "package": "future", "role": "Suggests" }, { "package": "future.apply", "role": "Suggests" }, { "package": "knitr", "role": "Suggests" }, { "package": "mockery", "role": "Suggests" }, { "package": "pkgload", "role": "Suggests" }, { "package": "purrr", "role": "Suggests" }, { "package": "readr", "role": "Suggests" }, { "package": "rmarkdown", "role": "Suggests" }, { "package": "sirt", "role": "Suggests" }, { "package": "stringr", "role": "Suggests" }, { "package": "testthat", "version": ">= 3.0.0", "role": "Suggests" }, { "package": "tidyr", "role": "Suggests" }, { "package": "vctrs", "role": "Suggests" }, { "package": "withr", "role": "Suggests" } ], "_owner": "shmercer", "_selfowned": true, "_usedby": 0, "_updates": [ { "week": "2025-48", "n": 59 }, { "week": "2025-49", "n": 53 }, { "week": "2025-50", "n": 13 }, { "week": "2025-51", "n": 59 }, { "week": "2025-52", "n": 16 }, { "week": "2026-01", "n": 2 }, { "week": "2026-03", "n": 6 }, { "week": "2026-07", "n": 8 } ], "_tags": [ { "name": "v1.0.0", "date": "2025-12-10" }, { "name": "v1.1.0", "date": "2025-12-18" } ], "_topics": [ "anthropic", "assessment-tool", "automated-scoring", "bradley-terry", "elo-rating", "gemini", "large-language-models", "llm", "llm-as-a-judge", "ollama", "openai", "pairwise-comparison", "ranking", "together-ai", "writing-assessment", "writing-evaluation" ], "_stars": 3, "_contributors": [ { "user": "shmercer", "count": 324, "uuid": 64615368 } ], "_userbio": { "uuid": 64615368, "type": "user", "name": "Sterett Mercer" }, "_downloads": { "count": 474, "source": "https://cranlogs.r-pkg.org/downloads/total/last-month/pairwiseLLM" }, "_devurl": "https://github.com/shmercer/pairwisellm", "_pkgdown": "https://shmercer.github.io/pairwiseLLM/", "_searchresults": 14, "_rbuild": "4.6.0", "_assets": [ "extra/citation.cff", "extra/citation.html", "extra/citation.json", "extra/citation.txt", "extra/contents.json", "extra/NEWS.html", "extra/NEWS.txt", "extra/pairwiseLLM.html", "extra/readme.html", "extra/readme.md", "manual.pdf" ], "_homeurl": "https://github.com/shmercer/pairwisellm", "_realowner": "shmercer", "_cranurl": true, "_releases": [ { "version": "1.1.0", "date": "2025-12-22" } ], "_exports": [ "adaptive_get_logs", "adaptive_item_log", "adaptive_rank", "adaptive_rank_resume", "adaptive_rank_run_live", "adaptive_rank_start", "adaptive_results_history", "adaptive_round_log", "adaptive_step_log", "alternate_pair_order", "anthropic_compare_pair_live", "anthropic_create_batch", "anthropic_download_batch_results", "anthropic_get_batch", "anthropic_poll_batch_until_complete", "build_anthropic_batch_requests", "build_bt_data", "build_btl_results_data", "build_elo_data", "build_gemini_batch_requests", "build_openai_batch_requests", "build_prompt", "check_llm_api_keys", "check_positional_bias", "compute_reverse_consistency", "ensure_only_ollama_model_loaded", "estimate_llm_pairs_cost", "fit_bayes_btl_mcmc", "fit_bt_model", "fit_elo_model", "gemini_compare_pair_live", "gemini_create_batch", "gemini_download_batch_results", "gemini_get_batch", "gemini_poll_batch_until_complete", "get_prompt_template", "list_prompt_templates", "llm_compare_pair", "llm_download_batch_results", "llm_resume_multi_batches", "llm_submit_pairs_batch", "llm_submit_pairs_multi_batch", "load_adaptive_session", "make_adaptive_judge_llm", "make_pairs", "ollama_compare_pair_live", "openai_compare_pair_live", "openai_create_batch", "openai_download_batch_output", "openai_get_batch", "openai_poll_batch_until_complete", "openai_upload_batch_file", "parse_anthropic_batch_output", "parse_gemini_batch_output", "parse_openai_batch_output", "randomize_pair_order", "read_samples_df", "read_samples_dir", "register_prompt_template", "remove_prompt_template", "run_anthropic_batch_pipeline", "run_gemini_batch_pipeline", "run_openai_batch_pipeline", "sample_pairs", "sample_reverse_pairs", "save_adaptive_session", "set_prompt_template", "submit_anthropic_pairs_live", "submit_gemini_pairs_live", "submit_llm_pairs", "submit_ollama_pairs_live", "submit_openai_pairs_live", "submit_together_pairs_live", "summarize_adaptive", "summarize_bt_fit", "summarize_items", "summarize_refits", "together_compare_pair_live", "trait_description", "validate_session_dir", "write_openai_batch_file" ], "_datasets": [ { "name": "example_openai_batch_output", "title": "Example OpenAI Batch output (JSONL lines)", "object": "example_openai_batch_output", "file": "example_openai_batch_output.rda", "class": [ "character" ], "fields": [], "table": false, "tojson": true }, { "name": "example_writing_pairs", "title": "Example dataset of paired comparisons for writing samples", "object": "example_writing_pairs", "file": "example_writing_pairs.rda", "class": [ "tbl_df", "tbl", "data.frame" ], "fields": [ "ID1", "ID2", "better_id" ], "rows": 190, "table": true, "tojson": true }, { "name": "example_writing_results", "title": "Example canonical results table for writing comparisons", "object": "example_writing_results", "file": "example_writing_results.rda", "class": [ "tbl_df", "tbl", "data.frame" ], "fields": [ "pair_uid", "unordered_key", "ordered_key", "A_id", "B_id", "better_id", "winner_pos", "phase", "iter", "received_at", "backend", "model" ], "rows": 190, "table": true, "tojson": true }, { "name": "example_writing_samples", "title": "Example dataset of writing samples", "object": "example_writing_samples", "file": "example_writing_samples.rda", "class": [ "tbl_df", "tbl", "data.frame" ], "fields": [ "ID", "text", "quality_score" ], "rows": 20, "table": true, "tojson": true }, { "name": "example_writing_samples1000", "title": "Synthetic Writing Samples with Controlled Quality Levels (N = 1000)", "object": "example_writing_samples1000", "file": "example_writing_samples1000.rda", "class": [ "tbl_df", "tbl", "data.frame" ], "fields": [ "ID", "text", "quality_level", "theta_true", "prompt_id", "model", "created_at" ], "rows": 1000, "table": true, "tojson": true } ], "_help": [ { "page": "adaptive_get_logs", "title": "Retrieve canonical adaptive logs.", "concept": [ "adaptive logs" ], "topics": [ "adaptive_get_logs" ] }, { "page": "adaptive_item_log", "title": "Adaptive item log accessor.", "concept": [ "adaptive logs" ], "topics": [ "adaptive_item_log" ] }, { "page": "adaptive_rank", "title": "Run adaptive ranking end-to-end from data and model settings", "concept": [ "adaptive ranking" ], "topics": [ "adaptive_rank" ] }, { "page": "adaptive_rank_resume", "title": "Adaptive ranking resume", "concept": [ "adaptive ranking" ], "topics": [ "adaptive_rank_resume" ] }, { "page": "adaptive_rank_run_live", "title": "Adaptive ranking live runner", "concept": [ "adaptive ranking" ], "topics": [ "adaptive_rank_run_live" ] }, { "page": "adaptive_rank_start", "title": "Adaptive ranking", "concept": [ "adaptive ranking" ], "topics": [ "adaptive_rank_start" ] }, { "page": "adaptive_results_history", "title": "Adaptive results history in build_bt_data() format.", "concept": [ "adaptive logs" ], "topics": [ "adaptive_results_history" ] }, { "page": "adaptive_round_log", "title": "Adaptive round log accessor.", "concept": [ "adaptive logs" ], "topics": [ "adaptive_round_log" ] }, { "page": "adaptive_step_log", "title": "Adaptive step log accessor.", "concept": [ "adaptive logs" ], "topics": [ "adaptive_step_log" ] }, { "page": "alternate_pair_order", "title": "Deterministically alternate sample order in pairs", "topics": [ "alternate_pair_order" ] }, { "page": "anthropic_compare_pair_live", "title": "Live Anthropic (Claude) comparison for a single pair of samples", "topics": [ "anthropic_compare_pair_live" ] }, { "page": "anthropic_create_batch", "title": "Create an Anthropic Message Batch", "topics": [ "anthropic_create_batch" ] }, { "page": "anthropic_download_batch_results", "title": "Download Anthropic Message Batch results (.jsonl)", "topics": [ "anthropic_download_batch_results" ] }, { "page": "anthropic_get_batch", "title": "Retrieve an Anthropic Message Batch by ID", "topics": [ "anthropic_get_batch" ] }, { "page": "anthropic_poll_batch_until_complete", "title": "Poll an Anthropic Message Batch until completion", "topics": [ "anthropic_poll_batch_until_complete" ] }, { "page": "build_anthropic_batch_requests", "title": "Build Anthropic Message Batch requests from a tibble of pairs", "topics": [ "build_anthropic_batch_requests" ] }, { "page": "build_bt_data", "title": "Build Bradley-Terry comparison data from pairwise results", "topics": [ "build_bt_data" ] }, { "page": "build_btl_results_data", "title": "Build canonical 'results_tbl' data for Bayesian BTL MCMC", "topics": [ "build_btl_results_data" ] }, { "page": "build_elo_data", "title": "Build EloChoice comparison data from pairwise results", "topics": [ "build_elo_data" ] }, { "page": "build_gemini_batch_requests", "title": "Build Gemini batch requests from a tibble of pairs", "topics": [ "build_gemini_batch_requests" ] }, { "page": "build_openai_batch_requests", "title": "Build OpenAI batch JSONL lines for paired comparisons", "topics": [ "build_openai_batch_requests" ] }, { "page": "build_prompt", "title": "Build a concrete LLM prompt from a template", "topics": [ "build_prompt" ] }, { "page": "check_llm_api_keys", "title": "Check configured API keys for LLM backends", "topics": [ "check_llm_api_keys" ] }, { "page": "check_positional_bias", "title": "Check positional bias and bootstrap consistency reliability", "topics": [ "check_positional_bias" ] }, { "page": "compute_reverse_consistency", "title": "Compute consistency between forward and reverse pair comparisons", "topics": [ "compute_reverse_consistency" ] }, { "page": "ensure_only_ollama_model_loaded", "title": "Ensure only one Ollama model is loaded in memory", "topics": [ "ensure_only_ollama_model_loaded" ] }, { "page": "estimate_llm_pairs_cost", "title": "Estimate LLM token usage and cost for a set of pairwise comparisons", "topics": [ "estimate_llm_pairs_cost" ] }, { "page": "example_openai_batch_output", "title": "Example OpenAI Batch output (JSONL lines)", "topics": [ "example_openai_batch_output" ] }, { "page": "example_writing_pairs", "title": "Example dataset of paired comparisons for writing samples", "topics": [ "example_writing_pairs" ] }, { "page": "example_writing_results", "title": "Example canonical results table for writing comparisons", "topics": [ "example_writing_results" ] }, { "page": "example_writing_samples", "title": "Example dataset of writing samples", "topics": [ "example_writing_samples" ] }, { "page": "example_writing_samples1000", "title": "Synthetic Writing Samples with Controlled Quality Levels (N = 1000)", "topics": [ "example_writing_samples1000" ] }, { "page": "fit_bayes_btl_mcmc", "title": "Full Bayesian BTL inference via CmdStanR (adaptive-compatible)", "topics": [ "fit_bayes_btl_mcmc" ] }, { "page": "fit_bt_model", "title": "Fit a Bradley–Terry model with sirt and fallback to BradleyTerry2", "topics": [ "fit_bt_model" ] }, { "page": "fit_elo_model", "title": "Fit an EloChoice model to pairwise comparison data", "topics": [ "fit_elo_model" ] }, { "page": "gemini_compare_pair_live", "title": "Live Google Gemini comparison for a single pair of samples", "topics": [ "gemini_compare_pair_live" ] }, { "page": "gemini_create_batch", "title": "Create a Gemini Batch job from request objects", "topics": [ "gemini_create_batch" ] }, { "page": "gemini_download_batch_results", "title": "Download Gemini Batch results to a JSONL file", "topics": [ "gemini_download_batch_results" ] }, { "page": "gemini_get_batch", "title": "Retrieve a Gemini Batch job by name", "topics": [ "gemini_get_batch" ] }, { "page": "gemini_poll_batch_until_complete", "title": "Poll a Gemini Batch job until completion", "topics": [ "gemini_poll_batch_until_complete" ] }, { "page": "get_prompt_template", "title": "Retrieve a named prompt template", "topics": [ "get_prompt_template" ] }, { "page": "list_prompt_templates", "title": "List available prompt templates", "topics": [ "list_prompt_templates" ] }, { "page": "llm_compare_pair", "title": "Backend-agnostic live comparison for a single pair of samples", "topics": [ "llm_compare_pair" ] }, { "page": "llm_download_batch_results", "title": "Extract results from a pairwiseLLM batch object", "topics": [ "llm_download_batch_results" ] }, { "page": "llm_resume_multi_batches", "title": "Resume polling and download results for multiple batch jobs", "topics": [ "llm_resume_multi_batches" ] }, { "page": "llm_submit_pairs_batch", "title": "Submit pairs to an LLM backend via batch API", "topics": [ "llm_submit_pairs_batch" ] }, { "page": "llm_submit_pairs_multi_batch", "title": "Multi‑batch submission and polling wrappers", "topics": [ "llm_submit_pairs_multi_batch" ] }, { "page": "load_adaptive_session", "title": "Load an adaptive session from disk.", "concept": [ "adaptive persistence" ], "topics": [ "load_adaptive_session" ] }, { "page": "make_adaptive_judge_llm", "title": "Build an LLM judge function for adaptive ranking", "concept": [ "adaptive ranking" ], "topics": [ "make_adaptive_judge_llm" ] }, { "page": "make_pairs", "title": "Create all unordered pairs of writing samples", "topics": [ "make_pairs" ] }, { "page": "ollama_compare_pair_live", "title": "Live Ollama comparison for a single pair of samples", "topics": [ "ollama_compare_pair_live" ] }, { "page": "openai_compare_pair_live", "title": "Live OpenAI comparison for a single pair of samples", "topics": [ "openai_compare_pair_live" ] }, { "page": "openai_create_batch", "title": "Create an OpenAI batch from an uploaded file", "topics": [ "openai_create_batch" ] }, { "page": "openai_download_batch_output", "title": "Download the output file for a completed batch", "topics": [ "openai_download_batch_output" ] }, { "page": "openai_get_batch", "title": "Retrieve an OpenAI batch", "topics": [ "openai_get_batch" ] }, { "page": "openai_poll_batch_until_complete", "title": "Poll an OpenAI batch until it completes or fails", "topics": [ "openai_poll_batch_until_complete" ] }, { "page": "openai_upload_batch_file", "title": "Upload a JSONL batch file to OpenAI", "topics": [ "openai_upload_batch_file" ] }, { "page": "parse_anthropic_batch_output", "title": "Parse Anthropic Message Batch output into a tibble", "topics": [ "parse_anthropic_batch_output" ] }, { "page": "parse_gemini_batch_output", "title": "Parse Gemini batch JSONL output into a tibble of pairwise results", "topics": [ "parse_gemini_batch_output" ] }, { "page": "parse_openai_batch_output", "title": "Parse an OpenAI Batch output JSONL file", "topics": [ "parse_openai_batch_output" ] }, { "page": "print.adaptive_state", "title": "Print an adaptive state summary.", "topics": [ "print.adaptive_state" ] }, { "page": "print.pairwiseLLM_cost_estimate", "title": "Print a pairwiseLLM cost estimate", "topics": [ "print.pairwiseLLM_cost_estimate" ] }, { "page": "randomize_pair_order", "title": "Randomly assign samples to positions SAMPLE_1 and SAMPLE_2", "topics": [ "randomize_pair_order" ] }, { "page": "read_samples_df", "title": "Read writing samples from a data frame", "topics": [ "read_samples_df" ] }, { "page": "read_samples_dir", "title": "Read writing samples from a directory of .txt files", "topics": [ "read_samples_dir" ] }, { "page": "register_prompt_template", "title": "Register a named prompt template", "topics": [ "register_prompt_template" ] }, { "page": "remove_prompt_template", "title": "Remove a registered prompt template", "topics": [ "remove_prompt_template" ] }, { "page": "run_anthropic_batch_pipeline", "title": "Run an Anthropic batch pipeline for pairwise comparisons", "topics": [ "run_anthropic_batch_pipeline" ] }, { "page": "run_gemini_batch_pipeline", "title": "Run a Gemini batch pipeline for pairwise comparisons", "topics": [ "run_gemini_batch_pipeline" ] }, { "page": "run_openai_batch_pipeline", "title": "Run a full OpenAI batch pipeline for pairwise comparisons", "topics": [ "run_openai_batch_pipeline" ] }, { "page": "sample_pairs", "title": "Randomly sample pairs of writing samples", "topics": [ "sample_pairs" ] }, { "page": "sample_reverse_pairs", "title": "Sample reversed versions of a subset of pairs", "topics": [ "sample_reverse_pairs" ] }, { "page": "save_adaptive_session", "title": "Save an adaptive session to disk.", "concept": [ "adaptive persistence" ], "topics": [ "save_adaptive_session" ] }, { "page": "set_prompt_template", "title": "Get or set a prompt template for pairwise comparisons", "topics": [ "set_prompt_template" ] }, { "page": "submit_anthropic_pairs_live", "title": "Live Anthropic (Claude) comparisons for a tibble of pairs", "topics": [ "submit_anthropic_pairs_live" ] }, { "page": "submit_gemini_pairs_live", "title": "Live Google Gemini comparisons for a tibble of pairs", "topics": [ "submit_gemini_pairs_live" ] }, { "page": "submit_llm_pairs", "title": "Backend-agnostic live comparisons for a tibble of pairs", "topics": [ "submit_llm_pairs" ] }, { "page": "submit_ollama_pairs_live", "title": "Live Ollama comparisons for a tibble of pairs", "topics": [ "submit_ollama_pairs_live" ] }, { "page": "submit_openai_pairs_live", "title": "Live OpenAI comparisons for a tibble of pairs", "topics": [ "submit_openai_pairs_live" ] }, { "page": "submit_together_pairs_live", "title": "Live Together.ai comparisons for a tibble of pairs", "topics": [ "submit_together_pairs_live" ] }, { "page": "summarize_adaptive", "title": "Summarize an adaptive state.", "concept": [ "adaptive ranking" ], "topics": [ "summarize_adaptive" ] }, { "page": "summarize_bt_fit", "title": "Summarize a Bradley–Terry model fit", "topics": [ "summarize_bt_fit" ] }, { "page": "summarize_items", "title": "Summarize adaptive items", "topics": [ "summarize_items" ] }, { "page": "summarize_refits", "title": "Summarize adaptive refits", "topics": [ "summarize_refits" ] }, { "page": "together_compare_pair_live", "title": "Live Together.ai comparison for a single pair of samples", "topics": [ "together_compare_pair_live" ] }, { "page": "trait_description", "title": "Get a trait name and description for prompts", "topics": [ "trait_description" ] }, { "page": "validate_session_dir", "title": "Validate an adaptive session directory.", "concept": [ "adaptive persistence" ], "topics": [ "validate_session_dir" ] }, { "page": "write_openai_batch_file", "title": "Write an OpenAI batch table to a JSONL file", "topics": [ "write_openai_batch_file" ] } ], "_readme": "https://github.com/shmercer/pairwisellm/raw/HEAD/README.md", "_rundeps": [ "askpass", "cli", "curl", "dplyr", "generics", "glue", "httr2", "jsonlite", "lifecycle", "magrittr", "openssl", "pillar", "pkgconfig", "R6", "rappdirs", "rlang", "sys", "tibble", "tidyselect", "utf8", "vctrs", "withr" ], "_vignettes": [ { "source": "adaptive-pairing.Rmd", "filename": "adaptive-pairing.html", "title": "Adaptive Pairing and Ranking in pairwiseLLM", "engine": "knitr::rmarkdown", "headings": [ "Overview", "Why adaptive pairing?", "Reliability vs. precision", "Conceptual model", "The adaptive workflow", "Early vs. late stages of adaptation", "Early stage: global scale identification", "Late stage: local refinement", "Global identifiability and stopping", "How the adaptive algorithm works (schematic overview)", "High-level flow", "Early vs. late behavior", "Worked example: adaptive ranking of writing samples", "Run adaptive ranking", "Inspecting results", "Ranked items with uncertainty", "Step-level logs (pair selection audit trail)", "Pair result history", "Refit-level summaries and stopping diagnostics", "Downstream analyses", "When should you use adaptive ranking?", "Technical details", "Summary", "Citation" ], "created": "2026-02-10 21:06:53", "modified": "2026-02-10 23:34:14", "commits": 2 }, { "source": "advanced-batch-workflows.Rmd", "filename": "advanced-batch-workflows.html", "title": "Advanced: Submitting and Polling Multiple Batches", "engine": "knitr::rmarkdown", "headings": [ "1. Overview", "2. Setup and API Keys", "3. Example Data and Prompt Template", "4. Designing the Batch Grid", "5. Submitting Many Batches with the Multi‑Batch Helpers", "5.1 Create a run plan and output directory", "5.2 Submit all runs (no polling)", "6. Polling, Downloading, and Parsing (Resumable)", "6.1 Building a single merged results table (all runs)", "7. Resuming After Interruption", "8. Next Steps", "9. Citation" ], "created": "2025-12-05 21:39:39", "modified": "2026-02-10 23:34:14", "commits": 16 }, { "source": "bayesian-btl-adaptive-pairing-design.Rmd", "filename": "bayesian-btl-adaptive-pairing-design.html", "title": "Bayesian BTL Adaptive Pairing Design", "engine": "knitr::rmarkdown", "headings": [ "Bayesian BTL Adaptive Pairing Design", "Scope and Goals", "User-facing overview", "Global Definitions (normative)", "Indices", "Counts and keys", "Ordering precedence", "Terminology note (normative)", "1. Statistical Model", "1.1 Notation (common to all variants)", "1.2 Model options (choose one)", "Model A — Regular BTL (no lapse, no position bias)", "Model B — BTL + lapse ($\\epsilon$ )", "Model C — BTL + position bias ($b$)", "Model D — BTL + position bias + lapse ($b, \\epsilon$) (default)", "1.3 Priors and identifiability (all variants)", "2. Data Structures and Invariants", "2.1 Core Stores", "2.2 Hard Invariants", "3. Warm‑Start Design", "3.1 Shuffled Chain Construction (default)", "3.2 Early-Phase Coverage Completion", "4. Automatic Defaults (Functions of N)", "4.1 Scope of Automatic Defaults", "4.2 Exploration Rate (Coverage-Oriented, One-Pair Steps)", "Definition (per step)", "Interpretation under TrueSkill pairing", "Defaults", "Logging", "Late-stage exploration taper", "4.3 Refit Cadence", "4.4 Effective Sample Size (ESS) Thresholds", "4.5 Defaults Summary (normative)", "4.5.1 Global Identifiability State", "4.6 Round Cadence and Quotas (normative)", "4.6.1 Round size (committed comparisons)", "4.6.2 Round stage quotas", "Time-varying base fractions", "Global-identifiability taper", "Design intent", "5. Candidate Generation and Feasibility Safeguards", "5.0 Rank Strata and Percentile Top-Band (normative)", "5.0.1 Base strata (global)", "5.0.2 Percentile-based top band (required for large N)", "5.0.3 Stratum distance", "5.1 Candidate Pools (normative; stage-specific)", "5.1.1 Stage: anchor_link", "5.1.2 Stage: long_link", "5.1.3 Stage: mid_link", "5.1.4 Stage: local_link", "5.1.4.1 Local-link prioritization", "5.1.5 Candidate cap (C_max)", "5.1.6 Per-round exposure constraint (Pollitt-style; soft)", "5.2 Candidate Starvation Detection", "5.2.1 Canonical candidate pipeline (normative)", "5.3 Feasibility Fallback Ladder (within-stage)", "5.3.1 Within-stage fallback stages (in order)", "5.3.2 Duplicate allowance rule (normative; TrueSkill-based)", "Logging requirements (audit)", "5.4 Phase 3 Entry (Near-Stop Behavior)", "Entry criterion (evaluated only at refits)", "Normative behavior in Phase 3", "6. Utility Function (TrueSkill-based)", "6.1 Degree Control and Star Caps", "6.1.1 Star Caps (default enabled)", "Definition", "Conditional near-tie override", "6.1.2 Interaction with Utility", "6.1.3 Optional Continuous Degree Regularization (off by default)", "6.1.4 Logging and Audit", "7. Pair Selection Procedure (Round Controller + Step Engine)", "7.1 Rolling anchors (normative)", "7.2 Round controller (normative)", "7.2.1 Stage-specific partner targeting (selection scoring)", "7.2.2 Handling stage starvation (normative)", "7.2.3 Soft Pollitt exposure enforcement", "7.2.4 Refit cadence and continuity", "7.3 Degree control is enforced via the canonical pipeline (normative)", "7.3.1 Star caps (default enabled)", "7.3.2 Optional continuous degree regularization (off by default)", "7.4 Filter by invariants and duplicate policy (normative)", "Under-represented set (normative)", "7.5 Select the next pair (exploitation vs exploration)", "7.5.1 Coverage quota override (early phase)", "7.5.2 Exploration routing (per step)", "Exploration selection (with non-local connections)", "Exploitation selection", "Precedence rule", "7.6 Assign presentation order (A/B) and enforce order invariants (normative)", "7.7 Fallback ladder if starved", "8. MCMC Configuration and Diagnostics", "8.1 Chains and Parallelism", "8.2 Diagnostics Gate", "8.3 Reliability (EAP)", "9. Stopping Criteria", "9.1 MCMC Diagnostics Gate", "9.1.1 Divergences (must be zero)", "9.1.2 R-hat (Gelman–Rubin convergence statistic)", "9.1.3 Effective Sample Size (ESS)", "9.1.4 Why all three diagnostics are required", "9.1.5 Operational rule", "9.2 Reliability Gate (EAP)", "Condition", "Logged flag", "Interpretation", "9.3 Theta Stability Gate (Lagged)", "Eligibility", "Definitions", "Conditions", "Logged flags", "9.4 Rank Stability Gate (Lagged)", "9.5 Stopping Configuration Summary (Defaults)", "10. Outputs and Logs", "10.1 Per-Step Log (step_log)", "Step identity and outcome", "Selection routing and health", "Candidate counts (per step; aligned to canonical pipeline §5.2.1)", "Key endpoint diagnostics", "Round scheduling and strata fields", "Star-cap diagnostics (per step)", "10.2 Per-Refit Log (round_log)", "Run scale / sampling state", "Candidate/selection health (stepwise)", "Global identifiability and quota adaptation", "Imbalance / coverage diagnostics", "Model-parameter monitoring (posterior percentiles)", "TrueSkill monitoring (audit-only)", "Near-tie and credible-interval width diagnostics (BTL posterior; audit + interpretability)", "Additional global efficiency metrics (report-only; not used for stopping)", "Stopping gates and core posterior summaries", "Stop decision", "10.3 Item-Level Log (item_log)", "Contents (per refit)", "Access and summaries", "Disk writing", "11. Interpretation Guarantees", "12. Design Philosophy", "13. Citation" ], "created": "2026-02-10 21:06:53", "modified": "2026-02-11 04:37:32", "commits": 3 }, { "source": "getting-started.Rmd", "filename": "getting-started.html", "title": "Getting Started with pairwiseLLM", "engine": "knitr::rmarkdown", "headings": [ "1. Introduction", "2. Setting API Keys", "3. Example Writing Data", "4. Constructing Pairwise Comparisons", "5. Traits and Prompt Templates", "5.1 Using a built-in trait", "5.2 Using or customizing prompt templates", "6. Live Pairwise Comparisons", "7. Preparing Data for BT or Elo Modeling", "8. Bradley–Terry Modeling", "9. Elo Modeling", "10. Batch APIs (Large Jobs)", "10.1 Submit a batch", "10.2 Download results", "10.3 Multi‑Batch Jobs", "10.4 Estimating cost before you run", "11. Backend-Specific Tools", "11.1 OpenAI", "11.2 Anthropic", "11.3 Google Gemini", "11.4 Together.ai (live only)", "11.5 Ollama (local, live only)", "12. Troubleshooting", "Missing API keys", "Timeouts", "Positional bias", "13. Citation" ], "created": "2025-12-05 21:39:39", "modified": "2026-02-10 23:34:14", "commits": 19 }, { "source": "prompt-template-bias.Rmd", "filename": "prompt-template-bias.html", "title": "Prompt Template Positional Bias Testing", "engine": "knitr::rmarkdown", "headings": [ "1. Motivation", "2. Testing Process Summary", "3. Trait descriptions and custom traits", "3.1 Built-in traits", "3.2 Setting a different built-in trait", "3.3 Creating a custom trait", "3.4 Why traits matter for positional bias testing", "4. Example data used in tests", "5. Built-in prompt templates", "6. Forward and reverse pairs", "7. Thinking / Reasoning Configurations Used in Testing", "7.1 Anthropic (Claude 4.5 models)", "thinking = \"no_thinking\"", "thinking = \"with_thinking\"", "7.2 Gemini 3 Pro Preview", "Only thinking = \"with_thinking\" was used", "7.3 OpenAI (gpt-4.1, gpt-4o, gpt-5.1)", "thinking = \"with_thinking\" (gpt-5.1 only)", "7.4 TogetherAI (Deepseek-R1, Deepseek-V3, Kimi-K2, Qwen3)", "7.5 Summary Table of Backend-Specific Behavior", "8. Loading summary results", "8.1 Column definitions", "8.2 Interpreting the statistics", "9. Summary results by prompt", "9.1 Template test1", "9.1.1 Template text", "9.1.2 Summary table", "9.2 Template test2", "9.2.1 Template text", "9.2.2 Summary table", "9.3 Template test3", "9.3.1 Template text", "9.3.2 Summary table", "9.4 Template test4", "9.4.1 Template text", "9.4.2 Summary table", "9.5 Template test5", "9.5.1 Template text", "9.5.2 Summary table", "10. Per-backend summary", "10.1 Anthropic models", "10.2 Gemini models", "10.3 OpenAI models", "10.4 TogetherAI-hosted models", "11. Conclusion", "12. Citation" ], "created": "2025-12-11 19:40:28", "modified": "2026-02-10 23:34:14", "commits": 5 } ], "_score": 5.924279286061881, "_indexed": true, "_nocasepkg": "pairwisellm", "_universes": [ "shmercer" ], "_binaries": [ { "r": "4.7.0", "os": "linux", "version": "1.3.0", "date": "2026-05-12T06:41:52.000Z", "distro": "noble", "commit": "385b4d2a7c5c6a0e3b07e5c03c81c98110ad43f9", "fileid": "f779c070b10621053bd4edb45794a156d470f5f380faa6ac9c477fd5bcd13357", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/shmercer/actions/runs/25717803229" }, { "r": "4.6.0", "os": "linux", "version": "1.3.0", "date": "2026-05-12T06:42:23.000Z", "distro": "noble", "commit": "385b4d2a7c5c6a0e3b07e5c03c81c98110ad43f9", "fileid": "a91d8bf0dd77338e5570b595e7f6edd97493536d46f336c35418d5ed31e756b5", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/shmercer/actions/runs/25717803229" }, { "r": "4.5.3", "os": "mac", "version": "1.3.0", "date": "2026-05-12T06:41:11.000Z", "commit": "385b4d2a7c5c6a0e3b07e5c03c81c98110ad43f9", "fileid": "2cdda74b63f4d0026da93cb2bc6f34b385a363e8656a79aee4031bf33c68a4a6", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/shmercer/actions/runs/25717803229" }, { "r": "4.6.0", "os": "mac", "version": "1.3.0", "date": "2026-05-12T06:41:18.000Z", "commit": "385b4d2a7c5c6a0e3b07e5c03c81c98110ad43f9", "fileid": "3d7d6567e659c2231998f930d59020f69b4be6a3f6846e9cabd7c8b6ee45b232", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/shmercer/actions/runs/25717803229" }, { "r": "4.7.0", "os": "win", "version": "1.3.0", "date": "2026-05-12T06:40:56.000Z", "commit": "385b4d2a7c5c6a0e3b07e5c03c81c98110ad43f9", "fileid": "fc42cc9f293c4dc878585cdfdeb18835659dc208418a6d82f0f1c73da25855ab", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/shmercer/actions/runs/25717803229" }, { "r": "4.5.3", "os": "win", "version": "1.3.0", "date": "2026-05-12T06:40:51.000Z", "commit": "385b4d2a7c5c6a0e3b07e5c03c81c98110ad43f9", "fileid": "8d4e5e7e2324e1229839276609b9517159b7e4813d6bad2ebc20bfaaced90a1d", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/shmercer/actions/runs/25717803229" }, { "r": "4.6.0", "os": "win", "version": "1.3.0", "date": "2026-05-12T06:41:31.000Z", "commit": "385b4d2a7c5c6a0e3b07e5c03c81c98110ad43f9", "fileid": "4b29b262f9fdd6d5e16ceef13e28f85fd83917009593fccfb2aac16bdfaceb6f", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/shmercer/actions/runs/25717803229" }, { "r": "4.6.0", "os": "wasm", "version": "1.3.0", "date": "2026-05-22T10:58:04.000Z", "commit": "385b4d2a7c5c6a0e3b07e5c03c81c98110ad43f9", "fileid": "06af659fb7865b835fadb9046e97526dae86c03bf234ab9f38e4c0cdc4010bc9", "status": "success", "buildurl": "https://github.com/r-universe/shmercer/actions/runs/25717803229" } ] }