{
  "generated_at": "2026-07-25T16:05:48",
  "scope": {
    "result_root": "C:\\edocument.repo\\benchmark-rag\\retrieval-benchmarl-results",
    "old_excluded": true,
    "current_experiments": 8,
    "measured_rows": 208,
    "warmup_rows_excluded": 208,
    "common_cases": 26
  },
  "docx": {
    "path": "C:\\edocument.repo\\benchmark-rag\\-REPORT-001\\RAG-benchmark-academic-report-SOL.docx",
    "bytes": 3814774,
    "paragraphs": 288,
    "tables": 20,
    "inline_shapes": 31
  },
  "xlsx": {
    "path": "C:\\edocument.repo\\benchmark-rag\\-REPORT-001\\RAG-benchmark-results-SOL.xlsx",
    "bytes": 3347387,
    "sheets": [
      "README",
      "Overall_Results",
      "Pairwise_Contrasts",
      "Group_Results",
      "Case_Results",
      "Runtime_Diagnostics",
      "Provenance",
      "Model_Parameters",
      "Equations",
      "Method_Lucene",
      "Method_RAG",
      "Model_Analysis",
      "Mathematics",
      "E01_LUC",
      "E02_STD-WHOLE-RR",
      "E03_STD-CRIT-NO-RR",
      "E04_STD-CRIT-RR",
      "E05_SEL-WHOLE-NO-RR",
      "E06_SEL-WHOLE-RR",
      "E07_SEL-CRIT-NO-RR",
      "E08_SEL-CRIT-RR",
      "Figures"
    ],
    "experiment_sheets": 8,
    "embedded_images": 31,
    "native_charts": 8
  },
  "observed_macro_results": {
    "LUC": {
      "experiment_id": "lucene-manual-syntax-fuzzy-multilingual-01",
      "batch_id": "20260723-135511-9cd3de68",
      "precision_at_10": 0.5230769230769231,
      "recall_at_10": 0.40685862832623965,
      "ndcg_at_10": 0.6186331466900233,
      "mrr": 0.837051282051282,
      "mean_latency_ms": 197.30769230769232,
      "p95_latency_ms": 386.0,
      "relevant_in_final_lists": 467
    },
    "STD-WHOLE-RR": {
      "experiment_id": "selective-weighting-ablation-standard-01",
      "batch_id": "20260720-131949-08f0c1b7",
      "precision_at_10": 0.55,
      "recall_at_10": 0.40934051722007186,
      "ndcg_at_10": 0.6655389457805131,
      "mrr": 0.9423076923076923,
      "mean_latency_ms": 14290.538461538461,
      "p95_latency_ms": 17576.0,
      "relevant_in_final_lists": 256
    },
    "STD-CRIT-NO-RR": {
      "experiment_id": "standard-weight-criteria-reranking-disabled-01",
      "batch_id": "20260722-095549-eb23b551",
      "precision_at_10": 0.4538461538461539,
      "recall_at_10": 0.363356777051109,
      "ndcg_at_10": 0.5476328089551907,
      "mrr": 0.7993589743589743,
      "mean_latency_ms": 1931.5384615384614,
      "p95_latency_ms": 2141.0,
      "relevant_in_final_lists": 382
    },
    "STD-CRIT-RR": {
      "experiment_id": "standard-weight-criteria-reranking-01",
      "batch_id": "20260720-170236-8d7baeed",
      "precision_at_10": 0.5769230769230769,
      "recall_at_10": 0.445546138521847,
      "ndcg_at_10": 0.6967984185411958,
      "mrr": 0.9423076923076923,
      "mean_latency_ms": 14469.115384615385,
      "p95_latency_ms": 19823.0,
      "relevant_in_final_lists": 269
    },
    "SEL-WHOLE-NO-RR": {
      "experiment_id": "selective-weight-whole-description-reranking-disabled-01",
      "batch_id": "20260722-145931-aca849af",
      "precision_at_10": 0.49615384615384617,
      "recall_at_10": 0.38095516775172644,
      "ndcg_at_10": 0.5679727405782733,
      "mrr": 0.7875457875457875,
      "mean_latency_ms": 1667.0,
      "p95_latency_ms": 1824.0,
      "relevant_in_final_lists": 389
    },
    "SEL-WHOLE-RR": {
      "experiment_id": "selective-weight-whole-description-reranking-01",
      "batch_id": "20260722-080546-3e6f67df",
      "precision_at_10": 0.5653846153846154,
      "recall_at_10": 0.4286734278991364,
      "ndcg_at_10": 0.6809653325309124,
      "mrr": 0.9615384615384616,
      "mean_latency_ms": 14092.5,
      "p95_latency_ms": 18719.0,
      "relevant_in_final_lists": 266
    },
    "SEL-CRIT-NO-RR": {
      "experiment_id": "selective-weight-criteria-reranking-disabled-01",
      "batch_id": "20260722-142120-947cf272",
      "precision_at_10": 0.4653846153846154,
      "recall_at_10": 0.354975845679287,
      "ndcg_at_10": 0.5270276355729193,
      "mrr": 0.7701923076923076,
      "mean_latency_ms": 1775.576923076923,
      "p95_latency_ms": 2162.0,
      "relevant_in_final_lists": 381
    },
    "SEL-CRIT-RR": {
      "experiment_id": "selective-weight-criteria-reranking-01",
      "batch_id": "20260722-074644-b85ccb96",
      "precision_at_10": 0.573076923076923,
      "recall_at_10": 0.4369526380052696,
      "ndcg_at_10": 0.6891078750969292,
      "mrr": 0.9230769230769231,
      "mean_latency_ms": 14288.923076923076,
      "p95_latency_ms": 20243.0,
      "relevant_in_final_lists": 263
    }
  },
  "key_paired_comparisons": [
    {
      "treatment": "RAG Standard Criteria + rerank",
      "control": "RAG Standard Criteria no rerank",
      "metric": "ndcg_at_10",
      "cases": 26,
      "mean_difference": 0.14916561,
      "median_difference": 0.09437273999999995,
      "ci95": [
        0.0557262773269231,
        0.2526105202596154
      ],
      "wins": 16,
      "ties": 2,
      "losses": 8,
      "contrast": "Rerank effect: standard criteria"
    },
    {
      "treatment": "RAG Standard Criteria + rerank",
      "control": "RAG Standard Criteria no rerank",
      "metric": "total_ms",
      "cases": 26,
      "mean_difference": 12537.576923076924,
      "median_difference": 12482.0,
      "ci95": [
        11700.0375,
        13411.411538461536
      ],
      "wins": 26,
      "ties": 0,
      "losses": 0,
      "contrast": "Rerank effect: standard criteria"
    },
    {
      "treatment": "RAG Selective Criteria + rerank",
      "control": "RAG Selective Criteria no rerank",
      "metric": "ndcg_at_10",
      "cases": 26,
      "mean_difference": 0.16208023846153846,
      "median_difference": 0.15000126999999996,
      "ci95": [
        0.07069455849999999,
        0.25993661780769217
      ],
      "wins": 18,
      "ties": 1,
      "losses": 7,
      "contrast": "Rerank effect: selective criteria"
    },
    {
      "treatment": "RAG Selective Criteria + rerank",
      "control": "RAG Selective Criteria no rerank",
      "metric": "total_ms",
      "cases": 26,
      "mean_difference": 12513.346153846154,
      "median_difference": 12388.5,
      "ci95": [
        11557.884615384615,
        13479.28173076923
      ],
      "wins": 26,
      "ties": 0,
      "losses": 0,
      "contrast": "Rerank effect: selective criteria"
    },
    {
      "treatment": "RAG Selective Whole + rerank",
      "control": "RAG Selective Whole no rerank",
      "metric": "ndcg_at_10",
      "cases": 26,
      "mean_difference": 0.11299259269230769,
      "median_difference": 0.10148556000000003,
      "ci95": [
        0.05082197463461537,
        0.17817748582692294
      ],
      "wins": 19,
      "ties": 1,
      "losses": 6,
      "contrast": "Rerank effect: selective whole"
    },
    {
      "treatment": "RAG Selective Whole + rerank",
      "control": "RAG Selective Whole no rerank",
      "metric": "total_ms",
      "cases": 26,
      "mean_difference": 12425.5,
      "median_difference": 12078.5,
      "ci95": [
        11548.217307692308,
        13316.90096153846
      ],
      "wins": 26,
      "ties": 0,
      "losses": 0,
      "contrast": "Rerank effect: selective whole"
    },
    {
      "treatment": "RAG Selective Criteria + rerank",
      "control": "RAG Standard Criteria + rerank",
      "metric": "ndcg_at_10",
      "cases": 26,
      "mean_difference": -0.007690544230769235,
      "median_difference": 0.0,
      "ci95": [
        -0.030190746000000008,
        0.011903596894230711
      ],
      "wins": 8,
      "ties": 10,
      "losses": 8,
      "contrast": "Scope/weight effect: criteria + rerank"
    },
    {
      "treatment": "RAG Selective Criteria + rerank",
      "control": "RAG Standard Criteria + rerank",
      "metric": "total_ms",
      "cases": 26,
      "mean_difference": -180.19230769230768,
      "median_difference": -257.0,
      "ci95": [
        -376.1923076923077,
        16.193269230769147
      ],
      "wins": 10,
      "ties": 0,
      "losses": 16,
      "contrast": "Scope/weight effect: criteria + rerank"
    },
    {
      "treatment": "RAG Selective Criteria no rerank",
      "control": "RAG Standard Criteria no rerank",
      "metric": "ndcg_at_10",
      "cases": 26,
      "mean_difference": -0.02060517269230768,
      "median_difference": 0.0,
      "ci95": [
        -0.07273142970192309,
        0.02462582475000001
      ],
      "wins": 12,
      "ties": 4,
      "losses": 10,
      "contrast": "Scope/weight effect: criteria no rerank"
    },
    {
      "treatment": "RAG Selective Criteria no rerank",
      "control": "RAG Standard Criteria no rerank",
      "metric": "total_ms",
      "cases": 26,
      "mean_difference": -155.96153846153845,
      "median_difference": -80.5,
      "ci95": [
        -310.00288461538463,
        -58.65384615384615
      ],
      "wins": 6,
      "ties": 0,
      "losses": 20,
      "contrast": "Scope/weight effect: criteria no rerank"
    },
    {
      "treatment": "RAG Selective Criteria + rerank",
      "control": "RAG Selective Whole + rerank",
      "metric": "ndcg_at_10",
      "cases": 26,
      "mean_difference": 0.008142541538461535,
      "median_difference": 0.0,
      "ci95": [
        -0.020756495076923072,
        0.03664056794230769
      ],
      "wins": 12,
      "ties": 3,
      "losses": 11,
      "contrast": "Criteria vs whole: selective + rerank"
    },
    {
      "treatment": "RAG Selective Criteria + rerank",
      "control": "RAG Selective Whole + rerank",
      "metric": "total_ms",
      "cases": 26,
      "mean_difference": 196.42307692307693,
      "median_difference": 191.0,
      "ci95": [
        -70.96826923076922,
        465.69326923076915
      ],
      "wins": 17,
      "ties": 0,
      "losses": 9,
      "contrast": "Criteria vs whole: selective + rerank"
    },
    {
      "treatment": "RAG Selective Criteria no rerank",
      "control": "RAG Selective Whole no rerank",
      "metric": "ndcg_at_10",
      "cases": 26,
      "mean_difference": -0.040945104230769226,
      "median_difference": -0.004536069999999989,
      "ci95": [
        -0.137233207625,
        0.05121045257692305
      ],
      "wins": 11,
      "ties": 1,
      "losses": 14,
      "contrast": "Criteria vs whole: selective no rerank"
    },
    {
      "treatment": "RAG Selective Criteria no rerank",
      "control": "RAG Selective Whole no rerank",
      "metric": "total_ms",
      "cases": 26,
      "mean_difference": 108.57692307692308,
      "median_difference": 167.5,
      "ci95": [
        -99.84807692307692,
        265.03846153846155
      ],
      "wins": 21,
      "ties": 0,
      "losses": 5,
      "contrast": "Criteria vs whole: selective no rerank"
    },
    {
      "treatment": "RAG Standard Criteria + rerank",
      "control": "RAG Standard Whole + rerank",
      "metric": "ndcg_at_10",
      "cases": 26,
      "mean_difference": 0.031259472307692306,
      "median_difference": 0.0,
      "ci95": [
        -0.0014401307788461685,
        0.0688748998653846
      ],
      "wins": 11,
      "ties": 4,
      "losses": 11,
      "contrast": "Criteria vs whole: standard + rerank"
    },
    {
      "treatment": "RAG Standard Criteria + rerank",
      "control": "RAG Standard Whole + rerank",
      "metric": "total_ms",
      "cases": 26,
      "mean_difference": 178.57692307692307,
      "median_difference": 347.5,
      "ci95": [
        -116.27019230769231,
        483.42596153846125
      ],
      "wins": 15,
      "ties": 0,
      "losses": 11,
      "contrast": "Criteria vs whole: standard + rerank"
    },
    {
      "treatment": "RAG Standard Whole + rerank",
      "control": "Lucene",
      "metric": "ndcg_at_10",
      "cases": 26,
      "mean_difference": 0.04690579923076923,
      "median_difference": 0.08180511000000001,
      "ci95": [
        -0.02387125566346153,
        0.11778735099038455
      ],
      "wins": 14,
      "ties": 1,
      "losses": 11,
      "contrast": "RAG Standard Whole + rerank versus Lucene"
    },
    {
      "treatment": "RAG Standard Whole + rerank",
      "control": "Lucene",
      "metric": "total_ms",
      "cases": 26,
      "mean_difference": 14093.23076923077,
      "median_difference": 13920.0,
      "ci95": [
        13191.342307692308,
        15127.429807692306
      ],
      "wins": 26,
      "ties": 0,
      "losses": 0,
      "contrast": "RAG Standard Whole + rerank versus Lucene"
    },
    {
      "treatment": "RAG Standard Criteria no rerank",
      "control": "Lucene",
      "metric": "ndcg_at_10",
      "cases": 26,
      "mean_difference": -0.07100033846153847,
      "median_difference": -0.07778657500000002,
      "ci95": [
        -0.1613629012980769,
        0.01707715596153841
      ],
      "wins": 8,
      "ties": 1,
      "losses": 17,
      "contrast": "RAG Standard Criteria no rerank versus Lucene"
    },
    {
      "treatment": "RAG Standard Criteria no rerank",
      "control": "Lucene",
      "metric": "total_ms",
      "cases": 26,
      "mean_difference": 1734.2307692307693,
      "median_difference": 1537.5,
      "ci95": [
        1464.1528846153847,
        2193.620192307692
      ],
      "wins": 26,
      "ties": 0,
      "losses": 0,
      "contrast": "RAG Standard Criteria no rerank versus Lucene"
    },
    {
      "treatment": "RAG Standard Criteria + rerank",
      "control": "Lucene",
      "metric": "ndcg_at_10",
      "cases": 26,
      "mean_difference": 0.07816527153846153,
      "median_difference": 0.09913379,
      "ci95": [
        -0.0007120110673076836,
        0.15568686126923048
      ],
      "wins": 14,
      "ties": 1,
      "losses": 11,
      "contrast": "RAG Standard Criteria + rerank versus Lucene"
    },
    {
      "treatment": "RAG Standard Criteria + rerank",
      "control": "Lucene",
      "metric": "total_ms",
      "cases": 26,
      "mean_difference": 14271.807692307691,
      "median_difference": 14012.5,
      "ci95": [
        13242.409615384615,
        15419.169230769228
      ],
      "wins": 26,
      "ties": 0,
      "losses": 0,
      "contrast": "RAG Standard Criteria + rerank versus Lucene"
    },
    {
      "treatment": "RAG Selective Whole no rerank",
      "control": "Lucene",
      "metric": "ndcg_at_10",
      "cases": 26,
      "mean_difference": -0.050660406923076926,
      "median_difference": -0.0033164649999999907,
      "ci95": [
        -0.15155929841346155,
        0.048553585846153824
      ],
      "wins": 12,
      "ties": 1,
      "losses": 13,
      "contrast": "RAG Selective Whole no rerank versus Lucene"
    },
    {
      "treatment": "RAG Selective Whole no rerank",
      "control": "Lucene",
      "metric": "total_ms",
      "cases": 26,
      "mean_difference": 1469.6923076923076,
      "median_difference": 1242.5,
      "ci95": [
        1210.0375,
        1938.385576923077
      ],
      "wins": 26,
      "ties": 0,
      "losses": 0,
      "contrast": "RAG Selective Whole no rerank versus Lucene"
    },
    {
      "treatment": "RAG Selective Whole + rerank",
      "control": "Lucene",
      "metric": "ndcg_at_10",
      "cases": 26,
      "mean_difference": 0.06233218576923077,
      "median_difference": 0.08290953000000001,
      "ci95": [
        -0.006866988278846157,
        0.1311369164807692
      ],
      "wins": 16,
      "ties": 1,
      "losses": 9,
      "contrast": "RAG Selective Whole + rerank versus Lucene"
    },
    {
      "treatment": "RAG Selective Whole + rerank",
      "control": "Lucene",
      "metric": "total_ms",
      "cases": 26,
      "mean_difference": 13895.192307692309,
      "median_difference": 13313.0,
      "ci95": [
        12854.846153846154,
        15079.963461538462
      ],
      "wins": 26,
      "ties": 0,
      "losses": 0,
      "contrast": "RAG Selective Whole + rerank versus Lucene"
    },
    {
      "treatment": "RAG Selective Criteria no rerank",
      "control": "Lucene",
      "metric": "ndcg_at_10",
      "cases": 26,
      "mean_difference": -0.09160551115384614,
      "median_difference": -0.062709975,
      "ci95": [
        -0.18387735545192307,
        -0.0008910616826923994
      ],
      "wins": 7,
      "ties": 1,
      "losses": 18,
      "contrast": "RAG Selective Criteria no rerank versus Lucene"
    },
    {
      "treatment": "RAG Selective Criteria no rerank",
      "control": "Lucene",
      "metric": "total_ms",
      "cases": 26,
      "mean_difference": 1578.2692307692307,
      "median_difference": 1426.0,
      "ci95": [
        1387.7298076923078,
        1894.5019230769228
      ],
      "wins": 26,
      "ties": 0,
      "losses": 0,
      "contrast": "RAG Selective Criteria no rerank versus Lucene"
    },
    {
      "treatment": "RAG Selective Criteria + rerank",
      "control": "Lucene",
      "metric": "ndcg_at_10",
      "cases": 26,
      "mean_difference": 0.0704747273076923,
      "median_difference": 0.06643835999999997,
      "ci95": [
        -0.005921317009615399,
        0.14600829186538447
      ],
      "wins": 15,
      "ties": 1,
      "losses": 10,
      "contrast": "RAG Selective Criteria + rerank versus Lucene"
    },
    {
      "treatment": "RAG Selective Criteria + rerank",
      "control": "Lucene",
      "metric": "total_ms",
      "cases": 26,
      "mean_difference": 14091.615384615385,
      "median_difference": 13744.5,
      "ci95": [
        12988.670192307693,
        15264.583653846154
      ],
      "wins": 26,
      "ties": 0,
      "losses": 0,
      "contrast": "RAG Selective Criteria + rerank versus Lucene"
    }
  ],
  "checks": {
    "eight_current_runs": true,
    "identical_26_case_sets": true,
    "all_manifests_succeeded": true,
    "all_26_measured_succeeded": true,
    "all_26_warmups_present": true,
    "all_hashes_match": true,
    "all_required_top_level_artifacts": true,
    "all_judgments_resolved": true,
    "csv_macro_matches_aggregate": true,
    "eight_experiment_worksheets": true,
    "experiment_chart_sources_numeric": true,
    "experiment_one_page_configuration": true,
    "eight_native_experiment_charts": true,
    "thirty_one_word_figures": true,
    "thirty_one_excel_figures": true,
    "academic_method_sheets_present": true,
    "embedding_model_analysis_present": true,
    "reranker_model_analysis_present": true,
    "full_mathematics_present": true,
    "critical_model_interpretation_present": true,
    "requested_conclusion_present": true,
    "old_missing_result_claim_removed": true,
    "measured_result_count_stated": true,
    "lucene_values_present": true,
    "rag_values_present": true,
    "toc_field_present": true
  }
}