{
 "field_extraction_accuracy": {
  "ci95": [
   0.99,
   1
  ],
  "k": 379,
  "n": 379,
  "value": 1.0
 },
 "ground_truth_distribution": {
  "Exception": 43,
  "Needs review": 18,
  "No exception": 35,
  "Not relevant": 12
 },
 "n": 108,
 "variants": {
  "assisted": {
   "category_cases": {
    "correct": [
     "EVR-002",
     "EVR-003",
     "EVR-005",
     "EVR-006",
     "EVR-008",
     "EVR-009",
     "EVR-012",
     "EVR-014",
     "EVR-016",
     "EVR-017",
     "EVR-018",
     "EVR-020",
     "EVR-021",
     "EVR-024",
     "EVR-025",
     "EVR-028",
     "EVR-029",
     "EVR-030",
     "EVR-031",
     "EVR-032",
     "EVR-034",
     "EVR-035",
     "EVR-036",
     "EVR-037",
     "EVR-039",
     "EVR-040",
     "EVR-041",
     "EVR-043",
     "EVR-045",
     "EVR-046",
     "EVR-047",
     "EVR-048",
     "EVR-049",
     "EVR-050",
     "EVR-054",
     "EVR-056",
     "EVR-057",
     "EVR-059",
     "EVR-060",
     "EVR-061",
     "EVR-062",
     "EVR-063",
     "EVR-064",
     "EVR-065",
     "EVR-067",
     "EVR-068",
     "EVR-069",
     "EVR-072",
     "EVR-073",
     "EVR-074",
     "EVR-075",
     "EVR-076",
     "EVR-077",
     "EVR-078",
     "EVR-080",
     "EVR-083",
     "EVR-084",
     "EVR-087",
     "EVR-089",
     "EVR-090",
     "EVR-092",
     "EVR-093",
     "EVR-094",
     "EVR-095",
     "EVR-096",
     "EVR-097",
     "EVR-098",
     "EVR-100",
     "EVR-104",
     "EVR-107",
     "EVR-108"
    ],
    "correct abstention (ambiguous wording)": [
     "EVR-004",
     "EVR-015",
     "EVR-022",
     "EVR-023",
     "EVR-055",
     "EVR-079",
     "EVR-081",
     "EVR-086",
     "EVR-103"
    ],
    "correct abstention (contradictory sources)": [
     "EVR-007",
     "EVR-019",
     "EVR-038",
     "EVR-051",
     "EVR-071",
     "EVR-082",
     "EVR-088",
     "EVR-091",
     "EVR-099"
    ],
    "correct: irrelevant evidence identified": [
     "EVR-001",
     "EVR-013",
     "EVR-026",
     "EVR-027",
     "EVR-033",
     "EVR-042",
     "EVR-053",
     "EVR-058",
     "EVR-101",
     "EVR-102",
     "EVR-105",
     "EVR-106"
    ],
    "false alarm": [
     "EVR-011",
     "EVR-044"
    ],
    "missed exception": [
     "EVR-070",
     "EVR-085"
    ],
    "unnecessary abstention": [
     "EVR-010",
     "EVR-052",
     "EVR-066"
    ]
   },
   "confusion": {
    "Exception": {
     "Exception": 40,
     "Needs review": 1,
     "No exception": 2,
     "Not relevant": 0
    },
    "Needs review": {
     "Exception": 0,
     "Needs review": 18,
     "No exception": 0,
     "Not relevant": 0
    },
    "No exception": {
     "Exception": 2,
     "Needs review": 2,
     "No exception": 31,
     "Not relevant": 0
    },
    "Not relevant": {
     "Exception": 0,
     "Needs review": 0,
     "No exception": 0,
     "Not relevant": 12
    }
   },
   "error_categories": {
    "correct": 71,
    "correct abstention (ambiguous wording)": 9,
    "correct abstention (contradictory sources)": 9,
    "correct: irrelevant evidence identified": 12,
    "false alarm": 2,
    "missed exception": 2,
    "unnecessary abstention": 3
   },
   "metrics": {
    "FN_missed": 2,
    "FP": 2,
    "TP": 40,
    "abstention_rate_all_cases": {
     "ci95": [
      0.131,
      0.279
     ],
     "k": 21,
     "n": 108,
     "value": 0.194
    },
    "abstention_rate_on_decidable": {
     "ci95": [
      0.013,
      0.107
     ],
     "k": 3,
     "n": 78,
     "value": 0.038
    },
    "accuracy_all_cases": {
     "ci95": [
      0.872,
      0.968
     ],
     "k": 101,
     "n": 108,
     "value": 0.935
    },
    "correct_abstention_rate": {
     "ci95": [
      0.824,
      1
     ],
     "k": 18,
     "n": 18,
     "value": 1.0
    },
    "decidable_cases": 78,
    "exceptions_abstained": 1,
    "f1": 0.941,
    "false_negative_rate": {
     "ci95": [
      0.024,
      0.186
     ],
     "k": 3,
     "n": 43,
     "value": 0.07
    },
    "false_positive_rate": {
     "ci95": [
      0.016,
      0.186
     ],
     "k": 2,
     "n": 35,
     "value": 0.057
    },
    "irrelevant_evidence_rate": {
     "ci95": [
      0.757,
      1
     ],
     "k": 12,
     "n": 12,
     "value": 1.0
    },
    "overconfident_on_ambiguous": {
     "ci95": [
      0,
      0.176
     ],
     "k": 0,
     "n": 18,
     "value": 0.0
    },
    "precision": {
     "ci95": [
      0.842,
      0.987
     ],
     "k": 40,
     "n": 42,
     "value": 0.952
    },
    "recall": {
     "ci95": [
      0.814,
      0.976
     ],
     "k": 40,
     "n": 43,
     "value": 0.93
    },
    "recall_excluding_abstentions": {
     "ci95": [
      0.842,
      0.987
     ],
     "k": 40,
     "n": 42,
     "value": 0.952
    }
   }
  },
  "mock_text_only": {
   "category_cases": {
    "correct": [
     "EVR-002",
     "EVR-003",
     "EVR-005",
     "EVR-006",
     "EVR-008",
     "EVR-009",
     "EVR-010",
     "EVR-012",
     "EVR-014",
     "EVR-016",
     "EVR-017",
     "EVR-018",
     "EVR-020",
     "EVR-021",
     "EVR-024",
     "EVR-025",
     "EVR-028",
     "EVR-029",
     "EVR-030",
     "EVR-031",
     "EVR-032",
     "EVR-034",
     "EVR-035",
     "EVR-036",
     "EVR-037",
     "EVR-039",
     "EVR-040",
     "EVR-041",
     "EVR-043",
     "EVR-045",
     "EVR-046",
     "EVR-047",
     "EVR-048",
     "EVR-049",
     "EVR-050",
     "EVR-052",
     "EVR-054",
     "EVR-056",
     "EVR-057",
     "EVR-059",
     "EVR-060",
     "EVR-061",
     "EVR-062",
     "EVR-063",
     "EVR-064",
     "EVR-065",
     "EVR-066",
     "EVR-067",
     "EVR-068",
     "EVR-069",
     "EVR-072",
     "EVR-073",
     "EVR-074",
     "EVR-075",
     "EVR-076",
     "EVR-077",
     "EVR-078",
     "EVR-080",
     "EVR-083",
     "EVR-084",
     "EVR-087",
     "EVR-089",
     "EVR-090",
     "EVR-092",
     "EVR-093",
     "EVR-094",
     "EVR-095",
     "EVR-096",
     "EVR-097",
     "EVR-098",
     "EVR-100",
     "EVR-104",
     "EVR-107",
     "EVR-108"
    ],
    "correct abstention (ambiguous wording)": [
     "EVR-004",
     "EVR-015",
     "EVR-022",
     "EVR-023",
     "EVR-055",
     "EVR-079",
     "EVR-081",
     "EVR-086",
     "EVR-103"
    ],
    "correct: irrelevant evidence identified": [
     "EVR-001",
     "EVR-013",
     "EVR-026",
     "EVR-027",
     "EVR-033",
     "EVR-042",
     "EVR-053",
     "EVR-058",
     "EVR-101",
     "EVR-102",
     "EVR-105",
     "EVR-106"
    ],
    "false alarm": [
     "EVR-011",
     "EVR-044"
    ],
    "missed exception": [
     "EVR-070",
     "EVR-085"
    ],
    "overconfident on contradictory sources": [
     "EVR-007",
     "EVR-019",
     "EVR-038",
     "EVR-051",
     "EVR-071",
     "EVR-082",
     "EVR-088",
     "EVR-091",
     "EVR-099"
    ]
   },
   "confusion": {
    "Exception": {
     "Exception": 41,
     "Needs review": 0,
     "No exception": 2,
     "Not relevant": 0
    },
    "Needs review": {
     "Exception": 0,
     "Needs review": 9,
     "No exception": 9,
     "Not relevant": 0
    },
    "No exception": {
     "Exception": 2,
     "Needs review": 0,
     "No exception": 33,
     "Not relevant": 0
    },
    "Not relevant": {
     "Exception": 0,
     "Needs review": 0,
     "No exception": 0,
     "Not relevant": 12
    }
   },
   "error_categories": {
    "correct": 74,
    "correct abstention (ambiguous wording)": 9,
    "correct: irrelevant evidence identified": 12,
    "false alarm": 2,
    "missed exception": 2,
    "overconfident on contradictory sources": 9
   },
   "metrics": {
    "FN_missed": 2,
    "FP": 2,
    "TP": 41,
    "abstention_rate_all_cases": {
     "ci95": [
      0.044,
      0.151
     ],
     "k": 9,
     "n": 108,
     "value": 0.083
    },
    "abstention_rate_on_decidable": {
     "ci95": [
      0,
      0.047
     ],
     "k": 0,
     "n": 78,
     "value": 0.0
    },
    "accuracy_all_cases": {
     "ci95": [
      0.805,
      0.928
     ],
     "k": 95,
     "n": 108,
     "value": 0.88
    },
    "correct_abstention_rate": {
     "ci95": [
      0.29,
      0.71
     ],
     "k": 9,
     "n": 18,
     "value": 0.5
    },
    "decidable_cases": 78,
    "exceptions_abstained": 0,
    "f1": 0.953,
    "false_negative_rate": {
     "ci95": [
      0.013,
      0.155
     ],
     "k": 2,
     "n": 43,
     "value": 0.047
    },
    "false_positive_rate": {
     "ci95": [
      0.016,
      0.186
     ],
     "k": 2,
     "n": 35,
     "value": 0.057
    },
    "irrelevant_evidence_rate": {
     "ci95": [
      0.757,
      1
     ],
     "k": 12,
     "n": 12,
     "value": 1.0
    },
    "overconfident_on_ambiguous": {
     "ci95": [
      0.29,
      0.71
     ],
     "k": 9,
     "n": 18,
     "value": 0.5
    },
    "precision": {
     "ci95": [
      0.845,
      0.987
     ],
     "k": 41,
     "n": 43,
     "value": 0.953
    },
    "recall": {
     "ci95": [
      0.845,
      0.987
     ],
     "k": 41,
     "n": 43,
     "value": 0.953
    },
    "recall_excluding_abstentions": {
     "ci95": [
      0.845,
      0.987
     ],
     "k": 41,
     "n": 43,
     "value": 0.953
    }
   }
  },
  "rules_only": {
   "category_cases": {
    "correct": [
     "EVR-002",
     "EVR-003",
     "EVR-009",
     "EVR-012",
     "EVR-014",
     "EVR-016",
     "EVR-017",
     "EVR-018",
     "EVR-021",
     "EVR-024",
     "EVR-025",
     "EVR-028",
     "EVR-029",
     "EVR-030",
     "EVR-031",
     "EVR-032",
     "EVR-035",
     "EVR-036",
     "EVR-039",
     "EVR-040",
     "EVR-041",
     "EVR-043",
     "EVR-046",
     "EVR-047",
     "EVR-048",
     "EVR-049",
     "EVR-050",
     "EVR-054",
     "EVR-056",
     "EVR-057",
     "EVR-059",
     "EVR-060",
     "EVR-061",
     "EVR-063",
     "EVR-064",
     "EVR-065",
     "EVR-067",
     "EVR-069",
     "EVR-072",
     "EVR-073",
     "EVR-075",
     "EVR-078",
     "EVR-080",
     "EVR-083",
     "EVR-084",
     "EVR-087",
     "EVR-089",
     "EVR-094",
     "EVR-095",
     "EVR-096",
     "EVR-098",
     "EVR-100",
     "EVR-107",
     "EVR-108"
    ],
    "correct abstention (ambiguous wording)": [
     "EVR-004",
     "EVR-015",
     "EVR-022",
     "EVR-023",
     "EVR-055",
     "EVR-079",
     "EVR-081",
     "EVR-086",
     "EVR-103"
    ],
    "false alarm": [
     "EVR-011",
     "EVR-044"
    ],
    "irrelevant evidence not detected": [
     "EVR-001",
     "EVR-013",
     "EVR-026",
     "EVR-027",
     "EVR-033",
     "EVR-042",
     "EVR-053",
     "EVR-058",
     "EVR-101",
     "EVR-102",
     "EVR-105",
     "EVR-106"
    ],
    "missed exception": [
     "EVR-008",
     "EVR-010",
     "EVR-068",
     "EVR-070",
     "EVR-076",
     "EVR-085",
     "EVR-090"
    ],
    "overconfident on contradictory sources": [
     "EVR-007",
     "EVR-019",
     "EVR-038",
     "EVR-051",
     "EVR-071",
     "EVR-082",
     "EVR-088",
     "EVR-091",
     "EVR-099"
    ],
    "unnecessary abstention": [
     "EVR-005",
     "EVR-006",
     "EVR-020",
     "EVR-034",
     "EVR-037",
     "EVR-045",
     "EVR-052",
     "EVR-062",
     "EVR-066",
     "EVR-074",
     "EVR-077",
     "EVR-092",
     "EVR-093",
     "EVR-097",
     "EVR-104"
    ]
   },
   "confusion": {
    "Exception": {
     "Exception": 30,
     "Needs review": 6,
     "No exception": 7,
     "Not relevant": 0
    },
    "Needs review": {
     "Exception": 9,
     "Needs review": 9,
     "No exception": 0,
     "Not relevant": 0
    },
    "No exception": {
     "Exception": 2,
     "Needs review": 9,
     "No exception": 24,
     "Not relevant": 0
    },
    "Not relevant": {
     "Exception": 0,
     "Needs review": 12,
     "No exception": 0,
     "Not relevant": 0
    }
   },
   "error_categories": {
    "correct": 54,
    "correct abstention (ambiguous wording)": 9,
    "false alarm": 2,
    "irrelevant evidence not detected": 12,
    "missed exception": 7,
    "overconfident on contradictory sources": 9,
    "unnecessary abstention": 15
   },
   "metrics": {
    "FN_missed": 7,
    "FP": 2,
    "TP": 30,
    "abstention_rate_all_cases": {
     "ci95": [
      0.252,
      0.427
     ],
     "k": 36,
     "n": 108,
     "value": 0.333
    },
    "abstention_rate_on_decidable": {
     "ci95": [
      0.12,
      0.293
     ],
     "k": 15,
     "n": 78,
     "value": 0.192
    },
    "accuracy_all_cases": {
     "ci95": [
      0.489,
      0.672
     ],
     "k": 63,
     "n": 108,
     "value": 0.583
    },
    "correct_abstention_rate": {
     "ci95": [
      0.29,
      0.71
     ],
     "k": 9,
     "n": 18,
     "value": 0.5
    },
    "decidable_cases": 78,
    "exceptions_abstained": 6,
    "f1": 0.8,
    "false_negative_rate": {
     "ci95": [
      0.186,
      0.451
     ],
     "k": 13,
     "n": 43,
     "value": 0.302
    },
    "false_positive_rate": {
     "ci95": [
      0.016,
      0.186
     ],
     "k": 2,
     "n": 35,
     "value": 0.057
    },
    "irrelevant_evidence_rate": {
     "ci95": [
      0,
      0.243
     ],
     "k": 0,
     "n": 12,
     "value": 0.0
    },
    "overconfident_on_ambiguous": {
     "ci95": [
      0.29,
      0.71
     ],
     "k": 9,
     "n": 18,
     "value": 0.5
    },
    "precision": {
     "ci95": [
      0.799,
      0.983
     ],
     "k": 30,
     "n": 32,
     "value": 0.938
    },
    "recall": {
     "ci95": [
      0.549,
      0.814
     ],
     "k": 30,
     "n": 43,
     "value": 0.698
    },
    "recall_excluding_abstentions": {
     "ci95": [
      0.658,
      0.905
     ],
     "k": 30,
     "n": 37,
     "value": 0.811
    }
   }
  }
 },
 "wrong_extractions": []
}