From ddf40241a9fbdabbee290c5dd62be0d95db44a47 Mon Sep 17 00:00:00 2001 From: Christoph Auer <60343111+cau-git@users.noreply.github.com> Date: Mon, 7 Apr 2025 14:50:14 +0200 Subject: [PATCH] Add README for Docling-DPBench (#60) Signed-off-by: Christoph Auer --- docling_eval/cli/main.py | 2 +- docling_eval/datamodels/dataset_record.py | 2 +- docs/DP-Bench_benchmarks.md | 4 +- docs/Docling-DP-Bench_benchmarks.md | 159 + .../evaluation_DPBench_layout.json | 2338 +++ ...evaluation_DPBench_layout_mAP_0.5_0.95.png | Bin 0 -> 20735 bytes ...evaluation_DPBench_layout_mAP_0.5_0.95.txt | 44 + .../evaluation_DPBench_markdown_text.json | 2510 ++++ .../evaluation_DPBench_markdown_text.txt | 158 + .../evaluation_DPBench_markdown_text_BLEU.png | Bin 0 -> 18878 bytes .../evaluation_DPBench_markdown_text_F1.png | Bin 0 -> 19972 bytes ...on_DPBench_markdown_text_edit_distance.png | Bin 0 -> 19665 bytes ...valuation_DPBench_markdown_text_meteor.png | Bin 0 -> 20790 bytes ...uation_DPBench_markdown_text_precision.png | Bin 0 -> 20019 bytes ...valuation_DPBench_markdown_text_recall.png | Bin 0 -> 20435 bytes .../evaluation_DPBench_reading_order.json | 11698 ++++++++++++++++ ...luation_DPBench_reading_order_ARD_norm.png | Bin 0 -> 20108 bytes ...luation_DPBench_reading_order_ARD_norm.txt | 26 + ...ion_DPBench_reading_order_weighted_ARD.png | Bin 0 -> 20281 bytes ...ion_DPBench_reading_order_weighted_ARD.txt | 26 + ..._DPBench_table_structure-delta_row_col.png | Bin 0 -> 26618 bytes .../evaluation_DPBench_table_structure.json | 718 + ...Bench_table_structure_TEDS_struct-only.png | Bin 0 -> 19818 bytes ...Bench_table_structure_TEDS_struct-only.txt | 26 + ..._table_structure_TEDS_struct-with-text.png | Bin 0 -> 20196 bytes ..._table_structure_TEDS_struct-with-text.txt | 26 + 26 files changed, 17733 insertions(+), 4 deletions(-) create mode 100644 docs/Docling-DP-Bench_benchmarks.md create mode 100644 docs/evaluations/Docling-DPBench/evaluation_DPBench_layout.json create mode 100644 docs/evaluations/Docling-DPBench/evaluation_DPBench_layout_mAP_0.5_0.95.png create mode 100644 docs/evaluations/Docling-DPBench/evaluation_DPBench_layout_mAP_0.5_0.95.txt create mode 100644 docs/evaluations/Docling-DPBench/evaluation_DPBench_markdown_text.json create mode 100644 docs/evaluations/Docling-DPBench/evaluation_DPBench_markdown_text.txt create mode 100644 docs/evaluations/Docling-DPBench/evaluation_DPBench_markdown_text_BLEU.png create mode 100644 docs/evaluations/Docling-DPBench/evaluation_DPBench_markdown_text_F1.png create mode 100644 docs/evaluations/Docling-DPBench/evaluation_DPBench_markdown_text_edit_distance.png create mode 100644 docs/evaluations/Docling-DPBench/evaluation_DPBench_markdown_text_meteor.png create mode 100644 docs/evaluations/Docling-DPBench/evaluation_DPBench_markdown_text_precision.png create mode 100644 docs/evaluations/Docling-DPBench/evaluation_DPBench_markdown_text_recall.png create mode 100644 docs/evaluations/Docling-DPBench/evaluation_DPBench_reading_order.json create mode 100644 docs/evaluations/Docling-DPBench/evaluation_DPBench_reading_order_ARD_norm.png create mode 100644 docs/evaluations/Docling-DPBench/evaluation_DPBench_reading_order_ARD_norm.txt create mode 100644 docs/evaluations/Docling-DPBench/evaluation_DPBench_reading_order_weighted_ARD.png create mode 100644 docs/evaluations/Docling-DPBench/evaluation_DPBench_reading_order_weighted_ARD.txt create mode 100644 docs/evaluations/Docling-DPBench/evaluation_DPBench_table_structure-delta_row_col.png create mode 100644 docs/evaluations/Docling-DPBench/evaluation_DPBench_table_structure.json create mode 100644 docs/evaluations/Docling-DPBench/evaluation_DPBench_table_structure_TEDS_struct-only.png create mode 100644 docs/evaluations/Docling-DPBench/evaluation_DPBench_table_structure_TEDS_struct-only.txt create mode 100644 docs/evaluations/Docling-DPBench/evaluation_DPBench_table_structure_TEDS_struct-with-text.png create mode 100644 docs/evaluations/Docling-DPBench/evaluation_DPBench_table_structure_TEDS_struct-with-text.txt diff --git a/docling_eval/cli/main.py b/docling_eval/cli/main.py index 3f26ba9..423661c 100644 --- a/docling_eval/cli/main.py +++ b/docling_eval/cli/main.py @@ -257,7 +257,7 @@ def get_prediction_provider( elif provider_type == PredictionProviderType.TABLEFORMER: return TableFormerPredictionProvider( - do_visualization=True, + do_visualization=False, ignore_missing_predictions=True, ) diff --git a/docling_eval/datamodels/dataset_record.py b/docling_eval/datamodels/dataset_record.py index 75461ac..c7585fc 100644 --- a/docling_eval/datamodels/dataset_record.py +++ b/docling_eval/datamodels/dataset_record.py @@ -36,7 +36,7 @@ class DatasetRecord( alias="GroundTruthPictures", default=[] ) - mime_type: str = Field(default="") + mime_type: str = Field(default="application/pdf") modalities: List[EvaluationModality] = Field(default=[]) model_config = ConfigDict(arbitrary_types_allowed=True, populate_by_name=True) diff --git a/docs/DP-Bench_benchmarks.md b/docs/DP-Bench_benchmarks.md index 87aac9a..0b4202b 100644 --- a/docs/DP-Bench_benchmarks.md +++ b/docs/DP-Bench_benchmarks.md @@ -12,14 +12,14 @@ docling_eval create-gt --benchmark DPBench --output-dir ./benchmarks/DPBench-gt/ docling_eval create-eval \ --modality end-to-end \ --benchmark DPBench \ - --gt-dir ./benchmarks/DPBench-gt/ \ + --gt-dir ./benchmarks/DPBench-gt/gt_dataset/ \ --output-dir ./benchmarks/DPBench-e2e/ \ --prediction-provider docling # use full-document predictions from docling docling_eval create-eval \ --modality table_structure \ --benchmark DPBench \ - --gt-dir ./benchmarks/DPBench-gt/ \ + --gt-dir ./benchmarks/DPBench-gt/gt_dataset/ \ --output-dir ./benchmarks/DPBench-tables/ \ --prediction-provider tableformer # use tableformer predictions only ``` diff --git a/docs/Docling-DP-Bench_benchmarks.md b/docs/Docling-DP-Bench_benchmarks.md new file mode 100644 index 0000000..4698373 --- /dev/null +++ b/docs/Docling-DP-Bench_benchmarks.md @@ -0,0 +1,159 @@ +# Docling-DP-Bench Benchmarks + +[Docling-DP-Bench on HuggingFace](https://huggingface.co/datasets/ds4sd/docling-dpbench) +Docling-DP-Bench is a re-annotated version of the original `upstage/dpbench` dataset with Docling-native labels. + +Create Docling-DPBench evaluation datasets: + +```sh +# Download the GT straight from HuggingFace +huggingface-cli download --repo-type dataset --local-dir ./benchmarks/Docling-DPBench-gt/gt_dataset ds4sd/docling-dpbench +# Make predictions for different modalities. +docling_eval create-eval \ + --modality end-to-end \ + --benchmark DPBench \ + --gt-dir ./benchmarks/Docling-DPBench-gt/gt_dataset/ \ + --output-dir ./benchmarks/Docling-DPBench-e2e/ \ + --prediction-provider docling # use full-document predictions from docling + +docling_eval create-eval \ + --modality table_structure \ + --benchmark DPBench \ + --gt-dir ./benchmarks/Docling-DPBench-gt/gt_dataset/ \ + --output-dir ./benchmarks/Docling-DPBench-tables/ \ + --prediction-provider tableformer # use tableformer predictions only +``` + +## Layout Evaluation + +Create the evaluation report: + +```sh +docling_eval evaluate \ + --modality layout \ + --benchmark DPBench \ + --output-dir ./benchmarks/Docling-DPBench-e2e/ + +``` + +[Layout evaluation json](evaluations/Docling-DPBench/evaluation_DPBench_layout.json) + +Visualize the report: + +```sh +docling_eval visualize \ + --modality layout \ + --benchmark DPBench \ + --output-dir ./benchmarks/Docling-DPBench-e2e/ +``` + +[mAP[0.5:0.95] report](evaluations/Docling-DPBench/evaluation_DPBench_layout_mAP_0.5_0.95.txt) + +![mAP[0.5:0.95] plot](evaluations/Docling-DPBench/evaluation_DPBench_layout_mAP_0.5_0.95.png) + + +## TableFormer Evaluation + +Create the evaluation report: + +```sh +docling_eval evaluate \ + --modality table_structure \ + --benchmark DPBench \ + --output-dir ./benchmarks/Docling-DPBench-tables/ +``` + + +Visualize the report: + +[Tableformer evaluation json](evaluations/Docling-DPBench/evaluation_DPBench_tableformer.json) + +Visualize the report: + +```sh +docling_eval visualize \ + --modality table_structure \ + --benchmark DPBench \ + --output-dir ./benchmarks/Docling-DPBench-tables/ +``` + +![TEDS plot](evaluations/Docling-DPBench/evaluation_DPBench_tableformer-delta_row_col.png) + +![TEDS struct only plot](evaluations/Docling-DPBench/evaluation_DPBench_table_structure_TEDS_struct-only.png) + +[TEDS struct only report](evaluations/Docling-DPBench/evaluation_DPBench_table_structure_TEDS_struct-only.txt) + +![TEDS struct with text plot](evaluations/Docling-DPBench/evaluation_DPBench_table_structure_TEDS_struct-with-text.png) + +[TEDS struct with text report](evaluations/Docling-DPBench/evaluation_DPBench_table_structure_TEDS_struct-with-text.txt) + + +## Reading order Evaluation + +Create the evaluation report: + +```sh +docling_eval evaluate \ + --modality reading_order \ + --benchmark DPBench \ + --output-dir ./benchmarks/Docling-DPBench-e2e/ +``` + +[Reading order json](evaluations/Docling-DPBench/evaluation_DPBench_reading_order.json) + +Visualize the report: + +```sh +docling_eval visualize \ + --modality reading_order \ + --benchmark DPBench \ + --output-dir ./benchmarks/Docling-DPBench-e2e/ +``` + +![ARD plot](evaluations/Docling-DPBench/evaluation_DPBench_reading_order_ARD_norm.png) + +[ARD report](evaluations/Docling-DPBench/evaluation_DPBench_reading_order_ARD_norm.txt) + +![Weighted ARD plot](evaluations/Docling-DPBench/evaluation_DPBench_reading_order_weighted_ARD.png) + +[Weighted ARD report](evaluations/Docling-DPBench/evaluation_DPBench_reading_order_weighted_ARD.txt) + + +## Markdown text Evaluation + +Create the evaluation report: + +```sh +docling_eval evaluate \ + --modality markdown_text \ + --benchmark DPBench \ + --output-dir ./benchmarks/Docling-DPBench-e2e/ +``` + +[Markdown text json](evaluations/Docling-DPBench/evaluation_DPBench_markdown_text.json) + + +Visualize the report: + +```sh +docling_eval visualize \ + --modality markdown_text \ + --benchmark DPBench \ + --output-dir ./benchmarks/Docling-DPBench-e2e/ +``` + + +[Markdown text report](evaluations/Docling-DPBench/evaluation_DPBench_markdown_text.txt) + + +![BLEU plot](evaluations/Docling-DPBench/evaluation_DPBench_markdown_text_BLEU.png) + +![Edit distance plot](evaluations/Docling-DPBench/evaluation_DPBench_markdown_text_edit_distance.png) + +![F1 plot](evaluations/Docling-DPBench/evaluation_DPBench_markdown_text_F1.png) + +![Meteor plot](evaluations/Docling-DPBench/evaluation_DPBench_markdown_text_meteor.png) + +![Precision plot](evaluations/Docling-DPBench/evaluation_DPBench_markdown_text_precision.png) + +![Recall plot](evaluations/Docling-DPBench/evaluation_DPBench_markdown_text_recall.png) diff --git a/docs/evaluations/Docling-DPBench/evaluation_DPBench_layout.json b/docs/evaluations/Docling-DPBench/evaluation_DPBench_layout.json new file mode 100644 index 0000000..d1ed544 --- /dev/null +++ b/docs/evaluations/Docling-DPBench/evaluation_DPBench_layout.json @@ -0,0 +1,2338 @@ +{ + "evaluations_per_class": [ + { + "label": "document_index", + "name": "Class mAP[0.5:0.95]", + "value": 1.0 + }, + { + "label": "picture", + "name": "Class mAP[0.5:0.95]", + "value": 0.9257975816726685 + }, + { + "label": "table", + "name": "Class mAP[0.5:0.95]", + "value": 0.9054929614067078 + }, + { + "label": "text", + "name": "Class mAP[0.5:0.95]", + "value": 0.6886561512947083 + }, + { + "label": "list_item", + "name": "Class mAP[0.5:0.95]", + "value": 0.5286591053009033 + }, + { + "label": "formula", + "name": "Class mAP[0.5:0.95]", + "value": 0.5131872296333313 + }, + { + "label": "section_header", + "name": "Class mAP[0.5:0.95]", + "value": 0.47209787368774414 + }, + { + "label": "footnote", + "name": "Class mAP[0.5:0.95]", + "value": 0.3772032856941223 + }, + { + "label": "caption", + "name": "Class mAP[0.5:0.95]", + "value": 0.14434443414211273 + } + ], + "evaluations_per_image": [ + { + "avg_weighted_label_matched_iou_50": 0.9993258714675903, + "avg_weighted_label_matched_iou_75": 0.9993258714675903, + "avg_weighted_label_matched_iou_90": 0.9993258714675903, + "avg_weighted_label_matched_iou_95": 0.9993258714675903, + "map_50": 1.0, + "map_75": 1.0, + "map_val": 1.0, + "name": "doc_d1437ae1c6bedfa2bbf72e4d04dee3a01896dd045024438ee7c4e775e0ab6e9c_page_000001.png-page-0", + "value": 0.9993258714675903 + }, + { + "avg_weighted_label_matched_iou_50": 0.9992531538009644, + "avg_weighted_label_matched_iou_75": 0.9992531538009644, + "avg_weighted_label_matched_iou_90": 0.9992531538009644, + "avg_weighted_label_matched_iou_95": 0.9992531538009644, + "map_50": 1.0, + "map_75": 1.0, + "map_val": 1.0, + "name": "doc_1cf54980192667fd7a6977b301a175e98be02b11651bbf78b6cae338858db2aa_page_000001.png-page-0", + "value": 0.9992531538009644 + }, + { + "avg_weighted_label_matched_iou_50": 0.9991626143455505, + "avg_weighted_label_matched_iou_75": 0.9991626143455505, + "avg_weighted_label_matched_iou_90": 0.9991626143455505, + "avg_weighted_label_matched_iou_95": 0.9991626143455505, + "map_50": 0.5, + "map_75": 0.5, + "map_val": 0.5, + "name": "doc_be866d1b599b669a33acbc07b07847d6c72011cf4d221b20d889ad5a90ab30f5_page_000001.png-page-0", + "value": 0.9991626143455505 + }, + { + "avg_weighted_label_matched_iou_50": 0.9985700845718384, + "avg_weighted_label_matched_iou_75": 0.9985700845718384, + "avg_weighted_label_matched_iou_90": 0.9985700845718384, + "avg_weighted_label_matched_iou_95": 0.9985700845718384, + "map_50": 0.5, + "map_75": 0.5, + "map_val": 0.5, + "name": "doc_9713894c940f2ad16f015d857757b003e4ecaaa1602afa72799c08526806f533_page_000001.png-page-0", + "value": 0.9985700845718384 + }, + { + "avg_weighted_label_matched_iou_50": 0.9983854293823242, + "avg_weighted_label_matched_iou_75": 0.9983854293823242, + "avg_weighted_label_matched_iou_90": 0.9983854293823242, + "avg_weighted_label_matched_iou_95": 0.9983854293823242, + "map_50": 1.0, + "map_75": 1.0, + "map_val": 1.0, + "name": "doc_a46e40b2b04dc83e4014fca656dae6df20af1fe7a420df306406409ea8c0db31_page_000001.png-page-0", + "value": 0.9983854293823242 + }, + { + "avg_weighted_label_matched_iou_50": 0.9983096122741699, + "avg_weighted_label_matched_iou_75": 0.9983096122741699, + "avg_weighted_label_matched_iou_90": 0.9983096122741699, + "avg_weighted_label_matched_iou_95": 0.9983096122741699, + "map_50": 1.0, + "map_75": 1.0, + "map_val": 1.0, + "name": "doc_0cf2e856478f66d96690a306b55ff40c01ab24cb26d048fe248eac8e3d66e65c_page_000001.png-page-0", + "value": 0.9983096122741699 + }, + { + "avg_weighted_label_matched_iou_50": 0.9979795217514038, + "avg_weighted_label_matched_iou_75": 0.9979795217514038, + "avg_weighted_label_matched_iou_90": 0.9979795217514038, + "avg_weighted_label_matched_iou_95": 0.9979795217514038, + "map_50": 0.5, + "map_75": 0.5, + "map_val": 0.5, + "name": "doc_daa5167329cda28a8a3841b6d5b8a37ee30273285f174af98ce5c0e5c76c0976_page_000001.png-page-0", + "value": 0.9979795217514038 + }, + { + "avg_weighted_label_matched_iou_50": 0.9979508519172668, + "avg_weighted_label_matched_iou_75": 0.9979508519172668, + "avg_weighted_label_matched_iou_90": 0.9979508519172668, + "avg_weighted_label_matched_iou_95": 0.9979508519172668, + "map_50": 1.0, + "map_75": 1.0, + "map_val": 1.0, + "name": "doc_934fbf534914863f6431eef38f5bf66fa91afd439ddf20fb1af0cf3225159ac1_page_000001.png-page-0", + "value": 0.9979508519172668 + }, + { + "avg_weighted_label_matched_iou_50": 0.9979238510131836, + "avg_weighted_label_matched_iou_75": 0.9979238510131836, + "avg_weighted_label_matched_iou_90": 0.9979238510131836, + "avg_weighted_label_matched_iou_95": 0.9979238510131836, + "map_50": 1.0, + "map_75": 1.0, + "map_val": 1.0, + "name": "doc_9877e33f447b370ad098531c25e340b9e786d1e84f4ac7365bab9c6746b57247_page_000001.png-page-0", + "value": 0.9979238510131836 + }, + { + "avg_weighted_label_matched_iou_50": 0.9977651238441467, + "avg_weighted_label_matched_iou_75": 0.9977651238441467, + "avg_weighted_label_matched_iou_90": 0.006217969581484795, + "avg_weighted_label_matched_iou_95": 0.006217969581484795, + "map_50": 1.0, + "map_75": 1.0, + "map_val": 0.800000011920929, + "name": "doc_7a0dc2cafa8fc77f28d78c90092a4cd31670dcf0ccfe8474a2765148448f44d7_page_000001.png-page-0", + "value": 0.9977651238441467 + }, + { + "avg_weighted_label_matched_iou_50": 0.9977610111236572, + "avg_weighted_label_matched_iou_75": 0.9977610111236572, + "avg_weighted_label_matched_iou_90": 0.9977610111236572, + "avg_weighted_label_matched_iou_95": 0.9977610111236572, + "map_50": 1.0, + "map_75": 1.0, + "map_val": 1.0, + "name": "doc_b862cd0d6f06c06ee5ab7729ed4e8ce58e6964eb0f1ab98b3865b57a4808216f_page_000001.png-page-0", + "value": 0.9977610111236572 + }, + { + "avg_weighted_label_matched_iou_50": 0.9977545738220215, + "avg_weighted_label_matched_iou_75": 0.003785280045121908, + "avg_weighted_label_matched_iou_90": 0.003785280045121908, + "avg_weighted_label_matched_iou_95": 0.003785280045121908, + "map_50": 0.6666666865348816, + "map_75": 0.3333333432674408, + "map_val": 0.4333333373069763, + "name": "doc_f17954b5fdd9abdb6b4ec921c26ef9949f8c3f34b35b4efae11d57f49efa8283_page_000001.png-page-0", + "value": 0.9977545738220215 + }, + { + "avg_weighted_label_matched_iou_50": 0.9974069595336914, + "avg_weighted_label_matched_iou_75": 0.9974069595336914, + "avg_weighted_label_matched_iou_90": 0.9974069595336914, + "avg_weighted_label_matched_iou_95": 0.9974069595336914, + "map_50": 0.75, + "map_75": 0.75, + "map_val": 0.75, + "name": "doc_e6023c1836d06942e984ed27b47b9de1677a107476544793a17058699a123435_page_000001.png-page-0", + "value": 0.9974069595336914 + }, + { + "avg_weighted_label_matched_iou_50": 0.9973869919776917, + "avg_weighted_label_matched_iou_75": 0.9973869919776917, + "avg_weighted_label_matched_iou_90": 0.9973869919776917, + "avg_weighted_label_matched_iou_95": 0.9973869919776917, + "map_50": 0.75, + "map_75": 0.75, + "map_val": 0.75, + "name": "doc_edafb0db61c16517e87d60ab4d1216bbabb43bcb39779d21707ab4b8ccf1e81c_page_000001.png-page-0", + "value": 0.9973869919776917 + }, + { + "avg_weighted_label_matched_iou_50": 0.9967820644378662, + "avg_weighted_label_matched_iou_75": 0.9967820644378662, + "avg_weighted_label_matched_iou_90": 0.9967820644378662, + "avg_weighted_label_matched_iou_95": 0.9967820644378662, + "map_50": 0.75, + "map_75": 0.75, + "map_val": 0.75, + "name": "doc_512a6e53e0a6661cb6bb77da54c354496ca75723048e2411ac5d1796a18cda8f_page_000001.png-page-0", + "value": 0.9967820644378662 + }, + { + "avg_weighted_label_matched_iou_50": 0.996722936630249, + "avg_weighted_label_matched_iou_75": 0.996722936630249, + "avg_weighted_label_matched_iou_90": 0.996722936630249, + "avg_weighted_label_matched_iou_95": 0.996722936630249, + "map_50": 0.6666666865348816, + "map_75": 0.6666666865348816, + "map_val": 0.6666666865348816, + "name": "doc_9aae5aa80d0ada0a9a0c1306993e96dcbdbec4cdf48766f56955cf8d3d8c46a4_page_000001.png-page-0", + "value": 0.996722936630249 + }, + { + "avg_weighted_label_matched_iou_50": 0.9966545104980469, + "avg_weighted_label_matched_iou_75": 0.9966545104980469, + "avg_weighted_label_matched_iou_90": 0.9966545104980469, + "avg_weighted_label_matched_iou_95": 0.9966545104980469, + "map_50": 1.0, + "map_75": 1.0, + "map_val": 1.0, + "name": "doc_6182c54724e702a8ed0936580d2096ab8b372a599589e731c66bd40ba5c252c0_page_000001.png-page-0", + "value": 0.9966545104980469 + }, + { + "avg_weighted_label_matched_iou_50": 0.9964510798454285, + "avg_weighted_label_matched_iou_75": 0.9964510798454285, + "avg_weighted_label_matched_iou_90": 0.9964510798454285, + "avg_weighted_label_matched_iou_95": 0.9964510798454285, + "map_50": 0.8762376308441162, + "map_75": 0.8762376308441162, + "map_val": 0.8762376308441162, + "name": "doc_a59c54dbc6c578c176e9c6627245740637da434d62a4460d865528ac3a2eecd4_page_000001.png-page-0", + "value": 0.9964510798454285 + }, + { + "avg_weighted_label_matched_iou_50": 0.9964224100112915, + "avg_weighted_label_matched_iou_75": 0.9964224100112915, + "avg_weighted_label_matched_iou_90": 0.9964224100112915, + "avg_weighted_label_matched_iou_95": 0.9964224100112915, + "map_50": 1.0, + "map_75": 1.0, + "map_val": 1.0, + "name": "doc_c75d25277c58e07bf91528ae08deccc80715b6f7f13b5ed571cf226463b364e8_page_000001.png-page-0", + "value": 0.9964224100112915 + }, + { + "avg_weighted_label_matched_iou_50": 0.99640953540802, + "avg_weighted_label_matched_iou_75": 0.99640953540802, + "avg_weighted_label_matched_iou_90": 0.99640953540802, + "avg_weighted_label_matched_iou_95": 0.99640953540802, + "map_50": 0.75, + "map_75": 0.75, + "map_val": 0.75, + "name": "doc_c98bfca166e04c05a1aa04e801e44dfc9282b1b112d230785873d1fb6930d4a4_page_000001.png-page-0", + "value": 0.99640953540802 + }, + { + "avg_weighted_label_matched_iou_50": 0.9963113069534302, + "avg_weighted_label_matched_iou_75": 0.9963113069534302, + "avg_weighted_label_matched_iou_90": 0.9963113069534302, + "avg_weighted_label_matched_iou_95": 0.9963113069534302, + "map_50": 1.0, + "map_75": 1.0, + "map_val": 1.0, + "name": "doc_bdf0e09acee7a07f7e60c6055128967b36b44564842983ad3b0f273f1c1c5914_page_000001.png-page-0", + "value": 0.9963113069534302 + }, + { + "avg_weighted_label_matched_iou_50": 0.9962734580039978, + "avg_weighted_label_matched_iou_75": 0.9962734580039978, + "avg_weighted_label_matched_iou_90": 0.012888174504041672, + "avg_weighted_label_matched_iou_95": 0.012888174504041672, + "map_50": 1.0, + "map_75": 1.0, + "map_val": 0.800000011920929, + "name": "doc_1039bce8b948fac48b0b9a42f8266700a104aca355c3681a544d65e6b136b8d5_page_000001.png-page-0", + "value": 0.9962734580039978 + }, + { + "avg_weighted_label_matched_iou_50": 0.996121346950531, + "avg_weighted_label_matched_iou_75": 0.996121346950531, + "avg_weighted_label_matched_iou_90": 0.996121346950531, + "avg_weighted_label_matched_iou_95": 0.9044751524925232, + "map_50": 0.948019802570343, + "map_75": 0.948019802570343, + "map_val": 0.9439356327056885, + "name": "doc_29d883eb26e7b9d08250b4bab1c51092d74e310e6acfa955f8fe28f1008accf5_page_000001.png-page-0", + "value": 0.996121346950531 + }, + { + "avg_weighted_label_matched_iou_50": 0.9960290193557739, + "avg_weighted_label_matched_iou_75": 0.9960290193557739, + "avg_weighted_label_matched_iou_90": 0.9960290193557739, + "avg_weighted_label_matched_iou_95": 0.9960290193557739, + "map_50": 1.0, + "map_75": 1.0, + "map_val": 1.0, + "name": "doc_d09e9a559e05904e7a73e547f179ed617b016a94065e045b9f46bbe0e2ae8528_page_000001.png-page-0", + "value": 0.9960290193557739 + }, + { + "avg_weighted_label_matched_iou_50": 0.9958966374397278, + "avg_weighted_label_matched_iou_75": 0.9958966374397278, + "avg_weighted_label_matched_iou_90": 0.9958966374397278, + "avg_weighted_label_matched_iou_95": 0.9958966374397278, + "map_50": 0.896039605140686, + "map_75": 0.896039605140686, + "map_val": 0.896039605140686, + "name": "doc_1c0d070f25babee3e3b5e2a842c54258f41939a7774c0e7c6d85fb7078e1196d_page_000001.png-page-0", + "value": 0.9958966374397278 + }, + { + "avg_weighted_label_matched_iou_50": 0.9958155155181885, + "avg_weighted_label_matched_iou_75": 0.9958155155181885, + "avg_weighted_label_matched_iou_90": 0.9958155155181885, + "avg_weighted_label_matched_iou_95": 0.9958155155181885, + "map_50": 0.75, + "map_75": 0.75, + "map_val": 0.75, + "name": "doc_cd160a4e2296bd7c23f19c26af5bb6ed772f766ee1eacfe9998703c38146b494_page_000001.png-page-0", + "value": 0.9958155155181885 + }, + { + "avg_weighted_label_matched_iou_50": 0.995690107345581, + "avg_weighted_label_matched_iou_75": 0.995690107345581, + "avg_weighted_label_matched_iou_90": 0.995690107345581, + "avg_weighted_label_matched_iou_95": 0.9947035908699036, + "map_50": 0.6000000238418579, + "map_75": 0.6000000238418579, + "map_val": 0.5958415865898132, + "name": "doc_fad8cdcee7f7884bddedf0ee485bf259f815315699f35ef82b2afca5fb28f68a_page_000001.png-page-0", + "value": 0.995690107345581 + }, + { + "avg_weighted_label_matched_iou_50": 0.9955419301986694, + "avg_weighted_label_matched_iou_75": 0.9955419301986694, + "avg_weighted_label_matched_iou_90": 0.9955419301986694, + "avg_weighted_label_matched_iou_95": 0.9955419301986694, + "map_50": 1.0, + "map_75": 1.0, + "map_val": 1.0, + "name": "doc_475c037beeba4ba75b3e749b51f3781a67e8e8f9db6a31772f7bf2ed2bd05d00_page_000001.png-page-0", + "value": 0.9955419301986694 + }, + { + "avg_weighted_label_matched_iou_50": 0.995496392250061, + "avg_weighted_label_matched_iou_75": 0.995496392250061, + "avg_weighted_label_matched_iou_90": 0.995496392250061, + "avg_weighted_label_matched_iou_95": 0.995496392250061, + "map_50": 1.0, + "map_75": 1.0, + "map_val": 1.0, + "name": "doc_750091cec2c31ca61ffaa40c282148258f40bf5db7356a7ed3f918a07c81ef0f_page_000001.png-page-0", + "value": 0.995496392250061 + }, + { + "avg_weighted_label_matched_iou_50": 0.9954509735107422, + "avg_weighted_label_matched_iou_75": 0.9954509735107422, + "avg_weighted_label_matched_iou_90": 0.9954509735107422, + "avg_weighted_label_matched_iou_95": 0.9954509735107422, + "map_50": 1.0, + "map_75": 1.0, + "map_val": 1.0, + "name": "doc_a546d4f7375892bde50a7ceb4e665e565000286a1c007c51105a3ca3fba87bf4_page_000001.png-page-0", + "value": 0.9954509735107422 + }, + { + "avg_weighted_label_matched_iou_50": 0.9953116774559021, + "avg_weighted_label_matched_iou_75": 0.9953116774559021, + "avg_weighted_label_matched_iou_90": 0.9953116774559021, + "avg_weighted_label_matched_iou_95": 0.9953116774559021, + "map_50": 1.0, + "map_75": 1.0, + "map_val": 1.0, + "name": "doc_3cc8da158da98e9b4e7b3c8ba7d1a74afc558fb6afc7ac8eadbe74afb4e7b14c_page_000001.png-page-0", + "value": 0.9953116774559021 + }, + { + "avg_weighted_label_matched_iou_50": 0.9951591491699219, + "avg_weighted_label_matched_iou_75": 0.9951591491699219, + "avg_weighted_label_matched_iou_90": 0.009132335893809795, + "avg_weighted_label_matched_iou_95": 0.009132335893809795, + "map_50": 1.0, + "map_75": 1.0, + "map_val": 0.800000011920929, + "name": "doc_4a23b09904c5fbb871fcf792e30b87c4b7cae0cdd0079b5ee0e3e34a353638b7_page_000001.png-page-0", + "value": 0.9951591491699219 + }, + { + "avg_weighted_label_matched_iou_50": 0.9951339364051819, + "avg_weighted_label_matched_iou_75": 0.9951339364051819, + "avg_weighted_label_matched_iou_90": 0.9951339364051819, + "avg_weighted_label_matched_iou_95": 0.9951339364051819, + "map_50": 1.0, + "map_75": 1.0, + "map_val": 1.0, + "name": "doc_6572d153a938b50167baca0e724b25a02adb927d3c6adb9f1985484b7b1791b6_page_000001.png-page-0", + "value": 0.9951339364051819 + }, + { + "avg_weighted_label_matched_iou_50": 0.99512779712677, + "avg_weighted_label_matched_iou_75": 0.99512779712677, + "avg_weighted_label_matched_iou_90": 0.99512779712677, + "avg_weighted_label_matched_iou_95": 0.99512779712677, + "map_50": 0.6666666865348816, + "map_75": 0.6666666865348816, + "map_val": 0.6666666865348816, + "name": "doc_16bc03112342160922798dc0b3220b260ad4ebe06deafb9ab3c67df528877717_page_000001.png-page-0", + "value": 0.99512779712677 + }, + { + "avg_weighted_label_matched_iou_50": 0.9948722720146179, + "avg_weighted_label_matched_iou_75": 0.9948722720146179, + "avg_weighted_label_matched_iou_90": 0.9948722720146179, + "avg_weighted_label_matched_iou_95": 0.9948722720146179, + "map_50": 0.948019802570343, + "map_75": 0.948019802570343, + "map_val": 0.948019802570343, + "name": "doc_eac01901729786f15ba498716c592370d50f00658c291495086ff49a8538c5c6_page_000001.png-page-0", + "value": 0.9948722720146179 + }, + { + "avg_weighted_label_matched_iou_50": 0.994820237159729, + "avg_weighted_label_matched_iou_75": 0.994820237159729, + "avg_weighted_label_matched_iou_90": 0.994820237159729, + "avg_weighted_label_matched_iou_95": 0.994820237159729, + "map_50": 0.9584158658981323, + "map_75": 0.9584158658981323, + "map_val": 0.9584158658981323, + "name": "doc_61070c2fe64a690c7c9cc97d12b76380e7a134feda3c5315f779306699a89a1b_page_000001.png-page-0", + "value": 0.994820237159729 + }, + { + "avg_weighted_label_matched_iou_50": 0.9946385622024536, + "avg_weighted_label_matched_iou_75": 0.9946385622024536, + "avg_weighted_label_matched_iou_90": 0.6634351015090942, + "avg_weighted_label_matched_iou_95": 0.6634351015090942, + "map_50": 0.6683168411254883, + "map_75": 0.6683168411254883, + "map_val": 0.6346534490585327, + "name": "doc_93ce1e59ab01c6352a98eca26b4a356b1e46e923cd60ea7145a458a3fbb5fd41_page_000001.png-page-0", + "value": 0.9946385622024536 + }, + { + "avg_weighted_label_matched_iou_50": 0.9945571422576904, + "avg_weighted_label_matched_iou_75": 0.9945571422576904, + "avg_weighted_label_matched_iou_90": 0.9945571422576904, + "avg_weighted_label_matched_iou_95": 0.9945571422576904, + "map_50": 1.0, + "map_75": 1.0, + "map_val": 1.0, + "name": "doc_b966d5aee5a17b377107ade67238d7215faef6303ac980080fd68534ab3ce5ae_page_000001.png-page-0", + "value": 0.9945571422576904 + }, + { + "avg_weighted_label_matched_iou_50": 0.9945335388183594, + "avg_weighted_label_matched_iou_75": 0.9945335388183594, + "avg_weighted_label_matched_iou_90": 0.9945335388183594, + "avg_weighted_label_matched_iou_95": 0.9945335388183594, + "map_50": 1.0, + "map_75": 1.0, + "map_val": 1.0, + "name": "doc_65be3f78649d4ee59fa39b9d2b94a44570e52ddbf31cb7213f92853bc23ceb16_page_000001.png-page-0", + "value": 0.9945335388183594 + }, + { + "avg_weighted_label_matched_iou_50": 0.9942327737808228, + "avg_weighted_label_matched_iou_75": 0.9942327737808228, + "avg_weighted_label_matched_iou_90": 0.9942327737808228, + "avg_weighted_label_matched_iou_95": 0.9942327737808228, + "map_50": 1.0, + "map_75": 1.0, + "map_val": 1.0, + "name": "doc_66c1d3c3d52df005397d2b458d43a0c2a5c1a2ba8ab8db4ed5a20ee07d7fe090_page_000001.png-page-0", + "value": 0.9942327737808228 + }, + { + "avg_weighted_label_matched_iou_50": 0.9941327571868896, + "avg_weighted_label_matched_iou_75": 0.9941327571868896, + "avg_weighted_label_matched_iou_90": 0.9941327571868896, + "avg_weighted_label_matched_iou_95": 0.9941327571868896, + "map_50": 1.0, + "map_75": 1.0, + "map_val": 1.0, + "name": "doc_d7f2535a168ba996bb0caccbcc719373580d555909bd5d46ed1748e8f3c17807_page_000001.png-page-0", + "value": 0.9941327571868896 + }, + { + "avg_weighted_label_matched_iou_50": 0.9941146373748779, + "avg_weighted_label_matched_iou_75": 0.015532199293375015, + "avg_weighted_label_matched_iou_90": 0.015532199293375015, + "avg_weighted_label_matched_iou_95": 0.015532199293375015, + "map_50": 1.0, + "map_75": 0.5, + "map_val": 0.75, + "name": "doc_321a42833cde040a5e4525c5f84ca2e7df18ee57649c9b47a0e5472e1357abb6_page_000001.png-page-0", + "value": 0.9941146373748779 + }, + { + "avg_weighted_label_matched_iou_50": 0.9932557344436646, + "avg_weighted_label_matched_iou_75": 0.9932557344436646, + "avg_weighted_label_matched_iou_90": 0.9932557344436646, + "avg_weighted_label_matched_iou_95": 0.9932557344436646, + "map_50": 1.0, + "map_75": 1.0, + "map_val": 1.0, + "name": "doc_2f972daaa81cb3949db12a39cf6910c4f9952135f7eb3850e1f60ef7d8b5aaf9_page_000001.png-page-0", + "value": 0.9932557344436646 + }, + { + "avg_weighted_label_matched_iou_50": 0.9932122230529785, + "avg_weighted_label_matched_iou_75": 0.9932122230529785, + "avg_weighted_label_matched_iou_90": 0.9932122230529785, + "avg_weighted_label_matched_iou_95": 0.9932122230529785, + "map_50": 0.75, + "map_75": 0.75, + "map_val": 0.75, + "name": "doc_7abbca82ea23276bbd534ca90a667ede80429690d2f06ae0e83459f75c42670f_page_000001.png-page-0", + "value": 0.9932122230529785 + }, + { + "avg_weighted_label_matched_iou_50": 0.9930915832519531, + "avg_weighted_label_matched_iou_75": 0.9930915832519531, + "avg_weighted_label_matched_iou_90": 0.9930915832519531, + "avg_weighted_label_matched_iou_95": 0.9930915832519531, + "map_50": 1.0, + "map_75": 1.0, + "map_val": 1.0, + "name": "doc_165dfeaf31832cc4547b6b638a033c1057ddb6384874c16c4574c6278856dd30_page_000001.png-page-0", + "value": 0.9930915832519531 + }, + { + "avg_weighted_label_matched_iou_50": 0.9928626418113708, + "avg_weighted_label_matched_iou_75": 0.9928626418113708, + "avg_weighted_label_matched_iou_90": 0.023027941584587097, + "avg_weighted_label_matched_iou_95": 0.023027941584587097, + "map_50": 1.0, + "map_75": 1.0, + "map_val": 0.800000011920929, + "name": "doc_646f014ab615ac22266a43be2691010032d9fd62368737d488ffcb912cd68fd2_page_000001.png-page-0", + "value": 0.9928626418113708 + }, + { + "avg_weighted_label_matched_iou_50": 0.992810845375061, + "avg_weighted_label_matched_iou_75": 0.992810845375061, + "avg_weighted_label_matched_iou_90": 0.992810845375061, + "avg_weighted_label_matched_iou_95": 0.992810845375061, + "map_50": 1.0, + "map_75": 1.0, + "map_val": 1.0, + "name": "doc_5cba8e49a880e4f6d1c1fba64f03b3d638fe26cfc00d77fddeb450334c8e0667_page_000001.png-page-0", + "value": 0.992810845375061 + }, + { + "avg_weighted_label_matched_iou_50": 0.9921263456344604, + "avg_weighted_label_matched_iou_75": 0.9921263456344604, + "avg_weighted_label_matched_iou_90": 0.8005243539810181, + "avg_weighted_label_matched_iou_95": 0.8005243539810181, + "map_50": 1.0, + "map_75": 1.0, + "map_val": 0.800000011920929, + "name": "doc_0d4c4e8911eb53569e91f3c53a1deb5951ad138e767ef5edff79b490053fc82a_page_000001.png-page-0", + "value": 0.9921263456344604 + }, + { + "avg_weighted_label_matched_iou_50": 0.9913201332092285, + "avg_weighted_label_matched_iou_75": 0.022576799616217613, + "avg_weighted_label_matched_iou_90": 0.022576799616217613, + "avg_weighted_label_matched_iou_95": 0.022576799616217613, + "map_50": 1.0, + "map_75": 0.5, + "map_val": 0.699999988079071, + "name": "doc_29dfa90e45920fd79e68e82f0f171119fc1b4d494aac703edf21c3ea74dd6fca_page_000001.png-page-0", + "value": 0.9913201332092285 + }, + { + "avg_weighted_label_matched_iou_50": 0.990705132484436, + "avg_weighted_label_matched_iou_75": 0.990705132484436, + "avg_weighted_label_matched_iou_90": 0.990705132484436, + "avg_weighted_label_matched_iou_95": 0.990705132484436, + "map_50": 1.0, + "map_75": 1.0, + "map_val": 1.0, + "name": "doc_f81cbaef6c20bdadec11a3e9622d381850f6f7d3ab059e3949a9a6a877fd144e_page_000001.png-page-0", + "value": 0.990705132484436 + }, + { + "avg_weighted_label_matched_iou_50": 0.9906885623931885, + "avg_weighted_label_matched_iou_75": 0.9906885623931885, + "avg_weighted_label_matched_iou_90": 0.9906885623931885, + "avg_weighted_label_matched_iou_95": 0.9906885623931885, + "map_50": 1.0, + "map_75": 1.0, + "map_val": 1.0, + "name": "doc_2ad0df434ac3f29ff1dfd1ce8e378a2badf1bb6b9aede5d18de2945460f55c1d_page_000001.png-page-0", + "value": 0.9906885623931885 + }, + { + "avg_weighted_label_matched_iou_50": 0.9900469183921814, + "avg_weighted_label_matched_iou_75": 0.9900469183921814, + "avg_weighted_label_matched_iou_90": 0.8807628750801086, + "avg_weighted_label_matched_iou_95": 0.8807628750801086, + "map_50": 0.75, + "map_75": 0.75, + "map_val": 0.675000011920929, + "name": "doc_94ba5468fcb6277721947697048846dc0d0551296be3b45f5918ab857d21dcc7_page_000001.png-page-0", + "value": 0.9900469183921814 + }, + { + "avg_weighted_label_matched_iou_50": 0.9900221824645996, + "avg_weighted_label_matched_iou_75": 0.9900221824645996, + "avg_weighted_label_matched_iou_90": 0.9900221824645996, + "avg_weighted_label_matched_iou_95": 0.5515025854110718, + "map_50": 1.0, + "map_75": 1.0, + "map_val": 0.9800000190734863, + "name": "doc_ba0e025d53c091e8d4bb87499ff69ed3428dcee325c8895ecdb40e973b4c835c_page_000001.png-page-0", + "value": 0.9900221824645996 + }, + { + "avg_weighted_label_matched_iou_50": 0.9898004531860352, + "avg_weighted_label_matched_iou_75": 0.9898004531860352, + "avg_weighted_label_matched_iou_90": 0.9898004531860352, + "avg_weighted_label_matched_iou_95": 0.9227567315101624, + "map_50": 1.0, + "map_75": 1.0, + "map_val": 0.9731848239898682, + "name": "doc_b4d90b6cc2d96d952b49dd227148c08018baffbfebd8d204705384f0a4a0c8a3_page_000001.png-page-0", + "value": 0.9898004531860352 + }, + { + "avg_weighted_label_matched_iou_50": 0.9891417026519775, + "avg_weighted_label_matched_iou_75": 0.9891417026519775, + "avg_weighted_label_matched_iou_90": 0.9891417026519775, + "avg_weighted_label_matched_iou_95": 0.9342519044876099, + "map_50": 1.0, + "map_75": 1.0, + "map_val": 0.9911321401596069, + "name": "doc_fcaa8c3de4757ff51e604d84c983c59c421e31b9989df5b74c930e0e8720e5ae_page_000001.png-page-0", + "value": 0.9891417026519775 + }, + { + "avg_weighted_label_matched_iou_50": 0.9888395667076111, + "avg_weighted_label_matched_iou_75": 0.9888395667076111, + "avg_weighted_label_matched_iou_90": 0.9888395667076111, + "avg_weighted_label_matched_iou_95": 0.6014910936355591, + "map_50": 0.6666666865348816, + "map_75": 0.6666666865348816, + "map_val": 0.6333333253860474, + "name": "doc_4a6baa0f883743ec25b853bae06a49d230ec010c973501945d46a16f9bb75aa9_page_000001.png-page-0", + "value": 0.9888395667076111 + }, + { + "avg_weighted_label_matched_iou_50": 0.9860407710075378, + "avg_weighted_label_matched_iou_75": 0.9860407710075378, + "avg_weighted_label_matched_iou_90": 0.9860407710075378, + "avg_weighted_label_matched_iou_95": 0.9860407710075378, + "map_50": 0.6666666865348816, + "map_75": 0.6666666865348816, + "map_val": 0.6666666865348816, + "name": "doc_462529434b77fc469f72c781c14663f8a787e4785af189268f5aeac71e823aba_page_000001.png-page-0", + "value": 0.9860407710075378 + }, + { + "avg_weighted_label_matched_iou_50": 0.9847614765167236, + "avg_weighted_label_matched_iou_75": 0.9847614765167236, + "avg_weighted_label_matched_iou_90": 0.8807675838470459, + "avg_weighted_label_matched_iou_95": 0.8807675838470459, + "map_50": 1.0, + "map_75": 1.0, + "map_val": 0.800000011920929, + "name": "doc_118cb93e60d052c799e7119026c02fd68ca57362f5ac749ee58c8de8556f9393_page_000001.png-page-0", + "value": 0.9847614765167236 + }, + { + "avg_weighted_label_matched_iou_50": 0.984495222568512, + "avg_weighted_label_matched_iou_75": 0.984495222568512, + "avg_weighted_label_matched_iou_90": 0.984495222568512, + "avg_weighted_label_matched_iou_95": 0.984495222568512, + "map_50": 0.6666666865348816, + "map_75": 0.6666666865348816, + "map_val": 0.6666666865348816, + "name": "doc_1f051b82d968ba073f99a2f78e15e82072c9986127614e36eb8c1fa1ccb0d261_page_000001.png-page-0", + "value": 0.984495222568512 + }, + { + "avg_weighted_label_matched_iou_50": 0.9837513566017151, + "avg_weighted_label_matched_iou_75": 0.9837513566017151, + "avg_weighted_label_matched_iou_90": 0.9576258659362793, + "avg_weighted_label_matched_iou_95": 0.9576258659362793, + "map_50": 1.0, + "map_75": 1.0, + "map_val": 0.8999999761581421, + "name": "doc_f4dea73d2b75fbc7d590e2085b14e9bee7e766da73e14f723a2580fb0f36c707_page_000001.png-page-0", + "value": 0.9837513566017151 + }, + { + "avg_weighted_label_matched_iou_50": 0.9835698008537292, + "avg_weighted_label_matched_iou_75": 0.9835698008537292, + "avg_weighted_label_matched_iou_90": 0.9835698008537292, + "avg_weighted_label_matched_iou_95": 0.9835698008537292, + "map_50": 0.6666666865348816, + "map_75": 0.6666666865348816, + "map_val": 0.6666666865348816, + "name": "doc_aa00d01e639cf879439b5f4c9313f1647bba27070a833569dee0b3eeb3476731_page_000001.png-page-0", + "value": 0.9835698008537292 + }, + { + "avg_weighted_label_matched_iou_50": 0.982866108417511, + "avg_weighted_label_matched_iou_75": 0.982866108417511, + "avg_weighted_label_matched_iou_90": 0.982866108417511, + "avg_weighted_label_matched_iou_95": 0.9004486799240112, + "map_50": 1.0, + "map_75": 1.0, + "map_val": 0.9912128448486328, + "name": "doc_64f31210764923368012a82e8741b32b217654d91f54fc9e0a13be613d63bbc1_page_000001.png-page-0", + "value": 0.982866108417511 + }, + { + "avg_weighted_label_matched_iou_50": 0.9823511242866516, + "avg_weighted_label_matched_iou_75": 0.9823511242866516, + "avg_weighted_label_matched_iou_90": 0.9823511242866516, + "avg_weighted_label_matched_iou_95": 0.6982664465904236, + "map_50": 1.0, + "map_75": 1.0, + "map_val": 0.9716171622276306, + "name": "doc_2cd17a32ee330a239e19c915738df0c27e8ec3635a60a7e16e2a0cf3868d4af3_page_000001.png-page-0", + "value": 0.9823511242866516 + }, + { + "avg_weighted_label_matched_iou_50": 0.9819358587265015, + "avg_weighted_label_matched_iou_75": 0.9819358587265015, + "avg_weighted_label_matched_iou_90": 0.9819358587265015, + "avg_weighted_label_matched_iou_95": 0.7832468152046204, + "map_50": 0.75, + "map_75": 0.75, + "map_val": 0.7278052568435669, + "name": "doc_ee9af89ec8d82b15d0fdff8908560e597e71cd4662d28472bc1a4d485d6c6712_page_000001.png-page-0", + "value": 0.9819358587265015 + }, + { + "avg_weighted_label_matched_iou_50": 0.980778694152832, + "avg_weighted_label_matched_iou_75": 0.980778694152832, + "avg_weighted_label_matched_iou_90": 0.6783071160316467, + "avg_weighted_label_matched_iou_95": 0.6660850048065186, + "map_50": 1.0, + "map_75": 1.0, + "map_val": 0.886056125164032, + "name": "doc_62c4b0340519f5963a37534d3abeb0546766cb53bc90504f663eb7cc321a6a6d_page_000001.png-page-0", + "value": 0.980778694152832 + }, + { + "avg_weighted_label_matched_iou_50": 0.9799604415893555, + "avg_weighted_label_matched_iou_75": 0.9799604415893555, + "avg_weighted_label_matched_iou_90": 0.9540870189666748, + "avg_weighted_label_matched_iou_95": 0.9540870189666748, + "map_50": 0.6262376308441162, + "map_75": 0.6262376308441162, + "map_val": 0.5762376189231873, + "name": "doc_d2773d8fcd2f2b18b6bd0b2952ba8dd56702e1f738a4f8a7b1e0e03c9cad9dba_page_000001.png-page-0", + "value": 0.9799604415893555 + }, + { + "avg_weighted_label_matched_iou_50": 0.9781196713447571, + "avg_weighted_label_matched_iou_75": 0.9781196713447571, + "avg_weighted_label_matched_iou_90": 0.8014415502548218, + "avg_weighted_label_matched_iou_95": 0.8014415502548218, + "map_50": 0.75, + "map_75": 0.75, + "map_val": 0.6499999761581421, + "name": "doc_733825a2a8367b6d61ceccd19b6781c3520730c0fff34ab4c2e8411f625f34be_page_000001.png-page-0", + "value": 0.9781196713447571 + }, + { + "avg_weighted_label_matched_iou_50": 0.978036105632782, + "avg_weighted_label_matched_iou_75": 0.978036105632782, + "avg_weighted_label_matched_iou_90": 0.978036105632782, + "avg_weighted_label_matched_iou_95": 0.8542413711547852, + "map_50": 0.6666666865348816, + "map_75": 0.6666666865348816, + "map_val": 0.6554455161094666, + "name": "doc_a7f0764fac3ab64711acc91fa88e2632f3e0d27b6d57ab58ee3b709b0889adba_page_000001.png-page-0", + "value": 0.978036105632782 + }, + { + "avg_weighted_label_matched_iou_50": 0.9752902984619141, + "avg_weighted_label_matched_iou_75": 0.6095676422119141, + "avg_weighted_label_matched_iou_90": 0.6095676422119141, + "avg_weighted_label_matched_iou_95": 0.6095676422119141, + "map_50": 1.0, + "map_75": 0.5, + "map_val": 0.699999988079071, + "name": "doc_31e7fbfc5b038d471b2fc82b248c4807152284bc35d6d5bd036c0aaa42504f17_page_000001.png-page-0", + "value": 0.9752902984619141 + }, + { + "avg_weighted_label_matched_iou_50": 0.974785566329956, + "avg_weighted_label_matched_iou_75": 0.974785566329956, + "avg_weighted_label_matched_iou_90": 0.974785566329956, + "avg_weighted_label_matched_iou_95": 0.7509828805923462, + "map_50": 0.8762376308441162, + "map_75": 0.8762376308441162, + "map_val": 0.8669554591178894, + "name": "doc_fbee4ae063533d20dc7339a3ffa2412d58504af06420fe15054657c87ed81a79_page_000001.png-page-0", + "value": 0.974785566329956 + }, + { + "avg_weighted_label_matched_iou_50": 0.972886860370636, + "avg_weighted_label_matched_iou_75": 0.972886860370636, + "avg_weighted_label_matched_iou_90": 0.972886860370636, + "avg_weighted_label_matched_iou_95": 0.6695253849029541, + "map_50": 0.5841584205627441, + "map_75": 0.5841584205627441, + "map_val": 0.5778327584266663, + "name": "doc_771caa3e8ec1bce887caac49940cab3c5b06a8a42ce120daa1999c30204aab33_page_000001.png-page-0", + "value": 0.972886860370636 + }, + { + "avg_weighted_label_matched_iou_50": 0.9700906276702881, + "avg_weighted_label_matched_iou_75": 0.8760496973991394, + "avg_weighted_label_matched_iou_90": 0.8760496973991394, + "avg_weighted_label_matched_iou_95": 0.8566318154335022, + "map_50": 1.0, + "map_75": 0.8547855019569397, + "map_val": 0.9052805304527283, + "name": "doc_e1a31e05b65ada20646a1136bdc3293f68e3641893a48674c791bc2b8fadffa9_page_000001.png-page-0", + "value": 0.9700906276702881 + }, + { + "avg_weighted_label_matched_iou_50": 0.9690712094306946, + "avg_weighted_label_matched_iou_75": 0.9632877707481384, + "avg_weighted_label_matched_iou_90": 0.89378821849823, + "avg_weighted_label_matched_iou_95": 0.14042127132415771, + "map_50": 1.0, + "map_75": 0.75, + "map_val": 0.75, + "name": "doc_00f0adaaa8358a28b4b4e83bc97dcd83a01f7283605b140c2be8e8d47bba8b6b_page_000001.png-page-0", + "value": 0.9690712094306946 + }, + { + "avg_weighted_label_matched_iou_50": 0.9681782126426697, + "avg_weighted_label_matched_iou_75": 0.9369664192199707, + "avg_weighted_label_matched_iou_90": 0.8437610268592834, + "avg_weighted_label_matched_iou_95": 0.6078407764434814, + "map_50": 0.7188118696212769, + "map_75": 0.5188118815422058, + "map_val": 0.5485148429870605, + "name": "doc_a266dd4ddaff135dc5b1ce3d960d4b29cfc96bee5f30f3fb879a0b730c57e6f7_page_000001.png-page-0", + "value": 0.9681782126426697 + }, + { + "avg_weighted_label_matched_iou_50": 0.968146026134491, + "avg_weighted_label_matched_iou_75": 0.968146026134491, + "avg_weighted_label_matched_iou_90": 0.968146026134491, + "avg_weighted_label_matched_iou_95": 0.968146026134491, + "map_50": 1.0, + "map_75": 1.0, + "map_val": 1.0, + "name": "doc_2183cb0d11b4e22a1fd54f2e79e9d91b280e63c24da91c4164d52a89732dd1b0_page_000001.png-page-0", + "value": 0.968146026134491 + }, + { + "avg_weighted_label_matched_iou_50": 0.9680723547935486, + "avg_weighted_label_matched_iou_75": 0.8330680727958679, + "avg_weighted_label_matched_iou_90": 0.8330680727958679, + "avg_weighted_label_matched_iou_95": 0.6061046719551086, + "map_50": 1.0, + "map_75": 0.8574257493019104, + "map_val": 0.895049512386322, + "name": "doc_603d71e12f52d9801a9d82995babf69681d923f493b8d49abfbab8662a88b376_page_000001.png-page-0", + "value": 0.9680723547935486 + }, + { + "avg_weighted_label_matched_iou_50": 0.9667116403579712, + "avg_weighted_label_matched_iou_75": 0.9667116403579712, + "avg_weighted_label_matched_iou_90": 0.9667116403579712, + "avg_weighted_label_matched_iou_95": 0.9667116403579712, + "map_50": 0.6666666865348816, + "map_75": 0.6666666865348816, + "map_val": 0.6666666865348816, + "name": "doc_78698056c30504beba802bc806711c307553d8b8aa39a6d8ff8388aafdc3953a_page_000001.png-page-0", + "value": 0.9667116403579712 + }, + { + "avg_weighted_label_matched_iou_50": 0.9659442901611328, + "avg_weighted_label_matched_iou_75": 0.7943786978721619, + "avg_weighted_label_matched_iou_90": 0.6187186241149902, + "avg_weighted_label_matched_iou_95": 0.5909274220466614, + "map_50": 0.8333333134651184, + "map_75": 0.6666666865348816, + "map_val": 0.6547854542732239, + "name": "doc_1a5c949c602fb587c39618443a00915582b2d20a32f959e5d3f4f3fc99ab9db5_page_000001.png-page-0", + "value": 0.9659442901611328 + }, + { + "avg_weighted_label_matched_iou_50": 0.9650094509124756, + "avg_weighted_label_matched_iou_75": 0.9650094509124756, + "avg_weighted_label_matched_iou_90": 0.5644127726554871, + "avg_weighted_label_matched_iou_95": 0.5644127726554871, + "map_50": 0.5420792102813721, + "map_75": 0.5420792102813721, + "map_val": 0.4925742447376251, + "name": "doc_85ce99d1f5e9e0661f3cda228e68c36ac7b735b3c78caef8165c906074c6e7ab_page_000001.png-page-0", + "value": 0.9650094509124756 + }, + { + "avg_weighted_label_matched_iou_50": 0.964936375617981, + "avg_weighted_label_matched_iou_75": 0.964936375617981, + "avg_weighted_label_matched_iou_90": 0.9271148443222046, + "avg_weighted_label_matched_iou_95": 0.9271148443222046, + "map_50": 1.0, + "map_75": 1.0, + "map_val": 0.8999999761581421, + "name": "doc_ed941b5cde32cfc778c225dcfa90c334d9995d70c46e7f508635073a3227784b_page_000001.png-page-0", + "value": 0.964936375617981 + }, + { + "avg_weighted_label_matched_iou_50": 0.9641724228858948, + "avg_weighted_label_matched_iou_75": 0.9641724228858948, + "avg_weighted_label_matched_iou_90": 0.9641724228858948, + "avg_weighted_label_matched_iou_95": 0.9641724228858948, + "map_50": 1.0, + "map_75": 1.0, + "map_val": 1.0, + "name": "doc_348bff737da27234885d263b9983064513fc33b1ff2ce858171e58f88c954c1a_page_000001.png-page-0", + "value": 0.9641724228858948 + }, + { + "avg_weighted_label_matched_iou_50": 0.9632815718650818, + "avg_weighted_label_matched_iou_75": 0.7895829677581787, + "avg_weighted_label_matched_iou_90": 0.7895829677581787, + "avg_weighted_label_matched_iou_95": 0.6787943243980408, + "map_50": 0.75, + "map_75": 0.5, + "map_val": 0.574999988079071, + "name": "doc_ee0a6175753e8cfc2f35fcbb1c5f7666f56969fb8111e326757835f062c65bb0_page_000001.png-page-0", + "value": 0.9632815718650818 + }, + { + "avg_weighted_label_matched_iou_50": 0.9630088210105896, + "avg_weighted_label_matched_iou_75": 0.9630088210105896, + "avg_weighted_label_matched_iou_90": 0.845522940158844, + "avg_weighted_label_matched_iou_95": 0.5369511842727661, + "map_50": 0.6262376308441162, + "map_75": 0.6262376308441162, + "map_val": 0.5741749405860901, + "name": "doc_9e22636736c43765307cbc5e876b0a18e340b52c909fc36043cefb38a7a11638_page_000001.png-page-0", + "value": 0.9630088210105896 + }, + { + "avg_weighted_label_matched_iou_50": 0.9629468321800232, + "avg_weighted_label_matched_iou_75": 0.9629468321800232, + "avg_weighted_label_matched_iou_90": 0.5600670576095581, + "avg_weighted_label_matched_iou_95": 0.3819303512573242, + "map_50": 0.7009900808334351, + "map_75": 0.7009900808334351, + "map_val": 0.6542574167251587, + "name": "doc_95f808815969717e4ef6390b2b789a8ea552a000a2b936c3c9371f695a71bd9e_page_000001.png-page-0", + "value": 0.9629468321800232 + }, + { + "avg_weighted_label_matched_iou_50": 0.961337685585022, + "avg_weighted_label_matched_iou_75": 0.961337685585022, + "avg_weighted_label_matched_iou_90": 0.8130836486816406, + "avg_weighted_label_matched_iou_95": 0.6036356091499329, + "map_50": 0.6658415794372559, + "map_75": 0.6658415794372559, + "map_val": 0.5568481683731079, + "name": "doc_020d140f50e8e8332a8b2099bc306d83ecf46b851e3ee6cc03e456ccd05411bd_page_000001.png-page-0", + "value": 0.961337685585022 + }, + { + "avg_weighted_label_matched_iou_50": 0.9594211578369141, + "avg_weighted_label_matched_iou_75": 0.9594211578369141, + "avg_weighted_label_matched_iou_90": 0.9594211578369141, + "avg_weighted_label_matched_iou_95": 0.7360147833824158, + "map_50": 1.0, + "map_75": 1.0, + "map_val": 0.9722772240638733, + "name": "doc_b2cd33ed246c6dca7d94b8937a8abbc44eb870a5007e5c07fcfbd2845e9b1d07_page_000001.png-page-0", + "value": 0.9594211578369141 + }, + { + "avg_weighted_label_matched_iou_50": 0.9577893614768982, + "avg_weighted_label_matched_iou_75": 0.9228880405426025, + "avg_weighted_label_matched_iou_90": 0.9094080328941345, + "avg_weighted_label_matched_iou_95": 0.9094080328941345, + "map_50": 1.0, + "map_75": 0.6262376308441162, + "map_val": 0.7378712892532349, + "name": "doc_c33c2993fe1e5c2945bb894f1a8e54c756753c344bd66f11024715e6375f07d1_page_000001.png-page-0", + "value": 0.9577893614768982 + }, + { + "avg_weighted_label_matched_iou_50": 0.9563427567481995, + "avg_weighted_label_matched_iou_75": 0.9563427567481995, + "avg_weighted_label_matched_iou_90": 0.9563427567481995, + "avg_weighted_label_matched_iou_95": 0.9563427567481995, + "map_50": 0.1651902049779892, + "map_75": 0.1651902049779892, + "map_val": 0.1651902049779892, + "name": "doc_827d21de372a2c26237ee1db526460851ae71c1867761776583535f532432e32_page_000001.png-page-0", + "value": 0.9563427567481995 + }, + { + "avg_weighted_label_matched_iou_50": 0.9559595584869385, + "avg_weighted_label_matched_iou_75": 0.9559595584869385, + "avg_weighted_label_matched_iou_90": 0.9202622175216675, + "avg_weighted_label_matched_iou_95": 0.7301286458969116, + "map_50": 1.0, + "map_75": 1.0, + "map_val": 0.9445544481277466, + "name": "doc_286f10070b1be903eb23c297dee52da2d8f108a4c4db79e0f6b62eda54050067_page_000001.png-page-0", + "value": 0.9559595584869385 + }, + { + "avg_weighted_label_matched_iou_50": 0.9544945955276489, + "avg_weighted_label_matched_iou_75": 0.9544945955276489, + "avg_weighted_label_matched_iou_90": 0.6762253046035767, + "avg_weighted_label_matched_iou_95": 0.5909298658370972, + "map_50": 0.800000011920929, + "map_75": 0.800000011920929, + "map_val": 0.7708085775375366, + "name": "doc_a5b975710668800f4d4712155bc597a028f0ffbfe162efef92ded23cb2c23b9f_page_000001.png-page-0", + "value": 0.9544945955276489 + }, + { + "avg_weighted_label_matched_iou_50": 0.9544895887374878, + "avg_weighted_label_matched_iou_75": 0.6668264865875244, + "avg_weighted_label_matched_iou_90": 0.45351094007492065, + "avg_weighted_label_matched_iou_95": 0.3887667953968048, + "map_50": 0.800000011920929, + "map_75": 0.49966996908187866, + "map_val": 0.5466006398200989, + "name": "doc_42a416ed4de970abf9cc8ed74dbc543df8d7cc6ed24547189c0e8c77e5fed1d5_page_000001.png-page-0", + "value": 0.9544895887374878 + }, + { + "avg_weighted_label_matched_iou_50": 0.9525103569030762, + "avg_weighted_label_matched_iou_75": 0.9372411370277405, + "avg_weighted_label_matched_iou_90": 0.858625054359436, + "avg_weighted_label_matched_iou_95": 0.42830342054367065, + "map_50": 0.8594059348106384, + "map_75": 0.6594059467315674, + "map_val": 0.6091089248657227, + "name": "doc_8a87099365535d996d3167c933ff78e2890f96e70d4a8e2291d0c420fe7093ba_page_000001.png-page-0", + "value": 0.9525103569030762 + }, + { + "avg_weighted_label_matched_iou_50": 0.9523462057113647, + "avg_weighted_label_matched_iou_75": 0.6315879821777344, + "avg_weighted_label_matched_iou_90": 0.6022234559059143, + "avg_weighted_label_matched_iou_95": 0.2966322898864746, + "map_50": 0.800000011920929, + "map_75": 0.6000000238418579, + "map_val": 0.6367656588554382, + "name": "doc_30628cf0d5ec4d1d9dd94853d0711fdf496cbaa199fcab19f6fd4a4ddd34965d_page_000001.png-page-0", + "value": 0.9523462057113647 + }, + { + "avg_weighted_label_matched_iou_50": 0.9514385461807251, + "avg_weighted_label_matched_iou_75": 0.9514385461807251, + "avg_weighted_label_matched_iou_90": 0.7721487879753113, + "avg_weighted_label_matched_iou_95": 0.4990311861038208, + "map_50": 1.0, + "map_75": 1.0, + "map_val": 0.8308581113815308, + "name": "doc_c1381b5a2db9ca221ea4d47f1e037d837679114eb70cafb1b7bb1d681d355d75_page_000001.png-page-0", + "value": 0.9514385461807251 + }, + { + "avg_weighted_label_matched_iou_50": 0.9509844779968262, + "avg_weighted_label_matched_iou_75": 0.6092579364776611, + "avg_weighted_label_matched_iou_90": 0.5560716986656189, + "avg_weighted_label_matched_iou_95": 0.4926658868789673, + "map_50": 1.0, + "map_75": 0.9095159769058228, + "map_val": 0.9108635783195496, + "name": "doc_ace262c34d2a1ad90acd6fce0e75610f64e96a8c35e03cd11505efd130953264_page_000001.png-page-0", + "value": 0.9509844779968262 + }, + { + "avg_weighted_label_matched_iou_50": 0.9506916999816895, + "avg_weighted_label_matched_iou_75": 0.7106003165245056, + "avg_weighted_label_matched_iou_90": 0.09215949475765228, + "avg_weighted_label_matched_iou_95": 0.09215949475765228, + "map_50": 1.0, + "map_75": 0.75, + "map_val": 0.7806105613708496, + "name": "doc_c8f35048844313b77c8c6d6a059b9f91a3b5e34f30e7a774a0f2d8fca82101eb_page_000001.png-page-0", + "value": 0.9506916999816895 + }, + { + "avg_weighted_label_matched_iou_50": 0.9490339159965515, + "avg_weighted_label_matched_iou_75": 0.9490339159965515, + "avg_weighted_label_matched_iou_90": 0.8140360116958618, + "avg_weighted_label_matched_iou_95": 0.19945424795150757, + "map_50": 1.0, + "map_75": 1.0, + "map_val": 0.8831683397293091, + "name": "doc_4b2666400d10ffed958b17ba6fc8b72a0526f1f346d130198afc2ba142537502_page_000001.png-page-0", + "value": 0.9490339159965515 + }, + { + "avg_weighted_label_matched_iou_50": 0.9486706852912903, + "avg_weighted_label_matched_iou_75": 0.5658518671989441, + "avg_weighted_label_matched_iou_90": 0.5582981109619141, + "avg_weighted_label_matched_iou_95": 0.5582981109619141, + "map_50": 1.0, + "map_75": 0.30816832184791565, + "map_val": 0.6424505114555359, + "name": "doc_f01f786e76db9a4e8e321342660e2a539d4b060e123b42a58ec84df128455d59_page_000001.png-page-0", + "value": 0.9486706852912903 + }, + { + "avg_weighted_label_matched_iou_50": 0.9469337463378906, + "avg_weighted_label_matched_iou_75": 0.9469337463378906, + "avg_weighted_label_matched_iou_90": 0.8774337768554688, + "avg_weighted_label_matched_iou_95": 0.39214926958084106, + "map_50": 1.0, + "map_75": 1.0, + "map_val": 0.9112211465835571, + "name": "doc_2bc220dfeed673f84b74c53248a4c4ef8ee5c01d1591188c2b770519de94e82d_page_000001.png-page-0", + "value": 0.9469337463378906 + }, + { + "avg_weighted_label_matched_iou_50": 0.9449262022972107, + "avg_weighted_label_matched_iou_75": 0.5118629336357117, + "avg_weighted_label_matched_iou_90": 0.5157959461212158, + "avg_weighted_label_matched_iou_95": 0.5157959461212158, + "map_50": 1.0, + "map_75": 0.5, + "map_val": 0.5942243933677673, + "name": "doc_0c00e243f924c0b33ccb0aede1a67b433927a1519cb33bfd704e23f6724dd560_page_000001.png-page-0", + "value": 0.9449262022972107 + }, + { + "avg_weighted_label_matched_iou_50": 0.9441550970077515, + "avg_weighted_label_matched_iou_75": 0.08444801717996597, + "avg_weighted_label_matched_iou_90": 0.03743501752614975, + "avg_weighted_label_matched_iou_95": 0.03134416043758392, + "map_50": 1.0, + "map_75": 0.6231110095977783, + "map_val": 0.7016094326972961, + "name": "doc_d6612de0987496c11003e3acc3632cc1a34f17c6b3473cafc2f736be9ba40308_page_000001.png-page-0", + "value": 0.9441550970077515 + }, + { + "avg_weighted_label_matched_iou_50": 0.9438775777816772, + "avg_weighted_label_matched_iou_75": 0.9438775777816772, + "avg_weighted_label_matched_iou_90": 0.9438775777816772, + "avg_weighted_label_matched_iou_95": 0.9341586828231812, + "map_50": 0.9628713130950928, + "map_75": 0.9628713130950928, + "map_val": 0.9441831707954407, + "name": "doc_e30779548a80ff14228753ece493dbce0b4329e426b5dbe61b19d8721fec6dee_page_000001.png-page-0", + "value": 0.9438775777816772 + }, + { + "avg_weighted_label_matched_iou_50": 0.9421811103820801, + "avg_weighted_label_matched_iou_75": 0.9421811103820801, + "avg_weighted_label_matched_iou_90": 0.6832085847854614, + "avg_weighted_label_matched_iou_95": 0.2397266924381256, + "map_50": 1.0, + "map_75": 1.0, + "map_val": 0.8626237511634827, + "name": "doc_43725982fe76bf6e308a001ad81cab4781e803e1d8574e9896be7cfdf2fa8732_page_000001.png-page-0", + "value": 0.9421811103820801 + }, + { + "avg_weighted_label_matched_iou_50": 0.9419163465499878, + "avg_weighted_label_matched_iou_75": 0.8383907079696655, + "avg_weighted_label_matched_iou_90": 0.5078065991401672, + "avg_weighted_label_matched_iou_95": 0.415530264377594, + "map_50": 0.6658415794372559, + "map_75": 0.523878276348114, + "map_val": 0.5307097434997559, + "name": "doc_906d54a21ef3c7bfac03f4bb613b0c79ef32fdf81b362450c79e98a96f88708a_page_000001.png-page-0", + "value": 0.9419163465499878 + }, + { + "avg_weighted_label_matched_iou_50": 0.9403232932090759, + "avg_weighted_label_matched_iou_75": 0.9403232932090759, + "avg_weighted_label_matched_iou_90": 0.8400506973266602, + "avg_weighted_label_matched_iou_95": 0.21792781352996826, + "map_50": 1.0, + "map_75": 1.0, + "map_val": 0.896039605140686, + "name": "doc_da3f562f120acab3005c9735a195e3c57322c6b1439f22c72d7951eaa2544985_page_000001.png-page-0", + "value": 0.9403232932090759 + }, + { + "avg_weighted_label_matched_iou_50": 0.9386529326438904, + "avg_weighted_label_matched_iou_75": 0.9386529326438904, + "avg_weighted_label_matched_iou_90": 0.6723020672798157, + "avg_weighted_label_matched_iou_95": 0.3447636663913727, + "map_50": 0.6000000238418579, + "map_75": 0.6000000238418579, + "map_val": 0.5425082445144653, + "name": "doc_32398c8e13acf26f1de76c730eaf6f57742037357a0eb25338b1fd7a5b816bf1_page_000001.png-page-0", + "value": 0.9386529326438904 + }, + { + "avg_weighted_label_matched_iou_50": 0.9383281469345093, + "avg_weighted_label_matched_iou_75": 0.9383281469345093, + "avg_weighted_label_matched_iou_90": 0.8591764569282532, + "avg_weighted_label_matched_iou_95": 0.19996272027492523, + "map_50": 1.0, + "map_75": 1.0, + "map_val": 0.8966336846351624, + "name": "doc_c00f679ff2c9acbd9ae3904fd350d0f24bea93fb90f44128c7c29a00483b6611_page_000001.png-page-0", + "value": 0.9383281469345093 + }, + { + "avg_weighted_label_matched_iou_50": 0.9373413920402527, + "avg_weighted_label_matched_iou_75": 0.796025812625885, + "avg_weighted_label_matched_iou_90": 0.796025812625885, + "avg_weighted_label_matched_iou_95": 0.17439353466033936, + "map_50": 1.0, + "map_75": 0.5, + "map_val": 0.6168316602706909, + "name": "doc_f23d48a7427bb9747681758df25281d6da6606363d6891058af006389747ee68_page_000001.png-page-0", + "value": 0.9373413920402527 + }, + { + "avg_weighted_label_matched_iou_50": 0.9362494349479675, + "avg_weighted_label_matched_iou_75": 0.9362494349479675, + "avg_weighted_label_matched_iou_90": 0.6494951248168945, + "avg_weighted_label_matched_iou_95": 0.08166538178920746, + "map_50": 1.0, + "map_75": 1.0, + "map_val": 0.793006420135498, + "name": "doc_4f6d9c52848acf2346013388db8ef25c40e6e2fd1232c49f2abfa7a21f65c82c_page_000001.png-page-0", + "value": 0.9362494349479675 + }, + { + "avg_weighted_label_matched_iou_50": 0.9358420968055725, + "avg_weighted_label_matched_iou_75": 0.9358420968055725, + "avg_weighted_label_matched_iou_90": 0.8806563019752502, + "avg_weighted_label_matched_iou_95": 0.6334714889526367, + "map_50": 0.9603960514068604, + "map_75": 0.9603960514068604, + "map_val": 0.8514065742492676, + "name": "doc_9c897807c023cf6c34295d03cccb262ed0cbef83bef63d4908ab6d3775b15597_page_000001.png-page-0", + "value": 0.9358420968055725 + }, + { + "avg_weighted_label_matched_iou_50": 0.9354779124259949, + "avg_weighted_label_matched_iou_75": 0.8993185758590698, + "avg_weighted_label_matched_iou_90": 0.8272649645805359, + "avg_weighted_label_matched_iou_95": 0.24133189022541046, + "map_50": 1.0, + "map_75": 0.6666666865348816, + "map_val": 0.6279428005218506, + "name": "doc_cd34bf0452e0bc9b7f26515c7426f9a197480478f52f464e67a0a5912221da69_page_000001.png-page-0", + "value": 0.9354779124259949 + }, + { + "avg_weighted_label_matched_iou_50": 0.9331861138343811, + "avg_weighted_label_matched_iou_75": 0.41669926047325134, + "avg_weighted_label_matched_iou_90": 0.2774311602115631, + "avg_weighted_label_matched_iou_95": 0.22559523582458496, + "map_50": 0.800000011920929, + "map_75": 0.628052830696106, + "map_val": 0.6396369934082031, + "name": "doc_c9c49f563c6f47d9c28f9caf13455ef61345a0c9be4bba251d064958f546c6f5_page_000001.png-page-0", + "value": 0.9331861138343811 + }, + { + "avg_weighted_label_matched_iou_50": 0.9306560754776001, + "avg_weighted_label_matched_iou_75": 0.9306560754776001, + "avg_weighted_label_matched_iou_90": 0.9306560754776001, + "avg_weighted_label_matched_iou_95": 0.9306560754776001, + "map_50": 0.7554455399513245, + "map_75": 0.7554455399513245, + "map_val": 0.7554455399513245, + "name": "doc_22d276b33b49ec941acbc323e24fa57ed07579760f472c136f226d755a0c3b13_page_000001.png-page-0", + "value": 0.9306560754776001 + }, + { + "avg_weighted_label_matched_iou_50": 0.9301129579544067, + "avg_weighted_label_matched_iou_75": 0.8316190838813782, + "avg_weighted_label_matched_iou_90": 0.22219663858413696, + "avg_weighted_label_matched_iou_95": 0.07811024785041809, + "map_50": 1.0, + "map_75": 0.6666666865348816, + "map_val": 0.6550055146217346, + "name": "doc_5d089e42c7c614828adb0ec78abd0c87d078aa01b79789d79aecea6411ff53e8_page_000001.png-page-0", + "value": 0.9301129579544067 + }, + { + "avg_weighted_label_matched_iou_50": 0.9283837676048279, + "avg_weighted_label_matched_iou_75": 0.320468008518219, + "avg_weighted_label_matched_iou_90": 0.016505038365721703, + "avg_weighted_label_matched_iou_95": 0.016505038365721703, + "map_50": 1.0, + "map_75": 0.5, + "map_val": 0.64207923412323, + "name": "doc_2fd524178510079b3618bd62ef2f0db3ff4e5f967afe578924c336d74e031f70_page_000001.png-page-0", + "value": 0.9283837676048279 + }, + { + "avg_weighted_label_matched_iou_50": 0.924868106842041, + "avg_weighted_label_matched_iou_75": 0.924868106842041, + "avg_weighted_label_matched_iou_90": 0.924868106842041, + "avg_weighted_label_matched_iou_95": 0.4495721459388733, + "map_50": 0.6881188154220581, + "map_75": 0.6881188154220581, + "map_val": 0.6506187915802002, + "name": "doc_a24a2d4ed5bc5e9aa585d4d29ea5b66737ea96bcaa84756d1b183e7c0ad77e91_page_000001.png-page-0", + "value": 0.924868106842041 + }, + { + "avg_weighted_label_matched_iou_50": 0.9229961633682251, + "avg_weighted_label_matched_iou_75": 0.6654443144798279, + "avg_weighted_label_matched_iou_90": 0.41392505168914795, + "avg_weighted_label_matched_iou_95": 0.41392505168914795, + "map_50": 0.6712871193885803, + "map_75": 0.37029704451560974, + "map_val": 0.5047920942306519, + "name": "doc_a012af9597cad209a43dc6a6a4543994e857a56fac9fdd3eed92a5e1b93f79b2_page_000001.png-page-0", + "value": 0.9229961633682251 + }, + { + "avg_weighted_label_matched_iou_50": 0.9204308390617371, + "avg_weighted_label_matched_iou_75": 0.49525871872901917, + "avg_weighted_label_matched_iou_90": 0.4819905161857605, + "avg_weighted_label_matched_iou_95": 0.14315322041511536, + "map_50": 0.75, + "map_75": 0.5, + "map_val": 0.4647029638290405, + "name": "doc_18fbe4800ce186f3338eade3d1800bde4e275396656df9781f0d84f4b6e22f0c_page_000001.png-page-0", + "value": 0.9204308390617371 + }, + { + "avg_weighted_label_matched_iou_50": 0.9150271415710449, + "avg_weighted_label_matched_iou_75": 0.9150271415710449, + "avg_weighted_label_matched_iou_90": 0.42739608883857727, + "avg_weighted_label_matched_iou_95": 0.42739608883857727, + "map_50": 0.5847772359848022, + "map_75": 0.5847772359848022, + "map_val": 0.5344059467315674, + "name": "doc_f974132511fc3655d9602cf317b6ae2ed040787671bb21e387322961b5ac30f1_page_000001.png-page-0", + "value": 0.9150271415710449 + }, + { + "avg_weighted_label_matched_iou_50": 0.9126367568969727, + "avg_weighted_label_matched_iou_75": 0.9126367568969727, + "avg_weighted_label_matched_iou_90": 0.7450672388076782, + "avg_weighted_label_matched_iou_95": 0.7450672388076782, + "map_50": 0.75, + "map_75": 0.75, + "map_val": 0.699999988079071, + "name": "doc_3b10e813590fb9c9f54883151a2dd4b4ccb407becc54fd516a82972dc261630b_page_000001.png-page-0", + "value": 0.9126367568969727 + }, + { + "avg_weighted_label_matched_iou_50": 0.9116213917732239, + "avg_weighted_label_matched_iou_75": 0.9116213917732239, + "avg_weighted_label_matched_iou_90": 0.9116213917732239, + "avg_weighted_label_matched_iou_95": 0.9116213917732239, + "map_50": 0.8903818726539612, + "map_75": 0.8903818726539612, + "map_val": 0.8903818726539612, + "name": "doc_196876d21778214061c0d970de83cf37204a90e0f58c3da7e9f3c8c7e803b770_page_000001.png-page-0", + "value": 0.9116213917732239 + }, + { + "avg_weighted_label_matched_iou_50": 0.909862756729126, + "avg_weighted_label_matched_iou_75": 0.909862756729126, + "avg_weighted_label_matched_iou_90": 0.909862756729126, + "avg_weighted_label_matched_iou_95": 0.7783505320549011, + "map_50": 0.995232880115509, + "map_75": 0.995232880115509, + "map_val": 0.9883021712303162, + "name": "doc_764990d134c06378c3f496bfae33215c8ff57a0d306cc984e6429a6ed48257df_page_000001.png-page-0", + "value": 0.909862756729126 + }, + { + "avg_weighted_label_matched_iou_50": 0.9062057733535767, + "avg_weighted_label_matched_iou_75": 0.9062057733535767, + "avg_weighted_label_matched_iou_90": 0.7126863598823547, + "avg_weighted_label_matched_iou_95": 0.45065003633499146, + "map_50": 0.6526402831077576, + "map_75": 0.6526402831077576, + "map_val": 0.5632425546646118, + "name": "doc_10817297774a93c1838650c06b79f518db43af558496041e256bd8c259057f79_page_000001.png-page-0", + "value": 0.9062057733535767 + }, + { + "avg_weighted_label_matched_iou_50": 0.8986648321151733, + "avg_weighted_label_matched_iou_75": 0.8887893557548523, + "avg_weighted_label_matched_iou_90": 0.5694871544837952, + "avg_weighted_label_matched_iou_95": 0.2943893074989319, + "map_50": 0.75, + "map_75": 0.6862623691558838, + "map_val": 0.5867486000061035, + "name": "doc_d6ce9ae24c7a15aa55e9a8c232c913cb8bcc1dc2d5d7f129258f4cbf6f46719d_page_000001.png-page-0", + "value": 0.8986648321151733 + }, + { + "avg_weighted_label_matched_iou_50": 0.8968093395233154, + "avg_weighted_label_matched_iou_75": 0.8537799715995789, + "avg_weighted_label_matched_iou_90": 0.20812468230724335, + "avg_weighted_label_matched_iou_95": 0.20812468230724335, + "map_50": 1.0, + "map_75": 0.7508251070976257, + "map_val": 0.7585808634757996, + "name": "doc_df678b583abe6afa6971fac9efa23189bd235105d697ae1b2b12d25e3b9fb8b4_page_000001.png-page-0", + "value": 0.8968093395233154 + }, + { + "avg_weighted_label_matched_iou_50": 0.8934210538864136, + "avg_weighted_label_matched_iou_75": 0.8934210538864136, + "avg_weighted_label_matched_iou_90": 0.8314065337181091, + "avg_weighted_label_matched_iou_95": 0.7706968188285828, + "map_50": 0.7788779139518738, + "map_75": 0.7788779139518738, + "map_val": 0.6343234181404114, + "name": "doc_93bc667fb64454b9df1688b9873ed8fbf3f548d58c81ec0ca3d9cc50e79fb667_page_000001.png-page-0", + "value": 0.8934210538864136 + }, + { + "avg_weighted_label_matched_iou_50": 0.8925890922546387, + "avg_weighted_label_matched_iou_75": 0.8925890922546387, + "avg_weighted_label_matched_iou_90": 0.8925890922546387, + "avg_weighted_label_matched_iou_95": 0.8925890922546387, + "map_50": 0.4830858111381531, + "map_75": 0.4830858111381531, + "map_val": 0.4830858111381531, + "name": "doc_275419814c80169ba0ca779343d8cf778903225b34dd40d0d63ef119b9892f78_page_000001.png-page-0", + "value": 0.8925890922546387 + }, + { + "avg_weighted_label_matched_iou_50": 0.8887479305267334, + "avg_weighted_label_matched_iou_75": 0.8887479305267334, + "avg_weighted_label_matched_iou_90": 0.6759513020515442, + "avg_weighted_label_matched_iou_95": 0.6759513020515442, + "map_50": 0.8762376308441162, + "map_75": 0.8762376308441162, + "map_val": 0.8257426023483276, + "name": "doc_caa98180e7cec3f27dfa28e4d67a66748ba4921bd3e2cc6d3cf5ceab4e765ec1_page_000001.png-page-0", + "value": 0.8887479305267334 + }, + { + "avg_weighted_label_matched_iou_50": 0.8865717649459839, + "avg_weighted_label_matched_iou_75": 0.8865717649459839, + "avg_weighted_label_matched_iou_90": 0.8865717649459839, + "avg_weighted_label_matched_iou_95": 0.45474013686180115, + "map_50": 0.75, + "map_75": 0.75, + "map_val": 0.7126237750053406, + "name": "doc_f99de206dc21ba8ea57548b1c0c007eb736da5f26c925dc99a4027806297bd8f_page_000001.png-page-0", + "value": 0.8865717649459839 + }, + { + "avg_weighted_label_matched_iou_50": 0.8845615983009338, + "avg_weighted_label_matched_iou_75": 0.8845615983009338, + "avg_weighted_label_matched_iou_90": 0.11128707975149155, + "avg_weighted_label_matched_iou_95": 0.05621664598584175, + "map_50": 1.0, + "map_75": 1.0, + "map_val": 0.8146039843559265, + "name": "doc_97d60e0c292030da220648b4ce7e8971adbe29731b7077d9c3c8d7c6dd778e42_page_000001.png-page-0", + "value": 0.8845615983009338 + }, + { + "avg_weighted_label_matched_iou_50": 0.8802587389945984, + "avg_weighted_label_matched_iou_75": 0.8802587389945984, + "avg_weighted_label_matched_iou_90": 0.17595285177230835, + "avg_weighted_label_matched_iou_95": 0.17595285177230835, + "map_50": 0.6000000238418579, + "map_75": 0.6000000238418579, + "map_val": 0.5348118543624878, + "name": "doc_5699486e354a93df094404bae0cd17381bfab180f9ab2b5c9f55428c07a106c5_page_000001.png-page-0", + "value": 0.8802587389945984 + }, + { + "avg_weighted_label_matched_iou_50": 0.8789843916893005, + "avg_weighted_label_matched_iou_75": 0.05081353336572647, + "avg_weighted_label_matched_iou_90": 0.017049936577677727, + "avg_weighted_label_matched_iou_95": 0.017049936577677727, + "map_50": 0.6683168411254883, + "map_75": 0.5016501545906067, + "map_val": 0.4764026403427124, + "name": "doc_a5528f4dd6f5ba0d52e6bf97a0aae99ede782fe97f4a0b4f96029e5ed7fe18b5_page_000001.png-page-0", + "value": 0.8789843916893005 + }, + { + "avg_weighted_label_matched_iou_50": 0.8786144852638245, + "avg_weighted_label_matched_iou_75": 0.8400722742080688, + "avg_weighted_label_matched_iou_90": 0.8400722742080688, + "avg_weighted_label_matched_iou_95": 0.8400722742080688, + "map_50": 0.8943894505500793, + "map_75": 0.8096809387207031, + "map_val": 0.8350934982299805, + "name": "doc_eaafa071048c2ec1309751325280ea3d354317cf67ebda400cdf4fd39503392b_page_000001.png-page-0", + "value": 0.8786144852638245 + }, + { + "avg_weighted_label_matched_iou_50": 0.8701573014259338, + "avg_weighted_label_matched_iou_75": 0.8701573014259338, + "avg_weighted_label_matched_iou_90": 0.8124291896820068, + "avg_weighted_label_matched_iou_95": 0.7568349242210388, + "map_50": 0.7009900808334351, + "map_75": 0.7009900808334351, + "map_val": 0.6009901165962219, + "name": "doc_22f4e5d348c45a3399f3c8a0e48756dd82a96ed9f5723a2a39183b554667d19a_page_000001.png-page-0", + "value": 0.8701573014259338 + }, + { + "avg_weighted_label_matched_iou_50": 0.8652716875076294, + "avg_weighted_label_matched_iou_75": 0.8652716875076294, + "avg_weighted_label_matched_iou_90": 0.8652716875076294, + "avg_weighted_label_matched_iou_95": 0.5844986438751221, + "map_50": 0.75, + "map_75": 0.75, + "map_val": 0.7250000238418579, + "name": "doc_a0edae1fa147c7bb78ebc493743a68ba4372b5ead31f2a2b146c35119462379e_page_000001.png-page-0", + "value": 0.8652716875076294 + }, + { + "avg_weighted_label_matched_iou_50": 0.8635362386703491, + "avg_weighted_label_matched_iou_75": 0.36877304315567017, + "avg_weighted_label_matched_iou_90": 0.055571671575307846, + "avg_weighted_label_matched_iou_95": 0.011053694412112236, + "map_50": 0.800000011920929, + "map_75": 0.5657708644866943, + "map_val": 0.554787814617157, + "name": "doc_37b2674a918c0c0782194d45b011f7c9ffca19ccc245545fb5e35027e7c67a9d_page_000001.png-page-0", + "value": 0.8635362386703491 + }, + { + "avg_weighted_label_matched_iou_50": 0.8595455884933472, + "avg_weighted_label_matched_iou_75": 0.6191399097442627, + "avg_weighted_label_matched_iou_90": 0.599827766418457, + "avg_weighted_label_matched_iou_95": 0.599827766418457, + "map_50": 0.7333333492279053, + "map_75": 0.5333333611488342, + "map_val": 0.5367656946182251, + "name": "doc_4682941b1a9a3ec96599d8188b673e8c6d1c4f2a555b5dc0d739c6c706815553_page_000001.png-page-0", + "value": 0.8595455884933472 + }, + { + "avg_weighted_label_matched_iou_50": 0.8586272597312927, + "avg_weighted_label_matched_iou_75": 0.6682833433151245, + "avg_weighted_label_matched_iou_90": 0.552062451839447, + "avg_weighted_label_matched_iou_95": 0.4943515360355377, + "map_50": 0.6916548609733582, + "map_75": 0.44165489077568054, + "map_val": 0.5033239126205444, + "name": "doc_39aec6caead187513fa9e2a26a3bbb54ba0a3a4ae8e879642ca1584330b0c285_page_000001.png-page-0", + "value": 0.8586272597312927 + }, + { + "avg_weighted_label_matched_iou_50": 0.8530619144439697, + "avg_weighted_label_matched_iou_75": 0.8530619144439697, + "avg_weighted_label_matched_iou_90": 0.8530619144439697, + "avg_weighted_label_matched_iou_95": 0.8530619144439697, + "map_50": 0.6666666865348816, + "map_75": 0.6666666865348816, + "map_val": 0.6666666865348816, + "name": "doc_7675bb9557d6e1e663237098f37aaecb61ab411b71034c2d397487ed0deb2ecd_page_000001.png-page-0", + "value": 0.8530619144439697 + }, + { + "avg_weighted_label_matched_iou_50": 0.8483885526657104, + "avg_weighted_label_matched_iou_75": 0.7628428936004639, + "avg_weighted_label_matched_iou_90": 0.6427671909332275, + "avg_weighted_label_matched_iou_95": 0.5168552994728088, + "map_50": 0.5911716222763062, + "map_75": 0.40429043769836426, + "map_val": 0.4453795254230499, + "name": "doc_4d4bf58bf1563cdfb71b2e1272e95c33d01625a9627694fafdf0c69b50816ea5_page_000001.png-page-0", + "value": 0.8483885526657104 + }, + { + "avg_weighted_label_matched_iou_50": 0.8379843831062317, + "avg_weighted_label_matched_iou_75": 0.8379843831062317, + "avg_weighted_label_matched_iou_90": 0.8379843831062317, + "avg_weighted_label_matched_iou_95": 0.8379843831062317, + "map_50": 0.6666666865348816, + "map_75": 0.6666666865348816, + "map_val": 0.6666666865348816, + "name": "doc_c04c230b8dfa00d0fabbe9730c078c840ccaa27f5f270d5899f35405784a535d_page_000001.png-page-0", + "value": 0.8379843831062317 + }, + { + "avg_weighted_label_matched_iou_50": 0.8310454487800598, + "avg_weighted_label_matched_iou_75": 0.6723902821540833, + "avg_weighted_label_matched_iou_90": 0.07023763656616211, + "avg_weighted_label_matched_iou_95": 0.05580037459731102, + "map_50": 0.6666666865348816, + "map_75": 0.5990098714828491, + "map_val": 0.540099024772644, + "name": "doc_2e4f2561d8624cab6140564d53aed11862cf5979572fc3d1205af105486e0b1f_page_000001.png-page-0", + "value": 0.8310454487800598 + }, + { + "avg_weighted_label_matched_iou_50": 0.8042868375778198, + "avg_weighted_label_matched_iou_75": 0.46401095390319824, + "avg_weighted_label_matched_iou_90": 0.4571266174316406, + "avg_weighted_label_matched_iou_95": 0.4571266174316406, + "map_50": 0.6113861203193665, + "map_75": 0.42450493574142456, + "map_val": 0.4310643672943115, + "name": "doc_f3e3d62292122d9be48e262e7d8e1f9363bab000e26437ffb6f2f94afc3c2294_page_000001.png-page-0", + "value": 0.8042868375778198 + }, + { + "avg_weighted_label_matched_iou_50": 0.8034886121749878, + "avg_weighted_label_matched_iou_75": 0.8034886121749878, + "avg_weighted_label_matched_iou_90": 0.8034886121749878, + "avg_weighted_label_matched_iou_95": 0.8034886121749878, + "map_50": 0.9049504995346069, + "map_75": 0.9049504995346069, + "map_val": 0.9049504995346069, + "name": "doc_2b12f031b9b42e6f54b76b7d4666df76b7c8410a676d63b0ad2afdb40388d623_page_000001.png-page-0", + "value": 0.8034886121749878 + }, + { + "avg_weighted_label_matched_iou_50": 0.803397536277771, + "avg_weighted_label_matched_iou_75": 0.18337038159370422, + "avg_weighted_label_matched_iou_90": 0.08443697541952133, + "avg_weighted_label_matched_iou_95": 0.08443697541952133, + "map_50": 0.75, + "map_75": 0.2953541576862335, + "map_val": 0.4181416630744934, + "name": "doc_12e9c991f3d907593bacefdcf9a46c2cb57b7fb599e7840f438df04e416aab9d_page_000001.png-page-0", + "value": 0.803397536277771 + }, + { + "avg_weighted_label_matched_iou_50": 0.7935387492179871, + "avg_weighted_label_matched_iou_75": 0.7935387492179871, + "avg_weighted_label_matched_iou_90": 0.7935387492179871, + "avg_weighted_label_matched_iou_95": 0.2201295793056488, + "map_50": 0.8333333134651184, + "map_75": 0.8333333134651184, + "map_val": 0.777062714099884, + "name": "doc_9720f5990852942d3d2e0c84e6dfb1db5c528f3811b1fce9d320e8fe55219b78_page_000001.png-page-0", + "value": 0.7935387492179871 + }, + { + "avg_weighted_label_matched_iou_50": 0.7925370931625366, + "avg_weighted_label_matched_iou_75": 0.4707385301589966, + "avg_weighted_label_matched_iou_90": 0.45806995034217834, + "avg_weighted_label_matched_iou_95": 0.45806995034217834, + "map_50": 0.800000011920929, + "map_75": 0.18943893909454346, + "map_val": 0.4382838308811188, + "name": "doc_bcb3dafc35b5e7476fd1b9cd6eccf5eeef936cd5b13ad846a4943f1e7797f4e9_page_000001.png-page-0", + "value": 0.7925370931625366 + }, + { + "avg_weighted_label_matched_iou_50": 0.7918540239334106, + "avg_weighted_label_matched_iou_75": 0.7918540239334106, + "avg_weighted_label_matched_iou_90": 0.7918540239334106, + "avg_weighted_label_matched_iou_95": 0.506723165512085, + "map_50": 0.5528053045272827, + "map_75": 0.5528053045272827, + "map_val": 0.538684606552124, + "name": "doc_a14ebf0077dbbb76dd91f3d7d0b66b66b4ab62260846cc75fb466b85a4a762b4_page_000001.png-page-0", + "value": 0.7918540239334106 + }, + { + "avg_weighted_label_matched_iou_50": 0.788382887840271, + "avg_weighted_label_matched_iou_75": 0.788382887840271, + "avg_weighted_label_matched_iou_90": 0.788382887840271, + "avg_weighted_label_matched_iou_95": 0.788382887840271, + "map_50": 0.6666666865348816, + "map_75": 0.6666666865348816, + "map_val": 0.6666666865348816, + "name": "doc_872d56afecf02a9a4ff4ec021e4347ead2b7f227e9299ff85fee4e3c4993788e_page_000001.png-page-0", + "value": 0.788382887840271 + }, + { + "avg_weighted_label_matched_iou_50": 0.7794774174690247, + "avg_weighted_label_matched_iou_75": 0.7060752511024475, + "avg_weighted_label_matched_iou_90": 0.7060752511024475, + "avg_weighted_label_matched_iou_95": 0.45324400067329407, + "map_50": 0.6875, + "map_75": 0.4375, + "map_val": 0.5229579210281372, + "name": "doc_0ce1ded16152ad18c45203c1dcba0287235b888266fbcd112396fdf7cd094fab_page_000001.png-page-0", + "value": 0.7794774174690247 + }, + { + "avg_weighted_label_matched_iou_50": 0.7674703598022461, + "avg_weighted_label_matched_iou_75": 0.7674703598022461, + "avg_weighted_label_matched_iou_90": 0.7674703598022461, + "avg_weighted_label_matched_iou_95": 0.44493943452835083, + "map_50": 1.0, + "map_75": 1.0, + "map_val": 0.9752475023269653, + "name": "doc_32b723d3d51157dd3e96717aff7efc3d13c5c6c942668b1fbce46bcbe8d94354_page_000001.png-page-0", + "value": 0.7674703598022461 + }, + { + "avg_weighted_label_matched_iou_50": 0.76442551612854, + "avg_weighted_label_matched_iou_75": 0.6262413859367371, + "avg_weighted_label_matched_iou_90": 0.4100084900856018, + "avg_weighted_label_matched_iou_95": 0.04523826763033867, + "map_50": 0.6428571343421936, + "map_75": 0.4728429913520813, + "map_val": 0.49259310960769653, + "name": "doc_ed046862e68a27a7260da60f3984484577f83500dea61020d043c7af8b76731f_page_000001.png-page-0", + "value": 0.76442551612854 + }, + { + "avg_weighted_label_matched_iou_50": 0.7533853054046631, + "avg_weighted_label_matched_iou_75": 0.7533853054046631, + "avg_weighted_label_matched_iou_90": 0.16541345417499542, + "avg_weighted_label_matched_iou_95": 0.12246809899806976, + "map_50": 0.6485148668289185, + "map_75": 0.6485148668289185, + "map_val": 0.531930685043335, + "name": "doc_14e279e87615e3e3bb697729ecd97a2c5f3cd1195d9b1a2733ebfde630fea23d_page_000001.png-page-0", + "value": 0.7533853054046631 + }, + { + "avg_weighted_label_matched_iou_50": 0.7423190474510193, + "avg_weighted_label_matched_iou_75": 0.4677715599536896, + "avg_weighted_label_matched_iou_90": 0.19580958783626556, + "avg_weighted_label_matched_iou_95": 0.19580958783626556, + "map_50": 0.7689769268035889, + "map_75": 0.5390539169311523, + "map_val": 0.5545654296875, + "name": "doc_cbb4a13ffd01d9f777fdb939451d6a21cea1b869ee50d79581451e3601df9ec8_page_000001.png-page-0", + "value": 0.7423190474510193 + }, + { + "avg_weighted_label_matched_iou_50": 0.7284966111183167, + "avg_weighted_label_matched_iou_75": 0.09818997234106064, + "avg_weighted_label_matched_iou_90": 0.09818997234106064, + "avg_weighted_label_matched_iou_95": 0.09818997234106064, + "map_50": 0.6666666865348816, + "map_75": 0.3333333432674408, + "map_val": 0.5, + "name": "doc_d276ca9a5ecb8d6d11359f515e50c8f78395548de4e3e2c49e38f5500ee40ebe_page_000001.png-page-0", + "value": 0.7284966111183167 + }, + { + "avg_weighted_label_matched_iou_50": 0.7168993353843689, + "avg_weighted_label_matched_iou_75": 0.19427651166915894, + "avg_weighted_label_matched_iou_90": 0.0, + "avg_weighted_label_matched_iou_95": 0.0, + "map_50": 0.8547855019569397, + "map_75": 0.10081008076667786, + "map_val": 0.3506210744380951, + "name": "doc_10f75857bb705a5441260399c3fca1080262cc0c271270825caf141cf7b31770_page_000001.png-page-0", + "value": 0.7168993353843689 + }, + { + "avg_weighted_label_matched_iou_50": 0.712614893913269, + "avg_weighted_label_matched_iou_75": 0.712614893913269, + "avg_weighted_label_matched_iou_90": 0.712614893913269, + "avg_weighted_label_matched_iou_95": 0.712614893913269, + "map_50": 0.8881188035011292, + "map_75": 0.8881188035011292, + "map_val": 0.8881188035011292, + "name": "doc_ab0f3ac6ce114304cc331404f20a780f57fcb060b5c96c54c69478d474997d1f_page_000001.png-page-0", + "value": 0.712614893913269 + }, + { + "avg_weighted_label_matched_iou_50": 0.7024937868118286, + "avg_weighted_label_matched_iou_75": 0.4502190947532654, + "avg_weighted_label_matched_iou_90": 0.3976336717605591, + "avg_weighted_label_matched_iou_95": 0.35544905066490173, + "map_50": 0.6858085989952087, + "map_75": 0.250495046377182, + "map_val": 0.3650891184806824, + "name": "doc_770c9434a3530bfb67ef32510b63e1da347dfd6c51e5db9e0b13e90c8b287895_page_000001.png-page-0", + "value": 0.7024937868118286 + }, + { + "avg_weighted_label_matched_iou_50": 0.691452145576477, + "avg_weighted_label_matched_iou_75": 0.6556690335273743, + "avg_weighted_label_matched_iou_90": 0.4655549228191376, + "avg_weighted_label_matched_iou_95": 0.4655549228191376, + "map_50": 0.6666666865348816, + "map_75": 0.5016501545906067, + "map_val": 0.5504950284957886, + "name": "doc_970804c04c81dd9cc3ba80f0063b9551d290e83934c4e2c258e376845b3d180f_page_000001.png-page-0", + "value": 0.691452145576477 + }, + { + "avg_weighted_label_matched_iou_50": 0.6873770356178284, + "avg_weighted_label_matched_iou_75": 0.6873770356178284, + "avg_weighted_label_matched_iou_90": 0.5610297322273254, + "avg_weighted_label_matched_iou_95": 0.5610297322273254, + "map_50": 0.8052805066108704, + "map_75": 0.8052805066108704, + "map_val": 0.6052805185317993, + "name": "doc_e5cd36c1ca7a2c476b14a19497ea75921899a21c6b510f63ef571abf0c000d5b_page_000001.png-page-0", + "value": 0.6873770356178284 + }, + { + "avg_weighted_label_matched_iou_50": 0.6813420057296753, + "avg_weighted_label_matched_iou_75": 0.6813420057296753, + "avg_weighted_label_matched_iou_90": 0.6813420057296753, + "avg_weighted_label_matched_iou_95": 0.6813420057296753, + "map_50": 0.5339934229850769, + "map_75": 0.5339934229850769, + "map_val": 0.5339934229850769, + "name": "doc_f0ab823f66709631e6226a937a8d68b52bde1e7ff6ef28086563c0646867c769_page_000001.png-page-0", + "value": 0.6813420057296753 + }, + { + "avg_weighted_label_matched_iou_50": 0.6662349700927734, + "avg_weighted_label_matched_iou_75": 0.6662349700927734, + "avg_weighted_label_matched_iou_90": 0.6662349700927734, + "avg_weighted_label_matched_iou_95": 0.6662349700927734, + "map_50": 0.4166666567325592, + "map_75": 0.4166666567325592, + "map_val": 0.4166666567325592, + "name": "doc_5179e446711cbe5535f394ec85fb83d4f25c56b7d180d0c3ce97878843afe59e_page_000001.png-page-0", + "value": 0.6662349700927734 + }, + { + "avg_weighted_label_matched_iou_50": 0.6636558175086975, + "avg_weighted_label_matched_iou_75": 0.6636558175086975, + "avg_weighted_label_matched_iou_90": 0.6636558175086975, + "avg_weighted_label_matched_iou_95": 0.5712635517120361, + "map_50": 0.41749176383018494, + "map_75": 0.41749176383018494, + "map_val": 0.41116613149642944, + "name": "doc_e786231794e24874e66321d9a2ce9d80050a73cd0e02fa358c88a1e87a183735_page_000001.png-page-0", + "value": 0.6636558175086975 + }, + { + "avg_weighted_label_matched_iou_50": 0.659218430519104, + "avg_weighted_label_matched_iou_75": 0.7290078401565552, + "avg_weighted_label_matched_iou_90": 0.7290078401565552, + "avg_weighted_label_matched_iou_95": 0.7290078401565552, + "map_50": 0.5676567554473877, + "map_75": 0.46666666865348816, + "map_val": 0.5171617269515991, + "name": "doc_c4416f86c1db7fee5dccc1a10e85b1aade1cd77803f4ee626a090e8bc26aa9a9_page_000001.png-page-0", + "value": 0.659218430519104 + }, + { + "avg_weighted_label_matched_iou_50": 0.6477729678153992, + "avg_weighted_label_matched_iou_75": 0.6477729678153992, + "avg_weighted_label_matched_iou_90": 0.6477729678153992, + "avg_weighted_label_matched_iou_95": 0.6477729678153992, + "map_50": 0.5, + "map_75": 0.5, + "map_val": 0.5, + "name": "doc_20c3068a37794e1e2db40c6227d57172ad308c8a81815c4c08970219586ef4ee_page_000001.png-page-0", + "value": 0.6477729678153992 + }, + { + "avg_weighted_label_matched_iou_50": 0.6475440263748169, + "avg_weighted_label_matched_iou_75": 0.5237763524055481, + "avg_weighted_label_matched_iou_90": 0.13985934853553772, + "avg_weighted_label_matched_iou_95": 0.0, + "map_50": 0.7702970504760742, + "map_75": 0.4000000059604645, + "map_val": 0.48514851927757263, + "name": "doc_12b156f795da26f2c74fb91e0c828427582d595d8bdf1d9acc695cf01e11bf10_page_000001.png-page-0", + "value": 0.6475440263748169 + }, + { + "avg_weighted_label_matched_iou_50": 0.6409308910369873, + "avg_weighted_label_matched_iou_75": 0.4044962525367737, + "avg_weighted_label_matched_iou_90": 0.49719303846359253, + "avg_weighted_label_matched_iou_95": 0.49719303846359253, + "map_50": 0.6000000238418579, + "map_75": 0.4000000059604645, + "map_val": 0.41999998688697815, + "name": "doc_e2b604a3fb1541b82b6af8caca05682dff0c7735e0a3a4fa7c6a68246fb60e57_page_000001.png-page-0", + "value": 0.6409308910369873 + }, + { + "avg_weighted_label_matched_iou_50": 0.6331600546836853, + "avg_weighted_label_matched_iou_75": 0.4861522614955902, + "avg_weighted_label_matched_iou_90": 0.46219000220298767, + "avg_weighted_label_matched_iou_95": 0.46219000220298767, + "map_50": 0.7091584205627441, + "map_75": 0.5841584205627441, + "map_val": 0.5520626902580261, + "name": "doc_4d5b682effaf9580928a7073980e3916687d168238bc96b6c17222f747eacde6_page_000001.png-page-0", + "value": 0.6331600546836853 + }, + { + "avg_weighted_label_matched_iou_50": 0.6294281482696533, + "avg_weighted_label_matched_iou_75": 0.0, + "avg_weighted_label_matched_iou_90": 0.0, + "avg_weighted_label_matched_iou_95": 0.0, + "map_50": 0.896039605140686, + "map_75": 0.0, + "map_val": 0.448019802570343, + "name": "doc_8d09961d139152e2bbe65e06bd7237b707dad351bd2ebc6fcbc3123f9ecd5086_page_000001.png-page-0", + "value": 0.6294281482696533 + }, + { + "avg_weighted_label_matched_iou_50": 0.6210302114486694, + "avg_weighted_label_matched_iou_75": 0.6210302114486694, + "avg_weighted_label_matched_iou_90": 0.6079462766647339, + "avg_weighted_label_matched_iou_95": 0.6131588816642761, + "map_50": 0.6509901285171509, + "map_75": 0.6509901285171509, + "map_val": 0.6284653544425964, + "name": "doc_b639c25ab157f6f4ecbdb7cd31724bf4c8869f03134f5e606c49dd5fbee2bb54_page_000001.png-page-0", + "value": 0.6210302114486694 + }, + { + "avg_weighted_label_matched_iou_50": 0.6201952695846558, + "avg_weighted_label_matched_iou_75": 0.6201952695846558, + "avg_weighted_label_matched_iou_90": 0.6201952695846558, + "avg_weighted_label_matched_iou_95": 0.6201952695846558, + "map_50": 0.3333333432674408, + "map_75": 0.3333333432674408, + "map_val": 0.3333333432674408, + "name": "doc_6d5a00597cdd1d363b56eb0fdc9f239a462b44fcb0be53a93d62f0d6e35d428f_page_000001.png-page-0", + "value": 0.6201952695846558 + }, + { + "avg_weighted_label_matched_iou_50": 0.614162802696228, + "avg_weighted_label_matched_iou_75": 0.3806822896003723, + "avg_weighted_label_matched_iou_90": 0.08673719316720963, + "avg_weighted_label_matched_iou_95": 0.027586812153458595, + "map_50": 0.6993399262428284, + "map_75": 0.4003300368785858, + "map_val": 0.4431683123111725, + "name": "doc_f151ca1888000160dc6ce2e46c3438bc13f9a034200b1ec5d81b03c77606eed6_page_000001.png-page-0", + "value": 0.614162802696228 + }, + { + "avg_weighted_label_matched_iou_50": 0.6000663042068481, + "avg_weighted_label_matched_iou_75": 0.6000663042068481, + "avg_weighted_label_matched_iou_90": 0.6000663042068481, + "avg_weighted_label_matched_iou_95": 0.43728142976760864, + "map_50": 0.75, + "map_75": 0.75, + "map_val": 0.7250000238418579, + "name": "doc_4b5f23f97fd273cacb8de95fc1799ed4dc4d5c15e9ed86d707cdf4be53eb7c7c_page_000001.png-page-0", + "value": 0.6000663042068481 + }, + { + "avg_weighted_label_matched_iou_50": 0.5758050680160522, + "avg_weighted_label_matched_iou_75": 0.5758050680160522, + "avg_weighted_label_matched_iou_90": 0.5758050680160522, + "avg_weighted_label_matched_iou_95": 0.5758050680160522, + "map_50": 0.45915842056274414, + "map_75": 0.45915842056274414, + "map_val": 0.45915842056274414, + "name": "doc_4c8f779a89fd348e058a875f9182d6b3070199531adaa187bc3a602b5f746df3_page_000001.png-page-0", + "value": 0.5758050680160522 + }, + { + "avg_weighted_label_matched_iou_50": 0.5566626787185669, + "avg_weighted_label_matched_iou_75": 0.32461249828338623, + "avg_weighted_label_matched_iou_90": 0.0, + "avg_weighted_label_matched_iou_95": 0.0, + "map_50": 0.6196369528770447, + "map_75": 0.08470846712589264, + "map_val": 0.2761276066303253, + "name": "doc_92ba5d5f6f6e627a3e9ddc00bcdd31b9c2c047e0461fff6c8fa08894a68b72da_page_000001.png-page-0", + "value": 0.5566626787185669 + }, + { + "avg_weighted_label_matched_iou_50": 0.5501596331596375, + "avg_weighted_label_matched_iou_75": 0.5136950612068176, + "avg_weighted_label_matched_iou_90": 0.08849658071994781, + "avg_weighted_label_matched_iou_95": 0.08849658071994781, + "map_50": 0.7788779139518738, + "map_75": 0.3894389569759369, + "map_val": 0.5729373097419739, + "name": "doc_4b667a2eb374a8566044a6c90f5641fb0aa4a7f2127350296f2a038b59c00208_page_000001.png-page-0", + "value": 0.5501596331596375 + }, + { + "avg_weighted_label_matched_iou_50": 0.5487409234046936, + "avg_weighted_label_matched_iou_75": 0.5487409234046936, + "avg_weighted_label_matched_iou_90": 0.5487409234046936, + "avg_weighted_label_matched_iou_95": 0.5487409234046936, + "map_50": 0.6666666865348816, + "map_75": 0.6666666865348816, + "map_val": 0.6666666865348816, + "name": "doc_747e0d37d78ca840e89dd820e032b90a350809453bdf96c7359204b4a324e933_page_000001.png-page-0", + "value": 0.5487409234046936 + }, + { + "avg_weighted_label_matched_iou_50": 0.5376645922660828, + "avg_weighted_label_matched_iou_75": 0.5376645922660828, + "avg_weighted_label_matched_iou_90": 0.4874265193939209, + "avg_weighted_label_matched_iou_95": 0.3385688066482544, + "map_50": 0.4455445408821106, + "map_75": 0.4455445408821106, + "map_val": 0.4164869785308838, + "name": "doc_99199a6ab7aff04e417c953ce0b87f56b1d9fc6a91d891b2c74720778ac3a95a_page_000001.png-page-0", + "value": 0.5376645922660828 + }, + { + "avg_weighted_label_matched_iou_50": 0.5117930173873901, + "avg_weighted_label_matched_iou_75": 0.5117930173873901, + "avg_weighted_label_matched_iou_90": 0.5117930173873901, + "avg_weighted_label_matched_iou_95": 0.5117930173873901, + "map_50": 0.5, + "map_75": 0.5, + "map_val": 0.5, + "name": "doc_0bd9bf7f193c5670d79bea2d81c123c7dc3094467ec9616245f81a148b063c77_page_000001.png-page-0", + "value": 0.5117930173873901 + }, + { + "avg_weighted_label_matched_iou_50": 0.5111064314842224, + "avg_weighted_label_matched_iou_75": 0.5111064314842224, + "avg_weighted_label_matched_iou_90": 0.5111064314842224, + "avg_weighted_label_matched_iou_95": 0.5111064314842224, + "map_50": 0.6677667498588562, + "map_75": 0.6677667498588562, + "map_val": 0.6677667498588562, + "name": "doc_4ba951e3da05a42523676d164548037c97a09984433bd37bf71b1f2a7acfd598_page_000001.png-page-0", + "value": 0.5111064314842224 + }, + { + "avg_weighted_label_matched_iou_50": 0.506821870803833, + "avg_weighted_label_matched_iou_75": 0.506821870803833, + "avg_weighted_label_matched_iou_90": 0.506821870803833, + "avg_weighted_label_matched_iou_95": 0.506821870803833, + "map_50": 0.3333333432674408, + "map_75": 0.3333333432674408, + "map_val": 0.3333333432674408, + "name": "doc_0ec2813813e8e7496d97aa4fa15dd414aa66cae2d6f62db818d30d9856ff0439_page_000001.png-page-0", + "value": 0.506821870803833 + }, + { + "avg_weighted_label_matched_iou_50": 0.4951573610305786, + "avg_weighted_label_matched_iou_75": 0.4951573610305786, + "avg_weighted_label_matched_iou_90": 0.49232831597328186, + "avg_weighted_label_matched_iou_95": 0.48283684253692627, + "map_50": 0.5, + "map_75": 0.5, + "map_val": 0.4563118815422058, + "name": "doc_9899a637992e35363176c7b7aca0633879da307327f5c434122856c639a6e9f1_page_000001.png-page-0", + "value": 0.4951573610305786 + }, + { + "avg_weighted_label_matched_iou_50": 0.46175795793533325, + "avg_weighted_label_matched_iou_75": 0.46175795793533325, + "avg_weighted_label_matched_iou_90": 0.46175795793533325, + "avg_weighted_label_matched_iou_95": 0.46175795793533325, + "map_50": 0.3125, + "map_75": 0.3125, + "map_val": 0.3125, + "name": "doc_0687b8e7d87922833102a8e792d999184f73fd8ccbc147bba73ab036a28ca776_page_000001.png-page-0", + "value": 0.46175795793533325 + }, + { + "avg_weighted_label_matched_iou_50": 0.4508671760559082, + "avg_weighted_label_matched_iou_75": 0.4508671760559082, + "avg_weighted_label_matched_iou_90": 0.4508671760559082, + "avg_weighted_label_matched_iou_95": 0.043584153056144714, + "map_50": 0.5, + "map_75": 0.5, + "map_val": 0.4831683039665222, + "name": "doc_8cfcbba6c953185e59b1edf5dfab0d36df31ef74a32548308deca21a8a96e615_page_000001.png-page-0", + "value": 0.4508671760559082 + }, + { + "avg_weighted_label_matched_iou_50": 0.4456102252006531, + "avg_weighted_label_matched_iou_75": 0.4665032625198364, + "avg_weighted_label_matched_iou_90": 0.29665011167526245, + "avg_weighted_label_matched_iou_95": 0.22371947765350342, + "map_50": 0.5, + "map_75": 0.395332396030426, + "map_val": 0.39995285868644714, + "name": "doc_54f70eba37eddc8a529fcb090d690185698ea98d3bd6a9a55ebace7ffb0042e8_page_000001.png-page-0", + "value": 0.4456102252006531 + }, + { + "avg_weighted_label_matched_iou_50": 0.40990638732910156, + "avg_weighted_label_matched_iou_75": 0.07446713000535965, + "avg_weighted_label_matched_iou_90": 0.07446713000535965, + "avg_weighted_label_matched_iou_95": 0.07446713000535965, + "map_50": 0.6666666865348816, + "map_75": 0.3333333432674408, + "map_val": 0.5, + "name": "doc_39a0d95abd6dc977dfc7986b57fbbd9a8331af6b5b603e023f41dfcbc67003c7_page_000001.png-page-0", + "value": 0.40990638732910156 + }, + { + "avg_weighted_label_matched_iou_50": 0.4036145806312561, + "avg_weighted_label_matched_iou_75": 0.4036145806312561, + "avg_weighted_label_matched_iou_90": 0.19418096542358398, + "avg_weighted_label_matched_iou_95": 0.009000754915177822, + "map_50": 0.6394389271736145, + "map_75": 0.6394389271736145, + "map_val": 0.5252475142478943, + "name": "doc_8bfb767df3506b4bd76b31e8823f59714c7abfc2a34ab11018e66fa2d9b5ed29_page_000001.png-page-0", + "value": 0.4036145806312561 + }, + { + "avg_weighted_label_matched_iou_50": 0.3754763901233673, + "avg_weighted_label_matched_iou_75": 0.3754763901233673, + "avg_weighted_label_matched_iou_90": 0.3754763901233673, + "avg_weighted_label_matched_iou_95": 0.3754763901233673, + "map_50": 0.4444444477558136, + "map_75": 0.4444444477558136, + "map_val": 0.4444444477558136, + "name": "doc_ce6853fb0ca94a72685ef90aa5ece8a180f95cd4a11386b2cf5fb8f29c35fe09_page_000001.png-page-0", + "value": 0.3754763901233673 + }, + { + "avg_weighted_label_matched_iou_50": 0.3746606111526489, + "avg_weighted_label_matched_iou_75": 0.3746606111526489, + "avg_weighted_label_matched_iou_90": 0.3746606111526489, + "avg_weighted_label_matched_iou_95": 0.06967957317829132, + "map_50": 0.3125, + "map_75": 0.3125, + "map_val": 0.3062500059604645, + "name": "doc_24ab9416df440de9d792a4034e177eb71933a6eed72d4cf1ead17d22a787702f_page_000001.png-page-0", + "value": 0.3746606111526489 + }, + { + "avg_weighted_label_matched_iou_50": 0.35681435465812683, + "avg_weighted_label_matched_iou_75": 0.35681435465812683, + "avg_weighted_label_matched_iou_90": 0.09661044925451279, + "avg_weighted_label_matched_iou_95": 0.008776781149208546, + "map_50": 0.7857142686843872, + "map_75": 0.7857142686843872, + "map_val": 0.603111743927002, + "name": "doc_63111d8ed6b32c2b3473840ace01896244527d3dc306d81410a3cc7d87a45226_page_000001.png-page-0", + "value": 0.35681435465812683 + }, + { + "avg_weighted_label_matched_iou_50": 0.35252872109413147, + "avg_weighted_label_matched_iou_75": 0.35252872109413147, + "avg_weighted_label_matched_iou_90": 0.35252872109413147, + "avg_weighted_label_matched_iou_95": 0.35252872109413147, + "map_50": 0.7103960514068604, + "map_75": 0.7103960514068604, + "map_val": 0.7103960514068604, + "name": "doc_39f32d6a01b8e434ad2fb16ff3896931153cdf4f85bca09194b71e1514711355_page_000001.png-page-0", + "value": 0.35252872109413147 + }, + { + "avg_weighted_label_matched_iou_50": 0.28757011890411377, + "avg_weighted_label_matched_iou_75": 0.3176450729370117, + "avg_weighted_label_matched_iou_90": 0.15807993710041046, + "avg_weighted_label_matched_iou_95": 0.0, + "map_50": 0.6683168411254883, + "map_75": 0.31683167815208435, + "map_val": 0.4118811786174774, + "name": "doc_a7914baa31ab36088e7d4199120e23630c800e99d8f96aa5a21091d7a67ff1cf_page_000001.png-page-0", + "value": 0.28757011890411377 + }, + { + "avg_weighted_label_matched_iou_50": 0.1603134274482727, + "avg_weighted_label_matched_iou_75": 0.11286555975675583, + "avg_weighted_label_matched_iou_90": 0.0653042271733284, + "avg_weighted_label_matched_iou_95": 0.0653042271733284, + "map_50": 1.0, + "map_75": 0.5, + "map_val": 0.6084158420562744, + "name": "doc_6916893bac49ba4fefa0a83eaf039b73c4adf00ed81ef97d4aad8f1a5cb90994_page_000001.png-page-0", + "value": 0.1603134274482727 + }, + { + "avg_weighted_label_matched_iou_50": 0.14368531107902527, + "avg_weighted_label_matched_iou_75": 0.14368531107902527, + "avg_weighted_label_matched_iou_90": 0.00778916385024786, + "avg_weighted_label_matched_iou_95": 0.0, + "map_50": 0.75, + "map_75": 0.75, + "map_val": 0.6313118934631348, + "name": "doc_c2b60a018b0503378c804fbf3f75b8ff956d30b3fcecdc30c37781b7dbe0c1c4_page_000001.png-page-0", + "value": 0.14368531107902527 + }, + { + "avg_weighted_label_matched_iou_50": 0.09470804035663605, + "avg_weighted_label_matched_iou_75": 0.09470804035663605, + "avg_weighted_label_matched_iou_90": 0.0011582998558878899, + "avg_weighted_label_matched_iou_95": 0.0, + "map_50": 0.75, + "map_75": 0.75, + "map_val": 0.625, + "name": "doc_c31d842c665085cb1379d441ef0b65869f29537b9252956d6105e6b4cc972311_page_000001.png-page-0", + "value": 0.09470804035663605 + }, + { + "avg_weighted_label_matched_iou_50": 0.06276185810565948, + "avg_weighted_label_matched_iou_75": 0.050391390919685364, + "avg_weighted_label_matched_iou_90": 0.0344405472278595, + "avg_weighted_label_matched_iou_95": 0.017872942611575127, + "map_50": 0.8341584205627441, + "map_75": 0.75, + "map_val": 0.7086633443832397, + "name": "doc_b9e5879b4dcd812e854cc06c6c3cb4f54231af28c919be63e27bef4c639551c9_page_000001.png-page-0", + "value": 0.06276185810565948 + }, + { + "avg_weighted_label_matched_iou_50": 0.02966148778796196, + "avg_weighted_label_matched_iou_75": 0.02966148778796196, + "avg_weighted_label_matched_iou_90": 0.03625364601612091, + "avg_weighted_label_matched_iou_95": 0.03625364601612091, + "map_50": 1.0, + "map_75": 1.0, + "map_val": 0.800000011920929, + "name": "doc_b2c11e30c537ecc6ac4d39ea937e512500882bc479ac6e5c3b380cf1627dc189_page_000001.png-page-0", + "value": 0.02966148778796196 + }, + { + "avg_weighted_label_matched_iou_50": 0.017453880980610847, + "avg_weighted_label_matched_iou_75": 0.0160814318805933, + "avg_weighted_label_matched_iou_90": 0.0160814318805933, + "avg_weighted_label_matched_iou_95": 0.0160814318805933, + "map_50": 0.44999998807907104, + "map_75": 0.25, + "map_val": 0.3499999940395355, + "name": "doc_eded7804127acce26e4b9e7138aa8c95d0d8886280a21148efb0ae9519941ede_page_000001.png-page-0", + "value": 0.017453880980610847 + }, + { + "avg_weighted_label_matched_iou_50": 0.016830099746584892, + "avg_weighted_label_matched_iou_75": 0.016830099746584892, + "avg_weighted_label_matched_iou_90": 0.016830099746584892, + "avg_weighted_label_matched_iou_95": 0.016830099746584892, + "map_50": 1.0, + "map_75": 1.0, + "map_val": 1.0, + "name": "doc_0592b0a1dff0c0b4a48b4504c035336bc9b30831fa2ac2ca8ef8b43a9b5df271_page_000001.png-page-0", + "value": 0.016830099746584892 + }, + { + "avg_weighted_label_matched_iou_50": 0.0059301480650901794, + "avg_weighted_label_matched_iou_75": 0.0059301480650901794, + "avg_weighted_label_matched_iou_90": 0.0059301480650901794, + "avg_weighted_label_matched_iou_95": 0.0059301480650901794, + "map_50": 1.0, + "map_75": 1.0, + "map_val": 1.0, + "name": "doc_3e5b086e7392fbe139a3c42dbcf1442d95c4d0d97a5671e15cbeee2191e59747_page_000001.png-page-0", + "value": 0.0059301480650901794 + } + ], + "image_mAP_stats": { + "bins": [ + 0.0, + 0.05, + 0.1, + 0.15000000000000002, + 0.2, + 0.25, + 0.30000000000000004, + 0.35000000000000003, + 0.4, + 0.45, + 0.5, + 0.55, + 0.6000000000000001, + 0.65, + 0.7000000000000001, + 0.75, + 0.8, + 0.8500000000000001, + 0.9, + 0.9500000000000001, + 1.0 + ], + "hist": [ + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 1.0, + 5.0, + 3.0, + 13.0, + 9.0, + 23.0, + 14.0, + 18.0, + 20.0, + 8.0, + 14.0, + 11.0, + 14.0, + 8.0, + 38.0 + ], + "mean": 0.7029211287945509, + "median": 0.6666666865348816, + "std": 0.20529796811035708, + "total": 200 + }, + "intersecting_labels": [ + "text", + "section_header", + "footnote", + "picture", + "caption", + "document_index", + "formula", + "list_item", + "table" + ], + "mAP": 0.6172709465026855, + "pred_labels": { + "caption": 29, + "document_index": 8, + "footnote": 81, + "formula": 38, + "list_item": 229, + "page_footer": 2, + "picture": 187, + "section_header": 213, + "table": 55, + "text": 814 + }, + "true_labels": { + "caption": 131, + "document_index": 8, + "footnote": 123, + "formula": 56, + "list_item": 246, + "picture": 168, + "section_header": 192, + "table": 55, + "text": 707, + "title": 1 + } +} \ No newline at end of file diff --git a/docs/evaluations/Docling-DPBench/evaluation_DPBench_layout_mAP_0.5_0.95.png b/docs/evaluations/Docling-DPBench/evaluation_DPBench_layout_mAP_0.5_0.95.png new file mode 100644 index 0000000000000000000000000000000000000000..cc15e7a4d6d32b0625c959aaadba77b097b7fef0 GIT binary patch literal 20735 zcmdVC2UJzrwk^6bCq&DD38WNM3?NBzFd+y65+$oh63IEERb~W5$;kklBp^w0uuy^| zVG|^y#0>&Ul>Ga&&bjB__wM`exBu^Nuia|Rh88=lHRqUP^wE1C>%Nl0ImUIn)=?-F zMp>CN$`r~nFA9aBYt1VBAA#Do9{dooJA27a#nQ;m@rsQh<-!#^s~eVfH%zaxIT+g5 znp#@$aq%4I;yKD@VrOS%E5glf{%^0~va~VgwzXXB#iy*blF_uKP?)ZeKMb*wF{TuX zl9}w8ld4W(!|l$ORC||}el*1%_gZuK-sP`%<^l}QkC-LJ`BRPh{IVxfDs^7*@*OeH z{182xZJ{Si5x_?`G9AS)QH~1_eG|%NUm1M)(ee3|$?utri~@ zwl(VepgEGV)4&^-ae3!$9bNpSjBRJwM4=R2UE_s+Y(Gs|L7}|av5bvE@%pfd!a$)M zy+c`pm-DQU!tWpcPFY5wT-xy8AP_vG{SSyStIa@W{u=h9{!n7Q5N8TMB`&qlZJ6kguOkd@B>QF`hENN z8>Th)^!7#Qh*OR}pV~ScuNvp4=l0{QZoW%{X#qaU@X~m|F>T(-OQ9#s>l{1XV}5j(8qWyl zJZw%gPLO`IYiw*x+5fQG#7OR_(e-SG<*Tk`Et;PQU7R0RjF}O%?08MR5c;vS^xV65 z?{@KA{qt*gw{17)mtCGq3z>H|?0kuDjWelzdA%Xt-(#jXP@X%|&+lb0pK-iizRTFu zl)99ZmsX}lVtcNmXrJtYa?QY7QyR6+>3U0^a~f80W`m@@?dhvmUzYgpP3ATHSk~IA z{pHJ-*~Ph0zKm~m(Q*w58r*8}sj{`BDF{CJI7Q}Nha*hVwYY^UN1 z4+{$mSIukAD0q8&tKQwn(la!a;Kw01_~I_>`7IM7b_35p`X4d-_>41qY+fh&Uc!2- zbL+PsRkE^rQynfEQzJ1a?<+PYlQZsrOuI4VWVpDi`d;^mC#x8DMJ->w(PXebF4!Z_ zIKo+4B}Sp4>)js>&rdEtru|mleRkqWm66J5M?POe>3Z(G=h$_TjBVU{Vl)96WjD7x zt&Ho@oBY=B~`>2INF+-9B@?g<=wp=mmJ%y@4mg! z{BrG<1BVVDKEK!P`^k`*bKW~%;4*|V)RJ_BRWb^uTFkD~s28aHj)J3wO9gircT-2& zbIs~wFSfk7@`p5D{g9cvgMTX%i-@Xu>s$Nld?hBI-&~1TdU4AlyLbD!xg_1(dg_JU z4Y3zbTNcbv6~csGH0gWlu4ZDb&+POFJyKJ(tzd3MR3os9orB|SxTs?zK3~A$+ohGO zR$VXg-g&(_HQM3Z=NH(nwsc=|-RqgTR0};8CO#YZa{m7NNm0jp@L?0QL?*^UzQ4xj7c>s@?x0%mR7@W(<7r6r+jw2uL=tsp6)QL zC1Bn%ai?PAPTn(yAD><9EO00Hzc4$L>@u1^oqqMhx`nX{Zp5TRrKoM6{P5?L(>R4n zH*VbUsh2P)*+4E9AKt9B*i-&Agx}Qfuxjjcm&w6inG2fMJ!PB47k_qR9UtM$a7ao@ zCTeAzUz(}$OgF69UKJ_XX}VbHaWF#2`V=lWu&}UD&tt|A_b@csq~DZWxEPO>EvM0F zJfojpoE9*u+;JgAXK z`9fI5*KvPx-q?dD*WQ}9_L)5N_U6C6p83S)&6@>v99rL+OZx>Mzj8}uZlLznqbei5 zT*py|^hDjY0||cGB_3*RxwTZMrPtIP$H2vFi_} zke6p4h7>N&9;tA*`26Ob3@>;7grACQsSnEq8JT;+o{M>}hKrdb`TZ*6)e=&tSLhs_ z6t|rixTq26_d-!(hP}$Qr!tt2ik#MWb2V$(#F$(lSJjxoxwB_?hE^G^Vqy*JHRK!0 zXws_~Gh-{44-?|i_RX{(Qr};mhp_f}i{PO`^)!;Ien@P@E?7DZm)a2ldEf}L8*^$OTp5jE53SZaZbV4es;oCCClmxpX=1O z8{&88?@@j(MO%c;O^*sPN&I~NV|<+N=FOXZ&Bl=eoSd8o4j=w#H{8;&^@w`qGcMhh zyv5le`qb0evsvb?C&tSUMh2a_y)L-B)Hf!WJD+0W%_HtIaon#Gn;0CIgKe;25AO6_ zYL9cTeR)>euldcDV#dJ26O@te4gKnyUZ2eY@SNS+K)te|!Iv z*>~d;G)2*tBt|c5YwH8J6gm6qV7^irUeEa-cU4>mb8y!GaGw8Bdg1cr=Rc;W1yA4E zz;plp{l2zr+dd>i-VYC%1<4<*0J#$dclLt^Lo+*ED(YWr(moG6Y}vZ?8cKoR0abTY z04k2-0e1HHPA)DX=lu@{GwV71UNx)OuHdn>;7CIm?H(RBM^-h^bME!$uNnHBVsL<$ zcZ^!I6lz((<5wxz>DFCe;5=@$zuXNMB{wkLBFiOCQ^{ZBQC{0M39H4 z_UG(u$jLue8O~UH^HAMp#;CUa)kecD8B~vj*^-Kiiuxqo)*l>mGyRdFEg5FJbK5N} z3Kty4E4T}z==*tj>+D85+MgY|_!n+NrO?AY4QGshd||59lB$-VPClT`rs9~@=tZNRYpFElSvVtmi^g?ZR}Y<@rFa;WfX-Vg0VUxw%#LRs>pR zxj8vynSOrrf~POiTtq@*k?I?KcyjMMQp{a~hw{g(MX5$rAtkM?ty{VE+Uk?_JCEvQ zecCbZ8QIX(sjTl?5y&0-WdFI4q9Up8p&_#zJNi|hy^;|Y6jVC9*|C~3toem6f87pW ze8N)_6sYgfEW>;8;zdmlA4G_>A99fDo$;PCl%x6uZXx#e=8<=q_MRZ|$Z`JJ@s5s; z17c!r_+1;1_~iK1toHRC*r&Z0d|AciO-y2cOib__Rs;y_Kj*{G%&lh^zdxViH1$n$ z%g&vqzy0=G>SSBN?BEe>?qq|8&)e5-2Uo3J89X;yD0!g}OO70#E=%)hHO~x2)iFwF zoWEyFNu~h=L`OxDqPmWWsp%niK`c-NJt=pj?pS8qOAS5evsl^2ARdE>ZeK~NrU!0r z-TL(^I1%;Ha?CS7e_oq)6Qf7!wj=vhPh5F~8WoRJ8h`QC9TnH&JB%zezE}8m;HszC z%N&Q{cz{5o93y(V@-{so}M1TLvJlRBjU6Og!1yD_;c$^w9|bh7qd39ip3JV281$;49b7yuNC(6 zy0hiw0RyOP;%iQ-e<~MN~5Vv z4mMPbb?XeFf(_DQd=XbKIaR0w$PBJt4bW?Vpn`!_cwZlqHeru-D^IDL& zbR9dKV*rRJ>s1u#j|4lEkjCt5#HRdSYfrUFlxW*Vye#(dYgB*u}@n_J^D>4?zlwiBpOBSX_J>5vVYoa=kwG1(v@( zok}7SL4ydiy{@bFJX&`$CKI4 z^2fiQ-eazjtS3ND`H9df`EZejT91)7VUpNqr$@mNyxXQnTkLLHJQ$ucEoiTJHXe6IH7#5$G%)bqDPm_5lFx_uL!Aan=rNI#m*+>D zM0K4WQK*QicI7L+%Or@5qu{88?o1w#-?F7NqhN8?Y=3!W5KnQOd$}KnwBHP}C+(>1 z(!z9TS6A0jafSGL@m*MTKryY1k22?Nh5At`1pUG`zRNJBmDI3LjOxR$4VZkzI^>;+e0hGwN5iD*nTB%TAqR+JfIjfJeu@#we z|422gz)}OB#sp8#)`U-NJ*HJpw^oiGS2NnCv_EjUlf(gZai*E}dV!lQ+*UeEX|>;7@dyS|JJA zg$q;KA4*C@7DfG|RFQOoHD5!Bm6s zWFCVO6M9{=h{Lxaq#PCT@k+kK%2D@dsgHIGHX28|Q%S?5o9ob^q9`7X4iX)&;q_6O z8#i8`e|G4}mkGchVMdKjtgI16yJgx>B(<+Xceu~f)3e>aN%z71`_^qJ3*4v@1laS8 zX1I!Q>tsC-3fduH)+mn(U0>in7dH4~a#9Tmm1SC7T)YA3lE1YRTYAkqn?7D>NB5(Z zS2ckt$f2LG%|}yGjtqu-lphyTE@?fZzPDcsAEt_9vuC=vxR^s0p{Sm1(_5Dwlazq& zKu&W0`yVb--{_Y{d3bmpJbI)yH#Kbkz3XJDTICV-*Pp^2JLsQn*<_I{>;7PpaH7qw zW9IsHsT61v?OWj<*VDAz-!hSN6k4w#YQhEsqSkFYT(;wBT!QdYRsX5MZ7nG&Dd1Ov zS=}ctPSR=T%o<-`L#u)$Aj=))&P7WoyQIVBkO;6u z6)ohE9%@X?H7#74vYm1P(5pj{5pbCpkk@_uR9pS^r5EU&MRebP1yAt%={p0+?w>mH zUCEJ-G(>-frM}o*^2iWNrnlH1>8V!|bVHXuj~3SyC!h47s!?T2x-}uqO@~E0HRDEW zgMxyNHihopOi8vK4!p}gGdBmod8Eq11)GO@P#tIf?$*k99D#2RxhUjOr|)j$JbF|f z8PJER#0&-cBo0!(da_<3LglcCh!%ifP2KMdmX?;KUub5!1#Zne;t41`@!!8&CFIa1pPqoe0|62l;W{Erm?>00LAQDImiBaho4eLp?k^keQzC?+KUm z?4Vntc!e6xzy6Kmd{pxyZv{vmjU*je#}>fdwDhX?rKM)gsfMJWha}`#zP)hGxK-ZY zze&c=KoVH{1wTNE12n5oJFqn+jDHO-8>Pndjqev({tz+!nd zBe^KY_9#{fabUp7d`MPmN@=?f8Jn&Be$HX#B3 zh=$JHEA7RKhj=zr+9USE;!czcpLze`10CVth*hF3)Llw9d7|e+=ZIm90dzIJLo17a z{CE=%wR*{pm;%Lc5f0hIDpdKmvy6<44T}qNeBz!cFNT?|EWWOZm5s^?>c>c9(~|3$ zOpqZm=11%UYHU>Z@?IkW{cM}Fh+@;~hHQ{_MT~`)_+0X7W`z#BWn^V9Dk&-DT+#RI z-b1)dZPdKSk&%&lPG45uzJ0qKoMqws*h5fwkw$yUDsOmdZvO4Jmpwh#!i8<6Po6yK zGj&8G>ClN2C*)cy(QU+mZldOv9t0id$J=%wHP`&Srt71qC2ftkAqUjOU%+m>@BJq! z@NJ*4h;{&qCf`1%tXsEE8OUA$HGoCJO{l`&SWQMo=56Q0I2x){xRDmZ5rpcq$B(<# zG{v1TZ;3*mxGtwzGO#a3+-$H9X`q}$N^JxazyZPMZa&H{hIE8Q# zy57AM@F1W@IaLe}Hs3hW0EWZb6O7V%S=n+`B?+6JdpHvoIhk*8v9rL;mM!|~*ez3_ z`0I61<&4v7UTbG6ki3~=7e~84cFvcz>PJZ3)0_)8It$wAh5Z0jM>se*uxY@+h`rW--dLc6ocr?b9N7ZPNhmdtB znHCuk_5*-}FvL6Z#BoCyN!^DGt=#kkzkY~M#RVl_XcRIqFb;lOe0BU z#0>S~BA7%YdM=M-_a>{2nr|-OL?(>7QhX;4E3A--a$nZeBv>qddPy1_e$SULFLv@7 z3AK3h;6w%x!d1Yc?Q^ZiU_B=`poaeOuL+M9BkXHSioso6_2XzQe8&qAL(O!?49_Kx zh3~~oIba*Y``R-tBVWFJY4N!;S}u_EmAC)=Qz%>`Sg(qp7!7^*Xb@!4*<(Lk&=u8X z+xGJ=tmg=BNHbPMe4ByfFr^aNfM?Kiup`23dPFDN<^x@z(kC`HmcO5g8Uq62ux4r$ z;+9@7fgS;!uaGX!&MjMBq35H2{qWFqmOy(_kkDE*S!YaT0E#xeHP=3N=}i%R-FMOu z{rK_2^zHq;OjsV(jEVI4=rD}V zv?bG0KgV7_8Ounzp8zi1L6e8~@7IE~CtZGw_-^nN3TYat016@=^I0@zXL)&VvTcot z+A+?)h-Fh`#Op2TrXj`6n)VA_r4bPk2gdQDan?>Rk6qpF1Zi5ejtY-_P(8(Mew%lp%yP zXuX(WUXa@$L9Vv*8jCZvOjGs z{Bypm<1zgLol@Vuj_HY-Y0_v?Wv0hXQD7AMS()v=F;p@fRE(%cexijrg?+ThOu+JM zrCz(=;oQ&S?Bvw**&y(kwmJxydb8wwYE%@vfXOE*eUF)Fa_G>Hh_GlqxOeZ<{LHvL zqP5hDg&<9$iinP6onBmAob3(N=Qpn2bNkMnL8uVb%Et)NJppzr9<9VKJ|k~2F|i3i zh;yd;Ghmi;rO4%d_hlAxogH)e;pVNmmp% zH~=~2*Zq3TnNJSXp2p>?-~WxHzR=V2(GH$dCqge^VXfaXO-B=8)0nL9N%O$B3OJ8l zW85pDGu`P~h*(M{6&wJ|g&rNKntDyPG!NJ%0`(vYZ6j6q;@ne3iCSEXrPE(aB*mcH zN&xiT#e3}@Q9wYio<`=A8)v#xWh9q&q2~^r7L$NxGF~Od2gLPj;0O8yoi>XVwEOyg zZK+~7u2TV5mO^sR!3d{Q__T)Ad)=NRS(?>Ehzq)T=Z+R=qakj{pYBLNksbmsPqG5o zhT!4IcXBMNsGduUg87@Ga0YU~0}6nv9&XQV`XR>#{>XIWE2wMIeqkQ1Li_feiu9O` z>khn%BM5MQ6wyz|zAB76#WDM|n`j3KT1KqexF;5ApyN9;C!r>~x-JWt)}2A}PzOob z;W{dg2B-Oluo9S2*`@ifr?FqX$X8stIk6BLT)J+q=7(?sh^I#)2v=%1@acu!-Xr-n zFDCxHdv^#bp2nFof1)RSMYJ?PwOR0U{H{|sVw`ENdmRQoojm8Wrxq<_x^=g-d0SR3 zPa;Ap2eo$3G>DvYrJD>OsH&Yhb*uZYP*5X$ABJz(Ze(o8c#PkEIT@4t$RQETl)G;vLtA)=|K`#ptxBk^Rrem6^?@e9Mzq0g)8}*9*z`iz;)Dy3E zMEuA5Ef0Kso5*n!u^%+)D!QrQYu#75bGSL}97uqgdfOwL#`>;y&vQ@OOr#mth-6HU zc6OrS=C7J>+Sn5&a_o4t!<9)1(c;COiF5MY6we?&}ub( zHl=#r_~&g9rXN(3_1c}Bor8hNgK)G$&~Q*Sr$92Ez`5Y(kPF}^MEmc*|4y~+ETBs$ zNCOjv7(K7?_SnXe&{~4Hy^PakiF0)!B$aHe!=yxv1}M!D^mo z&sCT8vW^?a83lj#BCAY2{|)uX_`Ku+`$J0;w*?mv8J8`2LrBCSll^IVo?&8&-GnDmlhiSzSbGho8c z6GKfkD^{=G4Q0%>;#g)7xofwnW@CkMY(*(ifGU=s1a~6T@zf!=z0IN3+z6o1R4>ze zP|jVVX`{BE|CP($Jbh5?ctHvBGB7Zdl$QQ_W4r!5Cn3bmIqg~6QQyeI5@CoGmz`lE zYB(H^px54i^r-4h1a!=4>S=KDHJ`*G2nv$VvB(TX2f>H-fd7xA!>tx~&B+)zUK2E8Z|xUtyF^Yx{aQ&@#ehtZBioQbTQJw3Nj z*av|D%BG$dT9vpSux!Wr6LFYm+)xA!gjR0~fT+ZW1gwx;FdLsBcatRJc*oHWwXR8uPqM2oME*@z5Zz&MnGM?x@A1 zwP-EEJ=pe8C_Va6V%9M-+Rp}GO!%F$xIXT#`@;M@@!V`Zt}fwT5wf~~cx}A@E4?-k zvT|@|N^j1dJGm<^0YKNRlB49xVM8hyyV zqHh^-|CGH2qoiA4cag!-H{hIyg@xz znAS(*UKJ*+qW$)U^$IDpJ1bYO&V4C{6$rF$JzdmvUO}M|0NVfY*S*VXIsCrZQ{{0z_0|!J#NH#)pl2C0@UI^Tuv2l-KY9aMCl4-sK3@b(sWOt|^I(qIM{HY2 zu`OxFz7cL?vI3SJI%rnx?dkZ0?2qQlyWlQ?PAZ8ibshkhY&zj%b4POI0JEFWQ9^j+ z=U`_)#Luq^uTm2l|DnND95I`4d_SyG#zM#NtD&akn&Ru^vY0Xp)?h;@rknq}o8JEK z4Sd$E6d|;F;Kww;=@H`@Sl$W11Q9BVp(enM25Od@yL$uLmT(F8mfPD%P%vgPdZFkv zK&;3xDBWn@S)fZeVKlrG(0b+k$S%1rOg0f1k0hM@^q}G(dU;#x2C`$xa9fpuFtD@E zS%SJ{igVhMYWR>yVAu-CNM{M&V>g9?gHZ9r0VO(`N883hzIttD`*Np5V5ehxd6&Rw zf&6(w+(+nxGR#{Wc@#l_{Bj0C?%O%_G$<%GD@!P5H-#c~vb&MtJDg4aI8&fpiBXEX z;O8kqZQI$Ui;0V?p{&Nt;5a1?3=C}ebq;{8Su#I;`lRDQoS(uqiOVko*rb~^ahbR0 zs1eZ(l2|gDJfqw~gfXE*twnJzH>$6vP#z1mP3@7A9h;a?MQn4(qT)0GfWC&Zq753T z4rB6Fw{3Eq!Af3i^<9q!{C$I&EY8KN-4 zk7xDwUe@6Bfk?L$Y6ne7i&1PsPc$C8566H0opL|1hkE@gV#5C&4Ws}2vjeV@bl>1_ zRKQM-WcRZoBX^ocfR^}ZlqTGcAnSYi0EuSo4iXJk*zD)`FC^HB)d)RX9Fp^S85ye( zOOzMjsAt_%g9*twu~)Hfc^BU|tn4AqQ<3&av~$4C={Ri2er>fQ-Q9#p%j05#S=X@5 zfvIb2zsz8m<8Q$cr|p|d`${q`GU zKO+-UAUi~WL+x}FLJ=6sx*cF**ptkRvYeSc%qnHzO@h@O`Y?>9;ecOGG(+%Q9J2U2 zS;w5eYe_jTS>iA(F)ocH5F!>q6QB+Zn8=cVMju1G9mGw5A_1qR7YskIh@S!QT?6Mt zS4SKvV&otBPN|u+94h4!kET!;P5&`tvo4m?!X_tF;H#`e@iL;@k_rbP z5sj9f@F!6CZS6JjPK*7WD~mv$(utb{6srJe9#Rx9gq~2tnook%8N_K#;@S*|DpiqL zxKbxdW^+4${27NimKsajYCKt*3iCOLbmV^8N)Tq>tP8U;bE~u(Li&sm2Q4z@1 zcYaB)L!zaS+i%mFB91X#iraXVpn&6u4uLbc-a6>*QHUsL!=66s8m}*1BtFpF>$d4= zAEIP#2P26OIovf;JGN#`Rc*o3tG-7ZW_ClNgCE`MJ{HfoIxLRps>@exDEq$25=Yah zCL+AlpW)3uUoWNRKaS+)Y@n1~B#SQQ^qo2mH(NZY7iT4Go!4BsqXt7wxTpq9lCrb{ z7%YeZbmcZ|Kvp0=JVrkFWl>QP|1gLN;_1;OMWvzHZGLtCMH(n-LCd7m=a&OM8F9Y4r~adyUFQC|{Sa_Ycdj5?0xS3$ZuQfU5-vX3xAY9i>Ln1F^+qybvgd z=x{(z7tsOh>h8iNYU4G>)u1G(Wn5R&2*mw#IbiYS0ZyTAkkUdF9pV9`dH(#enpq7s zuZxvz4=#q^vO|mr;%L|cs*G&yWl6Y^>xXx6h^8p1hSH&rUaHoB3)vMuW(sQU)wk^~eg$j-&4v#>Jgs_rB!ipzZi zNjC?ESzTRRi&yJMM+PgPf+Q|ndr@L8sx#SKd_@tlRSdT26xw%=q zc*ia7&S3^I2z-Q;i;X=5h&+g*rVLJF;^UFzLp(eep*Y*v-^V4b)GBCQ{`377#Q`7D zkxw}MJ;TGpCWZL+3@i<^`af5r4Tef0={UQ%h-7F$riIVlm^v2>WDL8(^Q#{oa2`4& zGu>v32C8$EzXl0U05;y0D^@@m&>-*=RHAJU$;dq#2tiP|hhSE}Foy%9ioRNYI?ZFj zg;1GjCu#u@k_om!&kVHV>~y`wDXVIp&em0%wU7noORCn zPl|~Sl-?#J1p>4)EZVi|BsJKY-~}gN4!S1TRuP-Zwzu5TmD1_d&4}oeCG7+5ssa5- z>6cFA2#(f;2+)LnglLtm^I;KvhN%T8CL(>r5QNqNEdMOMQ+zG9#O)}y1WujTggQX* z5N?#PYtK*qF#)z%M0cy-HpJjAVVfvakC!+Wa(*vgz3N2|f&?R4rJZAEdiLD8m{vt_ z^{7uQ)7@Xcrm8Vfww{5@5lNu@hsW~e%Ygy2_lz=2l^}kfMS&TK1<@tk*!lfS@g&+n zQ&3M!@sTjNnZV>Fth$Y|br0SiubzL0OFt*5+uhZ9&Sa@T<`moGyZD6pfmgm3LizfI z9-4!rDW$A5kP4F^JLAza`b;4W!&7sX3{BAVihzQS2nh+b7wp@&kN}6L zajcX>_pV;OYBy7Nc4eEm7Au4Bbavps>!UZ7okl!JfNoTne(L9E2Rk-`4C5IGX$+rs zyn^XWN8jzoZ9+*A!TUn;zd)h=Xos#}I_fskPTa*;()20Q%vPwwX+K_5io~5({P~ep zr@6t9@-q;3BZgEP@)jP*LGX#hN5j`%gLNShKxP<#W^BWn`bS5(!HK}OU3T^2N8 z0MXgOMIFJpgZ$Oxy_3%-)s6&3jMNG#dgS`kU{HeTGmm%#!%P1KD3>mnlJj}KV7-^V z`=me3-M|~wAsQ!B5k7nCi)#4o2q}H8BiW_&0?c2)kX4^!ZSATE08DQB51*7uAdGbpYd<_{{07LJOFotbU)^Fn;q31wV zLr~ZqD~;2db<~jArEZd&2Z@JM%Yz!B4+HJviJGoYm=IA#TH!~|W7fd9l+qh=EXnD) z0C(EvMfSKe_kVK%q{&`K=!ey~wjIpOF<_3Vrjn4@fZw}rJs{U2hik!YCKD^g-G0(Zjd2jWkLnE;n4T7c404kXB}L!geL&Dl%4H2?joFGlb&nUhX$mFt!@af z*`Gg6Fty<_oY6#}nyfP6&bJ>M{Q8l?w)Br0?=!u7=g#16_hnY$D0nIaLLj{>qYC#9 z4<9qwaJrCE#CQggrZu@M$Bq6nL5|^ZO)u_Z_wEER6W4(?py`6YdrrhrSeVES>#W^C z*?A9&UauVUHmhk9=M{hUnREvF+-RC%TyK-ENJ)A+x~vPb3@7Ma62?F{K2uO4Ofk!X z_91KS-?h8nBvLk|)HYp`lH`v!0KCMYSBb+(R2YU}UYlZ2s)O^4LSBYK#LmuMh68kA z4ul~`Zu`i;))gi~8Zq`|S*A5(QtKrYRH)P$HdhR>@*}@@vMlp}iMauzF{HnRqlG9L zq`!rFyjX|1DIHx_5(I&xQ9uCsL?vGhXjA+q^I7r~#l^|V1eq|23%iNZmOwcq3*D~)iTR(kPvfrFuWA>< zXY4~n*xAXUm>I?{IO=I<{js}Qcipy7eD?^y#336X@Ejlz?2J{H7foz_`*D11VW8!* z%a{hUgZfOIC3TXL3scpiqp?tBxRN&D75P1(wv`Y2iZDO?rwa3!>_@>4Nt(fUUFrR zaAlaB7_}=zhTw`vUsaUf8%hyE9?}TJhhAPg)3-+of9YXd#)h5ykF$mryD3C1A~tGJ zUl^~V!Dd3|cLz52Nm;F8i3PQ~kb-wRmzgiW3^j=mR2L#8#EYFt*~i3!u*DKA#$#_z z;YCelg2hEW=_pe1XqN@UN-Z);y7oQ13^%tIt<+^B;+z1uBNJ)FaR$R+ht(ol-wz)? z2)E-S*6pfTRYbhkuY4uyK)e$uVFKts2yGbyDSSFcXnn?}rybkis2YEB6?YW&tpTeQ~xL0uB9fsKpmNXSIxu<&ojT)#In>OEv5BGz0_F7h&3ieC`)(ZFy2I0xx zS#6Nn3Jt5@j+}RsHC{`d01R7k;@}g~2*lMtPH3CjWQ7?sT1VcibH3LhZjpWvX*-TM z(&1ZOw{c@5IwV*Sgis$fk%Aqs{Gwz?8xIF0u;G!?g?ig*!fJq2yN-w?CW*zxIZM(X z;#|`}f}^hJssUa}MOd{`K4a)(rvYzR0TQLyYk(xSrLJFl`S9=QQ14dOWvjgAxp;Yb z^{4wc!oje0%a*6z3#(SGiie~n+>xw$+)Vl%MoqoM0`_<+q&C;1glGhQZNy7e6I~cV zJM)yzO^r>}F1kMvW{q8Uwdzn1xbnxP%D;7U7;OEON_yIo*XoL->h=pBElXKkm-9DW z%~RTBtNhLS1DEG%I1`0gssDU$*FIK;7pbc?s#6yIuKM<0(jQ@uX|mNjbWh;bN;uJs>cU+v~34XxQxw9r?l%W<#8bx@E(y>nJc~KZ&TK=)f~?{M(JVthkk(M^bw1Hs#9Gbg2q29I=6b#1sk_iCB^nMJ?au?3mndb0F6MNOE zQ>Oru6M^&wp)w_LZpZF3Wt&svWn@0VCLa%=8zZ{VprPLY0`C>nSyKc&QD=!j4?APO zbu?DSxx$=#869Vbm}E!z4ZM+Csj$YXqc4BTPDTX|>*k~v%#X=Ujb8QV1IgQDH{eS8Ag{V|dimyayxV@j8mmsbUH z?Do*n(HYr_>iqcO(d^a7bE)#d#8z7hBG`B}g?z{QfDK-d=KP|7=tMI>leF+06aN=Y zA++29=k!*oAHy49esZ*s3ig;o+G)wLPauHli;Y9BE_hW1L9$mR<~0nMC2D$c`tF`- zjMKz}`-!_&?nf9qSiIxNcmwL!MZhlsxO>QXBTdB61DvU{=y+0D|iL$*LNs8gpxog+1-5l6L8C#F=UUo!yG3;7e91?dD*GkH|t-t;s2ag`F z^xs-g^&)Rg^!`?RkK$W7W-LN}oyiw7=?4)**9OQ#pGCTjD4ssJwY3Qe=4uh^0@h?_Jis2glp%eh>E3l;UFU zG2~0^IMGNO*P&4pXL5oZie#ApHVAH12Yd9L6zA$X1UH59`riLXRW)V8l3}^bk-sx9 zrAz|%CaY{SFI;Y<0=KX(I1si#$VM`L|I5@%k($B_gnq19<1e0N<1-b!)m7}=+;Ymi z=>Km8;~&p(AVQ~<5s*8PfHh_}{!;?(Y>O90fi9g}LAkV^VbhG>e?r#ro;!WnqG<%Q zOUHmt`3aWuI_UIWLRQboAQw(dIi7F;hSJ)e^pV*msTvH#4=w8OO$q*`SBWrbTyK4w z2#O-cF7TiAVR2@>pF&$Mjw3yP}WL8~O5GQOoOYO^}i%yOU!{%RFXsj~F zSJGorxblA0rHST9F=yq>Hk*ehFjM$TCN6!nJBDZmD4M1L&X{U|K_M#d=oa8)Kg<#* zpyf&8D&{zvyZX53zmW~F-9mZO2g&%j%BZ5Y^ufc2Dj3|vh$xq&WZ|CS-Ai{T!vrtY zIREkYkNtv8X*|+Uq?N#6*`#;gdY{UG2`nW@v5lzT;Vp`h5{bmL8<_IzyegQ3w5dt_ zjYAfd&8?`1q0zSEvUoq^-#;F;Y1Kf2749%TdA0eW6j9Y+FN4pqq&5IO7MLR;mfuj& ztONtvmmiV^N5>UspiGqLG&=~p**w%H4U;(bMCFUjLFjMC_sWzM7XA_9K3bGhwF#A=6{kgXTf z%3EfO^mYO^SD1E%`#=>6g48)drnxbgrQo;-Fc1vdA{4D{!#5ZS#pb@Hm4IcP$J~vx zj`O>x4fgFI3;5h;?b2-f703|pstr3XLolpDZ!wlNc8eonNVuW))wwE6;$8*5F(Q*- zf*-bD*3_*bQ{a|Htl+>jPdV^&^OOVJe$UmjSC?2YN9C zW~@Y$A6)9sd=%3`$BE5N6CU*^*I;ybxVzdDPWbNbZUdYE!>N&WGct)q2JzD%IplGE zuG&P=eU4Buf{^Aj2U{d?8!zd%=+&LDI@7Xkjbq*d2lBzJsFO%zxn{4cKZ>0s)+r-c z6tmS{YL`{J6u3;ro3>@8%)n7-hHgiZQu!yY7d_9fv3960J!Mh(Qw_?Vh{$2>}2NA{Je*vvQi+cio`0qYH9uNt=+bdwP2K z3r{G6Kt<=r-EN4SMhln318ABC?N&EyHZ|>*^vJ_fKJ7cbdg-9(kTbAlh`8%pe4qd7 zZ8>+pz3zu{H~?kk6S(DAd<_htr)Mz=_zD#D1w6ii`y-cCYFfnH1pO5mK{IbrE!hFn zxf*6IX%Gp*nK`tjr8{pUt0Khxpvvj0YQB5VD#f-O*V_Pock%znD=Tk>wH!}ikV0aF6Fi^LC?<`sHIuli5=q~V_9T@ES)@G3!D|b8 zDf`E<1XZpNFj+Iq@vhOS*T%mRLTGXyoa(LU1MjZa-SO68DwnB_T{HVZb6xcK7KH?#7!JKdRQ zXJ%5Sxp#j)JRSrGC2W|@~0R6#YW#Ee`OtVF~= zcmdfNPaL?*>QW|#NpPPTLGzYtfV@}rJQwUqZb8{DJAOR)S%bZ}{h(?$oo*^I_l?gM zY*gtm(o~Ymx|Jk1PG+qWvQ5p2c=ht7JcOD`sK!Q+QY-_w^(`QRS7NAqAU;Nk?V7!~ zVwg}ZdFIAEjxit1#yl`)h7->z#ux1xR1R$dm3rA>q|KE4VGkx;*|)ZA+8$Ad${hxk z7`ZfLbs6}*bUv=Q$eTs9@r<-|2&w_!hRQ-I6b4X>P0x;KRFj8YK*)W(`tBn&H>*eTv@59N$71Y*`u?`kr-XieK|Gfg8oBFkTa4dCVx-h z5lAKQdy?m81nRkz>*Tu#_$F{UrKU!J^{<3g+48Ty{_=(~b}b{L05RuRLX8?L_`d8K z2pGCY2Q{e`6JVj+k7-3|QbCZCr%+%_rMxm+RIp~qf%$~NXGtiTmpQ_g|G#-ySJ7}7 z9(P2{EvPjIQF73-5Z4EYDjOiE$J_)(QXbQYA%rA5WK}Xzk0X_cKqZr48myyuD}?h( z{{jX!!8fZnd1=p}6lPp0UX?%g@dyb)G8~IVJq+Rtjj(+s61EchbS=y;qBDMr%^n%( z##(3~$zIEBReN!2C86^bd~r}#k^wsqZcpK5LF2nmwk6A&Sm_Tyuz=EX{r$boa8;D$ z9%P~fpP{ViQip?yj5r}VD5GkbQsEIMW*N+iJl%fu(sLMiuEQ5f5(7J##2CbsgUj?t zA~<6;(%~JD4|?uA-X}acj>0)ap11&Jmo(`xC11kHQG%+QVO+!N4tz>3h1K<2N}f{- zb_aHJ^%*hj| zXkui(2xondE1Nr~mm?Oa}KsHpjH@7!sq8*a1op_ymevET(h_ze9Xz2Nl9T;Yk~H?dvb0 z;_?@NxBy))sW(B^BmxD+_P*|m-#G8ja\n\nor inversely\n\n\n\nwhere Ω denotes the number of corresponding micro-states and kB is Boltzmanns constant. '\n\nThis formula was from the beginning derived for simple cases, like an ideal gas. Nevertheless, it does represent a kind of universal truth in statistical mechanics: the number of possible micro-states corresponding to a given macro-state grows exponentially with the entropy. Although there are many complications when one tries to consider the entropy of the universe as a whole, I will still take it as the starting point for the discussion that the entropy (at a given time t ) is an exponential function of the total entropy as in (3). A more difficult question is if and how the constant W may vary with time, but for the purpose of the present paper, I will simply let it be constant.\n\nOne may of course argue that this can only be true when the universe is still quite ordered and the entropy is very far from reaching its maximum. But this is certainly what the situation is like in our universe today, and according to the computations in [10, 11], it would take an almost incredibly long time to reach such a state of maximal entropy. Thus, it will in the following be taken for granted that this time is much longer than the life-span of our universe.\n\n312", + "recall": 0.9583333333333334, + "true_md": "Probability, Combinatorics and Control\n\nbetween this and the fact that the development of the underlying wave function for the whole universe is unique.\n\nSummarizing:\n\nDefinition 1. A universe U is a chain of states (one state Ut for each moment of time t), with the property that the transition between adjacent states is always possible.\n\nDefinition 2. A multiverse M is the set of all possible universes U in the sense of Definition 1 together with a probability measure on this set.\n\nIt may of course be said that quantum mechanics should allow for transitions between all kinds of states, although the probability for most such transitions may be extremely small. In this extremely simplified treatment, I will assume that for a given state at a given moment of time t, the dynamical laws will only permit transitions to a very limited number of states at the previous and next moments, which will make the probabilistic part of the investigation particularly simple. However, modifications are called for near the endpoints (the Big Bang and the Big Crunch); see Section 5.\n\nAs it stands, the model presented so far is too simple to generate any results. In fact, there are no observable differences at all between the states, which mean that there are no measurable variables which could be related to the (so far non- specified) dynamics.\n\nThere are of course many different variables which we can choose to enrich this structure, and which ones to choose must depend on what properties we want to explain. For explaining the second law of thermodynamics, the obvious choice is the entropy.\n\n## 4. Entropy\n\nAccording to Boltzmann, the total entropy of a certain macro-state at a certain time is given by\n\n$$S ¼ kB ln Ω , (2)$$\n\nor inversely\n\n$$Ω ¼ W S , with W ¼ e 1=kB , (3)$$\n\nwhere Ω denotes the number of corresponding micro-states and kB is Boltzmann's constant.\n\nThis formula was from the beginning derived for simple cases, like an ideal gas. Nevertheless, it does represent a kind of universal truth in statistical mechanics: the number of possible micro-states corresponding to a given macro-state grows expo- nentially with the entropy. Although there are many complications when one tries to consider the entropy of the universe as a whole, I will still take it as the starting point for the discussion that the entropy (at a given time t) is an exponential function of the total entropy as in (3). A more difficult question is if and how the constant W may vary with time, but for the purpose of the present paper, I will simply let it be constant.\n\nOne may of course argue that this can only be true when the universe is still quite ordered and the entropy is very far from reaching its maximum. But this is certainly what the situation is like in our universe today, and according to the computations in [10, 11], it would take an almost incredibly long time to reach such a state of maximal entropy. Thus, it will in the following be taken for granted that this time is much longer than the life-span of our universe.\n\n312" + }, + { + "bleu": 0.8583846447877751, + "doc_id": "doc_b2cd33ed246c6dca7d94b8937a8abbc44eb870a5007e5c07fcfbd2845e9b1d07_page_000001.png", + "edit_distance": 0.11173184357541899, + "f1_score": 0.9433272394881171, + "meteor": 0.8954479943697132, + "precision": 0.9555555555555556, + "pred_md": "Combinatorial Cosmology DOI: http:/ /dx.doi.org/10.5772/intechopen.90696\n\n## 5. The dynamics\n\nThe next step is to construct a model for the dynamics. The idea, which essentially goes back to Boltzmann (see [12]), is that any given macro-state at any given time is extremely likely to develop into a state with higher entropy at the next moment of time, simply because there are so many more states with higher entropy than with lower entropy (compare with (3)). The problem with this in the present situation, however, is that this way of thinking in fact presupposes a preferred direction of time. Otherwise, given that the dynamical laws are time symmetric, why can we not similarly argue that the entropy should also grow when we go backward in time? (compare [9]).\n\nThere have been many attempts to avoid this problem by looking for defects in the symmetries. But my conclusion here is that we must actually accept Boltzmann s ' argument in both directions of time and hence we are led to the following:\n\nPrinciple 1 . At every moment of time t and for every state with entropy S , there are very many ' accessible states ' with higher entropy, both at the previous moment of time t /C0 1 and at the next one t þ 1. On the other hand, the chance for finding such accessible states with lower entropy, both at times t /C0 1 and t þ 1, is extremely small.\n\nThis principle also implies a shift of perspective in the search for time s arrow. ' Rather than trying to find the reason for the asymmetry, we must concentrate on understanding why we cannot observe the symmetric structure of the multiverse as a whole.\n\nAs still one more simplification, let us assume that the entropy can only change by /C6 1 during each unit of time. This assumption, however, has to be modified near the endpoints (BB and BC) for the following reason: it is a very important aspect of this approach to assume that physics during the first and last moments is very different from the rest of the time, since at these moments quantum phenomena can be expected to become global. To model this in a simple way, we can split the life-span of our multiverse up into three parts:\n\n\n\nHere the first and last parts may be called ' the extreme phases, ' which are characterized by the property that transition between very different states can be possible. During the ' normal phase ' in between on the other hand, physics is supposed to behave more or less as we are used to.\n\n## 6. Modeling the dynamics\n\nTo construct a miniature multiverse for computational purposes, one can proceed as follows: first of all, in the very small multiverses studied here, the extreme phases will only last for one single unit of time. Also, for ease of notation, let us put T 1 ¼ m , so that the moments of time can in this context be denoted as\n\n\n\nThe dynamics is specified by randomly choosing for each state at time t with entropy S , K edges to states at time t þ 1 with entropy S þ 1, and similarly K edges to states at time t /C0 1 with entropy S þ 1 (with obvious modifications at the endpoints). In this section, again to make everything as simple as possible, K will be set equal to 2. These random choices are in practice carried out by the random number\n\n313", + "recall": 0.9314079422382672, + "true_md": "Combinatorial Cosmology DOI: http://dx.doi.org/10.5772/intechopen.90696\n\n## 5. The dynamics\n\nThe next step is to construct a model for the dynamics. The idea, which essen- tially goes back to Boltzmann (see [12]), is that any given macro-state at any given time is extremely likely to develop into a state with higher entropy at the next moment of time, simply because there are so many more states with higher entropy than with lower entropy (compare with (3)). The problem with this in the present situation, however, is that this way of thinking in fact presupposes a preferred direction of time. Otherwise, given that the dynamical laws are time symmetric, why can we not similarly argue that the entropy should also grow when we go backward in time? (compare [9]).\n\nThere have been many attempts to avoid this problem by looking for defects in the symmetries. But my conclusion here is that we must actually accept Boltzmann's argument in both directions of time and hence we are led to the following:\n\nPrinciple 1. At every moment of time t and for every state with entropy S, there are very many 'accessible states' with higher entropy, both at the previous moment of time t /C0 1 and at the next one t þ 1. On the other hand, the chance for finding such accessible states with lower entropy, both at times t /C0 1 and t þ 1, is extremely small.\n\nThis principle also implies a shift of perspective in the search for time's arrow. Rather than trying to find the reason for the asymmetry, we must concentrate on understanding why we cannot observe the symmetric structure of the multiverse as a whole.\n\nAs still one more simplification, let us assume that the entropy can only change by /C6 1 during each unit of time. This assumption, however, has to be modified near the endpoints (BB and BC) for the following reason: it is a very important aspect of this approach to assume that physics during the first and last moments is very different from the rest of the time, since at these moments quantum phenomena can be expected to become global. To model this in a simple way, we can split the life-span of our multiverse up into three parts:\n\n$$/C0 T0, /C0 T1 ½ /C138 ∪ /C0 T1, T1 ½ /C138 ∪ T1, T0 ½ /C138 : (4)$$\n\nHere the first and last parts may be called 'the extreme phases,' which are characterized by the property that transition between very different states can be possible. During the 'normal phase' in between on the other hand, physics is supposed to behave more or less as we are used to.\n\n## 6. Modeling the dynamics\n\nTo construct a miniature multiverse for computational purposes, one can pro- ceed as follows: first of all, in the very small multiverses studied here, the extreme phases will only last for one single unit of time. Also, for ease of notation, let us put T1 ¼ m, so that the moments of time can in this context be denoted as\n\n$$/C0 m /C0 1, /C0 m, /C0 m þ 1, … , m /C0 1, m, m þ 1: (5)$$\n\nThe dynamics is specified by randomly choosing for each state at time t with entropy S, K edges to states at time t þ 1 with entropy S þ 1, and similarly K edges to states at time t /C0 1 with entropy S þ 1 (with obvious modifications at the end- points). In this section, again to make everything as simple as possible, K will be set equal to 2. These random choices are in practice carried out by the random number\n\n313" + }, + { + "bleu": 0.8465424877602751, + "doc_id": "doc_9c897807c023cf6c34295d03cccb262ed0cbef83bef63d4908ab6d3775b15597_page_000001.png", + "edit_distance": 0.14392803598200898, + "f1_score": 0.9413919413919414, + "meteor": 0.8815683690833797, + "precision": 0.9625468164794008, + "pred_md": "Combinatorial Cosmology DOI: http:/ /dx.doi.org/10.5772/intechopen.90696\n\nAs for the normal phase, the choice will, to start with, be the simplest possible one: each path is either possible or not, corresponding to the probability weights 1 and 0. During the extreme phases, this assumption is no longer reasonable. Again the model will be extremely simplified, but still it is based on physical intuition and, most importantly, completely time symmetric. Assume that the only types of edges having a non-neglectable chance of occurring during the extreme phase /C0 m /C0 1, /C0 m ½ /C138 are of the following two kinds: The first scenario is that the universe passes through the extreme phase into a state of zero entropy. The other scenario is that it passes into a state with high entropy (equal to 2 m ). Universes of one of these two types will be given the (un-normalized) probability 1 or p , respectively. Here p > 0 should be thought of as a very small number, at least when the size of the model becomes large. During the other extreme phase m m , þ /C138 1 , near the Big ½ Crunch, we make the completely symmetric assumption.\n\nRemark 3. These assumptions may perhaps seem somewhat arbitrary. And to a certain extent, this may be so. However, they do represent the following viewpoint of what may happen at the full cosmological scale: we may think of the Big Bang and the Big Crunch as states of complete order with zero volume and entropy. Such states can very well be metastable, very much like an oversaturated gas at a temperature below the point of condensation. If no disturbance takes place, such metastable states can very well continue to exist for a substantial period of time. In particular, a low-entropy state can have a very good chance of surviving the intense but extremely short extreme phase. On the other hand, if a sufficiently large disturbance occurs, then the metastable state may almost immediately decay into a very disordered state of high entropy.\n\nIt is not my intension to further argue in favor of this viewpoint here. The main thing in this chapter is to show that completely symmetric boundary conditions at the endpoints may give rise to a broken time symmetry.\n\nThe multiverse now splits up into four different kinds of paths:\n\n- · LL: The entropy is low (=0) at both ends ( /C0 m and m ).\n- · LH: The entropy is 0 at /C0 m and 2 m at m .\n- · HL: The entropy is 2 m at /C0 m and 0 at m .\n- · HH: The entropy is high ( ¼ 2 m ) at both ends ( /C0 m and m ).\n\nIf we now denote by NLL , NLH , NHL and NHH the number of paths of the indicated kinds, then with the above assumptions we also get the corresponding probability weights for the corresponding types as\n\n\n\nWe can now consider the following two types of broken time symmetry: Definition 4. A multiverse is said to exhibit a weak broken time symmetry if\n\n\n\nDefinition 5. A multiverse is said to exhibit a strong broken time symmetry if\n\n\n\nBoth these definitions should of course be made more precise when applied to specific models for the multiverse, e.g., by showing that the corresponding limits\n\n317", + "recall": 0.921146953405018, + "true_md": "Combinatorial Cosmology DOI: http://dx.doi.org/10.5772/intechopen.90696\n\nAs for the normal phase, the choice will, to start with, be the simplest possible\n\none: each path is either possible or not, corresponding to the probability weights 1 and 0. During the extreme phases, this assumption is no longer reasonable. Again the model will be extremely simplified, but still it is based on physical intuition and, most importantly, completely time symmetric. Assume that the only types of edges having a non-neglectable chance of occurring during the extreme phase /C0 m /C0 1, /C0 m ½ /C138 are of the following two kinds: The first scenario is that the universe passes through the extreme phase into a state of zero entropy. The other scenario is that it passes into a state with high entropy (equal to 2m). Universes of one of these two types will be given the (un-normalized) probability 1 or p, respectively. Here p>0 should be thought of as a very small number, at least when the size of the model becomes large. During the other extreme phase m, m þ 1 ½ /C138 , near the Big Crunch, we make the completely symmetric assumption.\n\nRemark 3. These assumptions may perhaps seem somewhat arbitrary. And to a certain extent, this may be so. However, they do represent the following viewpoint of what may happen at the full cosmological scale: we may think of the Big Bang and the Big Crunch as states of complete order with zero volume and entropy. Such states can very well be metastable, very much like an oversaturated gas at a tem- perature below the point of condensation. If no disturbance takes place, such meta- stable states can very well continue to exist for a substantial period of time. In particular, a low-entropy state can have a very good chance of surviving the intense but extremely short extreme phase. On the other hand, if a sufficiently large dis- turbance occurs, then the metastable state may almost immediately decay into a very disordered state of high entropy.\n\nIt is not my intension to further argue in favor of this viewpoint here. The main thing in this chapter is to show that completely symmetric boundary conditions at the endpoints may give rise to a broken time symmetry.\n\nThe multiverse now splits up into four different kinds of paths:\n\n• LL: The entropy is low (=0) at both ends ( /C0 mand m).\n\n• LH: The entropy is 0 at /C0 mand 2m at m.\n\n• HL: The entropy is 2m at /C0 mand 0 at m.\n\n• HH: The entropy is high ( ¼ 2m) at both ends ( /C0 mand m).\n\nIf we now denote by NLL, NLH, NHL and NHH the number of paths of the indicated kinds, then with the above assumptions we also get the corresponding probability weights for the corresponding types as\n\n$$PLL ¼ NLL, PLH ¼ pN LH , PHL ¼ pN HL , PHH ¼ p 2 NHH: (10)$$\n\nWe can now consider the following two types of broken time symmetry: Definition 4. A multiverse is said to exhibit a weak broken time symmetry if\n\n$$PLL ≪ PLH þ PHL: (11)$$\n\nDefinition 5. A multiverse is said to exhibit a strong broken time symmetry if\n\n$$PLL þ PHH ≪PLH þ PHL: (12)$$\n\nBoth these definitions should of course be made more precise when applied to specific models for the multiverse, e.g., by showing that the corresponding limits\n\n317" + }, + { + "bleu": 0.9553499030726316, + "doc_id": "doc_a5b975710668800f4d4712155bc597a028f0ffbfe162efef92ded23cb2c23b9f_page_000001.png", + "edit_distance": 0.052823315118397086, + "f1_score": 0.9741550695825049, + "meteor": 0.9592034327261377, + "precision": 0.9683794466403162, + "pred_md": "Probability, Combinatorics and Control\n\n\n\nequal zero when certain parameters tend to infinity in some well-defined way. However, it is worthwhile at this stage to note their implications for cosmology.\n\nThe strong broken symmetry in Definition 5 actually means that a monotonic behavior of the entropy is far more probable than a non-monotonic one. In the case of a weak broken symmetry, this is not necessarily so; it could very well be that the most probable scenario would be high entropy at both ends. Thus, this is definitely a weaker statement, but it can nevertheless be argued that it can be used to explain the time asymmetry that we observe, referring to a kind of anthropic principle: it is an obvious observational fact that we live in a universe with low entropy at at least one end. If the statement in Definition 4 is fulfilled, then clearly among such scenarios, the monotonic ones (LH and HL) are the by far most probable ones. Thus, since universes with high entropy at both ends would seem to be quite uninhabitable, one can argue that given the existence of an observer, then with almost certainty he must live in a universe with monotonic entropy.\n\nSumming up, both limits above can be used to argue in favor of time asymmetry. Nevertheless, at least to the mind of the author, the strong broken symmetry is the preferable one. This alternative will be further studied in Section 9.\n\n## 8. Numerical computations in the combinatorial multiverse\n\nWith the setup in Sections 6 and 7, we can now use Mathematica or MATLAB to generate instances of the combinatorial multiverse for small values of m and W and then compute the corresponding probability weights PLL , PLH , PHL and PHH . It is important to note that the matrices here can be treated as sparse, rather than as full matrices, which make the computations considerably faster.\n\nIn particular, in the case m ¼ 2 in Section 6 and with a randomly generated dynamics which is manifested by an adjacency matrix A , we can compute the power A 4 and read of the first row, which contains all the information we need about the paths from the state at t ¼ /C0 2 with S ¼ 0. So what do we find?\n\nIn Figure 3 , I have plotted the ratio NLL = NLH þ NHL ð Þ for the cases m ¼ 2 (light gray) and m ¼ 3 (dark gray) for values of W ranging from 3 to 30. What is actually displayed are the mean values of 1000 randomly generated matrices as above for each value of W . Although the picture clearly supports the claim that\n\nFigure 3. The ratio NLL = NLH þ NHL ð Þ as a function of W for the cases m ¼ 2 (light gray) and m ¼ 3 (dark gray) [4].\n\n318", + "recall": 0.98, + "true_md": "Probability, Combinatorics and Control\n\n$$lim PLL PLH þ PHL and lim PLL þ PHH PLH þ PHL (13)$$\n\nequal zero when certain parameters tend to infinity in some well-defined way. However, it is worthwhile at this stage to note their implications for cosmology.\n\nThe strong broken symmetry in Definition 5 actually means that a monotonic behavior of the entropy is far more probable than a non-monotonic one. In the case of a weak broken symmetry, this is not necessarily so; it could very well be that the most probable scenario would be high entropy at both ends. Thus, this is definitely a weaker statement, but it can nevertheless be argued that it can be used to explain the time asymmetry that we observe, referring to a kind of anthropic principle: it is an obvious observational fact that we live in a universe with low entropy at at least one end. If the statement in Definition 4 is fulfilled, then clearly among such scenarios, the monotonic ones (LH and HL) are the by far most probable ones. Thus, since universes with high entropy at both ends would seem to be quite uninhabitable, one can argue that given the existence of an observer, then with almost certainty he must live in a universe with monotonic entropy.\n\nSumming up, both limits above can be used to argue in favor of time asymmetry. Nevertheless, at least to the mind of the author, the strong broken symmetry is the preferable one. This alternative will be further studied in Section 9.\n\n## 8. Numerical computations in the combinatorial multiverse\n\nWith the setup in Sections 6 and 7, we can now use Mathematica or MATLAB to generate instances of the combinatorial multiverse for small values of m and W and then compute the corresponding probability weights PLL, PLH, PHL and PHH. It is important to note that the matrices here can be treated as sparse, rather than as full matrices, which make the computations considerably faster.\n\nIn particular, in the case m ¼ 2 in Section 6 and with a randomly generated dynamics which is manifested by an adjacency matrix A, we can compute the power A 4 and read of the first row, which contains all the information we need about the paths from the state at t 2 with S 0. So what do we find?\n\nIn Figure 3, I have plotted the ratio NLL= NLH þ NHL ð Þ for the cases m ¼ 2 (light gray) and m ¼ 3 (dark gray) for values of W ranging from 3 to 30. What is actually displayed are the mean values of 1000 randomly generated matrices as above for each value of W. Although the picture clearly supports the claim that\n\nFigure 3. The ratio NLL= NLH þ NHL ð Þ as a function of W for the cases m ¼ 2 (light gray) and m ¼ 3 (dark gray) [4].\n\n318" + }, + { + "bleu": 0.9228556000445189, + "doc_id": "doc_65be3f78649d4ee59fa39b9d2b94a44570e52ddbf31cb7213f92853bc23ceb16_page_000001.png", + "edit_distance": 0.05352112676056338, + "f1_score": 0.9402173913043479, + "meteor": 0.9491434219344517, + "precision": 0.9611111111111111, + "pred_md": "## Prologue\n\n## Programming and Understanding\n\nOne way to become aware of the precision required to unambiguously communicate a mathematical idea is to program it for a computer. Rather than using canned programs purely as an aid to visualization or numerical computation, we use computer programming in a functional style to encourage clear thinking. Programming forces us to be precise and unambiguous, without forcing us to be excessively rigorous. The computer does not tolerate vague descriptions or incomplete constructions. Thus the act of programming makes us keenly aware of our errors of reasoning or unsupported conclusions. 1\n\nAlthough this book is about differential geometry, we can show how thinking about programming can help in understanding in a more elementary context. The traditional use of Leibniz's notation and Newton's notation is convenient in simple situations, but in more complicated situations it can be a serious handicap to clear reasoning.\n\nA mechanical system is described by a Lagrangian function of the system state (time, coordinates, and velocities). A motion of the system is described by a path that gives the coordinates for each moment of time. A path is allowed if and only if it satisfies the Lagrange equations. Traditionally, the Lagrange equations are written\n\n\n\nWhat could this expression possibly mean?\n\nLet's try to write a program that implements Lagrange equations. What are Lagrange equations for? Our program must take a proposed path and give a result that allows us to decide if the path is allowed. This is already a problem; the equation shown above does not have a slot for a path to be tested.\n\n1 The idea of using computer programming to develop skills of clear thinking was originally advocated by Seymour Papert. An extensive discussion of this idea, applied to the education of young children, can be found in Papert [13].", + "recall": 0.9202127659574468, + "true_md": "## Prologue\n\n## Programming and Understanding\n\nOne way to become aware of the precision required to unam- biguously communicate a mathematical idea is to program it for a computer. Rather than using canned programs purely as an aid to visualization or numerical computation, we use computer programming in a functional style to encourage clear thinking. Programming forces us to be precise and unambiguous, without forcing us to be excessively rigorous. The computer does not toler- ate vague descriptions or incomplete constructions. Thus the act of programming makes us keenly aware of our errors of reasoning or unsupported conclusions. 1\n\nAlthough this book is about differential geometry, we can show how thinking about programming can help in understanding in a more elementary context. The traditional use of Leibniz's notation and Newton's notation is convenient in simple situations, but in more complicated situations it can be a serious handicap to clear reasoning.\n\nA mechanical system is described by a Lagrangian function of the system state (time, coordinates, and velocities). A motion of the system is described by a path that gives the coordinates for each moment of time. A path is allowed if and only if it satisfies the Lagrange equations. Traditionally, the Lagrange equations are written\n\n$$d dt ∂L ∂ ˙q - ∂L ∂q =0.$$\n\nWhat could this expression possibly mean?\n\nLet's try to write a program that implements Lagrange equa- tions. What are Lagrange equations for? Our program must take a proposed path and give a result that allows us to decide if the path is allowed. This is already a problem; the equation shown above does not have a slot for a path to be tested.\n\n1 The idea of using computer programming to develop skills of clear thinking was originally advocated by Seymour Papert. An extensive discussion of this idea, applied to the education of young children, can be found in Papert [13]." + }, + { + "bleu": 0.7817987681872078, + "doc_id": "doc_f81cbaef6c20bdadec11a3e9622d381850f6f7d3ab059e3949a9a6a877fd144e_page_000001.png", + "edit_distance": 0.21232876712328766, + "f1_score": 0.9101123595505618, + "meteor": 0.8520690634574707, + "precision": 0.9152542372881356, + "pred_md": "Prologue\n\nxvii\n\n## Functional Abstraction\n\nBut this corrected use of Leibniz notation is ugly. We had to introduce extraneous symbols ( q and ˙) in order to indicate the arq gument position specifying the partial derivative. Nothing would change here if we replaced q and ˙ q by a and b . 3 We can simplify the notation by admitting that the partial derivatives of the Lagrangian are themselves new functions, and by specifying the particular partial derivative by the position of the argument that is varied\n\n\n\nwhere ∂ L i is the function which is the partial derivative of the function L with respect to the i th argument. 4\n\nTwo different notions of derivative appear in this expression. The functions ∂ L 2 and ∂ L 1 , constructed from the Lagrangian L , have the same arguments as L . The derivative d/dt is an expression derivative. It applies to an expression that involves the variable t and it gives the rate of change of the value of the expression as the value of the variable t is varied.\n\nThese are both useful interpretations of the idea of a derivative. But functions give us more power. There are many equivalent ways to write expressions that compute the same value. For example 1 (1 / /r 1 + 1 /r 2 ) = ( r r 1 2 ) / r ( 1 + r 2 ). These expressions compute the same function of the two variables r 1 and r 2 . The first expression fails if r 1 = 0 but the second one gives the right value of the function. If we abstract the function, say as Π( r , r 1 2 ), we can ignore the details of how it is computed. The ideas become clearer because they do not depend on the detailed shape of the expressions.\n\n3 That the symbols q and ˙ q can be replaced by other arbitrarily chosen nonconflicting symbols without changing the meaning of the expression tells us that the partial derivative symbol is a logical quantifier, like forall and exists ( ∀ and ∃ ).\n\n4 The argument positions of the Lagrangian are indicated by indices starting with zero for the time argument.", + "recall": 0.9050279329608939, + "true_md": "Prologue\n\nxvii\n\n## Functional Abstraction\n\nBut this corrected use of Leibniz notation is ugly. We had to introduce extraneous symbols (q and ˙q) in order to indicate the ar- gument position specifying the partial derivative. Nothing would change here if we replaced q and ˙q by a and b. 3 We can sim- plify the notation by admitting that the partial derivatives of the Lagrangian are themselves new functions, and by specifying the particular partial derivative by the position of the argument that is varied\n\n$$d dt ((∂ 2 L)(t, w(t), d dt w(t))) - (∂ 1 L)(t, w(t), d dt w(t)) = 0,$$\n\nwhere ∂ i L is the function which is the partial derivative of the function L with respect to the ith argument. 4\n\nTwo different notions of derivative appear in this expression. The functions ∂ 2 L and ∂ 1 L, constructed from the Lagrangian L, have the same arguments as L. The derivative d/dt is an expression derivative. It applies to an expression that involves the variable t and it gives the rate of change of the value of the expression as the value of the variable t is varied.\n\nThese are both useful interpretations of the idea of a derivative. But functions give us more power. There are many equivalent ways to write expressions that compute the same value. For example 1/(1/r 1 +1/r 2 )=(r 1 r 2 )/(r 1 + r 2 ). These expressions compute the same function of the two variables r 1 and r 2 . The first expression fails if r 1 =0butthesecondonegivestheright value of the function. If we abstract the function, say as Π(r 1 ,r 2 ), we can ignore the details of how it is computed. The ideas become clearer because they do not depend on the detailed shape of the expressions.\n\n3 That the symbols q and ˙q can be replaced by other arbitrarily chosen non- conflicting symbols without changing the meaning of the expression tells us that the partial derivative symbol is a logical quantifier, like forall and exists (∀ and ∃).\n\n4 The argument positions of the Lagrangian are indicated by indices starting with zero for the time argument." + }, + { + "bleu": 0.6461909024749306, + "doc_id": "doc_e1a31e05b65ada20646a1136bdc3293f68e3641893a48674c791bc2b8fadffa9_page_000001.png", + "edit_distance": 0.3032967032967033, + "f1_score": 0.9352941176470588, + "meteor": 0.7359789328058983, + "precision": 0.9636363636363636, + "pred_md": "xviii\n\nPrologue\n\nSo let's get rid of the expression derivative d/dt and replace it with an appropriate functional derivative. If f is a function then we will write Df as the new function that is the derivative of f : 5\n\n\n\nTo do this for the Lagrange equation we need to construct a function to take the derivative of.\n\nGiven a configuration-space path w , there is a standard way to make the state-space path. We can abstract this method as a mathematical function Γ:\n\n\n\nUsing Γ we can write:\n\n\n\nIf we now define composition of functions ( f ◦ g )( x ) = f ( g x ( )), we can express the Lagrange equations entirely in terms of functions:\n\n\n\nThe functions ∂ L 1 and ∂ L 2 are partial derivatives of the function L . Composition with Γ[ w ] evaluates these partials with coordinates and velocites appropriate for the path w , making functions of time. Applying D takes the time derivative. The Lagrange equation states that the difference of the resulting functions of time must be zero. This statement of the Lagrange equation is complete, unambiguous, and functional. It is not encumbered with the particular choices made in expressing the Lagrangian. For example, it doesn't matter if the time is named t or τ , and it has an explicit place for the path to be tested.\n\nThis expression is equivalent to a computer program: 6\n\n5 An explanation of functional derivatives is in Appendix B, page 202.\n\n6 The programs in this book are written in Scheme, a dialect of Lisp. The details of the language are not germane to the points being made. What is important is that it is mechanically interpretable, and thus unambiguous. In this book we require that the mathematical expressions be explicit enough", + "recall": 0.9085714285714286, + "true_md": "xviii\n\nPrologue\n\nSo let's get rid of the expression derivative d/dt and replace it with an appropriate functional derivative. If f is a function then we will write Df as the new function that is the derivative of f: 5\n\n$$(Df)(t)= d dx f(x) ∣ x=t .$$\n\nTo do this for the Lagrange equation we need to construct a function to take the derivative of.\n\nGiven a configuration-space path w, there is a standard way to make the state-space path. We can abstract this method as a mathematical function Γ:\n\n$$Γ[w](t)=(t,w(t), d dt w(t)).$$\n\nUsing Γ we can write:\n\n$$d dt ((∂ 2 L)(Γ[w](t))) - (∂ 1 L)(Γ[w](t)) = 0.$$\n\nIf we now define composition of functions (f ◦ g)(x)=f(g(x)), we can express the Lagrange equations entirely in terms of func- tions:\n\n$$D((∂ 2 L) ◦ (Γ[w])) - (∂ 1 L) ◦ (Γ[w]) = 0.$$\n\nThe functions ∂ 1 L and ∂ 2 L are partial derivatives of the func- tion L. Composition with Γ[w] evaluates these partials with coor- dinates and velocites appropriate for the path w, making functions of time. Applying D takes the time derivative. The Lagrange equation states that the difference of the resulting functions of time must be zero. This statement of the Lagrange equation is complete, unambiguous, and functional. It is not encumbered with the particular choices made in expressing the Lagrangian. For example, it doesn't matter if the time is named t or τ,andit has an explicit place for the path to be tested.\n\nThis expression is equivalent to a computer program: 6\n\n5 An explanation of functional derivatives is in Appendix B, page 202.\n\n6 The programs in this book are written in Scheme, a dialect of Lisp. The details of the language are not germane to the points being made. What is important is that it is mechanically interpretable, and thus unambiguous. In this book we require that the mathematical expressions be explicit enough" + }, + { + "bleu": 0.8125986351917156, + "doc_id": "doc_5cba8e49a880e4f6d1c1fba64f03b3d638fe26cfc00d77fddeb450334c8e0667_page_000001.png", + "edit_distance": 0.16535433070866143, + "f1_score": 0.948905109489051, + "meteor": 0.8502948247898391, + "precision": 0.9558823529411765, + "pred_md": "4\n\n## Basis Fields\n\nA vector field may be written as a linear combination of basis vector fields. If n is the dimension, then any set of n linearly independent vector fields may be used as a basis. The coordinate basis X is an example of a basis. 1 We will see later that not every basis is a coordinate basis: in order to be a coordinate basis, there must be a coordinate system such that each basis element is the directional derivative operator in a corresponding coordinate direction.\n\nLet e be a tuple of basis vector fields, such as the coordinate basis X . The general vector field v applied to an arbitrary manifold function f can be expressed as a linear combination\n\n\n\nwhere b is a tuple-valued coefficient function on the manifold. When expressed in a coordinate basis, the coefficients that specify the direction of the vector are naturally expressed as functions b i of the coordinates of the manifold point. Here, the coefficient function b is more naturally expressed as a tuple-valued function on the manifold. If b is the coefficient function expressed as a function of coordinates, then b = b ◦ χ is the coefficient function as a function on the manifold.\n\nThe coordinate-basis forms have a simple definition in terms of the coordinate-basis vectors and the coordinates (equation 3.40). With this choice, the dual property, equation (3.41), holds without further fuss. More generally, we can define a basis of one-forms ˜ e that is dual to e in that the property\n\n\n\nis satisfied, analogous to property (3.41). Figure 4.1 illustrates the duality of basis fields.\n\n1 We cannot say if the basis vectors are orthogonal or normalized until we introduce a metric.", + "recall": 0.9420289855072463, + "true_md": "4\n\n## Basis Fields\n\nA vector field may be written as a linear combination of basis vector fields. If n is the dimension, then any set of n linearly independent vector fields may be used as a basis. The coordinate basis X is an example of a basis. 1 We will see later that not every basis is a coordinate basis: in order to be a coordinate basis, there must be a coordinate system such that each basis element is the directional derivative operator in a corresponding coordinate direction.\n\nLet e be a tuple of basis vector fields, such as the coordinate basis X. The general vector field v applied to an arbitrary manifold function f can be expressed as a linear combination\n\n$$v(f)(m)=e(f)(m) b(m)= ∑ i e i (f)(m) b i (m), (4.1)$$\n\nwhere b is a tuple-valued coefficient function on the manifold. When expressed in a coordinate basis, the coefficients that specify the direction of the vector are naturally expressed as functions b i of the coordinates of the manifold point. Here, the coefficient function b is more naturally expressed as a tuple-valued function on the manifold. If b is the coefficient function expressed as a function of coordinates, then b = b ◦ χ is the coefficient function as a function on the manifold.\n\nThe coordinate-basis forms have a simple definition in terms of the coordinate-basis vectors and the coordinates (equation 3.40). With this choice, the dual property, equation (3.41), holds without further fuss. More generally, we can define a basis of one-forms ˜e that is dual to e in that the property\n\n$$˜e i (e j )(m)=δ i j (4.2)$$\n\nis satisfied, analogous to property (3.41). Figure 4.1 illustrates the duality of basis fields.\n\n1 We cannot say if the basis vectors are orthogonal or normalized until we introduce a metric." + }, + { + "bleu": 0.9837131808463916, + "doc_id": "doc_020d140f50e8e8332a8b2099bc306d83ecf46b851e3ee6cc03e456ccd05411bd_page_000001.png", + "edit_distance": 0.2751937984496124, + "f1_score": 0.9979209979209981, + "meteor": 0.9913883112873674, + "precision": 0.995850622406639, + "pred_md": "## 2. General Profile of MSMEs\n\nIn July 2020, the survey established a general profile of the MSMEs interviewed. The respondents updated the interviewers on the status of their business in each subsequent phase. Respondents whose business had permanently closed were only asked the reasons for closing (Section 2.4) and about government assistance programs (Section 7). The demographics of respondents and business characteristics (i.e., the proportions) remained roughly the same across all three survey phases.\n\nBusiness characteristics. Business size was determined by the number of staff at the time of interview. Following Government Decree number 25/ GOV, firms with five or less staff are microenterprises, those with six - 50 staff are small, and those with 51 - 99 staff are medium.\n\nMicro and small enterprises made up most of the respondents. Approximately 58% were microenterprises, 40% were small, and only two\n\nFigure 2.1: Surveyed MSMEs by size across sectors (%)\n\npercent were medium. The tourism MSME sample included a higher percentage of microenterprises than the other two sectors. All of the tourism and handicraft/ textile MSMEs interviewed were registered, or formal, constituting approximately 71% of the sample. The remainder (agriculture MSMEs) were informal, as they were individual farmers.\n\nmain products are silk and cotton products such as bags, clothes, and scarves, bamboo wicker, pottery, carvings, and mulberry paper products. MSMEs interviewed in the agriculture sector focused on the cultivation and trade of cash crops such as vegetables, cassava, banana, sugar cane, tea and coffee, livestock or fish, and rice.\n\nThe geographic focus of sampling sought to emulate the concentration of businesses nationwide. Interviewed MSMEs in the tourism and handicraft/ textile sectors were mainly based in Vientiane Capital, Luang Prabang, and Champasack provinces. For the agriculture sector, MSMEs were based in 12 provinces and the capital. Annex 1 provides the locations of respondents who participated in all three phases.\n\nThe tourism sub-sectors interviewed included lodging, restaurants and bars, and tour operators. Most handicraft/textile respondents were involved in production, with the remaining in sales. The\n\nDemographics of respondents. The overall gender ratio of interviewees was slightly skewed towards men (52%). Within the handicraft/textile sector, 80% were women, while the agriculture sector was dominated by male representatives (74%). The tourism sector respondents were 51% men. Most of the interviewees were MSME owners (80%), followed by managers (17%), while the other three percent comprised positions such as accountant, assistant, and deputy manager. More than half (58%) of interviewees were 36 to 55 years old; the youngest respondent was 23 and the eldest was 83.\n\n6", + "recall": 1.0, + "true_md": "## 2. General Profile of MSMEs\n\nIn July 2020, the survey established a general profile of the MSMEs interviewed. The respondents updated the interviewers on the status of their business in each subsequent phase. Respondents whose business had permanently closed were only asked the reasons for closing (Section 2.4) and about government assistance programs (Section 7). The demographics of respondents and business characteristics (i.e., the proportions) remained roughly the same across all three survey phases.\n\nBusiness characteristics. Business size was determined by the number of staff at the time of interview. Following Government Decree number 25/ GOV, firms with five or less staff are microenterprises, those with six - 50 staff are small, and those with 51 - 99 staff are medium.\n\nMicro and small enterprises made up most of the respondents. Approximately 58% were microenterprises, 40% were small, and only two percent were medium. The tourism MSME sample included a higher percentage of microenterprises than the other two sectors. All of the tourism and handicraft/ textile MSMEs interviewed were registered, or formal, constituting approximately 71% of the sample. The remainder (agriculture MSMEs) were informal, as they were individual farmers. \n\nFigure 2.1: Surveyed MSMEs by size across sectors (%)\n\nThe geographic focus of sampling sought to emulate the concentration of businesses nationwide. Interviewed MSMEs in the tourism and handicraft/ textile sectors were mainly based in Vientiane Capital, Luang Prabang, and Champasack provinces. For the agriculture sector, MSMEs were based in 12 provinces and the capital. Annex 1 provides the locations of respondents who participated in all three phases.\n\nThe tourism sub-sectors interviewed included lodging, restaurants and bars, and tour operators. Most handicraft/textile respondents were involved in production, with the remaining in sales. The main products are silk and cotton products such as bags, clothes, and scarves, bamboo wicker, pottery, carvings, and mulberry paper products. MSMEs interviewed in the agriculture sector focused on the cultivation and trade of cash crops such as vegetables, cassava, banana, sugar cane, tea and coffee, livestock or fish, and rice.\n\nDemographics of respondents. The overall gender ratio of interviewees was slightly skewed towards men (52%). Within the handicraft/textile sector, 80% were women, while the agriculture sector was dominated by male representatives (74%). The tourism sector respondents were 51% men. Most of the interviewees were MSME owners (80%), followed by managers (17%), while the other three percent comprised positions such as accountant, assistant, and deputy manager. More than half (58%) of interviewees were 36 to 55 years old; the youngest respondent was 23 and the eldest was 83." + }, + { + "bleu": 0.9562938926076634, + "doc_id": "doc_14e279e87615e3e3bb697729ecd97a2c5f3cd1195d9b1a2733ebfde630fea23d_page_000001.png", + "edit_distance": 0.21391752577319587, + "f1_score": 0.9929577464788734, + "meteor": 0.9846426064262254, + "precision": 0.9929577464788732, + "pred_md": "## 3. Impact on Business Operations\n\nThis section investigates the impact of public health measures on business operations. MSMEs were asked about their expectations for recovery and the main effects of COVID-19 on their businesses.\n\ncourse of the research period. The impacts of the lockdown from March 30 to May 4, 2020, were starkly felt, with only 30% of the MSMEs 'working as usual, ' while over half (58%) were temporarily completely closed.\n\n## 3.1. Status of Business Operations\n\nAs shown in Figure 3.1.1, the number of MSMEs 'working as usual' gradually increased over the\n\nIn the agriculture sector, a large majority of MSMEs (93% in July 2020, 98% in October 2020, and 99% in January 2021) were operating normally, though\n\nFigure 3.1.1: Status of operations during each survey phase (%)\n\nduring the first lockdown period, just over three quarters (77%) were working as usual. In contrast, 63% of firms from the tourism sector and 62% from the handicraft/textile sector were working as usual as of July 2020, rising to 80% of tourism and 82% of handicraft/textile firms as of January 2021 . During the lockdown period, tourism and handicraft/ textile MSMEs were the hardest hit with just 12% and 15% respectively working as usual. As shown in Table 3.1 .1 ., a majority of tourism and handicraft/ textile MSMEs were temporarily closed during the lockdown period. In the handicraft/textile sector, 30% of MSMEs were temporarily closed as of July 2020, reducing to 12% in January 2021. Similarly, in tourism, 27% of businesses were temporarily closed as of July 2020 and that reduced to 18% in January 2021. Figure 3.1.1 and Table 3.1 .1 do not reflect those MSMEs who were permanently closed; this was four in July 2020, 22 in October 2020, and 24 in January 2021. Of these 50 businesses who permanently closed during the research period, 30 were in the tourism sector, 18 in handicraft/textile, and two in agriculture.\n\n7", + "recall": 0.9929577464788732, + "true_md": "## 3. Impact on Business Operations\n\nThis section investigates the impact of public health measures on business operations. MSMEs were asked about their expectations for recovery and the main effects of COVID-19 on their businesses.\n\n## 3.1. Status of Business Operations\n\nAs shown in Figure 3.1.1, the number of MSMEs 'working as usual' gradually increased over the course of the research period. The impacts of the lockdown from March 30 to May 4, 2020, were starkly felt, with only 30% of the MSMEs 'working as usual,' while over half (58%) were temporarily completely closed.\n\nIn the agriculture sector, a large majority of MSMEs (93% in July 2020, 98% in October 2020, and 99% in January 2021) were operating normally, though during the first lockdown period, just over three quarters (77%) were working as usual. In contrast, 63% of firms from the tourism sector and 62% from the handicraft/textile sector were working as usual as of July 2020, rising to 80% of tourism and 82% of handicraft/textile firms as of January 2021. During the lockdown period, tourism and handicraft/ textile MSMEs were the hardest hit with just 12% and 15% respectively working as usual. As shown in Table 3.1.1., a majority of tourism and handicraft/ textile MSMEs were temporarily closed during the \n\nFigure 3.1.1: Status of operations during each survey phase (%)\n\nlockdown period. In the handicraft/textile sector, 30% of MSMEs were temporarily closed as of July 2020, reducing to 12% in January 2021. Similarly, in tourism, 27% of businesses were temporarily closed as of July 2020 and that reduced to 18% in January 2021. Figure 3.1.1 and Table 3.1.1 do not reflect those MSMEs who were permanently closed; this was four in July 2020, 22 in October 2020, and 24 in January 2021. Of these 50 businesses who permanently closed during the research period, 30 were in the tourism sector, 18 in handicraft/textile, and two in agriculture.\n\n7" + }, + { + "bleu": 0.9769423163515002, + "doc_id": "doc_22f4e5d348c45a3399f3c8a0e48756dd82a96ed9f5723a2a39183b554667d19a_page_000001.png", + "edit_distance": 0.0035587188612099642, + "f1_score": 0.9915966386554622, + "meteor": 0.9963790035587189, + "precision": 0.9915966386554622, + "pred_md": "Figure 6.1.1: Will they fire more staff in the next 2 months - across survey phases (%)\n\nFigure 6.1.2: Will they fire more staff in the next 2 months - across sectors and survey phases (%)\n\n## 6.2. Expectations for Re-Hiring Employees\n\nIn July 2020, 81% of the MSMEs that had laid off employees expected to re-hire all of them when the situation improved. This number reduced to 23% in October 2020 and further to just 7% in January 2021. 5 In July 2020, all MSMEs had plans to re-hire at least some of their staff. But in October 2020, 17% said they had no plans to re-hire and another 36% said they didn't know whether they would re-hire or not. In January 2021, 20% said they had no plans to re-hire and another 27% said they did not know. This question was only posed to those who had let staff go since the last survey round, and in October 2020 and January 2021, the base numbers reduced as fewer MSMEs reported letting staff go. In July 2020, 195 MSMEs\n\n5. The question on re-hiring was asked to those who had laid-off employees since the last survey. In the latter two survey rounds, respondents were asked about plans to re-hire staff whom they had let go since the previous interview, whereas in July 2020, they were asked about plans to re-hire staff they had let go since their business was first affected by the pandemic.\n\n23", + "recall": 0.9915966386554622, + "true_md": "Figure 6.1.1: Will they fire more staff in the next 2 months - across survey phases (%)\n\nFigure 6.1.2: Will they fire more staff in the next 2 months - across sectors and survey phases (%)\n\n## 6.2. Expectations for Re-Hiring Employees\n\nIn July 2020, 81% of the MSMEs that had laid off employees expected to re-hire all of them when the situation improved. This number reduced to 23% in October 2020 and further to just 7% in January 2021. 5 In July 2020, all MSMEs had plans to re-hire at least some of their staff. But in October 2020, 17% said they had no plans to re-hire and another 36% said they didn't know whether they would re-hire or not. In January 2021, 20% said they had no plans to re-hire and another 27% said they did not know. This question was only posed to those who had let staff go since the last survey round, and in October 2020 and January 2021, the base numbers reduced as fewer MSMEs reported letting staff go. In July 2020, 195 MSMEs \n\n5. /T\\_he question on re-hiring was asked to those who had laid-off employees since the last survey. In the latter two survey rounds, respondents were asked about plans to re-hire staff whom they had let go since the previous interview, whereas in July 2020, they were asked about plans to re-hire staff they had let go since their business was first affected by the pandemic.\n\n23" + }, + { + "bleu": 0.9385455425892445, + "doc_id": "doc_1a5c949c602fb587c39618443a00915582b2d20a32f959e5d3f4f3fc99ab9db5_page_000001.png", + "edit_distance": 0.18928571428571428, + "f1_score": 0.9928571428571429, + "meteor": 0.9715622011126747, + "precision": 0.9928571428571429, + "pred_md": "Figure 9.4.1: Challenges in importing amongst tourism MSMEs who import - all survey phases (%)\n\nThere were very few tourism MSMEs that exported in each survey round. The base is too small for any conclusive analysis.\n\n- · Devising new ways to reach customers through online markets or social media;\n\n## 9.5. Adapting to the New Normal: Changing Business Models\n\nIn all survey phases, several MSMEs in the tourism sector reported changing their business models. In July 2020, 167 tourism MSMEs mentioned that they changed their business model, in October 2020, 223 mentioned the same, and in January 2021, it was 183 MSMEs. Some changed models in more ways than one. The main ways across all phases that MSMEs made changes were:\n\n- · Adapting to social distancing;\n\n6. Compared to 38% in July 2020 and 22% in October 2020.\n\n- · Moving into new products and services in high demand during COVID-19;\n- · Reducing employee salaries.\n\nCompared to previous survey round results, in January 2021, tourism MSMEs had increasingly shifted towards adapting to social distancing to operate (57%). 6 Starting online marketing remained a popular choice, as nearly a quarter (24%) mentioned it in January 2021, compared to 28% in July 2020 and 31% in October 2020. Reducing employee salaries as an approach reduced considerably in January 2021 at 8% of responses compared to 21% in July 2020 and 24% in October 2020.\n\n39", + "recall": 0.9928571428571429, + "true_md": "Figure 9.4.1: Challenges in importing amongst tourism MSMEs who import - all survey phases (%)\n\nThere were very few tourism MSMEs that exported in each survey round. The base is too small for any conclusive analysis.\n\n## 9.5. Adapting to the New Normal: Changing Business Models\n\nIn all survey phases, several MSMEs in the tourism sector reported changing their business models. In July 2020, 167 tourism MSMEs mentioned that they changed their business model, in October 2020, 223 mentioned the same, and in January 2021, it was 183 MSMEs. Some changed models in more ways than one. The main ways across all phases that MSMEs made changes were:\n\n• Adapting to social distancing;\n\n• Devising new ways to reach customers through online markets or social media;\n\n• Moving into new products and services in high demand during COVID-19;\n\n• Reducing employee salaries. \n\nCompared to previous survey round results, in January 2021, tourism MSMEs had increasingly shifted towards adapting to social distancing to operate (57%). 6 Starting online marketing remained a popular choice, as nearly a quarter (24%) mentioned it in January 2021, compared to 28% in July 2020 and 31% in October 2020. Reducing employee salaries as an approach reduced considerably in January 2021 at 8% of responses compared to 21% in July 2020 and 24% in October 2020.\n\n6. Compared to 38% in July 2020 and 22% in October 2020.\n\n39" + }, + { + "bleu": 0.9911213540430709, + "doc_id": "doc_771caa3e8ec1bce887caac49940cab3c5b06a8a42ce120daa1999c30204aab33_page_000001.png", + "edit_distance": 0.30660377358490565, + "f1_score": 1.0, + "meteor": 0.9859187384447397, + "precision": 1.0, + "pred_md": "Thailand, Philippines and Indonesia in particular, identifying known experts at the national, subnational and community level. The survey and interviews with key informants asked key questions to regional experts on violent extremism to ascertain if hostile sentiments espoused are exacerbating insecurities for women.\n\nThe survey was made available in English, Bahasa, Thai and Tagalog. We used the Qualtrics platform to facilitate the ease of dissemination and response from home computers, iPads or mobile phone survey options. Qualtrics, one of the most widely used research platforms, supports the implementation of both large-scale survey and experimental study designs. It is administered online with responses gathered into a central and privacy protected database that only the approved researchers have access to.\n\nof the region that most experience violent extremism and terrorism. However, through our networks, where possible, we disseminated the survey throughout all ASEAN countries.\n\nIt is important to note the limitations of this six-month study. Although the survey was disseminated among all member states, the majority of expert respondents came from Indonesia, the Philippines and Thailand. While this can be regarded as highly selective rather than representative, it is important to note that Indonesia, the Philippines and Thailand are the countries that continue to face the most pressing threat of ongoing violent extremism and conflict.\n\nThe platform allows for the easy migration of data into various statistical packages, including STATA, the main statistical analysis package that we will use to analyse the data. A limitation of this study is that we were unable to translate the survey in all ASEAN languages, and there is a selection bias in that we are focussing the survey in areas\n\nThis is with the exception of Myanmar. Given the current political circumstances and challenges posed by COVID-19, on top of the short project time span, it was unfeasible to include Myanmar within the scope of this study. It is also important to note that the data derived from the surveys and interviews were based on the perceptions of experts and key informants, who are involved in peacebuilding, and on P/CVE strategies throughout the region. As a result, it is important to note the subjectivity of responses.\n\nFigure 1: Age by gender of respondents\n\nGender Analysis of Violent Extremism and the Impact of COVID-19 on Peace and Security in ASEAN\n\n26", + "recall": 1.0, + "true_md": "Thailand, Philippines and Indonesia in particular, identifying known experts at the national, subnational and community level. The survey and interviews with key informants asked key questions to regional experts on violent extremism to ascertain if hostile sentiments espoused are exacerbating insecurities for women. \n\nThe survey was made available in English, Bahasa, Thai and Tagalog. We used the Qualtrics platform to facilitate the ease of dissemination and response from home computers, iPads or mobile phone survey options. Qualtrics, one of the most widely used research platforms, supports the implementation of both large-scale survey and experimental study designs. It is administered online with responses gathered into a central and privacy protected database that only the approved researchers have access to. \n\nThe platform allows for the easy migration of data into various statistical packages, including STATA, the main statistical analysis package that we will use to analyse the data. A limitation of this study is that we were unable to translate the survey in all ASEAN languages, and there is a selection bias in that we are focussing the survey in areas of the region that most experience violent extremism and terrorism. However, through our networks, where possible, we disseminated the survey throughout all ASEAN countries.\n\nIt is important to note the limitations of this six-month study. Although the survey was disseminated among all member states, the majority of expert respondents came from Indonesia, the Philippines and Thailand. While this can be regarded as highly selective rather than representative, it is important to note that Indonesia, the Philippines and Thailand are the countries that continue to face the most pressing threat of ongoing violent extremism and conflict. \n\nThis is with the exception of Myanmar. Given the current political circumstances and challenges posed by COVID-19, on top of the short project time span, it was unfeasible to include Myanmar within the scope of this study. It is also important to note that the data derived from the surveys and interviews were based on the perceptions of experts and key informants, who are involved in peacebuilding, and on P/CVE strategies throughout the region. As a result, it is important to note the subjectivity of responses.\n\nFigure 1: Age by gender of respondents\n\nGender Analysis of Violent Extremism and the Impact of COVID-19 on Peace and Security in ASEAN\n\n26" + }, + { + "bleu": 0.980888407067061, + "doc_id": "doc_e786231794e24874e66321d9a2ce9d80050a73cd0e02fa358c88a1e87a183735_page_000001.png", + "edit_distance": 0.3187066974595843, + "f1_score": 1.0, + "meteor": 0.9792135611293815, + "precision": 1.0, + "pred_md": "tweets, videos) inciting violence towards religious minorities, ethnic minorities, the LGBTI community, and women and girls. Forty-four per cent of respondents had 'sometimes' seen extremist social media content inciting violence towards religious minorities, with 31% seeing this content 'very often'.\n\nBoth men and women acknowledged that they had 'sometimes' seen this content on social media (62% and 41%, respectively). Indonesia was the country from which most respondents had viewed this content 'very often' (50%). When collapsing the 'always' and 'very often' categories, 41% of Instagram users had often seen intolerant content, followed by 36% of WhatsApp users and 34% of Facebook users. Among the Twitter users in the sample, 48% had seen intolerant content towards religious minorities.\n\nrespondents had seen this content 'very often' (58%). Users of Facebook, WhatsApp and Instagram acknowledged that they had seen this content 'very often' (26%, 31% and 35% respectively).\n\nThirty-nine per cent of respondents acknowledged that they had 'sometimes'' seen social media content inciting violence towards the LGBTI community. Women saw this type of content more frequently than men (84%), and Indonesia was the country from which more respondents saw this content with a higher frequency (53% saw such content 'always' and 'very often'). Participants in the survey observed intolerant content directed towards the LGBTI community. For example, one participant from the Philippines observed that,\n\nWhen asked about how often social media content was inciting violence towards ethnic minorities, 46% of respondents had 'sometimes' seen this type of extremist social media content inciting violence towards ethnic minorities whereas only 27% have seen this content rarely or never. Women have seen such content more frequently than men (90%), and Indonesia was the country from which most\n\nThere were instances when women were humiliated in public and on social media after they were labelled as part of the LGBTQ+ community. The comments on posts regarding them were mostly commending their public humiliation (cutting their hair) instead of condemning the act '.\n\nFigure 3: Frequency of viewing extremist social media inciting violence toward women and girls\n\nGender Analysis of Violent Extremism and the Impact of COVID-19 on Peace and Security in ASEAN\n\n29", + "recall": 1.0, + "true_md": "tweets, videos) inciting violence towards religious minorities, ethnic minorities, the LGBTI community, and women and girls. Forty-four per cent of respondents had 'sometimes' seen extremist social media content inciting violence towards religious minorities, with 31% seeing this content 'very often'. \n\nBoth men and women acknowledged that they had 'sometimes' seen this content on social media (62% and 41%, respectively). Indonesia was the country from which most respondents had viewed this content 'very often' (50%). When collapsing the 'always' and 'very often' categories, 41% of Instagram users had often seen intolerant content, followed by 36% of WhatsApp users and 34% of Facebook users. Among the Twitter users in the sample, 48% had seen intolerant content towards religious minorities.\n\nWhen asked about how often social media content was inciting violence towards ethnic minorities, 46% of respondents had 'sometimes' seen this type of extremist social media content inciting violence towards ethnic minorities whereas only 27% have seen this content rarely or never. Women have seen such content more frequently than men (90%), and Indonesia was the country from which most respondents had seen this content 'very often' (58%). Users of Facebook, WhatsApp and Instagram acknowledged that they had seen this content 'very often' (26%, 31% and 35% respectively).\n\nThirty-nine per cent of respondents acknowledged that they had 'sometimes'' seen social media content inciting violence towards the LGBTI community. Women saw this type of content more frequently than men (84%), and Indonesia was the country from which more respondents saw this content with a higher frequency (53% saw such content 'always' and 'very often'). Participants in the survey observed intolerant content directed towards the LGBTI community. For example, one participant from the Philippines observed that, \n\nThere were instances when women were humiliated in public and on social media after they were labelled as part of the LGBTQ+ community. The comments on posts regarding them were mostly commending their public humiliation (cutting their hair) instead of condemning the act'. \n\nFigure 3: Frequency of viewing extremist social media inciting violence toward women and girls\n\nGender Analysis of Violent Extremism and the Impact of COVID-19 on Peace and Security in ASEAN\n\n29" + }, + { + "bleu": 0.9718456823584092, + "doc_id": "doc_a14ebf0077dbbb76dd91f3d7d0b66b66b4ab62260846cc75fb466b85a4a762b4_page_000001.png", + "edit_distance": 0.015503875968992248, + "f1_score": 0.9792147806004617, + "meteor": 0.985258602475118, + "precision": 0.9906542056074766, + "pred_md": "this content 'very often', 71% were from Indonesia and 28.6% were from Thailand. When asked about how often participants had heard of groups expressing the importance of men accompanying women when travelling to conflict zones, more respondents had heard this message with a higher frequency ('always' or 'very often', 37.1%) than those who had rarely or never heard it (34%). Forty-six per cent of respondents from Indonesia heard this message with a higher frequency, followed by the Philippines (38%) and Thailand (15%). When grouping the answer options of 'always', 'very often' and 'sometimes', 66% of respondents said they had heard groups stress the importance of women being accompanied by men when travelling to conflict areas.\n\nFigure 5: Importance of a male guardian accompanying women when travelling to conflict zones\n\nIn the second part of the survey, using a five-point Likert scale from 'strongly agree' to 'strongly disagree', participants were presented with a series of statements regarding how worried they were about intolerant content being espoused in the offline space by violent ex- tremist groups. Most respondents (77%) agreed (combining both 'strongly agree' and 'agree') that they were worried about intolerance in their communities, particularly respondents from Indonesia and the Philippines. Almost all respondents in the sample (93%) agreed that they were worried about violent extremism in their countries. This appeared to be a general concern among both men and women as 85% of men and 95% of women agreed that they were concerned.\n\nSignificantly, 89% of respondents agreed that religious extremism would impede women's rights. Half of the participants in Indonesia agreed they were concerned that religious extremism would hamper women's rights, 27% in Philippines and 16% in Thailand. Both men (84.6%) and women (89.2%) expressed their concerns on this issue. Furthermore, 91% of respondents agreed that religious extremism prioritizes men's rights over women's rights - 93.1% of women strongly agreed with the statement compared to 6.90% of men.\n\nFor example, one interviewee from Indonesia observed that the teachings of extremism have entered schools, such as high schools, and have also begun to penetrate student organizations. She observed that the teachings 'spread from the Middle East, bringing misogynistic teachings towards women as part of their subjugation strategy'. She acknowledged that it was part of the organizational strategy where women appeared to look empowered:\n\n'However, this is just manipulation; behind it is the practice of misogyny, women's consciousness, their bodies and minds are controlled, even though\n\nGender Analysis of Violent Extremism and the Impact of COVID-19 on Peace and Security in ASEAN\n\n31", + "recall": 0.9680365296803652, + "true_md": "this content 'very often', 71% were from Indonesia and 28.6% were from Thailand. When asked about how often participants had heard of groups expressing the importance of men accompanying women when travelling to conflict zones, more respondents had heard this message with a higher frequency ('always' or 'very often', 37.1%) than those who had rarely or never heard it (34%). Forty-six per cent of respondents from Indonesia heard this message with a higher frequency, followed by the Philippines (38%) and Thailand (15%). When grouping the answer options of 'always', 'very often' and 'sometimes', 66% of respondents said they had heard groups stress the importance of women being accompanied by men when travelling to conflict areas.\n\nFigure 5: Importance of a male guardian accompanying women when travelling to conflict zones\n\nIn the second part of the survey, using a five-point Likert scale from 'strong- ly agree' to 'strongly disagree', partic- ipants were presented with a series of statements regarding how worried they were about intolerant content being es- poused in the offline space by violent ex- tremist groups. Most respondents (77%) agreed (combining both 'strongly agree' and 'agree') that they were worried about intolerance in their communities, partic- ularly respondents from Indonesia and the Philippines. Almost all respondents in the sample (93%) agreed that they were worried about violent extremism in their countries. This appeared to be a general concern among both men and women as 85% of men and 95% of women agreed that they were concerned. \n\nSignificantly, 89% of respondents agreed that religious extremism would impede women's rights. Half of the participants in Indonesia agreed they were concerned that religious extremism would hamper women's rights, 27% in Philippines and 16% in Thailand. Both men (84.6%) and women (89.2%) expressed their concerns on this issue. Furthermore, 91% of respondents agreed that religious extremism prioritizes men's rights over women's rights - 93.1% of women strongly agreed with the statement compared to 6.90% of men.\n\nFor example, one interviewee from Indonesia observed that the teachings of extremism have entered schools, such as high schools, and have also begun to penetrate student organizations. She observed that the teachings 'spread from the Middle East, bringing misogynistic teachings towards women as part of their subjugation strategy'. She acknowledged that it was part of the organizational strategy where women appeared to look empowered:\n\n'However, this is just manipulation; behind it is the practice of misogyny, women's consciousness, their bodies and minds are controlled, even though \n\nGender Analysis of Violent Extremism and the Impact of COVID-19 on Peace and Security in ASEAN\n\n31" + }, + { + "bleu": 0.9912813312708183, + "doc_id": "doc_99199a6ab7aff04e417c953ce0b87f56b1d9fc6a91d891b2c74720778ac3a95a_page_000001.png", + "edit_distance": 0.003389830508474576, + "f1_score": 0.9968051118210862, + "meteor": 0.9996598222474866, + "precision": 0.9936305732484076, + "pred_md": "Figure 7: Respondents' reaction to the statement 'I am worried that misogynistic and hostile beliefs espoused by extremist groups result in violence towards women.'\n\nDISAGREE\n\nDuring the COVID-19 pandemic, 70% of respondents agreed that online radicalization and the proliferation of extremist propaganda had increased. Altogether, 76.9% and 92.9% of women agreed with the statement.\n\nOne interviewee from Indonesia noted that:\n\n'COVID has managed to restrict direct meetings to disseminate propaganda, misinformation and disinformation through most government's large-scale restrictions to prevent the virus' spread. However, the tendency to utilize online spaces to disseminate these has increased since the use of online activities is mandatory in various sectors, such as working and education. Most people certainly use online platforms to disseminate false information\n\nregarding the outbreak, as well as radical ideas targeted at people, including recruiting them as a part of groups.'\n\nFigure 8: Respondents' view to the statement, 'Online radicalization and the proliferation of extremist propaganda has increased during COVID-1'.\n\nAnother interviewee from Indonesia observed that:\n\n'(Based on my experience), during 2020-2021 one of the interesting things has been the impact of misinformation and disinformation related to COVID, affecting people's views and attitudes in responding to, preventing and handling of (the virus). At the beginning of the Indonesian government's policy on limiting religious activities in places of worship, this issue caused a strong, adverse reaction among extremist groups, giving rise to a narrative that the\n\nGender Analysis of Violent Extremism and the Impact of COVID-19 on Peace and Security in ASEAN\n\n36", + "recall": 1.0, + "true_md": "Figure 7: Respondents' reaction to the statement 'I am worried that misogynistic and hostile beliefs espoused by extremist groups result in violence towards women.'\n\nDuring the COVID-19 pandemic, 70% of respondents agreed that online radicalization and the proliferation of extremist propaganda had increased. Altogether, 76.9% and 92.9% of women agreed with the statement.\n\nOne interviewee from Indonesia noted that:\n\n'COVID has managed to restrict direct meetings to disseminate propaganda, misinformation and disinformation through most government's large-scale restrictions to prevent the virus' spread. However, the tendency to utilize online spaces to disseminate these has increased since the use of online activities is mandatory in various sectors, such as working and education. Most people certainly use online platforms to disseminate false information regarding the outbreak, as well as radical ideas targeted at people, including recruiting them as a part of groups.'\n\nFigure 8: Respondents' view to the statement, 'Online radicalization and the proliferation of extremist propaganda has increased during COVID-1'.\n\nAnother interviewee from Indonesia observed that:\n\n'(Based on my experience), during 2020-2021 one of the interesting things has been the impact of misinformation and disinformation related to COVID, affecting people's views and attitudes in responding to, preventing and handling of (the virus). At the beginning of the Indonesian government's policy on limiting religious activities in places of worship, this issue caused a strong, adverse reaction among extremist groups, giving rise to a narrative that the \n\nGender Analysis of Violent Extremism and the Impact of COVID-19 on Peace and Security in ASEAN\n\n36" + }, + { + "bleu": 1.0, + "doc_id": "doc_29dfa90e45920fd79e68e82f0f171119fc1b4d494aac703edf21c3ea74dd6fca_page_000001.png", + "edit_distance": 0.0, + "f1_score": 1.0, + "meteor": 0.996, + "precision": 1.0, + "pred_md": "## Table of Contents", + "recall": 1.0, + "true_md": "## Table of Contents" + }, + { + "bleu": 1.0, + "doc_id": "doc_00f0adaaa8358a28b4b4e83bc97dcd83a01f7283605b140c2be8e8d47bba8b6b_page_000001.png", + "edit_distance": 0.0, + "f1_score": 1.0, + "meteor": 0.9999954788773148, + "precision": 1.0, + "pred_md": "Civil Society Engagement\n\nelection integrity. The registration of local election observers runs until 25 May, and the NEC is still reviewing the application of nearly 5,000 observers.\n\nTable: The number of accredited observers as of 28 April 2022 15\n\n15 https://www.nec.gov.kh/khmer/content/5524\n\n17", + "recall": 1.0, + "true_md": "Civil Society Engagement\n\nelection integrity. The registration of local election observers runs until 25 May, and the NEC is still reviewing the application of nearly 5,000 observers.\n\nTable: The number of accredited observers as of 28 April 2022 15\n\n15 https://www.nec.gov.kh/khmer/content/5524\n\n17" + }, + { + "bleu": 0.9499444758938779, + "doc_id": "doc_d276ca9a5ecb8d6d11359f515e50c8f78395548de4e3e2c49e38f5500ee40ebe_page_000001.png", + "edit_distance": 0.045454545454545456, + "f1_score": 0.9811320754716982, + "meteor": 0.9952069295332667, + "precision": 0.9629629629629629, + "pred_md": "Political Parties, Candidates Registration and Election Campaign\n\n## Table: Provisional Results of Registration of Candidates on 8 March 2022 21 and Official Results of Registration of Candidates on 29 April 2022 22\n\n21 https://www.nec.gov.kh/khmer/content/5393\n\n22 https://www.nec.gov.kh/khmer/content/5525\n\n23", + "recall": 1.0, + "true_md": "Political Parties, Candidates Registration and Election Campaign\n\nTable: Provisional Results of Registration of Candidates on 8 March 2022 21 and Official Results of Registration of Candidates on 29 April 2022 22\n\n21 https://www.nec.gov.kh/khmer/content/5393\n\n22 https://www.nec.gov.kh/khmer/content/5525\n\n23" + }, + { + "bleu": 1.0, + "doc_id": "doc_934fbf534914863f6431eef38f5bf66fa91afd439ddf20fb1af0cf3225159ac1_page_000001.png", + "edit_distance": 0.0, + "f1_score": 1.0, + "meteor": 0.9976851851851852, + "precision": 1.0, + "pred_md": "ANFREL Pre-Election Assessment Mission Report\n\n24", + "recall": 1.0, + "true_md": "ANFREL Pre-Election Assessment Mission Report\n\n24" + }, + { + "bleu": 0.9832901031458049, + "doc_id": "doc_31e7fbfc5b038d471b2fc82b248c4807152284bc35d6d5bd036c0aaa42504f17_page_000001.png", + "edit_distance": 0.009433962264150943, + "f1_score": 0.9868421052631579, + "meteor": 0.9910333257890226, + "precision": 0.9911894273127754, + "pred_md": "8\n\nEncinas Franco and Laguna\n\n## Filipino Women in Electoral Politics\n\nThe nature and extent of Filipino women's political participation is a product of the country's colonial history, martial law, and democratization post-1986. Historians argue that Spain's strong Catholic traditions ushered in patriarchal norms and practices that were not present in the pre-Hispanic period. National hero, Jose Rizal, has documented this in his 'Letter to the Women of Malolos,' praising the women for advocating their right to education. Historians also found proof of women's contribution to the Philippine revolution (Camagay 1998). Decades later, the suffragist movement ushered in one of the first national issues to have brought Filipino women together. It was a hardfought battle; the movement had to contend with staunch opposition from antisuffragists in the Constitutional Convention that drafted the 1935 Constitution. The reluctance was expected because only 21-yearold Filipino men had been allowed to vote during the time. They framed their opposition based on traditional notions of womanhood and their role in the private sphere, foremost of which is motherhood. Another key argument against female suffrage was the idea that politics is supposed to be 'dirty' and that this would taint families if women took part in politics. The assumptions catered to the age-old public-private divide, strongly suggesting that only men are qualified to occupy the former.\n\nEventually, the 1935 Constitution granted women suffrage on the condition that more than 300,000 women would vote affirmatively in a plebiscite. When signing the law paving the way for the said plebiscite, President Manuel Quezon had this to say to Filipino men: 'Are you going to deprive our women of the opportunity to say how their lives are going to be regulated and is it fair for us to presume that men can always speak in this country for women?' (Official Gazette 1936). In April 1937, more than 400,000 women voted in favor of their right to vote and participate in political life. In 1946 and 1947, Filipinos elected the first woman member of the House of Representatives, and senator, respectively. Nonetheless, data from 1946 to 1992 indicate an uphill climb. For instance, in the 1949 and 1953 elections for the House of Representatives, only one woman was elected out of the 100 positions.", + "recall": 0.982532751091703, + "true_md": "8\n\nEncinas Franco and Laguna\n\n## Filipino Women in Electoral Politics\n\nThe nature and extent of Filipino women's political participation is a product of the country's colonial history, martial law, and democratization post-1986. Historians argue that Spain's strong Catholic traditions ushered in patriarchal norms and practices that were not present in the pre-Hispanic period. National hero, Jose Rizal, has documented this in his 'Letter to the Women of Malolos,' praising the women for advocating their right to education. Historians also found proof of women's contribution to the Philippine revolution (Camagay 1998). Decades later, the suffragist movement ushered in one of the first national issues to have brought Filipino women together. It was a hard- fought battle; the movement had to contend with staunch opposition from antisuffragists in the Constitutional Convention that drafted the 1935 Constitution. The reluctance was expected because only 21-year- old Filipino men had been allowed to vote during the time. They framed their opposition based on traditional notions of womanhood and their role in the private sphere, foremost of which is motherhood. Another key argument against female suffrage was the idea that politics is supposed to be 'dirty' and that this would taint families if women took part in politics. The assumptions catered to the age-old public-private divide, strongly suggesting that only men are qualified to occupy the former. \n\nEventually, the 1935 Constitution granted women suffrage on the condition that more than 300,000 women would vote affirmatively in a plebiscite. When signing the law paving the way for the said plebiscite, President Manuel Quezon had this to say to Filipino men: 'Are you going to deprive our women of the opportunity to say how their lives are going to be regulated and is it fair for us to presume that men can always speak in this country for women?' (Official Gazette 1936). In April 1937, more than 400,000 women voted in favor of their right to vote and participate in political life. In 1946 and 1947, Filipinos elected the first woman member of the House of Representatives, and senator, respectively. Nonetheless, data from 1946 to 1992 indicate an uphill climb. For instance, in the 1949 and 1953 elections for the House of Representatives, only one woman was elected out of the 100 positions. " + }, + { + "bleu": 0.9833639076249199, + "doc_id": "doc_c33c2993fe1e5c2945bb894f1a8e54c756753c344bd66f11024715e6375f07d1_page_000001.png", + "edit_distance": 0.009389671361502348, + "f1_score": 0.9933481152993348, + "meteor": 0.994813588385378, + "precision": 0.9911504424778761, + "pred_md": "Overcoming Barriers to Filipino Women's Political Representation\n\n9\n\nThe post-World War II period saw women participating in formal politics and even attempting to form a political party and an alliance supporting President Ramon Magsaysay's candidacy for the presidency (He served as president from 1953 to 1957), while the advent of the martial law period in 1972 witnessed feminist movements. Roces (2012, 6) attributes this to the burgeoning student movement and activism, so much so that by the time Marcos declared martial law, women were prepared to take on the resistance. Though inspired by North America's second-wave feminists, Filipino women were also drawn to the era's discourses and contexts, such as the Vietnam War and the civil rights movement.\n\nThe women's movement continued to flourish in the Cory Aquino regime (1986-1992). The democratic transition provided political opportunity structures and venues ensuring women's access to the state and nonstate spheres. The drafting of the 1987 Constitution was one such opportunity. The movement managed to advocate for important provisions paving the way for women's rights legislation from the 1980s to the present. The provision in the 1987 Constitution mandates the state to recognize 'the role of women in nation building and shall ensure the fundamental equality before the law of men and women' (Article 2, Section 14). This provision is said to be unique and is not even found in other countries' charters (Masilungan n.d.).\n\nThe post-Marcos period advanced the participation of women not only in civil society and nongovernment organizations but also in formal politics and bureaucracy. Several women from the movement joined formal politics, while others were invited by the Aquino and Ramos governments (1992-1998) to executive posts. The entry of women activists, NGO leaders, and those from the academe ensured that the new democracy would significantly help push measures promoting women's rights and gender equality. The House of Representative (HOR) and Philippine Commission on Women (PCW)'s 'How to Be a Gender-Responsive Legislator' (2021, 52) listed several recent laws responding to women's empowerment and gender equality.\n\n- · Republic Act No. 11313: Safe Spaces Act (April 17, 2019)\n- · Republic Act No. 11210: 105-Day Expanded Maternity Leave Law (March 11, 2019)", + "recall": 0.9955555555555555, + "true_md": "Overcoming Barriers to Filipino Women's Political Representation\n\n9\n\nThe post-World War II period saw women participating in formal politics and even attempting to form a political party and an alliance supporting President Ramon Magsaysay's candidacy for the presidency (He served as president from 1953 to 1957), while the advent of the martial law period in 1972 witnessed feminist movements. Roces (2012, 6) attributes this to the burgeoning student movement and activism, so much so that by the time Marcos declared martial law, women were prepared to take on the resistance. Though inspired by North America's second-wave feminists, Filipino women were also drawn to the era's discourses and contexts, such as the Vietnam War and the civil rights movement.\n\nThe women's movement continued to flourish in the Cory Aquino regime (1986-1992). The democratic transition provided political opportunity structures and venues ensuring women's access to the state and nonstate spheres. The drafting of the 1987 Constitution was one such opportunity. The movement managed to advocate for important provisions paving the way for women's rights legislation from the 1980s to the present. The provision in the 1987 Constitution mandates the state to recognize 'the role of women in nation building and shall ensure the fundamental equality before the law of men and women' (Article 2, Section 14). This provision is said to be unique and is not even found in other countries' charters (Masilungan n.d.). \n\nThe post-Marcos period advanced the participation of women not only in civil society and nongovernment organizations but also in formal politics and bureaucracy. Several women from the movement joined formal politics, while others were invited by the Aquino and Ramos governments (1992-1998) to executive posts. The entry of women activists, NGO leaders, and those from the academe ensured that the new democracy would significantly help push measures promoting women's rights and gender equality. The House of Representative (HOR) and Philippine Commission on Women (PCW)'s 'How to Be a Gender-Responsive Legislator' (2021, 52) listed several recent laws responding to women's empowerment and gender equality. \n\n• Republic Act No. 11313: Safe Spaces Act (April 17, 2019)\n\n• Republic Act No. 11210: 105-Day Expanded Maternity Leave Law (March 11, 2019)" + }, + { + "bleu": 0.9028780710785177, + "doc_id": "doc_97d60e0c292030da220648b4ce7e8971adbe29731b7077d9c3c8d7c6dd778e42_page_000001.png", + "edit_distance": 0.05454545454545454, + "f1_score": 0.9905956112852664, + "meteor": 0.9692291505951983, + "precision": 0.9875, + "pred_md": "Overcoming Barriers to Filipino Women's Political Representation\n\n11\n\n- · Republic Act No. 9501: Magna Carta for Micro, Small, and Medium Enterprises (May 23, 2008)\n- · Republic Act No. 9262: Anti-Violence Against Women and their Children Act of 2004 (March 8, 2004)\n- · Republic Act No. 9208 (May 26, 2003), as amended by Republic Act No. 10364 (February 6, 2013): Anti-Trafficking in Persons Act of 2003\n- · Republic Act No. 9178: Barangay Micro Business Enterprises Act of 2002 (November 13, 2002)\n- · Republic Act No. 8972: Solo Parent's Welfare Act (November 7, 2000)\n- · Republic Act No. 8505: Rape Victim Assistance and Protection Act (February 13, 1998)\n- · Republic Act No. 8504: Philippine AIDS Prevention and Control Act of 1998 (February 13, 1998)\n- · Republic Act No. 8353: Anti-Rape Law of 1997 (September 30, 1997)\n- · Republic Act No. 7877: Anti-Sexual Harassment Act of 1995 (February 14, 1995)\n\nDuring the first Aquino administration (1986-1992), three women sectoral representatives were appointed in Congress. Yet feminist activists such as Teresita Quintos-Deles and Jurgette Honculada's appointments were blocked by the House Committee on Appointments (Abao and Yang 2001, 19).\n\nWhile reliable electoral data during the Marcos regime is unavailable, it is safe to argue that the repressive regime hampered the participation of women in formal politics given the widespread militarization and electoral fraud characterizing the dictatorship. And even with the legal framework guaranteed by the transition, women found it difficult to enter formal politics, despite women's consistently high voter turnout during elections (Table 1).", + "recall": 0.9937106918238994, + "true_md": "Overcoming Barriers to Filipino Women's Political Representation\n\n11\n\n• Republic Act No. 9501: Magna Carta for Micro, Small, and Medium Enterprises (May 23, 2008)\n\n• Republic Act No. 9262: Anti-Violence Against Women and their Children Act of 2004 (March 8, 2004)\n\n• Republic Act No. 9208 (May 26, 2003), as amended by Republic Act No. 10364 (February 6, 2013): Anti-Trafficking in Persons Act of 2003 \n\n• Republic Act No. 9178: Barangay Micro Business Enterprises Act of 2002 (November 13, 2002)\n\n• Republic Act No. 8972: Solo Parent's Welfare Act (November 7, 2000)\n\n• Republic Act No. 8505: Rape Victim Assistance and Protection Act (February 13, 1998)\n\n• Republic Act No. 8504: Philippine AIDS Prevention and Control Act of 1998 (February 13, 1998) \n\n• Republic Act No. 8353: Anti-Rape Law of 1997 (September 30, 1997)\n\n• Republic Act No. 7877: Anti-Sexual Harassment Act of 1995 (February 14, 1995)\n\nDuring the first Aquino administration (1986-1992), three women sectoral representatives were appointed in Congress. Yet feminist activists such as Teresita Quintos-Deles and Jurgette Honculada's appointments were blocked by the House Committee on Appointments (Abao and Yang 2001, 19). \n\nWhile reliable electoral data during the Marcos regime is unavailable, it is safe to argue that the repressive regime hampered the participation of women in formal politics given the widespread militarization and electoral fraud characterizing the dictatorship. And even with the legal framework guaranteed by the transition, women found it difficult to enter formal politics, despite women's consistently high voter turnout during elections (Table 1)." + }, + { + "bleu": 0.9801394557335459, + "doc_id": "doc_4682941b1a9a3ec96599d8188b673e8c6d1c4f2a555b5dc0d739c6c706815553_page_000001.png", + "edit_distance": 0.011111111111111112, + "f1_score": 0.9871244635193132, + "meteor": 0.9894378752268703, + "precision": 0.9913793103448276, + "pred_md": "12\n\nEncinas Franco and Laguna\n\nTable 1: Percentage of Government Positions Held by Women During the Presidencies of Corazon Aquino and Fidel Ramos\n\nSource: Tancangco 1991 as cited in Valte (1992).\n\n## Current Situation: 2001-2019\n\nFilipino women are still very much a minority in the formal political sphere. It can also be observed that in executive positions such as the cabinet, few women are appointed, especially during President Fidel Ramos's time, compared to Cory Aquino's administration (Table 1). As mentioned above, the Philippines has made significant strides in legislating for women's rights. However, 35 years after redemocratization and 84 years after the grant of suffrage, participation of women in politics is still a work in progress, as in most countries.\n\nIn 2019, the overall percentage of women in all elective posts in the country was only about 20 percent (PCW 2021), barely reaching the 30 percent international requirement for women's political", + "recall": 0.9829059829059829, + "true_md": "12\n\nEncinas Franco and Laguna\n\nTable 1: Percentage of Government Positions Held by Women During the Presidencies of Corazon Aquino and Fidel Ramos\n\nSource: Tancangco 1991 as cited in Valte (1992). \n\n## Current Situation: 2001-2019\n\nFilipino women are still very much a minority in the formal political sphere. It can also be observed that in executive positions such as the cabinet, few women are appointed, especially during President Fidel Ramos's time, compared to Cory Aquino's administration (Table 1). As mentioned above, the Philippines has made significant strides in legislating for women's rights. However, 35 years after re- democratization and 84 years after the grant of suffrage, participation of women in politics is still a work in progress, as in most countries. \n\nIn 2019, the overall percentage of women in all elective posts in the country was only about 20 percent (PCW 2021), barely reaching the 30 percent international requirement for women's political " + }, + { + "bleu": 1.0, + "doc_id": "doc_f4dea73d2b75fbc7d590e2085b14e9bee7e766da73e14f723a2580fb0f36c707_page_000001.png", + "edit_distance": 0.0, + "f1_score": 1.0, + "meteor": 0.9999999436285809, + "precision": 1.0, + "pred_md": "Overcoming Barriers to Filipino Women's Political Representation\n\n15\n\nthe way for women to enter the House of Representatives. In 2019, 20 women from party lists have contributed to the increase in female legislators. However, the Party-List Law's implementation has been controversial owing to the entry of political dynasties and traditional politicians. The ideal that it serve as the gateway to political power of disadvantaged groups has been lost due to vague provisions in the law and subsequent Supreme Court decisions. The party list system has also been 'co-opted by the traditional political system or have become the training ground for future influence-peddling traditional politicians' (Tigno 2019). In other words, it has deviated from the idea of proportional representation practiced in other countries. Dynastic families took advantage of the system's flaws and used them to field relatives, including some women, to expand their political power. However, recent interviews with legislators from progressive party lists demonstrate a better understanding of women's issues than some representatives elected from single-member districts (Encinas-Franco 2022, 157).\n\nTable 2. Women-Members of the House of Representatives per Region, 2007-2019", + "recall": 1.0, + "true_md": "Overcoming Barriers to Filipino Women's Political Representation\n\n15\n\nthe way for women to enter the House of Representatives. In 2019, 20 women from party lists have contributed to the increase in female legislators. However, the Party-List Law's implementation has been controversial owing to the entry of political dynasties and traditional politicians. The ideal that it serve as the gateway to political power of disadvantaged groups has been lost due to vague provisions in the law and subsequent Supreme Court decisions. The party list system has also been 'co-opted by the traditional political system or have become the training ground for future influence-peddling traditional politicians' (Tigno 2019). In other words, it has deviated from the idea of proportional representation practiced in other countries. Dynastic families took advantage of the system's flaws and used them to field relatives, including some women, to expand their political power. However, recent interviews with legislators from progressive party lists demonstrate a better understanding of women's issues than some representatives elected from single-member districts (Encinas-Franco 2022, 157). \n\nTable 2. Women-Members of the House of Representatives per Region, 2007-2019 " + }, + { + "bleu": 1.0, + "doc_id": "doc_0ce1ded16152ad18c45203c1dcba0287235b888266fbcd112396fdf7cd094fab_page_000001.png", + "edit_distance": 0.0, + "f1_score": 1.0, + "meteor": 0.9999999452314623, + "precision": 1.0, + "pred_md": "16\n\nEncinas Franco and Laguna\n\nSource: HOR 2022. Computations made by the authors.\n\nOverall, the abovementioned situation indicates that Filipino women have gradually increased their presence in formal politics. In Asia, the Philippines and Taiwan are the only countries above the global average of 24.5 percent of women in parliament (Liu 2021). However, challenges remain as the increased participation of women comes from dysfunctional features of the country's political system: political dynasties and the Party-List law. Nonetheless, not all women from these groups are necessarily averse to women's issues.\n\n## Barriers to Filipino Women's Participation\n\nPrevious studies have identified political, economic, and cultural factors that impede women's participation in politics. However, context still matters since the perception of women's role in societies and the evolution of political systems differ. The following section examines some of these barriers.\n\nThe Philippine electoral system's 'first-past-the-post' electoral type, coupled with the lack of well-developed political parties, inhibits women's entry into politics. Encinas-Franco (2021) argues that '[w] ithout party discipline and institutionalized rules within parties, one", + "recall": 1.0, + "true_md": "16\n\nEncinas Franco and Laguna\n\nSource: HOR 2022. Computations made by the authors.\n\nOverall, the abovementioned situation indicates that Filipino women have gradually increased their presence in formal politics. In Asia, the Philippines and Taiwan are the only countries above the global average of 24.5 percent of women in parliament (Liu 2021). However, challenges remain as the increased participation of women comes from dysfunctional features of the country's political system: political dynasties and the Party-List law. Nonetheless, not all women from these groups are necessarily averse to women's issues. \n\n## Barriers to Filipino Women's Participation\n\nPrevious studies have identified political, economic, and cultural factors that impede women's participation in politics. However, context still matters since the perception of women's role in societies and the evolution of political systems differ. The following section examines some of these barriers. \n\nThe Philippine electoral system's 'first-past-the-post' electoral type, coupled with the lack of well-developed political parties, inhibits women's entry into politics. Encinas-Franco (2021) argues that '[w] ithout party discipline and institutionalized rules within parties, one " + }, + { + "bleu": 0.9794296992604028, + "doc_id": "doc_d09e9a559e05904e7a73e547f179ed617b016a94065e045b9f46bbe0e2ae8528_page_000001.png", + "edit_distance": 0.014705882352941176, + "f1_score": 0.9868766404199473, + "meteor": 0.9967908667657702, + "precision": 0.9791666666666666, + "pred_md": "EFB = empty fruit bunch. Source: Murdiyatmo (2021).\n\nHowever, the main obstacle with producing second-generation bioethanol is the cost of enzymes. Murdiyatmo (2021) stated that, at the pilot scale, the cost of enzymes is very high, i.e. Rp18,000 per litre of ethanol produced. Some studies provided the cost of enzymes in the US. NREL (2011), for instance, estimated that the cost of enzymes to produce second-generation bioethanol in the US was equivalent to around $0.34 per gallon or Rp1,529 per litre of ethanol produced, i.e. less than one-tenth of the cost of 2 enzymes in Indonesia.\n\nIn the next sub-sections, we analyse biodiesel and bioethanol introduction in Indonesia. In each sub-section, we first discuss the current supply and demand of the biofuels and the related conventional transport fuel. Second, we estimate the conventional transport fuel, i.e. gasoline and diesel fuel demand in road transportation during the period of 2020 50. Third, we estimate the volume of pure biofuel (fatty acid methyl ester -[FAME]/biodiesel and bioethanol) needs in scenarios, and in the amount of feedstock, i.e. CPO in biodiesel and molasses in bioethanol needed to meet the demand required in each scenario.\n\n## 2.1. Diesel and biodiesel use\n\nThe consumption of diesel fuel in Indonesia, used primarily for road freight transport, fluctuated between 2010 and 2019 as it correlated with the economic condition (Table 2.8). Diesel consumption in the industry sector decreased significantly, around 10% per year between 2010 and 2019, resulting from the shift to another energy type. During the same period, with some fluctuations, diesel production increased at 3.6% annual growth rate, while imports were cut by half from nearly 13 billion litres in 2010 to nearly 6.5 billion litres in 2018. The biodiesel blending rate increased from only 1% in 2010 to nearly 20% in 2019, representing a growing level of mandatory biodiesel programmes. Apparently, diesel imports dropped with the increase of the biodiesel (B100) blending rate.\n\n2 Assuming average inflation rate of 2% between 2011 and 2021 and an exchange rate of $1 = Rp14,131.\n\n11", + "recall": 0.9947089947089947, + "true_md": "EFB = empty fruit bunch. Source: Murdiyatmo (2021). \n\nHowever, the main obstacle with producing second-generation bioethanol is the cost of enzymes. Murdiyatmo (2021) stated that, at the pilot scale, the cost of enzymes is very high, i.e. Rp18,000 per litre of ethanol produced. Some studies provided the cost of enzymes in the US. NREL (2011), for instance, estimated that the cost of enzymes to produce second-generation bioethanol in the US was equivalent to around $0.34 per gallon or Rp1,529 2 per litre of ethanol produced, i.e. less than one-tenth of the cost of enzymes in Indonesia. \n\nIn the next sub-sections, we analyse biodiesel and bioethanol introduction in Indonesia. In each sub-section, we first discuss the current supply and demand of the biofuels and the related conventional transport fuel. Second, we estimate the conventional transport fuel, i.e. gasoline and diesel fuel demand in road transportation during the period of 2020-50. Third, we estimate the volume of pure biofuel (fatty acid methyl ester [FAME]/biodiesel and bioethanol) needs in scenarios, and in the amount of feedstock, i.e. CPO in biodiesel and molasses in bioethanol needed to meet the demand required in each scenario. \n\n## 2.1. Diesel and biodiesel use \n\nThe consumption of diesel fuel in Indonesia, used primarily for road freight transport, fluctuated between 2010 and 2019 as it correlated with the economic condition (Table 2.8). Diesel consumption in the industry sector decreased significantly, around 10% per year between 2010 and 2019, resulting from the shift to another energy type. During the same period, with some fluctuations, diesel production increased at 3.6% annual growth rate, while imports were cut by half from nearly 13 billion litres in 2010 to nearly 6.5 billion litres in 2018. The biodiesel blending rate increased from only 1% in 2010 to nearly 20% in 2019, representing a growing level of mandatory biodiesel programmes. Apparently, diesel imports dropped with the increase of the biodiesel (B100) blending rate. \n\n2 Assuming average inflation rate of 2% between 2011 and 2021 and an exchange rate of $1 = Rp14,131. " + }, + { + "bleu": 1.0, + "doc_id": "doc_275419814c80169ba0ca779343d8cf778903225b34dd40d0d63ef119b9892f78_page_000001.png", + "edit_distance": 0.0, + "f1_score": 1.0, + "meteor": 0.9999999929535726, + "precision": 1.0, + "pred_md": "pharmaceutical products (Casson, Muliastra, and Obidzinski, 2014). The development of biofuels from biomass has raised interest in expanding the palm oil plantation area. This is because palm oil is the main raw material for biodiesel in Indonesia.\n\nCPO is the primary product derived from the red fruit of the oil palm, while palm kernel oil, derived from the fruit's nut, is considered a secondary product. Oil palm biomass includes EFBs, palm mesocarps fibres (PMFs), PKS, oil palm fronds, oil palm trunks, as well as palm oil mill effluent (POME). Oil palm fronds account for 70% of the total oil palm biomass produced, while EFB accounts for 10% and oil palm trunks account for only about 5% of the total biomass produced.\n\nAccording to Harahap et al. (2019), Indonesia housed 11 million hectares (Mha) of oil palm plantations and produced 31 million tonnes (Mt) of CPO in 2015. Oil extraction from palm fruits occurs in palm oil mills. One tonne (t) of CPO production results in nearly 5 t of solid biomass waste, including EFBs, PKSs, PMFs, and POME; see Figure 3.3. This implies that, in 2015, Indonesia produced around 155 Mt of palm biomass residue.\n\nFigure 3.3. Biomass Use in Oil Palm Industry\n\nSource: Harahap et al. (2019).\n\nRegarding the potential for biodiesel, the previous Table 2.10 projected the demand of FAME for both B30 and B40 mandates using the volume of diesel fuel needed for the road transport sector. As shown, the FAME demand will reach 19.1 million kL in 2040 for the B30 mandate and 25.4 million kL for the B40 mandate. The current FAME production capacity is 12.85 million kL, indicating a shortage of supply to meet the 2040 demand for both the B30 and B40 mandates.\n\nIncreasing the capacity for FAME production implies that the demand for domestic CPO will continue to increase. The estimated CPO required to produce FAME in 2040 is also calculated above (Table 2.11). The estimated CPO consumption for B30 and B40 mandate in 2040 will be 17.5 and 23.4 million tonnes, respectively. This was calculated based on\n\n24", + "recall": 1.0, + "true_md": "pharmaceutical products (Casson, Muliastra, and Obidzinski, 2014). The development of biofuels from biomass has raised interest in expanding the palm oil plantation area. This is because palm oil is the main raw material for biodiesel in Indonesia. \n\nCPO is the primary product derived from the red fruit of the oil palm, while palm kernel oil, derived from the fruit's nut, is considered a secondary product. Oil palm biomass includes EFBs, palm mesocarps fibres (PMFs), PKS, oil palm fronds, oil palm trunks, as well as palm oil mill effluent (POME). Oil palm fronds account for 70% of the total oil palm biomass produced, while EFB accounts for 10% and oil palm trunks account for only about 5% of the total biomass produced. \n\nAccording to Harahap et al. (2019), Indonesia housed 11 million hectares (Mha) of oil palm plantations and produced 31 million tonnes (Mt) of CPO in 2015. Oil extraction from palm fruits occurs in palm oil mills. One tonne (t) of CPO production results in nearly 5 t of solid biomass waste, including EFBs, PKSs, PMFs, and POME; see Figure 3.3. This implies that, in 2015, Indonesia produced around 155 Mt of palm biomass residue. \n\nFigure 3.3. Biomass Use in Oil Palm Industry \n\nSource: Harahap et al. (2019). \n\nRegarding the potential for biodiesel, the previous Table 2.10 projected the demand of FAME for both B30 and B40 mandates using the volume of diesel fuel needed for the road transport sector. As shown, the FAME demand will reach 19.1 million kL in 2040 for the B30 mandate and 25.4 million kL for the B40 mandate. The current FAME production capacity is 12.85 million kL, indicating a shortage of supply to meet the 2040 demand for both the B30 and B40 mandates. \n\nIncreasing the capacity for FAME production implies that the demand for domestic CPO will continue to increase. The estimated CPO required to produce FAME in 2040 is also calculated above (Table 2.11). The estimated CPO consumption for B30 and B40 mandate in 2040 will be 17.5 and 23.4 million tonnes, respectively. This was calculated based on \n\n24" + }, + { + "bleu": 0.9031916760944635, + "doc_id": "doc_5699486e354a93df094404bae0cd17381bfab180f9ab2b5c9f55428c07a106c5_page_000001.png", + "edit_distance": 0.05454545454545454, + "f1_score": 0.9867841409691629, + "meteor": 0.9692266354391271, + "precision": 0.9824561403508771, + "pred_md": "scheme helped the biomass power capacity to increase by more than double in 7 years. Under the FIT scheme, biomass fuels for power generation are grouped into six categories.\n\n- · General wood: sawmill residues, import wood such as pellets and chips, palm kernel shell (PKS) and palm trunk\n- · Liquid biomass: palm oil\n- · Unutilised wood: domestic thinned wood\n- · Construction wood waste: wood waste salvaged from construction and other wood materials\n- · Waste materials and other biomass: pruned branched, paper, food waste, waste cooking oil, and black liquor\n- · Biogas: methane derived from sewage sludge, manure, and food waste.\n\nWhile inexpensive biomass sources such as wood waste from construction and waste materials, were the main fuels under the RPS, the domestic unutilised wood and the general wood whose tariff rates are set higher increased specifically (Figure 4.1, 4.2).\n\nFigure 4.1. Approved Capacity under the FIT Scheme\n\nFIT = feed-in-tariff.\n\nNote: Liquid biomass approved under the FIT scheme between FY2012 and FY2017 is included in general wood and no liquid biomass has been approved since FY2018.\n\nSource: METI (2021a).\n\n30", + "recall": 0.9911504424778761, + "true_md": "scheme helped the biomass power capacity to increase by more than double in 7 years. Under the FIT scheme, biomass fuels for power generation are grouped into six categories. \n\n• General wood: sawmill residues, import wood such as pellets and chips, palm kernel shell (PKS) and palm trunk \n\n• Liquid biomass: palm oil \n\n• Unutilised wood: domestic thinned wood \n\n• Construction wood waste: wood waste salvaged from construction and other wood materials \n\n• Waste materials and other biomass: pruned branched, paper, food waste, waste cooking oil, and black liquor \n\n• Biogas: methane derived from sewage sludge, manure, and food waste. \n\nWhile inexpensive biomass sources such as wood waste from construction and waste materials, were the main fuels under the RPS, the domestic unutilised wood and the general wood whose tariff rates are set higher increased specifically (Figure 4.1, 4.2). \n\nFigure 4.1. Approved Capacity under the FIT Scheme \n\nFIT = feed-in-tariff.\n\nNote: Liquid biomass approved under the FIT scheme between FY2012 and FY2017 is included in general wood and no liquid biomass has been approved since FY2018. \n\nSource: METI (2021a).\n\n30" + }, + { + "bleu": 0.9860504963980316, + "doc_id": "doc_4c8f779a89fd348e058a875f9182d6b3070199531adaa187bc3a602b5f746df3_page_000001.png", + "edit_distance": 0.007326007326007326, + "f1_score": 1.0, + "meteor": 0.9999984272415138, + "precision": 1.0, + "pred_md": "Figure 4.2. Operating Capacity under the FIT Scheme\n\nFIT = feed-in-tariff.\n\nSource: METI (2021a).\n\nThe newly approved capacity has stagnated lately because some strict measures reduced the accumulated idle capacity in the revised FIT Act of 2017. For instance, developers are required to have entered into the grid connection agreement with a utility company for an FIT approval and to submit a business plan for assessment of feasibility and sustainability. As a result, the approved biomass power capacity is about 160MW on average in FY2018 and FY2019.\n\nA recent change in the FIT scheme is that new projects of biomass co-firing with coal in the category of unutilised wood, general wood, and construction wood waste are no longer eligible for the FIT scheme from FY2019. The data collected after implementation 4 of the FIT scheme revealed that the generation costs of these biomass co-firing with coal are lower than the estimated costs of conventional biomass power plants in terms of capital expenditures, operation and maintenance, and fuels. Hence, biomass co-firing with coal does not have a rationale to receive support through the FIT scheme since it could make profits without it. For reference, Figure 4.3 illustrates a biomass co-firing ratio of the major power utilities' coal-fired power plants. Nearly half of the coal-fired power plants co-combusted biomass in FY2019 and most of them are less than 1% ratio of biomass.\n\n4 Biomass of waste materials co-firing with coal is not eligible for the FIT scheme from FY2021.\n\n31", + "recall": 1.0, + "true_md": "Figure 4.2. Operating Capacity under the FIT Scheme\n\nFIT = feed-in-tariff.\n\nSource: METI (2021a).\n\nThe newly approved capacity has stagnated lately because some strict measures reduced the accumulated idle capacity in the revised FIT Act of 2017. For instance, developers are required to have entered into the grid connection agreement with a utility company for an FIT approval and to submit a business plan for assessment of feasibility and sustainability. As a result, the approved biomass power capacity is about 160MW on average in FY2018 and FY2019. \n\nA recent change in the FIT scheme is that new projects of biomass co-firing with coal in the category of unutilised wood, general wood, and construction wood waste are no longer eligible for the FIT scheme from FY2019. 4 The data collected after implementation of the FIT scheme revealed that the generation costs of these biomass co-firing with coal are lower than the estimated costs of conventional biomass power plants in terms of capital expenditures, operation and maintenance, and fuels. Hence, biomass co-firing with coal does not have a rationale to receive support through the FIT scheme since it could make profits without it. For reference, Figure 4.3 illustrates a biomass co-firing ratio of the major power utilities' coal-fired power plants. Nearly half of the coal-fired power plants co-combusted biomass in FY2019 and most of them are less than 1% ratio of biomass. \n\n4 Biomass of waste materials co-firing with coal is not eligible for the FIT scheme from FY2021.\n\n31" + }, + { + "bleu": 1.0, + "doc_id": "doc_f0ab823f66709631e6226a937a8d68b52bde1e7ff6ef28086563c0646867c769_page_000001.png", + "edit_distance": 0.0, + "f1_score": 1.0, + "meteor": 0.9999999365530463, + "precision": 1.0, + "pred_md": "## 3. Perspective of supply and demand balance of wood pellets and cost structure in Japan\n\nAccording to a survey taken by the Japan Woody Bioenergy Association in FY2018 (from April 2018 to March 2019) with 55 biomass power generators, more than half of fuel for biomass power generation is domestically produced wood biomass at present in Japan in terms of weight (Figure 4.5).\n\nFigure 4.5. Breakdown of Biomass Power Generation Fuel in Japan\n\nPKS = palm kernel shell.\n\nNote: The share of fuel calculated in terms of biomass fuel weight ('Wood pellets', 'Construction wood waste',\n\n'Waste materials', 'Others': tonne; others: dry tonne).\n\nSource: Depicted by IEEJ based on Japan Woody Bioenergy Association (JWBA), 2020.\n\nWhen translating the survey result into energy form, it is estimated that, within biomass power generation using wood biomass ('Unutilised wood', 'General wood', and 'Construction wood waste'), around 30% of input fuel is met by import biomass fuel (Figure 4.6).\n\n38", + "recall": 1.0, + "true_md": "## 3. Perspective of supply and demand balance of wood pellets and cost structure in Japan \n\nAccording to a survey taken by the Japan Woody Bioenergy Association in FY2018 (from April 2018 to March 2019) with 55 biomass power generators, more than half of fuel for biomass power generation is domestically produced wood biomass at present in Japan in terms of weight (Figure 4.5). \n\nFigure 4.5. Breakdown of Biomass Power Generation Fuel in Japan \n\nPKS = palm kernel shell.\n\nNote: The share of fuel calculated in terms of biomass fuel weight ('Wood pellets', 'Construction wood waste', \n\n'Waste materials', 'Others': tonne; others: dry tonne). \n\nSource: Depicted by IEEJ based on Japan Woody Bioenergy Association (JWBA), 2020.\n\nWhen translating the survey result into energy form, it is estimated that, within biomass power generation using wood biomass ('Unutilised wood', 'General wood', and 'Construction wood waste'), around 30% of input fuel is met by import biomass fuel (Figure 4.6). \n\n38" + }, + { + "bleu": 1.0, + "doc_id": "doc_0687b8e7d87922833102a8e792d999184f73fd8ccbc147bba73ab036a28ca776_page_000001.png", + "edit_distance": 0.0, + "f1_score": 1.0, + "meteor": 0.9999998707974613, + "precision": 1.0, + "pred_md": "Figure 4.6. Input Biomass Fuel for Each Type of Biomass Power Generation\n\nPKS = palm kernel shell.\n\nHeat value used: Domestic logs and wood chips: 19.4 MJ/kg; Domestic wood pellets, Import pellets, chips: 15.5 MJ/kg; PKS: 18 MJ/kg; Construction wood waste, Other waste, and Others: assuming the same with wood pellets.\n\nSource: Depicted by IEEJ based on Japan Woody Bioenergy Association, 2020.\n\nAccording to Japan's trade statistics, its import of wood pellets has increased around 16 times from 2014 to 2019. Viet Nam and Canada are the largest suppliers of Japan's wood pellet imports (Figure 4.7). On the other hand, domestic wood pellet production stayed almost the same over the same period (Figure 4.8).\n\nFigure 4.7. Wood Pellets Import\n\nSource: Trade Statistics of Japan.\n\n39", + "recall": 1.0, + "true_md": "Figure 4.6. Input Biomass Fuel for Each Type of Biomass Power Generation \n\nPKS = palm kernel shell.\n\nHeat value used: Domestic logs and wood chips: 19.4 MJ/kg; Domestic wood pellets, Import pellets, chips: 15.5 MJ/kg; PKS: 18 MJ/kg; Construction wood waste, Other waste, and Others: assuming the same with wood pellets. \n\nSource: Depicted by IEEJ based on Japan Woody Bioenergy Association, 2020.\n\nAccording to Japan's trade statistics, its import of wood pellets has increased around 16 times from 2014 to 2019. Viet Nam and Canada are the largest suppliers of Japan's wood pellet imports (Figure 4.7). On the other hand, domestic wood pellet production stayed almost the same over the same period (Figure 4.8). \n\nFigure 4.7. Wood Pellets Import \n\nSource: Trade Statistics of Japan.\n\n39" + }, + { + "bleu": 1.0, + "doc_id": "doc_5179e446711cbe5535f394ec85fb83d4f25c56b7d180d0c3ce97878843afe59e_page_000001.png", + "edit_distance": 0.0, + "f1_score": 1.0, + "meteor": 0.9999999325679799, + "precision": 1.0, + "pred_md": "Figure 4.8. Domestic Wood Pellets Production\n\nSource: Forestry Agency, Ministry of Agriculture, Forestry and Fishery (MAFF), 2020.\n\nApplications of wood pellets in Japan include power generation, boilers, stoves, agriculture use, and others. Although the trade statistics do not specify the usage of the imported wood pellets, according to the Japan Wood Pellet Association (JPA), most are used for power generation.\n\nThe price of domestic wood pellets for power generation has a wide range. According to a survey of domestic wood pellet manufacturers undertaken by JPA in 2020, the average price of domestic wood pellets for power generation is around 14,000~29,000 ¥/tonne, while according to the Trade Statistics of Japan, the average cost, insurance, and freight (CIF) price of imported wood pellets is around 18,000 ¥/tonne in 2020 (Figure 4.9).\n\nFigure 4-9. Average Cost, Insurance, and Freight Prices of Wood Pellets and Wood Chips\n\nAverage price = import value/import tonne.\n\nSource: Estimated by IEEJ based on Trade Statistics of Japan.\n\n40", + "recall": 1.0, + "true_md": "Figure 4.8. Domestic Wood Pellets Production \n\nSource: Forestry Agency, Ministry of Agriculture, Forestry and Fishery (MAFF), 2020. \n\nApplications of wood pellets in Japan include power generation, boilers, stoves, agriculture use, and others. Although the trade statistics do not specify the usage of the imported wood pellets, according to the Japan Wood Pellet Association (JPA), most are used for power generation. \n\nThe price of domestic wood pellets for power generation has a wide range. According to a survey of domestic wood pellet manufacturers undertaken by JPA in 2020, the average price of domestic wood pellets for power generation is around 14,000~29,000 ¥/tonne, while according to the Trade Statistics of Japan, the average cost, insurance, and freight (CIF) price of imported wood pellets is around 18,000 ¥/tonne in 2020 (Figure 4.9). \n\nFigure 4-9. Average Cost, Insurance, and Freight Prices of Wood Pellets and Wood Chips \n\nAverage price = import value/import tonne.\n\nSource: Estimated by IEEJ based on Trade Statistics of Japan.\n\n40" + }, + { + "bleu": 0.9658380832974983, + "doc_id": "doc_20c3068a37794e1e2db40c6227d57172ad308c8a81815c4c08970219586ef4ee_page_000001.png", + "edit_distance": 0.017142857142857144, + "f1_score": 0.9942857142857142, + "meteor": 0.9982562023926599, + "precision": 0.9886363636363636, + "pred_md": "- iii. Looking at cost items, the cost of raw woods procurement will be highest share at 42%, followed by labour cost at 35%, electricity cost of the fabrication department at 10% (refer to figure 5-2). For this analysis, $35 per tonne is assumed for raw wood costs and this assumption will be crucial to maintain the economics of this business model.\n- iv. This business model will be operating cost-oriented not capital cost-oriented (refer to figure 5.1); thus, management of raw wood cost, labour cost, and electricity cost is essential. Few variations of capital cost will not affect this business seriously.\n- v. Assumed selling price of wood pellet is $100 per tonne and appropriate.\n\nFigure 5.1. Operating Cost Structure by the Three Departments of A Company\n\nSource: Author.\n\nFigure 5.2. Operating Cost Structure by the Cost Items of a Company\n\nSource: Author.\n\n50", + "recall": 1.0, + "true_md": "iii. Looking at cost items, the cost of raw woods procurement will be highest share at 42%, followed by labour cost at 35%, electricity cost of the fabrication department at 10% (refer to figure 5-2). For this analysis, $35 per tonne is assumed for raw wood costs and this assumption will be crucial to maintain the economics of this business model. \n\niv. This business model will be operating cost-oriented not capital cost-oriented (refer to figure 5.1); thus, management of raw wood cost, labour cost, and electricity cost is essential. Few variations of capital cost will not affect this business seriously. \n\n v. Assumed selling price of wood pellet is $100 per tonne and appropriate. \n\nFigure 5.1. Operating Cost Structure by the Three Departments of A Company \n\nSource: Author. \n\nFigure 5.2. Operating Cost Structure by the Cost Items of a Company \n\nSource: Author. \n\n50" + }, + { + "bleu": 0.9884197113412908, + "doc_id": "doc_e6023c1836d06942e984ed27b47b9de1677a107476544793a17058699a123435_page_000001.png", + "edit_distance": 0.006557377049180328, + "f1_score": 0.9901639344262295, + "meteor": 0.9937683057766045, + "precision": 0.993421052631579, + "pred_md": "## 1. Shipping as a vector for marine IAS List of Philippine Ports is in Appendix 3\n\nShipping remains as the only scientifically documented pathway for marine biological invasion in the Philippines with the introduction and invasion of the South American mussel Mytella strigata (Vallejo et al. 2017). This invasive was first recorded from the South Harbor of Manila in 2014 and has been known to have spread throughout Manila Bay, to Lingayen Gulf, Aparri, Cagayan and Batangas Port in the Philippines. It has since then reported in Singapore, Taiwan, Hong Kong, India, Malaysia, the Gulf of Thailand, and Sri Lanka.\n\nFigure 2 . Foulers from the South Harbor of Manila Bay. Photo by SAILS-PORTEC Manila Bay\n\nMytella was likely spread through hull fouling and ballast water release. In the Philippines its spread to other ports was likely through small vessel hull fouling as the first adult samples were recorded from the fishing boat FV Ocean in 2015 which was docked in Manila Bay. An intensive monitoring of the South Harbor area in 2014 resulted in the detection of the first cohort of recruits in Manila Bay. The likely first introduction by ballast water release or by biofouling was in December 2013 and the first cohort of recruits was detected in July 2014.\n\nThere are at least 15 marine non-indigenous species ship hull fouling recorded from Manila Bay's South Harbor (Vallejo et al. 2019; Trinidad et al 2017.) Only Mytella is considered invasive enough to have wide scale ecological and economic impacts. The most numerous species is the wellstudied Hydroides elegans , which is a known ship fouler with a present pantropical distribution.\n\n6", + "recall": 0.9869281045751634, + "true_md": "## 1. Shipping as a vector for marine IAS List of Philippine Ports is in Appendix 3 \n\nShipping remains as the only scientifically documented pathway for marine biological invasion in the Philippines with the introduction and invasion of the South American mussel Mytella strigata (Vallejo et al. 2017). This invasive was first recorded from the South Harbor of Manila in 2014 and has been known to have spread throughout Manila Bay, to Lingayen Gulf, Aparri, Cagayan and Batangas Port in the Philippines. It has since then reported in Singapore, Taiwan, Hong Kong, India, Malaysia, the Gulf of Thailand, and Sri Lanka. \n\nFigure 2. Foulers from the South Harbor of Manila Bay. Photo by SAILS-PORTEC Manila Bay \n\nMytella was likely spread through hull fouling and ballast water release. In the Philippines its spread to other ports was likely through small vessel hull fouling as the first adult samples were recorded from the fishing boat FV Ocean in 2015 which was docked in Manila Bay. An intensive monitoring of the South Harbor area in 2014 resulted in the detection of the first cohort of recruits in Manila Bay. The likely first introduction by ballast water release or by biofouling was in December 2013 and the first cohort of recruits was detected in July 2014. \n\nThere are at least 15 marine non-indigenous species ship hull fouling recorded from Manila Bay's South Harbor (Vallejo et al. 2019; Trinidad et al 2017.) Only Mytella is considered invasive enough to have wide scale ecological and economic impacts. The most numerous species is the well- studied Hydroides elegans, which is a known ship fouler with a present pantropical distribution. \n\n6 " + }, + { + "bleu": 0.9926194218256498, + "doc_id": "doc_9aae5aa80d0ada0a9a0c1306993e96dcbdbec4cdf48766f56955cf8d3d8c46a4_page_000001.png", + "edit_distance": 0.007407407407407408, + "f1_score": 0.9942857142857142, + "meteor": 0.999254080195038, + "precision": 0.9886363636363636, + "pred_md": "The other potentially invasive fouler is the tropical American Mytilopsis sallei and M. adamsi which has been recorded invasive in Singapore, Australia, Thailand among other regions. While they are recorded from the Manila South Harbor, there is no evidence that it is invasive as it exists in low abundances.\n\nFigure 3. Non-indigenous macrofoulers from Manila Bay with IAS, Mytilopsis sallei and Mytella strigata (=charruana). (From Trinidad et aL 2019)\n\nNewer estimates (2021) on the number of possible IAS in Manila Bay is likely more than 30 species based on more intensive biofouling ecological monitoring and the use environmental DNA in detecting species. When research started in 2006 on IAS in Manila Bay, 3 species were initially observed.\n\n7", + "recall": 1.0, + "true_md": "The other potentially invasive fouler is the tropical American Mytilopsis sallei and M. adamsi which has been recorded invasive in Singapore, Australia, Thailand among other regions. While they are recorded from the Manila South Harbor, there is no evidence that it is invasive as it exists in low abundances. \n\nFigure 3. Non-indigenous macrofoulers from Manila Bay with IAS, Mytilopsis sallei and Mytella strigata (=charruana). (From Trinidad et aL 2019) \n\nNewer estimates (2021) on the number of possible IAS in Manila Bay is likely more than 30 species based on more intensive biofouling ecological monitoring and the use environmental DNA in detecting species. When research started in 2006 on IAS in Manila Bay, 3 species were initially observed. " + }, + { + "bleu": 0.9882643885833784, + "doc_id": "doc_bdf0e09acee7a07f7e60c6055128967b36b44564842983ad3b0f273f1c1c5914_page_000001.png", + "edit_distance": 0.006756756756756757, + "f1_score": 0.990033222591362, + "meteor": 0.9935787554004252, + "precision": 0.9933333333333333, + "pred_md": "estuarine influenced areas. Batangas, Cebu and Iloilo are located very near to protected areas and tourism areas. Batangas is within the center of the center of global marine biodiversity while Cebu is in the Mactan key biodiversity area. Manila has the highest number of foreign shipcalls while Cebu has the highest domestic shipcalls and second to Manila in international shipcalls.\n\nTable 1. Top 10 ports in the Philippines in shipcalls (2020 data from PPA, CPA and SBMA)\n\nThe port of Manila has been documented to have a significant number of possible IAS. The ongoing SAILS-PORTEC research program has detected IAS in Davao, Cebu and Matnog ports. These ports are adjacent to specific oil tanker pathways/routes. In Luzon where the refineries and oil storage facilities are located such as Batangas, are at higher risk. These loading ports are at high risk for IAS/MNIS and these are located near to international ports.\n\nThe shipcall statistics in Table 1 represent the year 2020, when the COVID 19 pandemic caused a global and domestic maritime transport slowdown. The average reduction in shipcalls is around 40%. Nonetheless, Manila and Cebu are likely the main ports that need to be closely monitored for potential IAS bioinvasion. In 2018, before the COVID-19 pandemic, Manila was experiencing port congestion with a report that ships may stay at berth for five days (Wallis, 2019). This will increase the risks for biofouling. Based on the 2021 statistics from the PPA, the average berthing time has been reduced to 1 day. This is a result of less shipping traffic due to the pandemic.\n\n10", + "recall": 0.9867549668874173, + "true_md": "estuarine influenced areas. Batangas, Cebu and Iloilo are located very near to protected areas and tourism areas. Batangas is within the center of the center of global marine biodiversity while Cebu is in the Mactan key biodiversity area. Manila has the highest number of foreign shipcalls while Cebu has the highest domestic shipcalls and second to Manila in international shipcalls. \n\nTable 1. Top 10 ports in the Philippines in shipcalls (2020 data from PPA, CPA and SBMA) \n\nThe port of Manila has been documented to have a significant number of possible IAS. The on- going SAILS-PORTEC research program has detected IAS in Davao, Cebu and Matnog ports. These ports are adjacent to specific oil tanker pathways/routes. In Luzon where the refineries and oil storage facilities are located such as Batangas, are at higher risk. These loading ports are at high risk for IAS/MNIS and these are located near to international ports. \n\nThe shipcall statistics in Table 1 represent the year 2020, when the COVID 19 pandemic caused a global and domestic maritime transport slowdown. The average reduction in shipcalls is around 40%. Nonetheless, Manila and Cebu are likely the main ports that need to be closely monitored for potential IAS bioinvasion. In 2018, before the COVID-19 pandemic, Manila was experiencing port congestion with a report that ships may stay at berth for five days (Wallis, 2019). This will increase the risks for biofouling. Based on the 2021 statistics from the PPA, the average berthing time has been reduced to 1 day. This is a result of less shipping traffic due to the pandemic. \n\n10 " + }, + { + "bleu": 1.0, + "doc_id": "doc_edafb0db61c16517e87d60ab4d1216bbabb43bcb39779d21707ab4b8ccf1e81c_page_000001.png", + "edit_distance": 0.0, + "f1_score": 1.0, + "meteor": 0.9999999614728914, + "precision": 1.0, + "pred_md": "Figure 6. Mytella strigata biofouling green mussel farms in Bacoor City, Cavite, Manila Bay Photo from https://businessmirror.com.ph/2020/02/17/fake-tahong-invades-bacoor-mussel-farms/\n\n## 5. Natural dispersal\n\nDispersal by purely natural means is not included as a pathway of biological invasions (Gaston 1996). Examples include range expansion by flight or any other medium of natural locomotion or transport. However if human created or crafted material is involved in rafting dispersal of IAS, then this may be considered as a case of biological invasion. The 2011 Great East Japan earthquake generated a large tsunami that caused an unprecedented biological transoceanic rafting event from the northwestern Pacific coastline of Japan towards North America on the eastern Pacific(Carlton et al. 2017). Millions of human made objects from small plastics to large docks and whole ships were cast adrift in the Pacific (Murray et al. 2018). This provided a substrate for biofoulers. Large debris could carry up to 20 to 30 mega-species of biofoulers (Carlton et al. 2017). These biofouled debris can constitute an IAS risk (Therriault 2017).\n\nWhile a tsunami is a relatively rare event, a more common one is fouler dispersal by rafting on coastal currents of floating plastic debris, wood and, bamboo. Marine litter often originate from\n\n14", + "recall": 1.0, + "true_md": "Figure 6. Mytella strigata biofouling green mussel farms in Bacoor City, Cavite, Manila Bay Photo from https://businessmirror.com.ph/2020/02/17/fake-tahong-invades-bacoor-mussel-farms/ \n\n## 5. Natural dispersal \n\nDispersal by purely natural means is not included as a pathway of biological invasions (Gaston 1996). Examples include range expansion by flight or any other medium of natural locomotion or transport. However if human created or crafted material is involved in rafting dispersal of IAS, then this may be considered as a case of biological invasion. The 2011 Great East Japan earthquake generated a large tsunami that caused an unprecedented biological transoceanic rafting event from the northwestern Pacific coastline of Japan towards North America on the eastern Pacific(Carlton et al. 2017). Millions of human made objects from small plastics to large docks and whole ships were cast adrift in the Pacific (Murray et al. 2018). This provided a substrate for biofoulers. Large debris could carry up to 20 to 30 mega-species of biofoulers (Carlton et al. 2017). These biofouled debris can constitute an IAS risk (Therriault 2017). \n\nWhile a tsunami is a relatively rare event, a more common one is fouler dispersal by rafting on coastal currents of floating plastic debris, wood and, bamboo. Marine litter often originate from \n\n14 " + }, + { + "bleu": 0.919776608417899, + "doc_id": "doc_c9c49f563c6f47d9c28f9caf13455ef61345a0c9be4bba251d064958f546c6f5_page_000001.png", + "edit_distance": 0.043478260869565216, + "f1_score": 0.9763779527559056, + "meteor": 0.9750785668633973, + "precision": 0.9738219895287958, + "pred_md": "consumption onsite or offsite. Food Service Establishments (FSE) refers to the business engaged in the Food Service Industry. For purposes of the survey, the FSE is segmented into:\n\n- · full-service restaurants, with full menu and waiting service;\n- · limited-service restaurants or quick service restaurants (QSR), with full menu but pay-as-you-order such as fast food or turo-turo type 8;\n- · cafes/bars/pop-ups (selected menu with few chairs and tables);\n- · kiosks and stalls (purely retail, to be consumed elsewhere); and\n- · catering or 100% home delivery.\n\nFull-service restaurants, limited-service restaurants and cafes/bars/pop-ups may also offer 'to go' or 'take away' services.\n\nFigure 1. FSI Segmentation\n\n- b. Plastic. The Baseline Study looked into the extent of Plastic use of FSEs in Dasmariñas City. Plastics are categorized by food grade. The six food grades are 1) Polyethylene 9 Terephthalate: clear, tough plastic such as soft drinks, juice and water, (2) High Density Polyethylene: white or colored plastic such as milk containers, (3) Polyvinyl Chloride: hard rigid clear plastic such as cordial bottles; (4) Low Density Polyethylene: soft, flexible such as squeezable bottles; 5) Polypropylene: hard but flexible plastics such as microwave ware; takeaway containers, some yogurt or jam containers and hinged lunch boxes, and (6) Polystyrene: rigid, brittle plastics such as small tubes and margarine or butter container. See Figure 1 . Plastic litter found in the rivers are of categories 1-6. There are also other plastics that do not fall under food grade 1-6.\n\n8 Filipino word for restaurants where a menu of cooked or ready-to-eat food are on display and clients point to their choice of food and pay as they take their food to their tables or ask for take-out packaging.\n\n9 Food grade plastics refer to plastic containers, tools or other supplies made of plastics that are cleared to be used for food preparation, handling, and service.\n\n18\n\nStudy on Plastics Use and Waste Management in the Food Service Industry", + "recall": 0.9789473684210527, + "true_md": "consumption onsite or o/ffsite. Food Service Establishments (FSE) refers to the business engaged in the Food Service Industry. For purposes of the survey, the FSE is segmented into:\n\n• full-service restaurants, with full menu and waiting service;\n\n• limited-service restaurants or quick service restaurants (QSR), with full menu but pay-as-you-order such as fast food or turo-turo type 8;\n\n• cafes/bars/pop-ups (selected menu with few chairs and tables);\n\n• kiosks and stalls (purely retail, to be consumed elsewhere); and\n\n• catering or 100% home delivery.\n\nFull-service restaurants, limited-service restaurants and cafes/bars/pop-ups may also o/ffer 'to go' or 'take away' services.\n\nFigure 1. FSI Segmentation\n\nb. Plastic. The Baseline Study looked into the extent of Plastic use of FSEs in Dasmariñas City. Plastics are categorized by food grade. 9 The six food grades are 1) Polyethylene Terephthalate: clear, tough plastic such as soft drinks, juice and water, (2) High Density Polyethylene: white or colored plastic such as milk containers, (3) Polyvinyl Chloride: hard rigid clear plastic such as cordial bottles; (4) Low Density Polyethylene: soft, /flexible such as squeezable bottles; 5) Polypropylene: hard but /flexible plastics such as microwave ware; takeaway containers, some yogurt or jam containers and hinged lunch boxes, and (6) Polystyrene: rigid, brittle plastics such as small tubes and margarine or butter container. See Figure 1. Plastic litter found in the rivers are of categories 1-6. There are also other plastics that do not fall under food grade 1-6. \n\n8 Filipino word for restaurants where a menu of cooked or ready-to-eat food are on display and clients point to their choice of food and pay as they take their food to their tables or ask for take-out packaging.\n\n9 Food grade plastics refer to plastic containers, tools or other supplies made of plastics that are cleared to be used for food preparation, handling, and service.\n\n18\n\nStudy on Plastics Use and Waste Management in the Food Service Industry" + }, + { + "bleu": 0.9606193082355066, + "doc_id": "doc_30628cf0d5ec4d1d9dd94853d0711fdf496cbaa199fcab19f6fd4a4ddd34965d_page_000001.png", + "edit_distance": 0.07575757575757576, + "f1_score": 1.0, + "meteor": 0.9984907989528191, + "precision": 1.0, + "pred_md": "very much interested to know more about plastics as well as the plastics types that can be reused or recycled. Almost all respondents (87.8% ) are interested in approaches to recycle plastics. 87% (20) are interested in improving waste management systems in their LGUs.\n\n- d. Awareness of Plastics Ordinance. About 68% of respondents know that there is a city ordinance on plastics, while 52% are aware of the provincial plastic ordinance. 9% do not know of any ordinance and 17% do not know whether or not there is a plastic ordinance. In the same way, only 70% knows of the implementation of an ordinance regulating or prohibiting Single Use Plastics. 30% of the respondents are not aware of the ordinance.\n\n## 6.2 Waste Management\n\n- a. Waste Management Fee Collection. At the Barangay level, only 5 respondent barangays - Sampaloc II, H-2, Salitran-II, San Roque-Sta. Cristina II, and Salawag - collect waste management fees.\n- b. Waste Management Budget. Majority of the respondents (44%) do not know the budget allocation of their LGUS for waste management. 12% of respondents replied that their LGUs have no allocation for waste management while 32% of respondents replied that their budget allocation is below 5% of their LGU budget. Only 8% of respondents replied that their budget allocation for waste management is between 10-20% if the LGU budget. See Figure 20 .\n- c. Waste Collection and Segregation. For 70% of the respondents, wastes are collected by the city government. 35% responded that barangays collect their wastes and still,\n\nFigure 20. Percentage of LGU Budget Allocated for Waste Management\n\nStudy on Plastics Use and Waste Management in the Food Service Industry\n\n49", + "recall": 1.0, + "true_md": "very much interested to know more about plastics as well as the plastics types that can be reused or recycled. Almost all respondents (87.8% ) are interested in approaches to recycle plastics. 87% (20) are interested in improving waste management systems in their LGUs. \n\nd. Awareness of Plastics Ordinance. About 68% of respondents know that there is a city ordinance on plastics, while 52% are aware of the provincial plastic ordinance. 9% do not know of any ordinance and 17% do not know whether or not there is a plastic ordinance. In the same way, only 70% knows of the implementation of an ordinance regulating or prohibiting Single Use Plastics. 30% of the respondents are not aware of the ordinance. \n\n## 6.2 Waste Management\n\na. Waste Management Fee Collection. At the Barangay level, only 5 respondent barangays - Sampaloc II, H-2, Salitran-II, San Roque-Sta. Cristina II, and Salawag - collect waste management fees.\n\nb. Waste Management Budget. Majority of the respondents (44%) do not know the budget allocation of their LGUS for waste management. 12% of respondents replied that their LGUs have no allocation for waste management while 32% of respondents replied that their budget allocation is below 5% of their LGU budget. Only 8% of respondents replied that their budget allocation for waste management is between 10-20% if the LGU budget. See Figure 20.\n\nFigure 20. Percentage of LGU Budget Allocated for Waste Management\n\nc. Waste Collection and Segregation. For 70% of the respondents, wastes are collected by the city government. 35% responded that barangays collect their wastes and still, \n\nStudy on Plastics Use and Waste Management in the Food Service Industry\n\n49" + }, + { + "bleu": 0.9190843670954895, + "doc_id": "doc_32398c8e13acf26f1de76c730eaf6f57742037357a0eb25338b1fd7a5b816bf1_page_000001.png", + "edit_distance": 0.024336283185840708, + "f1_score": 0.9655172413793105, + "meteor": 0.9780868348635601, + "precision": 0.967479674796748, + "pred_md": "The World Bank/PEMSEA Assessment of Policies and Regulations to Guide Country Dialogue at National Level to Reduce Plastic Waste in the Philippines indicated:\n\n'Despite these efforts, there seemed to be very limited information that shows the effectiveness of the bans on reducing plastics and litter, or even diversion from landfills in the country. For the majority of LGUs in the country, however, there seemed to be no clear documentation and reporting of progress and updated waste data possibly due to the difficulty and complexity of data generation and assessment. Another possible constraint is that the scope of the LGU ordinances vary and covered different kinds of SUPP, including the exemptions, which makes integration of the various reports, if available, a challenge.'\n\nThe World Bank/PEMSEA report also recommended that a baseline assessment be conducted to obtain a better understanding which SUPP are the most prevalent and problematic in the Philippines and to also identify the sources and extent and impacts of mismanagement.\n\n- b. Extended producer responsibility (EPR). EPR schemes use a combination of regulatory approaches to extend manufacturers' responsibility for single-use plastic products throughout their life cycle, including to the end-of-life stage. These schemes are aimed at decreasing the overall environmental impact from a product and its packaging. The primary responsibility under EPR lies with the producer, who makes design and marketing decisions. In most European countries, product manufacturers are charged a fee for every piece of packaging they put onto the market based on the reusability or recyclability of the packaging, supported by technical analysis. These fees are intended to cover some or all of the costs of collection, sorting and recycling. Since the recycling of plastic packaging costs more than it yields, companies will benefit from a more costeffective system of packaging.\n- c. Regulated Storage, Manufacture and Use of plastics. India required its states to enforce existing rules on the storage, manufacture, and use of some single-use plastics in lieu of a nationwide ban. Meanwhile, the Department of Environment and Natural Resources (DENR) is yet to issue a list of non-environmentally accepted products (NEAP) as provided in Republic Act 9003 or the Ecological Solid Waste Management Act, passed a decade ago. This will include single use plastics in all product forms per technical advice of the Department of Science and\n\nFigure 27. Soft drinks can with the message 'Recycle Me'\n\n64\n\nStudy on Plastics Use and Waste Management in the Food Service Industry", + "recall": 0.9635627530364372, + "true_md": "The World Bank/PEMSEA Assessment of Policies and Regulations to Guide Country Dialogue at National Level to Reduce Plastic Waste in the Philippines indicated: \n\n'Despite these e/fforts, there seemed to be very limited information that shows the e/ffectiveness of the bans on reducing plastics and litter, or even diversion from land/fills in the country. For the majority of LGUs in the country, however, there seemed to be no clear documentation and reporting of progress and updated waste data possibly due to the di/fficulty and complexity of data generation and assessment. Another possible constraint is that the scope of the LGU ordinances vary and covered di/fferent kinds of SUPP, including the exemptions, which makes integration of the various reports, if available, a challenge.'\n\nThe World Bank/PEMSEA report also recommended that a baseline assessment be conducted to obtain a better understanding which SUPP are the most prevalent and problematic in the Philippines and to also identify the sources and extent and impacts of mismanagement.\n\nb. Extended producer responsibility (EPR). EPR schemes use a combination of regulatory approaches to extend manufacturers' responsibility for single-use plastic products throughout their life cycle, including to the end-of-life stage. These schemes are aimed at decreasing the overall environmental impact from a product and its packaging. The primary responsibility under EPR lies with the producer, who makes design and marketing decisions. In most European countries, product manufacturers are charged a fee for every piece of packaging they put onto the market based on the reusability or recyclability of the packaging, supported by technical analysis. These fees are intended to cover some or all of the costs of collection, sorting and recycling. Since the recycling of plastic packaging costs more than it yields, companies will bene/fit from a more cost- e/ffective system of packaging.\n\n## c. Regulated Storage, Manufacture and Use of \n\nplastics. India required its states to enforce existing rules on the storage, manufacture, and use of some single-use plastics in lieu of a nationwide ban. Meanwhile, the Department of Environment and Natural Resources (DENR) is yet to issue a list of non-environmentally accepted products (NEAP) as provided in Republic Act 9003 or the Ecological Solid Waste Management Act, passed a decade ago. This will include single use plastics in all product forms per technical advice of the Department of Science and \n\n Figure 27. Soft drinks can with the message 'Recycle Me'\n\n64\n\nStudy on Plastics Use and Waste Management in the Food Service Industry" + }, + { + "bleu": 0.9334691747538143, + "doc_id": "doc_42a416ed4de970abf9cc8ed74dbc543df8d7cc6ed24547189c0e8c77e5fed1d5_page_000001.png", + "edit_distance": 0.03480278422273782, + "f1_score": 0.9714285714285715, + "meteor": 0.9818546803664715, + "precision": 0.9692982456140351, + "pred_md": "## Replace\n\n- l. Replace Plastics with Recyclable Materials. Plastics can be replaced by material made from polypropylene, a material type that is 100% recyclable. However, recyclable materials should have a forward linkage - link to a recycler who is willing to take on the recyclables. Paper-based wrappers are another alternative for bagels and sandwich papers. Containers and packaging can use plastics with a certain percentage of recycled content and designed to be recyclable or reusable. Highly recyclable packaging is of little benefit if it is not disposed of correctly. The success of a recyclable package is an equal demand from recycling companies through improved recyclability of packaging and investments in efficient recycling facilities and systems. This requires investment and innovation since quality and availability are still often a stumbling block for companies to use recycled plastic. The recyclability of plastic packaging can often be improved by:\n- · choosing a common type of plastic (such as PE, PP or PET);\n- · choosing a common color (white or transparent); and\n- · avoiding combinations of materials, such as plastic windows in cardboard packaging. Watermarking technology is also being developed so that packaging can be more easily recognized by sorters.\n\n## Trash\n\n- m. Waste Segregation and Segregated Bins. Shakey's Philippines implementation of waste segregation and 3R (Reduce, Reuse, Recycle) in its corporate office is one good testament of compliance to RA 9003. The country's premier pizza restaurant has installed 'Stop Before You Drop' trash bins for the implementation of company-wide proper waste management. The bins are labeled to indicate the different types of waste to aid in proper disposal and culture development of its employees. Waste collected are weighed on a daily basis to aid in monitoring wastages and to map out more waste management initiatives. 56\n\n## n. In-store Sorting and Recycling Bins.\n\n- McDonalds has installed sorting and recycling points in select restaurants in its markets. It also improved its recycling bin signage to make the recycling process easier to understand. McDonald's Germany, Austria, Czech Republic and Slovakia on the other hand, collect customer waste to sort for recycling. initiatives. 57\n\nFigure 32. In-store Sorting and Recycling Bins, McDonalds\n\n56 https://www.shakeyspizza.ph/images/asm-2021/PIZZA\\_ASM\\_2020\\_Report.pdf\n\n57 https://corporate.mcdonalds.com/corpmcd/our-purpose-and-impact/our-planet/packaging-and-waste.html\n\n76\n\nStudy on Plastics Use and Waste Management in the Food Service Industry", + "recall": 0.973568281938326, + "true_md": "## Replace\n\nl. Replace Plastics with Recyclable Materials. Plastics can be replaced by material made from polypropylene, a material type that is 100% recyclable. However, recyclable materials should have a forward linkage - link to a recycler who is willing to take on the recyclables. Paper-based wrappers are another alternative for bagels and sandwich papers. Containers and packaging can use plastics with a certain percentage of recycled content and designed to be recyclable or reusable. Highly recyclable packaging is of little bene/fit if it is not disposed of correctly. The success of a recyclable package is an equal demand from recycling companies through improved recyclability of packaging and investments in e/fficient recycling facilities and systems. This requires investment and innovation since quality and availability are still often a stumbling block for companies to use recycled plastic. The recyclability of plastic packaging can often be improved by:\n\n• choosing a common type of plastic (such as PE, PP or PET);\n\n• choosing a common color (white or transparent); and\n\n• avoiding combinations of materials, such as plastic windows in cardboard packaging. Watermarking technology is also being developed so that packaging can be more easily recognized by sorters.\n\n## Trash\n\nm. Waste Segregation and Segregated Bins. Shakey's Philippines implementation of waste segregation and 3R (Reduce, Reuse, Recycle) in its corporate o/ffice is one good testament of compliance to RA 9003. The country's premier pizza restaurant has installed 'Stop Before You Drop' trash bins for the implementation of company-wide proper waste management. The bins are labeled to indicate the di/fferent types of waste to aid in proper disposal and culture development of its employees. Waste collected are weighed on a daily basis to aid in monitoring wastages and to map out more waste management initiatives. 56\n\n## n. In-store Sorting and Recycling Bins.\n\nMcDonalds has installed sorting and recycling points in select restaurants in its markets. It also improved its recycling bin signage to make the recycling process easier to understand. McDonald's Germany, Austria, Czech Republic and Slovakia on the other hand, collect customer waste to sort for recycling. initiatives. 57\n\nFigure 32. In-store Sorting and Recycling Bins, McDonalds\n\n56 https://www.shakeyspizza.ph/images/asm-2021/PIZZA\\_ASM\\_2020\\_Report.pdf\n\n57 https://corporate.mcdonalds.com/corpmcd/our-purpose-and-impact/our-planet/packaging-and-waste.html\n\n76\n\nStudy on Plastics Use and Waste Management in the Food Servic" + }, + { + "bleu": 1.0, + "doc_id": "doc_9899a637992e35363176c7b7aca0633879da307327f5c434122856c639a6e9f1_page_000001.png", + "edit_distance": 0.0, + "f1_score": 1.0, + "meteor": 0.99999941682461, + "precision": 1.0, + "pred_md": "two meetings are related to the initial meeting of VNR and as particular human rights focus. 73\n\nDiagram 2\n\nParticipation of Institutions in the VNR Meeting of\n\nIndonesia 2021. 74\n\nThe distribution of participating institutions in VNR-related meetings are as follows:\n\nDiagram 3\n\nDistribution of Participating Institutions within VNR Meeting of Indonesia 2021. 75\n\n74 Data is processed based on: ibid., 332-345.\n\n75 Data is processed based on: Kementerian PPN / Bappenas, 'Annexes Indonesia's VNR 2021' (n. 68) , 332-345.\n\n14", + "recall": 1.0, + "true_md": "two meetings are related to the initial meeting of VNR and as particular human rights focus. 73 \n\nDiagram 2 Participation of Institutions in the VNR Meeting of Indonesia 2021. 74 \n\nThe distribution of participating institutions in VNR-related meetings are as follows: \n\nDiagram 3 Distribution of Participating Institutions within VNR Meeting of Indonesia 2021. 75 \n\n74 Data is processed based on: ibid., 332-345. \n\n75 Data is processed based on: Kementerian PPN / Bappenas, 'Annexes Indonesia's VNR 2021' (n. 68), 332-345. \n\n14 " + }, + { + "bleu": 1.0, + "doc_id": "doc_8a87099365535d996d3167c933ff78e2890f96e70d4a8e2291d0c420fe7093ba_page_000001.png", + "edit_distance": 0.0, + "f1_score": 1.0, + "meteor": 0.9999999872786485, + "precision": 1.0, + "pred_md": "be used as a good opportunity to learn from each other and increase the capacity of human rights institutions in various countries. 94\n\nWhat works in other countries, can be learned and developed according to the situation in Indonesia. 95 Partnerships can be carried out formally through a memorandum of understanding or with a partnerships agreement for potential strategic partners. 96\n\n## 3.2.6. SDGs Dissemination in Social Media\n\nInformation dissemination in the digital era is closely related to the use of social media. Therefore, the dissemination of the SDGs through social media platforms owned by the Komnas HAM needs to be optimized as a way to increase public participation to be active as 'agents' of the Komnas HAM in Indonesia. To be able to achieve this, the community needs to first receive education about the SDGs to clearly understand the focus of each goal and its derivatives. Once there is a fairly good understanding at the level of the general public, especially those who interact with the Komnas HAM's social media, an easier way to report SDGs related to human rights violations can be formulated.\n\nThe Komnas HAM, for example, has used social media Instagram, Twitter, and YouTube. There has been an increase in the frequency of Instagram social media uploads from 2019-2020 from 111 uploads in 2019 to 198 uploads in 2020. The variety of content uploaded by the Komnas HAM on Instagram is also increasingly diverse with the following details:\n\nDiagram 4\n\nDistribution of @komnas.ham Instagram Content (2019-2020)\n\nIf observed from the Komnas HAM's Instagram account within the 2019-2020 period, the SDGs have only been mentioned explicitly twice in the following contents:\n\n94 See also Komnas HAM, 'The NHRI Practice and Experience in Indonesia, Kyrgyzstan, and Palestine\n\nin Supporting Sustainable Development Goals Achievements' (n. 93).\n\n95 Ibid.\n\n96 Ibid.\n\n18", + "recall": 1.0, + "true_md": "be used as a good opportunity to learn from each other and increase the capacity of human rights institutions in various countries. 94 \n\n What works in other countries, can be learned and developed according to the situation in Indonesia. 95 Partnerships can be carried out formally through a memorandum of understanding or with a partnerships agreement for potential strategic partners. 96 \n\n## 3.2.6. SDGs Dissemination in Social Media \n\n Information dissemination in the digital era is closely related to the use of social media. Therefore, the dissemination of the SDGs through social media platforms owned by the Komnas HAM needs to be optimized as a way to increase public participation to be active as 'agents' of the Komnas HAM in Indonesia. To be able to achieve this, the community needs to first receive education about the SDGs to clearly understand the focus of each goal and its derivatives. Once there is a fairly good understanding at the level of the general public, especially those who interact with the Komnas HAM's social media, an easier way to report SDGs related to human rights violations can be formulated. \n\nThe Komnas HAM, for example, has used social media Instagram, Twitter, and YouTube. There has been an increase in the frequency of Instagram social media uploads from 2019-2020 from 111 uploads in 2019 to 198 uploads in 2020. The variety of content uploaded by the Komnas HAM on Instagram is also increasingly diverse with the following details: \n\nDiagram 4 Distribution of @komnas.ham Instagram Content (2019-2020)\n\n If observed from the Komnas HAM's Instagram account within the 2019-2020 period, the SDGs have only been mentioned explicitly twice in the following contents: \n\n94 See also Komnas HAM, 'The NHRI Practice and Experience in Indonesia, Kyrgyzstan, and Palestine in Supporting Sustainable Development Goals Achievements' (n. 93). \n\n95 Ibid. \n\n96 Ibid. \n\n18 " + }, + { + "bleu": 0.9628956085256684, + "doc_id": "doc_ce6853fb0ca94a72685ef90aa5ece8a180f95cd4a11386b2cf5fb8f29c35fe09_page_000001.png", + "edit_distance": 0.015748031496062992, + "f1_score": 0.9793103448275863, + "meteor": 0.9850255634357763, + "precision": 0.9861111111111112, + "pred_md": "Diagram 5\n\nDistribution of Komnas HAM's YouTube Content (20192020)\n\nAs of 1 December 2021, the Komnas HAM's YouTube channel has 2,290 subscribers with 185,676 total views. In the 2019-2020 period, content that specifically discusses the SDGs explicitly cannot be found on the Komnas HAM's YouTube. Nevertheless, on 15 December 2021, the Tanggap Rasa Podcast with the title of 'Podcast #EP32: SDGs dan Anak Muda' (Translation: 'Podcast #EP32: SDGs and Youth') has been broadcast and can increase the awareness and understanding of the citizen on the SDGs, especially towards young generations.\n\nFigure 4\n\nKomnas HAM's YouTube channel as of 1 December 2021\n\n21", + "recall": 0.9726027397260274, + "true_md": "Diagram 5 Distribution of Komnas HAM's YouTube Content (2019- 2020) \n\nAs of 1 December 2021, the Komnas HAM's YouTube channel has 2,290 subscribers with 185,676 total views. In the 2019-2020 period, content that specifically discusses the SDGs explicitly cannot be found on the Komnas HAM's YouTube. Nevertheless, on 15 December 2021, the Tanggap Rasa Podcast with the title of 'Podcast #EP32: SDGs dan Anak Muda' (Translation: 'Podcast #EP32: SDGs and Youth') has been broadcast and can increase the awareness and understanding of the citizen on the SDGs, especially towards young generations. \n\nFigure 4 Komnas HAM's YouTube channel as of 1 December 2021 \n\n21 " + }, + { + "bleu": 1.0, + "doc_id": "doc_f99de206dc21ba8ea57548b1c0c007eb736da5f26c925dc99a4027806297bd8f_page_000001.png", + "edit_distance": 0.0, + "f1_score": 1.0, + "meteor": 0.9999998823952235, + "precision": 1.0, + "pred_md": "In this content, DPN Argentina provides a brief explanation of the SDGs and the 2030 Agenda action plans, and most importantly, their role in advancing the 2030 Agenda through the SDGs Monitoring and Evaluation Program with a focus on certain thematic areas. These focuses allow DPN Argentina to investigate through monitoring and preparing reports on the development of public policies and actions of organizations responsible for compliance with the SDGs, as well as proposals, and recommendations to strengthen related processes.\n\nFurthermore, DPN Argentina also regularly uploads commemorations of days related to the SDGs by also including the SDGs logo in each of these uploads. Examples of such greetings are as follows:\n\nFigure 6\n\nDPN Argentina Content: World Health Day Celebration (7 April 2021). 98\n\n98 DPN Argentina, 'Día Mundial de la #Salud', accessed on 5 December 2021,https://twitter.com/D PNArgentina/status/1379765916259483648.\n\n23", + "recall": 1.0, + "true_md": " In this content, DPN Argentina provides a brief explanation of the SDGs and the 2030 Agenda action plans, and most importantly, their role in advancing the 2030 Agenda through the SDGs Monitoring and Evaluation Program with a focus on certain thematic areas. These focuses allow DPN Argentina to investigate through monitoring and preparing reports on the development of public policies and actions of organizations responsible for compliance with the SDGs, as well as proposals, and recommendations to strengthen related processes. \n\n Furthermore, DPN Argentina also regularly uploads commemorations of days related to the SDGs by also including the SDGs logo in each of these uploads. Examples of such greetings are as follows: \n\nFigure 6 DPN Argentina Content: World Health Day Celebration (7 April 2021). 98 \n\n98 DPN Argentina, 'Día Mundial de la #Salud', accessed on 5 December 2021,https://twitter.com/D PNArgentina/status/1379765916259483648. \n\n23 " + }, + { + "bleu": 0.9900352876807745, + "doc_id": "doc_f974132511fc3655d9602cf317b6ae2ed040787671bb21e387322961b5ac30f1_page_000001.png", + "edit_distance": 0.005649717514124294, + "f1_score": 0.9915966386554622, + "meteor": 0.9946311605451884, + "precision": 0.9943820224719101, + "pred_md": "Thailand, Malaysia, and Singapore. In these three countries, per capita GDP fell between 4 percent to 7 percent. 3\n\nFigure 1.2. Per capita GDP growth in 2020\n\nSource : World Bank (2022a)\n\nIt is also noteworthy that in two of these major destination countries - Thailand and Malaysia - the most-affected sectors were also ones heavily reliant on migrant workers. In Thailand, affected sectors include manufacturing, construction, agriculture, fishing, seafood processing, domestic work, and hospitality (United Nations Thematic Working Group, 2019; ILO, 2020). In Malaysia, migrant workers were, in 2019, especially prevalent in manufacturing (705,000), construction (435,000), services (306,000), plantation (282,000), agriculture (160,000), and domestic work (127,000) (Wahab, 2020a; Theng, Noor and Khalidi, 2020).\n\nThe construction sector in Malaysia crashed in the second quarter of 2020 and did not experience growth again until the second quarter of 2021, before suffering negative growth again the next quarter after a COVID-19 resurgence. Accommodation and dining establishments which includes many tourism-related jobs, fared even worse. Furthermore, wholesale trade and related activities in Malaysia have not recovered to pre-pandemic levels, even after growing in the first two quarters of 2021. In Thailand, the construction sector avoided a massive output decline similar to Malaysia's, although it did decline in the first quarter of 2020. However, manufacturing, accommodation, and wholesale trade in Thailand all suffered large contractions due to travel restrictions, supply chain disruptions, and weak aggregate demand, and, despite some recovery in the second quarter of 2021, remain well below prepandemic levels (Table 1.1).\n\n3 The Philippine economy was hit hardest because of the length and severity of the movement restrictions imposed in the country (Olanday and Rigby, 2020).\n\nASEAN Migration Outlook\n\n13", + "recall": 0.9888268156424581, + "true_md": "Thailand, Malaysia, and Singapore. In these three countries, per capita GDP fell between 4 percent to 7 percent. 3\n\nFigure 1.2. Per capita GDP growth in 2020\n\nSource: World Bank (2022a)\n\nIt is also noteworthy that in two of these major destination countries - Thailand and Malaysia - the most-affected sectors were also ones heavily reliant on migrant workers. In Thailand, affected sectors include manufacturing, construction, agriculture, fishing, seafood processing, domestic work, and hospitality (United Nations Thematic Working Group, 2019; ILO, 2020). In Malaysia, migrant workers were, in 2019, especially prevalent in manufacturing (705,000), construction (435,000), services (306,000), plantation (282,000), agriculture (160,000), and domestic work (127,000) (Wahab, 2020a; Theng, Noor and Khalidi, 2020).\n\nThe construction sector in Malaysia crashed in the second quarter of 2020 and did not experience growth again until the second quarter of 2021, before suffering negative growth again the next quarter after a COVID-19 resurgence. Accommodation and dining establishments which includes many tourism-related jobs, fared even worse. Furthermore, wholesale trade and related activities in Malaysia have not recovered to pre-pandemic levels, even after growing in the first two quarters of 2021. In Thailand, the construction sector avoided a massive output decline similar to Malaysia's, although it did decline in the first quarter of 2020. However, manufacturing, accommodation, and wholesale trade in Thailand all suffered large contractions due to travel restrictions, supply chain disruptions, and weak aggregate demand, and, despite some recovery in the second quarter of 2021, remain well below pre- pandemic levels (Table 1.1).\n\n3 The Philippine economy was hit hardest because of the length and severity of the movement restrictions imposed in the country (Olanday and Rigby, 2020).\n\nASEAN Migration Outlook\n\n13" + }, + { + "bleu": 1.0, + "doc_id": "doc_a24a2d4ed5bc5e9aa585d4d29ea5b66737ea96bcaa84756d1b183e7c0ad77e91_page_000001.png", + "edit_distance": 0.0, + "f1_score": 1.0, + "meteor": 0.9999999938835304, + "precision": 1.0, + "pred_md": "2020 and 2021, and, for approximately half of AMS, working hours lost were higher in 2021 compared to 2020 (Figure 1.3). The disruptions in global supply chains because of travel and transport restrictions hit some AMS particularly hard because of supply needs from other countries.\n\nDespite these tremendous job losses, many countries also experienced labour shortages due to previously unprecedented demand for certain products, such as rubber gloves in Malaysia and for fishery products in Thailand. The return of migrant workers to their home countries contributed to significant labour shortages (Lee and David, 2021; Sriring and Staporncharnchai, 2021). 4 COVID-related movement restrictions caused many workers to withdraw from the labour force (especially women) and labour force participation rates declined in most countries. 5 This was the case for Indonesia, Malaysia, the Philippines, and Viet Nam (Figure 1.4). According to the ILO (2021c), female employment in AMS in 2020 was 3.9 percent lower than the expected level, which is markedly less than the 2.7 percent figure for male employment. 6 The impact of the pandemic on employment is evident in lower labour force participation, lower working hours, and higher unemployment rates in most countries (Figure 1.5).\n\nFigure 1.3. Decline in weekly working hours compared to 2019 (percent)\n\nSource : ILO (2022a)\n\n4 There are of course long-standing reasons for the labour shortages in these sectors, which accounts for their high reliance for migrant workers, including poor working conditions, that is prone to abuse, and lack of attractiveness for local workers (Looi, 2020; Ng, 2020; ILO, 2015).\n\n5 McKinsey Global Institute (2020) estimates that at the beginning of the pandemic, women accounted for more than half of total job losses from COVID-19 though they made up only two-fifths of the global labour force. This is because they are overrepresented in sectors hardest hit by the pandemic: accommodation and food services; retail and wholesale trade; and other services, such as arts, recreation, and public administration.\n\n6 This is equivalent to saying there is greater increase in unemployment or inactivity for women compared to men. According to the report, one reason is the increase in unpaid care responsibilities for women as schools closed (ILO, 2021c).\n\nASEAN Migration Outlook\n\n15", + "recall": 1.0, + "true_md": "2020 and 2021, and, for approximately half of AMS, working hours lost were higher in 2021 compared to 2020 (Figure 1.3). The disruptions in global supply chains because of travel and transport restrictions hit some AMS particularly hard because of supply needs from other countries.\n\nDespite these tremendous job losses, many countries also experienced labour shortages due to previously unprecedented demand for certain products, such as rubber gloves in Malaysia and for fishery products in Thailand. The return of migrant workers to their home countries contributed to significant labour shortages (Lee and David, 2021; Sriring and Staporncharnchai, 2021). 4 COVID-related movement restrictions caused many workers to withdraw from the labour force (especially women) and labour force participation rates declined in most countries. 5 This was the case for Indonesia, Malaysia, the Philippines, and Viet Nam (Figure 1.4). According to the ILO (2021c), female employment in AMS in 2020 was 3.9 percent lower than the expected level, which is markedly less than the 2.7 percent figure for male employment. 6 The impact of the pandemic on employment is evident in lower labour force participation, lower working hours, and higher unemployment rates in most countries (Figure 1.5).\n\nFigure 1.3. Decline in weekly working hours compared to 2019 (percent)\n\nSource: ILO (2022a)\n\n4 There are of course long-standing reasons for the labour shortages in these sectors, which accounts for their high reliance for migrant workers, including poor working conditions, that is prone to abuse, and lack of attractiveness for local workers (Looi, 2020; Ng, 2020; ILO, 2015).\n\n5 McKinsey Global Institute (2020) estimates that at the beginning of the pandemic, women accounted for more than half of total job losses from COVID-19 though they made up only two-fifths of the global labour force. This is because they are overrepresented in sectors hardest hit by the pandemic: accommodation and food services; retail and wholesale trade; and other services, such as arts, recreation, and public administration. \n\n6 This is equivalent to saying there is greater increase in unemployment or inactivity for women compared to men. According to the report, one reason is the increase in unpaid care responsibilities for women as schools closed (ILO, 2021c).\n\nASEAN Migration Outlook\n\n15" + }, + { + "bleu": 0.9103430263851291, + "doc_id": "doc_6d5a00597cdd1d363b56eb0fdc9f239a462b44fcb0be53a93d62f0d6e35d428f_page_000001.png", + "edit_distance": 0.046511627906976744, + "f1_score": 0.9892473118279569, + "meteor": 0.9975195046347914, + "precision": 0.9787234042553191, + "pred_md": "Figure 1.6. Alien temporary work permits, Thailand\n\nSource : Department of Employment, Thailand (2022)\n\nFigure 1.7. Non-citizen population in Malaysia (in thousands)\n\nSource : Department of Statistics, Malaysia (2022). Figure for 2021 is an estimate.\n\n## Figure 1.8. Singapore foreign workforce stock (in thousands)\n\n: Compilation by Manpower Research & Statistics Department (Ministry of Manpower,\n\nSource Singapore, 2022).\n\nASEAN Migration Outlook\n\n19", + "recall": 1.0, + "true_md": "Figure 1.6. Alien temporary work permits, Thailand\n\nSource: Department of Employment, Thailand (2022)\n\nFigure 1.7. Non-citizen population in Malaysia (in thousands)\n\nSource: Department of Statistics, Malaysia (2022). Figure for 2021 is an estimate.\n\nFigure 1.8. Singapore foreign workforce stock (in thousands)\n\nSource: Compilation by Manpower Research & Statistics Department (Ministry of Manpower, Singapore, 2022). \n\nASEAN Migration Outlook\n\n19" + }, + { + "bleu": 1.0, + "doc_id": "doc_a012af9597cad209a43dc6a6a4543994e857a56fac9fdd3eed92a5e1b93f79b2_page_000001.png", + "edit_distance": 0.0, + "f1_score": 1.0, + "meteor": 0.9999999882376601, + "precision": 1.0, + "pred_md": "decline in 2020 in absolute numbers and as a percentage of 2019 deployment (Figure 1.9b). 9\n\nFigure 1.9b. Deployment of Overseas Foreign Workers by sex, new hires only (in thousands)\n\nSource : Philippine Statistics Authority (2022)\n\n## 1.5. Migrant Workers More at Risk of COVID-19 Infection\n\nCOVID-19 infection among migrants appears to be higher than among non-migrant groups (Hintermeier et al., 2020). Migrant workers are disproportionately exposed to COVID-19 because of the nature of their work and their living conditions. Many migrant workers performed essential services, including jobs in healthcare, selected manufacturing, transportation, logistics, construction, and maintenance, which continued during periods of movement restrictions (OECD, ADBI and ILO, 2021). Many migrant workers also have less access to personal protective equipment and testing and treatment facilities (OECD, ADBI and ILO, 2021). The lack of access was especially true for undocumented migrants.\n\nAdditionally, migrant workers employed in plantations far away from urban centres had limited access to information and testing. High rates of infection were also linked to overcrowded housing conditions, including shared facilities and sleeping areas, which increase the risk of transmission (ASEAN MP , 2021). Many workers in processing or assembly plants worked in conditions where physical distancing was rarely observed.\n\nIn Malaysia, out of 2,188 positive cases recorded nationwide on 25 November 2020, 1,511 were foreign workers employed by Top Glove Corporation Bhd., one of the world's largest personal protective equipment (PPE) manufacturers ( The Straits Times , 2020; Ngui, 2020). Many other migrant workers were employed as delivery agents, public transport drivers, or restaurant waiters, and are in constant contact with the general public. Infection risk is also higher\n\n9 Keeping in mind that for 2020 the figures are only up to October of the year.\n\nASEAN Migration Outlook\n\n21", + "recall": 1.0, + "true_md": "decline in 2020 in absolute numbers and as a percentage of 2019 deployment (Figure 1.9b). 9\n\nFigure 1.9b. Deployment of Overseas Foreign Workers by sex, new hires only (in thousands)\n\nSource: Philippine Statistics Authority (2022)\n\n## 1.5. Migrant Workers More at Risk of COVID-19 Infection\n\nCOVID-19 infection among migrants appears to be higher than among non-migrant groups (Hintermeier et al., 2020). Migrant workers are disproportionately exposed to COVID-19 because of the nature of their work and their living conditions. Many migrant workers performed essential services, including jobs in healthcare, selected manufacturing, transportation, logistics, construction, and maintenance, which continued during periods of movement restrictions (OECD, ADBI and ILO, 2021). Many migrant workers also have less access to personal protective equipment and testing and treatment facilities (OECD, ADBI and ILO, 2021). The lack of access was especially true for undocumented migrants.\n\nAdditionally, migrant workers employed in plantations far away from urban centres had limited access to information and testing. High rates of infection were also linked to overcrowded housing conditions, including shared facilities and sleeping areas, which increase the risk of transmission (ASEAN MP, 2021). Many workers in processing or assembly plants worked in conditions where physical distancing was rarely observed. \n\nIn Malaysia, out of 2,188 positive cases recorded nationwide on 25 November 2020, 1,511 were foreign workers employed by Top Glove Corporation Bhd., one of the world's largest personal protective equipment (PPE) manufacturers (The Straits Times, 2020; Ngui, 2020). Many other migrant workers were employed as delivery agents, public transport drivers, or restaurant waiters, and are in constant contact with the general public. Infection risk is also higher \n\n9 Keeping in mind that for 2020 the figures are only up to October of the year.\n\nASEAN Migration Outlook\n\n21" + }, + { + "bleu": 1.0, + "doc_id": "doc_c4416f86c1db7fee5dccc1a10e85b1aade1cd77803f4ee626a090e8bc26aa9a9_page_000001.png", + "edit_distance": 0.0, + "f1_score": 1.0, + "meteor": 0.9999998682968357, + "precision": 1.0, + "pred_md": "Figure 1.10. Migrant remittances inflows (in US$ billion)\n\nSource : World Bank and KNOMAD (2021)\n\nTable 1.4. Growth in migrant remittance inflows\n\nSource : World Bank and KNOMAD (2021)\n\nIn the Philippines, of the returning Filipino migrant workers in 2020, 55 percent earned a monthly income of between PHP20,000 and PHP50,000, and 19 percent earned between PHP5000 and PHP20,000. Before their return, 50 percent reported remitting amounts ranging from PHP10,000 to PHP20,000 (US$200 to US$400) monthly. It is highly unlikely that the families of these migrant workers would have savings to rely on after they lost their jobs. Additionally, 83 percent of these workers were still unemployed after three months, resulting in a 60 percent drop in household income for 48 percent of the returned migrant workers.\n\n26\n\nASEAN Migration Outlook", + "recall": 1.0, + "true_md": "Figure 1.10. Migrant remittances inflows (in US$ billion)\n\nSource: World Bank and KNOMAD (2021)\n\nTable 1.4. Growth in migrant remittance inflows\n\nSource: World Bank and KNOMAD (2021)\n\nIn the Philippines, of the returning Filipino migrant workers in 2020, 55 percent earned a monthly income of between PHP20,000 and PHP50,000, and 19 percent earned between PHP5000 and PHP20,000. Before their return, 50 percent reported remitting amounts ranging from PHP10,000 to PHP20,000 (US$200 to US$400) monthly. It is highly unlikely that the families of these migrant workers would have savings to rely on after they lost their jobs. Additionally, 83 percent of these workers were still unemployed after three months, resulting in a 60 percent drop in household income for 48 percent of the returned migrant workers. \n\n26\n\nASEAN Migration Outlook" + }, + { + "bleu": 0.9679810921763, + "doc_id": "doc_63111d8ed6b32c2b3473840ace01896244527d3dc306d81410a3cc7d87a45226_page_000001.png", + "edit_distance": 0.02531645569620253, + "f1_score": 0.9790209790209791, + "meteor": 0.9974090027585754, + "precision": 0.958904109589041, + "pred_md": "J\n\nailed for Doing Business\n\n## Executive Summary\n\n6\n\nI ndia suffers from 'regulatory cholesterol' that is getting in the way of doing business. The legislations, rules and regulations enacted by the Union and State governments have over time created barriers to the smooth flow of ideas, organisation, money, entrepreneurship and through them the creation of jobs, wealth and GDP.\n\nThe presence of hostile clauses in these laws, rules and regulations has grown since Independence, surviving three decades of economic reforms initiated in 1991. The biggest challenges come from the continuance of imprisonment as a tool of control. As automation increases in the coming years, the pre-Independence 1940s-style administrative controls meant to protect labour will prove counter-productive in 21 st -century India.\n\nThere are 1,536 laws that govern doing business in India, of which 678 are implemented at the Union level. Within these laws is a web of 69,233 compliances, of which 25,537 are at the Union level. These compliances need to be communicated to the governments through 6,618 annual filings, 2,282 (34.5 percent) at the Union level and at the states, 4,336.\n\nThese changes in compliance requirements occur constantly and add to business uncertainty. In the 12 months up to 31 December 2021, there have been 3,577 regulatory changes;", + "recall": 1.0, + "true_md": "## Executive Summary\n\nI ndia suffers from 'regulatory cholesterol' that is getting in the way of doing business. The legislations, rules and regulations enacted by the Union and State governments have over time created barriers to the smooth flow of ideas, organisation, money, entrepreneurship and through them the creation of jobs, wealth and GDP.\n\nThe presence of hostile clauses in these laws, rules and regulations has grown since Independence, surviving three decades of economic reforms initiated in 1991. The biggest challenges come from the continuance of imprisonment as a tool of control. As automation increases in the coming years, the pre-Independence 1940s-style administrative controls meant to protect labour will prove counter-productive in 21 st -century India.\n\nThere are 1,536 laws that govern doing business in India, of which 678 are implemented at the Union level. Within these laws is a web of 69,233 compliances, of which 25,537 are at the Union level. These compliances need to be communicated to the governments through 6,618 annual filings, 2,282 (34.5 percent) at the Union level and at the states, 4,336.\n\nThese changes in compliance requirements occur constantly and add to business uncertainty. In the 12 months up to 31 December 2021, there have been 3,577 regulatory changes; " + }, + { + "bleu": 0.9554691949352742, + "doc_id": "doc_c2b60a018b0503378c804fbf3f75b8ff956d30b3fcecdc30c37781b7dbe0c1c4_page_000001.png", + "edit_distance": 0.04032258064516129, + "f1_score": 0.965986394557823, + "meteor": 0.9819957676963994, + "precision": 0.9530201342281879, + "pred_md": "J\n\nailed for Doing Business\n\n## III. Regulatory cholesterol\n\nT his report defines 'regulatory cholesterol' as the policy actions of the three arms of the State, i.e. the executive, the legislature, and the judiciary, using the instruments of legislations, rules, regulations or orders, to create or raise barriers to a smooth flow of ideas, organisation, money and most importantly, the flow of the entrepreneurial spirit. In India, a wrong political choice in the early decades of Independence has created a policy fraternity that shuns data and causalities and leans on rhetoric and ideologies to frame economic policies. Inflation in the 1970s, for instance, was not caused by hoarders and speculators; it was a matter of supply and demand. 'Excoriating, coercing, or imprisoning the hoarders and speculators changes nothing in terms of creating new supply,' write Vijay Kelkar and Ajay Shah. 28 'The economic theory of people hostile to economic forces is wrong.'\n\nBy taking one policy tool -imprisonment - this report highlights the excesses of overregulation and the resultant regulatory cholesterol while doing business in India. Although the biggest constituency at the receiving end of these laws is that of entrepreneurs running forprofit firms and corporations, this regulatory overreach also impacts not-for-profits such as schools and hospitals-both necessary institutions for India with a huge demand. Step\n\n16", + "recall": 0.9793103448275862, + "true_md": "## III. Regulatory cholesterol\n\nT his report defines 'regulatory cholesterol' as the policy actions of the three arms of the State, i.e. the executive, the legislature, and the judiciary, using the instruments of legislations, rules, regulations or orders, to create or raise barriers to a smooth flow of ideas, organisation, money and most importantly, the flow of the entrepreneurial spirit. In India, a wrong political choice in the early decades of Independence has created a policy fraternity that shuns data and causalities and leans on rhetoric and ideologies to frame economic policies. Inflation in the 1970s, for instance, was not caused by hoarders and speculators; it was a matter of supply and demand. 'Excoriating, coercing, or imprisoning the hoarders and speculators changes nothing in terms of creating new supply,' write Vijay Kelkar and Ajay Shah. 28 'The economic theory of people hostile to economic forces is wrong.'\n\nBy taking one policy tool - imprisonment - this report highlights the excesses of overregulation and the resultant regulatory cholesterol while doing business in India. Although the biggest constituency at the receiving end of these laws is that of entrepreneurs running for- profit firms and corporations, this regulatory overreach also impacts not-for-profits such as schools and hospitals-both necessary institutions for India with a huge demand. Step " + }, + { + "bleu": 0.8858751546029152, + "doc_id": "doc_24ab9416df440de9d792a4034e177eb71933a6eed72d4cf1ead17d22a787702f_page_000001.png", + "edit_distance": 0.07407407407407407, + "f1_score": 0.9649122807017544, + "meteor": 0.9803607843137254, + "precision": 0.9482758620689655, + "pred_md": "ailed for Doing Business\n\nJ\n\n## TABLE 22: COMMERCIAL LAWS WITH MORE THAN 100 IMPRISONMENT CLAUSES\n\nSource: TeamLease Regtech\n\n## TABLE 23: IMPRISONMENT CLAUSES IN ENVIRONMENT, HEALTH AND SAFETY LAWS\n\nSource: TeamLease Regtech\n\nNOTE: The inconsistency in number of laws is because a single law could have multiple clauses on criminality; it could have a few clauses of less than three months and few of between three and five years.\n\n78", + "recall": 0.9821428571428571, + "true_md": "Jailed for Doing Business\n\nTABLE 22: COMMERCIAL LAWS WITH MORE THAN 100 IMPRISONMENT CLAUSES\n\nSource: TeamLease Regtech\n\nTABLE 23: IMPRISONMENT CLAUSES IN ENVIRONMENT, HEALTH AND SAFETY LAWS\n\nSource: TeamLease Regtech\n\nNOTE: The inconsistency in number of laws is because a single law could have multiple clauses on criminality; it could have a few clauses of less than three months and few of between three and five years.\n\n78" + }, + { + "bleu": 0.8669680567283442, + "doc_id": "doc_0ec2813813e8e7496d97aa4fa15dd414aa66cae2d6f62db818d30d9856ff0439_page_000001.png", + "edit_distance": 0.08163265306122448, + "f1_score": 0.9863013698630138, + "meteor": 0.991042584434655, + "precision": 0.972972972972973, + "pred_md": "Appendices\n\n## TABLE 28: BREAKDOWN OF IMPRISONMENT CLAUSES IN STATE LAWS\n\nSource: TeamLease Regtech\n\n## TABLE 29: STATES WITH MORE THAN 1,000 IMPRISONMENT CLAUSES\n\nSources: TeamLease Regtech, and Reserve Bank of India for GSDPs\n\nExchange rate: Rs 75 to USD\n\n81", + "recall": 1.0, + "true_md": "Appendices\n\nTABLE 28: BREAKDOWN OF IMPRISONMENT CLAUSES IN STATE LAWS\n\nSource: TeamLease Regtech\n\nTABLE 29: STATES WITH MORE THAN 1,000 IMPRISONMENT CLAUSES\n\nSources: TeamLease Regtech, and Reserve Bank of India for GSDPs\n\nExchange rate: Rs 75 to USD\n\n81" + }, + { + "bleu": 0.8250175837803845, + "doc_id": "doc_970804c04c81dd9cc3ba80f0063b9551d290e83934c4e2c258e376845b3d180f_page_000001.png", + "edit_distance": 0.10344827586206896, + "f1_score": 0.9866666666666666, + "meteor": 0.9883681688302922, + "precision": 0.9736842105263158, + "pred_md": "Appendices\n\n## TABLE 35: UNION-STATE BREAKDOWN OF IMPRISONMENT CLAUSES BY CATEGORIES\n\n## TABLE 36: THREE CASE STUDIES ON MANUFACTURING COMPLIANCES*\n\n* These are real data from three companies operating in the automotive components business\n\n## TABLE 37: BREAKDOWN OF IMPRISONMENT CLAUSES IN MANUFACTURING CASE STUDIES*\n\n* In Table 36\n\n85", + "recall": 1.0, + "true_md": "Appendices\n\nTABLE 35: UNION-STATE BREAKDOWN OF IMPRISONMENT CLAUSES BY CATEGORIES\n\nTABLE 36: THREE CASE STUDIES ON MANUFACTURING COMPLIANCES*\n\n* These are real data from three companies operating in the automotive components business\n\nTABLE 37: BREAKDOWN OF IMPRISONMENT CLAUSES IN MANUFACTURING CASE STUDIES*\n\n* In Table 36\n\n85" + }, + { + "bleu": 0.7845332138125909, + "doc_id": "doc_39a0d95abd6dc977dfc7986b57fbbd9a8331af6b5b603e023f41dfcbc67003c7_page_000001.png", + "edit_distance": 0.13636363636363635, + "f1_score": 0.9354838709677418, + "meteor": 0.9617886321399769, + "precision": 0.90625, + "pred_md": "ailed for Doing Business\n\nJ\n\n## TABLE 38: THREE CASE STUDIES ON NBFC COMPLIANCES*\n\n* These are real data from three NBFCs\n\n## TABLE 39: BREAKDOWN OF IMPRISONMENT CLAUSES IN NBFC CASE STUDIES*\n\n* In table 38\n\n86", + "recall": 0.9666666666666667, + "true_md": "Jailed for Doing Business\n\nTABLE 38: THREE CASE STUDIES ON NBFC COMPLIANCES*\n\n* These are real data from three NBFCs\n\nTABLE 39: BREAKDOWN OF IMPRISONMENT CLAUSES IN NBFC CASE STUDIES*\n\n* In table 38\n\n86" + }, + { + "bleu": 0.9007690507972514, + "doc_id": "doc_df678b583abe6afa6971fac9efa23189bd235105d697ae1b2b12d25e3b9fb8b4_page_000001.png", + "edit_distance": 0.04878048780487805, + "f1_score": 0.9743589743589742, + "meteor": 0.9510030307403665, + "precision": 0.9743589743589743, + "pred_md": "## Restrictions on Land Ownership by Foreigners in Selected Jurisdictions\n\nJune 2023\n\nLL File No. 2023-022255 LRA-D-PUB-002612\n\nThe Law Library of Congress, Global Legal Research Directorate (202) 707-5080 · law@loc.gov · http://www.law.gov", + "recall": 0.9743589743589743, + "true_md": "## Restrictions on Land Ownership by Foreigners in Selected Jurisdictions \n\nJune 2023 \n\nLL File No. 2023-022255 LRA-D-PUB-002612 \n\nThe Law Library of Congress, Global Legal Research Directorate (202) 707-5080 • law@loc.gov • http://www.law.gov" + }, + { + "bleu": 0.8742007910271749, + "doc_id": "doc_4f6d9c52848acf2346013388db8ef25c40e6e2fd1232c49f2abfa7a21f65c82c_page_000001.png", + "edit_distance": 0.11531190926275993, + "f1_score": 0.9937106918238994, + "meteor": 0.9946648480594008, + "precision": 0.9957983193277311, + "pred_md": "## Restrictions on Land Ownership by Foreigners in Selected Jurisdictions\n\nStaff of the Global Legal Research Directorate\n\n## I. Introduction\n\nThis report, prepared by the research staff of the Law Library of Congress, surveys 39 jurisdictions regarding whether, and if so how, they restrict ownership of land by foreigners. 1 The jurisdictions surveyed were among those with the highest gross domestic product according to 2021 World Bank data, selected to ensure broadly representative coverage. 2\n\nWe identified 10 countries that do not restrict land ownership by foreigners: Belgium , France , Germany , Ireland , Japan , the Netherlands , Norway , Portugal , Sweden , and the United Kingdom .\n\nWe found that the following countries do not permit foreign ownership of land, although exceptions may apply in some cases or other rights to land may be acquired: China Indonesia , , Nigeria , Philippines , and Thailand .\n\nAmong the other jurisdictions surveyed, some have restrictions that apply to different types of land, including agricultural, residential, and commercial land. Other types of restriction are based on the location of the land, such as near the border or military establishments. Some jurisdictions restrict particular categories of foreigners from land ownership. Some require special permission or approval for foreigners before they can acquire land.\n\nOwnership of agricultural land by foreigners is restricted by some provinces of Canada , and by Egypt India , (restricted for diplomatic personnel, nonresidents of Indian origin and nonresident citizens without registration), Iran Poland , (permit required), and Russia Argentina Brazil . , , and Turkey restrict ownership of rural or local land to a percentage of the total land of the local jurisdiction.\n\nArticle XVII of the General Agreement on Trade in Services (GATS) obligates members to provide national treatment to other members, i.e., 'treatment no less favourable than that it accords to its own.' 3 If land ownership restrictions result in less favorable treatment of foreigners, GATS\n\n1 The surveyed jurisdictions are Argentina Australia , , Austria , Belgium Brazil Canada Chile China Egypt , , , , , , Finland , Germany Greece India Indonesia Iran Ireland Israel , , , , , , , Italy , Japan Mexico , , the Netherlands , New Zealand Nigeria Norway Philippines Poland Portugal Russia Saudi Arabia South Africa South , , , , , , , , , Korea Spain Sweden Switzerland Taiwan Thailand Turkey United Arab Emirates , , , , , , , , and the United Kingdom .\n\n2 World Bank Databank, Gross Domestic Product 2021 (Jan. 15, 2023), https://perma.cc/GP7Y-Z8K8.\n\n3 General Agreement on Trade in Services (GATS), Apr. 15, 1994, Marrakesh Agreement Establishing the World Trade Organization, Annex 1B, art. XVII, 1869 U.N.T.S. 183, 33 I.L.M. 1167 (1994), https://perma.cc/Z89YSEVS.\n\nThe Law Library of Congress\n\n1", + "recall": 0.9916317991631799, + "true_md": "## Restrictions on Land Ownership by Foreigners in Selected Jurisdictions \n\nStaff of the Global Legal Research Directorate \n\n## I. Introduction \n\nThis report, prepared by the research staff of the Law Library of Congress, surveys 39 jurisdictions regarding whether, and if so how, they restrict ownership of land by foreigners. 1 The jurisdictions surveyed were among those with the highest gross domestic product according to 2021 World Bank data, selected to ensure broadly representative coverage. 2 \n\nWe identified 10 countries that do not restrict land ownership by foreigners: Belgium, France, Germany, Ireland, Japan, the Netherlands, Norway, Portugal, Sweden, and the United Kingdom. \n\nWe found that the following countries do not permit foreign ownership of land, although exceptions may apply in some cases or other rights to land may be acquired: China, Indonesia, Nigeria, Philippines, and Thailand. \n\nAmong the other jurisdictions surveyed, some have restrictions that apply to different types of land, including agricultural, residential, and commercial land. Other types of restriction are based on the location of the land, such as near the border or military establishments. Some jurisdictions restrict particular categories of foreigners from land ownership. Some require special permission or approval for foreigners before they can acquire land. \n\nOwnership of agricultural land by foreigners is restricted by some provinces of Canada, and by Egypt, India (restricted for diplomatic personnel, nonresidents of Indian origin and nonresident citizens without registration), Iran, Poland (permit required), and Russia. Argentina, Brazil, and Turkey restrict ownership of rural or local land to a percentage of the total land of the local jurisdiction. \n\nArticle XVII of the General Agreement on Trade in Services (GATS) obligates members to provide national treatment to other members, i.e., 'treatment no less favourable than that it accords to its own.' 3 If land ownership restrictions result in less favorable treatment of foreigners, GATS \n\n1 The surveyed jurisdictions are Argentina, Australia, Austria, Belgium, Brazil, Canada, Chile, China, Egypt, Finland, Germany, Greece, India, Indonesia, Iran, Ireland, Israel, Italy, Japan, Mexico, the Netherlands, New Zealand, Nigeria, Norway, Philippines, Poland, Portugal, Russia, Saudi Arabia, South Africa, South Korea, Spain, Sweden, Switzerland, Taiwan, Thailand, Turkey, United Arab Emirates, and the United Kingdom. \n\n2 World Bank Databank, Gross Domestic Product 2021 (Jan. 15, 2023), https://perma.cc/GP7Y-Z8K8. \n\n3 General Agreement on Trade in Services (GATS), Apr. 15, 1994, Marrakesh Agreement Establishing the World Trade Organization, Annex 1B, art. XVII, 1869 U.N.T.S. 183, 33 I.L.M. 1167 (1994), https://perma.cc/Z89Y- SEVS. \n\nThe Law Library of Congress \n\n1 " + }, + { + "bleu": 0.9913784026649441, + "doc_id": "doc_c1381b5a2db9ca221ea4d47f1e037d837679114eb70cafb1b7bb1d681d355d75_page_000001.png", + "edit_distance": 0.005050505050505051, + "f1_score": 0.9952380952380951, + "meteor": 0.9972148756952771, + "precision": 0.990521327014218, + "pred_md": "Restrictions on Land Ownership by Foreigners in Selected Jurisdictions\n\nmembers should specify this in their schedule of specific commitments. 4 Reservation of the ability to lease or own land to nationals is one such treatment; therefore, it should be listed in the schedule as a limitation on national treatment. 5 This applies to services that the GATS covers. 6\n\nSome jurisdictions do not list foreign land ownership on their schedules, but restrict it for national security or similar interests. 7 Such jurisdictions include Australia and Finland (national interest), Chile and Greece (border area), Russia (national security), and Spain (zones of interest to national defense and the military). Several other jurisdictions that also restrict ownership for national security purposes have entered restrictions on their GATS schedules. Such jurisdictions include Argentina and Mexico (border area), Iran (sensitive areas), South Korea (military bases and installation protection zones), Taiwan (lands within fortified and military areas and adjacent to the national frontiers), and Turkey (designated military zones).\n\nThere are other various restri ctions on foreigners' land ownership. Figure 1 below shows in simplified format the surveyed jurisdictions that impose particular categories of restrictions. On page 4, a color-coded map sets forth which jurisdictions permit foreign acquisition, prohibit it, or impose restrictions. A Comparative Summary Table beginning on page 5 presents the essential findings of our study for each jurisdiction. Lastly, the textual surveys for each jurisdiction provide further detail.\n\n4 Id. art. XX.\n\n5 Julia Nielson & Daria Taglioni, A Quick Guide to the GATS and Mode 4 , OECD, World Bank, IOM Seminar on Trade and Migration (Nov. 12-14, 2003), at 11, https://perma.cc/B8XW-LNZ4.\n\n6 World Trade Organization, The General Agreement on Trade in Services (GATS): Objectives, Coverage and Disciplines , Question 3 , https://perma.cc/4J7Y-WAG7 . It states, '[t]he GATS applies in principle to all service sectors, with two exceptions.'\n\n7 See GATS art. XIV General Exceptions.\n\nThe Law Library of Congress\n\n2", + "recall": 1.0, + "true_md": "Restrictions on Land Ownership by Foreigners in Selected Jurisdictions \n\nmembers should specify this in their schedule of specific commitments. 4 Reservation of the ability to lease or own land to nationals is one such treatment; therefore, it should be listed in the schedule as a limitation on national treatment. 5 This applies to services that the GATS covers. 6 \n\nSome jurisdictions do not list foreign land ownership on their schedules, but restrict it for national security or similar interests. 7 Such jurisdictions include Australia and Finland (national interest), Chile and Greece (border area), Russia (national security), and Spain (zones of interest to national defense and the military). Several other jurisdictions that also restrict ownership for national security purposes have entered restrictions on their GATS schedules. Such jurisdictions include Argentina and Mexico (border area), Iran (sensitive areas), South Korea (military bases and installation protection zones), Taiwan (lands within fortified and military areas and adjacent to the national frontiers), and Turkey (designated military zones). \n\nThere are other various restrictions on foreigners' land ownership. Figure 1 below shows in simplified format the surveyed jurisdictions that impose particular categories of restrictions. On page 4, a color-coded map sets forth which jurisdictions permit foreign acquisition, prohibit it, or impose restrictions. A Comparative Summary Table beginning on page 5 presents the essential findings of our study for each jurisdiction. Lastly, the textual surveys for each jurisdiction provide further detail. \n\n4 Id. art. XX. \n\n5 Julia Nielson & Daria Taglioni, A Quick Guide to the GATS and Mode 4, OECD, World Bank, IOM Seminar on Trade and Migration (Nov. 12-14, 2003), at 11, https://perma.cc/B8XW-LNZ4. \n\n6 World Trade Organization, The General Agreement on Trade in Services (GATS): Objectives, Coverage and Disciplines, Question 3, https://perma.cc/4J7Y-WAG7. It states, '[t]he GATS applies in principle to all service sectors, with two exceptions.' \n\n7 See GATS art. XIV General Exceptions. \n\nThe Law Library of Congress \n\n2 " + }, + { + "bleu": 1.0, + "doc_id": "doc_1039bce8b948fac48b0b9a42f8266700a104aca355c3681a544d65e6b136b8d5_page_000001.png", + "edit_distance": 0.0, + "f1_score": 1.0, + "meteor": 0.9999375, + "precision": 1.0, + "pred_md": "Restrictions on Land Ownership by Foreigners in Selected Jurisdictions\n\n## Comparative Summary Table\n\nThe Law Library of Congress\n\n5", + "recall": 1.0, + "true_md": "Restrictions on Land Ownership by Foreigners in Selected Jurisdictions \n\n## Comparative Summary Table \n\nThe Law Library of Congress \n\n5 " + }, + { + "bleu": 1.0, + "doc_id": "doc_1cf54980192667fd7a6977b301a175e98be02b11651bbf78b6cae338858db2aa_page_000001.png", + "edit_distance": 0.0, + "f1_score": 1.0, + "meteor": 0.9998518518518519, + "precision": 1.0, + "pred_md": "Restrictions on Land Ownership by Foreigners in Selected Jurisdictions\n\nThe Law Library of Congress\n\n6", + "recall": 1.0, + "true_md": "Restrictions on Land Ownership by Foreigners in Selected Jurisdictions \n\nThe Law Library of Congress \n\n6 " + }, + { + "bleu": 1.0, + "doc_id": "doc_d1437ae1c6bedfa2bbf72e4d04dee3a01896dd045024438ee7c4e775e0ab6e9c_page_000001.png", + "edit_distance": 0.0, + "f1_score": 1.0, + "meteor": 0.9998518518518519, + "precision": 1.0, + "pred_md": "Restrictions on Land Ownership by Foreigners in Selected Jurisdictions\n\nThe Law Library of Congress\n\n7", + "recall": 1.0, + "true_md": "Restrictions on Land Ownership by Foreigners in Selected Jurisdictions \n\nThe Law Library of Congress \n\n7 " + }, + { + "bleu": 0.9834032442208696, + "doc_id": "doc_475c037beeba4ba75b3e749b51f3781a67e8e8f9db6a31772f7bf2ed2bd05d00_page_000001.png", + "edit_distance": 0.009345794392523364, + "f1_score": 0.9938080495356038, + "meteor": 0.9967237248281295, + "precision": 0.9938080495356038, + "pred_md": "## THIS BOOK'S APPROACH\n\nThis book's approach is premised on a simple assumption: because behavioral economics is foremost a 'test-and-learn' field of scientific inquiry that evolves according to experimental outcomes and practical, policy-orientated applications of the knowledge garnered from these outcomes, so too should students test-and-learn. Studying and practicing behavioral economics should occur simultaneously, which, in turn, suggests a course taught more according to a practicum approach than in a traditionally styled lecture format. As such, the book's information and lessons are presented in a succinct and precise format.\n\nThe goal of this textbook is to help students experience behavioral economics through actual participation in the same experiments and economic games that have served as the foundations for, and shaped the contours of, the field. With the help of this book, students have the opportunity to learn behavioral economics firsthand and, in the process, create their own data and experiences. They will learn about themselves-about how they make private and public choices under experimental conditions-at the same time as they learn about the field of behavioral economics itself. They will be both the subjects and students of behavioral economics. What better way to learn?\n\n## HOMO ECONOMICUS VS. HOMO SAPIENS\n\nFor ease of reference and exposition, we henceforth refer to the type of individual construed by the traditional rational-choice model as Homo economicus , a peculiar subspecies of human beings that is unfailingly omniscient, dispassionate, and self-interested when it comes to making choices. Homo sapiens , on the other hand, represents the rest of us-the often-flawed reasoners and sometimesaltruistic competitors who are prone to making decisions based primarily on emotion and heuristics. , 1 2\n\n## THE TEXTBOOK'S DIFFERENT SECTIONS\n\nThe textbook consists of four sections that, taken together, portray in full the eclectic methodologies comprising the field of behavioral economics. Sections 1 and 2 present the thought and actual\n\n- 1. Homo economicus is Latin for 'economic man.' Persky (1995) traces its use back to the late 1800s when it was used by critics of John Stuart Mill's work on political economy. In contrast (and, as we will see, with no small touch of irony) Homo sapiens is Latin for 'wise man.' For a deep dive into evolution of Homo sapiens , particularly from the start of the Cognitive Revolution 70,000 years ago, see Harari (2015).\n- 2. We have all heard the saying that 'words matter.' The titles and descriptions we use to distinguish people and their behaviors (e.g., Homo economicus vs. Homo sapiens ) can reinforce or diminish behaviors such as pride in cultural heritage, respect for the living world, and trust in community, a process known as 'crowding out' of 'intrinsic motivation and commitment.' As an example of this phenomenon, Bauer et al. (2012) asked participants in an online survey to imagine themselves as one of four households facing a water shortage due to a drought affecting their shared well. The survey assigned the label 'consumers' to half of the participants and 'individuals' to the other half. Those imagining themselves as consumers reported feeling less personal responsibility to reduce their water demand, and less trust in others to do the same, than did those referred to as individuals. As we are about to learn, behavioral economics is all about exposing these types of 'framing effects' existing in the 'real world' inhabited by Homo sapiens .\n\nBEHAVIORAL ECONOMICS PRACTICUM XIX", + "recall": 0.9938080495356038, + "true_md": "## THIS BOOK'S APPROACH \n\nThis book's approach is premised on a simple assumption: because behavioral economics is foremost a 'test-and-learn' field of scientific inquiry that evolves according to experimental outcomes and practical, policy-orientated applications of the knowledge garnered from these outcomes, so too should students test-and-learn. Studying and practicing behavioral economics should occur simultaneously, which, in turn, suggests a course taught more according to a practicum approach than in a traditionally styled lecture format. As such, the book's information and lessons are presented in a succinct and precise format. \n\nThe goal of this textbook is to help students experience behavioral economics through actual participation in the same experiments and economic games that have served as the foundations for, and shaped the contours of, the field. With the help of this book, students have the opportunity to learn behavioral economics firsthand and, in the process, create their own data and experiences. They will learn about themselves-about how they make private and public choices under experimental conditions-at the same time as they learn about the field of behavioral economics itself. They will be both the subjects and students of behavioral economics. What better way to learn? \n\n## HOMO ECONOMICUS VS. HOMO SAPIENS \n\nFor ease of reference and exposition, we henceforth refer to the type of individual construed by the traditional rational-choice model as Homo economicus, a peculiar subspecies of human beings that is unfailingly omniscient, dispassionate, and self-interested when it comes to making choices. Homo sapiens, on the other hand, represents the rest of us-the often-flawed reasoners and sometimes- altruistic competitors who are prone to making decisions based primarily on emotion and heuristics. 1 , 2 \n\n## THE TEXTBOOK'S DIFFERENT SECTIONS \n\nThe textbook consists of four sections that, taken together, portray in full the eclectic methodologies comprising the field of behavioral economics. Sections 1 and 2 present the thought and actual \n\n1. Homo economicus is Latin for 'economic man.' Persky (1995) traces its use back to the late 1800s when it was used by critics of John Stuart Mill's work on political economy. In contrast (and, as we will see, with no small touch of irony) Homo sapiens is Latin for 'wise man.' For a deep dive into evolution of Homo sapiens, particularly from the start of the Cognitive Revolution 70,000 years ago, see Harari (2015). \n\n2. We have all heard the saying that 'words matter.' The titles and descriptions we use to distinguish people and their behaviors (e.g., Homo economicus vs. Homo sapiens) can reinforce or diminish behaviors such as pride in cultural heritage, respect for the living world, and trust in community, a process known as 'crowding out' of 'intrinsic motivation and commitment.' As an example of this phenomenon, Bauer et al. (2012) asked participants in an online survey to imagine themselves as one of four households facing a water shortage due to a drought affecting their shared well. The survey assigned the label 'consumers' to half of the participants and 'individuals' to the other half. Those imagining themselves as consumers reported feeling less personal responsibility to reduce their water demand, and less trust in others to do the same, than did those referred to as individuals. As we are about to learn, behavioral economics is all about exposing these types of 'framing effects' existing in the 'real world' inhabited by Homo sapiens. \n\nBEHAVIORAL ECONOMICS PRACTICUM XIX" + }, + { + "bleu": 0.9950351810422626, + "doc_id": "doc_0cf2e856478f66d96690a306b55ff40c01ab24cb26d048fe248eac8e3d66e65c_page_000001.png", + "edit_distance": 0.0, + "f1_score": 1.0, + "meteor": 0.9999999985851455, + "precision": 1.0, + "pred_md": "laboratory experiments that have formed key pillars of the field, such as those experiments depicted in Examples 1 and 2 in the book's Introduction section. The thought experiments in Section 1 are, for the most part, re-castings of the simple cognitive tests devised by psychologists and economists over the past three-to-four decades to illustrate the fallacies, miscalculations, and biases distinguishing Homo sapiens from Homo economicus . Similarly, the laboratory experiments presented in Section 2 are, for the most part, re-castings of the seminal experiments conducted by Kahneman and Tversky (among many others). These experiments helped motivate the revised theories of human choice behavior, such as Kahneman and Tversky's (1979) Prospect Theory, which form another pillar of behavioral economics. Alongside these experiments, Section 2 presents the revised theories of human choice behavior with varying degrees of rigor. This is where the theoretical bases of Homo economicus ' rational choice behavior are examined, and where key refinements to this theory are developed-theoretical refinements underpinning the myriad departures from rational choice behavior we witness Homo sapiens make in this section's laboratory and field experiments (and which are examined further in Sections 3 and 4).\n\nSection 3 submerses the student in the world of behavioral game theory. Here we explore games such as Ultimatum Bargaining presented in Example 5. We follow Camerer (2003)'s lead, first by characterizing the games analytically (i.e., identifying solution, or equilibrium, concepts that are predicted to result when members of Homo economicus play the games), and then by discussing empirical results obtained from corresponding field experiments conducted with Homo sapiens . It is within the context of these games and field experiments that theories of social interaction are tested concerning inter alia trust and trustworthiness, honesty, fairness, reciprocity, etc. As with the thought and laboratory experiments presented in Sections 1 and 2, the games and field experiments presented in Section 3 are meant to be replicated with students as subjects and the instructor as the experimenter, or researcher.\n\nFinally, Section 4 wades into the vast sea of empirical research and choice architecture. Here the student explores studies reporting on (1) the outcomes of actual policy nudges, such as the SMarT retirement-savings plan presented in Example 3 of the Introduction, (2) analyses of secondary datasets to test for choice behavior consistent with the revised theories discussed in Section 2, such as the test for loss aversion in Example 4 of the Introduction, and (3) analyses of primary datasets obtained from novel field experiments to further test the revised theories. The main purpose of this section is not only to introduce the student to interesting empirical studies and policy adaptations in the field of behavioral economics, but also, in the process, to incubate in the student an abiding appreciation for the obscure settings that sometimes lend themselves to such study. 3\n\n## THE TEXTBOOK'S DIFFERENT LEVELS OF RIGOR\n\nBecause the mathematical and computational rigor of material presented in this textbook varies throughout, particularly in Sections 2 - 4, the extent of the rigor used in the presentation of a given topic is indicated with superscripts. Topics without a superscript are considered basic and universal enough that backgrounds in economics, mathematics, or statistics are not required for the reader to understand the material. Topics with a single asterisk (*) indicate that higher mathematical reasoning skills are recommended for the reader to fully grasp the material. Topics with a double\n\n3. Our approach to studying behavioral economics is focused on the underlying laboratory experimentation and behavioral games that form the bedrock of the field. As such, we eschew delving into related fields such as neuroeconomics and auction theory. See Cartwright (2018) and Just (2013) for introductions to the former and latter fields, respectively. XX ARTHUR J. CAPLAN", + "recall": 1.0, + "true_md": "laboratory experiments that have formed key pillars of the field, such as those experiments depicted in Examples 1 and 2 in the book's Introduction section. The thought experiments in Section 1 are, for the most part, re-castings of the simple cognitive tests devised by psychologists and economists over the past three-to-four decades to illustrate the fallacies, miscalculations, and biases distinguishing Homo sapiens from Homo economicus. Similarly, the laboratory experiments presented in Section 2 are, for the most part, re-castings of the seminal experiments conducted by Kahneman and Tversky (among many others). These experiments helped motivate the revised theories of human choice behavior, such as Kahneman and Tversky's (1979) Prospect Theory, which form another pillar of behavioral economics. Alongside these experiments, Section 2 presents the revised theories of human choice behavior with varying degrees of rigor. This is where the theoretical bases of Homo economicus' rational choice behavior are examined, and where key refinements to this theory are developed-theoretical refinements underpinning the myriad departures from rational choice behavior we witness Homo sapiens make in this section's laboratory and field experiments (and which are examined further in Sections 3 and 4). \n\nSection 3 submerses the student in the world of behavioral game theory. Here we explore games such as Ultimatum Bargaining presented in Example 5. We follow Camerer (2003)'s lead, first by characterizing the games analytically (i.e., identifying solution, or equilibrium, concepts that are predicted to result when members of Homo economicus play the games), and then by discussing empirical results obtained from corresponding field experiments conducted with Homo sapiens. It is within the context of these games and field experiments that theories of social interaction are tested concerning inter alia trust and trustworthiness, honesty, fairness, reciprocity, etc. As with the thought and laboratory experiments presented in Sections 1 and 2, the games and field experiments presented in Section 3 are meant to be replicated with students as subjects and the instructor as the experimenter, or researcher. \n\nFinally, Section 4 wades into the vast sea of empirical research and choice architecture. Here the student explores studies reporting on (1) the outcomes of actual policy nudges, such as the SMarT retirement-savings plan presented in Example 3 of the Introduction, (2) analyses of secondary datasets to test for choice behavior consistent with the revised theories discussed in Section 2, such as the test for loss aversion in Example 4 of the Introduction, and (3) analyses of primary datasets obtained from novel field experiments to further test the revised theories. The main purpose of this section is not only to introduce the student to interesting empirical studies and policy adaptations in the field of behavioral economics, but also, in the process, to incubate in the student an abiding appreciation for the obscure settings that sometimes lend themselves to such study. 3 \n\n## THE TEXTBOOK'S DIFFERENT LEVELS OF RIGOR \n\nBecause the mathematical and computational rigor of material presented in this textbook varies throughout, particularly in Sections 2 - 4, the extent of the rigor used in the presentation of a given topic is indicated with superscripts. Topics without a superscript are considered basic and universal enough that backgrounds in economics, mathematics, or statistics are not required for the reader to understand the material. Topics with a single asterisk (*) indicate that higher mathematical reasoning skills are recommended for the reader to fully grasp the material. Topics with a double \n\n3. Our approach to studying behavioral economics is focused on the underlying laboratory experimentation and behavioral games that form the bedrock of the field. As such, we eschew delving into related fields such as neuroeconomics and auction theory. See Cartwright (2018) and Just (2013) for introductions to the former and latter fields, respectively. XX ARTHUR J. CAPLAN" + }, + { + "bleu": 0.9909858841296846, + "doc_id": "doc_9877e33f447b370ad098531c25e340b9e786d1e84f4ac7365bab9c6746b57247_page_000001.png", + "edit_distance": 0.003289473684210526, + "f1_score": 0.9982847341337907, + "meteor": 0.9981869591510246, + "precision": 1.0, + "pred_md": "survey responses and outcomes from the experiments and games. This spreadsheet is linked to the students' randomly assigned course ID (CID) numbers. The other spreadsheet, which is linked to their university student ID numbers and their names, compiles their performances on quizzes, homework, and exams assigned throughout the semester.\n\nAt the risk of sounding draconian, this is a course where it may make sense to base upwards of 50% of a student's grade upon their in-person attendance, which would entail carefully taking role at the beginning of each class. If the class meets 30 times face-to-face during the semester, for example, their grade attributable to attendance would then drop by 3.33 percentage points for each missed class (excused absences withstanding). Granted, students who foresee having difficulty attending class in-person throughout the semester would likely choose to drop the course immediately. For those students who remain, the remaining 50% of their course grade would then be based upon their quizzes, homework, and exam scores.\n\nThe issue of how best to convey written information to the student a priori (i.e., before conducting a given experiment or game) also looms large in a participatory-learning setting such as this, especially if the instructor desires to obtain unbiased responses from the students (or more practically, to control for potential biases). For example, the first set of thought experiments presented in Section 1 is meant to demonstrate firsthand to the students the extent to which automatic, knee-jerk responses from what Kahneman (2011) identifies as the System 1 portion of the brain can result in miscalculations. Students who choose to read ahead (small in number though these types of students may be) potentially skew the distribution of responses away from its otherwise true representation of these miscalculations. Such skewness may be tolerable for strictly educational purposes, where the goal is to demonstrate that at least a certain percentage of students are prone to miscalculation. But if the instructor also hopes to compile student responses into a dataset amenable for statistical analysis, then this type of potential bias draws into question the validity of the data. 2\n\nTo help control for potential biases associated with students having read ahead about the game or experiment they are now participating in, I recommend including the following question on each Response Card: 'Did you read about this topic ahead of time?' (see Appendix A). Answers to this question provide a control for the level of student foreknowledge, which is the potential bias of concern.\n\nI am personally unaware of any studies that have looked at how well students learn the lessons of behavioral economics in a cumulative sense over a span of time (e.g., an entire semester) and across a variety of experiments and games. In other words, I know of no studies that estimate the extent to which individuals who begin a course in behavioral economics as bona fide Homo sapiens evolve toward ' Homo economism ' in their individual and social choices. The pedagogy promoted in this textbook-in particular, the data it generates-offers instructors the opportunity to empirically test the hypothesis that students make this evolution.\n\n2. Note that this potential biasedness problem also extends to the laboratory experiments of Section 2 and games of Section 3. BEHAVIORAL ECONOMICS PRACTICUM XXV", + "recall": 0.9965753424657534, + "true_md": "survey responses and outcomes from the experiments and games. This spreadsheet is linked to the students' randomly assigned course ID (CID) numbers. The other spreadsheet, which is linked to their university student ID numbers and their names, compiles their performances on quizzes, homework, and exams assigned throughout the semester. \n\nAt the risk of sounding draconian, this is a course where it may make sense to base upwards of 50% of a student's grade upon their in-person attendance, which would entail carefully taking role at the beginning of each class. If the class meets 30 times face-to-face during the semester, for example, their grade attributable to attendance would then drop by 3.33 percentage points for each missed class (excused absences withstanding). Granted, students who foresee having difficulty attending class in-person throughout the semester would likely choose to drop the course immediately. For those students who remain, the remaining 50% of their course grade would then be based upon their quizzes, homework, and exam scores. \n\nThe issue of how best to convey written information to the student a priori (i.e., before conducting a given experiment or game) also looms large in a participatory-learning setting such as this, especially if the instructor desires to obtain unbiased responses from the students (or more practically, to control for potential biases). For example, the first set of thought experiments presented in Section 1 is meant to demonstrate firsthand to the students the extent to which automatic, knee-jerk responses from what Kahneman (2011) identifies as the System 1 portion of the brain can result in miscalculations. Students who choose to read ahead (small in number though these types of students may be) potentially skew the distribution of responses away from its otherwise true representation of these miscalculations. Such skewness may be tolerable for strictly educational purposes, where the goal is to demonstrate that at least a certain percentage of students are prone to miscalculation. But if the instructor also hopes to compile student responses into a dataset amenable for statistical analysis, then this type of potential bias draws into question the validity of the data. 2 \n\nTo help control for potential biases associated with students having read ahead about the game or experiment they are now participating in, I recommend including the following question on each Response Card: 'Did you read about this topic ahead of time?' (see Appendix A). Answers to this question provide a control for the level of student foreknowledge, which is the potential bias of concern. \n\nI am personally unaware of any studies that have looked at how well students learn the lessons of behavioral economics in a cumulative sense over a span of time (e.g., an entire semester) and across a variety of experiments and games. In other words, I know of no studies that estimate the extent to which individuals who begin a course in behavioral economics as bona fide Homo sapiens evolve toward 'Homo economism' in their individual and social choices. The pedagogy promoted in this textbook-in particular, the data it generates-offers instructors the opportunity to empirically test the hypothesis that students make this evolution. \n\n2. Note that this potential biasedness problem also extends to the laboratory experiments of Section 2 and games of Section 3. BEHAVIORAL ECONOMICS PRACTICUM XXV" + }, + { + "bleu": 0.9454902880327203, + "doc_id": "doc_a546d4f7375892bde50a7ceb4e665e565000286a1c007c51105a3ca3fba87bf4_page_000001.png", + "edit_distance": 0.02356902356902357, + "f1_score": 0.9969418960244648, + "meteor": 0.9975850043378012, + "precision": 0.9939024390243902, + "pred_md": "- 6. Warning : This question concerns a politically charged event that occurred on January 18, 2019, at the Indigenous People's March in Washington, D.C. After reading this account of what happened at the march, and viewing this video of the event, which of the effects presented in this chapter do you think best describes this episode in our nation's history?\n- 7. Think of a situation in your own life when you framed information (either wittingly or unwittingly) in such a way that helped pre-determine an outcome. Describe the situation and how you framed the information. Was the outcome improved or worsened as a result of how you framed the information?\n- 8. After having learned about the Anchoring Effect in this chapter, do you think you will ever fall for something like this again?\n- 9. When someone admonishes you 'not to judge a book by its cover,' or as British management journalist Robert Heller once noted, 'Never ignore a gut feeling, but never believe that it's enough,' what heuristic(s) is he unwittingly advising you to avoid using?\n- 10. Browse the internet for information about an effect that was not discussed in this chapter. Can you classify this effect as a special case of a Priming or Framing Effect? Explain.\n- 11. Browse the internet for a heuristic other than the Affect and Availability Heuristics described in this chapter. Explain the heuristic.\n- 12. It's one thing to detect the existence of a Silo Effect and quite another to measure its\n\n24 ARTHUR J. CAPLAN", + "recall": 1.0, + "true_md": "6. Warning: This question concerns a politically charged event that occurred on January 18, 2019, at the Indigenous People's March in Washington, D.C. After reading this account of what happened at the march, and viewing this video of the event, which of the effects presented in this chapter do you think best describes this episode in our nation's history? \n\n7. Think of a situation in your own life when you framed information (either wittingly or unwittingly) in such a way that helped pre-determine an outcome. Describe the situation and how you framed the information. Was the outcome improved or worsened as a result of how you framed the information? \n\n8. After having learned about the Anchoring Effect in this chapter, do you think you will ever fall for something like this again? \n\n9. When someone admonishes you 'not to judge a book by its cover,' or as British management journalist Robert Heller once noted, 'Never ignore a gut feeling, but never believe that it's enough,' what heuristic(s) is he unwittingly advising you to avoid using? \n\n10. Browse the internet for information about an effect that was not discussed in this chapter. Can you classify this effect as a special case of a Priming or Framing Effect? Explain. \n\n11. Browse the internet for a heuristic other than the Affect and Availability Heuristics described in this chapter. Explain the heuristic. \n\n12. It's one thing to detect the existence of a Silo Effect and quite another to measure its \n\n24 ARTHUR J. CAPLAN" + }, + { + "bleu": 1.0, + "doc_id": "doc_c04c230b8dfa00d0fabbe9730c078c840ccaa27f5f270d5899f35405784a535d_page_000001.png", + "edit_distance": 0.0, + "f1_score": 1.0, + "meteor": 0.9999999915709975, + "precision": 1.0, + "pred_md": "(Niederle and Vesterlund 2007)\n\nIn other words, while women shy away from competition, men are drawn to it.\n\nTurning to Task 4, recall that although this choice is very similar to that of Task 3, Task 4's choice eliminates the prospect of having to subsequently participate in a competition. Thus, only in Task 3 could a gender gap in preference for competition have played a role in the choice of compensation scheme. As the figure below shows, there is no statistically significant gender gap in the choice of compensation scheme in Task 4 based upon perceived ranking in Task 1. A higher percentage of women than men who guessed their Task 1 ranking to be low (i.e., at level '3') chose the tournament scheme in Task 4, while the percentages were reversed for those participants who guessed their Task 1 rankings to be high (at levels '1' and '2'). But because the two lines in the figure remain close together, these differences are not statistically significant (i.e., we should treat the groups' respective choices as being no different from one another).\n\n(Niederle and Vesterlund 2007)\n\nThis result from Task 4 cements the authors' finding that women shy away from actual competition slated to occur at a future point in time, not implicit competition based upon their interpretations of how their past performance compares with others. 10\n\n10. In a related study of the performances of men and women in professional judo fights for bronze medals (of all things!), Cohen-Zada et al. (2017) find that men's performances are significantly affected by what the authors' call \"psychological momentum\", while women's is not. Psychological momentum is defined as the tendency of an outcome (such as a win in an initial judo match) to be followed by a similar outcome (a win in a subsequent match) that is not caused by any strategic incentives of the players. The authors point out that this result is consistent with evidence in the biological literature that\n\nBEHAVIORAL ECONOMICS PRACTICUM 111", + "recall": 1.0, + "true_md": "(Niederle and Vesterlund 2007) \n\nIn other words, while women shy away from competition, men are drawn to it. \n\nTurning to Task 4, recall that although this choice is very similar to that of Task 3, Task 4's choice eliminates the prospect of having to subsequently participate in a competition. Thus, only in Task 3 could a gender gap in preference for competition have played a role in the choice of compensation scheme. As the figure below shows, there is no statistically significant gender gap in the choice of compensation scheme in Task 4 based upon perceived ranking in Task 1. A higher percentage of women than men who guessed their Task 1 ranking to be low (i.e., at level '3') chose the tournament scheme in Task 4, while the percentages were reversed for those participants who guessed their Task 1 rankings to be high (at levels '1' and '2'). But because the two lines in the figure remain close together, these differences are not statistically significant (i.e., we should treat the groups' respective choices as being no different from one another). \n\n(Niederle and Vesterlund 2007) \n\nThis result from Task 4 cements the authors' finding that women shy away from actual competition slated to occur at a future point in time, not implicit competition based upon their interpretations of how their past performance compares with others. 10 \n\n10. In a related study of the performances of men and women in professional judo fights for bronze medals (of all things!), Cohen-Zada et al. (2017) find that men's performances are significantly affected by what the authors' call \"psychological momentum\", while women's is not. Psychological momentum is defined as the tendency of an outcome (such as a win in an initial judo match) to be followed by a similar outcome (a win in a subsequent match) that is not caused by any strategic incentives of the players. The authors point out that this result is consistent with evidence in the biological literature that \n\nBEHAVIORAL ECONOMICS PRACTICUM 111" + }, + { + "bleu": 0.9526008557531807, + "doc_id": "doc_b966d5aee5a17b377107ade67238d7215faef6303ac980080fd68534ab3ce5ae_page_000001.png", + "edit_distance": 0.02109704641350211, + "f1_score": 0.9966329966329965, + "meteor": 0.9978444679912547, + "precision": 0.9932885906040269, + "pred_md": "- 8. Suppose Evelyn the Environmental Economist is presenting her case in a public meeting for why raising the price of municipal water in the face of persistent drought conditions would be a good thing for the community, when someone in the audience yells out, 'That's unfair for seniors and others living on fixed incomes.' How might Evelyn frame her response in a way that dispels the audience's concerns about the fairness of a price increase?\n- 9. How would the indifference curve in Figure 6.1 change when drawn for a person who suffers from guilt but not envy? Draw the curve.\n- 10. Can you recall an example from your own life where you exhibited an Endowment Effect that ultimately led to regret?\n- 11. The Gender Gap experiment discussed in this chapter measured gender differences in terms of how males and females deal with competitive situations. Think of another situation where a gender gap may exist and design an experiment to test for it.\n- 12. It was shown in this chapter that a Homo economicus who exhibits convex-shaped indifference curves exhibits an Endowment Effect. Does this result still hold if Homo economicus exhibits linearly shaped indifference curves, as depicted in the figure below? Show your result using this graph.\n\nBEHAVIORAL ECONOMICS PRACTICUM 117", + "recall": 1.0, + "true_md": "8. Suppose Evelyn the Environmental Economist is presenting her case in a public meeting for why raising the price of municipal water in the face of persistent drought conditions would be a good thing for the community, when someone in the audience yells out, 'That's unfair for seniors and others living on fixed incomes.' How might Evelyn frame her response in a way that dispels the audience's concerns about the fairness of a price increase? \n\n9. How would the indifference curve in Figure 6.1 change when drawn for a person who suffers from guilt but not envy? Draw the curve. \n\n10. Can you recall an example from your own life where you exhibited an Endowment Effect that ultimately led to regret? \n\n11. The Gender Gap experiment discussed in this chapter measured gender differences in terms of how males and females deal with competitive situations. Think of another situation where a gender gap may exist and design an experiment to test for it. \n\n12. It was shown in this chapter that a Homo economicus who exhibits convex-shaped indifference curves exhibits an Endowment Effect. Does this result still hold if Homo economicus exhibits linearly shaped indifference curves, as depicted in the figure below? Show your result using this graph. \n\nBEHAVIORAL ECONOMICS PRACTICUM 117" + }, + { + "bleu": 1.0, + "doc_id": "doc_6182c54724e702a8ed0936580d2096ab8b372a599589e731c66bd40ba5c252c0_page_000001.png", + "edit_distance": 0.0, + "f1_score": 1.0, + "meteor": 0.9999999911696823, + "precision": 1.0, + "pred_md": "Now, how do we solve for the game's analytical equilibrium? 12\n\nHere, Player 2 applies backward induction to find what's known as a Perfect Bayesian Equilibrium (PBE). As we already know, if Player 2 is the weak type and Player 1 has chosen to invade, then Player 2 should concede. If he is the strong type, then Player 2 should fight. We also know that Player 1 recognizes that she gets a payoff of $0 if she concedes in the first round, regardless of Player 2's type. If she instead chooses to invade in the first round, then Player 1's expected payoff from invading is . This is merely the weighted average of Player 1's expected payoff when Player 2 is weak and her expected payoff when Player 2 is strong. Thus, invade is a better strategy than concede for Player 1 when . In other words, if the probability that Player 1 assigns to Player 2 being weak is greater than one-sixth, Player 1 should choose to invade in the first round. Otherwise, Player 1 should concede and be done with it.\n\nWhat's the outcome when you and your classmates play this more complicated version of the Escalation Game?\n\n## BURNING BRIDGES GAME\n\nThis game shares starkly similar features with the Escalation Game, but there is no uncertainty (thus, the analytical equilibrium is an SPE rather than a PBE). The SPE has much to say about the relationship between two tenacious competitors. Spaniel (2011) portrays the game as follows:\n\n12. This equilibrium is known as a Perfect Bayesian Equilibrium (PBE) rather than an SPE because of the uncertainty that at least one of the players is forced to contend with. Similar to Nash, Thomas Bayes is considered a towering figure. He was an 18th-century English statistician, philosopher, and Presbyterian minister who is known for formulating a specific case of the theorem that bears his name: Bayes Theorem. Bayes never published his theory himself-his notes were edited and published posthumously.\n\n132 ARTHUR J. CAPLAN", + "recall": 1.0, + "true_md": "Now, how do we solve for the game's analytical equilibrium? 12 \n\nHere, Player 2 applies backward induction to find what's known as a Perfect Bayesian Equilibrium (PBE). As we already know, if Player 2 is the weak type and Player 1 has chosen to invade, then Player 2 should concede. If he is the strong type, then Player 2 should fight. We also know that Player 1 recognizes that she gets a payoff of $0 if she concedes in the first round, regardless of Player 2's type. If she instead chooses to invade in the first round, then Player 1's expected payoff from invading is . This is merely the weighted average of Player 1's expected payoff when Player 2 is weak and her expected payoff when Player 2 is strong. Thus, invade is a better strategy than concede for Player 1 when . In other words, if the probability that Player 1 assigns to Player 2 being weak is greater than one-sixth, Player 1 should choose to invade in the first round. Otherwise, Player 1 should concede and be done with it. \n\nWhat's the outcome when you and your classmates play this more complicated version of the Escalation Game? \n\n## BURNING BRIDGES GAME \n\nThis game shares starkly similar features with the Escalation Game, but there is no uncertainty (thus, the analytical equilibrium is an SPE rather than a PBE). The SPE has much to say about the relationship between two tenacious competitors. Spaniel (2011) portrays the game as follows: \n\n12. This equilibrium is known as a Perfect Bayesian Equilibrium (PBE) rather than an SPE because of the uncertainty that at least one of the players is forced to contend with. Similar to Nash, Thomas Bayes is considered a towering figure. He was an 18th-century English statistician, philosopher, and Presbyterian minister who is known for formulating a specific case of the theorem that bears his name: Bayes Theorem. Bayes never published his theory himself-his notes were edited and published posthumously. \n\n132 ARTHUR J. CAPLAN" + }, + { + "bleu": 0.9574182855664638, + "doc_id": "doc_c98bfca166e04c05a1aa04e801e44dfc9282b1b112d230785873d1fb6930d4a4_page_000001.png", + "edit_distance": 0.02348993288590604, + "f1_score": 0.9880239520958083, + "meteor": 0.9975855180342325, + "precision": 0.9763313609467456, + "pred_md": "one of the two players is allowed to communicate with the other player (i.e., there is 'one-way communication') the players coordinate their choices 96% of the time! However, with simultaneous two-way communication between the two players, they coordinate only 42% of the time! Explain what happened.\n\n- 10. We demonstrated how to solve for the Penalty Kick game's mixed-strategy equilibrium. Suppose you were new to the game of soccer (or football) and assigned to play the goalie position. After watching the following YouTube video, what strategy might make the most sense for you to adopt on penalty kicks: https://www.youtube.com/watch?v=3yWZZR9ZodI.\n- 11. The map below identifies (with red markers) the locations of gas stations in Salt Lake City, Utah (Utah's capital city). Do these gas station locations depict a pure strategy equilibrium for the Hotelling Game? Explain.\n- 12. In this chapter, we learned that when an individual acquires private information about something, this added information does not necessarily make the individual better off. In particular, when an individual (say, Player 1) acquires private information about something of common interest to both himself and another individual (say, Player 2), and Player 2 knows Player 1 has acquired this private information, Player 1 could actually be made worse off as a result of Player 2 changing her strategy in response to the fact that she knows Player 1 now has additional information. Whew! Can you think of a real-life example where the acquisition\n\nSource: Google Maps\n\nBEHAVIORAL ECONOMICS PRACTICUM 175", + "recall": 1.0, + "true_md": "one of the two players is allowed to communicate with the other player (i.e., there is 'one-way communication') the players coordinate their choices 96% of the time! However, with simultaneous two-way communication between the two players, they coordinate only 42% of the time! Explain what happened. \n\n10. We demonstrated how to solve for the Penalty Kick game's mixed-strategy equilibrium. Suppose you were new to the game of soccer (or football) and assigned to play the goalie position. After watching the following YouTube video, what strategy might make the most sense for you to adopt on penalty kicks: https://www.youtube.com/watch?v=3yWZZR9ZodI. \n\n11. The map below identifies (with red markers) the locations of gas stations in Salt Lake City, Utah (Utah's capital city). Do these gas station locations depict a pure strategy equilibrium for the Hotelling Game? Explain. \n\n12. In this chapter, we learned that when an individual acquires private information about something, this added information does not necessarily make the individual better off. In particular, when an individual (say, Player 1) acquires private information about something of common interest to both himself and another individual (say, Player 2), and Player 2 knows Player 1 has acquired this private information, Player 1 could actually be made worse off as a result of Player 2 changing her strategy in response to the fact that she knows Player 1 now has additional information. Whew! Can you think of a real-life example where the acquisition \n\nBEHAVIORAL ECONOMICS PRACTICUM 175" + }, + { + "bleu": 0.994100037519881, + "doc_id": "doc_a59c54dbc6c578c176e9c6627245740637da434d62a4460d865528ac3a2eecd4_page_000001.png", + "edit_distance": 0.005917159763313609, + "f1_score": 1.0, + "meteor": 0.9994050106161839, + "precision": 1.0, + "pred_md": "(Pope and Schweitzer 2011)\n\nTo reiterate, this study's main econometric results reveal a negative effect on sinking a putt when the typical golfer is putting for birdie, and a positive effect on putting for bogey. Consistent with the previous graphs, these numerical results suggest that the typical professional golfer is more likely to sink a put for bogey and less likely to sink the putt for birdie (i.e., the typical golfer is indeed loss averse). 10\n\n## ARE CIGARETTE SMOKERS HYPERBOLIC TIME DISCOUNTERS?\n\nRecall from Chapter 4 the distinction between time-consistent exponential time discounters ( Homo economicus ) and potentially time-inconsistent hyperbolic discounters ( Homo sapiens ). The discounting time paths for exponential versus hyperbolic discounting looked like this:\n\n10. A negative effect associated with putting for double bogey suggests that the typical golfer suppresses his inclination for loss aversion when putting for a score worse than bogey.\n\nBEHAVIORAL ECONOMICS PRACTICUM 193", + "recall": 1.0, + "true_md": "Pope and Schweitzer 2011) \n\nTo reiterate, this study's main econometric results reveal a negative effect on sinking a putt when the typical golfer is putting for birdie, and a positive effect on putting for bogey. Consistent with the previous graphs, these numerical results suggest that the typical professional golfer is more likely to sink a put for bogey and less likely to sink the putt for birdie (i.e., the typical golfer is indeed loss averse). 10 \n\n## ARE CIGARETTE SMOKERS HYPERBOLIC TIME DISCOUNTERS? \n\nRecall from Chapter 4 the distinction between time-consistent exponential time discounters (Homo economicus) and potentially time-inconsistent hyperbolic discounters (Homo sapiens). The discounting time paths for exponential versus hyperbolic discounting looked like this: \n\n10. A negative effect associated with putting for double bogey suggests that the typical golfer suppresses his inclination for loss aversion when putting for a score worse than bogey. \n\nBEHAVIORAL ECONOMICS PRACTICUM 193" + }, + { + "bleu": 0.9823771291847392, + "doc_id": "doc_7675bb9557d6e1e663237098f37aaecb61ab411b71034c2d397487ed0deb2ecd_page_000001.png", + "edit_distance": 0.017391304347826087, + "f1_score": 0.9937888198757763, + "meteor": 0.9982328696348571, + "precision": 0.9876543209876543, + "pred_md": "## (Yoeli et al. 2013)\n\nOn a final note, Yoeli et al. provide evidence that indirect reciprocity among Homo sapiens is unique to public goods. Their hypothesis is that choosing not to participate in a demand response program should carry the threat of social sanctions only if participation is considered to be for the public good. To test their hypothesis, the authors solicited an additional 1,000 customers with exactly the same treatments as described above, except that the informational materials the customers received ahead of time to entice them to participate in the demand response program were stripped of any language\n\nBEHAVIORAL ECONOMICS PRACTICUM 213", + "recall": 1.0, + "true_md": "(Yoeli et al. 2013) \n\nOn a final note, Yoeli et al. provide evidence that indirect reciprocity among Homo sapiens is unique to public goods. Their hypothesis is that choosing not to participate in a demand response program should carry the threat of social sanctions only if participation is considered to be for the public good. To test their hypothesis, the authors solicited an additional 1,000 customers with exactly the same treatments as described above, except that the informational materials the customers received ahead of time to entice them to participate in the demand response program were stripped of any language \n\nBEHAVIORAL ECONOMICS PRACTICUM 213" + }, + { + "bleu": 0.9869074503141916, + "doc_id": "doc_c75d25277c58e07bf91528ae08deccc80715b6f7f13b5ed571cf226463b364e8_page_000001.png", + "edit_distance": 0.005509641873278237, + "f1_score": 0.9938837920489296, + "meteor": 0.9947553334977014, + "precision": 0.9938837920489296, + "pred_md": "[markets] build loyalty and-more important-make people want to extend themselves to the degree that corporations need today: to be flexible, concerned, and willing to pitch in. That's what a social relationship delivers.' (page 90)\n\nHence, in the less-predictable world of Homo sapiens , businesses must decide the extent to which they participate with their employees and customers in monetary and/or social markets.\n\nAs a follow-on to Heyman and Ariely's (2004) experiments exploring the payment-effort trade-off, Vohs et al. (2006) sought to understand the behavioral psychology underscoring the trade-off. In its most general terms, the authors' hypothesis is that money makes Homo sapiens feel self-sufficient and behave accordingly. When reminded of money, people desire to be free from dependency upon others and prefer that others not depend upon them. Vohs et al. designed several experiments to test this hypothesis from a variety of angles.\n\nIn one experiment, the authors found that participants (a sample of University of Minnesota students) who were reminded about money-both Monopoly money and real money-in the context of a series of word descrambling tasks worked longer at the tasks than participants in a non-moneyprimed control group before requesting help from the experimenter. 25 In subsequent experiments with different groups of students, Vohs et al. found that (1) participants in a high-money treatment worked significantly longer than participants in a low-money treatment before asking for help from another available participant, (2) participants in a money-primed treatment volunteered to help code fewer data sheets than did participants in the non-money-primed control condition, (3) participants in a high-money treatment volunteered to gather fewer pencils that had spilled onto the floor than did participants in a low-money treatment, and (4) participants in a money-primed treatment donated significantly less money to a university student fund than participants in the non-money primed control. Three final experiments tested the effects of money on social intimacy, desire to engage in leisure activities alone, and preference to work alone. As expected, participants who were primed with money ahead of time were subsequently less socially intimate and exhibited a stronger preference for engaging in leisure activities and working alone.\n\nSo yes, Vohs et al.'s experiments suggest that money makes Homo sapiens feel self-sufficient and behave accordingly.\n\n## PRICE AND THE PLACEBO EFFECT\n\nIs it possible that the magnitudes of placebo effects experienced by Homo sapiens (e.g., through medical therapies or medications) are somehow influenced by the prices we pay for them? To investigate this possibility, Waber et al. (2008) studied the effect of price on a group of Homo sapiens ' analgesic responses to placebo pills. Over 80 healthy volunteers in Boston, MA were recruited via an online advertisement to participate in a field experiment where each participant was informed by a brochure about a purported new opioid analgesic recently approved by the Food and Drug Administration. The opioid was described as similar to codeine but with a faster onset time. In reality, and not disclosed to the participants, the pill was a placebo. After randomization, half of the participants were informed that the drug had a regular price of $2.50 per pill ('regular price'), and half of the participants that\n\n25. The descrambling task consisted of 30 sets of five jumbled words. Participants created sensible phrases using four of the five words. In the control and play-money treatment, the phrases primed neutral concepts (e.g., 'cold it desk outside is' became 'it is cold outside'). In the real-money treatment, 15 of the phrases primed the concept of money (e.g., 'high a salary desk paying' became 'a high-paying salary'), whereas the remaining 15 were neutral phrases. Participants in the playmoney treatment were primed with money by a stack of Monopoly money in their visual periphery while completing the neutral descrambling task.\n\n220 ARTHUR J. CAPLAN", + "recall": 0.9938837920489296, + "true_md": "[markets] build loyalty and-more important-make people want to extend themselves to the degree that corporations need today: to be flexible, concerned, and willing to pitch in. That's what a social relationship delivers.' (page 90) \n\nHence, in the less-predictable world of Homo sapiens, businesses must decide the extent to which they participate with their employees and customers in monetary and/or social markets. \n\nAs a follow-on to Heyman and Ariely's (2004) experiments exploring the payment-effort trade-off, Vohs et al. (2006) sought to understand the behavioral psychology underscoring the trade-off. In its most general terms, the authors' hypothesis is that money makes Homo sapiens feel self-sufficient and behave accordingly. When reminded of money, people desire to be free from dependency upon others and prefer that others not depend upon them. Vohs et al. designed several experiments to test this hypothesis from a variety of angles. \n\nIn one experiment, the authors found that participants (a sample of University of Minnesota students) who were reminded about money-both Monopoly money and real money-in the context of a series of word descrambling tasks worked longer at the tasks than participants in a non-money- primed control group before requesting help from the experimenter. 25 In subsequent experiments with different groups of students, Vohs et al. found that (1) participants in a high-money treatment worked significantly longer than participants in a low-money treatment before asking for help from another available participant, (2) participants in a money-primed treatment volunteered to help code fewer data sheets than did participants in the non-money-primed control condition, (3) participants in a high-money treatment volunteered to gather fewer pencils that had spilled onto the floor than did participants in a low-money treatment, and (4) participants in a money-primed treatment donated significantly less money to a university student fund than participants in the non-money primed control. Three final experiments tested the effects of money on social intimacy, desire to engage in leisure activities alone, and preference to work alone. As expected, participants who were primed with money ahead of time were subsequently less socially intimate and exhibited a stronger preference for engaging in leisure activities and working alone. \n\nSo yes, Vohs et al.'s experiments suggest that money makes Homo sapiens feel self-sufficient and behave accordingly. \n\n## PRICE AND THE PLACEBO EFFECT \n\nIs it possible that the magnitudes of placebo effects experienced by Homo sapiens (e.g., through medical therapies or medications) are somehow influenced by the prices we pay for them? To investigate this possibility, Waber et al. (2008) studied the effect of price on a group of Homo sapiens' analgesic responses to placebo pills. Over 80 healthy volunteers in Boston, MA were recruited via an online advertisement to participate in a field experiment where each participant was informed by a brochure about a purported new opioid analgesic recently approved by the Food and Drug Administration. The opioid was described as similar to codeine but with a faster onset time. In reality, and not disclosed to the participants, the pill was a placebo. After randomization, half of the participants were informed that the drug had a regular price of $2.50 per pill ('regular price'), and half of the participants that \n\n25. The descrambling task consisted of 30 sets of five jumbled words. Participants created sensible phrases using four of the five words. In the control and play-money treatment, the phrases primed neutral concepts (e.g., 'cold it desk outside is' became 'it is cold outside'). In the real-money treatment, 15 of the phrases primed the concept of money (e.g., 'high a salary desk paying' became 'a high-paying salary'), whereas the remaining 15 were neutral phrases. Participants in the play- money treatment were primed with money by a stack of Monopoly money in their visual periphery while completing the neutral descrambling task. \n\n220 ARTHUR J. CAPLAN" + }, + { + "bleu": 0.9962370482284509, + "doc_id": "doc_872d56afecf02a9a4ff4ec021e4347ead2b7f227e9299ff85fee4e3c4993788e_page_000001.png", + "edit_distance": 0.0037105751391465678, + "f1_score": 0.9980657640232108, + "meteor": 0.9996276959532957, + "precision": 0.9961389961389961, + "pred_md": "## (Kaza et al. 2018)\n\nCanada is currently the world's largest producer of MSW per capita. At slightly more than 36 metric tons per person per year, Canadians generate roughly 10 tons more MSW per person annually than the next highest garbage producers, Bulgarians and Americans (Tiseo, 2021). Summiting a list like this is obviously not in any country's best interest-there are no kudos for reaching the top of the heap, so to speak. Is it therefore possible that those nations reaching the top will take the lead in reversing course?\n\nHalifax is one Canadian city that apparently has. On August 1st, 2015, the city began providing a 'green nudge' to citizens living in its urban core area with the introduction of the Clear Bag Policy, a policy designed to nudge households toward more responsible sorting of their waste, which, in turn, would result in an overall reduction in the total amount of waste generated. As Akbulut-Yuksel and Boulatoff point out, under the new policy, households were mandated to replace their black garbage bags, traditionally used for the disposal of their refuse, with clear, transparent bags. The Clear Bag Policy allowed households to put out the same number of garbage bags at the curb (six every other week), but all waste destined for the landfill was required to be disposed of in a clear bag (except for one dark bag permitted for privacy's sake). This allowed waste collectors to screen and refuse any bags containing materials that should otherwise have been diverted from the landfill, such as recyclables, food waste, and hazardous waste. Clear bags also made apparent to everyone, neighbors and passersby alike, a given household's waste-generation and disposal habits. 33\n\nTo test the Clear Bag Policy's impact on a typical household's generation of MSW, Akbulut-Yuksel and Boulatoff designed a quasi-experiment spanning the period from January 6, 2014, to July 28, 2017, with January 6, 2014, to July 31, 2015, serving as the pre-treatment period and August 1, 2015, to July 28, 2017, serving as the post-treatment period. MSW data collected during this time span\n\n33. As Akbulut-Yuksel and Boulatoff point out, Halifax households are required to sort waste in four ways: (1) recyclable containers (plastics, glass, and aluminum) are put in a transparent blue bag, (2) paper and cardboard are put in a separate bag, (3) organic food waste goes in a green bin provided by the city, and (4) the remaining waste (refuse) goes into garbage bags. Recyclable materials are collected each week, while garbage and organic waste are each collected every other week on opposite weeks (except in the summer months when, thank goodness, organic waste is collected on a weekly basis). 234 ARTHUR J. CAPLAN", + "recall": 1.0, + "true_md": "(Kaza et al. 2018) \n\nCanada is currently the world's largest producer of MSW per capita. At slightly more than 36 metric tons per person per year, Canadians generate roughly 10 tons more MSW per person annually than the next highest garbage producers, Bulgarians and Americans (Tiseo, 2021). Summiting a list like this is obviously not in any country's best interest-there are no kudos for reaching the top of the heap, so to speak. Is it therefore possible that those nations reaching the top will take the lead in reversing course? \n\nHalifax is one Canadian city that apparently has. On August 1st, 2015, the city began providing a 'green nudge' to citizens living in its urban core area with the introduction of the Clear Bag Policy, a policy designed to nudge households toward more responsible sorting of their waste, which, in turn, would result in an overall reduction in the total amount of waste generated. As Akbulut-Yuksel and Boulatoff point out, under the new policy, households were mandated to replace their black garbage bags, traditionally used for the disposal of their refuse, with clear, transparent bags. The Clear Bag Policy allowed households to put out the same number of garbage bags at the curb (six every other week), but all waste destined for the landfill was required to be disposed of in a clear bag (except for one dark bag permitted for privacy's sake). This allowed waste collectors to screen and refuse any bags containing materials that should otherwise have been diverted from the landfill, such as recyclables, food waste, and hazardous waste. Clear bags also made apparent to everyone, neighbors and passersby alike, a given household's waste-generation and disposal habits. 33 \n\nTo test the Clear Bag Policy's impact on a typical household's generation of MSW, Akbulut-Yuksel and Boulatoff designed a quasi-experiment spanning the period from January 6, 2014, to July 28, 2017, with January 6, 2014, to July 31, 2015, serving as the pre-treatment period and August 1, 2015, to July 28, 2017, serving as the post-treatment period. MSW data collected during this time span \n\n33. As Akbulut-Yuksel and Boulatoff point out, Halifax households are required to sort waste in four ways: (1) recyclable containers (plastics, glass, and aluminum) are put in a transparent blue bag, (2) paper and cardboard are put in a separate bag, (3) organic food waste goes in a green bin provided by the city, and (4) the remaining waste (refuse) goes into garbage bags. Recyclable materials are collected each week, while garbage and organic waste are each collected every other week on opposite weeks (except in the summer months when, thank goodness, organic waste is collected on a weekly basis). 234 ARTHUR J. CAPLAN" + }, + { + "bleu": 0.9850996156111327, + "doc_id": "doc_2cd17a32ee330a239e19c915738df0c27e8ec3635a60a7e16e2a0cf3868d4af3_page_000001.png", + "edit_distance": 0.009302325581395349, + "f1_score": 0.9924242424242425, + "meteor": 0.9947209672424148, + "precision": 0.9924242424242424, + "pred_md": "WITH CHATGPT\n\n## СREATING SLIDES\n\n## 01 - Find Open Educational Resources\n\nStart by searching for information on platforms like OER Commons, where authors share their materials freely, ensuring no copyright issues.\n\n## 02- Prepare Your Content\n\nSummarize or extract the key points from the materials you've found. This will be the content for your slides.\n\n## 03- Generate Slides with ChatGPT\n\nProvide the summarized content to ChatGPT and instruct it to create a structured outline for Google Slides, including titles, main points, and any specific instructions for slide design.\n\n## 04 - Create App Script Code\n\nAfter finalizing the slide structure, ask ChatGPT to generate a Google Apps Script code that can create these slides automatically.\n\n## 05 - Execute in Google Apps Script\n\nOpen Google Apps Script, start a new project, and paste the code provided by ChatGPT. Run the script to auto-generate your slide deck.\n\n## 06 - Edit and Customize\n\nOnce the slides are created, you can further edit and customize them in Google Slides according to your needs.\n\nINTERESTED IN FREE AI-CONSULTANCE OR COLLABORATION WITH US?\n\nEMAIL REBECCA.ALLEN@MSJ.EDU F OR MORE INFORMATION", + "recall": 0.9924242424242424, + "true_md": " WITH CHATGPT\n\n## СREATING SLIDES\n\n## 01 - Find Open Educational Resources\n\nStart by searching for information on platforms like OER Commons, where authors share their materials freely, ensuring no copyright issues.\n\n## 02- Prepare Your Content\n\nSummarize or extract the key points from the materials you've found. This will be the content for your slides.\n\n## 03- Generate Slides with ChatGPT\n\nProvide the summarized content to ChatGPT and instruct it to create a structured outline for Google Slides, including titles, main points, and any specific instructions for slide design.\n\n## 04 - Create App Script Code\n\nAfter finalizing the slide structure, ask ChatGPT to generate a Google Apps Script code that can create these slides automatically.\n\n## 05 - Execute in Google Apps Script\n\nOpen Google Apps Script, start a new project, and paste the code provided by ChatGPT. Run the script to auto-generate your slide deck.\n\n## 06 - Edit and Customize\n\nOnce the slides are created, you can further edit and customize them in Google Slides according to your needs.\n\nINTERESTED IN FREE AI-CONSULTANCE OR COLLABORATION WITH US?\n\nEMAIL REBECCA.ALLEN@MSJ.EDU FOR MORE INFORMATION " + }, + { + "bleu": 0.9163334759038975, + "doc_id": "doc_f01f786e76db9a4e8e321342660e2a539d4b060e123b42a58ec84df128455d59_page_000001.png", + "edit_distance": 0.046511627906976744, + "f1_score": 0.9867841409691629, + "meteor": 0.973857934883517, + "precision": 0.9824561403508771, + "pred_md": "An overview of each actor's role in this ecosystem is described below.\n\n## Publishers\n\nPublishers work to 'make public' scholarly work in the form of textbooks, journals, and monographs, and represent a wide range of publishing approaches, business models, budgets, and institutional affiliations. With our focus on monographs, the two most significant groups are large commercial publishers and university presses. These publish the vast majority of monographs in circulation, although in recent years, smaller open access publishers have also begun to emerge.\n\nThe role of publishers includes (among other things):\n\n- · acquisitions and list curation\n- · editorial work and coordinating peer review\n- · design and production (for various formats, typically: print, digital PDF, and EPUB)\n- · distribution and marketing of finished products into various channels (libraries, aggregators, stores) where readers can access books\n\n6 | The Scholarly Publishing Ecosystem", + "recall": 0.9911504424778761, + "true_md": "An overview of each actor's role in this ecosystem is described below.\n\n## Publishers\n\nPublishers work to 'make public' scholarly work in the form of textbooks, journals, and monographs, and represent a wide range of publishing approaches, business models, budgets, and institutional affiliations. With our focus on monographs, the two most significant groups are large commercial publishers and university presses. These publish the vast majority of monographs in circulation, although in recent years, smaller open access publishers have also begun to emerge.\n\nThe role of publishers includes (among other things):\n\n• acquisitions and list curation\n\n• editorial work and coordinating peer review\n\n• design and production (for various formats, typically: print, digital PDF, and EPUB)\n\n• distribution and marketing of finished products into various channels (libraries, aggregators, stores) where readers can access books\n\n6 | The Scholarly Publishing Ecosystem" + }, + { + "bleu": 0.957767687479268, + "doc_id": "doc_c8f35048844313b77c8c6d6a059b9f91a3b5e34f30e7a774a0f2d8fca82101eb_page_000001.png", + "edit_distance": 0.023809523809523808, + "f1_score": 0.9861751152073733, + "meteor": 0.9867599161650242, + "precision": 0.981651376146789, + "pred_md": "## The Scholarly Publishing Cycle\n\nHaving explored the scholarly publishing ecosystem and its primary relationships, we can update the cycle as follows:\n\nOur project set out to explore and address the shortfall in serving the scholarly reader identified in this section. This shortfall is made clear in two connected points:\n\n- · Scholarly readers are not just content consumers; scholarly reading is an act of creation as well.\n- · Publishers and aggregators are not incentivized to create better tools to support scholarly reading.\n\nFrom here, this report will consider the experiences of publishers, librarians and readers through a synthesis of interviews conducted with several members of each group, as well as a short online survey aimed at readers. We will then share some of our own philosophy on the future of scholarly reading, then detail the path forward we see for our own work in the area.\n\n10 | The Scholarly Publishing Ecosystem", + "recall": 0.9907407407407407, + "true_md": "## The Scholarly Publishing Cycle\n\nHaving explored the scholarly publishing ecosystem and its primary relationships, we can update the cycle as follows:\n\nOur project set out to explore and address the shortfall in serving the scholarly reader identified in this section. This shortfall is made clear in two connected points:\n\n• Scholarly readers are not just content consumers; scholarly reading is an act of creation as well.\n\n• Publishers and aggregators are not incentivized to create better tools to support scholarly reading.\n\nFrom here, this report will consider the experiences of publishers, librarians and readers through a synthesis of interviews conducted with several members of each group, as well as a short online survey aimed at readers. We will then share some of our own philosophy on the future of scholarly reading, then detail the path forward we see for our own work in the area.\n\n10 | The Scholarly Publishing Ecosystem" + }, + { + "bleu": 1.0, + "doc_id": "doc_1f051b82d968ba073f99a2f78e15e82072c9986127614e36eb8c1fa1ccb0d261_page_000001.png", + "edit_distance": 0.0, + "f1_score": 1.0, + "meteor": 0.9999999676129074, + "precision": 1.0, + "pred_md": "An example of a conceptual map created by one of our interviewees\n\nIt seemed at times that the remarkable freedom of writing freeform allowed these languages to form, but it was difficult, if not impossible, to replicate that freedom on available digital tools. Printing out articles or chapters of interest and annotating them with pen or pencil is still seen as the way to go by many. Having physical copies on hand also means easier management as this benefits from the very natural use of space for arranging things, e.g.: 'The pile on the right contains my primary sources; on the left are things I've flagged as potentially interesting and to revisit.' Often mentioned was the use of digital editions for quick consultation and search, but print versions for in-depth reading and annotation. Most collect important works in print.\n\nWhile some note taking did take place alongside annotation, each of our researchers would reach a point where they needed to take the texts they had read and turn the notes, quotes, and other takeaways into something they could then begin to incorporate into their writing. Again, the approaches to this varied widely, and depended on the tools used initially. Some would take handwritten annotations and highlighting and type them into a word processor. Others would export annotations from tools in whatever\n\n32 | Considering Scholarly Readers", + "recall": 1.0, + "true_md": "An example of a conceptual map created by one of our interviewees\n\nIt seemed at times that the remarkable freedom of writing freeform allowed these languages to form, but it was difficult, if not impossible, to replicate that freedom on available digital tools. Printing out articles or chapters of interest and annotating them with pen or pencil is still seen as the way to go by many. Having physical copies on hand also means easier management as this benefits from the very natural use of space for arranging things, e.g.: 'The pile on the right contains my primary sources; on the left are things I've flagged as potentially interesting and to revisit.' Often mentioned was the use of digital editions for quick consultation and search, but print versions for in-depth reading and annotation. Most collect important works in print.\n\nWhile some note taking did take place alongside annotation, each of our researchers would reach a point where they needed to take the texts they had read and turn the notes, quotes, and other takeaways into something they could then begin to incorporate into their writing. Again, the approaches to this varied widely, and depended on the tools used initially. Some would take handwritten annotations and highlighting and type them into a word processor. Others would export annotations from tools in whatever\n\n32 | Considering Scholarly Readers" + }, + { + "bleu": 0.519083796763266, + "doc_id": "doc_4b667a2eb374a8566044a6c90f5641fb0aa4a7f2127350296f2a038b59c00208_page_000001.png", + "edit_distance": 0.4383561643835616, + "f1_score": 0.8409090909090908, + "meteor": 0.8945399877793061, + "precision": 0.7254901960784313, + "pred_md": "## Print vs. Digital\n\nWhy do some researchers abhor digital and favor print, or vice-versa? The classic print vs. digital debate was necessary for us to understand readers' preferences with each\n\nQ11 What factors influence your choice of print? (select all that apply)\n\nformat.\n\n## Q12 What factors influence your choice of digital? (select all that apply)\n\nOnline Survey |\n\n39", + "recall": 1.0, + "true_md": "## Print vs. Digital\n\nWhy do some researchers abhor digital and favor print, or vice-versa? The classic print vs. digital debate was necessary for us to understand readers' preferences with each format.\n\nOnline Survey | 39" + }, + { + "bleu": 0.9642483698224412, + "doc_id": "doc_3e5b086e7392fbe139a3c42dbcf1442d95c4d0d97a5671e15cbeee2191e59747_page_000001.png", + "edit_distance": 0.025906735751295335, + "f1_score": 1.0, + "meteor": 0.9900985406820844, + "precision": 1.0, + "pred_md": "## CONTENTS\n\nvii\n\n| About the Publisher | vii |\n|--------------------------------------------------|-------|\n| About This Project | ix |\n| Acknowledgments | xi |\n| LABMANUAL | |\n| Experiment #1: Hydrostatic Pressure | 3 |\n| Experiment #2: Bernoulli's Theorem Demonstration | 13 |\n| Experiment #3: Energy Loss in Pipe Fittings | 24 |\n| Experiment #4: Energy Loss in Pipes | 33 |\n| Experiment #5: Impact of a Jet | 43 |\n| Experiment #6: Orifice and Free Jet Flow | 50 |\n| Experiment #7: Osborne Reynolds' Demonstration | 59 |\n| Experiment #8: Free and Forced Vortices | 66 |\n| Experiment #9: Flow Over Weirs | 76 |\n| Experiment #10: Pumps | 84 |\n| References | 101 |\n| Links by Chapter | 102 |\n| Image Credits | 104 |", + "recall": 1.0, + "true_md": "## CONTENTS \n\n| About the Publisher | vii |\n|--------------------------------------------------|-----------|\n| About This Project | ix |\n| Acknowledgments | xi |\n| | LABMANUAL |\n| Experiment #1: Hydrostatic Pressure | 3 |\n| Experiment #2: Bernoulli's Theorem Demonstration | 13 |\n| Experiment #3: Energy Loss in Pipe Fittings | 24 |\n| Experiment #4: Energy Loss in Pipes | 33 |\n| Experiment #5: Impact of a Jet | 43 |\n| Experiment #6: Orifice and Free Jet Flow | 50 |\n| Experiment #7: Osborne Reynolds' Demonstration | 59 |\n| Experiment #8: Free and Forced Vortices | 66 |\n| Experiment #9: Flow Over Weirs | 76 |\n| Experiment #10: Pumps | 84 |\n| References | 101 |\n| Links by Chapter | 102 |\n| Image Credits | 104 |" + }, + { + "bleu": 0.9247626920288713, + "doc_id": "doc_462529434b77fc469f72c781c14663f8a787e4785af189268f5aeac71e823aba_page_000001.png", + "edit_distance": 0.0430622009569378, + "f1_score": 0.9751243781094527, + "meteor": 0.9779339853300734, + "precision": 0.98, + "pred_md": "the jet velocity can be assumed to remain constant. Therefore, the horizontal distance traveled by jet (x) in time (t) is equal to:\n\nThe vertical component of the trajectory of the jet will have a constant acceleration downward due to the force of gravity. Therefore, at any time, t, the y-position of the jet may be calculated as:\n\nRearranging Equation (8) gives:\n\nSubstitution of t and v from Equations 9 and 2 into Equation 7 results in:\n\nEquations (10) can be rearranged to find C v:\n\nTherefore, for steady flow conditions (i.e., constant h in the head tank), the value of C v can be determined from the x, y coordinates of the jet trajectory. A graph of x plotted against will have a slope of 2 C . v\n\n## 7.2. DETERMINATION OF THE COEFFICIENT OF DISCHARGE\n\nIf C d is assumed to be constant, then a graph of Q plotted against (Equation 6) will be linear, and the slope of this graph will be:\n\nEXPERIMENT #6: ORIFICE AND FREE JET FLOW 53", + "recall": 0.9702970297029703, + "true_md": "the jet velocity can be assumed to remain constant. Therefore, the horizontal distance traveled by jet (x) in time (t) is equal to: \n\nThe vertical component of the trajectory of the jet will have a constant acceleration downward due to the force of gravity. Therefore, at any time, t, the y-position of the jet may be calculated as: \n\nRearranging Equation (8) gives: \n\nSubstitution of t and v from Equations 9 and 2 into Equation 7 results in: \n\nEquations (10) can be rearranged to find Cv: \n\nTherefore, for steady flow conditions (i.e., constant h in the head tank), the value of Cv can be determined from the x, y coordinates of the jet trajectory. A graph of x plotted against will have a slope of 2Cv. \n\n## 7.2. DETERMINATION OF THE COEFFICIENT OF DISCHARGE \n\nIf Cd is assumed to be constant, then a graph of Q plotted against (Equation 6) will be linear, and the slope of this graph will be: \n\nEXPERIMENT #6: ORIFICE AND FREE JET FLOW 53" + }, + { + "bleu": 0.9847317428952738, + "doc_id": "doc_cd160a4e2296bd7c23f19c26af5bb6ed772f766ee1eacfe9998703c38146b494_page_000001.png", + "edit_distance": 0.008333333333333333, + "f1_score": 0.9865470852017937, + "meteor": 0.9920797389892377, + "precision": 0.990990990990991, + "pred_md": "in the flow. There is also a transitional stage between laminar and turbulent flows, in which the dye stream will wander about and show intermittent bursts of mixing, followed by a more laminar behavior.\n\nThe Reynolds number ( Re ), provides a useful way of characterizing the flow. It is defined as:\n\nwhere ( ) is the kinematic viscosity of the water (Figure 7.2), v is the mean flow velocity and d is the diameter of the pipe.\n\nThe Reynolds number is a dimensionless parameter that is the ratio of the inertial (destabilizing) force to the viscosity (stabilizing) force. As Re increases, the inertial force becomes relatively larger, and the flow destabilizes and becomes fully turbulent.\n\nThe Reynolds experiment determines the critical Reynolds number for pipe flow at which laminar flow ( Re<2000 ) becomes transitional ( 2000<Re<4000 ) and the transitional flow becomes turbulent ( Re>4000 ). The advantage of using a critical Reynolds number, instead of critical velocity, is that the results of the experiments are applicable to all Newtonian fluid flows in pipes with a circular crosssection.\n\nFigure 7.2: Kinematic Viscosity of Water at Atmospheric Pressure.\n\nEXPERIMENT #7: OSBORNE REYNOLDS' DEMONSTRATION 61", + "recall": 0.9821428571428571, + "true_md": "in the flow. There is also a transitional stage between laminar and turbulent flows, in which the dye stream will wander about and show intermittent bursts of mixing, followed by a more laminar behavior. \n\nThe Reynolds number (Re), provides a useful way of characterizing the flow. It is defined as: \n\nwhere ( ) is the kinematic viscosity of the water (Figure 7.2), v is the mean flow velocity and d is the diameter of the pipe. \n\nThe Reynolds number is a dimensionless parameter that is the ratio of the inertial (destabilizing) force to the viscosity (stabilizing) force. As Re increases, the inertial force becomes relatively larger, and the flow destabilizes and becomes fully turbulent. \n\nThe Reynolds experiment determines the critical Reynolds number for pipe flow at which laminar flow (Re<2000 ) becomes transitional (2000<Re<4000 ) and the transitional flow becomes turbulent (Re>4000). The advantage of using a critical Reynolds number, instead of critical velocity, is that the results of the experiments are applicable to all Newtonian fluid flows in pipes with a circular cross- section. \n\nFigure 7.2: Kinematic Viscosity of Water at Atmospheric Pressure. \n\nEXPERIMENT #7: OSBORNE REYNOLDS' DEMONSTRATION 61" + }, + { + "bleu": 1.0, + "doc_id": "doc_fad8cdcee7f7884bddedf0ee485bf259f815315699f35ef82b2afca5fb28f68a_page_000001.png", + "edit_distance": 0.0, + "f1_score": 1.0, + "meteor": 0.9999999365530463, + "precision": 1.0, + "pred_md": "Figure 8.1: a) P6238 CUSSONS free and forced vortex apparatus, b) push-in orifices, c) free vortex measuring caliper, d) force vortex measuring probes\n\n## 7. THEORY\n\nTwo types of vortices are distinguished in the dynamics of the motion: forced and free vortices. The forced vortex is caused by external forces on the fluid, such as the impeller of a pump, and the free vortex naturally occurs in the flow and can be observed in a drain or in the atmosphere of a tornado.\n\n## 7.1. FREE VORTEX\n\nA free vortex is formed when water flows out of a vessel through a central hole in the base (Figure 8.2). The degree of the rotation depends on the initial disturbance. In a free cylindrical vortex, the velocity varies inversely with the distance from the axis of rotation (Figure 8.3).\n\nThe equation governing the surface profile is derived from the Bernoulli's theorem:\n\nSubstituting Equation (1) into (2) will give a new expression:\n\nor:\n\n68 APPLIED FLUID MECHANICS LAB MANUAL", + "recall": 1.0, + "true_md": "Figure 8.1: a) P6238 CUSSONS free and forced vortex apparatus, b) push-in orifices, c) free vortex measuring caliper, d) force vortex measuring probes \n\n## 7. THEORY \n\nTwo types of vortices are distinguished in the dynamics of the motion: forced and free vortices. The forced vortex is caused by external forces on the fluid, such as the impeller of a pump, and the free vortex naturally occurs in the flow and can be observed in a drain or in the atmosphere of a tornado. \n\n## 7.1. FREE VORTEX \n\nA free vortex is formed when water flows out of a vessel through a central hole in the base (Figure 8.2). The degree of the rotation depends on the initial disturbance. In a free cylindrical vortex, the velocity varies inversely with the distance from the axis of rotation (Figure 8.3). \n\nThe equation governing the surface profile is derived from the Bernoulli's theorem: \n\nSubstituting Equation (1) into (2) will give a new expression: \n\nor: \n\n68 APPLIED FLUID MECHANICS LAB MANUAL" + }, + { + "bleu": 0.9037419518509685, + "doc_id": "doc_7abbca82ea23276bbd534ca90a667ede80429690d2f06ae0e83459f75c42670f_page_000001.png", + "edit_distance": 0.06349206349206349, + "f1_score": 0.9826989619377162, + "meteor": 0.9640335115181761, + "precision": 0.9793103448275862, + "pred_md": "- · Adjust the point gauge to read 10 mm greater than the datum.\n- · Record the reading as h .\n- · Turn on the pump, and slightly adjust the flow until the water level coincides with the point gauge. Check that the level has stabilized before taking readings.\n- · Measure the flow rate using the volumetric tank.\n- · Observe the shape of the nappe and take pictures of it.\n\nNote : The surface of the water will fall as it approaches the weir. This is particularly noticeable at high flow rates by high heads. To obtain an accurate measurement of the undisturbed water level above the crest of the weir, it is necessary to place the measuring gauge at a distance of at least three times the head above the weir.\n\n- · Increase the flow by opening the bench regulating valve to set the heads above the datum level in 10 mm increments until the regulating valve is fully open. Take care not to allow spillage to occur over the plate top that is adjacent to the notch. At each condition, measure the flow rate and observe the shape of the nappe.\n\nNote : To obtain a sufficiently accurate result, collect around 25 liters of water each time, or collect the water for at least 120 seconds.\n\n- · Close the regulating valve, stop the pump, and then replace the weir with the V-notch.\n- · Repeat the experiment with the V-notch weir plate, but with 5 mm increments in water surface elevation.\n- · Collect seven head and discharge readings for each weir.\n\nFigure 9.3: Position of the notch and Vernier height gauge to set the datum.\n\n80 APPLIED FLUID MECHANICS LAB MANUAL", + "recall": 0.9861111111111112, + "true_md": "• Adjust the point gauge to read 10 mm greater than the datum. \n\n• Record the reading as h. \n\n• Turn on the pump, and slightly adjust the flow until the water level coincides with the point gauge. Check that the level has stabilized before taking readings. \n\n• Measure the flow rate using the volumetric tank. \n\n• Observe the shape of the nappe and take pictures of it. \n\nNote: The surface of the water will fall as it approaches the weir. This is particularly noticeable at high flow rates by high heads. To obtain an accurate measurement of the undisturbed water level above the crest of the weir, it is necessary to place the measuring gauge at a distance of at least three times the head above the weir. \n\n• Increase the flow by opening the bench regulating valve to set the heads above the datum level in 10 mm increments until the regulating valve is fully open. Take care not to allow spillage to occur over the plate top that is adjacent to the notch. At each condition, measure the flow rate and observe the shape of the nappe. \n\nNote: To obtain a sufficiently accurate result, collect around 25 liters of water each time, or collect the water for at least 120 seconds. \n\n• Close the regulating valve, stop the pump, and then replace the weir with the V-notch. \n\n• Repeat the experiment with the V-notch weir plate, but with 5 mm increments in water surface elevation. \n\n• Collect seven head and discharge readings for each weir. \n\nFigure 9.3: Position of the notch and Vernier height gauge to set the datum. \n\n80 APPLIED FLUID MECHANICS LAB MANUAL" + }, + { + "bleu": 0.9103637165083404, + "doc_id": "doc_f17954b5fdd9abdb6b4ec921c26ef9949f8c3f34b35b4efae11d57f49efa8283_page_000001.png", + "edit_distance": 0.2608695652173913, + "f1_score": 0.9342560553633218, + "meteor": 0.7367947675198122, + "precision": 0.9440559440559441, + "pred_md": "MOHAVE COMMUNITY COLLEGE BIO181\n\n## Table of Contents\n\n| Measurement Lab worksheet...................................................................................... 3 |\n|-----------------------------------------------------------------------------------------------------------------------------|\n| Scientific Method Lab.................................................................................................. 6 |\n| Chemistry of the Cell ~ But this is biology!........................................... 9 |\n| Biological Macromolecules and Their Indicators............................. 10 |\n| Worksheet for Chemistry of the Cell....................................................... 12 |\n| How molecules move in a liquid............................................................................. 12 |\n| How molecules move in a solid.............................................................................. 12 |\n| Introduction to Light Microscopes:........................................................................... 16 |\n| CellularBiology……………………………………………………………………………………………32 |\n| A cell is the smallest unit of life known to our planet................... 33 |\n| Cellular Microscopy......................................................................................... 34 |\n| Viewing prepared slides under a microscope................................. 34 |\n| Viewing live cells under a microscope............................................... 34 |\n| Cellular Biology Worksheet....................................................................................... 35 |\n| Osmosis and Diffusion ............................................................................................... 39 |\n| Enzymatic Activity Lab.............................................................................................. 45 |\n| Cellular Respiration Lab............................................................................................ 49 |\n| Photosynthesis Lab ................................................................................................... 61 |\n| Observing Stomata, Guard Cells and Chloroplasts............................................. 65 |\n| Cellular Replication ................................................................................................... 66 |\n| Growth and the Creation of Life......................................................................... 66 |\n| Visualizing the Cell Cycle, Mitosis, and Meiosis............................................. 67 |\n| Whenitall goes wrong…..................................................................................... 68 |\n| Cellular Replication Worksheet ......................................................................... 69 |\n| Mammalian Gametogenesis .............................................................................. 72 |\n| Genetic Crosses......................................................................................................... 75 |\n| MENDELIAN GENETICS, PROBABILITY, PEDIGREES ANDCHI-SQUARE STATISTICS . 80 |\n| Chi-Square Data Table................................................................................................... 92 |\n\n1", + "recall": 0.9246575342465754, + "true_md": "MOHAVE COMMUNITY COLLEGE BIO181 \n\n## Table of Contents \n\n| Measurement Lab worksheet...................................................................................... | 3 |\n|--------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------|\n| Scientific Method Lab.................................................................................................. | 6 |\n| Chemistry of the Cell ~ But this is biology!........................................... | 9 |\n| Biological Macromolecules and Their Indicators............................. | 10 |\n| Worksheet for Chemistry of the Cell ....................................................... | 12 |\n| How molecules move in a liquid............................................................................. | 12 |\n| How molecules move in a solid.............................................................................. | 12 |\n| Introduction to Light Microscopes:........................................................................... | 16 |\n| | CellularBiology.........................................................................................................32 |\n| A cell is the smallest unit of life known to our planet................... | 33 |\n| Cellular Microscopy ......................................................................................... | 34 |\n| Viewing prepared slides under a microscope. ................................ | 34 |\n| Viewing live cells under a microscope. .............................................. | 34 |\n| Cellular Biology Worksheet ....................................................................................... | 35 |\n| Osmosis and Diffusion ............................................................................................... | 39 |\n| Enzymatic Activity Lab.............................................................................................. | 45 |\n| Cellular Respiration Lab............................................................................................ | 49 |\n| Photosynthesis Lab ................................................................................................... | 61 |\n| Observing Stomata, Guard Cells and Chloroplasts............................................. | 65 |\n| Cellular Replication ................................................................................................... | 66 |\n| Growth and the Creation of Life......................................................................... | 66 |\n| Visualizing the Cell Cycle, Mitosis, and Meiosis............................................. | 67 |\n| When it all goes wrong........................................................................................ | 68 |\n| Cellular Replication Worksheet ......................................................................... | 69 |\n| Mammalian Gametogenesis | .............................................................................. |\n| | 72 |\n| Genetic Crosses......................................................................................................... | 75 |\n| MENDELIAN GENETICS, PROBABILITY, PEDIGREES AND CHI-SQUARE STATISTICS . | 80 |\n| Chi-Square Data Table................................................................................................... | 92 |\n\n1 " + }, + { + "bleu": 0.96146750070041, + "doc_id": "doc_be866d1b599b669a33acbc07b07847d6c72011cf4d221b20d889ad5a90ab30f5_page_000001.png", + "edit_distance": 0.06976744186046512, + "f1_score": 1.0, + "meteor": 0.9288869245828455, + "precision": 1.0, + "pred_md": "MOHAVE COMMUNITY COLLEGE BIO181\n\n| Genetics Lab - Blood Disorders.............................................................................. 94 |\n|----------------------------------------------------------------------------------------------------------------------------|\n| Human Traits Governed by Mendelian Genetics................................................... 97 |\n| 1. Record your phenotype and genotype for the following Mendelian traits:.. 97 |\n| Human Traits not Governed by Mendelian Genetics............................................ 98 |\n| Human Genetics Problems................................................................................... 100 |\n| Pedigree Analysis ................................................................................................. 102 |\n| Practice Problems................................................................................................. 102 |\n| Lab Materials......................................................................................................... 104 |\n| Contributors and Attributions .............................................................................. 104 |\n| From Gene to Protein via Transcription and Translation.................................... 105 |\n\n2", + "recall": 1.0, + "true_md": "MOHAVE COMMUNITY COLLEGE BIO181 \n\n| Genetics Lab - Blood Disorders .............................................................................. | 94 |\n|------------------------------------------------------------------------------------------------------------------------|------|\n| Human Traits Governed by Mendelian Genetics................................................... | 97 |\n| 1. Record your phenotype and genotype for the following Mendelian traits:.. | 97 |\n| Human Traits not Governed by Mendelian Genetics ............................................ | 98 |\n| Human Genetics Problems ................................................................................... | 100 |\n| Pedigree Analysis ................................................................................................. | 102 |\n| Practice Problems................................................................................................. | 102 |\n| Lab Materials......................................................................................................... | 104 |\n| Contributors and Attributions .............................................................................. | 104 |\n| From Gene to Protein via Transcription and Translation.................................... | 105 |\n\n2 " + }, + { + "bleu": 0.8597167488177693, + "doc_id": "doc_12e9c991f3d907593bacefdcf9a46c2cb57b7fb599e7840f438df04e416aab9d_page_000001.png", + "edit_distance": 0.05555555555555555, + "f1_score": 0.9966101694915253, + "meteor": 0.9940330543522036, + "precision": 0.9932432432432432, + "pred_md": "MOHAVE COMMUNITY COLLEGE BIO181\n\n- 5. Sample problem: If the ocular has a 10x lens and the objective has a 45x lens the total magnification is 10 x 45 = 450x\n\n## Changing objectives:\n\n- 1. When changing objectives from scanning power to lower power to high power the following changes will occur:\n- a. The size of the field of view decreases\n- b. The field of view becomes darker\n- c. The size of the image increases\n- d. The resolution (ability to see detail) increases\n- e. The working distance between the slide and the objective lens decreases\n- f. The depth of focus (thickness of the specimen that is visible) is reduced\n- 2. When changing from scanning to low power the field of view gets smaller. In fact, every time you increase the power of the objective, the field gets smaller.\n\n## Steps for Using the Microscope:\n\n- 1. Place the slide on the stage lining it up with the rectangle and using the stage clip to hold it in place.\n- 2. Click the nosepiece to the lowest (shortest) setting, the scanning objective lens or 4x .\n- 3. Look into the eyepiece.\n- 4. Use the coarse adjustment knob to bring the specimen into view. The specimen must be in focus before moving to the next steps.\n- 5. Rotate the nosepiece to the low-power objective or 10x .\n- 6. Refocus using the coarse adjustment knob.\n- 7. Move the slide to get a centered view.\n- 8. Now use the fine adjustment knob to get the specimen in perfect focus.\n- 9. Your slide MUST be focused on low power before attempting this next step.\n\n20", + "recall": 1.0, + "true_md": "MOHAVE COMMUNITY COLLEGE BIO181 \n\n5. Sample problem: If the ocular has a 10x lens and the objective has a 45x lens the total magnification is 10 x 45 = 450x \n\n## Changing objectives: \n\n1. When changing objectives from scanning power to lower power to high power the following changes will occur: \n\na. The size of the field of view decreases \n\nb. The field of view becomes darker \n\nc. The size of the image increases \n\nd. The resolution (ability to see detail) increases \n\ne. The working distance between the slide and the objective lens decreases \n\nf. The depth of focus (thickness of the specimen that is visible) is reduced \n\n2. When changing from scanning to low power the field of view gets smaller. In fact, every time you increase the power of the objective, the field gets smaller. \n\n## Steps for Using the Microscope: \n\n1. Place the slide on the stage lining it up with the rectangle and using the stage clip to hold it in place. \n\n2. Click the nosepiece to the lowest (shortest) setting, the scanning objective lens or 4x. \n\n3. Look into the eyepiece. \n\n4. Use the coarse adjustment knob to bring the specimen into view. The specimen must be in focus before moving to the next steps. \n\n5. Rotate the nosepiece to the low-power objective or 10x. \n\n6. Refocus using the coarse adjustment knob. \n\n7. Move the slide to get a centered view. \n\n8. Now use the fine adjustment knob to get the specimen in perfect focus. \n\n9. Your slide MUST be focused on low power before attempting this next step. \n\n20 " + }, + { + "bleu": 0.5800291859425345, + "doc_id": "doc_a5528f4dd6f5ba0d52e6bf97a0aae99ede782fe97f4a0b4f96029e5ed7fe18b5_page_000001.png", + "edit_distance": 0.2992125984251969, + "f1_score": 0.9221556886227547, + "meteor": 0.8609095883190223, + "precision": 0.8651685393258427, + "pred_md": "MOHAVE COMMUNITY COLLEGE BIO181\n\n- · Transfer pipettes\n- · Test tube rack\n- · 4 large (20 ml) test tubes or small Erlenmeyer flasks for larger volumes\n- · Large plastic tray\n- · Masking tape or lab tape\n- · Large weigh boat (4/group)\n- · Metric ruler\n- · Electronic balance\n- · Spatula\n- · Weigh paper\n- · Red food coloring (optional)\n\nFigure 3. Saccharometer\n\nTable 2. Contents of Saccharometers when testing fermentation with various yeast concentrations.\n\n## *Double these amounts if using saccharometers that have a 15-cm vertical tube. See table below\n\n## Saccharometer DI Water Glucose Solution Yeast Suspension\n\n1\n\n16 ml\n\n12 ml\n\n0 ml\n\n58", + "recall": 0.9871794871794872, + "true_md": "MOHAVE COMMUNITY COLLEGE BIO181 \n\n• Transfer pipettes \n\n• Test tube rack \n\n• 4 large (20 ml) test tubes or small Erlenmeyer flasks for larger volumes \n\n• Large plastic tray \n\n• Masking tape or lab tape \n\n• Large weigh boat (4/group) \n\n• Metric ruler \n\n• Electronic balance \n\n• Spatula \n\n• Weigh paper \n\n• Red food coloring (optional) \n\nFigure 3. Saccharometer \n\nTable 2. Contents of Saccharometers when testing fermentation with various yeast concentrations. \n\n## *Double these amounts if using saccharometers that have a 15-cm vertical tube. See table below \n\n58 " + }, + { + "bleu": 0.9495755926290338, + "doc_id": "doc_10f75857bb705a5441260399c3fca1080262cc0c271270825caf141cf7b31770_page_000001.png", + "edit_distance": 0.05730659025787966, + "f1_score": 0.9746835443037974, + "meteor": 0.9939270433541624, + "precision": 0.9506172839506173, + "pred_md": "MOHAVE COMMUNITY COLLEGE BIO181\n\n## Saccharometer DI Water Glucose Solution Yeast Suspension\n\n## Employing Steps in the Scientific Method:\n\n- 1. Record the Question that is being investigated in this experiment.\n\n\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\n\n- 2. Record a Hypothesis for the question stated above.\n\n\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\n\n- 3. Predict the results of the experiment based on your hypothesis (if/then).\n\n\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\n\n- 4. Perform the experiment below and collect your data.\n\n## Procedure:\n\n- 1. Prepare yeast suspension: Add 7 grams yeast to 50 ml warm tap water. Stir to mix. Alternatively, you can use the yeast suspension from Part 2. Optional: Add a few drops of red food coloring to the yeast to increase contrast, allowing easier measuring of the height of yeast in saccharometers.\n- 2. Label 4 test tubes and 4 saccharometers # 1- 4. Use a transfer pipette to add the appropriate amount of glucose and distilled water listed in Table 2 to the corresponding labeled test tubes.\n- 3. Use a transfer pipette to add the appropriate amount of yeast solution listed in Table 1 to the corresponding labeled test tubes. It is important to work carefully and quickly after adding the yeast solution to the glucose and water.\n- 4. Carefully pour the contents of the test tubes into the correspondingly labeled saccharometer, ensuring that the solutions are well mixed.\n- 5. Carefully tilt the saccharometers to allow any air bubbles that are trapped in the arms of the vertical tube to escape.\n- 6. Begin the timer for the experiment and measure the size of any bubbles (in mm) that are trapped in the vertical arms of the saccharometers. Record this measurement as the 0 time point.\n- 7. Position the saccharometers on the large plastic tray, positioning them around a plastic weigh boat to catch any fermentation overflow that may occur.\n\n59", + "recall": 1.0, + "true_md": "MOHAVE COMMUNITY COLLEGE BIO181 \n\n## Employing Steps in the Scientific Method:\n\n1. Record the Question that is being investigated in this experiment. \n\n\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_ \n\n2. Record a Hypothesis for the question stated above. \n\n\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_ \n\n3. Predict the results of the experiment based on your hypothesis (if/then). \n\n\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_\\_ \n\n4. Perform the experiment below and collect your data. \n\n## Procedure: \n\n1. Prepare yeast suspension: Add 7 grams yeast to 50 ml warm tap water. Stir to mix. Alternatively, you can use the yeast suspension from Part 2. Optional: Add a few drops of red food coloring to the yeast to increase contrast, allowing easier measuring of the height of yeast in saccharometers. \n\n2. Label 4 test tubes and 4 saccharometers # 1- 4. Use a transfer pipette to add the appropriate amount of glucose and distilled water listed in Table 2 to the corresponding labeled test tubes. \n\n3. Use a transfer pipette to add the appropriate amount of yeast solution listed in Table 1 to the corresponding labeled test tubes. It is important to work carefully and quickly after adding the yeast solution to the glucose and water. \n\n4. Carefully pour the contents of the test tubes into the correspondingly labeled saccharometer, ensuring that the solutions are well mixed. \n\n5. Carefully tilt the saccharometers to allow any air bubbles that are trapped in the arms of the vertical tube to escape. \n\n6. Begin the timer for the experiment and measure the size of any bubbles (in mm) that are trapped in the vertical arms of the saccharometers. Record this measurement as the 0 time point. \n\n7. Position the saccharometers on the large plastic tray, positioning them around a plastic weigh boat to catch any fermentation overflow that may occur. \n\n59 " + }, + { + "bleu": 0.9790099053232284, + "doc_id": "doc_39aec6caead187513fa9e2a26a3bbb54ba0a3a4ae8e879642ca1584330b0c285_page_000001.png", + "edit_distance": 0.1095890410958904, + "f1_score": 1.0, + "meteor": 0.9998050166140506, + "precision": 1.0, + "pred_md": "MOHAVE COMMUNITY COLLEGE BIO181\n\n## Cellular Replication\n\n## Growth and the Creation of Life\n\nOne of the characteristics of living things is the ability to replicate and pass on genetic information to the next generation. Cell division in individual bacteria and archaea usually occurs by binary fission. Mitochondria and chloroplasts also replicate by binary fission, which is evidence of the evolutionary relationship between these organelles and prokaryotes.\n\nCell division in eukaryotes is more complex. It requires chromosomes. It is controlled in the cell cycle, which is the cell to manage a complicated process of duplicating the nucleus, other organelles, and multiple linear divided into three parts: interphase, mitosis, and cytokinesis. We spilt those further for ease of study.\n\nLet's start with interphase, which is broken into three stages. In the first growth phase (G1), the cell grows and prepares to duplicate its DNA. In the synthesis phase (S), the chromosomes are replicated. In the second growth phase (G2), the cell prepares to divide.\n\nCellular Cycle and Replication\n\nA step by step guide to growing a human!\n\nMitosis and Meiosis\n\nSimiliar processes with VERY different results!\n\n66", + "recall": 1.0, + "true_md": "MOHAVE COMMUNITY COLLEGE BIO181 \n\n## Cellular Replication \n\n## Growth and the Creation of Life \n\nOne of the characteristics of living things is the ability to replicate and pass on genetic information to the next generation. Cell division in individual bacteria and archaea usually occurs by binary fission. Mitochondria and chloroplasts also replicate by binary fission, which is evidence of the evolutionary relationship between these organelles and prokaryotes. \n\nCell division in eukaryotes is more complex. It requires the cell to manage a complicated process of duplicating the nucleus, other organelles, and multiple linear chromosomes. It is controlled in the cell cycle, which is divided into three parts: interphase, mitosis, and cytokinesis. We spilt those further for ease of study. Let's start with interphase, which is broken into three stages. In the first growth phase (G1), the cell grows and prepares to duplicate its DNA. In the synthesis phase (S), the chromosomes are replicated. In the second growth phase (G2), the cell prepares to divide. \n\nCellular Cycle and Replication \n\nA step by step guide to growing a human! \n\nMitosis and Meiosis \n\nSimiliar processes with VERY different results! \n\n66 " + }, + { + "bleu": 0.988023243499516, + "doc_id": "doc_b9e5879b4dcd812e854cc06c6c3cb4f54231af28c919be63e27bef4c639551c9_page_000001.png", + "edit_distance": 0.004629629629629629, + "f1_score": 0.9961389961389961, + "meteor": 0.9995346976603383, + "precision": 0.9923076923076923, + "pred_md": "MOHAVE COMMUNITY COLLEGE BIO181\n\nchromosome. Meiosis and mitosis are both nuclear divisions that result in new daughter cells. However, the two processes have significant differences. Fill out the following chart comparing the two forms of nuclear division.\n\n- 5. Using your beads, strings, and magnets recreate the process of meiosis. Ensuring you have two different colored beads, demonstrate the process of crossing over. When you think you have it down, flag your instructor over. Have them sign off on your handiwork. Instructor signature:\n\n6. By now hopefully you've noticed that these processes are denoted with '2n' and 'n' in various places. This is a reference to the number of sets of chromosomes that cell has at any given moment. Autosomal human cells are 2n. Gametes are 1n. Mitosis begins with one 2n cell and ends with two 2n cells. Meiosis begins with one 2n cell and ends with 4 1n cells. Sketch those two processes here to show every time the 'n' classification changes. (Hint: draw every step, it'll make your life easier, even if it takes a little bit longer!)\n\n71", + "recall": 1.0, + "true_md": "MOHAVE COMMUNITY COLLEGE BIO181 \n\nchromosome. Meiosis and mitosis are both nuclear divisions \n\nthat result in new daughter cells. However, the two processes have significant differences. Fill out the following chart comparing the two forms of nuclear division. \n\n5. Using your beads, strings, and magnets recreate the process of meiosis. Ensuring you have two different colored beads, demonstrate the process of crossing over. When you think you have it down, flag your instructor over. Have them sign off on your handiwork. Instructor signature: \n\n6. By now hopefully you've noticed that these processes are denoted with '2n' and 'n' in various places. This is a reference to the number of sets of chromosomes that cell has at any given moment. Autosomal human cells are 2n. Gametes are 1n. Mitosis begins with one 2n cell and ends with two 2n cells. Meiosis begins with one 2n cell and ends with 4 1n cells. Sketch those two processes here to show every time the 'n' classification changes. (Hint: draw every step, it'll make your life easier, even if it takes a little bit longer!) \n\n71 " + }, + { + "bleu": 0.9547649375192397, + "doc_id": "doc_6916893bac49ba4fefa0a83eaf039b73c4adf00ed81ef97d4aad8f1a5cb90994_page_000001.png", + "edit_distance": 0.025477707006369428, + "f1_score": 0.9822485207100592, + "meteor": 0.9858210264093508, + "precision": 0.9764705882352941, + "pred_md": "MOHAVE COMMUNITY COLLEGE BIO181\n\nSickle cell hemoglobin and normal hemoglobin differ in only a single amino acid out of more than 100 amino acids in the complete hemoglobin protein. This difference in a single amino acid results in the different properties of sickle cell hemoglobin compared to normal hemoglobin.\n\nHemoglobin is carried inside red blood cells. Normal hemoglobin dissolves in the watery cytosol of red blood cells. Sickle cell hemoglobin is less soluble in the cytosol because:\n\n- · Valine (Val) is much less water-soluble than glutamic acid (Glu).\n- · Amino acid 6 is in a crucial location on the outer surface of the hemoglobin protein.\n\nThe chart on the next page shows how the lower solubility of sickle cell hemoglobin results in the symptoms of sickle cell anemia.\n\n29a. Circle the arrows in the chart that represent transcription + translation.\n\n115", + "recall": 0.9880952380952381, + "true_md": "MOHAVE COMMUNITY COLLEGE BIO181 \n\nSickle cell hemoglobin and normal hemoglobin differ in only a single amino acid out of more than 100 amino acids in the complete hemoglobin protein. This difference in a single amino acid results in the different properties of sickle cell hemoglobin compared to normal hemoglobin. \n\nHemoglobin is carried inside red blood cells. Normal hemoglobin dissolves in the watery cytosol of red blood cells. Sickle cell hemoglobin is less soluble in the cytosol because: \n\n• Valine (Val) is much less water-soluble than glutamic acid (Glu). \n\n• Amino acid 6 is in a crucial location on the outer surface of the hemoglobin protein. \n\nThe chart on the next page shows how the lower solubility of sickle cell hemoglobin results in the symptoms of sickle cell anemia. \n\n29a. Circle the arrows in the chart that represent transcription + translation. \n\n115 " + }, + { + "bleu": 0.8210185119657157, + "doc_id": "doc_f151ca1888000160dc6ce2e46c3438bc13f9a034200b1ec5d81b03c77606eed6_page_000001.png", + "edit_distance": 0.14219114219114218, + "f1_score": 0.9442970822281167, + "meteor": 0.9569029151240551, + "precision": 0.9035532994923858, + "pred_md": "MOHAVE COMMUNITY COLLEGE BIO181\n\n- 16. Place the tubes in a balanced configuration in the microcentrifuge and spin for 3 minutes.\n- 17. Carefully pour off the supernatant from both tubes. Do not disturb the nucleic acid pellets. Invert the tubes and tap them gently on the surface of a clean paper towel to drain them thoroughly.\n- 18. Briefly spin the tubes in a balanced configuration in the microcentrifuge to bring any remaining ethanol to the bottom of the tube. Then use the micropipette to remove any remaining ethanol. Use a fresh tip for each tube. Be careful not to disturb the nucleic acid pellet.\n- 19. Allow the tubes to dry by leaving the tube caps open for 3-5 minutes. Inspect each tube carefully to ensure that the tube interior is completely dry.\n\n***Congratulations, you have just completed the miniprep plasmid DNA extraction!!!***\n\n## Restriction Enzyme Digest Prep (switch to the 1- 20μL micropipette):\n\n20. Use a micropipette to add 10 μL of tris -EDTA solution (TE) to each tube. Use a new tip for each tube. Dissolve the pellets by pipetting in and out. Rinse the sides of the tube several times, concentrating on the area where the nucleic acid pellet or particles were observed. Check that no particles remain in the pipet tip or on the side of the tube. Use the entire contents of each tube in the restriction digest that follows.\n\n## II. Set Up the Restriction Digests of the 'Suspect' and 'Evidence' DNA\n\n*Store on ice\n\nNOTE: Your instructor will assign you to use either 'Evidence A' DNA or 'Evidence B' DNA\n\n- 1. Label the three 1.5-mL microcentrifuge tubes in which you will perform the restriction digests: 'S1' for Suspect 1, 'S2' for Suspect 2, and either 'EA' for Evidence A or 'EB' for Evidence B. All three samples will be digested by the restriction enzymes BamHI and HindIII.\n- 2. Use the table below (next page) as a checklist while adding reagents to each reaction. Read down each column, adding the same reagent to all appropriate tubes. To avoid cross contamination, use a fresh pipet tip each time you add a reagent to a tube.\n\n132", + "recall": 0.9888888888888889, + "true_md": "MOHAVE COMMUNITY COLLEGE BIO181 \n\n17. Carefully pour off the supernatant from both tubes. Do not disturb the nucleic acid pellets. Invert the tubes and tap them gently on the surface of a clean paper towel to drain them thoroughly. \n\n18. Briefly spin the tubes in a balanced configuration in the microcentrifuge to bring any remaining ethanol to the bottom of the tube. Then use the micropipette to remove any remaining ethanol. Use a fresh tip for each \n\n19. Allow the tubes to dry by leaving the tube caps open for 3-5 minutes. Inspect each tube carefully to \n\n***Congratulations, you have just completed the miniprep plasmid DNA extraction!!!***\n\n## Restriction Enzyme Digest Prep (switch to the 1- 20-μL micropipette):\n\n20. Use a micropipette to add 10 μL of tris-EDTA solution (TE) to each tube. Use a new tip for each tube. Dissolve the pellets by pipetting in and out. Rinse the sides of the tube several times, concentrating on the area where the nucleic acid pellet or particles were observed. Check that no particles remain in the pipet tip or on the side of the tube. Use the entire contents of each tube in the restriction digest that \n\n## II. Set Up the Restriction Digests of the 'Suspect' and 'Evidence' DNA \n\n*Store on ice\n\nYour instructor will assign you to use either 'Evidence A' DNA or 'Evidence B' DNA \n\n1. Label the three 1.5-mL microcentrifuge tubes in which you will perform the restriction digests: 'S1' for Suspect 1, 'S2' for Suspect 2, and either 'EA' for Evidence A or 'EB' for Evidence B. All three samples will be\n\n2. Use the table below (next page) as a checklist while adding reagents to each reaction. Read down each column, adding the same reagent to all appropriate tubes. To avoid cross contamination, use a fresh pipet tip each time you add a reagent to a tube. \n\n132 " + }, + { + "bleu": 0.47889934802505324, + "doc_id": "doc_770c9434a3530bfb67ef32510b63e1da347dfd6c51e5db9e0b13e90c8b287895_page_000001.png", + "edit_distance": 0.45741324921135645, + "f1_score": 0.7619047619047619, + "meteor": 0.8045986045900205, + "precision": 0.6233766233766234, + "pred_md": "MOHAVE COMMUNITY COLLEGE BIO181\n\n- 3. Mix reagents by pipetting gently up and down.\n- 4. Incubate all of the reaction tubes for 1 hour at 37 C. o\n\nNOTE: Your instructor will freeze your completed restriction digests at -20 C until the next lab period. o\n\n## III. Electrophorese Digests\n\n## Reagents:\n\n- · Restriction digests from Part II, on ice\n- · 10x loading dye, 10 𝜇𝜇 L\n\nSupplies and Equipment\n\n- · Gel electrophoresis chamber with agarose gel in gel tray, power supply\n- · 1-20 𝜇𝜇 L Micropipette and pipet tips\n\n## Load the Gel\n\n- 1. Use a micropipette to add 2 𝜇𝜇 L of 10× loading dye to a reaction tube. Use the pipet tip and gently pipet up and down a couple of times to mix the 10× loading dye with the digested DNA. Use a new pipet tip and repeat for each digest.\n- 2. Use a micropipette to load the contents of each reaction tube (20 𝜇𝜇 L total) into a separate well in the gel. Use a fresh pipet tip for each reaction tube and write down the order in which the samples are loaded.\n\nNOTE: Be careful not to punch the tip of the pipet through the bottom or side of the well.\n\n## While loading,\n\n- · steady the pipet over the well using two hands. You may wish to place one or both elbows on the lab bench to steady your hands.\n- · be careful to expel any air in the pipet tip end before loading the gel. If an air bubble forms a cap over the well, the sample will flow into the buffer around the edges of the well.\n\n133", + "recall": 0.9795918367346939, + "true_md": "MOHAVE COMMUNITY COLLEGE BIO181\n\no\n\no\n\n## III. Electrophorese Digests \n\nReagents:\n\n• \n\n• \n\nSupplies and Equipment \n\n• \n\n• 1-20 𝜇𝜇L Micropipette and pipet tips \n\n## Load the Gel \n\n1. Use a micropipette to add 2 𝜇𝜇L of 10× loading dye to a reaction tube. Use the pipet tip and gently pipet up and down a couple of times to mix the 10× loading dye with the digested DNA. Use a new pipet tip and repeat for each digest. \n\n2. Use a micropipette to load the contents of each reaction tube (20 𝜇𝜇L total) into a separate well in the gel. Use a fresh pipet tip for each reaction tube and write down the order in which the samples are loaded. \n\nWhile loading, \n\n• steady the pipet over the well using two hands. You may wish to place one or both elbows on \n\n• be careful to expel any air in the pipet tip end before loading the gel. If an air bubble forms a \n\n133 " + }, + { + "bleu": 0.7700754483129463, + "doc_id": "doc_12b156f795da26f2c74fb91e0c828427582d595d8bdf1d9acc695cf01e11bf10_page_000001.png", + "edit_distance": 0.2222222222222222, + "f1_score": 0.9346733668341709, + "meteor": 0.8349351059864263, + "precision": 0.9789473684210527, + "pred_md": "## The Data Journey\n\nTo get started, let's consider the data visualization in Figure 1.1 1 below.\n\nThe underlying raw data went through many stages before it was presented to you in this data visualization. The information had to be:\n\n- · Collected via surveys\n- · Inputted into a database\n- · Stored on secure servers\n- · Cleaned for accuracy and consistency\n- · Analyzed to understand the trends\n- · Presented as a bar graph\n\n1. Statistics Canada. Table 32-10-0364-01 Area, production and farm gate value of marketed fruits. Data is reproduced and distributed on an \"as is\" basis with the permission of Statistics Canada. Retrieved January 9th, 2022. DOI: https://doi.org/10.25318/3210036401-eng. Statistics Canada Open Licence: https://www.statcan.gc.ca/en/reference/licence\n\n4 | The Data Journey", + "recall": 0.8942307692307693, + "true_md": "## The Data Journey \n\nTo get started, let's consider the data visualization 1 in Figure 1.1 below. \n\nFigure 1.1. Production of apples, blueberries, cranberries, graphs, and strawberrie s in British Columbia, 2016-2020. \n\nThe underlying raw data went through many stages before it was presented to you in this data visualization. The information had to be: \n\n• Collected via surveys \n\n• Inputted into a database \n\n• Stored on secure servers \n\n• Cleaned for accuracy and consistency \n\n• Analyzed to understand the trends \n\n• Presented as a bar graph \n\n1. Statistics Canada. Table 32-10-0364-01 Area, production and farm gate value of marketed fruits. Data is reproduced and distributed on an \"as is\" basis with the permission of Statistics Canada. Retrieved January 9th, 2022. DOI: https://doi.org/10.25318/3210036401-eng. Statistics Canada Open Licence: https://www.statcan.gc.ca/en/reference/licence \n\n4 | The Data Journey" + }, + { + "bleu": 0.8495800581222052, + "doc_id": "doc_bcb3dafc35b5e7476fd1b9cd6eccf5eeef936cd5b13ad846a4943f1e7797f4e9_page_000001.png", + "edit_distance": 0.16304347826086957, + "f1_score": 0.9186991869918698, + "meteor": 0.8599748912142899, + "precision": 0.9912280701754386, + "pred_md": "## False Causation\n\nCorrelation does not imply causation.\n\nIf you've ever taken a statistics or data analysis course, you have almost certainly come across this common phrase. It means that, just because two trends seem to fluctuate alongside each other, it doesn't prove that one causes the other or that they are related in a meaningful way.\n\nReview Figure 2.10 23 below, which shows a line graph of the\n\n- 2. Statistics Canada. Table 37-10-0079-01 Registered apprenticeship training, registrations by major trade groups and sex. Data is reproduced and distributed on an \"as is\" basis with the permission of Statistics Canada. Retrieved February 2nd, 2022. DOI: https://doi.org/ 10.25318/3710007901-eng. Statistics Canada Open Licence: https://www.statcan.gc.ca/en/reference/licence\n- 3. Statistics Canada. Table 32-10-0364-01 Area, production and farm gate\n\n46 | Misleading Data Visualizations", + "recall": 0.8560606060606061, + "true_md": "Figure 2.9. A pie chart displaying 12 categories of television viewing in Ontario in 2004 provides too much visual information , making it hard to read. \n\n## False Causation \n\nCorrelation does not imply causation. \n\nIf you've ever taken a statistics or data analysis course, you have almost certainly come across this common phrase. It means that, just because two trends seem to fluctuate alongside each other, it doesn't prove that one causes the other or that they are related in a meaningful way. \n\nReview Figure 2.10 23 below, which shows a line graph of the \n\n2. Statistics Canada. Table 37-10-0079-01 Registered apprenticeship training, registrations by major trade groups and sex. Data is reproduced and distributed on an \"as is\" basis with the permission of Statistics Canada. Retrieved February 2nd, 2022. DOI: https://doi.org/ 10.25318/3710007901-eng. Statistics Canada Open Licence: https://www.statcan.gc.ca/en/reference/licence \n\n3. Statistics Canada. Table 32-10-0364-01 Area, production and farm gate \n\n46 | Misleading Data Visualizations" + }, + { + "bleu": 0.9757776324508989, + "doc_id": "doc_2183cb0d11b4e22a1fd54f2e79e9d91b280e63c24da91c4164d52a89732dd1b0_page_000001.png", + "edit_distance": 0.015625, + "f1_score": 1.0, + "meteor": 0.9999847412109375, + "precision": 1.0, + "pred_md": "ways. Review Figure 2.16 below, which is a line graph of the 8 percentage of Canadian vs. foreign television programmes watched in New Brunswick from 2000 to 2004. Because of the similar colours of the lines, it is difficult for the reader to understand which line graph corresponds to which colour from the legend.\n\n8. Statistics Canada. Table 22-10-0097-01 Television viewing time of all television stations, by province, content and type of programme. Data is reproduced and distributed on an \"as is\" basis with the permission of Statistics Canada. Retrieved February 2nd, 2022. DOI: https://doi.org/ 10.25318/2210009701-eng. Statistics Canada Open Licence: https://www.statcan.gc.ca/en/reference/licence\n\n54 | Misleading Data Visualizations", + "recall": 1.0, + "true_md": "ways. Review Figure 2.16 8 below, which is a line graph of the percentage of Canadian vs. foreign television programmes watched in New Brunswick from 2000 to 2004. Because of the similar colours of the lines, it is difficult for the reader to understand which line graph corresponds to which colour from the legend. \n\n8. Statistics Canada. Table 22-10-0097-01 Television viewing time of all television stations, by province, content and type of programme. Data is reproduced and distributed on an \"as is\" basis with the permission of Statistics Canada. Retrieved February 2nd, 2022. DOI: https://doi.org/ 10.25318/2210009701-eng. Statistics Canada Open Licence: https://www.statcan.gc.ca/en/reference/licence \n\n54 | Misleading Data Visualizations" + }, + { + "bleu": 0.8880297804584464, + "doc_id": "doc_e2b604a3fb1541b82b6af8caca05682dff0c7735e0a3a4fa7c6a68246fb60e57_page_000001.png", + "edit_distance": 0.11920529801324503, + "f1_score": 0.934010152284264, + "meteor": 0.891420722076715, + "precision": 1.0, + "pred_md": "## Closure\n\nClosure refers to our mind completing missing portions of a design. There must be enough parts available for the image to be 'filled in'; if the image is too abstract, there are minimal reference points for the mind to complete it. See Figure 4.4 4 for an example of how our mind automatically imagine a line connecting the 2 broken ones.\n\n4. Statistics Canada. Table 18-10-0002-01 Monthly average retail prices for food and other selected products. Data is reproduced and distributed on an \"as is\" basis with the permission of Statistics Canada. Retrieved February 2nd, 2022. DOI: https://doi.org/10.25318/1810000201-eng. Statistics Canada Open Licence: https://www.statcan.gc.ca/en/ reference/licence\n\nGestalt's Principles | 89", + "recall": 0.8761904761904762, + "true_md": "Figure 4.3- Ontario area (in square feet) used to harvest mushroom s over the years. \n\n## Closure \n\nClosure refers to our mind completing missing portions of a design. There must be enough parts available for the image to be 'filled in'; if the image is too abstract, there are minimal reference points for the mind to complete it. See Figure 4.4 4 for an example of how our mind automatically imagine a line connecting the 2 broken ones. \n\n4. Statistics Canada. Table 18-10-0002-01 Monthly average retail prices for food and other selected products. Data is reproduced and distributed on an \"as is\" basis with the permission of Statistics Canada. Retrieved February 2nd, 2022. DOI: https://doi.org/10.25318/1810000201-eng. Statistics Canada Open Licence: https://www.statcan.gc.ca/en/ reference/licence \n\nGestalt's Principles | 89" + }, + { + "bleu": 1.0, + "doc_id": "doc_603d71e12f52d9801a9d82995babf69681d923f493b8d49abfbab8662a88b376_page_000001.png", + "edit_distance": 0.0, + "f1_score": 1.0, + "meteor": 0.9999999737314428, + "precision": 1.0, + "pred_md": "Suppose your business just purchased a $100,000 asset that has a 3-year useful life, and falls into 3-year class of assets. Using the SL method, the depreciation expense each year for the next 3 years would be:\n\nNote that the book value or basis of the asset (acquisition cost - accumulated depreciation) would be $0 after it has been fully depreciated at the end of 4 years. Because of the half-year convention, it takes 4 years to depreciate the asset, even though it falls into the 3-year classification.\n\nDepreciation expense for the same asset using the MACRS method would be calculated as:\n\nNote again that the depreciation expense using MACRS is higher in the early years and lower in later years than with the SL method and that the book value after 4 years is again zero. Businesses often use MACRS for tax purposes and SL for profit reporting. Can you think of any reasons why?\n\nSome businesses that invest small amounts in capital assets are allowed to deduct up to $1,000,000 of the cost of acquired depreciable property as a current expenditure instead of a capital expenditure. This is known as direct expensing, and is available only to businesses that don't make large capital purchases each year. The allowable expensing amount is reduced by one dollar for each dollar of capital investment expenditure over $2,500,000 during the year. Other restrictions also apply.\n\n42 | Ch. 3. The Federal Tax System", + "recall": 1.0, + "true_md": "Suppose your business just purchased a $100,000 asset that has a 3-year useful life, and falls into 3-year class of assets. Using the SL method, the depreciation expense each year for the next 3 years would be: \n\nNote that the book value or basis of the asset (acquisition cost - accumulated depreciation) would be $0 after it has been fully depreciated at the end of 4 years. Because of the half-year convention, it takes 4 years to depreciate the asset, even though it falls into the 3-year classification. \n\nDepreciation expense for the same asset using the MACRS method would be calculated as: \n\nNote again that the depreciation expense using MACRS is higher in the early years and lower in later years than with the SL method and that the book value after 4 years is again zero. Businesses often use MACRS for tax purposes and SL for profit reporting. Can you think of any reasons why? \n\nSome businesses that invest small amounts in capital assets are allowed to deduct up to $1,000,000 of the cost of acquired depreciable property as a current expenditure instead of a capital expenditure. This is known as direct expensing, and is available only to businesses that don't make large capital purchases each year. The allowable expensing amount is reduced by one dollar for each dollar of capital investment expenditure over $2,500,000 during the year. Other restrictions also apply. \n\n42 | Ch. 3. The Federal Tax System" + }, + { + "bleu": 1.0, + "doc_id": "doc_a0edae1fa147c7bb78ebc493743a68ba4372b5ead31f2a2b146c35119462379e_page_000001.png", + "edit_distance": 0.0, + "f1_score": 1.0, + "meteor": 0.9999987661145441, + "precision": 1.0, + "pred_md": "Figure 13.3. Graph of Projection Estimates Open Template in Microsoft Excel\n\nHaving obtained price forecasts, our next step would be to re-estimate CR for GCS based on the forecasted prices. In addition, we may use the confidence interval forecasts to find a most optimistic forecast using the upper confidence interval forecasts and a pessimistic forecast using the lower bound forecasts.\n\n298 | Ch. 13. Homogeneous Investment Types", + "recall": 1.0, + "true_md": "Figure 13.3. Graph of Projection Estimates Open Template in Microsoft Excel \n\nHaving obtained price forecasts, our next step would be to re-estimate CR for GCS based on the forecasted prices. In addition, we may use the confidence interval forecasts to find a most optimistic forecast using the upper confidence interval forecasts and a pessimistic forecast using the lower bound forecasts. \n\n298 | Ch. 13. Homogeneous Investment Types" + }, + { + "bleu": 0.9277985563092971, + "doc_id": "doc_54f70eba37eddc8a529fcb090d690185698ea98d3bd6a9a55ebace7ffb0042e8_page_000001.png", + "edit_distance": 0.0625, + "f1_score": 0.9706840390879479, + "meteor": 0.9355579467414015, + "precision": 0.9675324675324676, + "pred_md": "\n\nn the case that the distributions were identically distributed with expected value and variance of and , each partner would face the same expected value as before, . But, the variance of their individual earnings would be , half of what it was before without combining their businesses. Furthermore, the standard deviation of the earnings each partner would face would be:\n\n\n\nAnd if n partners joined together, then they would each face the same expected value as before, but the variance each partner would receive is . We now illustrate these important results.\n\nAssume that business one's earnings are determined by outcomes associated with the toss of a fair coin. If the outcome of the coin toss is tails, the firm pays (loses) $5,000. If the toss is a heads, the firm wins $8,000. Thus, the firm wins either $8,000 or loses $5,000 and earns on average (.5) (-5,000) + (.5) (8,000) = $1500.\n\nThe standard deviation of this risky outcomes is:\n\n\n\nFurthermore, assuming a normal distribution, 68% of the time, the average outcome will be between the mean and plus or minus one standard deviation: ($1,500 + $6,500) = $8,000 and ($1,500 - $6,500) = -$5,000.\n\nNow suppose that two persons decide to combine their operations and share the average of the outcomes. Then the possible outcomes of two coin tosses are two heads (H, H) which earns on average $16,000 / 2 = $8,000 and occurs with a probability of .25; two tails (T, T) which earns on average -$10,000 / 2 = -$5,000 and occurs with a probability of .25, and one head and one tail (H, T) or one tail and one head (T, H) which both earn on average $3,000 / 2 = $1,500 and each occurs with a probability of .25. The expected value for each of the two players can now can be expressed as:\n\n\n\nThe two players now receive on average the same as before, $1,500, but consider the standard deviation of the average outcome:\n\n340 | Ch. 15. Homogeneous Risk Measures", + "recall": 0.9738562091503268, + "true_md": "$$(15.19) $$\n\nn the case that the distributions were identically distributed with expected value and variance of and , each partner would face the same expected value as before, . But, the variance of their individual earnings would be , half of what it was before without combining their businesses. Furthermore, the standard deviation of the earnings each partner would face would be: \n\n$$(15.20) $$\n\nAnd if n partners joined together, then they would each face the same expected value as before, but the variance each partner would receive is . We now illustrate these important results. \n\nAssume that business one's earnings are determined by outcomes associated with the toss of a fair coin. If the outcome of the coin toss is tails, the firm pays (loses) $5,000. If the toss is a heads, the firm wins $8,000. Thus, the firm wins either $8,000 or loses $5,000 and earns on average (.5) (-5,000) + (.5) (8,000) = $1500. \n\nThe standard deviation of this risky outcomes is: \n\n$$(15.21) $$\n\nFurthermore, assuming a normal distribution, 68% of the time, the average outcome will be between the mean and plus or minus one standard deviation: ($1,500 + $6,500) = $8,000 and ($1,500 - $6,500) = -$5,000. \n\nNow suppose that two persons decide to combine their operations and share the average of the outcomes. Then the possible outcomes of two coin tosses are two heads (H, H) which earns on average $16,000 / 2 = $8,000 and occurs with a probability of .25; two tails (T, T) which earns on average -$10,000 / 2 = -$5,000 and occurs with a probability of .25, and one head and one tail (H, T) or one tail and one head (T, H) which both earn on average $3,000 / 2 = $1,500 and each occurs with a probability of .25. The expected value for each of the two players can now can be expressed as: \n\n$$(15.22) $$\n\nThe two players now receive on average the same as before, $1,500, but consider the standard deviation of the average outcome: \n\n340 | Ch. 15. Homogeneous Risk Measures" + }, + { + "bleu": 0.928573111041468, + "doc_id": "doc_95f808815969717e4ef6390b2b789a8ea552a000a2b936c3c9371f695a71bd9e_page_000001.png", + "edit_distance": 0.05660377358490566, + "f1_score": 0.9431818181818183, + "meteor": 0.949638228931259, + "precision": 0.9222222222222223, + "pred_md": "Table 15.6. Observations of Returns on the Firm's Portfolio of Investments r t p and on a Potential New Investment (a Challenger).\n\nAnother way to represent the two rates of return measures and their relationship to each other is to represent them in a two dimensional scatter graph.\n\nWe may visually observe how the two sets of rates of return move together by drawing a line through the points on the graph in such a way as to minimize the squared distance from the point to the line. Our scatter graph is identified as Figure 15.3.\n\nFigure 15.3. Scatter Graph of Returns on the Firm's Portfolio of Investments and Returns on the Potential New Investment\n\nThe relationship between the returns on the new investment and the firm's portfolio can be expressed as:\n\n\n\nCh. 15. Homogeneous Risk Measures | 349", + "recall": 0.9651162790697675, + "true_md": "Table 15.6. Observations of Returns on the Firm's Portfolio of Investments rt p and on a Potential New Investment (a Challenger). \n\nAnother way to represent the two rates of return measures and their relationship to each other is to represent them in a two dimensional scatter graph. \n\nWe may visually observe how the two sets of rates of return move together by drawing a line through the points on the graph in such a way as to minimize the squared distance from the point to the line. Our scatter graph is identified as Figure 15.3. \n\nFigure 15.3. Scatter Graph of Returns on the Firm's Portfolio of Investments and Returns on the Potential New Investment \n\nThe relationship between the returns on the new investment and the firm's portfolio can be expressed as: \n\n$$(15.42) $$\n\nCh. 15. Homogeneous Risk Measures | 349" + }, + { + "bleu": 0.9200825152135937, + "doc_id": "doc_32b723d3d51157dd3e96717aff7efc3d13c5c6c942668b1fbce46bcbe8d94354_page_000001.png", + "edit_distance": 0.07236842105263158, + "f1_score": 0.9733333333333333, + "meteor": 0.9937347820104396, + "precision": 0.948051948051948, + "pred_md": "Figure 17.2. Year-to-year changes in housing prices.\n\nInflationary, nominal, and real interest rates. To understand price volatility of durables, it is necessary to describe inflationary, nominal, and real interest rates. Recall from your earlier training that the inflation rate i is equal to the rate of change in average prices, changes often linked to monetary or fiscal policies of governments. The nominal interest rate r depends on the rate of inflation and a real component that is dependent on factors other than the rate of inflation such as changing market conditions or changes in productivity. To describe the effects of inflation on the nominal interest, let one plus the nominal interest rate r equal one plus the real rate r * times one plus the inflation rate so i that:\n\nCh. 17. Land Investments | 385", + "recall": 1.0, + "true_md": "Inflationary, nominal, and real interest rates. To understand price volatility of durables, it is necessary to describe inflationary, nominal, and real interest rates. Recall from your earlier training that the inflation rate i is equal to the rate of change in average prices, changes often linked to monetary or fiscal policies of governments. The nominal interest rate r depends on the rate of inflation and a real component that is dependent on factors other than the rate of inflation such as changing market conditions or changes in productivity. To describe the effects of inflation on the nominal interest, let one plus the nominal interest rate r equal one plus the real rate r * times one plus the inflation rate i so that: \n\nCh. 17. Land Investments | 385" + }, + { + "bleu": 0.9429726226359606, + "doc_id": "doc_eded7804127acce26e4b9e7138aa8c95d0d8886280a21148efb0ae9519941ede_page_000001.png", + "edit_distance": 0.05191256830601093, + "f1_score": 0.965174129353234, + "meteor": 0.9897607751011512, + "precision": 0.941747572815534, + "pred_md": "## Fish species on IUCN Red List\n\nPotosi Pupfish\n\nCyprinodon alvarezi\n\nLa Palma Pupfish\n\nCyprinodon longidorsalis\n\nButterfly Splitfin\n\nAmeca splendens\n\nGolden Skiffia\n\nSkiffia francesae\n\nTable 6.1: Four fish species on IUCN Red List \"Extinct in the Wild\" held in public aquariums.\n\nPublic aquariums, because of their inhouse expertise, can act quickly to collect and breed rare fish. Actions to prevent the extinction of the Barrens Topminnow include monitoring populations and propagating and stocking juveniles into existing or newly created spring habitats. The Tennessee Aquarium assisted with propagations and developed a program called 'Keeper Kids,' where students on spring break help feed the Barrens Topminnows in a behind-the-scenes experience.\n\nFigure 6.3: Photo of the critically endangered Butterfly Splitfin (Ameca spendens).\n\nThe breeding colonies of the Butterfly Splitfin (Figure 6.3) at the London Zoo and elsewhere serve as ark populations essential to the survival of this species. Butterfly Splitfins are endemic to the Río Ameca in western Mexico and almost extinct in the wild. Actions such as nonnative fish removal, stream restoration, and sanctuary designation may take decades before eventual introduction and survival in the wild. The Tennessee Aquarium is part of a large partnership to guide hatchery augmentation and recovery of the rarest darter in North America (U.S. Fish and Wildlife Service 2019). The Conasauga Logperch ( Percina jenkinsi ), a federally endangered darter (Percidae), is found only in a 30-mile (48 km) stretch of the Conasauga River in Georgia and Tennessee (Moyer et al. 2015).\n\nFigure 6.4: Lake Sturgeon (Acipenser fulvescens).\n\nThe Banggai Cardinalfish ( Pterapogon kauderni ), a small, endangered tropical cardinalfish in the family Apogonidae, is now bred and displayed in numerous public aquariums after overharvest in the wild drove wild populations to near extinction. Consequently, most Banggai Cardinalfish sold to hobbyists in the United States and European Union today are captive bred.\n\n132 | Public Aquariums and Their Role in Education, Science, and Conservation", + "recall": 0.9897959183673469, + "true_md": "## Fish species on IUCN Red List \n\nTable 6.1: Four fish species on IUCN Red List \"Extinct in the Wild\" held in public aquariums. \n\nPublic aquariums, because of their in- house expertise, can act quickly to collect and breed rare fish. Actions to prevent the extinction of the Barrens Topminnow include monitoring populations and propagating and stocking juveniles into existing or newly created spring habitats. The Tennessee Aquarium assisted with propagations and developed a program called 'Keeper Kids,' where students on spring break help feed the Barrens Topminnows in a behind-the-scenes experience. \n\nFigure 6.3: Photo of the critically endangered Butterfly Splitfin (Ameca spendens). \n\nThe breeding colonies of the Butterfly Splitfin (Figure 6.3) at the London Zoo and elsewhere serve as ark populations essential to the survival of this species. Butterfly Splitfins are endemic to the Río Ameca in western Mexico and almost extinct in the wild. Actions such as nonnative fish removal, stream restoration, and sanctuary designation may take decades before eventual introduction and survival in the wild. The Tennessee Aquarium is part of a large partnership to guide hatchery augmentation and recovery of the rarest darter in North America (U.S. Fish and Wildlife Service 2019). The Conasauga Logperch (Percina jenkinsi), a federally endangered darter (Percidae), is found only in a 30-mile (48 km) stretch of the Conasauga River in Georgia and Tennessee (Moyer et al. 2015). \n\nFigure 6.4: Lake Sturgeon (Acipenser fulvescens). \n\nThe Banggai Cardinalfish (Pterapogon kauderni), a small, endangered tropical cardinalfish in the family Apogonidae, is now bred and displayed in numerous public aquariums after overharvest in the wild drove wild populations to near extinction. Consequently, most Banggai Cardinalfish sold to hobbyists in the United States and European Union today are captive bred. \n\n132 | Public Aquariums and Their Role in Education, Science, and Conservation" + }, + { + "bleu": 0.9822512579569583, + "doc_id": "doc_733825a2a8367b6d61ceccd19b6781c3520730c0fff34ab4c2e8411f625f34be_page_000001.png", + "edit_distance": 0.010186757215619695, + "f1_score": 0.9886914378029079, + "meteor": 0.9903151502148873, + "precision": 0.9902912621359223, + "pred_md": "## 7.6 Examples of Women's Impact\n\nSportfishing . Among those who fish for sport, only 27% of U.S. anglers are female (Burkett and Carter 2020). Underrepresentation of females in sportfishing is ironic, as the first publication on fly-fishing, dating from the 15th century, was written by Dame Juliana Berners, entitled Treatyse of Fysshynge with an Angle , a publication that heavily influenced novelty of the sport for European enthusiasts. Though sometimes invisible, women are slowly changing the world of sportfishing by breaking stereotypes. Future growth of sportfishing will rely on female anglers, instructors, and guides. Here I share a few examples on women making a substantial impact through their passion toward fishing. These examples demonstrate women who loved and valued what they did. If the paucity of female role models discourages females from seeing the relevance of fishing to them, these examples should inspire.\n\nFrederick Buller (2013) chronicled the very long list of large Atlantic Salmon caught by female anglers, which are outnumbered 200 to 1 by male salmon anglers. Georgina Ballantine holds the British record for a 64-pound rod-caught Atlantic Salmon from River Tay, Scotland, in 1922 (Figure 7.5). Joan Wulff was introduced to fly-fishing by her father when she was ten and won several fly-fishing accuracy championships before winning the 1951 Fishermen's Distance competition against allmale competitors. She became the first female spokesperson for Garcia Corporation in 1959 and advocated for women anglers in her writings for Outdoor Life and Rod & Reel . Today, females make up 30% of participants in the sport of fly-fishing (Recreational Fishing and Boating Foundation 2021). Joan Wulff participated in many distance casting events and did trick casting. She snapped a cigarette from the mouth of Johnny Carson on the TV show 'Who Do You Trust?' (Fogt 2017). Starting in 1978, Wulff opened a flycasting school on the Upper Beaverkill River in New York. Her FlyCasting Techniques , published in 1987, and New Fly-Casting Techniques , published in 2012, are classic guides to learning her techniques. When asked about her favorite fish, she would respond, 'Whatever I'm fishing for,' and her favorite place to fish was 'Wherever I am.'\n\nFigure 7.5: Georgina Ballantine holds the British record for a 64-pound rod-caught salmon from River Tay, Scotland in 1922.\n\nMost avid bass anglers can identify Roland Martin, Bill Dance, and Jimmy Houston, who dominated competitive bass fishing in the first decade of Bass Anglers Sportsman Society (B.A.S.S.) and have had TV fishing shows for decades. Kim Bain-Moore began competing in bass tournaments at age 19 and in 2009 became the first woman to compete in the Bassmaster Classic tournament. Only three females have been inducted into the Bass Fishing Hall of Fame. The first was Christine Houston, who organized the first-ever all women's bass club, the 'Tulsa Bass Belles.' But female participation in competitive bass fishing never took off as expected. Fewer that one in five readers of Field & Stream , Outdoor Life , and Bassmaster magazines are female (Carini and Weber 2017).\n\nGender and Fishing | 155", + "recall": 0.9870967741935484, + "true_md": "## 7.6 Examples of Women's Impact \n\nSportfishing. Among those who fish for sport, only 27% of U.S. anglers are female (Burkett and Carter 2020). Underrepresentation of females in sportfishing is ironic, as the first publication on fly-fishing, dating from the 15th century, was written by Dame Juliana Berners, entitled Treatyse of Fysshynge with an Angle, a publication that heavily influenced novelty of the sport for European enthusiasts. Though sometimes invisible, women are slowly changing the world of sportfishing by breaking stereotypes. Future growth of sportfishing will rely on female anglers, instructors, and guides. Here I share a few examples on women making a substantial impact through their passion toward fishing. These examples demonstrate women who loved and valued what they did. If the paucity of female role models discourages females from seeing the relevance of fishing to them, these examples should inspire. \n\nFrederick Buller (2013) chronicled the very long list of large Atlantic Salmon caught by female anglers, which are outnumbered 200 to 1 by male salmon anglers. Georgina Ballantine holds the British record for a 64-pound rod-caught Atlantic Salmon from River Tay, Scotland, in 1922 (Figure 7.5). Joan Wulff was introduced to fly-fishing by her father when she was ten and won several fly-fishing accuracy championships before winning the 1951 Fishermen's Distance competition against all- male competitors. She became the first female spokesperson for Garcia Corporation in 1959 and advocated for women anglers in her writings for Outdoor Life and Rod & Reel. Today, females make up 30% of participants in the sport of fly-fishing (Recreational Fishing and Boating Foundation 2021). Joan Wulff participated in many distance casting events and did trick casting. She snapped a cigarette from the mouth of Johnny Carson on the TV show 'Who Do You Trust?' (Fogt 2017). Starting in 1978, Wulff opened a fly- casting school on the Upper Beaverkill River in New York. Her Fly- Casting Techniques, published in 1987, and New Fly-Casting Techniques, published in 2012, are classic guides to learning her techniques. When asked about her favorite fish, she would respond, 'Whatever I'm fishing for,' and her favorite place to fish was 'Wherever I am.' \n\nFigure 7.5: Georgina Ballantine holds the British record for a 64-pound rod-caught salmon from River Tay, Scotland in 1922. \n\nMost avid bass anglers can identify Roland Martin, Bill Dance, and Jimmy Houston, who dominated competitive bass fishing in the first decade of Bass Anglers Sportsman Society (B.A.S.S.) and have had TV fishing shows for decades. Kim Bain-Moore began competing in bass tournaments at age 19 and in 2009 became the first woman to compete in the Bassmaster Classic tournament. Only three females have been inducted into the Bass Fishing Hall of Fame. The first was Christine Houston, who organized the first-ever all women's bass club, the 'Tulsa Bass Belles.' But female participation in competitive bass fishing never took off as expected. Fewer that one in five readers of Field & Stream, Outdoor Life, and Bassmaster magazines are female (Carini and Weber 2017). \n\nGender and Fishing | 155" + }, + { + "bleu": 1.0, + "doc_id": "doc_4a6baa0f883743ec25b853bae06a49d230ec010c973501945d46a16f9bb75aa9_page_000001.png", + "edit_distance": 0.0, + "f1_score": 1.0, + "meteor": 0.9999998926454167, + "precision": 1.0, + "pred_md": "What's unique about the growth of Alligator Gars is their fast growth in the first years of life followed by slower growth (Figure 8.6; Figure 8.7). Juvenile Alligator Gars quickly transition to fish-eating habits (Butler et al. 2018). A fish diet means the juveniles grow at 4-5 mm per day in the first three months of life, so that by the end of the first growing season they may reach 1.5 to 2 feet in length (~40-70 cm) and 8-10 pounds in weight (Sakaris et al. 2019). Despite their fast growth, young Alligator Gars are preyed upon by many larger fish.\n\nFigure 8.6: Growth in length of Alligator Gar in Texas. Figure 8.7: Growth in weight of Alligator Gar in Texas. Long description.\n\nFigure 8.7: Growth in weight of Alligator Gar in Texas.\n\nAngling and Conservation of Living Fishy Dinosaurs | 171", + "recall": 1.0, + "true_md": "What's unique about the growth of Alligator Gars is their fast growth in the first years of life followed by slower growth (Figure 8.6; Figure 8.7). Juvenile Alligator Gars quickly transition to fish-eating habits (Butler et al. 2018). A fish diet means the juveniles grow at 4-5 mm per day in the first three months of life, so that by the end of the first growing season they may reach 1.5 to 2 feet in length (~40-70 cm) and 8-10 pounds in weight (Sakaris et al. 2019). Despite their fast growth, young Alligator Gars are preyed upon by many larger fish. \n\nFigure 8.6: Growth in length of Alligator Gar in Texas. Figure 8.7: Growth in weight of Alligator Gar in Texas. Long description. \n\nFigure 8.7: Growth in weight of Alligator Gar in Texas. \n\nAngling and Conservation of Living Fishy Dinosaurs | 171" + }, + { + "bleu": 0.9945229581687481, + "doc_id": "doc_ed941b5cde32cfc778c225dcfa90c334d9995d70c46e7f508635073a3227784b_page_000001.png", + "edit_distance": 0.003110419906687403, + "f1_score": 1.0, + "meteor": 0.9999998796303329, + "precision": 1.0, + "pred_md": "Fly fishers targeting trout had an important influence in developing and sustaining conservation programs, although they were sometimes criticized for exclusive or single-interest advocacy. Here I review the history of trout fishing and fly-fishing with special focus on the Rocky Mountain West, where fly fishers first exerted their influence on conservation ethics and sportfishing policy. Although many individuals and organizations played roles, I concentrate on only two: Fly Fishers International (FFI) and Trout Unlimited (TU). These two organizations had similar interests in conservation, but important differences prevented them from working together on a unified goal of conservation. The legacy of fly-fishing demonstrates the importance of passion, persistence, and partnerships in fish conservation.\n\nTrout and salmon are the only sport fish native to the Western states, and fly-fishing here became more than a leisure activity. Norman Maclean's novel, A River Runs through It (1976), begins, 'In our family there was no clear line between religion and fly fishing.' Later Maclean writes that 'Something within fishermen tries to 1 make fishing into a world perfect and apart.' The iconography of Western fly-fishing that Maclean and others wrote about was created by anglers, fisheries managers, tourists, guides, businesses, and region promoters. The history of Rocky Mountain fly-fishing parallels the history of the expansion of our Western frontier as well as fisheries management (Brown 2015). Although Henry David Thoreau (1862) maintained that 'In wildness is the preservation of the world,' humans are part of the trout fishing system and helped create, destroy, maintain, and restore the trout fishing we have today.\n\nThe first trout fishers were Native Americans. Native Americans used a variety of fishing methods, including weirs, spears, nets, traps, baskets, hook-and-line methods, and baits. They also caught fish by hand via tickling. Tickling for trout involves rubbing the underbelly of a trout with fingers to get the trout to go into a trance, after which they can then easily be thrown onto the bank (Martindale 1901). Native Americans were more patient than others. This method is different from noodling for catfish, where the noodler uses fingers as bait and grabs the catfish by its mouth. Native Americans also caught fish by fly-fishing with deer-hair flies, according to the writings of early American naturalist William Bartram (1739-1823) (Monahan, no date).\n\nThe story of Rocky Mountain trout fishing begins with displacement of Native Americans from their historical fishing and hunting grounds. Uninhabited wilderness had to be created through the dispossession of Native people before it could be preserved (Spence 1999). Explorers, trappers, pioneers, soldiers, and homesteaders brought fishing gear to frontier outposts. The Lewis and Clark Expedition (1804-1806) included a designated angler named Silas Goodrich. The expedition first described several new species of fish, including the Yellowstone Cutthroat Trout and Westslope Cutthroat Trout, caught by Goodrich. Later military expeditions spent time trout fishing in addition to fighting Native Americans. Custer's Last Stand at Little Bighorn might have been avoided if he'd joined a column of reinforcements under General George Crook. Crook's soldiers were comfortably camped close by on Goose Creek near the Tongue River-fishing, not fighting (Monnett 1993; Owens 2002a; Lessner 2010).\n\n1. Although Maclean and other writers use the term fishermen, women are active anglers and contribute significantly to the sport.\n\nFly-Fishing's Legacy for Conservation | 191", + "recall": 1.0, + "true_md": "Fly fishers targeting trout had an important influence in developing and sustaining conservation programs, although they were sometimes criticized for exclusive or single-interest advocacy. Here I review the history of trout fishing and fly-fishing with special focus on the Rocky Mountain West, where fly fishers first exerted their influence on conservation ethics and sportfishing policy. Although many individuals and organizations played roles, I concentrate on only two: Fly Fishers International (FFI) and Trout Unlimited (TU). These two organizations had similar interests in conservation, but important differences prevented them from working together on a unified goal of conservation. The legacy of fly-fishing demonstrates the importance of passion, persistence, and partnerships in fish conservation. \n\nTrout and salmon are the only sport fish native to the Western states, and fly-fishing here became more than a leisure activity. Norman Maclean's novel, A River Runs through It (1976), begins, 'In our family there was no clear line between religion and fly fishing.' Later Maclean writes that 'Something within fishermen 1 tries to make fishing into a world perfect and apart.' The iconography of Western fly-fishing that Maclean and others wrote about was created by anglers, fisheries managers, tourists, guides, businesses, and region promoters. The history of Rocky Mountain fly-fishing parallels the history of the expansion of our Western frontier as well as fisheries management (Brown 2015). Although Henry David Thoreau (1862) maintained that 'In wildness is the preservation of the world,' humans are part of the trout fishing system and helped create, destroy, maintain, and restore the trout fishing we have today. \n\nThe first trout fishers were Native Americans. Native Americans used a variety of fishing methods, including weirs, spears, nets, traps, baskets, hook-and-line methods, and baits. They also caught fish by hand via tickling. Tickling for trout involves rubbing the underbelly of a trout with fingers to get the trout to go into a trance, after which they can then easily be thrown onto the bank (Martindale 1901). Native Americans were more patient than others. This method is different from noodling for catfish, where the noodler uses fingers as bait and grabs the catfish by its mouth. Native Americans also caught fish by fly-fishing with deer-hair flies, according to the writings of early American naturalist William Bartram (1739-1823) (Monahan, no date). \n\nThe story of Rocky Mountain trout fishing begins with displacement of Native Americans from their historical fishing and hunting grounds. Uninhabited wilderness had to be created through the dispossession of Native people before it could be preserved (Spence 1999). Explorers, trappers, pioneers, soldiers, and homesteaders brought fishing gear to frontier outposts. The Lewis and Clark Expedition (1804-1806) included a designated angler named Silas Goodrich. The expedition first described several new species of fish, including the Yellowstone Cutthroat Trout and Westslope Cutthroat Trout, caught by Goodrich. Later military expeditions spent time trout fishing in addition to fighting Native Americans. Custer's Last Stand at Little Bighorn might have been avoided if he'd joined a column of reinforcements under General George Crook. Crook's soldiers were comfortably camped close by on Goose Creek near the Tongue River-fishing, not fighting (Monnett 1993; Owens 2002a; Lessner 2010). \n\n1. Although Maclean and other writers use the term fishermen, women are active anglers and contribute significantly to the sport. \n\nFly-Fishing's Legacy for Conservation | 191" + }, + { + "bleu": 0.9182782016589127, + "doc_id": "doc_ee0a6175753e8cfc2f35fcbb1c5f7666f56969fb8111e326757835f062c65bb0_page_000001.png", + "edit_distance": 0.045454545454545456, + "f1_score": 0.9884169884169884, + "meteor": 0.9744665940622189, + "precision": 0.9846153846153847, + "pred_md": "Figure 10.2: Positive attributes reported by recreational anglers in the United States. Long description.\n\nOver time, an angler's motivation may change from a catch orientation to emphasize noncatch motivations, such as being outdoors or passing on their passion for fishing (McKenna 2013). The progression often follows these stages:\n\n- · Stage 1: I just want to catch a fish!\n- · Stage 2: I want to catch a lot of fish!\n- · Stage 3: I want to catch big fish.\n- · Stage 4: I'm just happy to be out fishing.\n- · Stage 5: I want to pass on my knowledge and passion for fishing.\n\nStudies of angler characteristics confirm that there is no such thing as an 'average' angler. Rather, anglers are a heterogeneous and changing group. Therefore, we can segment anglers in distinct categories for analysis (Bryan 1977; Kyle et al. 2007; Beardmore et al. 2013; TenHarmsel et al. 2019). For example, Magee (2018) categorized recreational anglers into five distinct fisher classes with differing motivations (Table 10.1).\n\n216 | Recreational Fishing and Keep Fish Wet", + "recall": 0.9922480620155039, + "true_md": "Figure 10.2: Positive attributes reported by recreational anglers in the United States. Long description. \n\nOver time, an angler's motivation may change from a catch orientation to emphasize noncatch motivations, such as being outdoors or passing on their passion for fishing (McKenna 2013). The progression often follows these stages: \n\n• Stage 1: I just want to catch a fish! \n\n• Stage 2: I want to catch a lot of fish! \n\n• Stage 3: I want to catch big fish. \n\n• Stage 4: I'm just happy to be out fishing. \n\n• Stage 5: I want to pass on my knowledge and passion for fishing. \n\nStudies of angler characteristics confirm that there is no such thing as an 'average' angler. Rather, anglers are a heterogeneous and changing group. Therefore, we can segment anglers in distinct categories for analysis (Bryan 1977; Kyle et al. 2007; Beardmore et al. 2013; TenHarmsel et al. 2019). For example, Magee (2018) categorized recreational anglers into five distinct fisher classes with differing motivations (Table 10.1). \n\n216 | Recreational Fishing and Keep Fish Wet" + }, + { + "bleu": 1.0, + "doc_id": "doc_a7f0764fac3ab64711acc91fa88e2632f3e0d27b6d57ab58ee3b709b0889adba_page_000001.png", + "edit_distance": 0.0, + "f1_score": 1.0, + "meteor": 0.9999999940495793, + "precision": 1.0, + "pred_md": "Figure 10.5: Frequency distribution displays the number of angler days resulting in differing catch per day for a hypothetical 8 fish per day creel limit and estimated change if creel limit is reduced to 4 fish per day. Long description.\n\nCreel limits are one of many elements that may be used by anglers to define fishing success. When more fish are harvested per trip, anglers rate fishing higher. High creel limits may cause anglers to have unrealistic expectations about the potential supply of fish compared to the demand (Cook et al. 2001). Creel limit reductions may be unsuccessful in reducing angler harvest or affecting fish populations. The hypothetical angler success graph (Figure 10.5) demonstrates that a reduction in creel from 8 to 4 would affect only a few trips and result in a small harvest reduction. Furthermore, creel limits are applied on a per-angler basis, so they cannot control total harvest if total fishing effort increases or if noncompliance is high. Finally, since anglers have a variety of motivations, they likely respond differently to regulation changes (Beard et al. 2011).\n\nThe ethic of fairness is involved in setting creel limit regulations because many anglers do not harvest a single fish during an angling trip. In Wisconsin lakes, Walleye harvest was not equally distributed. Only 7.4% of Walleye angler trips were successful in harvesting at least one Walleye, and <1% harvested a limit during a fishing trip (Staggs 1989). In Minnesota, anglers were slightly more successful, where 27.2% of angler trips ended with a harvest of at least one Walleye and about 1% harvesting a limit. The ideal creel limit would distribute the catch among more anglers and prevent overuse by a few individuals.\n\nLong-term trends in panfish populations (i.e., Bluegill, Yellow Perch, Black Crappie, Pumpkinseed, and Rock Bass) in Wisconsin lakes showed significant declines due to overfishing (Rypel et al. 2016). The daily limit for panfish was 50 aggregate per day from 1967 through 1998, which was reduced to 25 in 1998. Further reduction in daily limits for panfish (10) to improve undesirable small sizes of Bluegill populations increased both mean length and mean maximum length relative to sizes in control lakes (Jacobson 2005; Rypel et al. 2015).\n\n226 | Recreational Fishing and Keep Fish Wet", + "recall": 1.0, + "true_md": "Figure 10.5: Frequency distribution displays the number of angler days resulting in differing catch per day for a hypothetical 8 fish per day creel limit and estimated change if creel limit is reduced to 4 fish per day. Long description. \n\nCreel limits are one of many elements that may be used by anglers to define fishing success. When more fish are harvested per trip, anglers rate fishing higher. High creel limits may cause anglers to have unrealistic expectations about the potential supply of fish compared to the demand (Cook et al. 2001). Creel limit reductions may be unsuccessful in reducing angler harvest or affecting fish populations. The hypothetical angler success graph (Figure 10.5) demonstrates that a reduction in creel from 8 to 4 would affect only a few trips and result in a small harvest reduction. Furthermore, creel limits are applied on a per-angler basis, so they cannot control total harvest if total fishing effort increases or if noncompliance is high. Finally, since anglers have a variety of motivations, they likely respond differently to regulation changes (Beard et al. 2011). \n\nThe ethic of fairness is involved in setting creel limit regulations because many anglers do not harvest a single fish during an angling trip. In Wisconsin lakes, Walleye harvest was not equally distributed. Only 7.4% of Walleye angler trips were successful in harvesting at least one Walleye, and <1% harvested a limit during a fishing trip (Staggs 1989). In Minnesota, anglers were slightly more successful, where 27.2% of angler trips ended with a harvest of at least one Walleye and about 1% harvesting a limit. The ideal creel limit would distribute the catch among more anglers and prevent overuse by a few individuals. \n\nLong-term trends in panfish populations (i.e., Bluegill, Yellow Perch, Black Crappie, Pumpkinseed, and Rock Bass) in Wisconsin lakes showed significant declines due to overfishing (Rypel et al. 2016). The daily limit for panfish was 50 aggregate per day from 1967 through 1998, which was reduced to 25 in 1998. Further reduction in daily limits for panfish (10) to improve undesirable small sizes of Bluegill populations increased both mean length and mean maximum length relative to sizes in control lakes (Jacobson 2005; Rypel et al. 2015). \n\n226 | Recreational Fishing and Keep Fish Wet" + }, + { + "bleu": 0.9928710875764828, + "doc_id": "doc_78698056c30504beba802bc806711c307553d8b8aa39a6d8ff8388aafdc3953a_page_000001.png", + "edit_distance": 0.00404040404040404, + "f1_score": 0.9979633401221996, + "meteor": 0.9977623052980117, + "precision": 0.9959349593495935, + "pred_md": "Figure 11.2: Arapaima gigas displayed in the Siam Centre, Bangkok.\n\nArapaima is an important flagship genus for flooded forest ecosystem and human floodplain communities. Flagship taxa are used as a symbol to promote conservation awareness (Caro 2010). Their large size makes them a true freshwater megafauna like crocodiles, river dolphins, and other large fish. Freshwater megafauna face many threats, and 71% of these species are in decline (He et al. 2017, 2018). Arapaima continue to face intense fishing throughout their range (Watson et al. 2021). However, freshwater megafauna like the Arapaima have fewer conservation resources and efforts than marine or terrestrial megafaunas.\n\nFishing, in general, and fishing for Arapaim a in particular, is a central element of the local economy and culture in Amazonia. Because these fish are obligate breathers, they are traditionally harvested by fishers using harpoons at the time when they surface to breathe. Men typically fish from canoes and search for signs of Arapaima near the surface. As they near the Arapaima , the harpooner throws the harpoon by hand. This is a specialized type of fishing, and the local fishers possess knowledge of the behavior that increases their likelihood of catching one. With appropriate training, fishers' participation in management processes can contribute to the conservation and governance of these small-scale fisheries.\n\nMany populations of Arapaima have been driven to local extinction due to overfishing (Castello et al. 2015a; Gurdak 2019a; Watson et al. 2021; Freitas and Sousa 2021). Much of the catch is illegal, with most specimens being caught below the minimum size limit or during the closed season (Cavole et al. 2015). The small-scale fishers are geographically dispersed, and governments in these regions have insufficient resources to devote to enforcing fishing rules. The riverine fishers who target Arapaima are marginalized and have limited formal education. Yet, compliance with regulations is essential to prevent overfishing and local extinction.\n\nArapaima represent only a small fraction of the fisheries harvest, but they are culturally important and symbolic as a flagship genus of tropical South American fisheries and floodplain management and conservation. Reducing the threats to Arapaima will also provide protections for many of the highly migratory fish of the Amazon basin. Collectively, the migratory fish contribute most of the fishery's landings in the basin (Duponchelle et al. 2021). Migratory fish depend on multiple, distant, but interconnected habitats during their life cycle. Any threat to one of the habitats or the corridor that connects them can influence these important food fish (Goulding et al. 2019).\n\nIntegrating Fishers in the Management of Arapaima | 251", + "recall": 1.0, + "true_md": "Figure 11.2: Arapaima gigas displayed in the Siam Centre, Bangkok. \n\nArapaima is an important flagship genus for flooded forest ecosystem and human floodplain communities. Flagship taxa are used as a symbol to promote conservation awareness (Caro 2010). Their large size makes them a true freshwater megafauna like crocodiles, river dolphins, and other large fish. Freshwater megafauna face many threats, and 71% of these species are in decline (He et al. 2017, 2018). Arapaima continue to face intense fishing throughout their range (Watson et al. 2021). However, freshwater megafauna like the Arapaima have fewer conservation resources and efforts than marine or terrestrial megafaunas. \n\nFishing, in general, and fishing for Arapaima in particular, is a central element of the local economy and culture in Amazonia. Because these fish are obligate breathers, they are traditionally harvested by fishers using harpoons at the time when they surface to breathe. Men typically fish from canoes and search for signs of Arapaima near the surface. As they near the Arapaima, the harpooner throws the harpoon by hand. This is a specialized type of fishing, and the local fishers possess knowledge of the behavior that increases their likelihood of catching one. With appropriate training, fishers' participation in management processes can contribute to the conservation and governance of these small-scale fisheries. \n\nMany populations of Arapaima have been driven to local extinction due to overfishing (Castello et al. 2015a; Gurdak 2019a; Watson et al. 2021; Freitas and Sousa 2021). Much of the catch is illegal, with most specimens being caught below the minimum size limit or during the closed season (Cavole et al. 2015). The small-scale fishers are geographically dispersed, and governments in these regions have insufficient resources to devote to enforcing fishing rules. The riverine fishers who target Arapaima are marginalized and have limited formal education. Yet, compliance with regulations is essential to prevent overfishing and local extinction. \n\nArapaima represent only a small fraction of the fisheries harvest, but they are culturally important and symbolic as a flagship genus of tropical South American fisheries and floodplain management and conservation. Reducing the threats to Arapaima will also provide protections for many of the highly migratory fish of the Amazon basin. Collectively, the migratory fish contribute most of the fishery's landings in the basin (Duponchelle et al. 2021). Migratory fish depend on multiple, distant, but interconnected habitats during their life cycle. Any threat to one of the habitats or the corridor that connects them can influence these important food fish (Goulding et al. 2019). \n\nIntegrating Fishers in the Management of Arapaima | 251" + }, + { + "bleu": 0.9786551599333881, + "doc_id": "doc_747e0d37d78ca840e89dd820e032b90a350809453bdf96c7359204b4a324e933_page_000001.png", + "edit_distance": 0.021367521367521368, + "f1_score": 0.996, + "meteor": 0.997821345569431, + "precision": 0.9920318725099602, + "pred_md": "## Top 10 tuna fishing nations (2018)\n\nFigure 12.8: Top tuna fishing nations based on landings of seven tuna species in 2018. Long description.\n\nToday most tuna are captured in purse seines, and longlines are the second-most-common gear. Indonesia and Japan are consistently the top-two fishing nations (Figure 12.8). Five of the top tuna fishing nations-Japan, Taiwan (Republic of China), Spain, Korea, and the USA-have large fishing fleets that operate far from their home waters, whereas the others have large local or regional fleets. New technologies, such as sonar, have made tuna fishing much more effective. In response, the use of spotter planes is banned for fishing Atlantic Bluefin Tuna in the Mediterranean (Di Natale 2020). Many recreational tuna boats also use spotter planes in the eastern Atlantic Ocean, although the traditionalist harpoon fishers shun the technology (Whynott 1995; Decker 2016).\n\nThe Pacific Ocean has consistently had the highest landings, about 66% of the world's tuna catch. The western and central Pacific Ocean is where many artisanal and industrial fisheries overlap. For the small island nations, fishing provides a major source of income, jobs, and food security (Bell et al. 2019). Yet, Pacific island nations have not fully realized the economic potential with the global tuna industry, despite the fact that 80% of it is caught within their exclusive economic zones (EEZs, i.e., within 200 miles). The 1982 United Nations Convention on the Law of the Sea awarded coastal states sovereign rights to (1) exploit and manage all living resources within their EEZ, (2) exclude distant water fleets in favor of developing their own fleets, and (3) charge distant water fleets rent for access. Eight island nations-the Federated States of Micronesia, Kiribati, Marshall Islands, Nauru, Palau, Papua New Guinea, Solomon Islands and Tuvalu, which support 80% of the purse-seine catch in their waters-formed an alliance and require collective bargaining to set rents for access by foreign vessels. The alliance also prioritized domestic over foreign vessels and set limits on the number of purse-seine vessels. The issue of sovereignty over tuna that migrate freely among EEZs remains a concern for small island nations (Bailey et al. 2012). Working to establish fair and equitable allocations of total allowable catches to the many parties will require more equitable sharing with the larger tuna-fishing nations.\n\n282 | Conserving Tuna: The Most Commercially Valuable Fish on Earth", + "recall": 1.0, + "true_md": "Figure 12.8: Top tuna fishing nations based on landings of seven tuna species in 2018. Long description. \n\nToday most tuna are captured in purse seines, and longlines are the second-most-common gear. Indonesia and Japan are consistently the top-two fishing nations (Figure 12.8). Five of the top tuna fishing nations-Japan, Taiwan (Republic of China), Spain, Korea, and the USA-have large fishing fleets that operate far from their home waters, whereas the others have large local or regional fleets. New technologies, such as sonar, have made tuna fishing much more effective. In response, the use of spotter planes is banned for fishing Atlantic Bluefin Tuna in the Mediterranean (Di Natale 2020). Many recreational tuna boats also use spotter planes in the eastern Atlantic Ocean, although the traditionalist harpoon fishers shun the technology (Whynott 1995; Decker 2016). \n\nThe Pacific Ocean has consistently had the highest landings, about 66% of the world's tuna catch. The western and central Pacific Ocean is where many artisanal and industrial fisheries overlap. For the small island nations, fishing provides a major source of income, jobs, and food security (Bell et al. 2019). Yet, Pacific island nations have not fully realized the economic potential with the global tuna industry, despite the fact that 80% of it is caught within their exclusive economic zones (EEZs, i.e., within 200 miles). The 1982 United Nations Convention on the Law of the Sea awarded coastal states sovereign rights to (1) exploit and manage all living resources within their EEZ, (2) exclude distant water fleets in favor of developing their own fleets, and (3) charge distant water fleets rent for access. Eight island nations-the Federated States of Micronesia, Kiribati, Marshall Islands, Nauru, Palau, Papua New Guinea, Solomon Islands and Tuvalu, which support 80% of the purse-seine catch in their waters-formed an alliance and require collective bargaining to set rents for access by foreign vessels. The alliance also prioritized domestic over foreign vessels and set limits on the number of purse-seine vessels. The issue of sovereignty over tuna that migrate freely among EEZs remains a concern for small island nations (Bailey et al. 2012). Working to establish fair and equitable allocations of total allowable catches to the many parties will require more equitable sharing with the larger tuna-fishing nations. \n\n282 | Conserving Tuna: The Most Commercially Valuable Fish on Earth" + }, + { + "bleu": 0.9934731095160807, + "doc_id": "doc_aa00d01e639cf879439b5f4c9313f1647bba27070a833569dee0b3eeb3476731_page_000001.png", + "edit_distance": 0.0037105751391465678, + "f1_score": 0.9945553539019963, + "meteor": 0.996474273759294, + "precision": 0.9963636363636363, + "pred_md": "There is no question that fishing is the major factor driving grouper stocks on the downward spiral, but those that have large spawning aggregations are most vulnerable to declines (Coleman et al. 1996; Asch and Erisman 2018; Sadovy de Mitcheson et al. 2020). Because it takes a long time for scientists to obtain needed life history information, fisheriesindependent survey data, and catch history, grouper populations may be overfished long before data are even available for a stock assessment. Without formal stock assessments, general indicators of population status are based on catch trends. Very few grouper stocks that have spawning aggregations are managed sustainably. In a recent global analysis of the status of populations that form spawning aggregations, 45% were unknown, 33% were decreasing, and 5% were already gone (Figure 13.5). Only 12% had stable populations, and 5% were increasing.\n\nFigure 13.5: Current known status reflecting changes of exploited grouper aggregations globally, as noted by fisher interviews, monitoring, or underwater surveys (N = 509). Long description.\n\nOf the 167 species of grouper, 9.6% are vulnerable, 4.8% are near threatened, 1.2% are endangered, and 0.6% are critically endangered (Figure 13.6). The majority of species (68.9%) are classified as least concern and 15% are data deficient, with insufficient data for classification. The larger (>50 cm total length) and long-lived (>20 years) species of grouper that also had smaller geographic ranges were most likely to be endangered or critically endangered (Luiz et al. 2016). Market prices for grouper are escalating, and other lower-valued fish are often mislabeled or substituted.\n\nFigure 13.6: Categories of all grouper species (N = 167) according to the IUCN Red List (IUCN Red List Assessments, updated November 2018). Long description.\n\nTo protect grouper from overfishing, many measures are being implemented, such as minimum and slot-size limits, recreational bag limits, commercial fishing quotas, gear and seasonal controls, marine protected areas, and limited entry (Rocklin et al. 2022). The effectiveness will depend on traits of the species and the local context. Regulations to prevent marketing of undersize fish will mitigate growth overfishing. Allowing smaller fish to reach maturity at least once before harvest will mitigate recruitment overfishing. Size-limit regulations focused on protecting spawning-size fish may be ineffective for deepwater recreational fishing. Grouper have a physoclistous (i.e., closed) swim bladder, making them particularly susceptible to ruptured swim bladders, bloating, stomach distention, and protruding eyes caused by rapid decompression when hauled to the surface (Brulé et al. 2015). The proportion of grouper with distended stomachs was 70% in one study of commercial hook-and-line fishing and as high as 95% for Red\n\n312 | Grouper and Spawning Aggregations", + "recall": 0.9927536231884058, + "true_md": "There is no question that fishing is the major factor driving grouper stocks on the downward spiral, but those that have large spawning aggregations are most vulnerable to declines (Coleman et al. 1996; Asch and Erisman 2018; Sadovy de Mitcheson et al. 2020). Because it takes a long time for scientists to obtain needed life history information, fisheries- independent survey data, and catch history, grouper populations may be overfished long before data are even available for a stock assessment. Without formal stock assessments, general indicators of population status are based on catch trends. Very few grouper stocks that have spawning aggregations are managed sustainably. In a recent global analysis of the status of populations that form spawning aggregations, 45% were unknown, 33% were decreasing, and 5% were already gone (Figure 13.5). Only 12% had stable populations, and 5% were increasing. \n\nFigure 13.5: Current known status reflecting changes of exploited grouper aggregations globally, as noted by fisher interviews, monitoring, or underwater surveys (N = 509). Long description. \n\nOf the 167 species of grouper, 9.6% are vulnerable, 4.8% are near threatened, 1.2% are endangered, and 0.6% are critically endangered (Figure 13.6). The majority of species (68.9%) are classified as least concern and 15% are data deficient, with insufficient data for classification. The larger (>50 cm total length) and long-lived (>20 years) species of grouper that also had smaller geographic ranges were most likely to be endangered or critically endangered (Luiz et al. 2016). Market prices for grouper are escalating, and other lower-valued fish are often mislabeled or substituted. \n\nFigure 13.6: Categories of all grouper species (N = 167) according to the IUCN Red List (IUCN Red List Assessments, updated November 2018). Long description. \n\nTo protect grouper from overfishing, many measures are being implemented, such as minimum and slot-size limits, recreational bag limits, commercial fishing quotas, gear and seasonal controls, marine protected areas, and limited entry (Rocklin et al. 2022). The effectiveness will depend on traits of the species and the local context. Regulations to prevent marketing of undersize fish will mitigate growth overfishing. Allowing smaller fish to reach maturity at least once before harvest will mitigate recruitment overfishing. Size-limit regulations focused on protecting spawning-size fish may be ineffective for deepwater recreational fishing. Grouper have a physoclistous (i.e., closed) swim bladder, making them particularly susceptible to ruptured swim bladders, bloating, stomach distention, and protruding eyes caused by rapid decompression when hauled to the surface (Brulé et al. 2015). The proportion of grouper with distended stomachs was 70% in one study of commercial hook-and-line fishing and as high as 95% for Red \n\n312 | Grouper and Spawning Aggregations" + }, + { + "bleu": 0.0, + "doc_id": "doc_827d21de372a2c26237ee1db526460851ae71c1867761776583535f532432e32_page_000001.png", + "edit_distance": 0.0037105751391465678, + "f1_score": 0.9945553539019963, + "meteor": 0.996474273759294, + "precision": 0.9963636363636363, + "pred_md": "", + "recall": 0.9927536231884058, + "true_md": "## \n\n## \n\n## " + }, + { + "bleu": 0.8482630847746898, + "doc_id": "doc_e30779548a80ff14228753ece493dbce0b4329e426b5dbe61b19d8721fec6dee_page_000001.png", + "edit_distance": 0.12809315866084425, + "f1_score": 0.9505962521294719, + "meteor": 0.8851669827969265, + "precision": 0.96875, + "pred_md": "2\n\nNumerical Methods for Ordinary Differential Equations\n\nalso plays an important role in error analysis (investigating the difference between the numerical approximation and the solution).\n\nCalculating with only a finite subset of the rational numbers has many consequences. For example: a computer cannot distinguish between two polynomials of sufficiently high degree. Consequently, methods based on the main theorem of algebra (i.e. that an n th degree polynomial has exactly n complex zeros) cannot be trusted. Errors that follow from the use of finitely many digits are called rounding errors (Section 1.4).\n\nAn important aspect of numerical mathematics is the emphasis on efficiency. Contrary to ordinary mathematics, numerical mathematics considers an increase in efficiency, i.e. a decrease of the number of operations and/or amount of storage required, as an essential improvement. Progress in this aspect is of great practical importance and the end of this development has not been reached yet. Here, the creative mind will meet many challenges. On top of that, revolutions in computer architecture will overturn much conventional wisdom.\n\n## 1.3 Why numerical mathematics?\n\nAbig advantage of numerical mathematics is that it can provide answers to problems that do not admit closed-form solutions. Consider for example the integral\n\n\n\nThis is an expression for the arc length of one arc of the curve y x ( ) = sin x , which does not have a solution in closed form. A numerical method, however, can approximate this integral in a very simple way (Chapter 5). An additional advantage is that a numerical method only uses standard function evaluations and the operations addition, subtraction, multiplication and division. Because these are exactly the operations a computer can perform, numerical mathematics and computers form a perfect combination.\n\nAn advantage of analytical methods is that the solution is given by a mathematical formula. From this, insight in the behavior and the properties of the solution can be gained. For numerical approximations, however, this is not the case. In that case, visualization tools may be used to gain insight in the behavior of the solution. Using a numerical method to draw a graph of a function is usually a more useful tool than evaluating the solution at a large number of points.\n\n## 1.4 Rounding errors\n\nA computer uses a finite representation of the all numbers in R . These are stored in a computer in the form in which, by definition, d 1 > 0 and 0 ≤ di < β . The normalization is needed in order to prevent a waste of digits and to make the representation unambiguous. We call the value in equation (1.1) a floating point number (representation) in which 0. d d 1 2 . . . dn is called the mantissa , β the base and e (integer) the exponent , where L < e < U . Characteristic values for | L | and U are in the range [ 100, 1000 , often, ] β = 2 (binary representation) and n = 24 ( single precision) or n = 53 ( double precision). Most computers and software packages (Matlab) satisfy the IEEE-754 standard, and hence provide single- 1 and double-precision 2 computations.\n\n\n\nLet for x ∈ R\n\n\n\n1 http://en.wikipedia.org/wiki/Single-precision\\_floating-point\\_format\n\n2 http://en.wikipedia.org/wiki/Double-precision\\_floating-point\\_format", + "recall": 0.9331103678929766, + "true_md": "2\n\nNumerical Methods for Ordinary Differential Equations\n\nalso plays an important role in error analysis (investigating the difference between the numerical approximation and the solution).\n\nCalculating with only a finite subset of the rational numbers has many consequences. For exam- ple: a computer cannot distinguish between two polynomials of sufficiently high degree. Conse- quently, methods based on the main theorem of algebra (i.e. that an nth degree polynomial has exactly n complex zeros) cannot be trusted. Errors that follow from the use of finitely many digits are called rounding errors (Section 1.4).\n\nAn important aspect of numerical mathematics is the emphasis on efficiency. Contrary to or- dinary mathematics, numerical mathematics considers an increase in efficiency, i.e. a decrease of the number of operations and/or amount of storage required, as an essential improvement. Progress in this aspect is of great practical importance and the end of this development has not been reached yet. Here, the creative mind will meet many challenges. On top of that, revolutions in computer architecture will overturn much conventional wisdom.\n\n## 1.3 Why numerical mathematics?\n\nAbig advantage of numerical mathematics is that it can provide answers to problems that do not admit closed-form solutions. Consider for example the integral\n\n$$π ∫ 0 √ 1 + cos 2 xdx.$$\n\nThis is an expression for the arc length of one arc of the curve y(x) = sin x, which does not have a solution in closed form. A numerical method, however, can approximate this integral in a very simple way (Chapter 5). An additional advantage is that a numerical method only uses stan- dard function evaluations and the operations addition, subtraction, multiplication and division. Because these are exactly the operations a computer can perform, numerical mathematics and computers form a perfect combination.\n\nAn advantage of analytical methods is that the solution is given by a mathematical formula. From this, insight in the behavior and the properties of the solution can be gained. For numerical approximations, however, this is not the case. In that case, visualization tools may be used to gain insight in the behavior of the solution. Using a numerical method to draw a graph of a function is usually a more useful tool than evaluating the solution at a large number of points.\n\n## 1.4 Rounding errors\n\nA computer uses a finite representation of the all numbers in R . These are stored in a computer in the form\n\n$$± 0.d 1 d 2 . . . dn · β e , (1.1)$$\n\nin which, by definition, d 1 > 0 and 0 ≤ d i < β. The normalization is needed in order to prevent a waste of digits and to make the representation unambiguous. We call the value in equation (1.1) a floating point number (representation) in which 0.d 1 d 2 . . . dn is called the mantissa, β the base and e (integer) the exponent, where L < e < U. Characteristic values for | L | and U are in the range [100, 1000], often, β = 2 (binary representation) and n = 24 (single precision) or n = 53 (double precision). Most computers and software packages (Matlab) satisfy the IEEE-754 standard, and hence provide single- 1 and double-precision 2 computations.\n\nLet for x ∈ R\n\n$$0.d 1 . . . dn · β e ≤ x < 0.d 1 d 2 . . . (dn + 1) · β e ,$$\n\n1 http://en.wikipedia.org/wiki/Single-precision\\_floating-point\\_format\n\n2 http://en.wikipedia.org/wiki/Double-precision\\_floating-point\\_format" + }, + { + "bleu": 0.8628123231278753, + "doc_id": "doc_b4d90b6cc2d96d952b49dd227148c08018baffbfebd8d204705384f0a4a0c8a3_page_000001.png", + "edit_distance": 0.11909650924024641, + "f1_score": 0.9453681710213778, + "meteor": 0.8879463094104648, + "precision": 0.9660194174757282, + "pred_md": "## Chapter 3\n\n## Numerical differentiation\n\n## 3.1 Introduction\n\nEveryone who possesses a car and/or a driver's licence is familiar with speeding tickets. In The Netherlands, speeding tickets are usually processed in a fully automated fashion, and the perpetrator will receive the tickets within a couple of weeks after the offence. The Dutch police optimized the procedures of speed control such that this effort has become very profitable to the Dutch government. Various strategies for speed control are carried out by police forces, which are all based on the position of the vehicle at consecutive times. The actual velocity follows from the first-order derivative of the position of the vehicle with respect to time. Since no explicit formula for this position is available, the velocity can only be estimated using an approximation of the velocity based on several discrete vehicle positions at discrete times. This motivates the use of approximate derivatives, also called numerical derivatives . If the police want to know whether the offender drove faster before speed detection (in other words, whether the perpetrator hit the brakes after having seen the police patrol), or whether the driver was already accelerating, then they are also interested in the acceleration of the 'bad guy'. This acceleration can be estimated using numerical approximations of the second-order derivative of the car position with respect to time.\n\nSince the time-interval of recording is nonzero, the velocity is not determined exactly in general. In this chapter, the resulting error, referred to as the truncation error , is estimated using Taylor series. In most cases, the truncation error increases with an increasing size of the recording interval (Sections 3.2 and 3.4). Next to the truncation error, the measurement of the position of the vehicle is also prone to measurement errors. Issues that influence the results are, for example, parallax, the measurement equipment, and in some cases even the performance of the police officer (in car-videoing and laser control). These measurement errors provide an additional deterioration of the approximation of the speed and acceleration. The impact of measurement errors on approximations of derivatives is treated in Section 3.3.\n\n## 3.2 Simple difference formulae for the first derivative\n\nSuppose f is a continuously differentiable function. The forward difference is defined as\n\n\n\nin which h is called the step size . By definition,\n\n", + "recall": 0.9255813953488372, + "true_md": "## Chapter 3\n\n## Numerical differentiation\n\n## 3.1 Introduction\n\nEveryone who possesses a car and/or a driver's licence is familiar with speeding tickets. In The Netherlands, speeding tickets are usually processed in a fully automated fashion, and the perpetrator will receive the tickets within a couple of weeks after the offence. The Dutch police optimized the procedures of speed control such that this effort has become very profitable to the Dutch government. Various strategies for speed control are carried out by police forces, which are all based on the position of the vehicle at consecutive times. The actual velocity follows from the first-order derivative of the position of the vehicle with respect to time. Since no explicit formula for this position is available, the velocity can only be estimated using an approximation of the velocity based on several discrete vehicle positions at discrete times. This motivates the use of approximate derivatives, also called numerical derivatives. If the police want to know whether the offender drove faster before speed detection (in other words, whether the perpetrator hit the brakes after having seen the police patrol), or whether the driver was already accelerating, then they are also interested in the acceleration of the 'bad guy'. This acceleration can be estimated using numerical approximations of the second-order derivative of the car position with respect to time.\n\nSince the time-interval of recording is nonzero, the velocity is not determined exactly in general. In this chapter, the resulting error, referred to as the truncation error, is estimated using Taylor se- ries. In most cases, the truncation error increases with an increasing size of the recording interval (Sections 3.2 and 3.4). Next to the truncation error, the measurement of the position of the vehicle is also prone to measurement errors. Issues that influence the results are, for example, paral- lax, the measurement equipment, and in some cases even the performance of the police officer (in car-videoing and laser control). These measurement errors provide an additional deteriora- tion of the approximation of the speed and acceleration. The impact of measurement errors on approximations of derivatives is treated in Section 3.3.\n\n## 3.2 Simple difference formulae for the first derivative\n\nSuppose f is a continuously differentiable function. The forward difference is defined as\n\n$$Q f (h) = f (x + h) - f (x) h , h > 0,$$\n\nin which h is called the step size. By definition,\n\n$$lim h → 0 f (x + h) - f (x) h = f′(x),$$" + }, + { + "bleu": 0.44948237024990595, + "doc_id": "doc_64f31210764923368012a82e8741b32b217654d91f54fc9e0a13be613d63bbc1_page_000001.png", + "edit_distance": 0.45737704918032784, + "f1_score": 0.9085545722713865, + "meteor": 0.5736174167948361, + "precision": 0.9390243902439024, + "pred_md": "Chapter 3. Numerical differentiation\n\n35\n\nNote that the exact error equals\n\n\n\nIn this example the error estimate is very reliable.\n\nTo receive a better approximation the error estimate can be added to the approximation:\n\n\n\nIn the above example, the value of p was computed using Richardson's extrapolation. However, using Theorem 3.2.1, it is clear that p = 1, and this value could have been used immediately in equation (3.13b) in order to determine cph p . In practice, more complex situations are found, and the following complications may occur:\n\n- -It is not known whether higher-order derivatives exist and/or are bounded.\n- -The final result is a combination of various approximation methods. The influence of these approximations on p is not always clear.\n- -During implementation of the algorithm in a computer program, errors may be made.\n\nTo reveal any of these complications it is good practice to verify whether the calculated p is close to the p that follows from theory.\n\n## 3.7.3 Formulae of higher accuracy from Richardson's extrapolation ∗\n\nIn several applications the value of p in (3.10) is known. In that case Richardson's extrapolation can be used to determine formulae of higher accuracy.\n\nThis is done by making use of the fact that the error estimates for Q h ( ) and Q ( 2 h ) equal\n\n\n\nMultiplying equation (3.15a) by 2 p and subtracting equation (3.15b) from this yields\n\n\n\nsuch that\n\n\n\nThis means that\n\nThe value ( 2 p Q h ( ) -Q ( 2 h )) / 2 ( p -1 ) is a new approximation formula for M with an accuracy that is one order higher than the order of Q h ( ) .\n\n\n\n## Example 3.7.2 (Forward difference of higher accuracy)\n\nAs an example, the forward-difference method is considered. The error in the forward-difference formula may be written as\n\n\n\nand the difference for 2 h equals\n\n", + "recall": 0.88, + "true_md": "Chapter 3. Numerical differentiation\n\n35\n\nNote that the exact error equals\n\n$$M - Q(h) = e - 2.7525 . . . = - 0.0342 . . . .$$\n\nIn this example the error estimate is very reliable.\n\nTo receive a better approximation the error estimate can be added to the approximation:\n\n$$Q(h) + cph p = 2.7525 . . . - 0.0348 . . . = 2.7177 . . . .$$\n\nIn the above example, the value of p was computed using Richardson's extrapolation. However, using Theorem 3.2.1, it is clear that p = 1, and this value could have been used immediately in equation (3.13b) in order to determine cph p . In practice, more complex situations are found, and the following complications may occur:\n\n- It is not known whether higher-order derivatives exist and/or are bounded.\n\n- The final result is a combination of various approximation methods. The influence of these approximations on p is not always clear.\n\n- During implementation of the algorithm in a computer program, errors may be made.\n\nTo reveal any of these complications it is good practice to verify whether the calculated p is close to the p that follows from theory.\n\n## 3.7.3 Formulae of higher accuracy from Richardson's extrapolation ∗\n\nIn several applications the value of p in (3.10) is known. In that case Richardson's extrapolation can be used to determine formulae of higher accuracy.\n\nThis is done by making use of the fact that the error estimates for Q(h) and Q(2h) equal\n\n$$M - Q(h) = cph p + O (h p+1 ), (3.15a) M - Q(2h) = cp(2h) p + O (h p+1 ) . (3.15b)$$\n\nMultiplying equation (3.15a) by 2 p and subtracting equation (3.15b) from this yields\n\n$$2 p (M - Q(h)) - (M - Q(2h)) = 2 p (cph p ) - cp(2h) p + O (h p+1 ),$$\n\nsuch that\n\n$$(2 p - 1)M - 2 p Q(h) + Q(2h) = O (h p+1 ).$$\n\nThis means that\n\n$$M= 2 p Q(h) - Q(2h) 2 p - 1 + O (h p+1 ). (3.16)$$\n\nThe value (2 p Q(h) - Q(2h))/(2 p - 1) is a new approximation formula for M with an accuracy that is one order higher than the order of Q(h).\n\n## Example 3.7.2 (Forward difference of higher accuracy)\n\nAs an example, the forward-difference method is considered. The error in the forward-difference formula may be written as\n\n$$f ′(x) - Q f (h) = c 1 h + O (h 2 ), (3.17)$$\n\nand the difference for 2h equals\n\n$$f ′(x) - Q f (2h) = c 1 2h + O (h 2 ). (3.18)$$" + }, + { + "bleu": 0.7925149351869011, + "doc_id": "doc_764990d134c06378c3f496bfae33215c8ff57a0d306cc984e6429a6ed48257df_page_000001.png", + "edit_distance": 0.2106537530266344, + "f1_score": 0.9122807017543859, + "meteor": 0.8131202660621888, + "precision": 0.9454545454545454, + "pred_md": "## Chapter 4\n\n## Nonlinear equations\n\n## 4.1 Introduction\n\nThe pressure drop in a fluid in motion is examined. For a flow in a pipe with a circular cross section of diameter D (meter), the Reynolds number, Re , is given by\n\n\n\nin which v ( m s / ) is the average flow velocity and ν ( m 2 / s ) is the viscosity of the fluid. The flow is called laminar if Re < 2100 (low flow velocity) and turbulent if Re > 3000. For 2100 ≤ Re ≤ 3000, the flow is neither laminar nor turbulent.\n\nFor turbulent flows, the pressure drop between inflow and outflow is given by\n\n\n\nin which w is a friction coefficient, ρ ( kg / m 3 ) is the fluid density, L ( m ) is the length and g ( m s / 2 ) is the acceleration of gravity. If the fluid contains particles (sand, paper fibers), then the friction coefficient w satisfies the equation\n\n\n\nin which k is a parameter known from experiments.\n\nIn this chapter, numerical methods will be discussed that can be used to determine w if the values of Re and k are known.\n\n## 4.2 Definitions\n\nIn this chapter, various iterative methods will be considered to solve nonlinear equations of the form f ( p ) = 0. The point p is called a zero of the function f , or a root of the equation f ( x ) = 0. First, some useful definitions and concepts are introduced.\n\n## Convergence\n\n/negationslash\n\nEach numerical method generates a sequence { pn } = p 0, p 1, p 2, . . . which should converge to p : lim n → ∞ pn = p . Assume that the sequence indeed converges, with pn = p for all n . If there exist positive constants λ and α satisfying\n\n", + "recall": 0.8813559322033898, + "true_md": "## Chapter 4\n\n## Nonlinear equations\n\n## 4.1 Introduction\n\nThe pressure drop in a fluid in motion is examined. For a flow in a pipe with a circular cross section of diameter D (meter), the Reynolds number, Re, is given by\n\n$$Re = Dv ν ,$$\n\nin which v (m/s) is the average flow velocity and ν (m 2 /s) is the viscosity of the fluid. The flow is called laminar if Re < 2100 (low flow velocity) and turbulent if Re > 3000. For 2100 ≤ Re ≤ 3000, the flow is neither laminar nor turbulent.\n\nFor turbulent flows, the pressure drop between inflow and outflow is given by\n\n$$P out - P in = ρwLv 2 2gD ,$$\n\nin which w is a friction coefficient, ρ (kg/m 3 ) is the fluid density, L (m) is the length and g (m/s 2 ) is the acceleration of gravity. If the fluid contains particles (sand, paper fibers), then the friction coefficient w satisfies the equation\n\n$$1 √w = ln(Re√w) + 14 - 5.6 k k ,$$\n\nin which k is a parameter known from experiments.\n\nIn this chapter, numerical methods will be discussed that can be used to determine w if the values of Re and k are known.\n\n## 4.2 Definitions\n\nIn this chapter, various iterative methods will be considered to solve nonlinear equations of the form f(p) = 0. The point p is called a zero of the function f, or a root of the equation f (x) = 0. First, some useful definitions and concepts are introduced.\n\n## Convergence\n\nEach numerical method generates a sequence { pn } = p 0 , p 1 , p 2 , . . . which should converge to p: limn → ∞ p n = p. Assume that the sequence indeed converges, with pn = p for all n. If there exist positive constants λ and α satisfying\n\n$$lim n → ∞ | p - p n+1 | | p - pn | α = λ, (4.1)$$" + }, + { + "bleu": 0.9768999637813832, + "doc_id": "doc_ba0e025d53c091e8d4bb87499ff69ed3428dcee325c8895ecdb40e973b4c835c_page_000001.png", + "edit_distance": 0.014084507042253521, + "f1_score": 0.9868766404199475, + "meteor": 0.9940655630157996, + "precision": 0.9842931937172775, + "pred_md": "organizations to navigate successfully the global digital economy. Finally each of the identified competences, within the Framework will correspond to the different e-learning modules (PR2) and e-game levels (PR3)\n\n## Reference frameworks:\n\n- ⮚ GreenComp - 'The European Sustainability Competence Framework' (1), responds to the growing need for people to improve and develop the knowledge, skills and attitudes to live, work and act in a sustainable manner.\n\nGreenComp is a reference framework for sustainability competences. It provides a common ground to learners and guidance to educators, providing a consensual definition of what sustainability as a competence entails. It is designed to support education and training programmes for lifelong learning. It is written for all learners, irrespective of their age and their education level and in any learning setting -formal, non-formal and informal. Sustainability competences can help learners become systemic and critical thinkers, as well as develop agency, and form a knowledge basis for everyone who cares abou t our planet's present and future state. The aim of GreenComp is to foster a sustainability mindset by helping users develop the knowledge, skills and attitudes to think, plan and act with empathy, responsibility, and care for our planet.\n\nGreen- Comp is the result of a robust research methodology that has involved a large and diverse group of experts and stakeholders, to build a consensus on an agreed proposal. It provides a general reference model that everyone involved in lifelong learning can use to design learning opportunities aimed at developing sustainability competences and to assess progress in supporting education and training for sustainability.\n\nGreenComp consists of 12 competences organised into the four main areas below:\n\nThis project has been funded with the support of the European Commission. This publication reflects the views only of the author and the Commission cannot be held responsible for any use which may be made of the information contained therein.\n\nProject No: : 2021-2-FR02-KA220-YOU-000048126", + "recall": 0.9894736842105263, + "true_md": "organizations to navigate successfully the global digital economy. Finally each of the identified competences, within the Framework will correspond to the different e-learning modules (PR2) and e-game levels (PR3) \n\n## Reference frameworks: \n\n⮚ GreenComp - 'The European Sustainability Competence Framework'(1), responds to the growing need for people to improve and develop the knowledge, skills and attitudes to live, work and act in a sustainable manner. \n\nGreenComp is a reference framework for sustainability competences. It provides a common ground to learners and guidance to educators, providing a consensual definition of what sustainability as a competence entails. It is designed to support education and training programmes for lifelong learning. It is written for all learners, irrespective of their age and their education level and in any learning setting - formal, non-formal and informal. Sustainability competences can help learners become systemic and critical thinkers, as well as develop agency, and form a knowledge basis for everyone who cares about our planet's present and future state. The aim of GreenComp is to foster a sustainability mindset by helping users develop the knowledge, skills and attitudes to think, plan and act with empathy, responsibility, and care for our planet. \n\nGreen- Comp is the result of a robust research methodology that has involved a large and diverse group of experts and stakeholders, to build a consensus on an agreed proposal. It provides a general reference model that everyone involved in lifelong learning can use to design learning opportunities aimed at developing sustainability competences and to assess progress in supporting education and training for sustainability. \n\nGreenComp consists of 12 competences organised into the four main areas below: \n\nThis project has been funded with the support of the European Commission. This publication reflects the views only of the author and the Commission cannot be held responsible for any use which may be made of the information contained therein.\n\nProject No: : 2021-2-FR02-KA220-YOU-000048126" + }, + { + "bleu": 1.0, + "doc_id": "doc_b862cd0d6f06c06ee5ab7729ed4e8ce58e6964eb0f1ab98b3865b57a4808216f_page_000001.png", + "edit_distance": 0.0, + "f1_score": 1.0, + "meteor": 0.9999990931646378, + "precision": 1.0, + "pred_md": "## 3. RECOLLECTION OF NATIONAL INITIATIVES\n\nPartners were also asked to recollect initiatives from their respective countries that represented the core values and practices of a Circular Economy or Social Entrepreneurship:\n\nThis project has been funded with the support of the European Commission. This publication reflects the views only of the author and the Commission cannot be held responsible for any use which may be made of the information contained therein.\n\nProject No: : 2021-2-FR02-KA220-YOU-000048126", + "recall": 1.0, + "true_md": "## 3. RECOLLECTION OF NATIONAL INITIATIVES \n\nPartners were also asked to recollect initiatives from their respective countries that represented the core values and practices of a Circular Economy or Social Entrepreneurship: \n\nThis project has been funded with the support of the European Commission. This publication reflects the views only of the author and the Commission cannot be held responsible for any use which may be made of the information contained therein.\n\nProject No: : 2021-2-FR02-KA220-YOU-000048126" + }, + { + "bleu": 0.9663068422437632, + "doc_id": "doc_6572d153a938b50167baca0e724b25a02adb927d3c6adb9f1985484b7b1791b6_page_000001.png", + "edit_distance": 0.019704433497536946, + "f1_score": 0.9767441860465117, + "meteor": 0.9890639120254359, + "precision": 0.9692307692307692, + "pred_md": "As seen in this chart of responses, we were very satisfied to reach diversity in age groups, with all groups being represented by over 10%. The main group reached was of ages 36-45, and the least represented was the youngest age group of 18-25.\n\nRegarding the education level of responders, we were satisfied to receive a very high level of responses with Bachelor's or higher d egrees, with the significant share of others coming from\n\nUpper Secondary-educated participants. There was also a small representation of non-formal training, as well as >1% representation for other options.\n\nFor responders' profession, the most commo n answers representing 19.7% equally, were Youth Workers and Project Managers, although practising Social Entrepreneurs were also well represented, along with an 8% response rate from self-declared circular economy experts.\n\nThis project has been funded with the support of the European Commission. This publication reflects the views only of the author and the Commission cannot be held responsible for any use which may be made of the information contained therein.\n\nProject No: : 2021-2-FR02-KA220-YOU-000048126", + "recall": 0.984375, + "true_md": "As seen in this chart of responses, we were very satisfied to reach diversity in age groups, with all groups being represented by over 10%. The main group reached was of ages 36-45, and the least represented was the youngest age group of 18-25. \n\nRegarding the education level of responders, we were satisfied to receive a very high level of responses with Bachelor's or higher degrees, with the significant share of others coming from \n\nUpper Secondary-educated participants. There was also a small representation of non-formal training, as well as >1% representation for other options. \n\nFor responders' profession, the most common answers representing 19.7% equally, were Youth Workers and Project Managers, although practising Social Entrepreneurs were also well represented, along with an 8% response rate from self-declared circular economy experts. \n\nThis project has been funded with the support of the European Commission. This publication reflects the views only of the author and the Commission cannot be held responsible for any use which may be made of the information contained therein.\n\nProject No: : 2021-2-FR02-KA220-YOU-000048126" + }, + { + "bleu": 0.9546636256311006, + "doc_id": "doc_750091cec2c31ca61ffaa40c282148258f40bf5db7356a7ed3f918a07c81ef0f_page_000001.png", + "edit_distance": 0.0273972602739726, + "f1_score": 0.9739130434782608, + "meteor": 0.9739260151803225, + "precision": 0.9824561403508771, + "pred_md": "With this in mind, here we have the 7 key competence areas selected to form a part of EcoCircle's Competence Framework:\n\nThis project has been funded with the support of the European Commission. This publication reflects the views only of the author and the Commission cannot be held responsible for any use which may be made of the information contained therein.\n\nProject No: : 2021-2-FR02-KA220-YOU-000048126", + "recall": 0.9655172413793104, + "true_md": "With this in mind, here we have the 7 key competence areas selected to form a part of Eco- Circle's Competence Framework: \n\nThis project has been funded with the support of the European Commission. This publication reflects the views only of the author and the Commission cannot be held responsible for any use which may be made of the information contained therein.\n\nProject No: : 2021-2-FR02-KA220-YOU-000048126" + }, + { + "bleu": 1.0, + "doc_id": "doc_a46e40b2b04dc83e4014fca656dae6df20af1fe7a420df306406409ea8c0db31_page_000001.png", + "edit_distance": 0.0, + "f1_score": 1.0, + "meteor": 0.9999971528790087, + "precision": 1.0, + "pred_md": "## 6. ECO CIRCLE COMPETENCE FRAMEWORK\n\nThis project has been funded with the support of the European Commission. This publication reflects the views only of the author and the Commission cannot be held responsible for any use which may be made of the information contained therein.\n\nProject No: : 2021-2-FR02-KA220-YOU-000048126", + "recall": 1.0, + "true_md": "## 6. ECO CIRCLE COMPETENCE FRAMEWORK \n\nThis project has been funded with the support of the European Commission. This publication reflects the views only of the author and the Commission cannot be held responsible for any use which may be made of the information contained therein.\n\nProject No: : 2021-2-FR02-KA220-YOU-000048126" + }, + { + "bleu": 0.9919355277549052, + "doc_id": "doc_ab0f3ac6ce114304cc331404f20a780f57fcb060b5c96c54c69478d474997d1f_page_000001.png", + "edit_distance": 0.008, + "f1_score": 1.0, + "meteor": 0.9927942028440652, + "precision": 1.0, + "pred_md": "CHAPTER 1.\n\n## CALIFORNIA\n\nJAMES GLAPA-GROSSKLAG\n\n## COURSE MARKING DRIVERS\n\nSB1359 was passed in September 2016, going into force in January 2018. The law 'requires California Community Colleges and California State Universities and requests the University of California system to include a symbol/logo in the online campus course schedule by January 1, 2018 for courses that exclusively use digital course materials that are free of charge to students and therefore not required to be purchased.'\n\nThe potential scale of impact is significant. With 114 colleges serving 2.1 million students, the California Community Colleges (CCCs) comprise the largest public system of higher education in the US. The California State University (CSU) with 23 campuses serving nearly 500,000 students, is the largest four-year public university system in the US. Notably, the law does not apply to the state's research-focused University of California.\n\nFigure 1.1: Zero Cost Textbook Logo\n\n## IMPLEMENTATION\n\nBetween the passage of the law in 2016 and the implementation of the law in 2018, both the CCCs and CSU systems engaged in outreach to the field. The CCCs' system office issued a memo to college leadership explaining the requirements and created a sample logo that colleges could choose to adopt. The CSU system's Affordable Learning Solutions team engaged the field with a series of webinars and FAQs.\n\nPRICE TRANSPARENCY 1", + "recall": 1.0, + "true_md": "## CHAPTER 1. \n\n## CALIFORNIA \n\nJAMES GLAPA-GROSSKLAG \n\n## COURSE MARKING DRIVERS \n\nSB1359 was passed in September 2016, going into force in January 2018. The law 'requires California Community Colleges and California State Universities and requests the University of California system to include a symbol/logo in the online campus course schedule by January 1, 2018 for courses that exclusively use digital course materials that are free of charge to students and therefore not required to be purchased.' \n\nThe potential scale of impact is significant. With 114 colleges serving 2.1 million students, the California Community Colleges (CCCs) comprise the largest public system of higher education in the US. The California State University (CSU) with 23 campuses serving nearly 500,000 students, is the largest four-year public university system in the US. Notably, the law does not apply to the state's research-focused University of California. \n\nFigure 1.1: Zero Cost Textbook Logo \n\n## IMPLEMENTATION \n\nBetween the passage of the law in 2016 and the implementation of the law in 2018, both the CCCs and CSU systems engaged in outreach to the field. The CCCs' system office issued a memo to college leadership explaining the requirements and created a sample logo that colleges could choose to adopt. The CSU system's Affordable Learning Solutions team engaged the field with a series of webinars and FAQs. \n\nPRICE TRANSPARENCY 1" + }, + { + "bleu": 0.9880642756959294, + "doc_id": "doc_16bc03112342160922798dc0b3220b260ad4ebe06deafb9ab3c67df528877717_page_000001.png", + "edit_distance": 0.006779661016949152, + "f1_score": 0.9910447761194029, + "meteor": 0.993556980015472, + "precision": 0.9940119760479041, + "pred_md": "should adopt two separate designators to mark no-cost vs. low-cost, but the council felt it was better to simplify the process and allow for some OER providers that have fees associated with their services.\n\nAt this point in time, the application of the #NOLO designator was a manual process. It required the addition of the designator to the section title prior to registration and then its removal after add/drop to ensure the label didn't appear on the student transcript. This process severely hampered our longterm reporting abilities. In total, four colleges adopted the #NOLO designator in this fashion.\n\nTo assist in greater faculty and institutional adoption as well as improve data capture, the CSCU OER Advisory Council made a formal recommendation to the provost's academic council in Spring 2018 to implement the #NOLO designator as a course section attribute within the student information system. In addition to adding a course section attribute, a student-facing course search filter was added as well as an additional column within the course search results page.\n\nFigure 2.1: Filtered Search Option for NOLO Sections.\n\nFigure 2.2: Added Column in Results for NOLO Designator.\n\nThe request to implement the designator within the student information system was supported in Fall 2018 by the president's cabinet. The ability to mark courses was enabled late Fall 2018 and the student-facing features were enabled in January 2019. Each institutional representative on the OER council engaged with their local governance structures to request a vote for adoption.\n\n4 BOYOUNG CHAE, KEVIN CORCORAN, MICHAEL DALY, ANN FIDDLER, JEFF GALLANT, JAMES GLAPA-GROSSKLAG, AMY HOFER, AND", + "recall": 0.9880952380952381, + "true_md": "should adopt two separate designators to mark no-cost vs. low-cost, but the council felt it was better to simplify the process and allow for some OER providers that have fees associated with their services. \n\nAt this point in time, the application of the #NOLO designator was a manual process. It required the addition of the designator to the section title prior to registration and then its removal after add/drop to ensure the label didn't appear on the student transcript. This process severely hampered our long- term reporting abilities. In total, four colleges adopted the #NOLO designator in this fashion. \n\nTo assist in greater faculty and institutional adoption as well as improve data capture, the CSCU OER Advisory Council made a formal recommendation to the provost's academic council in Spring 2018 to implement the #NOLO designator as a course section attribute within the student information system. In addition to adding a course section attribute, a student-facing course search filter was added as well as an additional column within the course search results page. \n\nFigure 2.1: Filtered Search Option for NOLO Sections. \n\nFigure 2.2: Added Column in Results for NOLO Designator. \n\nThe request to implement the designator within the student information system was supported in Fall 2018 by the president's cabinet. The ability to mark courses was enabled late Fall 2018 and the student-facing features were enabled in January 2019. Each institutional representative on the OER council engaged with their local governance structures to request a vote for adoption. \n\n4 BOYOUNG CHAE, KEVIN CORCORAN, MICHAEL DALY, ANN FIDDLER, JEFF GALLANT, JAMES GLAPA-GROSSKLAG, AMY HOFER, AND" + }, + { + "bleu": 0.985518479652992, + "doc_id": "doc_4ba951e3da05a42523676d164548037c97a09984433bd37bf71b1f2a7acfd598_page_000001.png", + "edit_distance": 0.010554089709762533, + "f1_score": 1.0, + "meteor": 0.9904912085461055, + "precision": 1.0, + "pred_md": "CHAPTER 7.\n\nTEXAS\n\nMICHELLE REED\n\n## COURSE MARKING DRIVERS\n\nI've worked at the University of Texas at Arlington (UTA) for the last three years as Open Education Librarian and was recently promoted to the leadership team as Director of Open Educational Resources following a half-million-dollar investment in OER from university administration. It was in my first year as Open Education Librarian that the Texas Legislature passed Senate Bill 810 (SB810), which requires institutions of higher education across the state to provide searchable information to students about OER-only courses. A strong definition of OER was provided:\n\n'teaching, learning, and research resources that reside in the public domain or have been released under an intellectual property license that allows for free use, reuse, modification, and sharing with others, including full courses, course materials, modules, textbooks, streaming videos, tests, software, and any other tools, materials, or techniques used to support access to knowledge.'\n\nHowever, Texas was not given a very long implementation window. The bill passed in June 2017, effective immediately, with a compliance deadline of Spring 2018. We in higher education know a change of this scope, and impacting as many stakeholders as course marking does, takes longer. A recent survey commissioned by the Digital Higher Education Consortium of Texas (DigiTex) and administered in May 2019 shows only 59 respondents of the 158 two-and four-year institutions that received the statewide survey have a course marking solution in place. The findings were presented in Open Educational Resources (OER) in Texas Higher Education, 2019 . 1\n\n1. Jimes, C., Karaglani, A., Petrides, L., Rios, J., Sebesta, J., & Torre, K. (2019). Open Educational Resources (OER) in Texas Higher Education, 2019 . Austin, TX: Digital Higher Education Consortium of Texas and Texas Higher Education Coordinating Board; Half Moon Bay, CA: Institute for the Study of Knowledge Management in Education.\n\nPRICE TRANSPARENCY 17", + "recall": 1.0, + "true_md": "## CHAPTER 7. \n\n## TEXAS \n\nMICHELLE REED \n\n## COURSE MARKING DRIVERS \n\nI've worked at the University of Texas at Arlington (UTA) for the last three years as Open Education Librarian and was recently promoted to the leadership team as Director of Open Educational Resources following a half-million-dollar investment in OER from university administration. It was in my first year as Open Education Librarian that the Texas Legislature passed Senate Bill 810 (SB810), which requires institutions of higher education across the state to provide searchable information to students about OER-only courses. A strong definition of OER was provided: \n\n'teaching, learning, and research resources that reside in the public domain or have been released under an intellectual property license that allows for free use, reuse, modification, and sharing with others, including full courses, course materials, modules, textbooks, streaming videos, tests, software, and any other tools, materials, or techniques used to support access to knowledge.' \n\nHowever, Texas was not given a very long implementation window. The bill passed in June 2017, effective immediately, with a compliance deadline of Spring 2018. We in higher education know a change of this scope, and impacting as many stakeholders as course marking does, takes longer. A recent survey commissioned by the Digital Higher Education Consortium of Texas (DigiTex) and administered in May 2019 shows only 59 respondents of the 158 two-and four-year institutions that received the statewide survey have a course marking solution in place. The findings were presented in Open Educational Resources (OER) in Texas Higher Education, 2019. 1 \n\n1. Jimes, C., Karaglani, A., Petrides, L., Rios, J., Sebesta, J., & Torre, K. (2019). Open Educational Resources (OER) in Texas Higher Education, 2019. Austin, TX: Digital Higher Education Consortium of Texas and Texas Higher Education Coordinating Board; Half Moon Bay, CA: Institute for the Study of Knowledge Management in Education. \n\nPRICE TRANSPARENCY 17" + }, + { + "bleu": 1.0, + "doc_id": "doc_512a6e53e0a6661cb6bb77da54c354496ca75723048e2411ac5d1796a18cda8f_page_000001.png", + "edit_distance": 0.0, + "f1_score": 1.0, + "meteor": 0.999999782605671, + "precision": 1.0, + "pred_md": "Figure 7.1: Texas OER landscape survey results show terms used in course schedules\n\n## IMPLEMENTATION\n\nLocally, we implemented a quick and free solution that reflects the constraints of system capabilities, no financial support, and a local directive to vet every course to be tagged. Based on what was feasible in the short term and conversations with key stakeholders (i.e., registrar, early OER adopters, curriculum coordinators, student representatives, and the campus store), we incorporated an 'educational resources cost' option into an existing 'course attribute' drop-down menu under the system's advanced search options.\n\n18 BOYOUNG CHAE, KEVIN CORCORAN, MICHAEL DALY, ANN FIDDLER, JEFF GALLANT, JAMES GLAPA-GROSSKLAG, AMY HOFER, AND", + "recall": 1.0, + "true_md": "Figure 7.1: Texas OER landscape survey results show terms used in course schedules \n\n## IMPLEMENTATION \n\nLocally, we implemented a quick and free solution that reflects the constraints of system capabilities, no financial support, and a local directive to vet every course to be tagged. Based on what was feasible in the short term and conversations with key stakeholders (i.e., registrar, early OER adopters, curriculum coordinators, student representatives, and the campus store), we incorporated an 'educational resources cost' option into an existing 'course attribute' drop-down menu under the system's advanced search options. \n\n18 BOYOUNG CHAE, KEVIN CORCORAN, MICHAEL DALY, ANN FIDDLER, JEFF GALLANT, JAMES GLAPA-GROSSKLAG, AMY HOFER, AND" + }, + { + "bleu": 0.6987024682441614, + "doc_id": "doc_b2c11e30c537ecc6ac4d39ea937e512500882bc479ac6e5c3b380cf1627dc189_page_000001.png", + "edit_distance": 0.1111111111111111, + "f1_score": 1.0, + "meteor": 0.9341241334545795, + "precision": 1.0, + "pred_md": "## Contents\n\n1\n\n97\n\n| 1. | Front Matter | 1 |\n|------|---------------------------------------------|-----|\n| 2. | Introduction to Researching Wicked Problems | 3 |\n| 3. | Our Mental Shortcuts | 13 |\n| 4. | Identifying a Topic | 25 |\n| 5. | Types of Sources | 38 |\n| 6. | Access & Searching | 55 |\n| 7. | SIFTing Information | 67 |\n| 8. | Evaluating News Sources | 80 |\n| 9. | Audience, Presentation & Citation | 88 |\n| | Instructor Resources | 97 |", + "recall": 1.0, + "true_md": "## Contents \n\n| 1. Front Matter | 1 |\n|------------------------------------------------|-----|\n| 2. Introduction to Researching Wicked Problems | 3 |\n| 3. Our Mental Shortcuts | 13 |\n| 4. Identifying a Topic | 25 |\n| 5. Types of Sources | 38 |\n| 6. Access & Searching | 55 |\n| 7. SIFTing Information | 67 |\n| 8. Evaluating News Sources | 80 |\n| 9. Audience, Presentation & Citation | 88 |\n| Instructor Resources | 97 |" + }, + { + "bleu": 0.9794642728696622, + "doc_id": "doc_3cc8da158da98e9b4e7b3c8ba7d1a74afc558fb6afc7ac8eadbe74afb4e7b14c_page_000001.png", + "edit_distance": 0.020833333333333332, + "f1_score": 0.9798657718120807, + "meteor": 0.9870266035217048, + "precision": 0.9733333333333334, + "pred_md": "## Fact-Checking 2\n\nIn this context, we are talking about fact-checking that is done before a source is published. Over the last two decades there has been an increase in fact checking as an activity that takes place after a source has been published, a practice discussed in more detail in the chapter, SIFTing Information.\n\nFact checkers verify that the names, dates, and facts in a work (usually an article or book) are correct. For example, they may contact a person who is quoted in a proposed news article and ask the person whether this quotation is correct, or how to spell the person's name. Factcheckers are primarily useful in catching accidental mistakes.\n\nThe number of people employed in fact-checking varies by publication. Some organizations have substantial fact-checking departments. Others may hire freelancers per piece, or may combine fact-checking with other duties. Magazines are more likely to use fact checkers than newspapers. Television and radio programs rarely employ dedicated fact checkers, and instead expect others, including senior staff, to engage in fact-checking in addition to their other duties.\n\n- 2. Content in this section is adapted from the Wikipedia entry 'Fact-checking' (https:/ /en.wikipedia.org/wiki/ Fact-checking) and is used under a CC BY-SA 3.0 license.\n\n48 | Types of Sources", + "recall": 0.9864864864864865, + "true_md": "## Fact-Checking 2 \n\nIn this context, we are talking about fact-checking that is done before a source is published. Over the last two decades there has been an increase in fact checking as an activity that takes place after a source has been published, a practice discussed in more detail in the chapter, SIFTing Information. \n\nFact checkers verify that the names, dates, and facts in a work (usually an article or book) are correct. For example, they may contact a person who is quoted in a proposed news article and ask the person whether this quotation is correct, or how to spell the person's name. Fact- checkers are primarily useful in catching accidental mistakes. \n\nThe number of people employed in fact-checking varies by publication. Some organizations have substantial fact-checking departments. Others may hire freelancers per piece, or may combine fact-checking with other duties. Magazines are more likely to use fact checkers than newspapers. Television and radio programs rarely employ dedicated fact checkers, and instead expect others, including senior staff, to engage in fact-checking in addition to their other duties. \n\n2. Content in this section is adapted from the Wikipedia entry 'Fact-checking' (https://en.wikipedia.org/wiki/ Fact-checking) and is used under a CC BY-SA 3.0 license. \n\n48 | Types of Sources" + }, + { + "bleu": 1.0, + "doc_id": "doc_e5cd36c1ca7a2c476b14a19497ea75921899a21c6b510f63ef571abf0c000d5b_page_000001.png", + "edit_distance": 0.0, + "f1_score": 1.0, + "meteor": 0.9999999712213304, + "precision": 1.0, + "pred_md": "## Stop\n\nCheck your emotions. If a claim causes strong emotion - anger, glee, pride, vindication - STOP. You must fact-check this claim. Remember from the chapter, Our Mental Shortcuts, that we more readily accept information that confirms our beliefs (confirmation bias) and we tend to think less critically about that kind of information than we do about information that challenges our beliefs (motivated reasoning.) A strong emotional reaction is a sign that these cognitive biases are at work. Remember, these mental shortcuts don't make us bad people, we all have them. But we do need to account for them if we want to move toward better information.\n\nIn addition, if you get lost while working on the other moves, or hit dead ends, or find yourself going down an increasingly confusing rabbit hole during your investigation, STOP. Back up and start over knowing what you know now. You're likely to take a more informed path with different search terms and better decisions.\n\nIn these chapters we're focusing on researching a wicked problem, but the SIFT method is a great thing to use before you share information on social media. Often we feel compelled to share the things that evoke the strongest feelings, but those strong feelings are a good sign that those things need to be checked before they are shared.\n\nSIFTing Information | 69", + "recall": 1.0, + "true_md": "## Stop \n\nCheck your emotions. If a claim causes strong emotion - anger, glee, pride, vindication - STOP. You must fact-check this claim. Remember from the chapter, Our Mental Shortcuts, that we more readily accept information that confirms our beliefs (confirmation bias) and we tend to think less critically about that kind of information than we do about information that challenges our beliefs (motivated reasoning.) A strong emotional reaction is a sign that these cognitive biases are at work. Remember, these mental shortcuts don't make us bad people, we all have them. But we do need to account for them if we want to move toward better information. \n\nIn addition, if you get lost while working on the other moves, or hit dead ends, or find yourself going down an increasingly confusing rabbit hole during your investigation, STOP. Back up and start over knowing what you know now. You're likely to take a more informed path with different search terms and better decisions. \n\nIn these chapters we're focusing on researching a wicked problem, but the SIFT method is a great thing to use before you share information on social media. Often we feel compelled to share the things that evoke the strongest feelings, but those strong feelings are a good sign that those things need to be checked before they are shared. \n\nSIFTing Information | 69" + }, + { + "bleu": 1.0, + "doc_id": "doc_118cb93e60d052c799e7119026c02fd68ca57362f5ac749ee58c8de8556f9393_page_000001.png", + "edit_distance": 0.009615384615384616, + "f1_score": 0.9881422924901185, + "meteor": 0.9946881011125499, + "precision": 0.984251968503937, + "pred_md": "to expand this section to include notes, tips and feedback from TWP instructors. If you use these materials, please let me know how it went, what worked for you, and any suggested changes or additions. I'd love to hear from you at chwixson (at) plymouth (dot) edu or fill out as much of [this form] as you'd like.\n\n## Introduction\n\nThroughout the chapters, I tried to generate Reflection & Discussion Questions that could be used either as in class (whole group or think/pair/share) discussion prompts or as written reflections assigned out of class. If your students generate any written answers to any of the Reflection & Discussion Questions in this chapter, I would be very interested to see them.\n\n## Our Mental Shortcuts\n\nIf you'd like to reinforce Kahneman's ideas about System 1 and System 2 thinking the video below (12 minutes) is very good, (thanks to Mike Davidson for this suggestion.)\n\n/ /www.youtube.com/embed/UBVV8pch1dM\n\nReflection & Discussion Question 1: Taking Stock of What You Already Know\n\n98 | Instructor Resources", + "recall": 0.9920634920634921, + "true_md": "to expand this section to include notes, tips and feedback from TWP instructors. If you use these materials, please let me know how it went, what worked for you, and any suggested changes or additions. I'd love to hear from you at chwixson (at) plymouth (dot) edu or fill out as much of [this form] as you'd like. \n\n## Introduction \n\nThroughout the chapters, I tried to generate Reflection & Discussion Questions that could be used either as in class (whole group or think/pair/share) discussion prompts or as written reflections assigned out of class. If your students generate any written answers to any of the Reflection & Discussion Questions in this chapter, I would be very interested to see them. \n\n## Our Mental Shortcuts \n\nIf you'd like to reinforce Kahneman's ideas about System 1 and System 2 thinking the video below (12 minutes) is very good, (thanks to Mike Davidson for this suggestion.) \n\n//www.youtube.com/embed/UBVV8pch1dM \n\nReflection & Discussion Question 1: Taking Stock of What You Already Know \n\n98 | Instructor Resources" + }, + { + "bleu": 1.0, + "doc_id": "doc_0d4c4e8911eb53569e91f3c53a1deb5951ad138e767ef5edff79b490053fc82a_page_000001.png", + "edit_distance": 0.0, + "f1_score": 1.0, + "meteor": 0.9999999832164076, + "precision": 1.0, + "pred_md": "be a starting point for asking questions too, but I would recommend against brainstorming as the only strategy towards topic and question identification since it does not enable students to get to topics they didn't know existed.\n\nI struggle with getting students to actually read the sources we find together in our research consultations. They seem to want to do all the searching first and all the reading later. No matter how I tell them it's iterative and you need to go back and forth between reading and searching many many times, the messages wasn't landing. This chapter is my next iteration in how to talk about the research process, but I really don't now what the secret recipe is yet. Let me know if you think this one lands.\n\n## Types of Sources\n\nI am a big fan of Mike Caulfield's information literacy work (see the next chapter, SIFTing Information.) Sometimes I have found my attempts to use his strategies in the classroom were hard for students. For example, when I've tried the exercise about the American Academy of Pediatrics and the American College of Pediatricians (Reflection & Discussion Question 1) without first talking about professional organizations, students rarely got how they were different, and it did not build their confidence.\n\nIt's hard to identify a legitimate professional association if you've never heard of the concept of professional associations. This chapter may be long, but I felt it was important to enumerate at least some of the dimensions of the sources they may find, so that when we get to Caulfield's SIFT method they are set up for success.\n\n102 | Instructor Resources", + "recall": 1.0, + "true_md": "be a starting point for asking questions too, but I would recommend against brainstorming as the only strategy towards topic and question identification since it does not enable students to get to topics they didn't know existed. \n\nI struggle with getting students to actually read the sources we find together in our research consultations. They seem to want to do all the searching first and all the reading later. No matter how I tell them it's iterative and you need to go back and forth between reading and searching many many times, the messages wasn't landing. This chapter is my next iteration in how to talk about the research process, but I really don't now what the secret recipe is yet. Let me know if you think this one lands. \n\n## Types of Sources \n\nI am a big fan of Mike Caulfield's information literacy work (see the next chapter, SIFTing Information.) Sometimes I have found my attempts to use his strategies in the classroom were hard for students. For example, when I've tried the exercise about the American Academy of Pediatrics and the American College of Pediatricians (Reflection & Discussion Question 1) without first talking about professional organizations, students rarely got how they were different, and it did not build their confidence. \n\nIt's hard to identify a legitimate professional association if you've never heard of the concept of professional associations. This chapter may be long, but I felt it was important to enumerate at least some of the dimensions of the sources they may find, so that when we get to Caulfield's SIFT method they are set up for success. \n\n102 | Instructor Resources" + }, + { + "bleu": 0.9529214815407602, + "doc_id": "doc_d7f2535a168ba996bb0caccbcc719373580d555909bd5d46ed1748e8f3c17807_page_000001.png", + "edit_distance": 0.02631578947368421, + "f1_score": 0.9916897506925207, + "meteor": 0.9853504882156151, + "precision": 0.988950276243094, + "pred_md": "Other advice that might smooth the way for this exercise is to remind students right before they start that we aren't interested in what these organizations' websites say about themselves, but what they can learn about them from the rest of the internet. Encourage use of Wikipedia for this type of source research. Encourage them to slow down and to practice 'click restraint' once they have Googled one of these orgs. What can they learn from looking at just the search results page, without clicking through to anything? What is the overall impression from a variety of results?\n\n- · Center for Consumer Freedom: Many of the Google search results (with or without including the search term funding) indicate this is astroturing. A look at the Wikipedia page tells us that this org was started by a pretty well known PR guy and the sidebar lists their focus as 'represents the interests of restaurant and food companies' and their method as 'lobbying.'\n- · National Consumers League: Students may note that it has been around since 1899, has no critical results on the first page of Google results, and even has an entry in the Encyclopedia Britannica.\n- · One Fair Wage: a legitimately grass-roots effort to raise the minimum wage for restaurant workers.\n- · Save Our Tips: This is one case where adding the word funding to the search helps a bit. If we do that we find sources indicating that this group is funded in part by the National Restaurant Association and a conservative strategy and consulting group. Not what you would expect for a grassroots effort lead by waitstaff.\n\n104 | Instructor Resources", + "recall": 0.9944444444444445, + "true_md": "Other advice that might smooth the way for this exercise is to remind students right before they start that we aren't interested in what these organizations' websites say about themselves, but what they can learn about them from the rest of the internet. Encourage use of Wikipedia for this type of source research. Encourage them to slow down and to practice 'click restraint' once they have Googled one of these orgs. What can they learn from looking at just the search results page, without clicking through to anything? What is the overall impression from a variety of results? \n\n• Center for Consumer Freedom: Many of the Google search results (with or without including the search term funding) indicate this is astroturing. A look at the Wikipedia page tells us that this org was started by a pretty well known PR guy and the sidebar lists their focus as 'represents the interests of restaurant and food companies' and their method as 'lobbying.' \n\n• National Consumers League: Students may note that it has been around since 1899, has no critical results on the first page of Google results, and even has an entry in the Encyclopedia Britannica. \n\n• One Fair Wage: a legitimately grass-roots effort to raise the minimum wage for restaurant workers. \n\n• Save Our Tips: This is one case where adding the word funding to the search helps a bit. If we do that we find sources indicating that this group is funded in part by the National Restaurant Association and a conservative strategy and consulting group. Not what you would expect for a grassroots effort lead by waitstaff. \n\n104 | Instructor Resources" + }, + { + "bleu": 0.9445349601651277, + "doc_id": "doc_66c1d3c3d52df005397d2b458d43a0c2a5c1a2ba8ab8db4ed5a20ee07d7fe090_page_000001.png", + "edit_distance": 0.03296703296703297, + "f1_score": 0.9915492957746479, + "meteor": 0.9832368836066779, + "precision": 0.9887640449438202, + "pred_md": "- of any individual to color their decisions, even when they're acting in good faith.\n- · Credentials: Academic credentials tend to represent a significant commitment of time towards gaining mastery of a subject, and therefore requiring a particular degree may increase the likelihood of accurate information. However, not all groups are equally represented in higher education. Degree completion is uneven across race and income factors (among others), making academia not demographically representative of our society as a whole. Some perspectives are therefore systematically underrepresented in groups with advanced degrees.\n- · Peer Review: Peer review sometimes only results in collaborative improvements to a work. It can also prevent the publication of very obviously flawed or poorly executed or analyzed research. Very new or radical ideas may be initially rejected because they are such a departure from existing dogma. Peer review is largely a practice of academia, therefore has the same exclusionary problems mentioned in the credentials section. It is possible for individual reviewers to act in a biased or unethical way to prevent the publication of some works.\n- · Fact Checking: Not a lot of downside here. Let me know if your students come up with anything good.\n- · Domains: For some top level domains (mostly just .gov and .edu) looking at the domain provides some assurance that the web content there is an official communication of a particular institution. There really isn't any problem with domains excluding\n\n106 | Instructor Resources", + "recall": 0.9943502824858758, + "true_md": "of any individual to color their decisions, even when they're acting in good faith. \n\n• Credentials: Academic credentials tend to represent a significant commitment of time towards gaining mastery of a subject, and therefore requiring a particular degree may increase the likelihood of accurate information. However, not all groups are equally represented in higher education. Degree completion is uneven across race and income factors (among others), making academia not demographically representative of our society as a whole. Some perspectives are therefore systematically underrepresented in groups with advanced degrees. \n\n• Peer Review: Peer review sometimes only results in collaborative improvements to a work. It can also prevent the publication of very obviously flawed or poorly executed or analyzed research. Very new or radical ideas may be initially rejected because they are such a departure from existing dogma. Peer review is largely a practice of academia, therefore has the same exclusionary problems mentioned in the credentials section. It is possible for individual reviewers to act in a biased or unethical way to prevent the publication of some works. \n\n• Fact Checking: Not a lot of downside here. Let me know if your students come up with anything good. \n\n• Domains: For some top level domains (mostly just .gov and .edu) looking at the domain provides some assurance that the web content there is an official communication of a particular institution. There really isn't any problem with domains excluding \n\n106 | Instructor Resources" + }, + { + "bleu": 0.9193298899751058, + "doc_id": "doc_165dfeaf31832cc4547b6b638a033c1057ddb6384874c16c4574c6278856dd30_page_000001.png", + "edit_distance": 0.04059040590405904, + "f1_score": 0.9896193771626298, + "meteor": 0.9855807923341604, + "precision": 0.9862068965517241, + "pred_md": "- 1. Edward Bernays\n- 2. Wikipedia. Public Relations\n- 3. Pinterest. Retrieved June 10, 2021.\n- 4. Bernays, Edward. Crystalizing Public Opinion.\n- 5. Encyclopedia of Propaganda\n\nPossible directions for the discussion:\n\n- · What the sources suggest about the level of research. Do sources like Wikipedia and Pinterest indicate a deep engagement with the topic? What about the Encyclopedia of Propaganda? Call back to the chapter, Identifying a Topic, encyclopedias are good preliminary sources, but if research stops with an overview source, how valuable is it?\n- · Ways in which the citations are ambiguous. Is enough information provided that readers can find the original information? Is number 1 about that person or written by that person? Is number 4 a book or an article? It has implications for how we would look for it. For number 5, there is more than one book with the title Encyclopedia of Propaganda, and also it's unlikely they meant to refer to the whole encyclopedia.\n- · The difference between discovering a source on a social media platform and citing the content. Is enough information given to find the Pinterest source? Revisit the creator concept from the chapter, Types of Sources. Social media companies distribute but do not create content, so they are not the ones that should be cited. Opportunity to talk about specific sources students have found on social media\n\n114 | Instructor Resources", + "recall": 0.9930555555555556, + "true_md": "1. Edward Bernays \n\n2. Wikipedia. Public Relations \n\n3. Pinterest. Retrieved June 10, 2021. \n\n4. Bernays, Edward. Crystalizing Public Opinion. \n\n5. Encyclopedia of Propaganda \n\nPossible directions for the discussion: \n\n• What the sources suggest about the level of research. Do sources like Wikipedia and Pinterest indicate a deep engagement with the topic? What about the Encyclopedia of Propaganda? Call back to the chapter, Identifying a Topic, encyclopedias are good preliminary sources, but if research stops with an overview source, how valuable is it? \n\n• Ways in which the citations are ambiguous. Is enough information provided that readers can find the original information? Is number 1 about that person or written by that person? Is number 4 a book or an article? It has implications for how we would look for it. For number 5, there is more than one book with the title Encyclopedia of Propaganda, and also it's unlikely they meant to refer to the whole encyclopedia. \n\n• The difference between discovering a source on a social media platform and citing the content. Is enough information given to find the Pinterest source? Revisit the creator concept from the chapter, Types of Sources. Social media companies distribute but do not create content, so they are not the ones that should be cited. Opportunity to talk about specific sources students have found on social media \n\n114 | Instructor Resources" + }, + { + "bleu": 0.7790142188460469, + "doc_id": "doc_d6612de0987496c11003e3acc3632cc1a34f17c6b3473cafc2f736be9ba40308_page_000001.png", + "edit_distance": 0.08333333333333333, + "f1_score": 0.9965635738831616, + "meteor": 0.9905567899709637, + "precision": 0.9931506849315068, + "pred_md": "## HOW CAN YOU HELP?\n\n## As a boater:\n\n- Check tidal conditions beforehand\n- Stay within marked channels\n- Pay attention to buoys and markers\n- Do not run aground\n- If you run aground, call for help\n- Wear polarized sunglasses\n- Take a safe boating course\n\n## As a developer:\n\n- Do careful mapping of seagrass in potential areas for development\n- Avoid dredging and filling\n- Learn about existing regulations\n\n## As a homeowner:\n\n- Diminish fertilizer use (use soaking, rain gardens, and native plants instead)\n- Dispose of pet waste properly\n- Keep seagrass in mind during construction (for example, build high docks with grating instead of planks)\n\n## As anyone who wants to help:\n\n- Urge politicians to establish stricter water quality regulations\n- Mobilize to give seagrass an 'endangered' status\n- Follow established laws for seagrass protection\n- Reach out to environmental organizations and volunteer in restoration projects\n- Challenge the misconception that seagrass is 'ugly' and 'useless'\n- Tell your friends and family about the importance of this ecosystem\n\n## FURTHER RESOURCES\n\n## SEAGRASS IN SOUTH FLORIDA\n\nWHY I T I S I M P O RTANT & WHAT YOU CAN DO CC0, 2022", + "recall": 1.0, + "true_md": "## HOW CAN YOU HELP?\n\n## As a boater:\n\nCheck tidal conditions beforehand\n\nStay within marked channels\n\nPay attention to buoys and markers\n\nDo not run aground\n\nIf you run aground, call for help\n\nWear polarized sunglasses\n\nTake a safe boating course\n\n## As a developer:\n\nDo careful mapping of seagrass in potential areas for development\n\nAvoid dredging and filling\n\nLearn about existing regulations\n\n## As a homeowner:\n\nDiminish fertilizer use (use soaking, rain gardens, and native plants instead)\n\nDispose of pet waste properly\n\nKeep seagrass in mind during construction (for example, build high docks with grating instead of planks)\n\n## As anyone who wants to help: \n\nUrge politicians to establish stricter water quality regulations\n\nMobilize to give seagrass an 'endangered' status \n\nFollow established laws for seagrass protection\n\nReach out to environmental organizations and volunteer in restoration projects \n\nChallenge the misconception that seagrass is 'ugly' and 'useless'\n\nTell your friends and family about the importance of this ecosystem \n\n## FURTHER RESOURCES\n\n## SEAGRASS IN SOUTH FLORIDA \n\nWHY I T I S I M P O RTANT & WHAT YOU CAN DO CC0, 2022" + }, + { + "bleu": 0.9783378132570437, + "doc_id": "doc_286f10070b1be903eb23c297dee52da2d8f108a4c4db79e0f6b62eda54050067_page_000001.png", + "edit_distance": 0.016036655211912942, + "f1_score": 0.9195402298850576, + "meteor": 0.9911154581489545, + "precision": 0.8955223880597015, + "pred_md": "3Btg2 -26 to 31 in; dark grayish brown (10YR 4/2) crushed, silty clay; common coarse prominent dark yellowish brown (10YR 4/6) moist irregular mottles throughout; moderate medium prismatic structure parting to moderate coarse subangular blocky; extremely hard, very firm; common very fine and fine roots throughout; common very fine moderate continuity tubular pores; common distinct continuous very dark grayish brown (10YR 3/2), moist, clay films on vertical and horizontal faces of peds; strongly acid; clear wavy boundary. (0 to 15 in thick)\n\n3Btg3 -31 to 35 in; grayish brown (10YR 5/2) crushed, silty clay; common fine prominent dark yellowish brown (10YR 4/6) moist irregular mottles throughout; moderate medium subangular blocky structure; very hard, friable; common very fine and fine roots throughout; common very fine moderate continuity tubular pores; few faint continuous dark grayish brown (10YR 4/2), moist, clay films on vertical and horizontal faces of peds; common medium rounded very dark grayish brown (10YR 3/2) soft clay bodies pedogenic throughout and few medium rounded white (10YR 8/1) soft nests of gypsum pedogenic throughout; strongly acid; clear wavy boundary. (0 to 10 in thick)\n\n3Btg4 -35 to 42 in; grayish brown (10YR 5/2) crushed, silty clay loam; common fine prominent dark yellowish brown (10YR 4/6) moist irregular mottles throughout and common fine prominent yellowish brown (10YR 5/8) moist irregular mottles throughout; weak coarse prismatic structure parting to moderate medium subangular blocky; very hard, friable; common very fine and fine roots throughout; common very fine and fine moderate continuity tubular pores; few faint discontinuous dark grayish brown (10YR 4/2), moist, clay films on vertical faces of peds and few distinct continuous very dark grayish brown (10YR 3/2) moist, silt coats in root channels and/or pores; few medium rounded white (10YR 8/1) soft nests of gypsum pedogenic throughout; strongly acid; gradual wavy boundary. (0 to 10 in thick)\n\n3Btg5/E -42 to 54 in; dark grayish brown (10YR 4/2) exterior, silty clay loam; common fine prominent dark yellowish brown (10YR 4/6) moist irregular mottles throughout; moderate coarse prismatic structure parting to moderate medium subangular blocky; hard, friable; common very and fine roots throughout; many very fine and fine moderate continuity tubular pores; few faint discontinuous dark grayish brown (10YR 4/2) moist clay films on vertical faces of peds and few distinct continuous very dark grayish brown (10YR 3/2) moist, silt coats in root channels and/or pores; strongly acid; gradual wavy boundary. (0 to 15 in thick)\n\n3Btg6/E -54 to 69 in; light brownish gray (10YR 6/2) exterior, silty clay loam; common coarse prominent dark yellowish brown (10YR 4/6) moist irregular mottles throughout and common coarse prominent dark reddish brown (5YR 3/4) moist irregular mottles throughout; moderate coarse prismatic structure parting to weak coarse subangular blocky; slightly hard, very friable; common very fine and fine roots throughout; many very fine and fine moderate continuity tubular pores; few faint continuous grayish brown (10YR 5/2), moist, clay films on vertical faces of peds and few distinct continuous dark grayish brown(10YR 4/2) moist silt coats in root channels and/or pores; common fine rounded black (N 2/0) soft iron/manganese concretions pedogenic throughout; strongly acid; gradual wavy boundary. (0 to 20 in thick)\n\n3Btg7/E -69 to 86 in; light brownish gray (10YR 6/2) exterior, silty clay loam; common coarse prominent dark yellowish brown (10YR 4/6) moist irregular mottles throughout and common fine prominent dark brown (7.5YR 3/4.) moist irregular mottles throughout; weak coarse prismatic structure; slightly hard, very friable; few very fine roots throughout; common very fine and fine moderate continuity tubular pores; few faint discontinuous dark grayish brown (10YR 4/2), moist, clay films on vertical faces of peds and few distinct continuous grayish brown (10YR 5/2) moist, silt coats in root channels and/or pores; common fine rounded black (N 2/0) soft iron/manganese concretions pedogenic throughout and few medium irregular brown (10YR 5/3) soft clay bodies pedogenic in cracks; very strongly acid; clear smooth boundary. (0 to 20 in thick)\n\n3Btg8/E -86 to 97 in; 80% light brownish gray (2.5Y 6/2) exterior, and 15% yellowish brown (10YR 5/8), exterior, and 5% strong brown (7.5 YR 4/6), exterior, silty clay loam; moderate coarse prismatic structure parting to weak coarse\n\nSoil Formation | 27", + "recall": 0.9448818897637795, + "true_md": "3Btg2-26 to 31 in; dark grayish brown (10YR 4/2) crushed, silty clay; common coarse prominent dark yellowish brown (10YR 4/6) moist irregular mottles throughout; moderate medium prismatic structure parting to moderate coarse subangular blocky; extremely hard, very firm; common very fine and fine roots throughout; common very fine moderate continuity tubular pores; common distinct continuous very dark grayish brown (10YR 3/2), moist, clay films on vertical and horizontal faces of peds; strongly acid; clear wavy boundary. (0 to 15 in thick) \n\n3Btg3-31 to 35 in; grayish brown (10YR 5/2) crushed, silty clay; common fine prominent dark yellowish brown (10YR 4/6) moist irregular mottles throughout; moderate medium subangular blocky structure; very hard, friable; common very fine and fine roots throughout; common very fine moderate continuity tubular pores; few faint continuous dark grayish brown (10YR 4/2), moist, clay films on vertical and horizontal faces of peds; common medium rounded very dark grayish brown (10YR 3/2) soft clay bodies pedogenic throughout and few medium rounded white (10YR 8/1) soft nests of gypsum pedogenic throughout; strongly acid; clear wavy boundary. (0 to 10 in thick) \n\n3Btg4-35 to 42 in; grayish brown (10YR 5/2) crushed, silty clay loam; common fine prominent dark yellowish brown (10YR 4/6) moist irregular mottles throughout and common fine prominent yellowish brown (10YR 5/8) moist irregular mottles throughout; weak coarse prismatic structure parting to moderate medium subangular blocky; very hard, friable; common very fine and fine roots throughout; common very fine and fine moderate continuity tubular pores; few faint discontinuous dark grayish brown (10YR 4/2), moist, clay films on vertical faces of peds and few distinct continuous very dark grayish brown (10YR 3/2) moist, silt coats in root channels and/or pores; few medium rounded white (10YR 8/1) soft nests of gypsum pedogenic throughout; strongly acid; gradual wavy boundary. (0 to 10 in thick) \n\n3Btg5/E-42 to 54 in; dark grayish brown (10YR 4/2) exterior, silty clay loam; common fine prominent dark yellowish brown (10YR 4/6) moist irregular mottles throughout; moderate coarse prismatic structure parting to moderate medium subangular blocky; hard, friable; common very and fine roots throughout; many very fine and fine moderate continuity tubular pores; few faint discontinuous dark grayish brown (10YR 4/2) moist clay films on vertical faces of peds and few distinct continuous very dark grayish brown (10YR 3/2) moist, silt coats in root channels and/or pores; strongly acid; gradual wavy boundary. (0 to 15 in thick) \n\n3Btg6/E-54 to 69 in; light brownish gray (10YR 6/2) exterior, silty clay loam; common coarse prominent dark yellowish brown (10YR 4/6) moist irregular mottles throughout and common coarse prominent dark reddish brown (5YR 3/4) moist irregular mottles throughout; moderate coarse prismatic structure parting to weak coarse subangular blocky; slightly hard, very friable; common very fine and fine roots throughout; many very fine and fine moderate continuity tubular pores; few faint continuous grayish brown (10YR 5/2), moist, clay films on vertical faces of peds and few distinct continuous dark grayish brown(10YR 4/2) moist silt coats in root channels and/or pores; common fine rounded black (N 2/0) soft iron/manganese concretions pedogenic throughout; strongly acid; gradual wavy boundary. (0 to 20 in thick) \n\n3Btg7/E-69 to 86 in; light brownish gray (10YR 6/2) exterior, silty clay loam; common coarse prominent dark yellowish brown (10YR 4/6) moist irregular mottles throughout and common fine prominent dark brown (7.5YR 3/4.) moist irregular mottles throughout; weak coarse prismatic structure; slightly hard, very friable; few very fine roots throughout; common very fine and fine moderate continuity tubular pores; few faint discontinuous dark grayish brown (10YR 4/2), moist, clay films on vertical faces of peds and few distinct continuous grayish brown (10YR 5/2) moist, silt coats in root channels and/or pores; common fine rounded black (N 2/0) soft iron/manganese concretions pedogenic throughout and few medium irregular brown (10YR 5/3) soft clay bodies pedogenic in cracks; very strongly acid; clear smooth boundary. (0 to 20 in thick) \n\n3Btg8/E-86 to 97 in; 80% light brownish gray (2.5Y 6/2) exterior, and 15% yellowish brown (10YR 5/8), exterior, and 5% strong brown (7.5 YR 4/6), exterior, silty clay loam; moderate coarse prismatic structure parting to weak coarse \n\nSoil Formation | 27" + }, + { + "bleu": 0.9330236386998406, + "doc_id": "doc_ed046862e68a27a7260da60f3984484577f83500dea61020d043c7af8b76731f_page_000001.png", + "edit_distance": 0.03323262839879154, + "f1_score": 0.9969418960244648, + "meteor": 0.9956650257223771, + "precision": 0.9939024390243902, + "pred_md": "Record your observations in Table 13.2.\n\n## Table 13.2. Effect of cations on flocculation of a clay suspension.\n\n## Activity 4. Determining CEC by replacing adsorbed cations.\n\nIn this activity, you will titrate the filtrate with a 0.01 molar solution of NaOH using phenolphthalein as an indicator. Phenolphthalein changes from colorless to faint pink when the quantity of OH ions added via the NaOH equals the -quantity of H ions in the solution (that is, when the pH is raised to 7). For this activity, assume the soil samples have + been extracted and the filtrates are now available for analysis.\n\n- 1. Place 10 ml of each filtrate into separate 125 ml flasks. This 10 ml quantity is the amount of filtrate from 1.0 gram of soil.\n- 2. Add 10 drops of the phenolphthalein indicator.\n- 3. Titrate the extract with the NaOH solution to a faint pink endpoint. The titration must be done very carefully to obtain meaningful results. If you put too much NaOH in the flask and get a bright pink color, discard the solution and repeat the process. In the table below, record the milliliters of NaOH solution used to achieve the endpoint.\n\nCalculate the CEC and record your data in Table 13.3.\n\nHere is an example of how to calculate the CEC, assuming 2.5 mL of NaOH was required to achieve an end point. The reaction occurring during titration is\n\nThus, one mole of NaOH reacts with one mole of H . Therefore, at the phenolphthalein end point, moles of NaOH added + = moles of H+ in solution.\n\nThe solution of 0.01 molar NaOH contains 1 cmol charge per liter (1 cmolc/L). Therefore 2.5 mL NaOH contains\n\nThus, the CEC is\n\n114 | Soil Colloids", + "recall": 1.0, + "true_md": " Record your observations in Table 13.2. \n\nTable 13.2. Effect of cations on flocculation of a clay suspension. \n\n## Activity 4. Determining CEC by replacing adsorbed cations. \n\nIn this activity, you will titrate the filtrate with a 0.01 molar solution of NaOH using phenolphthalein as an indicator. Phenolphthalein changes from colorless to faint pink when the quantity of OH - ions added via the NaOH equals the quantity of H + ions in the solution (that is, when the pH is raised to 7). For this activity, assume the soil samples have been extracted and the filtrates are now available for analysis. \n\n1. Place 10 ml of each filtrate into separate 125 ml flasks. This 10 ml quantity is the amount of filtrate from 1.0 gram of soil. \n\n2. Add 10 drops of the phenolphthalein indicator. \n\n3. Titrate the extract with the NaOH solution to a faint pink endpoint. The titration must be done very carefully to obtain meaningful results. If you put too much NaOH in the flask and get a bright pink color, discard the solution and repeat the process. In the table below, record the milliliters of NaOH solution used to achieve the endpoint. \n\n Calculate the CEC and record your data in Table 13.3. \n\nHere is an example of how to calculate the CEC, assuming 2.5 mL of NaOH was required to achieve an end point. The reaction occurring during titration is \n\nThus, one mole of NaOH reacts with one mole of H + . Therefore, at the phenolphthalein end point, moles of NaOH added = moles of H+ in solution. \n\nThe solution of 0.01 molar NaOH contains 1 cmol charge per liter (1 cmolc/L). Therefore 2.5 mL NaOH contains \n\nThus, the CEC is \n\n114 | Soil Colloids" + }, + { + "bleu": 0.9877602812634523, + "doc_id": "doc_2e4f2561d8624cab6140564d53aed11862cf5979572fc3d1205af105486e0b1f_page_000001.png", + "edit_distance": 0.006968641114982578, + "f1_score": 1.0, + "meteor": 0.9992913344358411, + "precision": 1.0, + "pred_md": "## Activity 5. Calculating versus estimating CEC\n\nThere are two ways you can calculate the CEC: the sum of cations method and the mineralogy method.\n\n## The Sum-of-Cations Method\n\nIf you have a soil analysis where the quantities of all cations in the soil are listed, simply summing all those exchangeable quantities will yield the CEC you found in the preceding problems.\n\n## The 'Mineralogy' Method\n\nAs you know from your reading and class discussion, clay minerals have a range of values for CEC. If the mineralogy of the clay fraction is known (that is, the type and amounts of each clay mineral), then the CEC can be approximated.\n\nTo make these calculations easier, Table 13.4 contains representative values for CEC to use in all calculations for this class unless otherwise noted. In nature, however, these soil colloids will have a range of values.\n\n## Table 13.4. Typical CEC of various soil colloids.\n\nAs an example of this mineralogy approach to CEC calculations, consider a soil having 100% clay where the clay is 100% kaolinite. The CEC would then be 10 cmolc/kg. If a soil contains only 10% kaolinite (or 10 kg clay in 100 kg soil), however, this clay would contribute\n\nA prairie soil contains 30% clay. This clay sized fraction is dominantly montmorillonite. The soil also contains 5% humus (organic matter).\n\nUsing the mineralogy method, what is the cation exchange capacity (CEC) contributed by the clay?\n\n120 | Soil Colloids", + "recall": 1.0, + "true_md": "## Activity 5. Calculating versus estimating CEC \n\nThere are two ways you can calculate the CEC: the sum of cations method and the mineralogy method. \n\n## The Sum-of-Cations Method \n\nIf you have a soil analysis where the quantities of all cations in the soil are listed, simply summing all those exchangeable quantities will yield the CEC you found in the preceding problems. \n\n## The 'Mineralogy' Method \n\nAs you know from your reading and class discussion, clay minerals have a range of values for CEC. If the mineralogy of the clay fraction is known (that is, the type and amounts of each clay mineral), then the CEC can be approximated. \n\nTo make these calculations easier, Table 13.4 contains representative values for CEC to use in all calculations for this class unless otherwise noted. In nature, however, these soil colloids will have a range of values. \n\nTable 13.4. Typical CEC of various soil colloids. \n\nAs an example of this mineralogy approach to CEC calculations, consider a soil having 100% clay where the clay is 100% kaolinite. The CEC would then be 10 cmolc/kg. If a soil contains only 10% kaolinite (or 10 kg clay in 100 kg soil), however, this clay would contribute \n\nA prairie soil contains 30% clay. This clay sized fraction is dominantly montmorillonite. The soil also contains 5% humus (organic matter). \n\n Using the mineralogy method, what is the cation exchange capacity (CEC) contributed by the clay? \n\n120 | Soil Colloids" + }, + { + "bleu": 0.9530482646641534, + "doc_id": "doc_18fbe4800ce186f3338eade3d1800bde4e275396656df9781f0d84f4b6e22f0c_page_000001.png", + "edit_distance": 0.028662420382165606, + "f1_score": 0.9872881355932203, + "meteor": 0.9815535926918355, + "precision": 0.9872881355932204, + "pred_md": "The acidic cations adsorbed on the negative exchange sites are called the reserve ( also residual or potential) and saltreplaceable ( also exchangeable) acidity. The reserve and salt-replaceable acidity controls the level of soluble or active acidity in the soil solution. Only the active acidity is measured in a routine pH determination. The reserve and saltreplaceable acidity is always many times higher than the active acidity.\n\nA soil is acid when hydrogen ions predominate in the soil. The degree of acidity is expressed in terms of pH, which is defined as the negative logarithm of the hydrogen ion activity. Therefore, the pH of a 0.01-molar hydrogen ion solution is\n\nAt pH 7, the concentration of H+ ions and OH- ions are equal, and the soil or solution is neutral. At pH values less than 7, the soil is acid; at values more than 7, the soil is alkaline. Most soils vary in pH from about 4 to 10. Soils in areas with high rainfall are generally acid with a pH less than 7. Soils developed in high-lime deposits often will be alkaline. Soils high in calcium seldom have pH values higher than 7.5, but the presence of large amounts of calcium carbonate may cause the pH to be as high as 8.5. Where the pH is higher than 8.5, an excess of sodium is highly probable.\n\nThe most desirable soil pH for most crops in Kansas is 6.8. However, crops like blueberries need a lower pH, and other crops, like alfalfa, need a higher pH. At soil pH less than 5.8, several problems may occur:\n\n- · Al and Mn toxicity\n- · Inhibited growth of N-fixing bacteria\n- · Possible deficiencies in Mg and/or Ca.\n- · P deficiency (P reacts with Fe and Al)\n- · At more than pH 7.5, other problems may occur:\n- · Deficiency of Fe, Mn, Cu, or Zn\n- · P deficiency (P reacts with Ca)\n\n## Buffering Capacity\n\nBuffering capacity is a measure of the soil's ability to resist a change in pH, directly related to the magnitude of the exchange capacity. Small fluctuations in acid or base content can occur without a noticeable pH change as cations are adsorbed or released from the exchange complex. Soils with the largest cation exchange capacity have the greatest buffering of a pH change. In other words, two soils may have the same pH (active acidity in soil solution), but the one with the largest cation exchange capacity will have the most acidity stored in reserve and therefore the highest buffering capacity or ability to resist a change in pH. For this reason, it takes less lime to increase the pH of a sandy soil (low CEC) by a given amount than it takes to increase the pH of a clay soil (higher CEC) the same amount.\n\n## Sources of Soil Acidity\n\nControlling soil pH is vital to optimal use and productivity of soils. Adding lime is the most effective and practical way to raise the pH of acid soils. Elemental sulfur, iron sulfate, or aluminum sulfate can be used to reduce soil pH. Because acidity is a concern in Kansas, we will focus on raising soil pH. Understanding the following equations should help you understand the sources of soil acidity and soil reactions to lime.\n\n124 | Soil Acidity and Adjusting Soil pH", + "recall": 0.9872881355932204, + "true_md": "The acidic cations adsorbed on the negative exchange sites are called the reserve (also residual or potential) and salt- replaceable (also exchangeable) acidity. The reserve and salt-replaceable acidity controls the level of soluble or active acidity in the soil solution. Only the active acidity is measured in a routine pH determination. The reserve and salt- replaceable acidity is always many times higher than the active acidity. \n\nA soil is acid when hydrogen ions predominate in the soil. The degree of acidity is expressed in terms of pH, which is defined as the negative logarithm of the hydrogen ion activity. Therefore, the pH of a 0.01-molar hydrogen ion solution is \n\nAt pH 7, the concentration of H+ ions and OH- ions are equal, and the soil or solution is neutral. At pH values less than 7, the soil is acid; at values more than 7, the soil is alkaline. Most soils vary in pH from about 4 to 10. Soils in areas with high rainfall are generally acid with a pH less than 7. Soils developed in high-lime deposits often will be alkaline. Soils high in calcium seldom have pH values higher than 7.5, but the presence of large amounts of calcium carbonate may cause the pH to be as high as 8.5. Where the pH is higher than 8.5, an excess of sodium is highly probable. \n\nThe most desirable soil pH for most crops in Kansas is 6.8. However, crops like blueberries need a lower pH, and other crops, like alfalfa, need a higher pH. At soil pH less than 5.8, several problems may occur: \n\n• Al and Mn toxicity \n\n• Inhibited growth of N-fixing bacteria \n\n• Possible deficiencies in Mg and/or Ca. \n\n• P deficiency (P reacts with Fe and Al) \n\n• At more than pH 7.5, other problems may occur: \n\n• Deficiency of Fe, Mn, Cu, or Zn \n\n• P deficiency (P reacts with Ca) \n\n## Buffering Capacity \n\nBuffering capacity is a measure of the soil's ability to resist a change in pH, directly related to the magnitude of the exchange capacity. Small fluctuations in acid or base content can occur without a noticeable pH change as cations are adsorbed or released from the exchange complex. Soils with the largest cation exchange capacity have the greatest buffering of a pH change. In other words, two soils may have the same pH (active acidity in soil solution), but the one with the largest cation exchange capacity will have the most acidity stored in reserve and therefore the highest buffering capacity or ability to resist a change in pH. For this reason, it takes less lime to increase the pH of a sandy soil (low CEC) by a given amount than it takes to increase the pH of a clay soil (higher CEC) the same amount. \n\n## Sources of Soil Acidity \n\nControlling soil pH is vital to optimal use and productivity of soils. Adding lime is the most effective and practical way to raise the pH of acid soils. Elemental sulfur, iron sulfate, or aluminum sulfate can be used to reduce soil pH. Because acidity is a concern in Kansas, we will focus on raising soil pH. Understanding the following equations should help you understand the sources of soil acidity and soil reactions to lime. \n\n124 | Soil Acidity and Adjusting Soil pH" + }, + { + "bleu": 0.9846346559133785, + "doc_id": "doc_d6ce9ae24c7a15aa55e9a8c232c913cb8bcc1dc2d5d7f129258f4cbf6f46719d_page_000001.png", + "edit_distance": 0.0091324200913242, + "f1_score": 1.0, + "meteor": 0.9999979590056952, + "precision": 1.0, + "pred_md": "Soils with the same pH may require different amounts of limestone due to differences in CEC, which would imply differences in buffering capacities. For example, consider the amount of limestone necessary to raise the base saturation of two soils from 70% to 90% when one soil has a CEC of 15 cmolc/kg, and the other has a CEC of 40 cmolc/kg.\n\nLastly, soil pH is governed by base saturation. If other factors are constant, the lower the pH, the more lime that is required to achieve a desired pH. This is because at a low pH, a larger percentage of the CEC is occupied by acid cations, which requires larger amounts of lime to neutralize.\n\n## Activity 1: Determining pH With Indicator Strips (Field Method)\n\nOf the several techniques available for determining pH, one that can be used easily in the field is the indicator strip method. This technique uses the principle of pH sensitivity of certain dyes, which cause differences in color across a range in pH. With the soils provided, complete the following pH determination:\n\nWeigh 10.0 g of soil into a small plastic cup. Add 20 ml of distilled water and stir. Allow to stand for 5 minutes, occasionally stirring.\n\nUsing the pH indicator strips provided, dip the strip into the cup until the tip is wetted. Determine the pH by comparing the color change of the pH test strip to the color chart.\n\nRecord the soil pH in Table 14.1.\n\n## Activity 2: Determining Soil pH with a pH Meter\n\nLaboratory pH meters are more accurate than pH dyes and strips. The pH meter measures the hydrogen ion activity [H ] + by measuring the electric potential across a thin, porous glass membrane at the base of the electrode. This potential changes in response to [H ], and by standardizing the instrument with buffers of known pH, we can measure the pH of + any solution, including soil solutions.\n\nUsing the samples prepared in Activity 1, carefully place the electrode in the suspension. Gently swirl the electrode in the solution, and note the pH reading. Wait for the pH meter to reach a steady reading, indicated by the word 'ready' on the screen.\n\nRecord the value for this 1:2 soil-water suspension in Table 14.1.\n\nSoil Acidity and Adjusting Soil pH | 127", + "recall": 1.0, + "true_md": "Soils with the same pH may require different amounts of limestone due to differences in CEC, which would imply differences in buffering capacities. For example, consider the amount of limestone necessary to raise the base saturation of two soils from 70% to 90% when one soil has a CEC of 15 cmolc/kg, and the other has a CEC of 40 cmolc/kg. \n\nLastly, soil pH is governed by base saturation. If other factors are constant, the lower the pH, the more lime that is required to achieve a desired pH. This is because at a low pH, a larger percentage of the CEC is occupied by acid cations, which requires larger amounts of lime to neutralize. \n\n## Activity 1: Determining pH With Indicator Strips (Field Method) \n\nOf the several techniques available for determining pH, one that can be used easily in the field is the indicator strip method. This technique uses the principle of pH sensitivity of certain dyes, which cause differences in color across a range in pH. With the soils provided, complete the following pH determination: \n\nWeigh 10.0 g of soil into a small plastic cup. Add 20 ml of distilled water and stir. Allow to stand for 5 minutes, occasionally stirring. \n\nUsing the pH indicator strips provided, dip the strip into the cup until the tip is wetted. Determine the pH by comparing the color change of the pH test strip to the color chart. \n\n Record the soil pH in Table 14.1. \n\n## Activity 2: Determining Soil pH with a pH Meter \n\nLaboratory pH meters are more accurate than pH dyes and strips. The pH meter measures the hydrogen ion activity [H + ] by measuring the electric potential across a thin, porous glass membrane at the base of the electrode. This potential changes in response to [H + ], and by standardizing the instrument with buffers of known pH, we can measure the pH of any solution, including soil solutions. \n\nUsing the samples prepared in Activity 1, carefully place the electrode in the suspension. Gently swirl the electrode in the solution, and note the pH reading. Wait for the pH meter to reach a steady reading, indicated by the word 'ready' on the screen. \n\n Record the value for this 1:2 soil-water suspension in Table 14.1. \n\nSoil Acidity and Adjusting Soil pH | 127" + }, + { + "bleu": 0.8902306595546721, + "doc_id": "doc_37b2674a918c0c0782194d45b011f7c9ffca19ccc245545fb5e35027e7c67a9d_page_000001.png", + "edit_distance": 0.05790645879732739, + "f1_score": 0.9931972789115648, + "meteor": 0.9732824531045366, + "precision": 0.9909502262443439, + "pred_md": "- · Lime is recommended if pH < 5.8\n- · Depth is in inches\n- · Used if cash flow is limited or in lime availability problem areas in Central and Western Kansas\n- · Lime is recommended if pH < 5.5\n\nThis buffer contains chromium (Cr), a toxic heavy metal. Therefore, your lab instructor will perform the SMP buffer analysis. As a class, determine which soil-water mixtures from Activity 1 need lime (pH ≤ 6.4). To those solutions, add 10 ml of the SMP buffer solution, and stir with a glass rod. Allow the mixtures to stand for 30 minutes, which should be enough time for the acid cations to be displaced from the CEC and forced into solution. Read the pH on meter.\n\nAssuming the desired pH is 6.0 (i.e. use the middle equation), calculate the lime requirement, show your work below, and record your results in Table 14.1.\n\n## Activity 5: Evaluating Liming Materials\n\nThe type of liming material and the size or fineness of the material determine how efficiently liming materials raise soil pH. This experiment was actually initiated earlier in the semester to allow time for the liming agents to react. Amending the soil with several different liming agents allows us assess the effects of particle size and liming material based on the relative changes in soil. The treatments included the following:\n\n- · Reagent grade CaCO3\n- · Reagent grade CaO\n- · Reagent grade CaSO4\n- · Coarse dolomitic limestone (35 mesh)\n- · Fine dolomitic limestone (120 mesh)\n- · Control (no amendments)\n\nWhen this experiment was initiated, each lab section was divided into six groups, with each group responsible for one of the six treatments. Your laboratory instructor assigned a treatment to your group, and you completed the following steps:\n\n- 1. Label four plastic bags\n- 2. Weigh 20 g of air-dry soil into each plastic bag.\n- 3. Weigh 0.1 gram of designated liming material onto weighing paper.\n- 4. Add the liming material to the soil and mix thoroughly to distribute evenly in the soil.\n- 5. Add a few mL of water to each bag and mix.\n- 6. Close the bags to start incubation.\n\nNow that the liming agents have had time to react, you will collect the results.\n\n130 | Soil Acidity and Adjusting Soil pH", + "recall": 0.9954545454545455, + "true_md": "• Lime is recommended if pH < 5.8 \n\n• Depth is in inches \n\n• Used if cash flow is limited or in lime availability problem areas in Central and Western Kansas \n\n• Lime is recommended if pH < 5.5 \n\nThis buffer contains chromium (Cr), a toxic heavy metal. Therefore, your lab instructor will perform the SMP buffer analysis. As a class, determine which soil-water mixtures from Activity 1 need lime (pH ≤ 6.4). To those solutions, add 10 ml of the SMP buffer solution, and stir with a glass rod. Allow the mixtures to stand for 30 minutes, which should be enough time for the acid cations to be displaced from the CEC and forced into solution. Read the pH on meter. \n\n Assuming the desired pH is 6.0 (i.e. use the middle equation), calculate the lime requirement, show your work below, and record your results in Table 14.1. \n\n## Activity 5: Evaluating Liming Materials \n\nThe type of liming material and the size or fineness of the material determine how efficiently liming materials raise soil pH. This experiment was actually initiated earlier in the semester to allow time for the liming agents to react. Amending the soil with several different liming agents allows us assess the effects of particle size and liming material based on the relative changes in soil. The treatments included the following: \n\n• Reagent grade CaCO3 \n\n• Reagent grade CaO \n\n• Reagent grade CaSO4 \n\n• Coarse dolomitic limestone (35 mesh) \n\n• Fine dolomitic limestone (120 mesh) \n\n• Control (no amendments) \n\nWhen this experiment was initiated, each lab section was divided into six groups, with each group responsible for one of the six treatments. Your laboratory instructor assigned a treatment to your group, and you completed the following steps: \n\n1. Label four plastic bags \n\n2. Weigh 20 g of air-dry soil into each plastic bag. \n\n3. Weigh 0.1 gram of designated liming material onto weighing paper. \n\n4. Add the liming material to the soil and mix thoroughly to distribute evenly in the soil. \n\n5. Add a few mL of water to each bag and mix. \n\n6. Close the bags to start incubation. \n\nNow that the liming agents have had time to react, you will collect the results. \n\n130 | Soil Acidity and Adjusting Soil pH" + }, + { + "bleu": 1.0, + "doc_id": "doc_4d5b682effaf9580928a7073980e3916687d168238bc96b6c17222f747eacde6_page_000001.png", + "edit_distance": 0.0, + "f1_score": 1.0, + "meteor": 0.9999999599588955, + "precision": 1.0, + "pred_md": "## cropping.\n\nTable adapted from Jones et al. (1988) with permission. †Strip cropping uses a four-year rotation of row crop followed by one year of a small grain and two years of meadow (forages) for RGMM, or uses two years of row crops followed by one year of small grain and one year of meadow for RRGM. Meadow includes alfalfa, clover, grass, etc.\n\nHow does the erosion rate under contour tillage compare to the tolerable erosion rate?\n\nHow does the erosion rate under contour tillage compare to the erosion rate under conservation tillage alone?\n\nNext we will test the impact of installing terraces on the landscape. Using Table 16.5, determine the Pt factor. When terraces are installed, contour tillage is usually used as well. Also, note that installing a terrace results in a shorter length of the slope (because the terrace stops water from continuing to run down slope), so this calculation is performed for each terrace individually. Also note that the net P factor is determined by multiplying the\n\nPc and Pt values together, or writing the RUSLE as follows:\n\nTable 16.5. Conservation practice (P) values for terraces with underground outlets or waterways.\n\n146 | Soil Erosion and Conservation", + "recall": 1.0, + "true_md": "## cropping. \n\nTable adapted from Jones et al. (1988) with permission. †Strip cropping uses a four-year rotation of row crop followed by one year of a small grain and two years of meadow (forages) for RGMM, or uses two years of row crops followed by one year of small grain and one year of meadow for RRGM. Meadow includes alfalfa, clover, grass, etc. \n\n How does the erosion rate under contour tillage compare to the tolerable erosion rate? \n\n How does the erosion rate under contour tillage compare to the erosion rate under conservation tillage alone? \n\nNext we will test the impact of installing terraces on the landscape. Using Table 16.5, determine the Pt factor. When terraces are installed, contour tillage is usually used as well. Also, note that installing a terrace results in a shorter length of the slope (because the terrace stops water from continuing to run down slope), so this calculation is performed for each terrace individually. Also note that the net P factor is determined by multiplying the Pc and Pt values together, or writing the RUSLE as follows: \n\nTable 16.5. Conservation practice (P) values for terraces with underground outlets or waterways. \n\n146 | Soil Erosion and Conservation" + }, + { + "bleu": 0.9541897568896189, + "doc_id": "doc_7a0dc2cafa8fc77f28d78c90092a4cd31670dcf0ccfe8474a2765148448f44d7_page_000001.png", + "edit_distance": 0.030470914127423823, + "f1_score": 1.0, + "meteor": 0.996796359709009, + "precision": 1.0, + "pred_md": "## Contents\n\n| Acknowledgment of Country | v |\n|-----------------------------------------------------------------------------------------|------|\n| Accessibility Information | vi |\n| Acknowledgments | vii |\n| About the Authors | viii |\n| Introduction | 1 |\n| Part I. Chapter One - Exploring Your Data | |\n| Section 1.1: Data and Types of Statistical Variables | 3 |\n| Section 1.2: Descriptive Statistics | 5 |\n| Section 1.3: Missing Data | 6 |\n| Section 1.4: Checking Values | 7 |\n| Section 1.5: Normality | 8 |\n| Section 1.6: Outliers | 9 |\n| Section 1.7: Chapter One Self-Test | 10 |\n| Part II. Chapter Two - Test Statistics, p Values, Confidence Intervals and Effect Sizes | |\n| Section 2.1: p Values | 12 |\n| Section 2.2: Significance | 13 |\n| Section 2.3: Confidence Intervals | 14 |\n| Section 2.4: Effect Sizes | 16 |\n| Section 2.5: Statistical Power | 17 |\n| Section 2.6: Chapter Two Self-Test | 18 |\n| Part III. Chapter Three - Comparing Two Group Means | |\n| Section 3.1: Looking at Group Differences | 20 |\n| Section 3.2: Between Versus Within Groups Analysis | 21 |\n| Section 3.3: Independent T-test Assumptions, Interpretation, and Write Up | 22 |\n| Section 3.4: Paired T-test Assumptions, Interpretation, and Write Up | 25 |\n| Section 3.5: Chapter Three Self-Test | 27 |\n| Part IV. Chapter Four - Comparing Associations Between Two Variables | |\n| Section 4.1: Examining Relationships | 29 |\n| Section 4.2: Correlation Assumptions, Interpretation, and Write Up | 31 |\n| Section 4.3: Chapter Four Self-Test | 33 |", + "recall": 1.0, + "true_md": "## Contents \n\n| Acknowledgment of Country | v |\n|-----------------------------------------------------------------------------------|-----------|\n| Accessibility Information | vi |\n| Acknowledgments | vii |\n| About the Authors | viii |\n| Introduction | 1 |\n| Part I. Chapter One - Exploring Your | Data |\n| Section 1.1: Data and Types of Statistical Variables | 3 |\n| Section 1.2: Descriptive Statistics | 5 |\n| Section 1.3: Missing Data | 6 |\n| Section 1.4: Checking Values | 7 |\n| Section 1.5: Normality | 8 |\n| Section 1.6: Outliers | 9 |\n| Section 1.7: Chapter One Self-Test | 10 |\n| Part II. Chapter Two - Test Statistics, p Values, Confidence Intervals and Effect | Sizes |\n| Section 2.1: p Values | 12 |\n| Section 2.2: Significance | 13 |\n| Section 2.3: Confidence Intervals | 14 |\n| Section 2.4: Effect Sizes | 16 |\n| Section 2.5: Statistical Power | 17 |\n| Section 2.6: Chapter Two Self-Test | 18 |\n| Part III. Chapter Three - Comparing Two Group | Means |\n| Section 3.1: Looking at Group Differences | 20 |\n| Section 3.2: Between Versus Within Groups Analysis | 21 |\n| Section 3.3: Independent T-test Assumptions, Interpretation, and Write Up | 22 |\n| Section 3.4: Paired T-test Assumptions, Interpretation, and Write Up | 25 |\n| Section 3.5: Chapter Three Self-Test | 27 |\n| Part IV. Chapter Four - Comparing Associations Between Two | Variables |\n| Section 4.1: Examining Relationships | 29 |\n| Section 4.2: Correlation Assumptions, Interpretation, and Write Up | 31 |\n| Section 4.3: Chapter Four Self-Test | 33 |" + }, + { + "bleu": 0.9211513556912155, + "doc_id": "doc_4a23b09904c5fbb871fcf792e30b87c4b7cae0cdd0079b5ee0e3e34a353638b7_page_000001.png", + "edit_distance": 0.07453416149068323, + "f1_score": 0.98989898989899, + "meteor": 0.9497142713192155, + "precision": 0.9865771812080537, + "pred_md": "## Part V. Chapter Five - Comparing Associations Between Multiple Variables\n\n| Section 5.1: The Linear Model | 35 |\n|---------------------------------------------------------------------------------------------|------|\n| Section 5.2: Simple Regression Assumptions, Interpretation, and Write Up | 36 |\n| Section 5.3: Multiple Regression Explanation, Assumptions, Interpretation, and Write Up | 39 |\n| Section 5.4: Hierarchical Regression Explanation, Assumptions, Interpretation, and Write Up | 43 |\n| Section 5.5: Chapter Five Self-Test | 47 |\n| Part VI. Chapter Six - Comparing Three or More Group Means | |\n| Section 6.1: Between Versus Within Group Analyses | 49 |\n| Section 6.2: One-Way ANOVA Assumptions, Interpretation, and Write Up | 51 |\n| Section 6.3 Repeated Measures ANOVA Assumptions, Interpretation, and Write Up | 54 |\n| Section 6.4: Chapter Six Self-Test | 62 |\n| Part VII. Chapter Seven - Moderation and Mediation Analyses | |\n| Section 7.1: Mediation and Moderation Models | 64 |\n| Section 7.2: Mediation Assumptions, The PROCESS Macro, Interpretation, and Write Up | 66 |\n| Section 7.3: Moderation Models, Assumptions, Interpretation, and Write Up | 69 |\n| Section 7.4: Chapter Seven Self-Test | 73 |\n| Part VIII. Chapter Eight - Factor Analysis and Scale Reliability | |\n| Section 8.1: Factor Analysis Definitions | 75 |\n| Section 8.2: EFA versus CFA | 76 |\n| Section 8.3: EFA Steps with Factor Extraction | 78 |\n| Section 8.4: EFA Determining the Number of Factors | 80 |\n| Section 8.5: EFA Interpretation | 84 |\n| Section 8.6: EFA Write Up | 86 |\n| Section 8.7: Scale Reliability | 87 |\n| Section 8.8: Chapter Eight Self-Test | 89 |\n| Part IX. Chapter Nine - Nonparametric Statistics | |\n| Section 9.1: Nonparametric Definitions | 91 |\n| Section 9.2: Choosing Appropriate Tests | 93 |\n| Section 9.3: Comparing Two Independent Conditions: The Mann- Whitney U Test | 94 |\n| Section 9.4: Comparing Two Dependent Conditions or Paired Samples - Wilcoxon Sign-Rank Test | 96 |\n| Section 9.5: Differences Between Several Independent Groups: The Kruskal-Wallis Test | 98 |\n| Section 9.6: Chapter Nine Self-Test | 100 |\n\nReferences\n\n101", + "recall": 0.9932432432432432, + "true_md": "## Part V. Chapter Five - Comparing Associations Between Multiple Variables \n\n| Part V. Chapter Five - Comparing Associations Between Multiple | Variables |\n|---------------------------------------------------------------------------------------------|-------------|\n| Section 5.1: The Linear Model | 35 |\n| Section 5.2: Simple Regression Assumptions, Interpretation, and Write Up | 36 |\n| Section 5.3: Multiple Regression Explanation, Assumptions, Interpretation, and Write Up | 39 |\n| Section 5.4: Hierarchical Regression Explanation, Assumptions, Interpretation, and Write Up | 43 |\n| Section 5.5: Chapter Five Self-Test | 47 |\n| Part VI. Chapter Six - Comparing Three or More Group | Means |\n| Section 6.1: Between Versus Within Group Analyses | 49 |\n| Section 6.2: One-Way ANOVA Assumptions, Interpretation, and Write Up | 51 |\n| Section 6.3 Repeated Measures ANOVA Assumptions, Interpretation, and Write Up | 54 |\n| Section 6.4: Chapter Six Self-Test | 62 |\n| Part VII. Chapter Seven - Moderation and Mediation | Analyses |\n| Section 7.1: Mediation and Moderation Models | 64 |\n| Section 7.2: Mediation Assumptions, The PROCESS Macro, Interpretation, and Write Up | 66 |\n| Section 7.3: Moderation Models, Assumptions, Interpretation, and Write Up | 69 |\n| Section 7.4: Chapter Seven Self-Test | 73 |\n| Part VIII. Chapter Eight - Factor Analysis and Scale | Reliability |\n| Section 8.1: Factor Analysis Definitions | 75 |\n| Section 8.2: EFA versus CFA | 76 |\n| Section 8.3: EFA Steps with Factor Extraction | 78 |\n| Section 8.4: EFA Determining the Number of Factors | 80 |\n| Section 8.5: EFA Interpretation | 84 |\n| Section 8.6: EFA Write Up | 86 |\n| Section 8.7: Scale Reliability | 87 |\n| Section 8.8: Chapter Eight Self-Test | 89 |\n| Part IX. Chapter Nine - Nonparametric | Statistics |\n| Section 9.1: Nonparametric Definitions | 91 |\n| Section 9.2: Choosing Appropriate Tests | 93 |\n| Section 9.3: Comparing Two Independent Conditions: The Mann-Whitney U Test | 94 |\n| Section 9.4: Comparing Two Dependent Conditions or Paired Samples - Wilcoxon Sign-Rank Test | 96 |\n| Section 9.5: Differences Between Several Independent Groups: The Kruskal-Wallis Test | 98 |\n| Section 9.6: Chapter Nine Self-Test | 100 |\n| References | 101 |" + }, + { + "bleu": 0.981980651635335, + "doc_id": "doc_85ce99d1f5e9e0661f3cda228e68c36ac7b735b3c78caef8165c906074c6e7ab_page_000001.png", + "edit_distance": 0.01020408163265306, + "f1_score": 1.0, + "meteor": 0.9907984645349123, + "precision": 1.0, + "pred_md": "## Humanity's Home Base.\n\nFigure 1. This image shows the Western hemisphere as viewed from space 35,400 kilometers (about 22,000 miles) above Earth. Data about the land surface from one satellite was combined with another satellite's data about the clouds to create the image. (credit: modification of work by R. Stockli, A. Nelson, F. Hasler,\n\n## NASA/ GSFC/ NOAA/ USGS)\n\nOur nearest astronomical neighbor is Earth's satellite, commonly called the Moon . Figure 2 shows Earth and the Moon drawn to scale on the same diagram. Notice how small we have to make these bodies to fit them on the page with the right scale. The Moon's distance from Earth is about 30 times Earth's diameter, or approximately 384,000 kilometers, and it takes about a month for the Moon to revolve around Earth. The Moon's diameter is 3476 kilometers, about one fourth the size of Earth.\n\nEarth and Moon, Drawn to Scale.\n\n10 | Chapter 1 Section 1.6: A Tour of the Universe", + "recall": 1.0, + "true_md": "## Humanity's Home Base.\n\nFigure 1. This image shows the Western hemisphere as viewed from space 35,400 kilometers (about 22,000 miles) above Earth. Data about the land surface from one satellite was combined with another satellite's data about the clouds to create the image. (credit: modification of work by R. Stockli, A. Nelson, F. Hasler,\n\n## NASA/ GSFC/ NOAA/ USGS)\n\nOur nearest astronomical neighbor is Earth's satellite, commonly called the Moon. Figure 2 shows Earth and the Moon drawn to scale on the same diagram. Notice how small we have to make these bodies to fit them on the page with the right scale. The Moon's distance from Earth is about 30 times Earth's diameter, or approximately 384,000 kilometers, and it takes about a month for the Moon to revolve around Earth. The Moon's diameter is 3476 kilometers, about one fourth the size of Earth.\n\n## Earth and Moon, Drawn to Scale.\n\n10 | Chapter 1 Section 1.6: A Tour of the Universe" + }, + { + "bleu": 0.9686907986501687, + "doc_id": "doc_caa98180e7cec3f27dfa28e4d67a66748ba4921bd3e2cc6d3cf5ceab4e765ec1_page_000001.png", + "edit_distance": 0.026041666666666668, + "f1_score": 0.9958847736625516, + "meteor": 0.9887109721540848, + "precision": 0.9918032786885246, + "pred_md": "## Tycho Brahe's Observatory\n\nThree years after the publication of Copernicus' De Revolutionibus , Tycho Brahe was born to a family of Danish nobility. He developed an early interest in astronomy and, as a young man, made significant astronomical observations. Among these was a careful study of what we now know was an exploding star that flared up to great brilliance in the night sky. His growing reputation gained him the patronage of the Danish King Frederick II, and at the age of 30, Brahe was able to establish a fine astronomical observatory on the North Sea island of Hven (Figure 1). Brahe was the last and greatest of the pre-telescopic observers in Europe.\n\nTycho Brahe (1546-1601) and Johannes Kepler (1571-1630).\n\n(b)\n\nFigure 1 . (a) A stylized engraving shows Tycho Brahe using his instruments to measure the altitude of celestial objects above the horizon. The large curved instrument in the foreground allowed\n\nChapter 3 Orbits and Gravity Section 3.1: The Laws of Planetary Motion | 99", + "recall": 1.0, + "true_md": "## Tycho Brahe's Observatory\n\nThree years after the publication of Copernicus' De Revolutionibus, Tycho Brahe was born to a family of Danish nobility. He developed an early interest in astronomy and, as a young man, made significant astronomical observations. Among these was a careful study of what we now know was an exploding star that flared up to great brilliance in the night sky. His growing reputation gained him the patronage of the Danish King Frederick II, and at the age of 30, Brahe was able to establish a fine astronomical observatory on the North Sea island of Hven (Figure 1). Brahe was the last and greatest of the pre-telescopic observers in Europe.\n\n## Tycho Brahe (1546-1601) and Johannes Kepler (1571-1630).\n\nFigure 1. (a) A stylized engraving shows Tycho Brahe using his instruments to measure the altitude of celestial objects above the horizon. The large curved instrument in the foreground allowed\n\nChapter 3 Orbits and Gravity Section 3.1: The Laws of Planetary Motion | 99" + }, + { + "bleu": 1.0, + "doc_id": "doc_94ba5468fcb6277721947697048846dc0d0551296be3b45f5918ab857d21dcc7_page_000001.png", + "edit_distance": 0.0, + "f1_score": 1.0, + "meteor": 0.9999999801222537, + "precision": 1.0, + "pred_md": "radiation at other wavelengths, as shown in (Figure 1). Just as you can catch more rain with a garbage can than with a coffee cup, large telescopes gather much more light than your eye can. Second, there is an instrument attached to the telescope that sorts the incoming radiation by wavelength. Sometimes the sorting is fairly crude. For example, we might simply want to separate blue light from red light so that we can determine the temperature of a star. But at other times, we want to see individual spectral lines to determine what an object is made of, or to measure its speed (as explained in the Radiation and Spectra chapter). Third, we need some type of detector , a device that senses the radiation in the wavelength regions we have chosen and permanently records the observations.\n\n## Orion Region at Different Wavelengths.\n\nFigure 1. The same part of the sky looks different when observed with instruments that are sensitive to different bands of the spectrum. (a) Visible light: this shows part of the Orion region as the human eye sees it, with dotted lines added to show the figure of the mythical hunter, Orion. (b) X-rays: here, the view emphasizes the point-like X-ray sources nearby. The colors are artificial, changing from yellow to white to blue with increasing energy of the X-rays. The bright, hot stars in Orion are still seen in this image, but so are many other objects located at very different\n\n276 | Chapter 6 Astronomical Instruments Section 6.1: Telescopes", + "recall": 1.0, + "true_md": "radiation at other wavelengths, as shown in (Figure 1). Just as you can catch more rain with a garbage can than with a coffee cup, large telescopes gather much more light than your eye can. Second, there is an instrument attached to the telescope that sorts the incoming radiation by wavelength. Sometimes the sorting is fairly crude. For example, we might simply want to separate blue light from red light so that we can determine the temperature of a star. But at other times, we want to see individual spectral lines to determine what an object is made of, or to measure its speed (as explained in the Radiation and Spectra chapter). Third, we need some type of detector, a device that senses the radiation in the wavelength regions we have chosen and permanently records the observations.\n\n## Orion Region at Different Wavelengths.\n\nFigure 1. The same part of the sky looks different when observed with instruments that are sensitive to different bands of the spectrum. (a) Visible light: this shows part of the Orion region as the human eye sees it, with dotted lines added to show the figure of the mythical hunter, Orion. (b) X-rays: here, the view emphasizes the point-like X-ray sources nearby. The colors are artificial, changing from yellow to white to blue with increasing energy of the X-rays. The bright, hot stars in Orion are still seen in this image, but so are many other objects located at very different\n\n276 | Chapter 6 Astronomical Instruments Section 6.1: Telescopes" + }, + { + "bleu": 0.9820454470830212, + "doc_id": "doc_3b10e813590fb9c9f54883151a2dd4b4ccb407becc54fd516a82972dc261630b_page_000001.png", + "edit_distance": 0.014492753623188406, + "f1_score": 0.9910447761194029, + "meteor": 0.9927529716746932, + "precision": 0.9880952380952381, + "pred_md": "vapor and other gases, making it useless. Only in the vacuum of space can optical elements be cooled to hundreds of degrees below freezing and still remain operational.\n\nThe first orbiting infrared observatory, launched in 1983, was the Infrared Astronomical Satellite (IRAS), built as a joint project by the United States, the Netherlands, and Britain. IRAS was equipped with a 0.6-meter telescope cooled to a temperature of less than 10 K. For the first time, the infrared sky could be seen as if it were night, rather than through a bright foreground of atmospheric and telescope emissions. IRAS carried out a rapid but comprehensive survey of the entire infrared sky over a 10-month period, cataloging about 350,000 sources of infrared radiation. Since then, several other infrared telescopes have operated in space with much better sensitivity and resolution due to improvements in infrared detectors. The most powerful of these infrared telescopes is the 0.85-meter Spitzer Space Telescope, which launched in 2003. A few of its observations are shown in Figure 2. With infrared observations, astronomers can detect cooler parts of cosmic objects, such as the dust clouds around star nurseries and the remnants of dying stars, that visible-light images don't reveal.\n\nObservations from the Spitzer Space Telescope (SST).\n\nFlame nebula\n\nFigure 2. These infrared images-a region of star formation, the remnant of an exploded star, and a region where an old star is\n\n336 | Chapter 6 Section 6.5: Observations outside Earth's Atmosphere", + "recall": 0.9940119760479041, + "true_md": "vapor and other gases, making it useless. Only in the vacuum of space can optical elements be cooled to hundreds of degrees below freezing and still remain operational.\n\nThe first orbiting infrared observatory, launched in 1983, was the Infrared Astronomical Satellite (IRAS), built as a joint project by the United States, the Netherlands, and Britain. IRAS was equipped with a 0.6-meter telescope cooled to a temperature of less than 10 K. For the first time, the infrared sky could be seen as if it were night, rather than through a bright foreground of atmospheric and telescope emissions. IRAS carried out a rapid but comprehensive survey of the entire infrared sky over a 10-month period, cataloging about 350,000 sources of infrared radiation. Since then, several other infrared telescopes have operated in space with much better sensitivity and resolution due to improvements in infrared detectors. The most powerful of these infrared telescopes is the 0.85-meter Spitzer Space Telescope, which launched in 2003. A few of its observations are shown in Figure 2. With infrared observations, astronomers can detect cooler parts of cosmic objects, such as the dust clouds around star nurseries and the remnants of dying stars, that visible-light images don't reveal.\n\n## Observations from the Spitzer Space Telescope (SST).\n\nFigure 2. These infrared images-a region of star formation, the remnant of an exploded star, and a region where an old star is\n\n336 | Chapter 6 Section 6.5: Observations outside Earth's Atmosphere" + }, + { + "bleu": 0.9905864253524023, + "doc_id": "doc_fbee4ae063533d20dc7339a3ffa2412d58504af06420fe15054657c87ed81a79_page_000001.png", + "edit_distance": 0.13526570048309178, + "f1_score": 1.0, + "meteor": 0.999549028647166, + "precision": 1.0, + "pred_md": "Figure 7.3. You can read more about KSU's marketing approach in Marking Open and Affordable Courses (Hare, Kirschner, and Reed 2020).\n\nFor an even simpler graphic, we can look to Kansas State University. KSU's Open/Alternative Textbook Initiative developed their OER icon, a book with an 'O' on the cover, to be recognizable even at a small scale. This was done because it would be used as a marking denoting the use of open materials in their course schedule. This graphic is clear, easy to read, and emblematic of the initiative itself, by representing open textbooks with a book icon.\n\n## Aligning with Your Identity\n\nLike KSU did with their OER icon, your branding should be reflective of your initiative's work in some way. Think about your audience and what you want them to feel when they see your program's marketing on campus. Does your program have a unique name or tagline that influences the way you present it (e.g., playful, bold, colorful, or innovative)?\n\nA great example of a program whose name and messaging align clearly with their work is Central Virginia Community College (CVCC). CVCC uses the tagline 'OpenEd CVCC: Innovation and Affordability' as their program's name and their icon features this theme of innovation through graphics of light bulbs, gears, and representations of various disciplines.\n\nFigure 7.4. You can read more about CVCC's marketing approach in Marking Open and Affordable Courses (Hare, Kirschner, and Reed 2020).\n\nCVCC's logo is more complex than the ones we shared in our 'simple' section. However, this isn't a problem in their case. Keep in mind that the simplicity of any graphic will depend on where and how it's used. CVCC's logo might have more going on than KSU's icon, but it is meant to be used at a larger scale, so it can accommodate this complexity. If your logo will be used in print materials or as a smaller icon, that's when you'll want to focus on simpler designs. For graphics that will be displayed more prominently, though, a larger graphic works fine.\n\n90 | PROGRAM MANAGEMENT", + "recall": 1.0, + "true_md": "Figure 7.3. You can read more about KSU's marketing approach in Marking Open and Affordable Courses (Hare, Kirschner, and Reed 2020). \n\nFor an even simpler graphic, we can look to Kansas State University. KSU's Open/Alternative Textbook Initiative developed their OER icon, a book with an 'O' on the cover, to be recognizable even at a small scale. This was done because it would be used as a marking denoting the use of open materials in their course schedule. This graphic is clear, easy to read, and emblematic of the initiative itself, by representing open textbooks with a book icon. \n\n## Aligning with Your Identity \n\nLike KSU did with their OER icon, your branding should be reflective of your initiative's work in some way. Think about your audience and what you want them to feel when they see your program's marketing on campus. Does your program have a unique name or tagline that influences the way you present it (e.g., playful, bold, colorful, or innovative)? \n\nFigure 7.4. You can read more about CVCC's marketing approach in Marking Open and Affordable Courses (Hare, Kirschner, and Reed 2020). \n\nA great example of a program whose name and messaging align clearly with their work is Central Virginia Community College (CVCC). CVCC uses the tagline 'OpenEd CVCC: Innovation and Affordability' as their program's name and their icon features this theme of innovation through graphics of light bulbs, gears, and representations of various disciplines. \n\nCVCC's logo is more complex than the ones we shared in our 'simple' section. However, this isn't a problem in their case. Keep in mind that the simplicity of any graphic will depend on where and how it's used. CVCC's logo might have more going on than KSU's icon, but it is meant to be used at a larger scale, so it can accommodate this complexity. If your logo will be used in print materials or as a smaller icon, that's when you'll want to focus on simpler designs. For graphics that will be displayed more prominently, though, a larger graphic works fine. \n\n90 | PROGRAM MANAGEMENT" + }, + { + "bleu": 1.0, + "doc_id": "doc_62c4b0340519f5963a37534d3abeb0546766cb53bc90504f663eb7cc321a6a6d_page_000001.png", + "edit_distance": 0.0, + "f1_score": 1.0, + "meteor": 0.9999999894599106, + "precision": 1.0, + "pred_md": "## Promotional Materials\n\nA good promotional strategy should include multiple facets, from physical materials to digital communications. Below, we've compiled a table of promotional materials you might use on campus, and examples of each type.\n\nTable 7.1. Types of promotional materials\n\nGet in contact with partners at your institution to learn more about the processes and options available to you and how you can best leverage the support at your disposal. If you have a marketing team available to you that orders pens and other materials for campus events, get in contact with them about their vendors and how you can leverage their existing workflows for ordering materials to support your OER Program. This might be as simple as ordering buttons and posters through your University Printing Office, or it may require you to browse a third party's marketing catalog or to create materials yourself, if you lack funding for your work.\n\n## Annual Events\n\nCreating promotional materials and graphics can make your OER program recognizable on your college's campus, but just because you've created materials doesn't mean that people will find or learn from them. As a program manager, you will need to find ways to implement your messaging and events on campus. Leveraging annual events like Open Education Week in March and International Open Access Week in October can ground your work in a given time of year and focus your programming around a topic or theme (Open Education Global, n.d.; SPARC, n.d.). The Open Education Week website lists past events and provides downloadable promotional materials to help you kickstart your event planning and coordination. If these weeks regularly conflict with other events at your institution, that's okay. You can celebrate Open Education Week the week before or after it falls. So long as you are consistent in the general time you hold these events, they will still gain recognition at your institution and faculty will come to expect them.\n\n92 | PROGRAM MANAGEMENT", + "recall": 1.0, + "true_md": "## Promotional Materials \n\nA good promotional strategy should include multiple facets, from physical materials to digital communications. Below, we've compiled a table of promotional materials you might use on campus, and examples of each type. \n\nTable 7.1. Types of promotional materials \n\nGet in contact with partners at your institution to learn more about the processes and options available to you and how you can best leverage the support at your disposal. If you have a marketing team available to you that orders pens and other materials for campus events, get in contact with them about their vendors and how you can leverage their existing workflows for ordering materials to support your OER Program. This might be as simple as ordering buttons and posters through your University Printing Office, or it may require you to browse a third party's marketing catalog or to create materials yourself, if you lack funding for your work. \n\n## Annual Events \n\nCreating promotional materials and graphics can make your OER program recognizable on your college's campus, but just because you've created materials doesn't mean that people will find or learn from them. As a program manager, you will need to find ways to implement your messaging and events on campus. Leveraging annual events like Open Education Week in March and International Open Access Week in October can ground your work in a given time of year and focus your programming around a topic or theme (Open Education Global, n.d.; SPARC, n.d.). The Open Education Week website lists past events and provides downloadable promotional materials to help you kickstart your event planning and coordination. If these weeks regularly conflict with other events at your institution, that's okay. You can celebrate Open Education Week the week before or after it falls. So long as you are consistent in the general time you hold these events, they will still gain recognition at your institution and faculty will come to expect them. \n\n92 | PROGRAM MANAGEMENT" + }, + { + "bleu": 1.0, + "doc_id": "doc_ee9af89ec8d82b15d0fdff8908560e597e71cd4662d28472bc1a4d485d6c6712_page_000001.png", + "edit_distance": 0.0, + "f1_score": 1.0, + "meteor": 0.9999999599588955, + "precision": 1.0, + "pred_md": "Figure 12.2. A set of open textbooks printed in bulk are featured in this photo. Open textbooks from the Open Course Library, picture by Tom Caswell, CC BY 2.0.\n\n## What tool(s) do you typically use in your course?\n\nAsk whether the instructor utilizes your institution's course management system (Canvas, Blackboard, etc.), or a separate course website to communicate and share content with students. This may affect the tools and practices you recommend.\n\n## What supporting materials do you utilize for this course?\n\nIf the instructor relies on self-grading homework platforms or ancillary presentations and lecture notes from publishers, you will want to discuss the various free and low-cost options available to replace that content (See Chapter 15, Finding Ancillaries for OER).\n\nAlternatively, does the instructor already supplement their course materials with course notes or materials they have personally created? Often, when traditional materials are lacking or require supplement, instructors will create notes, reading lists, or other content to 'back up' any traditional, commercial content used in their course. This instructor-created content can be reused with OER as well, or even adapted into a new open resource in the future.\n\n164 | SUPPORTING OER ADOPTION", + "recall": 1.0, + "true_md": "Figure 12.2. A set of open textbooks printed in bulk are featured in this photo. Open textbooks from the Open Course Library, picture by Tom Caswell, CC BY 2.0. \n\n## What tool(s) do you typically use in your course? \n\nAsk whether the instructor utilizes your institution's course management system (Canvas, Blackboard, etc.), or a separate course website to communicate and share content with students. This may affect the tools and practices you recommend. \n\n## What supporting materials do you utilize for this course? \n\nIf the instructor relies on self-grading homework platforms or ancillary presentations and lecture notes from publishers, you will want to discuss the various free and low-cost options available to replace that content (See Chapter 15, Finding Ancillaries for OER). \n\nAlternatively, does the instructor already supplement their course materials with course notes or materials they have personally created? Often, when traditional materials are lacking or require supplement, instructors will create notes, reading lists, or other content to 'back up' any traditional, commercial content used in their course. This instructor-created content can be reused with OER as well, or even adapted into a new open resource in the future. \n\n164 | SUPPORTING OER ADOPTION" + }, + { + "bleu": 0.9798276558958281, + "doc_id": "doc_4b5f23f97fd273cacb8de95fc1799ed4dc4d5c15e9ed86d707cdf4be53eb7c7c_page_000001.png", + "edit_distance": 0.012269938650306749, + "f1_score": 1.0, + "meteor": 0.9987443475657043, + "precision": 1.0, + "pred_md": "## Version History\n\nThis page provides a record of edits and changes made to this book since its initial publication. Whenever edits or updates are made in the text, we provide a record and description of those changes here. If the change is minor, the version number increases by 0.1. If the edits involve substantial updates, the edition number increases to the next whole number.\n\nThe files posted alongside this book always reflect the most recent version. If you find an error in this book, please let us know in the Rebus Community forum, where reported errors will be visible to others.\n\nWe will contact the author, make the necessary changes, and replace all file types as soon as possible. Once we receive the updated files, this Version History page will be updated to reflect the edits made.\n\n## Version History\n\n## Version History", + "recall": 1.0, + "true_md": "## Version History \n\nThis page provides a record of edits and changes made to this book since its initial publication. Whenever edits or updates are made in the text, we provide a record and description of those changes here. If the change is minor, the version number increases by 0.1. If the edits involve substantial updates, the edition number increases to the next whole number. \n\nThe files posted alongside this book always reflect the most recent version. If you find an error in this book, please let us know in the Rebus Community forum, where reported errors will be visible to others. \n\nWe will contact the author, make the necessary changes, and replace all file types as soon as possible. Once we receive the updated files, this Version History page will be updated to reflect the edits made. \n\n## Version History \n\nVersion History " + }, + { + "bleu": 0.7525725691178221, + "doc_id": "doc_92ba5d5f6f6e627a3e9ddc00bcdd31b9c2c047e0461fff6c8fa08894a68b72da_page_000001.png", + "edit_distance": 0.18604651162790697, + "f1_score": 0.9824561403508772, + "meteor": 0.9752654243947515, + "precision": 0.9655172413793104, + "pred_md": "Upstage aims to enrich your business by providing Easy-to-Apply AI solutions\n\n## Our Purpose\n\n## Our Mission\n\n## Making AI Beneficial\n\nEasy-to-apply AI, Everywhere\n\n## What We Do\n\n## Providing the world's best and easy-to-use AI solutions for everyone\n\n- • Plug-and-play to cross/multi-cloud system\n- • Ensuring performance tailored to customer data via retraining\n- · Providing a platform that allows easy distribution and management of AI solutions\n- • AI consulting service to help AI transformation\n\n3", + "recall": 1.0, + "true_md": "## Upstage aims to enrich your business by providing Easy-to-Apply AI solutions\n\n## Our Purpose\n\nMaking AI Beneficial\n\n## Our Mission\n\nEasy-to-apply AI, Everywhere\n\n## What We Do\n\nProviding the world's best and easy-to-use AI solutions for everyone\n\n• Plug-and-play to cross/multi-cloud system\n\n• Ensuring performance tailored to customer data via retraining\n\n• Providing a platform that allows easy distribution and management of AI solutions\n\n• AI consulting service to help AI transformation\n\n3" + }, + { + "bleu": 0.6893409630302637, + "doc_id": "doc_c31d842c665085cb1379d441ef0b65869f29537b9252956d6105e6b4cc972311_page_000001.png", + "edit_distance": 0.2727272727272727, + "f1_score": 0.8979591836734693, + "meteor": 0.9616242748772871, + "precision": 0.8148148148148148, + "pred_md": "## AI Pack\n\n## Upstage offers 3 AI packs that process unstructured information and data, making a tangible impact on your business\n\n## OCR\n\nProduct semantic search\n\nRecommendation\n\n11", + "recall": 1.0, + "true_md": "AI Pack\n\n## Upstage offers 3 AI packs that process unstructured information and data, making a tangible impact on your business\n\n11" + }, + { + "bleu": 0.8941136542390052, + "doc_id": "doc_cbb4a13ffd01d9f777fdb939451d6a21cea1b869ee50d79581451e3601df9ec8_page_000001.png", + "edit_distance": 0.21333333333333335, + "f1_score": 1.0, + "meteor": 0.9673321639339987, + "precision": 1.0, + "pred_md": "Recommendation Pack: Track Record\n\n## Recommendation pack shows outstanding performance of 1.7~2.6 times that of competing models even when using commercial service data\n\n## Comparison with Beauty Commerce Recommendation Models\n\n## Education Content Platform PoC Case\n\nRecommendation model Hit Ratio comparison\n\nComparison Case of Domestic Subscription Platform Recommendation Model Comparison of quantitative evaluations among personalized content recommendations\n\nComparison of prediction rates of correct/incorrect answers based on personalized questions\n\n20", + "recall": 1.0, + "true_md": "Recommendation Pack: Track Record\n\n## Recommendation pack shows outstanding performance of 1.7~2.6 times that of competing models even when using commercial service data\n\n## Comparison with Beauty Commerce Recommendation Models\n\nRecommendation model Hit Ratio comparison\n\n## Comparison Case of Domestic Subscription Platform Recommendation Model\n\nComparison of quantitative evaluations among personalized content recommendations\n\n## Education Content Platform PoC Case\n\nComparison of prediction rates of correct/incorrect answers based on personalized questions\n\n20" + }, + { + "bleu": 0.6931692668621272, + "doc_id": "doc_a7914baa31ab36088e7d4199120e23630c800e99d8f96aa5a21091d7a67ff1cf_page_000001.png", + "edit_distance": 0.38164251207729466, + "f1_score": 0.8768472906403941, + "meteor": 0.8531116853176899, + "precision": 0.8018018018018018, + "pred_md": "## Semantic Search Pack: Value\n\n## SS Pack allows businesses to access further data more rapidly\n\nThe SS Pack can reduce the information acquisition time by returning all the information that matches the user's search intent.\n\nThe performance optimized for individual search systems is maintained by automatic updates of real-time search log records, augmented by Upstage's technological know-how.\n\n## Optimal Attempt\n\n## Higher Return of Information\n\n## Cutting-Edge Technology\n\n## Reduced Information Acquisition Time\n\nUnlike existing search systems that only return information limited to the entered search keywords, SS Pack returns all relevant data that meet the user's search intent\n\nThe analysis of user logs saved in real-time allows us to further optimize the individual search services over time\n\nBy returning all semantic-based information of the search keywords, the time required for information acquisition is reduced drastically compared to that of traditional keyword-matching search systems\n\n1 Evaluated against 100 internal test queries. Comparison of the amount of information returned with at least one keyword included in the search term and the amount of returned information against that of SS Pack\n\n2 State-of-the-art, current highest level of results and performance\n\n22", + "recall": 0.967391304347826, + "true_md": "Semantic Search Pack: Value\n\n## SS Pack allows businesses to access further data more rapidly\n\nThe SS Pack can reduce the information acquisition time by returning all the information that matches the user's search intent.\n\nThe performance optimized for individual search systems is maintained by automatic updates of real-time search log records, augmented by Upstage's technological know-how.\n\n## 1.8X ↑ 1 Higher Return of Information\n\nUnlike existing search systems that only return information limited to the entered search keywords, SS Pack returns all relevant data that meet the user's search intent\n\n## Optimal Attempt Reduced Information Acquisition Time\n\nBy returning all semantic-based information of the search keywords, the time required for information acquisition is reduced drastically compared to that of traditional keyword-matching search systems\n\n## SOTA 2 Cutting-Edge Technology\n\nThe analysis of user logs saved in real-time allows us to further optimize the individual search services over time\n\n22" + }, + { + "bleu": 0.8913297116886038, + "doc_id": "doc_b639c25ab157f6f4ecbdb7cd31724bf4c8869f03134f5e606c49dd5fbee2bb54_page_000001.png", + "edit_distance": 0.2459425717852684, + "f1_score": 0.9266187050359712, + "meteor": 0.9375822074137942, + "precision": 0.9554896142433235, + "pred_md": "arXiv:2312.15166v2 [cs.CL] 29 Dec 2023\n\n## SOLAR 10.7B: Scaling Large Language Models with Simple yet Effective Depth Up-Scaling\n\nDahyun Kim , Chanjun Park ∗ ∗† , Sanghoon Kim ∗† , Wonsung Lee ∗† , Wonho Song Yunsu Kim, Hyeonwoo Kim, Yungi Kim, Hyeonju Lee, Jihoo Kim Changbae Ahn, Seonghoon Yang, Sukyung Lee, Hyunbyung Park, Gyoungjin Gim Mikyoung Cha, Hwalsuk Lee , Sunghun Kim † †\n\nUpstage AI, South Korea\n\n{kdahyun, chanjun.park,limerobot, wonsung.lee, hwalsuk.lee, hunkim}@upstage.ai\n\n## Abstract\n\nWe introduce SOLAR 10.7B, a large language model (LLM) with 10.7 billion parameters, demonstrating superior performance in various natural language processing (NLP) tasks. Inspired by recent efforts to efficiently up-scale LLMs, we present a method for scaling LLMs called depth up-scaling (DUS), which encompasses depthwise scaling and continued pretraining. In contrast to other LLM up-scaling methods that use mixture-of-experts, DUS does not require complex changes to train and inference efficiently. We show experimentally that DUS is simple yet effective in scaling up highperformance LLMs from small ones. Building on the DUS model, we additionally present SOLAR 10.7B-Instruct, a variant fine-tuned for instruction-following capabilities, surpassing Mixtral-8x7B-Instruct. SOLAR 10.7B is publicly available under the Apache 2.0 license, promoting broad access and application in the LLM field 1 .\n\nciently and effectively scale-up LLMs, they often require non-trivial changes to the training and inference framework (Gale et al., 2023), which hinders widespread applicability. Effectively and efficiently scaling up LLMs whilst also retaining the simplicity for ease of use is an important problem (Alberts et al., 2023; Fraiwan and Khasawneh, 2023; Sallam et al., 2023; Bahrini et al., 2023).\n\n## 1 Introduction\n\nThe field of natural language processing (NLP) has been significantly transformed by the introduction of large language models (LLMs), which have enhanced our understanding and interaction with human language (Zhang et al., 2023a). These advancements bring challenges such as the increased need to train ever larger models (Rae et al., 2021; Wang et al., 2023; Pan et al., 2023; Lian, 2023; Yao et al., 2023; Gesmundo and Maile, 2023) owing to the performance scaling law (Kaplan et al., 2020; Hernandez et al., 2021; Anil et al., 2023; Kaddour et al., 2023). To efficiently tackle the above, recent works in scaling language models such as a mixture of experts (MoE) (Shazeer et al., 2017; Komatsuzaki et al., 2022) have been proposed. While those approaches are able to effi-\n\n∗ Equal Contribution † Corresponding Author\n\n1\n\nhttps://huggingface.co/upstage/\n\nSOLAR-10.7B-v1.0\n\nInspired by Komatsuzaki et al. (2022), we present depth up-scaling (DUS), an effective and efficient method to up-scale LLMs whilst also remaining straightforward to use. DUS consists of scaling the base model along the depth dimension and continually pretraining the scaled model. Unlike (Komatsuzaki et al., 2022), DUS does not scale the model using MoE and rather use a depthwise scaling method analogous to Tan and Le (2019) which is adapted for the LLM architecture. Thus, there are no additional modules or dynamism as with MoE, making DUS immediately compatible with easy-to-use LLM frameworks such as HuggingFace (Wolf et al., 2019) with no changes to the training or inference framework for maximal efficiency. Furthermore, DUS is applicable to all transformer architectures, opening up new gateways to effectively and efficiently scale-up LLMs in a simple manner. Using DUS, we release SOLAR 10.7B, an LLM with 10.7 billion parameters, that outperforms existing models like Llama 2 (Touvron et al., 2023) and Mistral 7B (Jiang et al., 2023) in various benchmarks.\n\nWe have also developed SOLAR 10.7B-Instruct, a variant fine-tuned for tasks requiring strict adherence to complex instructions. It significantly outperforms the Mixtral-8x7B-Instruct model across various evaluation metrics, evidencing an advanced proficiency that exceeds the capabilities of even larger models in terms of benchmark performance.\n\nBy releasing SOLAR 10.7B under the Apache 2.0 license, we aim to promote collaboration and innovation in NLP. This open-source approach allows", + "recall": 0.8994413407821229, + "true_md": "arXiv:2312.15166v2 [cs.CL] 29 Dec 2023\n\n## SOLAR 10.7B: Scaling Large Language Models with Simple yet Effective Depth Up-Scaling\n\nDahyun Kim ∗ , Chanjun Park ∗† , Sanghoon Kim ∗† , Wonsung Lee ∗† , Wonho Song Yunsu Kim, Hyeonwoo Kim, Yungi Kim, Hyeonju Lee, Jihoo Kim Changbae Ahn, Seonghoon Yang, Sukyung Lee, Hyunbyung Park, Gyoungjin Gim Mikyoung Cha, Hwalsuk Lee † , Sunghun Kim †\n\nUpstage AI, South Korea\n\n{kdahyun, chanjun.park,limerobot, wonsung.lee, hwalsuk.lee, hunkim}@upstage.ai\n\n## Abstract\n\nWe introduce SOLAR 10.7B, a large language model (LLM) with 10.7 billion parameters, demonstrating superior performance in various natural language processing (NLP) tasks. In- spired by recent efforts to efficiently up-scale LLMs, we present a method for scaling LLMs called depth up-scaling (DUS), which encom- passes depthwise scaling and continued pre- training. In contrast to other LLM up-scaling methods that use mixture-of-experts, DUS does not require complex changes to train and infer- ence efficiently. We show experimentally that DUS is simple yet effective in scaling up high- performance LLMs from small ones. Building on the DUS model, we additionally present SO- LAR 10.7B-Instruct, a variant fine-tuned for instruction-following capabilities, surpassing Mixtral-8x7B-Instruct. SOLAR 10.7B is pub- licly available under the Apache 2.0 license, promoting broad access and application in the LLM field 1 .\n\n## 1 Introduction\n\nThe field of natural language processing (NLP) has been significantly transformed by the introduc- tion of large language models (LLMs), which have enhanced our understanding and interaction with human language (Zhang et al., 2023a). These ad- vancements bring challenges such as the increased need to train ever larger models (Rae et al., 2021; Wang et al., 2023; Pan et al., 2023; Lian, 2023; Yao et al., 2023; Gesmundo and Maile, 2023) ow- ing to the performance scaling law (Kaplan et al., 2020; Hernandez et al., 2021; Anil et al., 2023; Kaddour et al., 2023). To efficiently tackle the above, recent works in scaling language models such as a mixture of experts (MoE) (Shazeer et al., 2017; Komatsuzaki et al., 2022) have been pro- posed. While those approaches are able to effi- ciently and effectively scale-up LLMs, they often require non-trivial changes to the training and infer- ence framework (Gale et al., 2023), which hinders widespread applicability. Effectively and efficiently scaling up LLMs whilst also retaining the simplic- ity for ease of use is an important problem (Alberts et al., 2023; Fraiwan and Khasawneh, 2023; Sallam et al., 2023; Bahrini et al., 2023).\n\n∗ Equal Contribution † Corresponding Author\n\nhttps://huggingface.co/upstage/ SOLAR-10.7B-v1.0\n\nInspired by Komatsuzaki et al. (2022), we present depth up-scaling (DUS), an effective and efficient method to up-scale LLMs whilst also re- maining straightforward to use. DUS consists of scaling the base model along the depth dimension and continually pretraining the scaled model. Un- like (Komatsuzaki et al., 2022), DUS does not scale the model using MoE and rather use a depthwise scaling method analogous to Tan and Le (2019) which is adapted for the LLM architecture. Thus, there are no additional modules or dynamism as with MoE, making DUS immediately compatible with easy-to-use LLM frameworks such as Hug- gingFace (Wolf et al., 2019) with no changes to the training or inference framework for maximal efficiency. Furthermore, DUS is applicable to all transformer architectures, opening up new gate- ways to effectively and efficiently scale-up LLMs in a simple manner. Using DUS, we release SO- LAR 10.7B, an LLM with 10.7 billion parameters, that outperforms existing models like Llama 2 (Tou- vron et al., 2023) and Mistral 7B (Jiang et al., 2023) in various benchmarks.\n\nWe have also developed SOLAR 10.7B-Instruct, a variant fine-tuned for tasks requiring strict adher- ence to complex instructions. It significantly out- performs the Mixtral-8x7B-Instruct model across various evaluation metrics, evidencing an advanced proficiency that exceeds the capabilities of even larger models in terms of benchmark performance.\n\nBy releasing SOLAR 10.7B under the Apache 2.0 license, we aim to promote collaboration and in- novation in NLP. This open-source approach allows" + }, + { + "bleu": 0.935445450908611, + "doc_id": "doc_22d276b33b49ec941acbc323e24fa57ed07579760f472c136f226d755a0c3b13_page_000001.png", + "edit_distance": 0.05, + "f1_score": 0.9400749063670413, + "meteor": 0.962502564064584, + "precision": 0.9691119691119691, + "pred_md": "Figure 1: Depth up-scaling for the case with n = 32 , s = 48 , and m = 8 . Depth up-scaling is achieved through a dual-stage process of depthwise scaling followed by continued pretraining.\n\nfor wider access and application of these models by researchers and developers globally.\n\n## 2 Depth Up-Scaling\n\nTo efficiently scale-up LLMs, we aim to utilize pretrained weights of base models to scale up to larger LLMs (Komatsuzaki et al., 2022). While existing methods such as Komatsuzaki et al. (2022) use MoE(Shazeer et al., 2017) to scale-up the model architecture, we opt for a different depthwise scaling strategy inspired by Tan and Le (2019). We then continually pretrain the scaled model as just scaling the model without further pretraining degrades the performance.\n\nBase model. Any n -layer transformer architecture can be used but we select the 32-layer Llama 2 architecture as our base model. We initialize the Llama 2 architecture with pretrained weights from Mistral 7B, as it is one of the top performers compatible with the Llama 2 architecture. By adopting the Llama 2 architecture for our base model, we aim to leverage the vast pool of community resources while introducing novel modifications to further enhance its capabilities.\n\nDepthwise scaling. From the base model with n layers, we set the target layer count s for the scaled model, which is largely dictated by the available hardware.\n\nWith the above, the depthwise scaling process is as follows. The base model with n layers is duplicated for subsequent modification. Then, we remove the final m layers from the original model and the initial m layers from its duplicate, thus forming two distinct models with n -m layers. These two models are concatenated to form a scaled model with s = 2 ( · n -m ) layers. Note that n = 32 from our base model and we set s = 48 considering our hardware constraints and the efficiency of the scaled model, i.e., fitting between 7 and 13 billion parameters. Naturally, this leads to the removal of m = 8 layers. The depthwise scaling process with n = 32 , s = 48 , and m = 8 is depicted in 'Step 1: Depthwise Scaling' of Fig. 1.\n\nWenote that a method in the community that also scale the model in the same manner 2 as 'Step 1: Depthwise Scaling' of Fig. 1 has been concurrently developed.\n\nContinued pretraining. The performance of the depthwise scaled model initially drops below that of the base LLM. Thus, we additionally apply the continued pretraining step as shown in 'Step 2: Continued Pretraining' of Fig. 1. Experimentally, we observe rapid performance recovery of the scaled model during continued pretraining, a phenomenon also observed in Komatsuzaki et al. (2022). We consider that the particular way of depthwise scaling has isolated the heterogeneity in the scaled model which allowed for this fast performance recovery.\n\nDelving deeper into the heterogeneity of the scaled model, a simpler alternative to depthwise scaling could be to just repeat its layers once more, i.e., from n to 2 n layers. Then, the 'layer distance', or the difference in the layer indices in the base model, is only bigger than 1 where layers n and n +1 are connected, i.e., at the seam.\n\nHowever, this results in maximum layer distance at the seam, which may be too significant of a discrepancy for continued pretraining to quickly resolve. Instead, depthwise scaling sacrifices the 2 m middle layers, thereby reducing the discrepancy at the seam and making it easier for continued\n\n2 https://huggingface.co/Undi95/ Mistral-11B-v0.1", + "recall": 0.9127272727272727, + "true_md": "Figure 1: Depth up-scaling for the case with n = 32, s = 48, and m = 8. Depth up-scaling is achieved through a dual-stage process of depthwise scaling followed by continued pretraining.\n\nfor wider access and application of these models by researchers and developers globally.\n\n## 2 Depth Up-Scaling\n\nTo efficiently scale-up LLMs, we aim to utilize pre- trained weights of base models to scale up to larger LLMs (Komatsuzaki et al., 2022). While exist- ing methods such as Komatsuzaki et al. (2022) use MoE(Shazeer et al., 2017) to scale-up the model ar- chitecture, we opt for a different depthwise scaling strategy inspired by Tan and Le (2019). We then continually pretrain the scaled model as just scaling the model without further pretraining degrades the performance.\n\nBase model. Any n-layer transformer architec- ture can be used but we select the 32-layer Llama 2 architecture as our base model. We initialize the Llama 2 architecture with pretrained weights from Mistral 7B, as it is one of the top performers com- patible with the Llama 2 architecture. By adopting the Llama 2 architecture for our base model, we aim to leverage the vast pool of community re- sources while introducing novel modifications to further enhance its capabilities.\n\nDepthwise scaling. From the base model with n layers, we set the target layer count s for the scaled model, which is largely dictated by the available hardware.\n\nWith the above, the depthwise scaling process is as follows. The base model with n layers is duplicated for subsequent modification. Then, we remove the final m layers from the original model and the initial m layers from its duplicate, thus forming two distinct models with n - m layers. These two models are concatenated to form a scaled model with s = 2·(n-m) layers. Note that n = 32 from our base model and we set s = 48 considering our hardware constraints and the efficiency of the scaled model, i.e., fitting between 7 and 13 billion parameters. Naturally, this leads to the removal of m=8layers. The depthwise scaling process with n = 32, s = 48, and m = 8 is depicted in 'Step 1: Depthwise Scaling' of Fig. 1.\n\nWenote that a method in the community that also scale the model in the same manner 2 as 'Step 1: Depthwise Scaling' of Fig. 1 has been concurrently developed.\n\nContinued pretraining. The performance of the depthwise scaled model initially drops below that of the base LLM. Thus, we additionally apply the continued pretraining step as shown in 'Step 2: Continued Pretraining' of Fig. 1. Experimen- tally, we observe rapid performance recovery of the scaled model during continued pretraining, a phenomenon also observed in Komatsuzaki et al. (2022). We consider that the particular way of depthwise scaling has isolated the heterogeneity in the scaled model which allowed for this fast performance recovery.\n\nDelving deeper into the heterogeneity of the scaled model, a simpler alternative to depthwise scaling could be to just repeat its layers once more, i.e., from n to 2n layers. Then, the 'layer distance', or the difference in the layer indices in the base model, is only bigger than 1 where layers n and n + 1 are connected, i.e., at the seam.\n\nHowever, this results in maximum layer distance at the seam, which may be too significant of a discrepancy for continued pretraining to quickly resolve. Instead, depthwise scaling sacrifices the 2m middle layers, thereby reducing the discrep- ancy at the seam and making it easier for continued\n\n2 https://huggingface.co/Undi95/ Mistral-11B-v0.1" + }, + { + "bleu": 0.9402607164498324, + "doc_id": "doc_29d883eb26e7b9d08250b4bab1c51092d74e310e6acfa955f8fe28f1008accf5_page_000001.png", + "edit_distance": 0.05215123859191656, + "f1_score": 0.9502572898799313, + "meteor": 0.9699587614483001, + "precision": 0.9787985865724381, + "pred_md": "Table 1: Training datasets used for the instruction and alignment tuning stages, respectively. For the instruction tuning process, we utilized the Alpaca-GPT4 (Peng et al., 2023), OpenOrca (Mukherjee et al., 2023), and Synth. Math-Instruct datasets, while for the alignment tuning, we employed the Orca DPO Pairs (Intel, 2023), Ultrafeedback Cleaned (Cui et al., 2023; Ivison et al., 2023), and Synth. Math-Alignment datasets. The 'Total # Samples' indicates the total number of samples in the entire dataset. The 'Maximum # Samples Used' indicates the actual maximum number of samples that were used in training, which could be lower than the total number of samples in a given dataset. 'Open Source' indicates whether the dataset is open-sourced.\n\npretraining to quickly recover performance. We attribute the success of DUS to reducing such discrepancies in both the depthwise scaling and the continued pretraining steps. We also hypothesize that other methods of depthwise scaling could also work for DUS, as long as the discrepancy in the scaled model is sufficiently contained before the continued pretraining step.\n\nand call it 'Synth. Math-Instruct'.\n\nComparison to other up-scaling methods. Unlike Komatsuzaki et al. (2022), depthwise scaled models do not require additional modules like gating networks or dynamic expert selection. Consequently, scaled models in DUS do not necessitate a distinct training framework for optimal training efficiency, nor do they require specialized CUDA kernels for fast inference. A DUS model can seamlessly integrate into existing training and inference frameworks while maintaining high efficiency.\n\n## 3 Training Details\n\nAfter DUS, including continued pretraining, we perform fine-tuning of SOLAR 10.7B in two stages: 1) instruction tuning and 2) alignment tuning.\n\nInstruction tuning. In the instruction tuning stage, the model is trained to follow instructions in a QA format (Zhang et al., 2023b). We mostly use open-source datasets but also synthesize a math QA dataset to enhance the model's mathematical capabilities. A rundown of how we crafted the dataset is as follows. First, seed math data are collected from the Math (Hendrycks et al., 2021) dataset only, to avoid contamination with commonly used benchmark datasets such as GSM8K (Cobbe et al., 2021). Then, using a process similar to MetaMath (Yu et al., 2023), we rephrase the questions and answers of the seed math data. We use the resulting rephrased question-answer pairs as a QA dataset\n\nAlignment tuning. In the alignment tuning stage, the instruction-tuned model is further fine-tuned to be more aligned with human or strong AI ( e.g., GPT4 (OpenAI, 2023)) preferences using direct preference optimization (DPO) (Rafailov et al., 2023). Similar to the instruction tuning stage, we use mostly open-source datasets but also synthesize a math-focused alignment dataset utilizing the 'Synth. Math-Instruct' dataset mentioned in the instruction tuning stage.\n\nThe alignment data synthesis process is as follows. We take advantage of the fact that the rephrased question-answer pairs in Synth. Math-Instruct data are beneficial in enhancing the model's mathematical capabilities (see Sec. 4.3.1). Thus, we speculate that the rephrased answer to the rephrased question is a better answer than the original answer, possibly due to the interim rephrasing step. Consequently, we set the rephrased question as the prompt and use the rephrased answer as the chosen response and the original answer as the rejected response and create the {prompt, chosen, rejected} DPO tuple. We aggregate the tuples from the rephrased question-answer pairs and call the resulting dataset 'Synth. Math-Alignment'.\n\n## 4 Results\n\n## 4.1 Experimental Details\n\nTraining datasets. We present details regarding our training datasets for the instruction and alignment tuning stages in Tab. 1. We do not always use the entire dataset and instead subsample a set amount. Note that most of our training data is open-source, and the undisclosed datasets can be substituted for open-source alternatives such as the MetaMathQA (Yu et al., 2023) dataset.", + "recall": 0.9233333333333333, + "true_md": "Table 1: Training datasets used for the instruction and alignment tuning stages, respectively. For the instruction tuning process, we utilized the Alpaca-GPT4 (Peng et al., 2023), OpenOrca (Mukherjee et al., 2023), and Synth. Math-Instruct datasets, while for the alignment tuning, we employed the Orca DPO Pairs (Intel, 2023), Ultrafeedback Cleaned (Cui et al., 2023; Ivison et al., 2023), and Synth. Math-Alignment datasets. The 'Total # Samples' indicates the total number of samples in the entire dataset. The 'Maximum # Samples Used' indicates the actual maximum number of samples that were used in training, which could be lower than the total number of samples in a given dataset. 'Open Source' indicates whether the dataset is open-sourced.\n\npretraining to quickly recover performance. We attribute the success of DUS to reducing such dis- crepancies in both the depthwise scaling and the continued pretraining steps. We also hypothesize that other methods of depthwise scaling could also work for DUS, as long as the discrepancy in the scaled model is sufficiently contained before the continued pretraining step.\n\nComparison to other up-scaling methods. Un- like Komatsuzaki et al. (2022), depthwise scaled models do not require additional modules like gat- ing networks or dynamic expert selection. Conse- quently, scaled models in DUS do not necessitate a distinct training framework for optimal training efficiency, nor do they require specialized CUDA kernels for fast inference. A DUS model can seam- lessly integrate into existing training and inference frameworks while maintaining high efficiency.\n\n## 3 Training Details\n\nAfter DUS, including continued pretraining, we perform fine-tuning of SOLAR 10.7B in two stages: 1) instruction tuning and 2) alignment tuning.\n\nInstruction tuning. In the instruction tuning stage, the model is trained to follow instructions in a QA format (Zhang et al., 2023b). We mostly use open-source datasets but also synthesize a math QA dataset to enhance the model's mathematical capa- bilities. A rundown of how we crafted the dataset is as follows. First, seed math data are collected from the Math (Hendrycks et al., 2021) dataset only, to avoid contamination with commonly used bench- mark datasets such as GSM8K (Cobbe et al., 2021). Then, using a process similar to MetaMath (Yu et al., 2023), we rephrase the questions and an- swers of the seed math data. We use the resulting rephrased question-answer pairs as a QA dataset and call it 'Synth. Math-Instruct'.\n\nAlignment tuning. In the alignment tuning stage, the instruction-tuned model is further fine-tuned to be more aligned with human or strong AI (e.g., GPT4 (OpenAI, 2023)) preferences using direct preference optimization (DPO) (Rafailov et al., 2023). Similar to the instruction tuning stage, we use mostly open-source datasets but also synthe- size a math-focused alignment dataset utilizing the 'Synth. Math-Instruct' dataset mentioned in the instruction tuning stage.\n\nThe alignment data synthesis process is as follows. We take advantage of the fact that the rephrased question-answer pairs in Synth. Math-Instruct data are beneficial in enhancing the model's mathematical capabilities (see Sec. 4.3.1). Thus, we speculate that the rephrased answer to the rephrased question is a better answer than the orig- inal answer, possibly due to the interim rephrasing step. Consequently, we set the rephrased question as the prompt and use the rephrased answer as the chosen response and the original answer as the re- jected response and create the {prompt, chosen, rejected} DPO tuple. We aggregate the tuples from the rephrased question-answer pairs and call the resulting dataset 'Synth. Math-Alignment'.\n\n## 4 Results\n\n## 4.1 Experimental Details\n\nTraining datasets. We present details regarding our training datasets for the instruction and align- ment tuning stages in Tab. 1. We do not always use the entire dataset and instead subsample a set amount. Note that most of our training data is open-source, and the undisclosed datasets can be substituted for open-source alternatives such as the MetaMathQA (Yu et al., 2023) dataset." + }, + { + "bleu": 0.9238704340732101, + "doc_id": "doc_61070c2fe64a690c7c9cc97d12b76380e7a134feda3c5315f779306699a89a1b_page_000001.png", + "edit_distance": 0.2050561797752809, + "f1_score": 0.9328493647912885, + "meteor": 0.938825965901774, + "precision": 0.9518518518518518, + "pred_md": "Table 2: Evaluation results for SOLAR 10.7B and SOLAR 10.7B-Instruct along with other top-performing models. We report the scores for the six tasks mentioned in Sec. 4.1 along with the H6 score (average of six tasks). We also report the size of the models in units of billions of parameters. The type indicates the training stage of the model and is chosen from {Pretrained, Instruction-tuned, Alignment-tuned}. Models based on SOLAR 10.7B are colored purple. The best scores for H6 and the individual tasks are shown in bold.\n\nWe reformatted the instruction datasets with an Alpaca-styled chat template. For datasets such as OpenOrca, which are derived from FLAN (Longpre et al., 2023), we filter data that overlaps with the benchmark datasets (see Tab. 8 in Appendix. C for more information). The alignment datasets are in the {prompt, chosen, rejected} triplet format. We preprocess the alignment datasets following Zephyr (Tunstall et al., 2023).\n\nsmaller size, SOLAR 10.7B-Instruct scores the highest in terms of H6, even surpassing the recent top-performing open-source LLM Mixtral 8x7BInstruct-v0.1 or Qwen 72B. The above results indicate DUS can up-scale models that are capable of achieving state-of-the-art performance when finetuned. We also report data contamination results for SOLAR 10.7B-Instruct in Appendix C.\n\nEvaluation. In the HuggingFace Open LLM Leaderboard (Beeching et al., 2023), six types of evaluation methods are presented: ARC (Clark et al., 2018), HellaSWAG (Zellers et al., 2019), MMLU(Hendrycks et al., 2020), TruthfulQA (Lin et al., 2022), Winogrande (Sakaguchi et al., 2021), and GSM8K (Cobbe et al., 2021). We utilize these datasets as benchmarks for evaluation and also report the average scores for the six tasks, e.g., H6.\n\nModel merging. Model merging methods such as Yadav et al. (2023) can boost model performance without further training. We merge some of the models that we trained in both the instruction and alignment tuning stages. We implement our own merging methods although popular open source also exist such as MergeKit 3 .\n\n## 4.2 Main Results\n\nWe present evaluation results for our SOLAR 10.7B and SOLAR 10.7B-Instruct models along with other top-performing models in Tab. 2. SOLAR 10.7B outperforms other pretrained models of similar sizes, such as Qwen 14B and Mistral 7B, which shows that DUS is an effective method to up-scale base LLMs. Furthermore, despite the\n\n3 https://github.com/cg123/mergekit\n\n## 4.3 Ablation Studies\n\nWe present ablation studies for both the instruction and alignment tuning stages.\n\n## 4.3.1 Instruction Tuning\n\nAblation on the training datasets. We present ablation studies using different training datasets for the instruction tuning in Tab. 3. The ablated models are prefixed with SFT for supervised finetuning. 'SFT v1' only uses the Alpaca-GPT4 dataset, whereas 'SFT v2' also uses the OpenOrca dataset. 'SFT v3' uses the Synth. Math-Instruct dataset along with the datasets used in 'SFT v2'. Similarly, 'SFT v4' uses the Synth. Math-Instruct dataset along with the datasets used in 'SFT v1'.\n\nFirst, we analyze how Alpaca-GPT4 and OpenOrca affect the trained models. The first ablated model, 'SFT v1', which used only the AlpacaGPT4 dataset for training, resulted in 69 15 . for H6. When we add the OpenOrca dataset to train the second ablated model, 'SFT v2', the resulting H6 score is 69 21 . , which is little change from 69 15 . of 'SFT v1'. However, the task scores vary more as 'SFT v2' gets a substantially higher GSM8K score of 57 32 . compared to 52 24 . of 'SFT v1' but also gets noticeably lower scores across the board for ARC, HellaSwag, and TruthfulQA. This seems to", + "recall": 0.9145907473309609, + "true_md": "Table 2: Evaluation results for SOLAR 10.7B and SOLAR 10.7B-Instruct along with other top-performing models. We report the scores for the six tasks mentioned in Sec. 4.1 along with the H6 score (average of six tasks). We also report the size of the models in units of billions of parameters. The type indicates the training stage of the model and is chosen from {Pretrained, Instruction-tuned, Alignment-tuned}. Models based on SOLAR 10.7B are colored purple. The best scores for H6 and the individual tasks are shown in bold.\n\nWe reformatted the instruction datasets with an Alpaca-styled chat template. For datasets such as OpenOrca, which are derived from FLAN (Long- pre et al., 2023), we filter data that overlaps with the benchmark datasets (see Tab. 8 in Appendix. C for more information). The alignment datasets are in the {prompt, chosen, rejected} triplet format. We preprocess the alignment datasets following Zephyr (Tunstall et al., 2023).\n\nEvaluation. In the HuggingFace Open LLM Leaderboard (Beeching et al., 2023), six types of evaluation methods are presented: ARC (Clark et al., 2018), HellaSWAG (Zellers et al., 2019), MMLU(Hendrycks et al., 2020), TruthfulQA (Lin et al., 2022), Winogrande (Sakaguchi et al., 2021), and GSM8K (Cobbe et al., 2021). We utilize these datasets as benchmarks for evaluation and also re- port the average scores for the six tasks, e.g., H6.\n\nModel merging. Model merging methods such as Yadav et al. (2023) can boost model perfor- mance without further training. We merge some of the models that we trained in both the instruc- tion and alignment tuning stages. We implement our own merging methods although popular open source also exist such as MergeKit 3 .\n\n## 4.2 Main Results\n\nWe present evaluation results for our SOLAR 10.7B and SOLAR 10.7B-Instruct models along with other top-performing models in Tab. 2. SO- LAR 10.7B outperforms other pretrained models of similar sizes, such as Qwen 14B and Mistral 7B, which shows that DUS is an effective method to up-scale base LLMs. Furthermore, despite the smaller size, SOLAR 10.7B-Instruct scores the highest in terms of H6, even surpassing the recent top-performing open-source LLM Mixtral 8x7B- Instruct-v0.1 or Qwen 72B. The above results indi- cate DUS can up-scale models that are capable of achieving state-of-the-art performance when fine- tuned. We also report data contamination results for SOLAR 10.7B-Instruct in Appendix C.\n\n3 https://github.com/cg123/mergekit\n\n## 4.3 Ablation Studies\n\nWe present ablation studies for both the instruction and alignment tuning stages.\n\n## 4.3.1 Instruction Tuning\n\nAblation on the training datasets. We present ablation studies using different training datasets for the instruction tuning in Tab. 3. The ablated models are prefixed with SFT for supervised fine- tuning. 'SFT v1' only uses the Alpaca-GPT4 dataset, whereas 'SFT v2' also uses the OpenOrca dataset. 'SFT v3' uses the Synth. Math-Instruct dataset along with the datasets used in 'SFT v2'. Similarly, 'SFT v4' uses the Synth. Math-Instruct dataset along with the datasets used in 'SFT v1'.\n\nFirst, we analyze how Alpaca-GPT4 and OpenOrca affect the trained models. The first ab- lated model, 'SFT v1', which used only the Alpaca- GPT4 dataset for training, resulted in 69.15 for H6. When we add the OpenOrca dataset to train the second ablated model, 'SFT v2', the resulting H6 score is 69.21, which is little change from 69.15 of 'SFT v1'. However, the task scores vary more as 'SFT v2' gets a substantially higher GSM8K score of 57.32 compared to 52.24 of 'SFT v1' but also gets noticeably lower scores across the board for ARC, HellaSwag, and TruthfulQA. This seems to" + }, + { + "bleu": 0.9259703849081649, + "doc_id": "doc_2f972daaa81cb3949db12a39cf6910c4f9952135f7eb3850e1f60ef7d8b5aaf9_page_000001.png", + "edit_distance": 0.06116642958748222, + "f1_score": 0.9204819277108433, + "meteor": 0.9703585524133813, + "precision": 0.9227053140096618, + "pred_md": "Table 3: Ablation studies on the different datasets used for instruction tuning. 'SFT v3+v4' indicates that the model is merged from 'SFT v3' and 'SFT v4' by simply averaging the model weights. The best scores for H6 and the individual tasks are shown in bold.\n\nTable 4: Ablation studies on the different datasets used during the direct preference optimization (DPO) stage. 'SFT v3' is used as the SFT base model for DPO. We name ablated models with the 'DPO' prefix to indicate the alignment tuning stage. 'DPO v1+v2' indicates that the model is merged from 'DPO v1' and 'DPO v2' by simply averaging the model weights. The best scores for H6 and the individual tasks are shown in bold.\n\nTable 5: Ablation studies on the different SFT base models used during the direct preference optimization (DPO) stage. Ultrafeedback Clean and Synth. Math-Alignment datasets are used. We name ablated models with the 'DPO' prefix to indicate the alignment tuning stage. The best scores for H6 and the individual tasks are shown in bold.\n\nindicate that using OpenOrca results in a model that behaves differently from using only Alpaca-GPT4.\n\n## 4.3.2 Alignment Tuning\n\nSecond, we investigate whether Synth. MathInstruct dataset is beneficial. For 'SFT v3', we add the Synth. Math-Instruct dataset, which boosts GSM8K scores to 64 14 . and achieves comparable scores for the other tasks. Interestingly, when we add the Synth. Math-Instruct dataset to 'SFT v1' to train 'SFT v4', we get our highest H6 score of 70 88 . with higher scores than 'SFT v3' for all tasks. From the above, we can see that adding the Synth. Math-Instruct dataset is helpful.\n\nLastly, we see whether merging models trained with and without OpenOrca can boost performance. In the first analysis, we saw that using OpenOrca resulted in a model that behaved differently from the model that was trained without OpenOrca. Building on this intuition, we merge 'SFT v3' and 'SFT v4' as they are the best-performing models with and without OpenOrca. To our surprise, the resulting merged model 'SFT v3+v4' retains the high scores for non-GSM8K tasks from 'SFT v4' but also achieves a higher GSM8K score than 'SFT v3' or 'SFT v4'. Thus, we see that merging models that specialize in different tasks is a promising way to obtain a model that performs well generally.\n\nAs we utilize DPO for practical alignment tuning, there are additional aspects to ablate such as the SFT base models used. Thus, we present ablations for the different training datasets used for training, the different SFT base models to initialize the DPO model, and finally, the model merging strategy to obtain the final alignment-tuned model.\n\nAblation on the training datasets. We ablate on the different alignment datasets used during DPO in Tab. 4. We use 'SFT v3' as the SFT base model for DPO. 'DPO v1' only uses the Ultrafeedback Clean dataset while 'DPO v2' also used the Synth. Math-Alignment dataset.\n\nFirst, we test how Ultrafeedback Clean and Synth. Math-Alignment impacts model performance. For 'DPO v1', it achieves 73 06 . in H6, which is a substantial boost from the SFT base model score of 70 03 . . However, we note that while scores for tasks like ARC, HellaSwag, and TruthfulQA all improved by good margins, the score for GSM8K is 58 83 . , which is lower than the SFT base model score of 64 14 . . Adding Synth. Math-Alignment to train 'DPO v2', we see that the GSM8k score improves to 60 27 . , which is lower than the SFT base model but still higher than 'DPO v1'. Other task scores are also not nega-", + "recall": 0.9182692307692307, + "true_md": "Table 3: Ablation studies on the different datasets used for instruction tuning. 'SFT v3+v4' indicates that the model is merged from 'SFT v3' and 'SFT v4' by simply averaging the model weights. The best scores for H6 and the individual tasks are shown in bold.\n\nTable 4: Ablation studies on the different datasets used during the direct preference optimization (DPO) stage. 'SFT v3' is used as the SFT base model for DPO. We name ablated models with the 'DPO' prefix to indicate the alignment tuning stage. 'DPO v1+v2' indicates that the model is merged from 'DPO v1' and 'DPO v2' by simply averaging the model weights. The best scores for H6 and the individual tasks are shown in bold.\n\nTable 5: Ablation studies on the different SFT base models used during the direct preference optimization (DPO) stage. Ultrafeedback Clean and Synth. Math-Alignment datasets are used. We name ablated models with the 'DPO' prefix to indicate the alignment tuning stage. The best scores for H6 and the individual tasks are shown in bold.\n\nindicate that using OpenOrca results in a model that behaves differently from using only Alpaca-GPT4.\n\nSecond, we investigate whether Synth. Math- Instruct dataset is beneficial. For 'SFT v3', we add the Synth. Math-Instruct dataset, which boosts GSM8K scores to 64.14 and achieves comparable scores for the other tasks. Interestingly, when we add the Synth. Math-Instruct dataset to 'SFT v1' to train 'SFT v4', we get our highest H6 score of 70.88 with higher scores than 'SFT v3' for all tasks. From the above, we can see that adding the Synth. Math-Instruct dataset is helpful.\n\nLastly, we see whether merging models trained with and without OpenOrca can boost performance. In the first analysis, we saw that using OpenOrca re- sulted in a model that behaved differently from the model that was trained without OpenOrca. Build- ing on this intuition, we merge 'SFT v3' and 'SFT v4' as they are the best-performing models with and without OpenOrca. To our surprise, the result- ing merged model 'SFT v3+v4' retains the high scores for non-GSM8K tasks from 'SFT v4' but also achieves a higher GSM8K score than 'SFT v3' or 'SFT v4'. Thus, we see that merging models that specialize in different tasks is a promising way to obtain a model that performs well generally.\n\n## 4.3.2 Alignment Tuning\n\nAs we utilize DPO for practical alignment tuning, there are additional aspects to ablate such as the SFT base models used. Thus, we present ablations for the different training datasets used for training, the different SFT base models to initialize the DPO model, and finally, the model merging strategy to obtain the final alignment-tuned model.\n\nAblation on the training datasets. We ablate on the different alignment datasets used during DPO in Tab. 4. We use 'SFT v3' as the SFT base model for DPO. 'DPO v1' only uses the Ultrafeedback Clean dataset while 'DPO v2' also used the Synth. Math-Alignment dataset.\n\nFirst, we test how Ultrafeedback Clean and Synth. Math-Alignment impacts model perfor- mance. For 'DPO v1', it achieves 73.06 in H6, which is a substantial boost from the SFT base model score of 70.03. However, we note that while scores for tasks like ARC, HellaSwag, and Truth- fulQA all improved by good margins, the score for GSM8K is 58.83, which is lower than the SFT base model score of 64.14. Adding Synth. Math-Alignment to train 'DPO v2', we see that the GSM8k score improves to 60.27, which is lower than the SFT base model but still higher than 'DPO v1'. Other task scores are also not nega-" + }, + { + "bleu": 0.9079325705108671, + "doc_id": "doc_eac01901729786f15ba498716c592370d50f00658c291495086ff49a8538c5c6_page_000001.png", + "edit_distance": 0.36561743341404357, + "f1_score": 0.9205776173285197, + "meteor": 0.9168756533962219, + "precision": 0.940959409594096, + "pred_md": "Table 6: Performance comparison amongst the merge candidates. 'Cand. 1' and 'Cand. 2' are trained using the same setting as 'DPO v2' and 'DPO v3', respectively, but with slightly different hyper-parameters. The best scores for H6 and the individual tasks are shown in bold.\n\nTable 7: Ablation studies on the different merge methods used for obtaining the final model. We use 'Cand. 1' and 'Cand. 2' from Tab. 6 as our two models for merging. We name the merged models with the 'Merge' prefix to indicate they are merged. The best scores for H6 and the individual tasks are shown in bold.\n\ntively impacted by adding Synth. Math-Alignment. Thus, we can conclude that adding Synth. MathAlignment is beneficial for H6.\n\nThen, we experiment whether merging 'DPO v1' and 'DPO v2' is beneficial. Unfortunately, 'DPO v1+v2' scores 73 21 . in H6, which is worse than 'DPO v2'. More importantly, the gain in the GSM8K score from adding Synth. MathAlignment is gone, which is undesirable. One reason for this could be that 'DPO v2' is a strict improvement over 'DPO v1', unlike the case for merging 'SFT v3' and 'SFT v4' where the models had different strengths and weaknesses.\n\nTo utilize this for the alignment-tuned model as well, we train two models named 'Cand. 1' and 'Cand. 2' using the same training dataset and SFT base model as 'DPO v2' and 'DPO v3' but with different hyper-parameters to maximize each model's respective strengths. We compare 'Cand. 1' and 'Cand. 2' in Tab. 6 where we can see that 'Cand. 1' has high GSM8K scores but relatively low scores for the other tasks, whereas 'Cand. 2' has low scores for GSM8K but high scores for the other tasks. We merge these two models using various methods and ablate the results in Tab.. 7.\n\nAblation on the SFT base models. When applying DPO, we start from a model that is already instruction tuned ,i.e., the SFT base model and ablate on using different SFT base models. We use Ultrafeedback Clean and Synth. Math-Alignment datasets for this ablation. Each of the ablated models is trained as follows. 'DPO v2' uses 'SFT v3' as the base SFT model, while 'DPO v3' uses 'SFT v3+v4' as the SFT base model instead.\n\nNote that 'SFT v3+v4' has higher scores on all tasks compared to 'SFT v3', and the gap is especially large for ARC ( +1 45 . ) and GSM8K ( +2 43 . ). Surprisingly, the two models perform similarly in terms of H6. A closer look at the scores for the individual tasks shows only a small margin in the GSM8K scores, and other task scores show little difference. Thus, the performance gaps in certain tasks in the SFT base models do not always carry over to the alignment-tuned models.\n\nAblation on different merge methods. From Tab. 3, we saw that merging two models that have different strengths can be beneficial to performance.\n\nWe use two merge methods: 1) Average ( a , b ), where a and b denote the weighting for 'Cand. 1' and 'Cand. 2' when averaging weights and 2) SLERP (Shoemake, 1985). We use ( 0 5 . , 0 5 . ), ( 0 4 . , 0 6 . ), and ( 0 6 . , 0 4 . ) for Average ( a b , ). From Tab. 7, we can see that the different merge methods have little effect on the H6 scores. The scores for the individual tasks also do not differ by much, suggesting that as long as the merge candidates have sufficiently different strengths, the exact merge method may not be as crucial. Thus, we chose 'Merge v1' as our SOLAR 10.7B-Instruct model.\n\n## 5 Conclusion\n\nWe introduce SOLAR 10.7B and its fine-tuned variant SOLAR 10.7B-Instruct, which are depth upscaled (DUS) models with 10.7 billion parameters. They show superior performance over models like Llama 2, Mistral 7B, and Mixtral-7B-Instruct in essential NLP tasks while maintaining computational efficiency. Thus, DUS is effective in scaling-up highly performant LLMs from smaller ones. With more exploration, DUS could be further improved, paving a new path to efficiently scaling LLMs.", + "recall": 0.901060070671378, + "true_md": "Table 6: Performance comparison amongst the merge candidates. 'Cand. 1' and 'Cand. 2' are trained using the same setting as 'DPO v2' and 'DPO v3', respectively, but with slightly different hyper-parameters. The best scores for H6 and the individual tasks are shown in bold.\n\nTable 7: Ablation studies on the different merge methods used for obtaining the final model. We use 'Cand. 1' and 'Cand. 2' from Tab. 6 as our two models for merging. We name the merged models with the 'Merge' prefix to indicate they are merged. The best scores for H6 and the individual tasks are shown in bold.\n\ntively impacted by adding Synth. Math-Alignment. Thus, we can conclude that adding Synth. Math- Alignment is beneficial for H6.\n\nThen, we experiment whether merging 'DPO v1' and 'DPO v2' is beneficial. Unfortunately, 'DPO v1+v2' scores 73.21 in H6, which is worse than 'DPO v2'. More importantly, the gain in the GSM8K score from adding Synth. Math- Alignment is gone, which is undesirable. One reason for this could be that 'DPO v2' is a strict improvement over 'DPO v1', unlike the case for merging 'SFT v3' and 'SFT v4' where the models had different strengths and weaknesses.\n\nAblation on the SFT base models. When ap- plying DPO, we start from a model that is already instruction tuned ,i.e., the SFT base model and ab- late on using different SFT base models. We use Ultrafeedback Clean and Synth. Math-Alignment datasets for this ablation. Each of the ablated mod- els is trained as follows. 'DPO v2' uses 'SFT v3' as the base SFT model, while 'DPO v3' uses 'SFT v3+v4' as the SFT base model instead.\n\nNote that 'SFT v3+v4' has higher scores on all tasks compared to 'SFT v3', and the gap is espe- cially large for ARC (+1.45) and GSM8K (+2.43). Surprisingly, the two models perform similarly in terms of H6. A closer look at the scores for the individual tasks shows only a small margin in the GSM8K scores, and other task scores show little difference. Thus, the performance gaps in certain tasks in the SFT base models do not always carry over to the alignment-tuned models.\n\nAblation on different merge methods. From Tab. 3, we saw that merging two models that have different strengths can be beneficial to performance. To utilize this for the alignment-tuned model as well, we train two models named 'Cand. 1' and 'Cand. 2' using the same training dataset and SFT base model as 'DPO v2' and 'DPO v3' but with dif- ferent hyper-parameters to maximize each model's respective strengths. We compare 'Cand. 1' and 'Cand. 2' in Tab. 6 where we can see that 'Cand. 1' has high GSM8K scores but relatively low scores for the other tasks, whereas 'Cand. 2' has low scores for GSM8K but high scores for the other tasks. We merge these two models using various methods and ablate the results in Tab.. 7.\n\nWe use two merge methods: 1) Average (a, b), where a and b denote the weighting for 'Cand. 1' and 'Cand. 2' when averaging weights and 2) SLERP (Shoemake, 1985). We use (0.5, 0.5), (0.4, 0.6), and (0.6, 0.4) for Average (a, b). From Tab. 7, we can see that the different merge methods have little effect on the H6 scores. The scores for the individual tasks also do not differ by much, suggest- ing that as long as the merge candidates have suffi- ciently different strengths, the exact merge method may not be as crucial. Thus, we chose 'Merge v1' as our SOLAR 10.7B-Instruct model.\n\n## 5 Conclusion\n\nWe introduce SOLAR 10.7B and its fine-tuned vari- ant SOLAR 10.7B-Instruct, which are depth up- scaled (DUS) models with 10.7 billion parameters. They show superior performance over models like Llama 2, Mistral 7B, and Mixtral-7B-Instruct in es- sential NLP tasks while maintaining computational efficiency. Thus, DUS is effective in scaling-up highly performant LLMs from smaller ones. With more exploration, DUS could be further improved, paving a new path to efficiently scaling LLMs." + }, + { + "bleu": 0.8764534816545629, + "doc_id": "doc_196876d21778214061c0d970de83cf37204a90e0f58c3da7e9f3c8c7e803b770_page_000001.png", + "edit_distance": 0.07, + "f1_score": 0.9164733178654292, + "meteor": 0.9331992199186487, + "precision": 0.9518072289156626, + "pred_md": "## Acknowledgements\n\nWe would like to extend our gratitude to the teams at Hugging Face, particularly Clémentine Fourrier, Lewis Tunstall, Omar Sanseviero, and Philipp Schmid. Our appreciation also extends to the teams at AWS, notably Ritesh Vajaria, Gal Oshri, Jay Kwon, Brandon Lee, Effie Bae, and Rahul Sharma. We are grateful to the teams at Korea Telecom (KT), especially Jin Hyoung Lee, Jungsuk Park, Sungjoon Park, Hong-rae Wang, Kyeongsoo Jung, and Sunyoong Yoon, whose significant support has been instrumental in ensuring the broad compatibility of our model. Additionally, we would like to extend our thanks to the open community for their invaluable contributions and feedback.\n\n## Limitations\n\nOur study on the Depth Up-Scaling (DUS) has important limitations and considerations. One key limitation is the need for more thorough explorations of hyperparameters used in the DUS approach. Namely, we removed m = 8 layers from both ends of our base model, primarily due to hardware limitations. However, we have not yet determined if this value is optimal for enhancing performance. The extended time and cost of continued pretraining made it challenging to conduct more comprehensive experiments, which we aim to address in future work through various comparative analyses.\n\nIn terms of the model's broader implications, there are several points to note. The model's significant computational demands for training and inference might limit its use, especially for those with restricted computational resources. Additionally, like all machine learning models, it is vulnerable to biases in its training data, which could lead to skewed outcomes in certain situations. Furthermore, the substantial energy consumption required for training and operating the model raises environmental concerns, which are critical in the pursuit of sustainable AI development.\n\nLastly, while the fine-tuned variant of the model shows improved performance in following instructions, it still requires task-specific fine-tuning for optimal performance in specialized applications. This fine-tuning process can be resource-intensive and not always effective. Recognizing and addressing these limitations is essential for a comprehensive understanding of the proposed Large Language Model's capabilities and for guiding future research and development in the field of LLMs.\n\n## Ethics Statement\n\nWe conscientiously address and emphasize the commitment of SOLAR 10.7B in maintaining the highest ethical standards. First, we highlight that SOLAR 10.7B-Instruct has shown low levels of data contamination in our evaluations, a testament to our rigorous data handling and processing protocols. This aspect is crucial, as it underpins the reliability and integrity of the results obtained from SOLAR.\n\nFurthermore, during the course of our experiments, we ensured that all setups and methodologies employed steer clear of any potential ethical pitfalls. This preemptive consideration and avoidance of ethically questionable practices underscore our dedication to conducting research that is not only innovative but also responsible.\n\nAdditionally, we ensure that SOLAR complies with general ethical considerations in all aspects of its operation. This includes adherence to privacy norms, respect for intellectual property, and ensuring the absence of bias in our algorithms. Our commitment to these ethical principles is unwavering, and we believe it significantly contributes to the credibility and societal acceptance of SOLAR.\n\nIn conclusion, the ethical framework within which SOLAR operates is robust and comprehensive, ensuring that our advancements in this field are not only scientifically sound but also ethically responsible.\n\n## References\n\nIan L Alberts, Lorenzo Mercolli, Thomas Pyka, George Prenosil, Kuangyu Shi, Axel Rominger, and Ali Afshar-Oromieh. 2023. Large language models (llm) and chatgpt: what will the impact on nuclear medicine be? European journal of nuclear medicine and molecular imaging , 50(6):1549-1552.\n\nRohan Anil, Andrew M Dai, Orhan Firat, Melvin Johnson, Dmitry Lepikhin, Alexandre Passos, Siamak Shakeri, Emanuel Taropa, Paige Bailey, Zhifeng Chen, et al. 2023. Palm 2 technical report. arXiv preprint arXiv:2305.10403 .\n\nAram Bahrini, Mohammadsadra Khamoshifar, Hossein Abbasimehr, Robert J Riggs, Maryam Esmaeili, Rastin Mastali Majdabadkohne, and Morteza Pasehvar. 2023. Chatgpt: Applications, opportunities, and threats. In 2023 Systems and Information Engineering Design Symposium (SIEDS) , pages 274-279. IEEE.", + "recall": 0.883668903803132, + "true_md": "## Acknowledgements\n\nWe would like to extend our gratitude to the teams at Hugging Face, particularly Clémentine Four- rier, Lewis Tunstall, Omar Sanseviero, and Philipp Schmid. Our appreciation also extends to the teams at AWS, notably Ritesh Vajaria, Gal Oshri, Jay Kwon, Brandon Lee, Effie Bae, and Rahul Sharma. We are grateful to the teams at Korea Telecom (KT), especially Jin Hyoung Lee, Jungsuk Park, Sungjoon Park, Hong-rae Wang, Kyeongsoo Jung, and Sunyoong Yoon, whose significant support has been instrumental in ensuring the broad compati- bility of our model. Additionally, we would like to extend our thanks to the open community for their invaluable contributions and feedback.\n\n## Limitations\n\nOur study on the Depth Up-Scaling (DUS) has im- portant limitations and considerations. One key limitation is the need for more thorough explo- rations of hyperparameters used in the DUS ap- proach. Namely, we removed m = 8 layers from both ends of our base model, primarily due to hard- ware limitations. However, we have not yet deter- mined if this value is optimal for enhancing perfor- mance. The extended time and cost of continued pretraining made it challenging to conduct more comprehensive experiments, which we aim to ad- dress in future work through various comparative analyses.\n\nIn terms of the model's broader implications, there are several points to note. The model's sig- nificant computational demands for training and inference might limit its use, especially for those with restricted computational resources. Addition- ally, like all machine learning models, it is vulnera- ble to biases in its training data, which could lead to skewed outcomes in certain situations. Further- more, the substantial energy consumption required for training and operating the model raises environ- mental concerns, which are critical in the pursuit of sustainable AI development.\n\nLastly, while the fine-tuned variant of the model shows improved performance in following instruc- tions, it still requires task-specific fine-tuning for optimal performance in specialized applications. This fine-tuning process can be resource-intensive and not always effective. Recognizing and address- ing these limitations is essential for a comprehen- sive understanding of the proposed Large Language Model's capabilities and for guiding future research and development in the field of LLMs.\n\n## Ethics Statement\n\nWe conscientiously address and emphasize the commitment of SOLAR 10.7B in maintaining the highest ethical standards. First, we highlight that SOLAR 10.7B-Instruct has shown low levels of data contamination in our evaluations, a testament to our rigorous data handling and processing pro- tocols. This aspect is crucial, as it underpins the reliability and integrity of the results obtained from SOLAR.\n\nFurthermore, during the course of our experi- ments, we ensured that all setups and methodolo- gies employed steer clear of any potential ethical pitfalls. This preemptive consideration and avoid- ance of ethically questionable practices underscore our dedication to conducting research that is not only innovative but also responsible.\n\nAdditionally, we ensure that SOLAR complies with general ethical considerations in all aspects of its operation. This includes adherence to pri- vacy norms, respect for intellectual property, and ensuring the absence of bias in our algorithms. Our commitment to these ethical principles is unwaver- ing, and we believe it significantly contributes to the credibility and societal acceptance of SOLAR.\n\nIn conclusion, the ethical framework within which SOLAR operates is robust and comprehen- sive, ensuring that our advancements in this field are not only scientifically sound but also ethically responsible.\n\n## References\n\nIan L Alberts, Lorenzo Mercolli, Thomas Pyka, George Prenosil, Kuangyu Shi, Axel Rominger, and Ali Afshar-Oromieh. 2023. Large language models (llm) and chatgpt: what will the impact on nuclear medicine be? European journal of nuclear medicine and molecular imaging, 50(6):1549-1552.\n\nRohan Anil, Andrew M Dai, Orhan Firat, Melvin John- son, Dmitry Lepikhin, Alexandre Passos, Siamak Shakeri, Emanuel Taropa, Paige Bailey, Zhifeng Chen, et al. 2023. Palm 2 technical report. arXiv preprint arXiv:2305.10403.\n\nAram Bahrini, Mohammadsadra Khamoshifar, Hos- sein Abbasimehr, Robert J Riggs, Maryam Esmaeili, Rastin Mastali Majdabadkohne, and Morteza Pase- hvar. 2023. Chatgpt: Applications, opportunities, and threats. In 2023 Systems and Information Engi- neering Design Symposium (SIEDS), pages 274-279. IEEE." + }, + { + "bleu": 0.9461352082271325, + "doc_id": "doc_eaafa071048c2ec1309751325280ea3d354317cf67ebda400cdf4fd39503392b_page_000001.png", + "edit_distance": 0.03257328990228013, + "f1_score": 0.9600818833162742, + "meteor": 0.9689976005494595, + "precision": 0.975051975051975, + "pred_md": "Edward Beeching, Clémentine Fourrier, Nathan Habib, Sheon Han, Nathan Lambert, Nazneen Rajani, Omar Sanseviero, Lewis Tunstall, and Thomas Wolf. 2023. Open llm leaderboard. https://huggingface.co/spaces/ HuggingFaceH4/open\\_llm\\_leaderboard .\n\nTom Brown, Benjamin Mann, Nick Ryder, Melanie Subbiah, Jared D Kaplan, Prafulla Dhariwal, Arvind Neelakantan, Pranav Shyam, Girish Sastry, Amanda Askell, et al. 2020. Language models are few-shot learners. Advances in neural information processing systems , 33:1877-1901.\n\nPeter Clark, Isaac Cowhey, Oren Etzioni, Tushar Khot, Ashish Sabharwal, Carissa Schoenick, and Oyvind Tafjord. 2018. Think you have solved question answering? try arc, the ai2 reasoning challenge. arXiv preprint arXiv:1803.05457 .\n\nKarl Cobbe, Vineet Kosaraju, Mohammad Bavarian, Mark Chen, Heewoo Jun, Lukasz Kaiser, Matthias Plappert, Jerry Tworek, Jacob Hilton, Reiichiro Nakano, et al. 2021. Training verifiers to solve math word problems. arXiv preprint arXiv:2110.14168 .\n\nGanqu Cui, Lifan Yuan, Ning Ding, Guanming Yao, Wei Zhu, Yuan Ni, Guotong Xie, Zhiyuan Liu, and Maosong Sun. 2023. Ultrafeedback: Boosting language models with high-quality feedback. arXiv preprint arXiv:2310.01377 .\n\nChunyuan Deng, Yilun Zhao, Xiangru Tang, Mark Gerstein, and Arman Cohan. 2023. Investigating data contamination in modern benchmarks for large language models. arXiv preprint arXiv:2311.09783 .\n\nHanze Dong, Wei Xiong, Deepanshu Goyal, Rui Pan, Shizhe Diao, Jipeng Zhang, Kashun Shum, and Tong Zhang. 2023. Raft: Reward ranked finetuning for generative foundation model alignment. arXiv preprint arXiv:2304.06767 .\n\nMohammad Fraiwan and Natheer Khasawneh. 2023. A review of chatgpt applications in education, marketing, software engineering, and healthcare: Benefits, drawbacks, and research directions. arXiv preprint arXiv:2305.00237 .\n\nTrevor Gale, Deepak Narayanan, Cliff Young, and Matei Zaharia. 2023. Megablocks: Efficient sparse training with mixture-of-experts. Proceedings of Machine Learning and Systems , 5.\n\nAndrea Gesmundo and Kaitlin Maile. 2023. Composable function-preserving expansions for transformer architectures. arXiv preprint arXiv:2308.06103 .\n\nShahriar Golchin and Mihai Surdeanu. 2023. Time travel in llms: Tracing data contamination in large language models. arXiv preprint arXiv:2308.08493 .\n\nDan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, and Jacob Steinhardt. 2020. Measuring massive multitask language understanding. In International Conference on Learning Representations .\n\nDan Hendrycks, Collin Burns, Saurav Kadavath, Akul Arora, Steven Basart, Eric Tang, Dawn Song, and Jacob Steinhardt. 2021. Measuring mathematical problem solving with the math dataset. arXiv preprint arXiv:2103.03874 .\n\nDanny Hernandez, Jared Kaplan, Tom Henighan, and Sam McCandlish. 2021. Scaling laws for transfer. arXiv preprint arXiv:2102.01293 .\n\nChangho Hwang, Wei Cui, Yifan Xiong, Ziyue Yang, Ze Liu, Han Hu, Zilong Wang, Rafael Salas, Jithin Jose, Prabhat Ram, et al. 2023. Tutel: Adaptive mixture-of-experts at scale. Proceedings of Machine Learning and Systems , 5.\n\nIntel. 2023. Supervised fine-tuning and direct preference optimization on intel gaudi2.\n\nHamish Ivison, Yizhong Wang, Valentina Pyatkin, Nathan Lambert, Matthew Peters, Pradeep Dasigi, Joel Jang, David Wadden, Noah A. Smith, Iz Beltagy, and Hannaneh Hajishirzi. 2023. Camels in a changing climate: Enhancing lm adaptation with tulu 2.\n\nAlbert Q Jiang, Alexandre Sablayrolles, Arthur Mensch, Chris Bamford, Devendra Singh Chaplot, Diego de las Casas, Florian Bressand, Gianna Lengyel, Guillaume Lample, Lucile Saulnier, et al. 2023. Mistral 7b. arXiv preprint arXiv:2310.06825 .\n\nJean Kaddour, Oscar Key, Piotr Nawrot, Pasquale Minervini, and Matt J Kusner. 2023. No train no gain: Revisiting efficient training algorithms for transformer-based language models. arXiv preprint arXiv:2307.06440 .\n\nJared Kaplan, Sam McCandlish, Tom Henighan, Tom B Brown, Benjamin Chess, Rewon Child, Scott Gray, Alec Radford, Jeffrey Wu, and Dario Amodei. 2020. Scaling laws for neural language models. arXiv preprint arXiv:2001.08361 .\n\nAran Komatsuzaki, Joan Puigcerver, James Lee-Thorp, Carlos Riquelme Ruiz, Basil Mustafa, Joshua Ainslie, Yi Tay, Mostafa Dehghani, and Neil Houlsby. 2022. Sparse upcycling: Training mixture-ofexperts from dense checkpoints. arXiv preprint arXiv:2212.05055 .\n\nWing Lian. 2023. https://huggingface.co/ winglian/omega-3b .\n\nStephanie Lin, Jacob Hilton, and Owain Evans. 2022. Truthfulqa: Measuring how models mimic human falsehoods. In Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers) , pages 3214-3252.\n\nShayne Longpre, Le Hou, Tu Vu, Albert Webson, Hyung Won Chung, Yi Tay, Denny Zhou, Quoc V Le, Barret Zoph, Jason Wei, et al. 2023. The flan collection: Designing data and methods for effective instruction tuning. arXiv preprint arXiv:2301.13688 .", + "recall": 0.9455645161290323, + "true_md": "Edward Beeching, Clémentine Fourrier, Nathan Habib, Sheon Han, Nathan Lambert, Nazneen Rajani, Omar Sanseviero, Lewis Tunstall, and Thomas Wolf. 2023. Open llm leaderboard. https://huggingface.co/spaces/ HuggingFaceH4/open\\_llm\\_leaderboard.\n\nTom Brown, Benjamin Mann, Nick Ryder, Melanie Subbiah, Jared D Kaplan, Prafulla Dhariwal, Arvind Neelakantan, Pranav Shyam, Girish Sastry, Amanda Askell, et al. 2020. Language models are few-shot learners. Advances in neural information processing systems, 33:1877-1901.\n\nPeter Clark, Isaac Cowhey, Oren Etzioni, Tushar Khot, Ashish Sabharwal, Carissa Schoenick, and Oyvind Tafjord. 2018. Think you have solved question an- swering? try arc, the ai2 reasoning challenge. arXiv preprint arXiv:1803.05457.\n\nKarl Cobbe, Vineet Kosaraju, Mohammad Bavarian, Mark Chen, Heewoo Jun, Lukasz Kaiser, Matthias Plappert, Jerry Tworek, Jacob Hilton, Reiichiro Nakano, et al. 2021. Training verifiers to solve math word problems. arXiv preprint arXiv:2110.14168.\n\nGanqu Cui, Lifan Yuan, Ning Ding, Guanming Yao, Wei Zhu, Yuan Ni, Guotong Xie, Zhiyuan Liu, and Maosong Sun. 2023. Ultrafeedback: Boosting lan- guage models with high-quality feedback. arXiv preprint arXiv:2310.01377.\n\nChunyuan Deng, Yilun Zhao, Xiangru Tang, Mark Ger- stein, and Arman Cohan. 2023. Investigating data contamination in modern benchmarks for large lan- guage models. arXiv preprint arXiv:2311.09783.\n\nHanze Dong, Wei Xiong, Deepanshu Goyal, Rui Pan, Shizhe Diao, Jipeng Zhang, Kashun Shum, and Tong Zhang. 2023. Raft: Reward ranked finetuning for generative foundation model alignment. arXiv preprint arXiv:2304.06767.\n\nMohammad Fraiwan and Natheer Khasawneh. 2023. A review of chatgpt applications in education, market- ing, software engineering, and healthcare: Benefits, drawbacks, and research directions. arXiv preprint arXiv:2305.00237.\n\nTrevor Gale, Deepak Narayanan, Cliff Young, and Matei Zaharia. 2023. Megablocks: Efficient sparse training with mixture-of-experts. Proceedings of Machine Learning and Systems, 5.\n\nAndrea Gesmundo and Kaitlin Maile. 2023. Compos- able function-preserving expansions for transformer architectures. arXiv preprint arXiv:2308.06103.\n\nShahriar Golchin and Mihai Surdeanu. 2023. Time travel in llms: Tracing data contamination in large language models. arXiv preprint arXiv:2308.08493.\n\nDan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, and Jacob Steinhardt. 2020. Measuring massive multitask language under- standing. In International Conference on Learning Representations.\n\nDan Hendrycks, Collin Burns, Saurav Kadavath, Akul Arora, Steven Basart, Eric Tang, Dawn Song, and Ja- cob Steinhardt. 2021. Measuring mathematical prob- lem solving with the math dataset. arXiv preprint arXiv:2103.03874.\n\nDanny Hernandez, Jared Kaplan, Tom Henighan, and Sam McCandlish. 2021. Scaling laws for transfer. arXiv preprint arXiv:2102.01293.\n\nChangho Hwang, Wei Cui, Yifan Xiong, Ziyue Yang, Ze Liu, Han Hu, Zilong Wang, Rafael Salas, Jithin Jose, Prabhat Ram, et al. 2023. Tutel: Adaptive mixture-of-experts at scale. Proceedings of Machine Learning and Systems, 5.\n\nIntel. 2023. Supervised fine-tuning and direct prefer- ence optimization on intel gaudi2.\n\nHamish Ivison, Yizhong Wang, Valentina Pyatkin, Nathan Lambert, Matthew Peters, Pradeep Dasigi, Joel Jang, David Wadden, Noah A. Smith, Iz Belt- agy, and Hannaneh Hajishirzi. 2023. Camels in a changing climate: Enhancing lm adaptation with tulu 2.\n\nAlbert Q Jiang, Alexandre Sablayrolles, Arthur Men- sch, Chris Bamford, Devendra Singh Chaplot, Diego de las Casas, Florian Bressand, Gianna Lengyel, Guil- laume Lample, Lucile Saulnier, et al. 2023. Mistral 7b. arXiv preprint arXiv:2310.06825.\n\nJean Kaddour, Oscar Key, Piotr Nawrot, Pasquale Minervini, and Matt J Kusner. 2023. No train no gain: Revisiting efficient training algorithms for transformer-based language models. arXiv preprint arXiv:2307.06440.\n\nJared Kaplan, Sam McCandlish, Tom Henighan, Tom B Brown, Benjamin Chess, Rewon Child, Scott Gray, Alec Radford, Jeffrey Wu, and Dario Amodei. 2020. Scaling laws for neural language models. arXiv preprint arXiv:2001.08361.\n\nAran Komatsuzaki, Joan Puigcerver, James Lee-Thorp, Carlos Riquelme Ruiz, Basil Mustafa, Joshua Ainslie, Yi Tay, Mostafa Dehghani, and Neil Houlsby. 2022. Sparse upcycling: Training mixture-of- experts from dense checkpoints. arXiv preprint arXiv:2212.05055.\n\nWing Lian. 2023. https://huggingface.co/ winglian/omega-3b.\n\nStephanie Lin, Jacob Hilton, and Owain Evans. 2022. Truthfulqa: Measuring how models mimic human falsehoods. In Proceedings of the 60th Annual Meet- ing of the Association for Computational Linguistics (Volume 1: Long Papers), pages 3214-3252.\n\nShayne Longpre, Le Hou, Tu Vu, Albert Webson, Hyung Won Chung, Yi Tay, Denny Zhou, Quoc V Le, Barret Zoph, Jason Wei, et al. 2023. The flan collection: Designing data and methods for effective instruction tuning. arXiv preprint arXiv:2301.13688." + }, + { + "bleu": 0.9030410934642822, + "doc_id": "doc_fcaa8c3de4757ff51e604d84c983c59c421e31b9989df5b74c930e0e8720e5ae_page_000001.png", + "edit_distance": 0.05732484076433121, + "f1_score": 0.9305555555555556, + "meteor": 0.9463374237585681, + "precision": 0.9610655737704918, + "pred_md": "Subhabrata Mukherjee, Arindam Mitra, Ganesh Jawahar, Sahaj Agarwal, Hamid Palangi, and Ahmed Awadallah. 2023. Orca: Progressive learning from complex explanation traces of gpt-4. arXiv preprint arXiv:2306.02707 .\n\nOpenAI. 2023. Gpt-4 technical report.\n\nYu Pan, Ye Yuan, Yichun Yin, Zenglin Xu, Lifeng Shang, Xin Jiang, and Qun Liu. 2023. Reusing pretrained models by multi-linear operators for efficient training. arXiv preprint arXiv:2310.10699 .\n\nBaolin Peng, Chunyuan Li, Pengcheng He, Michel Galley, and Jianfeng Gao. 2023. Instruction tuning with gpt-4. arXiv preprint arXiv:2304.03277 .\n\nAlec Radford, Jeffrey Wu, Rewon Child, David Luan, Dario Amodei, Ilya Sutskever, et al. 2019. Language models are unsupervised multitask learners. OpenAI blog , 1(8):9.\n\nJack W Rae, Sebastian Borgeaud, Trevor Cai, Katie Millican, Jordan Hoffmann, Francis Song, John Aslanides, Sarah Henderson, Roman Ring, Susannah Young, et al. 2021. Scaling language models: Methods, analysis & insights from training gopher. arXiv preprint arXiv:2112.11446 .\n\nRafael Rafailov, Archit Sharma, Eric Mitchell, Stefano Ermon, Christopher D Manning, and Chelsea Finn. 2023. Direct preference optimization: Your language model is secretly a reward model. arXiv preprint arXiv:2305.18290 .\n\nOscar Sainz, Jon Ander Campos, Iker García-Ferrero, Julen Etxaniz, Oier Lopez de Lacalle, and Eneko Agirre. 2023. Nlp evaluation in trouble: On the need to measure llm data contamination for each benchmark. arXiv preprint arXiv:2310.18018 .\n\nKeisuke Sakaguchi, Ronan Le Bras, Chandra Bhagavatula, and Yejin Choi. 2021. Winogrande: An adversarial winograd schema challenge at scale. Communications of the ACM , 64(9):99-106.\n\nMalik Sallam, Nesreen Salim, Muna Barakat, and Alaa Al-Tammemi. 2023. Chatgpt applications in medical, dental, pharmacy, and public health education: A descriptive study highlighting the advantages and limitations. Narra J , 3(1):e103-e103.\n\nNoam Shazeer, Azalia Mirhoseini, Krzysztof Maziarz, Andy Davis, Quoc Le, Geoffrey Hinton, and Jeff Dean. 2017. Outrageously large neural networks: The sparsely-gated mixture-of-experts layer. arXiv preprint arXiv:1701.06538 .\n\nTianxiao Shen, Myle Ott, Michael Auli, and Marc'Aurelio Ranzato. 2019. Mixture models for diverse machine translation: Tricks of the trade. In International conference on machine learning , pages 5719-5728. PMLR.\n\nWeijia Shi, Anirudh Ajith, Mengzhou Xia, Yangsibo Huang, Daogao Liu, Terra Blevins, Danqi Chen, and Luke Zettlemoyer. 2023. Detecting pretraining data from large language models. arXiv preprint arXiv:2310.16789 .\n\nKen Shoemake. 1985. Animating rotation with quaternion curves. In Proceedings of the 12th annual conference on Computer graphics and interactive techniques , pages 245-254.\n\nMingxing Tan and Quoc Le. 2019. Efficientnet: Rethinking model scaling for convolutional neural networks. In International conference on machine learning , pages 6105-6114. PMLR.\n\nHugo Touvron, Louis Martin, Kevin Stone, Peter Albert, Amjad Almahairi, Yasmine Babaei, Nikolay Bashlykov, Soumya Batra, Prajjwal Bhargava, Shruti Bhosale, et al. 2023. Llama 2: Open foundation and fine-tuned chat models. arXiv preprint arXiv:2307.09288 .\n\nLewis Tunstall, Edward Beeching, Nathan Lambert, Nazneen Rajani, Kashif Rasul, Younes Belkada, Shengyi Huang, Leandro von Werra, Clémentine Fourrier, Nathan Habib, et al. 2023. Zephyr: Direct distillation of lm alignment. arXiv preprint arXiv:2310.16944 .\n\nPeihao Wang, Rameswar Panda, Lucas Torroba Hennigen, Philip Greengard, Leonid Karlinsky, Rogerio Feris, David Daniel Cox, Zhangyang Wang, and Yoon Kim. 2023. Learning to grow pretrained models for efficient transformer training. arXiv preprint arXiv:2303.00980 .\n\nYizhong Wang, Yeganeh Kordi, Swaroop Mishra, Alisa Liu, Noah A Smith, Daniel Khashabi, and Hannaneh Hajishirzi. 2022. Self-instruct: Aligning language model with self generated instructions. arXiv preprint arXiv:2212.10560 .\n\nJason Wei, Maarten Bosma, Vincent Y Zhao, Kelvin Guu, Adams Wei Yu, Brian Lester, Nan Du, Andrew M Dai, and Quoc V Le. 2021. Finetuned language models are zero-shot learners. arXiv preprint arXiv:2109.01652 .\n\nJason Wei, Yi Tay, Rishi Bommasani, Colin Raffel, Barret Zoph, Sebastian Borgeaud, Dani Yogatama, Maarten Bosma, Denny Zhou, Donald Metzler, et al. 2022a. Emergent abilities of large language models. arXiv preprint arXiv:2206.07682 .\n\nJason Wei, Xuezhi Wang, Dale Schuurmans, Maarten Bosma, Fei Xia, Ed Chi, Quoc V Le, Denny Zhou, et al. 2022b. Chain-of-thought prompting elicits reasoning in large language models. Advances in Neural Information Processing Systems , 35:24824-24837.\n\nThomas Wolf, Lysandre Debut, Victor Sanh, Julien Chaumond, Clement Delangue, Anthony Moi, Pierric Cistac, Tim Rault, Rémi Louf, Morgan Funtowicz, et al. 2019. Huggingface's transformers: State-ofthe-art natural language processing. arXiv preprint arXiv:1910.03771 .", + "recall": 0.9019230769230769, + "true_md": "Subhabrata Mukherjee, Arindam Mitra, Ganesh Jawa- har, Sahaj Agarwal, Hamid Palangi, and Ahmed Awadallah. 2023. Orca: Progressive learning from complex explanation traces of gpt-4. arXiv preprint arXiv:2306.02707.\n\nOpenAI. 2023. Gpt-4 technical report.\n\nYu Pan, Ye Yuan, Yichun Yin, Zenglin Xu, Lifeng Shang, Xin Jiang, and Qun Liu. 2023. Reusing pre- trained models by multi-linear operators for efficient training. arXiv preprint arXiv:2310.10699.\n\nBaolin Peng, Chunyuan Li, Pengcheng He, Michel Gal- ley, and Jianfeng Gao. 2023. Instruction tuning with gpt-4. arXiv preprint arXiv:2304.03277.\n\nAlec Radford, Jeffrey Wu, Rewon Child, David Luan, Dario Amodei, Ilya Sutskever, et al. 2019. Language models are unsupervised multitask learners. OpenAI blog, 1(8):9.\n\nJack W Rae, Sebastian Borgeaud, Trevor Cai, Katie Millican, Jordan Hoffmann, Francis Song, John Aslanides, Sarah Henderson, Roman Ring, Susan- nah Young, et al. 2021. Scaling language models: Methods, analysis & insights from training gopher. arXiv preprint arXiv:2112.11446.\n\nRafael Rafailov, Archit Sharma, Eric Mitchell, Stefano Ermon, Christopher D Manning, and Chelsea Finn. 2023. Direct preference optimization: Your language model is secretly a reward model. arXiv preprint arXiv:2305.18290.\n\nOscar Sainz, Jon Ander Campos, Iker García-Ferrero, Julen Etxaniz, Oier Lopez de Lacalle, and Eneko Agirre. 2023. Nlp evaluation in trouble: On the need to measure llm data contamination for each benchmark. arXiv preprint arXiv:2310.18018.\n\nKeisuke Sakaguchi, Ronan Le Bras, Chandra Bhagavat- ula, and Yejin Choi. 2021. Winogrande: An adver- sarial winograd schema challenge at scale. Commu- nications of the ACM, 64(9):99-106.\n\nMalik Sallam, Nesreen Salim, Muna Barakat, and Alaa Al-Tammemi. 2023. Chatgpt applications in medical, dental, pharmacy, and public health education: A descriptive study highlighting the advantages and limitations. Narra J, 3(1):e103-e103.\n\nNoam Shazeer, Azalia Mirhoseini, Krzysztof Maziarz, Andy Davis, Quoc Le, Geoffrey Hinton, and Jeff Dean. 2017. Outrageously large neural networks: The sparsely-gated mixture-of-experts layer. arXiv preprint arXiv:1701.06538.\n\nTianxiao Shen, Myle Ott, Michael Auli, and Marc'Aurelio Ranzato. 2019. Mixture models for diverse machine translation: Tricks of the trade. In International conference on machine learning, pages 5719-5728. PMLR.\n\nWeijia Shi, Anirudh Ajith, Mengzhou Xia, Yangsibo Huang, Daogao Liu, Terra Blevins, Danqi Chen, and Luke Zettlemoyer. 2023. Detecting pretraining data from large language models. arXiv preprint arXiv:2310.16789.\n\nKen Shoemake. 1985. Animating rotation with quater- nion curves. In Proceedings of the 12th annual con- ference on Computer graphics and interactive tech- niques, pages 245-254.\n\nMingxing Tan and Quoc Le. 2019. Efficientnet: Re- thinking model scaling for convolutional neural net- works. In International conference on machine learn- ing, pages 6105-6114. PMLR.\n\nHugo Touvron, Louis Martin, Kevin Stone, Peter Al- bert, Amjad Almahairi, Yasmine Babaei, Nikolay Bashlykov, Soumya Batra, Prajjwal Bhargava, Shruti Bhosale, et al. 2023. Llama 2: Open founda- tion and fine-tuned chat models. arXiv preprint arXiv:2307.09288.\n\nLewis Tunstall, Edward Beeching, Nathan Lambert, Nazneen Rajani, Kashif Rasul, Younes Belkada, Shengyi Huang, Leandro von Werra, Clémentine Fourrier, Nathan Habib, et al. 2023. Zephyr: Di- rect distillation of lm alignment. arXiv preprint arXiv:2310.16944.\n\nPeihao Wang, Rameswar Panda, Lucas Torroba Hen- nigen, Philip Greengard, Leonid Karlinsky, Roge- rio Feris, David Daniel Cox, Zhangyang Wang, and Yoon Kim. 2023. Learning to grow pretrained mod- els for efficient transformer training. arXiv preprint arXiv:2303.00980.\n\nYizhong Wang, Yeganeh Kordi, Swaroop Mishra, Al- isa Liu, Noah A Smith, Daniel Khashabi, and Han- naneh Hajishirzi. 2022. Self-instruct: Aligning lan- guage model with self generated instructions. arXiv preprint arXiv:2212.10560.\n\nJason Wei, Maarten Bosma, Vincent Y Zhao, Kelvin Guu, Adams Wei Yu, Brian Lester, Nan Du, An- drew M Dai, and Quoc V Le. 2021. Finetuned lan- guage models are zero-shot learners. arXiv preprint arXiv:2109.01652.\n\nJason Wei, Yi Tay, Rishi Bommasani, Colin Raffel, Barret Zoph, Sebastian Borgeaud, Dani Yogatama, Maarten Bosma, Denny Zhou, Donald Metzler, et al. 2022a. Emergent abilities of large language models. arXiv preprint arXiv:2206.07682.\n\nJason Wei, Xuezhi Wang, Dale Schuurmans, Maarten Bosma, Fei Xia, Ed Chi, Quoc V Le, Denny Zhou, et al. 2022b. Chain-of-thought prompting elicits rea- soning in large language models. Advances in Neural Information Processing Systems, 35:24824-24837.\n\nThomas Wolf, Lysandre Debut, Victor Sanh, Julien Chaumond, Clement Delangue, Anthony Moi, Pier- ric Cistac, Tim Rault, Rémi Louf, Morgan Funtowicz, et al. 2019. Huggingface's transformers: State-of- the-art natural language processing. arXiv preprint arXiv:1910.03771." + }, + { + "bleu": 0.9085635014878181, + "doc_id": "doc_2ad0df434ac3f29ff1dfd1ce8e378a2badf1bb6b9aede5d18de2945460f55c1d_page_000001.png", + "edit_distance": 0.05390070921985816, + "f1_score": 0.9302325581395349, + "meteor": 0.9484439151550638, + "precision": 0.9580838323353293, + "pred_md": "Peihao Wang, Rameswar Panda, Lucas Torroba Hennigen, Philip Greengard, Leonid Karlinsky, Rogerio Feris, David Daniel Cox, Zhangyang Wang, and Yoon Kim. 2023. Learning to grow pretrained models for efficient transformer training. arXiv preprint arXiv:2303.00980 .\n\nYizhong Wang, Yeganeh Kordi, Swaroop Mishra, Alisa Liu, Noah A Smith, Daniel Khashabi, and Hannaneh Hajishirzi. 2022. Self-instruct: Aligning language model with self generated instructions. arXiv preprint arXiv:2212.10560 .\n\nJason Wei, Maarten Bosma, Vincent Y Zhao, Kelvin Guu, Adams Wei Yu, Brian Lester, Nan Du, Andrew M Dai, and Quoc V Le. 2021. Finetuned language models are zero-shot learners. arXiv preprint arXiv:2109.01652 .\n\nJason Wei, Yi Tay, Rishi Bommasani, Colin Raffel, Barret Zoph, Sebastian Borgeaud, Dani Yogatama, Maarten Bosma, Denny Zhou, Donald Metzler, et al. 2022a. Emergent abilities of large language models. arXiv preprint arXiv:2206.07682 .\n\nJason Wei, Xuezhi Wang, Dale Schuurmans, Maarten Bosma, Fei Xia, Ed Chi, Quoc V Le, Denny Zhou, et al. 2022b. Chain-of-thought prompting elicits reasoning in large language models. Advances in Neural Information Processing Systems , 35:24824-24837.\n\nThomas Wolf, Lysandre Debut, Victor Sanh, Julien Chaumond, Clement Delangue, Anthony Moi, Pierric Cistac, Tim Rault, Rémi Louf, Morgan Funtowicz, et al. 2019. Huggingface's transformers: State-ofthe-art natural language processing. arXiv preprint arXiv:1910.03771 .\n\nPrateek Yadav, Derek Tam, Leshem Choshen, Colin Raffel, and Mohit Bansal. 2023. Ties-merging: Resolving interference when merging models. In Thirtyseventh Conference on Neural Information Processing Systems .\n\nChengrun Yang, Xuezhi Wang, Yifeng Lu, Hanxiao Liu, Quoc V Le, Denny Zhou, and Xinyun Chen. 2023. Large language models as optimizers. arXiv preprint arXiv:2309.03409 .\n\nYiqun Yao, Zheng Zhang, Jing Li, and Yequan Wang. 2023. 2x faster language model pre-training via masked structural growth. arXiv preprint arXiv:2305.02869 .\n\nLonghui Yu, Weisen Jiang, Han Shi, Jincheng Yu, Zhengying Liu, Yu Zhang, James T Kwok, Zhenguo Li, Adrian Weller, and Weiyang Liu. 2023. Metamath: Bootstrap your own mathematical questions for large language models. arXiv preprint arXiv:2309.12284 .\n\nZheng Yuan, Hongyi Yuan, Chuanqi Tan, Wei Wang, Songfang Huang, and Fei Huang. 2023. Rrhf: Rank responses to align language models with human feedback without tears. arXiv preprint arXiv:2304.05302 .\n\nRowan Zellers, Ari Holtzman, Yonatan Bisk, Ali Farhadi, and Yejin Choi. 2019. Hellaswag: Can a machine really finish your sentence? In Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics , pages 4791-4800.\n\nShengyu Zhang, Linfeng Dong, Xiaoya Li, Sen Zhang, Xiaofei Sun, Shuhe Wang, Jiwei Li, Runyi Hu, Tianwei Zhang, Fei Wu, et al. 2023. Instruction tuning for large language models: A survey. arXiv preprint arXiv:2308.10792 .\n\nWayne Xin Zhao, Kun Zhou, Junyi Li, Tianyi Tang, Xiaolei Wang, Yupeng Hou, Yingqian Min, Beichen Zhang, Junjie Zhang, Zican Dong, et al. 2023. A survey of large language models. arXiv preprint arXiv:2303.18223 .\n\nKun Zhou, Yutao Zhu, Zhipeng Chen, Wentong Chen, Wayne Xin Zhao, Xu Chen, Yankai Lin, Ji-Rong Wen, and Jiawei Han. 2023. Don't make your llm an evaluation benchmark cheater. arXiv preprint arXiv:2311.01964 .\n\nDaniel M Ziegler, Nisan Stiennon, Jeffrey Wu, Tom B Brown, Alec Radford, Dario Amodei, Paul Christiano, and Geoffrey Irving. 2019. Fine-tuning language models from human preferences. arXiv preprint arXiv:1909.08593 .", + "recall": 0.903954802259887, + "true_md": "Peihao Wang, Rameswar Panda, Lucas Torroba Hen- nigen, Philip Greengard, Leonid Karlinsky, Roge- rio Feris, David Daniel Cox, Zhangyang Wang, and Yoon Kim. 2023. Learning to grow pretrained mod- els for efficient transformer training. arXiv preprint arXiv:2303.00980.\n\nYizhong Wang, Yeganeh Kordi, Swaroop Mishra, Al- isa Liu, Noah A Smith, Daniel Khashabi, and Han- naneh Hajishirzi. 2022. Self-instruct: Aligning lan- guage model with self generated instructions. arXiv preprint arXiv:2212.10560.\n\nJason Wei, Maarten Bosma, Vincent Y Zhao, Kelvin Guu, Adams Wei Yu, Brian Lester, Nan Du, An- drew M Dai, and Quoc V Le. 2021. Finetuned lan- guage models are zero-shot learners. arXiv preprint arXiv:2109.01652.\n\nJason Wei, Yi Tay, Rishi Bommasani, Colin Raffel, Barret Zoph, Sebastian Borgeaud, Dani Yogatama, Maarten Bosma, Denny Zhou, Donald Metzler, et al. 2022a. Emergent abilities of large language models. arXiv preprint arXiv:2206.07682.\n\nJason Wei, Xuezhi Wang, Dale Schuurmans, Maarten Bosma, Fei Xia, Ed Chi, Quoc V Le, Denny Zhou, et al. 2022b. Chain-of-thought prompting elicits rea- soning in large language models. Advances in Neural Information Processing Systems, 35:24824-24837.\n\nThomas Wolf, Lysandre Debut, Victor Sanh, Julien Chaumond, Clement Delangue, Anthony Moi, Pier- ric Cistac, Tim Rault, Rémi Louf, Morgan Funtowicz, et al. 2019. Huggingface's transformers: State-of- the-art natural language processing. arXiv preprint arXiv:1910.03771.\n\nPrateek Yadav, Derek Tam, Leshem Choshen, Colin Raffel, and Mohit Bansal. 2023. Ties-merging: Re- solving interference when merging models. In Thirty- seventh Conference on Neural Information Process- ing Systems.\n\nChengrun Yang, Xuezhi Wang, Yifeng Lu, Hanxiao Liu, Quoc V Le, Denny Zhou, and Xinyun Chen. 2023. Large language models as optimizers. arXiv preprint arXiv:2309.03409.\n\nYiqun Yao, Zheng Zhang, Jing Li, and Yequan Wang. 2023. 2x faster language model pre-training via masked structural growth. arXiv preprint arXiv:2305.02869.\n\nLonghui Yu, Weisen Jiang, Han Shi, Jincheng Yu, Zhengying Liu, Yu Zhang, James T Kwok, Zhen- guo Li, Adrian Weller, and Weiyang Liu. 2023. Metamath: Bootstrap your own mathematical ques- tions for large language models. arXiv preprint arXiv:2309.12284.\n\nZheng Yuan, Hongyi Yuan, Chuanqi Tan, Wei Wang, Songfang Huang, and Fei Huang. 2023. Rrhf: Rank responses to align language models with human feedback without tears. arXiv preprint arXiv:2304.05302.\n\nRowan Zellers, Ari Holtzman, Yonatan Bisk, Ali Farhadi, and Yejin Choi. 2019. Hellaswag: Can a machine really finish your sentence? In Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics, pages 4791-4800.\n\nShengyu Zhang, Linfeng Dong, Xiaoya Li, Sen Zhang, Xiaofei Sun, Shuhe Wang, Jiwei Li, Runyi Hu, Tian- wei Zhang, Fei Wu, et al. 2023. Instruction tuning for large language models: A survey. arXiv preprint arXiv:2308.10792.\n\nWayne Xin Zhao, Kun Zhou, Junyi Li, Tianyi Tang, Xiaolei Wang, Yupeng Hou, Yingqian Min, Beichen Zhang, Junjie Zhang, Zican Dong, et al. 2023. A survey of large language models. arXiv preprint arXiv:2303.18223.\n\nKun Zhou, Yutao Zhu, Zhipeng Chen, Wentong Chen, Wayne Xin Zhao, Xu Chen, Yankai Lin, Ji-Rong Wen, and Jiawei Han. 2023. Don't make your llm an evaluation benchmark cheater. arXiv preprint arXiv:2311.01964.\n\nDaniel M Ziegler, Nisan Stiennon, Jeffrey Wu, Tom B Brown, Alec Radford, Dario Amodei, Paul Chris- tiano, and Geoffrey Irving. 2019. Fine-tuning lan- guage models from human preferences. arXiv preprint arXiv:1909.08593." + }, + { + "bleu": 0.8675070700101889, + "doc_id": "doc_2b12f031b9b42e6f54b76b7d4666df76b7c8410a676d63b0ad2afdb40388d623_page_000001.png", + "edit_distance": 0.07838179519595449, + "f1_score": 0.9010416666666667, + "meteor": 0.9293319118203978, + "precision": 0.9351351351351351, + "pred_md": "## A Contributions\n\nThe contributions of this study are as follows:\n\n- · Introduction of the SOLAR 10.7 BillionParameter Model : We have released the SOLAR 10.7B model, which is not only depthwise scaled but also continually pretrained. The availability of SOLAR 10.7B under the Apache 2.0 license permits commercial usage, enabling the integration of this advanced model into a diverse range of products and services. This bridges the gap between academic research and practical applications, fostering wider accessibility and utility in various fields.\n- · Superior Performance Across Diverse Benchmarks : SOLAR 10.7B excels in various benchmarks, outperforming established models like Llama 2 and Mistral 7B in reasoning, mathematics, and the MMLU framework.\n- · Advancement in Instruction-Following Capabilities : The introduction of SOLAR 10.7BInstruct, a variant fine-tuned for enhanced instruction-following abilities, marks a significant improvement in the model's ability to understand and execute complex instructions.\n\nDahyun Kim, Chanjun Park, Sanghoon Kim, and Wonsung Lee contributed equally to this paper. Sanghoon Kim led the Foundation Model part, with Dahyun Kim, Wonho Song, Yunsu Kim, and Hyeonwoo Kim. Chanjun Park led the Data and Evaluation (Data-Centric LLM) part, with Yungi Kim, Jihoo Kim, Changbae Ahn, Seonghoon Yang, Sukyung Lee, and Hyunbyung Park. Wonsung Lee led the Adaptation Modeling part, with Gyoungjin Gim, Hyeonju Lee, and Mikyoung Cha. Hwalsuk Lee performed the role of the overall project operation. All these individuals contributed to the creation of SOLAR 10.7B.\n\n## B Related Works and Background\n\n## B.1 Large Language Models\n\nFollowing the advent of context-based language models, various studies have revealed a 'scaling law' (Kaplan et al., 2020; Hernandez et al., 2021; Anil et al., 2023), demonstrating a positive correlation between the size of model and training data and model performance. This has led to the emergence of Large Language Models (LLMs). Unlike previous language models, LLMs possess the ability for In-context learning, including Zero-shot learning (Radford et al., 2019) and Few-shot learning (Brown et al., 2020), allowing them to perform new tasks without updating model weights. These capabilities of LLMs, not evident in smaller models, are referred to as Emergent abilities (Wei et al., 2022a).\n\n## B.2 Mixture of Experts\n\nIn the landscape of machine learning architectures, the Mixture of Experts (MoE) models like (Shazeer et al., 2017; Shen et al., 2019; Komatsuzaki et al., 2022) has gained attention for its capability to address the challenges posed by complex and heterogeneous data. MoE models offer notable benefits, including enhanced output diversity, allowing for the capture of intricate patterns within the input space. Moreover, their computational efficiency, especially when implemented in a sparse form, has made them valuable in scenarios where resource constraints are a consideration (Shazeer et al., 2017; Komatsuzaki et al., 2022).\n\nHowever, efficient implementation of MoE models poses a considerable challenge, primarily due to the intricacies associated with dynamic routing and load-imbalanced computation (Gale et al., 2023). Existing hardware and software for deep learning, such as TPUs and XLA compilers, often demand static knowledge of tensor shapes, making MoE implementation on TPU challenging.\n\nWhile GPU implementation offers more flexibility, sparse computation compatibility becomes a hurdle. Striking the right balance between fixing the size of each expert to facilitate efficient computation and maintaining model quality creates a tradeoff between information preservation and hardware efficiency. This tradeoff, in turn, necessitates careful consideration during hyperparameter tuning, adding a layer of complexity to the implementation of MoE models, potentially offsetting their advantages. Given the formidable challenges in MoE model implementation, it becomes almost inevitable for researchers and practitioners to resort to specialized tools and frameworks, such as Tutel (Hwang et al., 2023) or Megablocks (Gale et al., 2023).\n\nDeparting from the horizontal expansion characteristic of MoE models, the DUS method introduces model scaling in the vertical dimension. Notably, DUS does not introduce dynamism in the scaled model, which significantly reduces the com-", + "recall": 0.8693467336683417, + "true_md": "## A Contributions\n\nThe contributions of this study are as follows:\n\n• Introduction of the SOLAR 10.7 Billion- Parameter Model: We have released the SO- LAR 10.7B model, which is not only depth- wise scaled but also continually pretrained. The availability of SOLAR 10.7B under the Apache 2.0 license permits commercial us- age, enabling the integration of this advanced model into a diverse range of products and ser- vices. This bridges the gap between academic research and practical applications, fostering wider accessibility and utility in various fields.\n\n• Superior Performance Across Diverse Benchmarks: SOLAR 10.7B excels in var- ious benchmarks, outperforming established models like Llama 2 and Mistral 7B in reason- ing, mathematics, and the MMLU framework.\n\n• Advancement in Instruction-Following Ca- pabilities: The introduction of SOLAR 10.7B- Instruct, a variant fine-tuned for enhanced instruction-following abilities, marks a sig- nificant improvement in the model's ability to understand and execute complex instructions.\n\nDahyun Kim, Chanjun Park, Sanghoon Kim, and Wonsung Lee contributed equally to this pa- per. Sanghoon Kim led the Foundation Model part, with Dahyun Kim, Wonho Song, Yunsu Kim, and Hyeonwoo Kim. Chanjun Park led the Data and Evaluation (Data-Centric LLM) part, with Yungi Kim, Jihoo Kim, Changbae Ahn, Seonghoon Yang, Sukyung Lee, and Hyunbyung Park. Wonsung Lee led the Adaptation Modeling part, with Gyoungjin Gim, Hyeonju Lee, and Mikyoung Cha. Hwalsuk Lee performed the role of the overall project op- eration. All these individuals contributed to the creation of SOLAR 10.7B.\n\n## B Related Works and Background\n\n## B.1 Large Language Models\n\nFollowing the advent of context-based language models, various studies have revealed a 'scaling law' (Kaplan et al., 2020; Hernandez et al., 2021; Anil et al., 2023), demonstrating a positive corre- lation between the size of model and training data and model performance. This has led to the emer- gence of Large Language Models (LLMs). Un- like previous language models, LLMs possess the ability for In-context learning, including Zero-shot learning (Radford et al., 2019) and Few-shot learn- ing (Brown et al., 2020), allowing them to perform new tasks without updating model weights. These capabilities of LLMs, not evident in smaller mod- els, are referred to as Emergent abilities (Wei et al., 2022a).\n\n## B.2 Mixture of Experts\n\nIn the landscape of machine learning architectures, the Mixture of Experts (MoE) models like (Shazeer et al., 2017; Shen et al., 2019; Komatsuzaki et al., 2022) has gained attention for its capability to ad- dress the challenges posed by complex and hetero- geneous data. MoE models offer notable benefits, including enhanced output diversity, allowing for the capture of intricate patterns within the input space. Moreover, their computational efficiency, especially when implemented in a sparse form, has made them valuable in scenarios where resource constraints are a consideration (Shazeer et al., 2017; Komatsuzaki et al., 2022).\n\nHowever, efficient implementation of MoE mod- els poses a considerable challenge, primarily due to the intricacies associated with dynamic routing and load-imbalanced computation (Gale et al., 2023). Existing hardware and software for deep learning, such as TPUs and XLA compilers, often demand static knowledge of tensor shapes, making MoE implementation on TPU challenging.\n\nWhile GPU implementation offers more flexi- bility, sparse computation compatibility becomes a hurdle. Striking the right balance between fix- ing the size of each expert to facilitate efficient computation and maintaining model quality creates a tradeoff between information preservation and hardware efficiency. This tradeoff, in turn, necessi- tates careful consideration during hyperparameter tuning, adding a layer of complexity to the imple- mentation of MoE models, potentially offsetting their advantages. Given the formidable challenges in MoE model implementation, it becomes almost inevitable for researchers and practitioners to re- sort to specialized tools and frameworks, such as Tutel (Hwang et al., 2023) or Megablocks (Gale et al., 2023).\n\nDeparting from the horizontal expansion char- acteristic of MoE models, the DUS method intro- duces model scaling in the vertical dimension. No- tably, DUS does not introduce dynamism in the scaled model, which significantly reduces the com-" + }, + { + "bleu": 0.8691514227253168, + "doc_id": "doc_1c0d070f25babee3e3b5e2a842c54258f41939a7774c0e7c6d85fb7078e1196d_page_000001.png", + "edit_distance": 0.07542579075425791, + "f1_score": 0.9095674967234602, + "meteor": 0.9286705003594863, + "precision": 0.9559228650137741, + "pred_md": "plexity when compared to MoE. This shift in approach offers a unique and more straightforward way of working, moving away from conventional MoE challenges. Not only that, DUS also undergoes continued pretraining to quickly recover performance of the scaled model.\n\n## B.3 Prompt Engineering\n\nA key research area to harness the emergent abilities of LLMs is prompt engineering. Prompt engineering is the study of how to design inputs (prompts) that enable LLMs to better perform specific tasks. A prime example of this research is Chain-of-Thought (CoT) (Wei et al., 2022b), which proposes CoT prompting that decomposes multi-step problems into a series of intermediate reasoning steps. Moreover, efforts are underway to replace even such prompt engineering with LLMs (Yang et al., 2023).\n\n## B.4 Instruction Tuning\n\nTo enhance the steerability of LLMs, instruction tuning (Wei et al., 2021) has emerged as a learning technique. This involves fine-tuning LLMs using data formatted as (instruction, input, output) for various tasks (Wang et al., 2022). Instruction tuning allows for targeted adjustments, providing a more controlled and task-oriented improvement to the model's capabilities.\n\nBefore instruction tuning, existing methods faced challenges in effectively guiding and controlling the behavior of large language models (Zhang et al., 2023b). The sheer complexity of these models made it difficult to ensure precise and taskoriented responses. The need for a more targeted approach arose from the limitations of existing methods, leading to the development of instruction tuning. This targeted approach enables better control over the model's behavior, making it more suitable for specific tasks and improving its overall performance in alignment with user-defined objectives. Therefore, instruction tuning is computationally efficient and facilitates the rapid adaptation of LLMs to a specific domain without requiring extensive retraining or architectural changes.\n\n## B.5 Alignment Tuning\n\nLLM has been observed to generate sentences that may be perceived as linguistically incongruent by human readers since they learned not human intention, but only vast knowledge across various domains in the pretraining step (Ziegler et al., 2019).\n\nTo overcome this limitation and align with human intentions, previous research (Ziegler et al., 2019) have proposed Reinforcement Learning with Human Feedback (RLHF). RLHF operates by learning a reward model based on human preferences, employing reinforcement learning to guide the LLM towards prioritizing answers with the highest reward scores. This process enhances the safety, propriety, and overall quality of the generated responses. Despite demonstrating satisfactory performance, RLHF encounters challenges such as managing numerous hyperparameters and necessitating the incorporation of multiple models (policy, value, reward, and reference models).\n\nIn response to these challenges, the supervised fine-tuning based approaches have proposed, such as Rank Responses to align Human Feedback (RRHF) (Yuan et al., 2023), Reward rAnked FineTuning (RAFT) (Dong et al., 2023), and Direct Policy Optimization (DPO) (Intel, 2023). They avoid the complexities associated with reinforcement learning while achieving empirical performance comparable to RLHF. Among them, DPO that we used directly guides the LLM to increase the probability of positive responses and decrease the probability of negative responses through a \"direct\" approach. Interestingly, DPO demonstrates more stable learning results compared to RLHF, despite its simple training approach.\n\n## B.6 Data Contamination\n\nRecent researches (Zhou et al., 2023; Sainz et al., 2023; Golchin and Surdeanu, 2023; Deng et al., 2023) emphasize the need to measure whether a specific benchmark was used to train the large language models. There are three types of the data contamination: guideline, raw text and annotation (Sainz et al., 2023). Guideline contamination occurs when a model accesses detailed annotation guidelines for a dataset, providing advantages in specific tasks, and its impact should be considered, especially in zero and few-shot evaluations. Raw text contamination occurs when a model has access to the original text. Wikipedia is widely used as a pretraining data, but also as a source for creating new datasets. The caution is advised in the development of automatically annotated datasets sourced from the web. Annotation contamination occurs when the annotations of the specific benchmark are exposed during model training.", + "recall": 0.8675, + "true_md": "plexity when compared to MoE. This shift in ap- proach offers a unique and more straightforward way of working, moving away from conventional MoE challenges. Not only that, DUS also under- goes continued pretraining to quickly recover per- formance of the scaled model.\n\n## B.3 Prompt Engineering\n\nA key research area to harness the emergent abil- ities of LLMs is prompt engineering. Prompt en- gineering is the study of how to design inputs (prompts) that enable LLMs to better perform spe- cific tasks. A prime example of this research is Chain-of-Thought (CoT) (Wei et al., 2022b), which proposes CoT prompting that decomposes multi-step problems into a series of intermedi- ate reasoning steps. Moreover, efforts are under- way to replace even such prompt engineering with LLMs (Yang et al., 2023).\n\n## B.4 Instruction Tuning\n\nTo enhance the steerability of LLMs, instruction tuning (Wei et al., 2021) has emerged as a learning technique. This involves fine-tuning LLMs using data formatted as (instruction, input, output) for various tasks (Wang et al., 2022). Instruction tuning allows for targeted adjustments, providing a more controlled and task-oriented improvement to the model's capabilities.\n\nBefore instruction tuning, existing methods faced challenges in effectively guiding and control- ling the behavior of large language models (Zhang et al., 2023b). The sheer complexity of these mod- els made it difficult to ensure precise and task- oriented responses. The need for a more targeted approach arose from the limitations of existing methods, leading to the development of instruc- tion tuning. This targeted approach enables better control over the model's behavior, making it more suitable for specific tasks and improving its overall performance in alignment with user-defined objec- tives. Therefore, instruction tuning is computation- ally efficient and facilitates the rapid adaptation of LLMs to a specific domain without requiring extensive retraining or architectural changes.\n\n## B.5 Alignment Tuning\n\nLLM has been observed to generate sentences that may be perceived as linguistically incongruent by human readers since they learned not human inten- tion, but only vast knowledge across various do- mains in the pretraining step (Ziegler et al., 2019). To overcome this limitation and align with human intentions, previous research (Ziegler et al., 2019) have proposed Reinforcement Learning with Hu- man Feedback (RLHF). RLHF operates by learning a reward model based on human preferences, em- ploying reinforcement learning to guide the LLM towards prioritizing answers with the highest re- ward scores. This process enhances the safety, propriety, and overall quality of the generated re- sponses. Despite demonstrating satisfactory per- formance, RLHF encounters challenges such as managing numerous hyperparameters and necessi- tating the incorporation of multiple models (policy, value, reward, and reference models).\n\nIn response to these challenges, the supervised fine-tuning based approaches have proposed, such as Rank Responses to align Human Feedback (RRHF) (Yuan et al., 2023), Reward rAnked Fine- Tuning (RAFT) (Dong et al., 2023), and Direct Policy Optimization (DPO) (Intel, 2023). They avoid the complexities associated with reinforce- ment learning while achieving empirical perfor- mance comparable to RLHF. Among them, DPO that we used directly guides the LLM to increase the probability of positive responses and decrease the probability of negative responses through a \"di- rect\" approach. Interestingly, DPO demonstrates more stable learning results compared to RLHF, despite its simple training approach.\n\n## B.6 Data Contamination\n\nRecent researches (Zhou et al., 2023; Sainz et al., 2023; Golchin and Surdeanu, 2023; Deng et al., 2023) emphasize the need to measure whether a specific benchmark was used to train the large lan- guage models. There are three types of the data contamination: guideline, raw text and annota- tion (Sainz et al., 2023). Guideline contamination occurs when a model accesses detailed annotation guidelines for a dataset, providing advantages in specific tasks, and its impact should be considered, especially in zero and few-shot evaluations. Raw text contamination occurs when a model has ac- cess to the original text. Wikipedia is widely used as a pretraining data, but also as a source for cre- ating new datasets. The caution is advised in the development of automatically annotated datasets sourced from the web. Annotation contamina- tion occurs when the annotations of the specific benchmark are exposed during model training." + }, + { + "bleu": 0.70695618586421, + "doc_id": "doc_39f32d6a01b8e434ad2fb16ff3896931153cdf4f85bca09194b71e1514711355_page_000001.png", + "edit_distance": 0.10176991150442478, + "f1_score": 0.9112903225806452, + "meteor": 0.9576686915878817, + "precision": 0.889763779527559, + "pred_md": "## C Additional Information\n\nWe present additional information for the sake of space in the main paper.\n\nFiltered task names. We present task names we use to filter FLAN dervied datasets such as OpenOrca in Table 8.\n\n## Filtered Task Name\n\ntask228\\_arc\\_answer\\_generation\\_easy ai2\\_arcARCChallenge:1.0.0 ai2\\_arcARCEasy:1.0.0 task229\\_arc\\_answer\\_generation\\_hard hellaswag:1.1.0 task1389\\_hellaswag\\_completion cot\\_gsm8k cot\\_gsm8k\\_ii drop:2.0.0\n\nwinogrande:1.1.0\n\nTable 8: Task names that we use to filter data for FLAN derived datasets such as OpenOrca.\n\nTable 9: Data contamination test results for SOLAR 10.7B-Instruct. We show 'result < 0.1, %' values where a value higher than 0.9 indicates high probability of data contamination. HellaSwag and Winogrande datasets are not currently supported. We set SOLAR 10.7B as our reference model when performing the data contamination tests.\n\nResults on data contamination. To show the integrity of SOLAR 10.7B-Instruct, we also report the data contamination test (Shi et al., 2023) results in Table. 9. All four tested benchmark datasets yield results well below the contamination threshold, affirming the absence of data contamination in our model. One interesting point is that the value for GSM8K is noticeably higher than for other datasets, even without contamination. One potential reason for this is the stronger data similarity in math-related instruction datasets.", + "recall": 0.9338842975206612, + "true_md": "## C Additional Information\n\nWe present additional information for the sake of space in the main paper.\n\nFiltered task names. We present task names we use to filter FLAN dervied datasets such as OpenOrca in Table 8.\n\nTable 8: Task names that we use to filter data for FLAN derived datasets such as OpenOrca.\n\nTable 9: Data contamination test results for SOLAR 10.7B-Instruct. We show 'result < 0.1, %' values where a value higher than 0.9 indicates high probability of data contamination. HellaSwag and Winogrande datasets are not currently supported. We set SOLAR 10.7B as our reference model when performing the data contamina- tion tests.\n\nResults on data contamination. To show the in- tegrity of SOLAR 10.7B-Instruct, we also report the data contamination test (Shi et al., 2023) results in Table. 9. All four tested benchmark datasets yield results well below the contamination thresh- old, affirming the absence of data contamination in our model. One interesting point is that the value for GSM8K is noticeably higher than for other datasets, even without contamination. One potential reason for this is the stronger data similar- ity in math-related instruction datasets." + }, + { + "bleu": 0.661572934089486, + "doc_id": "doc_8d09961d139152e2bbe65e06bd7237b707dad351bd2ebc6fcbc3123f9ecd5086_page_000001.png", + "edit_distance": 0.1388888888888889, + "f1_score": 0.9795918367346937, + "meteor": 0.9784615892836497, + "precision": 0.96, + "pred_md": "## Contents\n\n- 1. Overview of OCR Pack\n- 2. Introduction of Product Services and Key Features\n\n6\n\n- 3. Product - Detail Specification\n- 4. Integration Policy\n\n5. FAQ", + "recall": 1.0, + "true_md": "## Contents\n\n1. Overview of OCR Pack \n\n2. Introduction of Product Services and Key Features\n\n3. Product - Detail Specification\n\n4. Integration Policy\n\n5. FAQ" + }, + { + "bleu": 0.8747982410187122, + "doc_id": "doc_8bfb767df3506b4bd76b31e8823f59714c7abfc2a34ab11018e66fa2d9b5ed29_page_000001.png", + "edit_distance": 0.20786516853932585, + "f1_score": 0.9565217391304348, + "meteor": 0.9881732427186057, + "precision": 0.9166666666666666, + "pred_md": "## Overview of OCR Pack\n\n## Base Model Performance Evaluation of Upstage OCR Pack\n\n## Upstage universal OCR model E2E performance evaluation 1\n\n## Upstage universal OCR model performance details: Document criteria\n\nScene (Photographed document image)\n\nDocument (Scanned document image)\n\n1 Performance based on universal model, additional performance improvement is possible by implementing specialized models according to business requirements\n\n- 2 A: Universal model of global leading AI company / B: Universal model of leading AI company in Korea , 2022. 5 Test criteria\n\n11\n\n3 Recall: Percentage of what the OCR model predicted to be True from those that were actually True 4 Precision: Percentage of what the OCR model classifies as True, which is actually True\n\n5 F1: Harmonic mean value of Recall and Precision\n\n6. Parsing-F1: Comparison of parsing model F1 of both companies for business registration document form. Company A is excluded from comparison due to the absence of the document parsing model.", + "recall": 1.0, + "true_md": "Overview of OCR Pack\n\n## Base Model Performance Evaluation of Upstage OCR Pack\n\n## Upstage universal OCR model E2E performance evaluation 1\n\n1 Performance based on universal model, additional performance improvement is possible by implementing specialized models according to business requirements\n\n2 A: Universal model of global leading AI company / B: Universal model of leading AI company in Korea, 2022. 5 Test criteria\n\n## Upstage universal OCR model performance details: Document criteria\n\n3 Recall: Percentage of what the OCR model predicted to be True from those that were actually True\n\n4 Precision: Percentage of what the OCR model classifies as True, which is actually True\n\n5 F1: Harmonic mean value of Recall and Precision\n\n6. Parsing-F1: Comparison of parsing model F1 of both companies for business registration document form. Company A is excluded from comparison due to the absence of the document parsing model." + }, + { + "bleu": 0.9306048591020997, + "doc_id": "doc_646f014ab615ac22266a43be2691010032d9fd62368737d488ffcb912cd68fd2_page_000001.png", + "edit_distance": 0.11764705882352941, + "f1_score": 0.9333333333333333, + "meteor": 0.9866959064327486, + "precision": 0.875, + "pred_md": "Introduction of product services and key features\n\n## Key Functions by Main Service Flow\n\nupstage'", + "recall": 1.0, + "true_md": "Introduction of product services and key features\n\n## Key Functions by Main Service Flow" + } + ], + "f1_score_stats": { + "bins": [ + 0.0, + 0.05, + 0.1, + 0.15000000000000002, + 0.2, + 0.25, + 0.30000000000000004, + 0.35000000000000003, + 0.4, + 0.45, + 0.5, + 0.55, + 0.6000000000000001, + 0.65, + 0.7000000000000001, + 0.75, + 0.8, + 0.8500000000000001, + 0.9, + 0.9500000000000001, + 1.0 + ], + "hist": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 1.0, + 2.0, + 38.0, + 158.0 + ], + "mean": 0.9748616502262734, + "median": 0.9885542131099482, + "std": 0.03333887558320575, + "total": 200 + }, + "meteor_stats": { + "bins": [ + 0.0, + 0.05, + 0.1, + 0.15000000000000002, + 0.2, + 0.25, + 0.30000000000000004, + 0.35000000000000003, + 0.4, + 0.45, + 0.5, + 0.55, + 0.6000000000000001, + 0.65, + 0.7000000000000001, + 0.75, + 0.8, + 0.8500000000000001, + 0.9, + 0.9500000000000001, + 1.0 + ], + "hist": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 1.0, + 2.0, + 0.0, + 3.0, + 11.0, + 18.0, + 164.0 + ], + "mean": 0.9689426968131607, + "median": 0.9895993251640107, + "std": 0.055870601320712664, + "total": 200 + }, + "precision_stats": { + "bins": [ + 0.0, + 0.05, + 0.1, + 0.15000000000000002, + 0.2, + 0.25, + 0.30000000000000004, + 0.35000000000000003, + 0.4, + 0.45, + 0.5, + 0.55, + 0.6000000000000001, + 0.65, + 0.7000000000000001, + 0.75, + 0.8, + 0.8500000000000001, + 0.9, + 0.9500000000000001, + 1.0 + ], + "hist": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 1.0, + 0.0, + 2.0, + 4.0, + 15.0, + 177.0 + ], + "mean": 0.9748618392096996, + "median": 0.9888571605934571, + "std": 0.043218000039770395, + "total": 200 + }, + "recall_stats": { + "bins": [ + 0.0, + 0.05, + 0.1, + 0.15000000000000002, + 0.2, + 0.25, + 0.30000000000000004, + 0.35000000000000003, + 0.4, + 0.45, + 0.5, + 0.55, + 0.6000000000000001, + 0.65, + 0.7000000000000001, + 0.75, + 0.8, + 0.8500000000000001, + 0.9, + 0.9500000000000001, + 1.0 + ], + "hist": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 11.0, + 26.0, + 163.0 + ], + "mean": 0.976191201264632, + "median": 0.9934769675335713, + "std": 0.03536853948670038, + "total": 200 + } +} \ No newline at end of file diff --git a/docs/evaluations/Docling-DPBench/evaluation_DPBench_markdown_text.txt b/docs/evaluations/Docling-DPBench/evaluation_DPBench_markdown_text.txt new file mode 100644 index 0000000..de5904d --- /dev/null +++ b/docs/evaluations/Docling-DPBench/evaluation_DPBench_markdown_text.txt @@ -0,0 +1,158 @@ +DPBench size: 200 + +DPBench markdown_text BLEU: mean=0.92 median=0.96 std=0.13 + +| BLEU | prob [%] | acc [%] | 1-acc [%] | total | +|----------------|------------|-----------|-------------|---------| +| (0.000, 0.050] | 0.5 | 0 | 100 | 1 | +| (0.050, 0.100] | 0 | 0.5 | 99.5 | 0 | +| (0.100, 0.150] | 0 | 0.5 | 99.5 | 0 | +| (0.150, 0.200] | 0.5 | 0.5 | 99.5 | 1 | +| (0.200, 0.250] | 0 | 1 | 99 | 0 | +| (0.250, 0.300] | 0 | 1 | 99 | 0 | +| (0.300, 0.350] | 0 | 1 | 99 | 0 | +| (0.350, 0.400] | 0 | 1 | 99 | 0 | +| (0.400, 0.450] | 0.5 | 1 | 99 | 1 | +| (0.450, 0.500] | 0.5 | 1.5 | 98.5 | 1 | +| (0.500, 0.550] | 0.5 | 2 | 98 | 1 | +| (0.550, 0.600] | 0.5 | 2.5 | 97.5 | 1 | +| (0.600, 0.650] | 0.5 | 3 | 97 | 1 | +| (0.650, 0.700] | 2 | 3.5 | 96.5 | 4 | +| (0.700, 0.750] | 0.5 | 5.5 | 94.5 | 1 | +| (0.750, 0.800] | 3 | 6 | 94 | 6 | +| (0.800, 0.850] | 3.5 | 9 | 91 | 7 | +| (0.850, 0.900] | 11 | 12.5 | 87.5 | 22 | +| (0.900, 0.950] | 21.5 | 23.5 | 76.5 | 43 | +| (0.950, 1.000] | 55 | 45 | 55 | 110 | + + +DPBench markdown_text F1: mean=0.97 median=0.99 std=0.03 + +| F1 | prob [%] | acc [%] | 1-acc [%] | total | +|----------------|------------|-----------|-------------|---------| +| (0.000, 0.050] | 0 | 0 | 100 | 0 | +| (0.050, 0.100] | 0 | 0 | 100 | 0 | +| (0.100, 0.150] | 0 | 0 | 100 | 0 | +| (0.150, 0.200] | 0 | 0 | 100 | 0 | +| (0.200, 0.250] | 0 | 0 | 100 | 0 | +| (0.250, 0.300] | 0 | 0 | 100 | 0 | +| (0.300, 0.350] | 0 | 0 | 100 | 0 | +| (0.350, 0.400] | 0 | 0 | 100 | 0 | +| (0.400, 0.450] | 0 | 0 | 100 | 0 | +| (0.450, 0.500] | 0 | 0 | 100 | 0 | +| (0.500, 0.550] | 0 | 0 | 100 | 0 | +| (0.550, 0.600] | 0 | 0 | 100 | 0 | +| (0.600, 0.650] | 0 | 0 | 100 | 0 | +| (0.650, 0.700] | 0 | 0 | 100 | 0 | +| (0.700, 0.750] | 0 | 0 | 100 | 0 | +| (0.750, 0.800] | 0.5 | 0 | 100 | 1 | +| (0.800, 0.850] | 0.5 | 0.5 | 99.5 | 1 | +| (0.850, 0.900] | 1 | 1 | 99 | 2 | +| (0.900, 0.950] | 19 | 2 | 98 | 38 | +| (0.950, 1.000] | 79 | 21 | 79 | 158 | + + +DPBench markdown_text precision: mean=0.97 median=0.99 std=0.04 + +| precision | prob [%] | acc [%] | 1-acc [%] | total | +|----------------|------------|-----------|-------------|---------| +| (0.000, 0.050] | 0 | 0 | 100 | 0 | +| (0.050, 0.100] | 0 | 0 | 100 | 0 | +| (0.100, 0.150] | 0 | 0 | 100 | 0 | +| (0.150, 0.200] | 0 | 0 | 100 | 0 | +| (0.200, 0.250] | 0 | 0 | 100 | 0 | +| (0.250, 0.300] | 0 | 0 | 100 | 0 | +| (0.300, 0.350] | 0 | 0 | 100 | 0 | +| (0.350, 0.400] | 0 | 0 | 100 | 0 | +| (0.400, 0.450] | 0 | 0 | 100 | 0 | +| (0.450, 0.500] | 0 | 0 | 100 | 0 | +| (0.500, 0.550] | 0 | 0 | 100 | 0 | +| (0.550, 0.600] | 0 | 0 | 100 | 0 | +| (0.600, 0.650] | 0.5 | 0 | 100 | 1 | +| (0.650, 0.700] | 0 | 0.5 | 99.5 | 0 | +| (0.700, 0.750] | 0.5 | 0.5 | 99.5 | 1 | +| (0.750, 0.800] | 0 | 1 | 99 | 0 | +| (0.800, 0.850] | 1 | 1 | 99 | 2 | +| (0.850, 0.900] | 2 | 2 | 98 | 4 | +| (0.900, 0.950] | 7.5 | 4 | 96 | 15 | +| (0.950, 1.000] | 88.5 | 11.5 | 88.5 | 177 | + + +DPBench markdown_text recall: mean=0.98 median=0.99 std=0.04 + +| recall | prob [%] | acc [%] | 1-acc [%] | total | +|----------------|------------|-----------|-------------|---------| +| (0.000, 0.050] | 0 | 0 | 100 | 0 | +| (0.050, 0.100] | 0 | 0 | 100 | 0 | +| (0.100, 0.150] | 0 | 0 | 100 | 0 | +| (0.150, 0.200] | 0 | 0 | 100 | 0 | +| (0.200, 0.250] | 0 | 0 | 100 | 0 | +| (0.250, 0.300] | 0 | 0 | 100 | 0 | +| (0.300, 0.350] | 0 | 0 | 100 | 0 | +| (0.350, 0.400] | 0 | 0 | 100 | 0 | +| (0.400, 0.450] | 0 | 0 | 100 | 0 | +| (0.450, 0.500] | 0 | 0 | 100 | 0 | +| (0.500, 0.550] | 0 | 0 | 100 | 0 | +| (0.550, 0.600] | 0 | 0 | 100 | 0 | +| (0.600, 0.650] | 0 | 0 | 100 | 0 | +| (0.650, 0.700] | 0 | 0 | 100 | 0 | +| (0.700, 0.750] | 0 | 0 | 100 | 0 | +| (0.750, 0.800] | 0 | 0 | 100 | 0 | +| (0.800, 0.850] | 0 | 0 | 100 | 0 | +| (0.850, 0.900] | 5.5 | 0 | 100 | 11 | +| (0.900, 0.950] | 13 | 5.5 | 94.5 | 26 | +| (0.950, 1.000] | 81.5 | 18.5 | 81.5 | 163 | + + +DPBench markdown_text edit_distance: mean=0.07 median=0.03 std=0.10 + +| edit_distance | prob [%] | acc [%] | 1-acc [%] | total | +|-----------------|------------|-----------|-------------|---------| +| (0.000, 0.050] | 62.5 | 0 | 100 | 125 | +| (0.050, 0.100] | 15 | 62.5 | 37.5 | 30 | +| (0.100, 0.150] | 8.5 | 77.5 | 22.5 | 17 | +| (0.150, 0.200] | 2.5 | 86 | 14 | 5 | +| (0.200, 0.250] | 4 | 88.5 | 11.5 | 8 | +| (0.250, 0.300] | 2 | 92.5 | 7.5 | 4 | +| (0.300, 0.350] | 1.5 | 94.5 | 5.5 | 3 | +| (0.350, 0.400] | 1 | 96 | 4 | 2 | +| (0.400, 0.450] | 1 | 97 | 3 | 2 | +| (0.450, 0.500] | 2 | 98 | 2 | 4 | +| (0.500, 0.550] | 0 | 100 | 0 | 0 | +| (0.550, 0.600] | 0 | 100 | 0 | 0 | +| (0.600, 0.650] | 0 | 100 | 0 | 0 | +| (0.650, 0.700] | 0 | 100 | 0 | 0 | +| (0.700, 0.750] | 0 | 100 | 0 | 0 | +| (0.750, 0.800] | 0 | 100 | 0 | 0 | +| (0.800, 0.850] | 0 | 100 | 0 | 0 | +| (0.850, 0.900] | 0 | 100 | 0 | 0 | +| (0.900, 0.950] | 0 | 100 | 0 | 0 | +| (0.950, 1.000] | 0 | 100 | 0 | 0 | + + +DPBench markdown_text meteor: mean=0.97 median=0.99 std=0.06 + +| meteor | prob [%] | acc [%] | 1-acc [%] | total | +|----------------|------------|-----------|-------------|---------| +| (0.000, 0.050] | 0 | 0 | 100 | 0 | +| (0.050, 0.100] | 0 | 0 | 100 | 0 | +| (0.100, 0.150] | 0 | 0 | 100 | 0 | +| (0.150, 0.200] | 0 | 0 | 100 | 0 | +| (0.200, 0.250] | 0 | 0 | 100 | 0 | +| (0.250, 0.300] | 0 | 0 | 100 | 0 | +| (0.300, 0.350] | 0 | 0 | 100 | 0 | +| (0.350, 0.400] | 0 | 0 | 100 | 0 | +| (0.400, 0.450] | 0 | 0 | 100 | 0 | +| (0.450, 0.500] | 0 | 0 | 100 | 0 | +| (0.500, 0.550] | 0 | 0 | 100 | 0 | +| (0.550, 0.600] | 0.5 | 0 | 100 | 1 | +| (0.600, 0.650] | 0 | 0.5 | 99.5 | 0 | +| (0.650, 0.700] | 0.5 | 0.5 | 99.5 | 1 | +| (0.700, 0.750] | 1 | 1 | 99 | 2 | +| (0.750, 0.800] | 0 | 2 | 98 | 0 | +| (0.800, 0.850] | 1.5 | 2 | 98 | 3 | +| (0.850, 0.900] | 5.5 | 3.5 | 96.5 | 11 | +| (0.900, 0.950] | 9 | 9 | 91 | 18 | +| (0.950, 1.000] | 82 | 18 | 82 | 164 | + + diff --git a/docs/evaluations/Docling-DPBench/evaluation_DPBench_markdown_text_BLEU.png b/docs/evaluations/Docling-DPBench/evaluation_DPBench_markdown_text_BLEU.png new file mode 100644 index 0000000000000000000000000000000000000000..20ebecbea3981eb809053a1bb221534628384468 GIT binary patch literal 18878 zcmdUX2UL|=m*qpstXO7If&oQPK$2uI>k|aY8B_!W2};gjSxSirNJf%@oFzySFoI+x zN(M=ik(`-*?Z5u+$un#9n(1DvN(J8g-uK;m&pG?-y$_GBN?+W(VebYCg|eA?>6{#e zvdV=*q3Kw+2H)ZT+R}}`ge}giSjd~|Sy*2;)1_RwZgI!R)WXQ%#(pbZGjjt|6JAd4 z6P(<~_usOxxMMEN#bx}@S8$q|>2sNzF8AOn>+f7rHK$OvTqpn2#7IONP$*P(>bcVj zHX*~Uwkp1LD=R;D|FLQRA0izWKWS2VWX<*_0MMw;{Y4`hXV4n7n!CrynN59 z{S=DJ=iezb6w2|3|KZ)p;pR7}KG*KFU>vuMYYwy*tQZ#=wPf9mM~rSbQ!uBn0e6^5HKGOmAoxQVO4qF+ui zSxb|NiHTh;J;rWoz;D1-$aXxU&2EsRE>2Mrx1sR#;Q89DyE=|Dt=2(Hcf1}JtSqaC z3R}wETeZ5otLv53a8v)c*E;lSCl4GrZJIabU*gMlD7ZX|N4MnG(6`r@rKF_NR+2Px zCuh4nVj_LtidA>52ewzJjL`|vOT2%GFP7Q zeZ~8D(ubypo9i-7wd-PJ&)QD(UYXi*&MjU!IZioQD^4!b&B4*JXLvYqEtAbD^4Mq2 zd=P68H)#<}RLd9$3y`?e;d0^0-pK8&(k&BvRzA&5^i5TWjK<*k$s=#rarC(>vy)1= zWGbhIW+&{ zm1k|5zKp)8LbRa6Y|41|+|1(W-rc*Sk6d|n^vID5$$8Vy#(T@-XWHy#p6unX4imFi z$hyo#_&r|CV>s|e{&|2;4yB%wZ0{Wdn#uE$SVA>z{3@@M>#PxcAa zSA>e-Vdq5Z7v8UBjxnr@W0#L|_h1n6)+@la8m=ra;d_3U{n#T#ZO3n6Klbw7@_J+3 z9CcXIzeRqjtMEyUkBoRMPDOjo_*y!KDs|Jm$igT4^h=fACaPx+Of_gLlZ`0cXZoeQ z{ED{2&j-TRBSy)?di0LY&1w3c&-iagb#z=?uy5=x_Sv~>S8vMcj~_ptJ#gT_Gag+x zExZ0m3tCa0d+!VC7w0CPuuL?Y76_$lXWe<!IL%#?5caFY%1No$$yT z9;l0tGHOhXk`5Kl)I2FJ-nQfD)pvYG->O(xS(}>+<_C@)DwYZtZ*7|{Vs?(OOF|^* zA-->3_C5NP>RC7^r{%h$heOThU514iHeS1hi7zQB=?EX+ZR8ufNur_mu3^DQj&oapzuIm=W+U3*yxuNg7+> z($04{!ot!W=FeXks9Wk!Pu0V5oe{lo>C#{8H*Cm>*7al*(UT4l?5y$+2{C$h(%8`G zWbmWMkJB>S%NNwK%U4dFa(VRV(d*3eV1dd@JcIR#=09?Eis%j%BisE9W1{8hZJ#jZ z<4HJmZxynbZ*X&gGwn#6V!T3}0-GO)di{&Mi=Is5)6+^gt4-+!hfGaPm--}DzTy@S zQx~R(<#9uPT-xHRSFetoYqznmNMA$85cBTcsYaude#exhv6>2IW~pZjihWq)O7Hig1qLyoA9GCA($0%oGGt0=lGTu|# zGcXX>mS^AGEGjBm8?Ve|+?u1vDibDU-6*jQ7vsbt znxxG$>ku20-*h~S8yb|gw6qLsW1c$A^#mv^oiuGraAmT)MAEzZ8>5_D7Zb5HV;>%D z+$(GmGg7ePWNU9#Q(UY?6S z+kQ+r=||Ozv;9n~?xrC}5590^sy@3#?51hFLaa9lE4*{@P3iLDM5VXyz}R01_0m0E zDb2Qy7M zMs17&!^6X|dA#FeHNL5J9EnADSy4T69OhDYacbNOj}O+4&$S*^!ny0|>LN#lJY>Q0 zeEiZfir$ad=rC^<6y8tbgbCIO9PHr2X{i(@}Qz%Y7B0HOos2Q{NwRv2$`tYvx+zMgR2n_TI5` zr{91T)=VmCv5=d&E?!wC)5D)bo&Cg#t3F4r$i>P<))qKBbEu@8@)Vm6BpDnPg57y} zaiA?PmqfYtbcx?Faz=J>s72sbZqHBlqm+i7IeV7hX6%~lloU$T7V+8V7d)8~fhLNP zLS@9JeDmjegBAFV+uo{@s)YN)>agIxt}u#N)go~)&aYm(HXdol+F_Bb?1K$E8QLaU zVwDoryiJC7QDsWjYN|2og@uJR<=M|L(w;hZ?oU!=dwML3)M_RsDkW}z@j8Gy_7$1a zVyxqIhC$5*>~ig7e+{YrdE+I=nsO}T!|X?dTy+6L?D>tpUB%k&6*7xJ_K(J4;Wutl zXwEdvoWURAOpcSXpH5)vl_YZN zbCU6t!giBCj6Yi2Ro*R`$T07dwr{p>cVxwmFD=cq$Em!2DK?Zp`~5MFL6h;IR3)Br zvR>UZmg;lFZe?j^YJM;|{${cFbk{I8TR8>_aQlD-=V*)fMR%f!TmSNbBZSEhb&Qoqk)>c+f@Rbe| zjYXC-9PcjPuwjE-wpq`JU4o4G+_{s%S2pkEOQ}>07k6Z#F3gN71E8c^k7{k+cTx&9 zr0Qo^Q5^svs@E0o+LY^OH}5%d%6qcuc5-9UIyyRki$nYE?Co!lb-YJXe}qr56l%ZNP z6f%lCWZkv2O}(JyUhl+=qGr1JogSE{KgsewGo_42%?Lcq=Jz#OFHJ2ZA<)82_ zV-0usUL~$y!EmTpAy)2ln^K0MVpmU(F5tgE2#akFC?l`}Pi;K&AxQX9hT`ku|z71WkdPU4cUuE!(&u&3$Ht!Wo*PmSC z=;2k0##ksQ13T<-q}QdFLK&Tgj>q*37zdx^U0V<0>aw#ZmCyF7>Tiy&(plIFBq= z7c0+dZ~xhCry_8ZtL7F|7SNwi2+)6R9rz3#WJT9K=eGx-6(|o*FXHZsqR^rUimX226rt07ph#5 z2cw^a^RmJ){vnUbk*;6K^x(mR{wD%$moiposHW-7cl*frhJ+l~aYyxwM*jPiqQr5u z3~^AgW#-NECSmOn5iI8~UPMt-CtdJF*;lj^C9SQk%l3GzAxq_0x%1Ue(t3Eu8KTQp z^2CeKn#oh+M0ksbR-`nu*f#x;Y<1)Kr`0MO49Nk2L2Mr@Wtq`{t2-cJA9Zct;zYMO3X_Jc%V#+&Xk% zA?(-mCbM&D+No-l1|@xK^l9`NS6!p?WtCwf^b8>$R8+n(w)2gA4qa}}H|ufBld(fW zVcESqz(PW$Cil3M-LSyg;_mL7q(9$#(n#f*qx_pdf|TtRe-_oDu9E(khK8mjSRlE# zGCaKZ(P4Vknbvjd)?Jzi4EDX>Il7|)Tt|l8l;Hzc*TGMZwv#%JFdC>&yj^giAyLbw zbG75rRD;@Eo7bpdb~2XjLt5d{zFphfZTlD%$OdJB z-0TW5Ucd#fP+OvG-EZoN7XiI_m+MJRbU$<#F9K|+XLeqELA#dj2h!EFk1;Z91Xz{E zg`Hel8c3j-u6{d^@_Z~zY2JWY0ThZM2 zTGGNdXgh1-V7_54m@)kwA&N^DqaQ)2D>bI-lC}UxpdW^ls>ePtUu74wusfxCChfS zbP7mKuC3kuxUajj7fo^iV9)F6kBk z+-dihTa3~yHHbqnQ&f?etF3wKsR=ze^(ADP!DARu5VJU796 zTxK-SL{{>4zS}MiWaebPzFu^DB|fa4OO7Y==9bYOS}ZTxCaSzXgU?9M)c?E?&$5ug z3=$?rS=*7nTpT+R4^FP7r6nD$yX!}LK&m7$`)R@SNI|E?tflVVH#b+Ts<^jp+qQJ4 zCA;k0T>VDoOW?S=M@Etw)AXa~E61Hz7Q(k}-(HuhTPos#oDs?=!LSkw%xi#6QOh*Z zj9Ym5>eVn!V_lshVOeum%Y+F2^h~Pd(p}^^0Z1ExgvP-6V;?hfG8&V3APu@sVNp>H zH0n8BfdE_}C2OCbzR%D^0^>@Us4Ahuw9qu?+VH8OU)W1JydyGUA-xjp*j^utE+JCy z0P=fIM1yIOnwZP;BOol%8s$B=FO4xWH+NGRInp3%%R4QdX2-5O0@Y7T+C_Z%GT4ir zbT9XfKUb|<#c;-<#I-x|^bF#{D>zvFVL?!zUM5p;KPXIe^+HB|9Ho<>l8)peT5ElIzF}N z(BIFMm$_24x2|+;>H07z<;m5=>Za37+%ZNo*Rb3@*Bv`&0kQBJoBL(K} z)?DibPl@FpV{1saxb(A#IZiY84LSCPb?k}S1zI{q9+p`nEoj2frk()pq=m8*8`e$` zeg5DWvNyYCPC8Ip>g0mj1Y#ek;1q}$1$<5&nA=~>6#;Z$j7d(^cAWBy*q8gPK1nkk z**C_tJ)fLncY0n06fqHT89e#dA75XQ_tqt<6Kqq1T+K-Lra&FturdlRpIP@85hvOU z?fH)Urfs4bCar3u#cg|Q%a1mAZf?#14LF@+ja{q7KrI#7v;mxd^tW#+jyN*HRzrcH z0QR{ix)S_Cx^VJs+(Wu&no%RqgNF}=hb~BFzb!?cG}*UlGqL?OQm0 zblPl)tw(|c>N2nLfwA#sI8z5Spj)7DM(TbWH8Hl&W0Xr$kI+FKT+Bf;H; zE($3l^rX?Z=g6tT4B~eMoO$^TYpc+lN3+RC#emWwJ7L(6l!y~)h|EgjVSak}7_cbl ze&wG(e?CJ?<2Z-BS&WoJXhJC|cd(ZzvT;BwalnIwx4?4|@(c-x<)bU-pAR1zc7Avu z3<1En;QCGM#{d|Z4ZArL0Kp^Y>1OcGrj<7%y-UNxne9r58sOkVQzNbF$avMcqxta{ znH?1fiF39+RK$7!XAk$N`H*Sf$kPcK(F+$ME$R1woRdfM%!?f1DtUX4PIcbf$1J>h zWqWkWhPh~c>PxR9s(O*OPhtajPX?ERD=mIIRj*DLz>pO>J8L5kUVdR-{?9-E40MD{AiBGXV8;Upg-7@lS1xVTMMAy;6~Nu_hFSe2}&VzW&!2v3NAZ5RT(ZJ zQEnP|RXE^l_EhR2Hb-7F`aL=(8uZRUoandOzZspJs5-kP(w6P9Z(pw-HY_6+c$lZB*Z+`iG`|rPBLpO9K z&u*#=EzK1qdxo%AlLL~2!N+*)=mU3vtK}ml1iQ`+u!O^W^2qVy#}y0GwFC&=4(W;q zd}Ipta$0TagoP!?gHcpT( z6sKQz0BNx77XqH~2@Jdi(c!Q>gfFx{-^Uz6B%Is88kEn@&hF9H)~-oZ7f1uM=4?Th z)uCB7Su5|?gQIR6pwh>gp5y5)yxmz}8X6&LZP02Gwm`~|D7xJ!P)dbhP_Vf{h_Yau z_&~@|uH2)&rlht<0j(9*)bQ5nv}N25Ane1>>FJYTk)r@m^ibSl3j+mA!|*ouC7w%c z&IJXFar+^Kz9JoC$X?%ZI5(_+;# zlYWkkojCt^7yr`ylzEn~f*Zg>o$ql~o;rO7o`A>PGd30?UZ-^m-M>O`fgq#TI7+)L z5X67mAH7=?;9!*S3rhA`w2eD52bpHBo=1XWH`fH*|18q@Iu$4kt%-y2{G4L=TaZ9I2bmE_s*)1XB zj!m7p8Z?ws6?XwV{f^2W%q{)03kvW49(!S#Fwvj^5nCF{y8?OK+$vgH#(dLe!*wnz zr%Ft;!5k79T*%#TOM7ashbeApwX z(K89Pn7V%@XvD>ZVryp?2W|!Zu=x-3&JU}>3ZEAj7blDar1MCa09eK0>2Mi(dWg0I zAb;u|?Uq}pLk4XU;L5Lri)%vZ|B5o?3o;Dg=?wCD+Dw0S{5qTx>2JXQDrB@b=)Bt#t~=)72k)&5xwL1#6%5bu-a^MeMV86tE6Lv zcoPK>ci9`XiBNCMKk#%#%?o^knA3r589rCHTazrBJ@uWph&W2&RG-`OOsBwrs>3*Bep8^_G{H zy#bdmxlZGc1xI`-Y(JgwaS23m&YjLba6J@V2L}g+f8GIVlz%}%!AX;rTc(enJRwS$ zLbNm^RSrTB;e1dyn-&}Z#L+q_VZlPRc=7zaJzvsXkZC8;1vNQKpf8yoYJ7F(!G>65 z%e{hko)>yB#sfAIZJ)q)x7}Px;Jzfi|G0mDieSeFyS1u9gv@=tyq-g(i3f@aq1i;C znB_6hQwX$$$o|clS=1(lg^h*f9E<@~Dc3(*%#G`k@(vAGb36MEhYg?sFob==z= zCa$dh?YA#AHS!=1l~q(!PAYAtP`swhj42_4cVFsNh9yEO(nh~n3t&s+GQtzK=~qWk zpNTu>ATf}*0_*QUnoGjtKk=hQ$a5g%_s7E=a2_XC9zE}2D!@W@bhuL-WE07p*Hfkt zgYCMW2-GHNicqUTPaQ`9qB>s2d9QfW1lLrmTvF>Q71U+pmMqze9t=-ehFf!$g9S{9 z;(3@_g@3)%*u<9&U7`URs(T}7S^@M8F7JRpzpS|fM$7zptOv>_Qci?`un zBscEntcB+y<4)%XdnR5!J_T-W?&)SQalEf@;1u+I$xx?g+xG0-%q-sGbi5tHSu!+P zNnBVNK7z%irNh*xPoHjBw=RWiH$8nU@KmHwRJoo{xnAk`KOfr$+;LBF*X)>vrludr zYbx13yf@=+*M)^8qQ<^44#o<0)v95oLfG(>MrcCXV6pN$t|~{^E}%=&6N$~Vn0tf5 z+r61V@G3ewMZjIf91DXdd$`Y_KO#+5TBZ$_>bUQ1+79$$d^bL=n;LFb20=?|NL{oH za~^clX>qg_HK+)6xz?uOjEK?=??56v0;#Xf@csn9+qBnU*y6VwP#^?StN6bL`|#Tj zSejJTB(Yy1Zt(Al0sh|yS7ATR82n-tT8HseCf_G1DiG8Nhx&LIM>Wo2NS+;jP-<>R zX#paQQ%a0`e}8Rhd%HFaV_%VlLp5QNbkP+OqbS8IT>-qS0ScS?dXc#wJ`a*u30(}l z`U9MO1?PHXN4eBtmfz9Lfx$nG(_e#NoI>U&boho%n-pe7S}hTaL+l9N{mP|pC|O{x z$R>h|C4L~N6mfu{q%#zT4%LLa(u~Bx={7W}(CP%8Kv+LWi7E&{5{n6_w`?+D`&83W zKGagK-Gh^k~M!GqS< zp1HZXU3y2sPk5?Bk}P#>XKZD>zY} z-50Y*eH^qCxHRuc1ma*%fVL9Iu)i;{bi9LGr|<_B6DEV8mKKu)aUmg9 z@FIPCPT{^-@7Pf+tgRdL!yTjWx?50QK!Vnih<^TjzkUFmpVz3;TU^+H6rO`9zu7Q= zdp=&;Pjdn}hLlW{JxdTrhgn%gT2R5;MRHkiT}eEz1Kcec*WRxp42EKy!qdla5lP}_ zDSUWhK!O2e9pV5B$uq=vkM857g_4*=*fskMd}DS659zvz{-r@pRPmm!Aw^`XQ;ok< z1YnmzRFlk0NC|j4I_{K8Ij<~bLbH!U|J}?bkcP9duvEB*L^aZo7rG!M(IkjbQi-_# zj`qU-BFe6#S5L`^{ygWO+=tvpG;I(FFOfHu&;u!|s041{gha-sq-AU{$E z!Gz#ay&1vW?W+tQ^_`Lm5tO}m?;hRYF(k7k{r@=(oK+LX8*d9a?oMGj zeB{Ugn1mW4U!gOhRwGn)LXj{SoE1P7~ji(+Y&-om;oQfZ*BO z47ouON+AFte>rzoSJy9f1grJM0Qa>5Il2|A16CpwoMALj5H;uZXA^`~(3>!1O9LEr zc4*o}I^3=8AWOSuQx#V6F!i6fv1f5{acHL1aB=qM_|#NJJ31iVL@jKQ{y%n^+#=+B z*|4y%>1oz5n=|_aOv=)y;E{_5;wDF(+&Dp1TyMwzKopx->O6e-kp0M!LMIX1@taX~ zz`GsCr6tykA>!+ydVyyisnUXDjJPszpMdqPpn~X(LC?*@ZNozS8-*e|zb%}*a3Kz6 z3Ih*=w8oohjXa64GrJ6ROmess7Z*dfy2BfB59^MP&fp0H{K}tfK@wVvt z^LJ-wXV*lJe=Ryb29|if#b$({IAn-t!j^GM)~GNbPz}1jJPn$kx&zI$8f7mI)M!8C z?J8CeITgU-K?IAvwHvMlY*NIWeZe}DA;wJ2< z+UO~I_PUV#XwKa@7HR}Uu9jmF>$I|DjT?`}UJRg%`A9t=bQ+B;j?K>P+Ydp$9wJF4 zx)~`GzK{beUa_fj0j248yACZYeEM*&Dcd}nM2A7wWd|q8#+21py}_W?q6iGB@RK}+ zB=3X zlB+wdb=)gLgkBK{H#JaaZY6{__;*cGYz8xpVD^XsVQtuy9@~bc)(^nSUlT$)!>Ra3 zWgAZx`cboZ?S#ayX^rF6g{d!^!V(G`BF|)L^oK2Fz zanKJymAkecS0wEykO3gm88{BERzG=sc4nwio~Y7Z2hN6QZo@)FlKOyCtPG%(iMEX$ z0yKJ ztSqr$hySh$PMbQ=@bFpuxGDz!>*Y(AvSj}Z4klf%b(?KbBN;VN@u;DF4n~U5oA~BM zC=q@v@MS6iG?`nm;T8VA#=HLzH}CzmuIR6h0s8V3-c&p5K&6Ue`gbdwYzDva20EZ_$DMQ2PmDABk4gUQ&MHJ{wT7b;*(9Ri}V67(j zvGniz%cN-KJ2YM|)pkSDg@;qiIolo?aLQ@FMDP1l6ZDg1Y_hoo!z&igZnG z4*FabbWOtQ54RU+@8;B~!g2Zj_Z_HHV1;~n=%NRFWAh|6a%$%%z_54=FoUQ{f0t$x z8mUW{s&qWXq9Ba^fRu0D8;A2ZwjyM?@fs-kDQJNr=6+bUtOqsLG}gmSip);4FVd7r z1HV$)&w7sq$^Dh5cr_uiiFa+Dt^)^5@E=uDjUo!ho|veK?-V}402W3evrjKPNwF-! zC#4*ZUi}W6FbLQLx2#XZKJBHG2m=D>;}L0;FOP~SdvzF`R;HUxEz!H`C!*D)tc2Id ztfJn%y&o@eF&E^Zjj3DxXNc>1D6Q)dDG&uj2(Q4G(uzZ-1RNZE>va&f=Q*IkF>0iEmK+9`6 z6klX@w72hkknq2}kMj7-ztC7g!g?R(e1e3TLhk=pWS0Lrm+eQSys)%z6yGw1TkjdE zh$wc%6N8Rk;pWYo(>no1J6N*CDa%;VXCl@~0Jj5G;&4ehPSq)^y)g6a*WtU(B?^4fRcl?c-W!A<`CAFGFl(K8XiNE#GkcqC#5XwwYc(p^h15jLuDW9{lo z>u6H63!DsFb20!>Mym4RNIcBKBABimr$B{~zLvCmj9mY9yQj5A4~2jrTqg*33NxUG zzO=S#f(*KIYi43X{@l59IXWxYMxa`#v(0g-uys3!SDze(%o4OeUtUJfc6T5 zL_7rJMp+&lhMMjYN}z0K3wL`iywx)Z882W`vVoKK$Qt(#oF0SIa6N^B9YEJAlt2XBU|W!iQ^je}60_ zI78^Ur2ghrB3bc4A>z=?QGobEC>d0;!x9n&MhOg*YnI@+OM10AYYHU=oq1{s&y0B8 zmRU6i2?+Q|f&?D%c*E8(iP4(@6VOq#+L8xN&1B(}z_k2I0^xnmxVe z-40BJG{D!BNT~F~mwi(*7Zw%4LnN=X{AFu7@=i?hlqfD?nbp?=Z3B7k$~U==~{>IV>1&zEhr@fxOZ(!njkL9d4X zIFq@dZ*cIKjEv0OJn<6$4+5M{pF)d5z|pYg9n^nGZ}bpDRpRL2$)JJvWSjjcdUsJ` zQ~UVfIiA}rO=BPK&3Xt9+!t{B8luUNjzNfbv>#|-qQyU4(zjCZ!v6$wivQ=qUC5D( zCEg#se#;hk#}%SH?P2ehV%~mfBPpY~zoWXk`Tb!zJm5FPxjIXq1mDORn9P$j^Im>1 zet6Wh^PLpq!oX`rw>ewcD3A3K4-m9pGE#3S8<&_gp~&h^a0SHbu!cwk#dgv3lZ<0_ z*s5+-KOwiwA`_YvOHI_e*WkLRlKdNU4I8gKZgCmJ&;AJqzOCLw`Qnw3GkX{Fi3+JV zi=~)%pPENKcM&Nm;WZQ!&9j)TI(=q;$2nZ&9iwg?ea64lr`8E>#glklSH1gFhf+~Q z-H)~T-wvO?UevutjKT!nqKCEXxVM&1r*IR@VaIYyy+0|=nEET4ALuEY`|>DtK*-(G zGXT8gszwV6k`|%MzF*;%~1p7ErV4rEZ|_1dR$lA=}Mq$y%OqJSvR<3LCdDd>$!>$6@SWJe6QWUerw$D_JE zc~U+(54n|vTD<<}<$NcOn!+p6S+XlgNW+6)q)6>FGwU^RBx8sXW#Nx~xg!&TJQD&R zCoqY`7naBMRk3JQdXq(}h!GP1UMa;W(z!^b3Z?ayF$o+is(8w;EqJ)Uuqw6O|S3 zMMc(53^A0YncwUY-S@Z7D=J@9PCdSl0^#(ava&M#RKn>OFI*^r#k^dX*tOFJA@_SJ z9=_}i*~;er-BrDfLV@~y*TO>Y+gf=fJk!_KZkJ-_WqmzlNr7LQjGBS4|JsHTxdix? zg@?AS0T4Gno{QFa%@(EvFR$OB6Uk>7)>YTW$`e$+p!yz1H~VQMZ3jH%iD>frVYO92 zQimiMg_>4US;>_v35L7|4QATV!St+qwpMii<~}Oa{KYoz0pN){Zodg};G+0H%x8K& z$bR5f<$&<6g3HfkijcoryGgbPx`{HG6l`P_(JQeSy+G%+f{C_&m$fKaER zzBK|Z!?%S$!<0khRj|4G#`m&6wN{v87>^)kurX*H6l1R5oAKO~)odFEdIYucC^Gu$ zHERs3o}bRK;xI2T7s#B0S-$p-u{trJqGba|Q^y!E2QY@hX8DhSKCp;i#)Zu}Xrt9Z zRuB^lv*YB`#scR8+s+5Or7*xO1zi4Y$$744ch|sxA!NIobPR&m+HATu`v{n7rJ1&i zvnq#8veoXX*Nbx#h3*!@NkfRRWl6DL-*Ib^N;WcGwLB5N5{SnRgy~Qy z`WwTLNuLmgFlsT}0B2Z0TY=xYL9_(nj?!kixtB07kZhfk2*}GM=kbLp-i8KD2ad_0B=vAXf4^I3~%*kjY8d$>L$VdBPx6O|RQ0 zi1#7;CjhM~6gWB3>-Yu+#;jnx+HN?bAw)J+H)2MWFgCD8m%y77%w8JwZPmI6oezJm zC#)L|0Na)=o(d(9 z!!EHncAt#Pp_MW~2d;)QMSOkGHIL*<(pG=ob&B;EXhLXbJL&2BtC;=)*aCje%oM~g zXW;>bKSRR9Jjrk5R{{`rRLLKkGAzmr`zfn*$W^4@VPgGT;BB3{JFiY z*xKKII|_poh^~xGjIA6yc<>CoQvtcT_Wt!D+gnRTJt3{(_}75K=c4Aolu42tA$Z9% zjCWsGm(;s7npRa** z=6Umh!yy7LaJM4-)S8x8ax9rynF#9(^ps(CA5j&6YREW?7910a+mMQ3l4TGKkLArpAw zJ?77XAsPn5!uHJ+L%}6Uu3m>)hDF=m(p-3Zky%7VTe>U5+b2ic+o98bbAb6j3KEbW zra~I0emDCQ!HPk*XJ7|{!Iaf9h**^+Xcx4>?OqRyCtEJl8IL>A1>h|+A^&6IJc#G> zP+MLznJp>Yqw{bJvk8uu-e@?iUCLUvpBY4Ls>(uyd`jOf(jvw10R70P-lUR2E#hq> zqmm1bFwpW~mY!}2?k{us7iU(JA7jqhh@3N4H0rAQ)VMowXdC zZ>Y8nT`F=AU%RP%?@3Gp^5J!LJOF3Bp!}ooWO|^Oq?1K5?{GgHu0hdM`oQgizy7zd z@4xMb%|c0)#hicw3>dm;jpX-4fEV}?pzTyVQcda5eoxS|OIiJEos0Nw37SOU3F6V+ z0l^w6X#l&SnwpQ~S=!nvCRw<~CE*?XX4#if0-cuU^shkZ!PJs01Un*c!zm-SJli#m zS+~ijWTshEl;k!t=P}Jq=J6DhG*mIdNgM)@dInQGh#w9!9mH)4&`}G;nOcqPTKD`k zEeuR#P?!iYI9H)F9W?vz12UWK#QZa6!Ea-+fb3)}oiQ2!J}MSOJ90Kh^*!3PWat70 zfXm(_Ya_B^u?B>SLFh$pJ#^s~npnCnDOl;rM4;7ZTOz*s1Ab(?q1Q#VScoNL#CeSP zq}xvDP2s%KZ`nr~aeK^Nc++|-#Vd$BN#59pjbzRZjaw2fOFjYFC9XbMI~fpf096;$ zKxg7~b1`&$g?29rw2w!a_a$YcLNfcW#)!^0$t&k;1bt z*M+8=9rB?vRC1_62gx45kDP_oB=*i&4A+oBa10~G;Ma=$61efx{H1C2M9o|!Z^DKU z`v~!6;zP-#VIC5)b&J_7ewmCRq;%4;6YCU)tk8i!1LL)L>@``|@0eyl5D&rwTLZr( zVJ7f<7RUyZ635aiNWBS}(5^Q~pSYrk|DN!t7CMq`km-`h{4hQgS=NONp)-luwOHlB zi&sbZLNYT1@xzPz0?JAqFmGtx7JACaUVR#9Fe;ny%>Qn^c-acA@2#AaWStQaf!9Y*P3h2F~=BlKe{A$VKe=1dJ2WI znR@Z80)?{5l|rHG+^`ORBUs(qjo(DA&t0)rG&iuey=JLTk-cVp>!!K&P2=nPZtGiG z8Jn94@bDew;p5(CWNm%RN|cw^^q-&LF}F12wKAvm;9GvWby3ZVLfLYS{GyANj54NB zWN%T=o>ICKIMU|eSTVg)^u162g2DO%q0-*%A`b!$o)zbcJdg@inxo(G>d=esD^`be zbo#vdPnHNfO0#<^dhX@&pJaT$Kb7xhQjUwu+{i-a&4R{+6epJ1mXXZ*XrjV}tR@?}8*hS}E_?^=CD`gdha%B_U&lF1j^$o80tDR>kYbca=jH~uh zD6XG=rqJQz4=Eccl&gGePU7F6?EnA$kBHo!>c4DfZ-1k!@aeo3J-x5PSdQv!yF=pg z{MQ0E{iLR2F0>`bxStw?U1*L?H-(rb9FONozvm7pkGvdqzQ>j+EgZk^=GVLE>~oj< zXUfJ$?!p7{EN9#pQf4(WEF74pw zD=TYauG3BGRI#L`0h85QKozEUWRtOL>Rm#3& zF~7JtRD3X`E=DoR%zl|SZ>c6X~_t5pH2+C4k!~ zYuk0{WOzh`U*Xd|dw6+yv5bcVoEEGVbL?%pMn+6$Mp|qOrrkC%U1#&Vz+(4jnZIDE z*vismVSRmltINtV!?N@8bc0OMcU)6w|Ni}vXCM8-AtEA@sC_;7M)jKq#`v|2Fv`rh z^ZiYRjp^#bvQ0cHbjhk-b7&G^h!uM&2H2j=cEd z*WMQMyqt*UUA)>i+57}ezO>dX6f)K#YWL<qocD-O1n%!i( z!G-Sf|PUbcVGFSS`*#+`Dovwvc=GBKBVc~D*mQ7qV35tLQ0uraw( zVzKNZ|C3$3scAuCb~SVJ^L|Av4#Sy4ERxQx>6w}S2SXfdLY3bl=0)|*uVde_8Htg6SX&MEH`U3bK60=W1eM$^>HJn z#>U2xr|z%gICSVUP5PLKNE5kOZ+Sq#)ReVDM`G?m$whv}D7o?;)oIBJ23Oo!{9*Ov zXspifIi~fo%I7`pX4`Dd)19XW>t#opQ)F-4h|Mvb8~<8pkmMH~9X*r~;N_*aYV~T_ zp)YUEV`e`7Oc6F~^*x-8Q&M^7&K=*;wyYD$9mDTMZx8#DxL#hIO-qZ2iV83&_xHn6 z87$m4%Xg)V)XcDCkR1B#vL^rZH-puk^=3U!V{mGE@m%F~?QhO)R17=6d2%Q*&!61= zn{!Y6rCxD1-LGj*F_A5JxcO;RR8)*Y#ACIT8_JP<&zMgb*iR4YuUfUrJu53KMms0d z?}Tl_nUL;oJ%Q_gZct1!)9UK#(tW&x%fMgA^oH}?*Y$4B)z#JKJ^hXu|9E8CPmgyz zB|oc^mpk9%qqBD*GCbTjI{J{+(%knwxwH$cPIYV6t@BS#KIS%qu=2nD>9L=*508G< z=q??{Yixdrd9>!xWuzi3;nMe_#H^8@wsA004X}v9@d8{XU*h>Z%M3%D3~(m&@7=#I ziz6b72$6Mk%+9bL@U;8>X_qV#-cw|rfu=U9^-H2o$YF}X-YBeHQwz?&YHiqsEsCM1*LHpVdX4N;qIOQ%Ky4IrA*+aG z_zS(6u?}4uf@vx9!;RDrQ;zr=NZzDJ6TNgo|OMxVSj- z&i8MTwqxxLT6s7dk*ETG%?D-u_1OGgajEXsb#F>F4S1o4Z1*C@oIDh&RAN@paNu!^ zk5j`R!v zl}C>s^MC*TT{!;mzJ2*(3;kh!&2dUG*VR*Q*tFT;eu8?O7bYtupRx(u9&X%YmWK%A z;N~8FxSa6nd$9dT*Wl6PNJ~Gm=jfV24o=tC?Z7#fhCq)b-RQyToS=I!fJ( zMJeacofBD;5=g3041S$yDQAge5-`C2&5pJ)EbrveG-C7HyhrGG^2%)9ilAp1lCgP) zcDF=BEA1lFJQvb!t;kuVwvt!x6%`dpB@tg7PEzBMbeM@}xzkC9eU$Z(Sct|=MxeSG z|8Z~a?&G&!2ix^BiPPq~{1JvMcHLW<#qE{a+uJ=80)^|NFCRcLCj}`&+UHAJhjUze zjsq7T-{sZo8NAroIXDKFXFD2nJ;Wm=oEG|9Gpzg(^Xh44Z*`pK&)E#tSwtIu{rEuC zb~F%W@F2jzqiqKVzq+x02|eT1-P_ASC99|WR$Xwqzq;#T-}#Zo_h)dM$}fI>7F8*9 zv+=0mWTTPH7VURe@18z=x-Q#pa%d*&#pll# zcN|eyLYUOwUA_L8Y4fp+jEoqhXHhHz0zV?eg{C&ySE>4de#cbd9`nWqr^#|*K^#$= zftqvjlFP|*L81}HwUHzjG{4t%TUuITqgwSx8b z4R6I!OKZQs+UxOMlJ?!>rpr$LW&~d7)YO!Wq@i#PYRgD_jz*>AN_L44Zz77o4J6dZ zj~-o~pXd?PFWD}RTFt4Qo&InOD-V@^(r3uDPTG-s#vhQ~}AwlFZLG=0!d5@UJv=#dna zT8*xuDnvZ3kK0)=RcUmkZdB>aYu3zy5lf z)Pb4Nwm2XFW5B!Sbc?Xo*4AT=bGO*p*!mma>n=3p(b%cv?sF;vvwgJUa0&^g_>!O% z(A8kzepyp95P0gEd1>)`vTxbh*~5$92A!Eo0o87pC#og}vl$~XWcCF+SZeZ4Ty#Eo zsr%V5O+BZ5f5pJ2PDG14n}BI^vP>yIttMxoHDBCehLQTCviF6!q^7wb2>lG4098$K zl3yBoM`@&)6*e?9Xr|x#gHg8(z*WF0&;o~h8zZ9;ZNAc_xrN`rTgsc;7(hbTymC%@ zDlRd%q%I8@M7VXmwzf7yOY!)0gU%kR0Rp-yA%JhlY9t>2m&=_^G5gNz+V@e_8WTUPJQ$V>vbE@f3m86(h>({=n@vjAipFVxcp6?9+ z5CfR#8XUY~E>ETMA=0q&q6|eha3_8Sfgk+o%8q|gU7V_wf6A8Ryln5FQMjoyrKl^^ zjfJ0G+S}TwwBI4jV}f5|BFp{+6N}T-Wq=mzwrPP1`4f(F@6I@+nYCfHuI5v8%U&^2 z1IoRP&+~P4xQENE>@@2hZc3U_8yhMfJwR40rjGAK4Qb#J03g$zXQt zc=5}Fq)=u6fH6yK_LXqTe4!^X*SXq3 zU@N{2et8?PJg;Tj#x5x-sT`*wd+pk_nHl#LN$dXS;ao0Hol!got3vO)^NBhwBrjKI z+L$148xud+SXjKk&4zQSh6agP6-wy7tbdiRm37-DIt{VT(Bd6F(NkK=Xr-unUOW5F zWfm3|zh-f9@sZLbf*Bq>cwqeTx1aol&4U?OCF0HVmQ|J)re6jGwAeIgA@i}tILwX; z$A3Q`Y(IE-aX8Al=MJOpRHX~8HdN=z72jtp;y3b|EsH(brR3#Raj5q`yNm?IshSWE zgnH34)vPTR`!eWf82p}e?4#@#d<)&?Ys0V z&Ilf@%>IHcl65D>vcr(1je(>Fn^WSJ8(WPu8aM$2Tie=FZjY!3i8}~g7?P?H?r;o4 z+HNwmV2eTBSEH3p4>u_m`VLjlsKV_bssE|1! zE!5SO6{nSHU7WjQpF8($qs>TjJxeN<@0d~5zPB1_a@gE*OJ-enRR3u7$7prc)0AfG z6WR_k6&;A{@(K=4`0ckfuej9|7H7wN_|5|kRamyVcUpFT-byX0t1K^<-NB`jea*?R zBCxu@IxNMg`d7Cu9xahiU(YA@4eAbeby*dRXdUUA)3 zxWAt3=+R40cMI06-XiuoD%f5MU5l{h%I6mc7+j+D(}xnWER~At&5|V(N3FPq3{yty zS%W0L|GABzAN!f%n3bhINs-uPdw%D_#5Q5nz}PGA-#E=b?e5Ou7XA}83x|1Pdb-|) zHp!Zip5Exw^JC9m^KsOUt?%Ytwa-i)x5Tc`8l2!Pkl{DuzRxc>m}j6Q06+3V`^bSg z$BE*DB-&XV$4}(|X3}ri@chY>CkR!W@yNLYML5J>o8~% zj=IZ!f+@+6B)P%5n6Pi3o^XkaYoXS>Uy1Jy9}PUD80pTY??85dec;6fmdmL2(+E?} zBS&PB08=XKG@aa?oSddRR%ohxe0-wF!6mP_qAYt#Vl27}IkhsAmmNz283E@+t@~f2 zpbwxgtNnfJf$dC8F|5vW`*kXEGOKA*v$K_!9Eank6PH)_{P}1bL((fX3tL-NTu#t= z=??bx`$M{RKg$K}+gZQ-@=IN+sm7ZHx3OAC5IP1wv|n5O@wr5ON$0N!N$kCx8#D zA8y{mAM`rUZqfi@%H8s_?OQxni{WSB(Jev8Eq-nBsNKnZWgn1Z5v$8m>QEjt`gTvg z^FUJRqeI-P34v_JE41azvE0Ssk(ij6$IqUrpxsH*nnev#&#_NA@4+&1>w7Vdx7F7j ztyjwZ1$zbuV{x=%(K*e`czJCfY0an&9kEB+segTH_2T4M-io&8hoyOj&6`syuNUI+ zTT)F2TQeq)R{vg}-YBbNkQkyD15*NB93bRdyxjsk`RG`Z$0O2X%?Kt$Aehb~*SL$OFFW_!--@JLVx3D?e zF3D-Wp0+e89}ym|mua+S{)rXEq81=Z3CNYZW1pED0vuR8o;P>ybk^;WlH{eqtSZB@ z8BOu5B?bj3o$AKwe1(UHXScY$b!3yEuKV)dYSi#JByoRnhg1ax zg<+@O%8-)I91}K;h;jLk}Kcsy64`+*|}u6_)o60C-c2ByCPNNYX`{ zbTK(2A0KSQM=Lj1nl$IR?u=b_8KO?!l7e0X)fNb}}cZ%ZtxMnW5A{X;)m zDaJE9Cx=-gVRB-kWa4Y_uzz7aGfB`uwlR_QM&EfgT@AvW=Q&Pa3Z+HkJB z3r~1+FI~L&I|u=@D?=#+jdp1s0nkM<^B1*=EvQ6HS-pDx{5ATx6YekSoip2JKags{ z!O{D0xu&M3F2zI@dqM?kqXJ$d)yaJ33eF7G%*@PR#40kUoG}c@eCXMU(My1}%!TFY z1%ITBeAX$WsFwjJL?TNocv^Sn(*cJVwWeEmhJ(3?L(N4O4FYDjgk$tlSlv17OQ}&#C;?TZ|A0_Whx5b-7Ls=Pq0@LE%J` zupsNA+>l~vxyKij;8`Sn1vLCqIImc61#+i&mSbpd+;ZQlr+BitCBCo+RzkPMP68N* zRz3YzDA@$3xvy?&>#}i{5$t znNh(Z@kXP9eHjE{;vEnWfFvB{yu2`wyEv+d{$mF-^IP3mRx@KF)iB6>M(AkGD~&d=qF0WnCtvEwbs`5eXR_i%CJ<(Qr0gDtJkrn}(MYXB z1!K;QO2HAdpQ^sFWB2aZ{(d856=@ocM#8$_cA*DrBH-O&Zf==`gaq}}n|>+@nj}k$ zl(9&u=AN-@%d}}&zT$w(pgVh&X+7`;>s?x3uZ)bOj3%bwlNzY6+IY1j+|ltiyS^Y4 zrz)q}wy6DQ9`=KgQ9(m9_3a67v-xVp1Wl3UnHF<~mdvJ>{EIavi8+zv459-3SY+Cg zovi;cs%?i;8eyl#eA@B~s4qEUbc>$-c(%71!bq3WINJ@i@{%J*!$``1bwoo2=oVMK z1QHPBS5#M##}Gh(RrKMmojk$|cF~Ga&j?Dn_Hhk@>vKWD8T3XW2=!xNV4t#xt0Qhn z4NI{d(=HDbQQW-ySQVhc*LQi6i^Br=(?$Hcgltj|BA~^YHw!nviAC%RRyl$cFwpFc zhW7+CwT3`~fU>*MnSYn?6AFroBuQf$BY%+hdo5S* zVfmqVZ(IxqW0=-%Uy7t&L1;yX(TpN)o!r)`{_29SDjQWhHi*WV>NrV12y9(f$f!$N zJklDHc8`#J>d8hgdYpMx6Eq`RB+CMXE9(-pTJG4{F`1j2gRhR`Hed1&B@HC5OInSl znc_K)7ej6L3Vm1F*7XjW^A$%QjtdEL*N6%fwJ8Z*Mm{+T6+;khV_|7&snT2Z`l+#w zj^VbfhR_b2))8T=1f4vZY{!NLbj!V@lXG@0>fLTIz~>jMmf?F#?=^hv+D5Z%fT z`ZFyp?XXI`uW?=Ui}|I=%B1)x@CCxzYs=9y$f72b9-lCY->0Vq5lw!D?oXczf}<4% zx#VZYJ^{Ae4;)zQ`1z_I_k)VW4^||PeChYeNrAR3TVcQ~2B(DCv0GpzNdsG0P*6aG zi)`A$5JMV2e{ac=D}+r&UDnss(jw%gEV!8u*%6hUf7~}83^?naF>&N5u*ES{B0uMc zptH>YqHmxo%3Qtr+HR)FklAdFsXHG8Az?I&>A130`qruPTpuPYhqSxJg9de~k_k2f zMJ2vm9k*Uy&Lafv<*av z*Ic_~_ewZ6ehoXZodsO$%e=fiQ0Q*}NcwC=9-IcWQpOoRqMZ%B?C_NL5ed3}(P-WZZHZ^ST_>QT3YI*@yhFlm zk@MRZHVzIVl{IHr9o)r!{M4O0>1ZAZ`POTDIDOq;(quY4kkR%?SeSk9rWj)XCZ_&x_D@8r5hL0a~vM(>LTLkYuEP^0~%kJI``m9 zKK{3&Y2?Xu`O`xUq=wNmFnE_6)FL#)FS_U$BZVTRo4)Ayu&c;1r63gG?)+elN=1)L6|aMeX$4hv|$ef|0yw39#s`A(3B z5AB;<);eiONj!)yLC0k|B~~fs4bUUkzI`Wglo(jInYX0M61e;O@9)yUpDF`>L%UIw zP})qPbc#LTp#+QBDIlgBusfuHq0}jC5ubX_;yiba^p&35_w3=awY8m}pBIVW!^jxj z*w}cpNKQ^p1$R=gnnHQ~`mnXmS|2n`RCFvawe1JksS4({PDKYkAFF@w{$Z{HxndW|$9?iH~t z%N9tdnu)Af$HX0(6!1I=>o)H;8f(vC7(tR&M6zb1q8ftm2vAAXHr2^RMfvpSpNAm$ zw!9Y?6YGYkGLDNMGNSLkkWZc%jMH#sSXfw!Ri7NxS*~Nplt|x|d3z)lrHoWY_4n5w zelsJ{(GN0ei|Ajn!d zX>PA-$&bemqYjnTwL|SF50TXIocd!u>mh!A5{k9pi3qV72b?vNONbPtqiINZoFXD> zSpP4`*NKZ;$5YbX_foFfAY3!!ZU|~+-r3(!G{dbf<1l-+$9_ZitS#MUSC?H&2#I(z zEJrLjOeu+@SIpSYSemxh=`pcdxEyvEa55F_eBlP*Q|7_DXw|D(meN1Y{(SI zE6tbXwBB_WOVqjW-yW?0VQvJHx9IgKnt9GS1TA!SUWH15&ERgyB{J?OGBN5)+e?iU=wj+R}XXk-b<8^VcWW<%fr})hB3a2pE*5o8>{_7zEF&a_iQu zNPQHa2Iz1k#O$7oF|Zi z+pno9DS4^HUkSx6MFWg_h7&mfn9zQ4BrR!^JNDwixx0aM*)Jf8ildU)_q+ zDs0-!27b|D?q&Gzzi-%aSmA7duzAJpR;Y+jTSUe1qxAl#nJ5QkeD|rUs^*M;+^ELm z@${+k$&;>U0^;K0mX?=y?Aa4AlK$ew3lJnJl8{C$Q_S#lhKH=_ZXs3{pdP)QpX@6& zNa`8<_jAaMbC}OWuSIEnckRAhuvko0)uoKg%o1SJ3(xjeBZif89qp(@;=sQSylV&L zZG%N#yLPSqiByn3j*gCdS#YhjvUjsLblV3zPEgOCJ$pPk#jH(TUSd8R7?hn#zh#Rm zw6#>zmPq`W>Pe(f$g^DB++_oTNIl0D?{0GaZ%#7dru)Zwdm~@J-q!{O)H56yItH+4 zur|trw19PWN<^*#dg-FSN}*ip-|^YQ6=ZcR;M;}SRx1xSW1MuN0TA)0b&7H0PVP&{ z#rQ85ih&%ysve^I#9ur8bd|{JJ7YPsC5%?J%Uf6_0%OWMI&_Fp;!`Zyiih}j_;_43 zE})40&vy8bGxb+~P~5k=lektw&#Yg&em%m(80yZ9KFW>#biJw#kUrq2^dEoxVLuoX zWxX_K;l>B>8;0XjT2yoam~*f#E3G0#a;#ewPN+`Dwfq$L5@O(z@aNM#AF02KN3#k& zFOl3#J94$8y}7HAfQd=Q&G|o{yP)Y`u-X$(O`Lb#;)3E4huAU5C~Ig?jZ=ARX%qkf zV7JiCXk=>i0i2wiQd}zWRe(|;KAfmE$;P$7-Y0Ut>SPkf34)v&1`gbg7Z4Z#qMij{=&Pk4mAYc1CuJqYGHTvf+O*+YEB7lF=31v*8K zbgRE_aFEl*#RXED!3h;;l%6kNj#O^jwrw)f2(cli6m3X>AWMuqOuEO=X1B2LqD+9? zn7MHA;zb@MatX}`LKK7F@(x^hW`t0jX_BPU)6>^L=wM#Z$#ql%eGA_HQmQTkxDBY| zGexK_lkv{@{%nOqlB+5k8T zClYW_Q9dVCBn#RMDn-?8!8U~;&1b`OqwijPdw$wme|&bf z0lbe@5;yLBRcMv=8ibf@{8m)R|8btW|2lF1!5{b^q3&MqJ-Q@qMZ<$b*w$Zv{WaNT z#o0}Qga^dtp02I~s27QBu{rq{C9+V4=2HSN5D2|YZV5aIIRuL7V zJqmh5bB;qAn&1fJb`wZVgDq)^N!nQDlere=%#f6G(25a+$Tkq> zA)$btinK7;{lowJV|csS1Y341l_Xek*kB%`hMhh8TTVF+H6Sj)>`OLOO<5G~BsWPa zfbOLizn)75m-D~CRJx-=LMooy85#SlSRlOW?A)^_&efIT)&m?0NS8KSECtaW@Os1p zV!dVp3e>(afK6W?u+Nc@PZqnNjLd5x$DDkIzh~UOosIhV$rCFhk0(!(csKsz9D4ub z9KN3|e*ABxE_9eo0*U>frOse%5~|lB;CVcqTp|-T-;cgq z`M*P~@K?ZV(4~}NsZY#)2hO`+x^?Rv5i6()I4E_1^wR^i7eIfifXrIAiSad-++yuf z0O6@X7RnZ8(QuFu?OzxH43#0Lk5sPTyqk?WSRLj@I9Cu!htT~%L{Ua1Mh6{>78(A; z+T4o&cpB?kVmG2E*mujGk_}p{9_A<#WQ6ZLb0|xV7(w0nnt0NhZtfN^aEFHa(v%gs z;)rJY(V`ZZeh>8rIx}=4>7%_mLg*jJL2x&O5Z2Go2Z3-^O>T=T5d`v9md%+Zoi$k< z$M4Do9uGx}6br+OznI+z^#47>o(O4f_Kt*Aw{2|dv7+%{yG$UEir(sUC6W}{%l??C z;2QM06LH6o8%`hTSk;N%X}Z;_G6v^{d=vQ~v)G+Ws2(+-<3~EW@N}-NJyTCka>xhA zBgl*a7GVesl33_JKw!bS$C|}&u*+3T^OFPv8T#Oi3K_br=kPCtKEL+h@lT~(Nzy{7$rO?xnGmw4`b8r+k5f}88XSJ)Z z;-k*Mmog)eXXC&^4Gde7wJI8*8-HKKQoE5i`!%GSLr|2U%PhczLnuVjI8^YgM>JXn zI`0?5_}{@fK<`r?`0AwRx1J&cC+v6K?U- zE}c58Z9SOf`gk=(p@8xxcJgHYwW~W}y_!fQ~KlDN;@PxHxPAM_ zw{_v$AU5|$g*bbHYdYfq>&`U#645NQEtFD=g-ur>x`0yWPhmZATLxs>-}}cxff(d0 z>&O3z)@qP6fYsbBV#%hn1-sk1Ms8)p9G2@chzVtwd5NS%=q{pSDMUyOpsTm}`}&kC zsW;V@!D?uqAVIB!t zOE`a)3S{@bbWc&{5M`ob^0Kn^pg+2C2%r`9s8eL5rE5WbOu;*S2yt0bGxpU1s5T&!$5k?!gB0v{HGK0D~rpmj@4|XbI zZvzF%h5Cc_uY$@+{?Q=>8Q}&B6(Q+`%-gxa6p^k3y?IHBDDIVJHqo;;-vnfw8rbdz zB5Xs9=`Qh=T%LLfp)VZSOFGjsYybuarM#tyC*fR4E<_v$_m+UvM9rM|Zv9{7uSbd* zU+N+(7eZI|41F0wCfKA)5E8>bgd{d9Ij2NVPC+B-xgA19W8)P-&&CEYrpi!agiM>g zOl|8Zm1ZZtVGbnou95 zHx-Rm^u?`3K7HT5UElKhUy7_wu&-})d4Mo18)Bt=|3(7^O02#!y{Y-BOt9FzcUmo) zm>l?_=r?axhBB3`Tf`t6ET%>TX?Csoemm$2yM@gHi64f32d5DjKhh#h@d+ZxuYH@M zq~iyODG|+2CQJ)#RIo5FA)~?#WHRPsk<_a}g6k$b22aH`50S7mT<6af>(5?fzLnS?S~hk6(s(SpyvL%G2`= z;1lt7idyx$3z@a{|6OPWTv1@5jV{)KiX}=d{$meKGX+5e0>AhlwOaA7b`TJ*JXl<# zz-^0dFDX~!D}kb``$27VI4_p_1)x0T3kK4*j+0E3Vbv$b%vT89`2+spGcapw5kn1|F}N!RC(vR^CAiQ#?x?l-WO0xh z)zO88bLlJ#q2pS+Zk=cxE=&3RsE@K+cW?Ky9~_|RnaetaUZz?iPe zAW2`e$is?8{#|)vgRK1>`;FF5;EsQECrBXCf{Gj9b~H-t?onj7hO+szk?JlrO zdA*zULog86Dn4TZ#73r%P(z_!M>xi#BJVR78Od@DG6e)h`m&f~_=Y^vxYEHGc33g8 zAG)4ulw1%S6(+|Qu)?R@Y~qKXr~EIT>r7}6-4cc^;4F3$hek(aUh&!1FfwG4D<&$c z4uata!#}^UzzI1SvTuopC-x%nP1U-5VxIdFuSRT0gm@%gZ~Lzgc1%pKtMZ?JVS5O+CACYFna^h!-o)Pz^JV?zF-fD zZ^O)+a1+)KZm)X>==_7!*8n)&zHQq9!cTaz!5tzp^@y}dSzn;ddtq9~Wv{sXu`QfH z9iKHoY6zitqpP$rYDqH>LDU@_C|SE7@Sm7-oa~@Bpcu!GkB@T~O^uH~2mSsJ^&y#+ zE^n$v`bKN|dxC(8Ih#~nxa-0o&tCLIG?Rh^T|EZq6PWnkd3VN8WMAS9nAZSPdXj}R zneH7%QPeZ7V@RvVr&}0(`T@N^zy9M)+cC2-0lm*Z!zdVI-jNHkL>1j8NhcUj>eVHv z+AgkY-ASr_=iT+h`=FNvP-TpHD$?O>i$lDA)_E|qmjtTb)Bjn+d^l?VuY-x+8(aZ{ zd?2yq-(5Xo`X9A4P1+6;9&8UJC}vF`&cU~l036a&MXQnK9e~rh2Hg!XmMH1xfc9_w zMW%`2qy_^~5~_~?JUDC{Teki3%Q4$gO;VeQlK@gD87F~Y;#>n}J3X9F?xvVzJg zPkB#HVZ^0%`A`4`x{F_uGz$Tmc*}6Uje#>QZETbQ2VryC&dNIWP8-k%Zov=C4KPbU zkod<*yyU#570fen{4d>`(}Qt|oNR1o;qxJf1dc6kYIBxtBI@3D7M5UEcEXX% zhHi2N_K#IdiaZklm7UlQ%*Q~yLOx?d>rH&FNQbJBehKkLe&~4=zQG!#ohGX=NYaL) zv+Z17bmIm~Lpm!8AXn-zKEQtPU>qI*t{nqspS6^e2~Qq9s@77D8>soT5p$n34(@Ff zRF$FeRtz&jH>rYnIuW%Gi_s0|R;Km$dw=UwHp5TcQaP^`9Lx~qmmlYbB7D0aHjEIaJL`bMAD0t>n!p0Vku{kOD zz`kVK3=zg%d}+L3{t5x9?N&b`9I$e4a3xeoIQ>D82f!~*jKSqWCsfIFDa=C$sl-DL z<4GhLfB_aR@%VtNPiUeDWvf7nWh;iX;R#hs2v_IC`D%~CX$EkHejG3n0h|vBDOtc( zRa#!Y0ZD;WOwy|VtCT#odDAAPn>TNIcd;Ejh?>(x69ZvUc>`egGncn7-7wYwIzR+? zgSIcn|5c)jIgRh{J3K{q3!{1JxsDkqZDCONn_KSRyXOVWg+9=Y1~(7NPKjka+J?gz zLZrcOV5IHMfR^DPy(#5bp%M9Mq;AgUo^Wq~`k#W|1+7OQmn$9R zWT*-d1k74&R6>KDS;QO?!9moTJlZrbQSgZI04x9*i}DjN6wOc|Pm0o1#b{L^0rw45 zk%t&v`}*5s`~!`Jeni9-|L61W|BEKb-J#r&hdT%>OO$*_MLl2Gn>Us=V%r_Px44ot z5>9;yM?xGxhzY3Q8;$R-k@WNKHFm2_ImMqxEBkpz#Ovot)|92;q)OtBC)gQODkyPNy4~8Y>s3M14MLM+7>4Mv%s#kV5bG% zlF@EB^_u(3A`_O9dvvWwTScH9_6?@5eWbtoD#m2HzkPd? zo_^eI2LD6k;*4i*ZWkh7IxjCRO~jt1dU|+lF+VDo?|hqKCdukuNYH(Jsa;A4JnvRc zW%vI4AbjsOeCjyN$@bGX95g_IflM43e*wn(l z@g7`U$eC?d1g)j?=Il?e4qZOIb?a7PF`@KdqatPtFTdXJ#7}re1F(urv`@bK@bQUjL{S?3$CYA8q!Ur%!e+3sza*=Vw(vhK9B7i8~NTqk) z)ypI8vhAYtaZI)m=Gm#`BA_i&<;xyXq%yS`I8e%yL^s~oXliI6CV?G$_XZBgu3xd@ zTAU{gJe=;tj&)G!%JJ3`X9sVD=OL8m5VaYbZ zy(|F4LGpT=&7G8wNSDAc9GT){U>0p`!Mi-zsgNiMw8xEkrJdV#GHqf~6P8jC5yklK zcF3@Ta20SO6mT@{)U8fgb1m9|q_oHM(lpY#wC8BqS%8aBE3Y;5eSeVpvgRe_t0-{ZYQRE!f}K8EkEwtJDiZ@n1Ck^IWm3h zj#oznj`jgH4#Jw-P!V)u!~;wrnHVOFFuX;9YR-2pxFjp#GOxZ1=N+AfC)%?pP%S^x zK3HJ;q1OAI);`}aRM@TcerLhhM@{bz)zB-|=*h^D_Jb@$UWFn9it1@dNXR3YgfZO; zcdb9H;mwQYt?AX|{S=rzZ#jGJoZpKB7cgiL;8p7+j)5hJYG$;_kQHCJ`Wsn6Zk8tc zT-t{dsPiPw)m%<))(Qr85U4`%CmaO}AqXi)80W$O;}G-HztI1Nhr>7Sh9{f6V8b7r z3VEXddC`L(IIQjl=|ex`p$#!)eU1EL^8;82zza)!Q7jmB-I+jj+hH$8mIcgKPnK{c zlGinmENHgcYR?AOc|Cv`j$_B_(lKTV2ErfEyD2vw=r30VEIXVJ7UcLY)}UdM7S5_K>} z7t>GV^%La)uYuaO*RBaXoHqtwyeCl#M?s=klQ#YVw>7DElk$0E8D-TWrEgtb zDc1_t$@(5W1#1bk@t(SO*UmsJlfjN8&kjIP2FFIqFS6SWRN@&vGE|8H)-$*VPMzHB zcDp`iH&ZWO9c^&k^@{T1?3fC{B$le@6k-s~v;nbIAgrf8ZsdhRUTnw9LNWV=?&p)C z3V7KMu@OT|SG2UWjKpOIZnUJn1pyolYuj0urSEPS{oqAfpw_@2K?Y!vqGd^mgn$yU z^}u-}R9l9wZ1|93wX#yl;6DDcAFevQ66h%z%mVW(OXO9e3c_mJKwz4 z8cKI0ncG5Hu7>2TjL~4sa-N3IO%cR&vSp7Hk&ys(0S%-gZBW4zP+^Thh|i>wu@R_L zU*OPU*YcywMy^f9Jg!3s=oNL8b$d`QBc`PEB6o(u6iFzsuUzX~1Fx;alh=Y5AQTQ6 z&OkheAwR~#Z9oLw+oP>9(@OEVV_y z>pZl+`a~T`VtVvjc=+&PZ;24yzBIf~>&j_ZgbB9__C%W4Z^6NT2wWSF{2AHu>1J#C zWhDOhAqRb5p?@9(fT@CF;y0AC`lMOcfvc1UCCKj)KxgvAR8JI7GTa2SHxUAe^anT= z4LS+~TS>$j1DBK&sBw}c*~mB`iCLnE!%i_kMw`Kem>;*A1C29C2rO5+)yXJ+WNdV0 zdCVna!r*AF=w+S-{~ zTk&!7aC7pUJY-^TZ(}FI#bx;)ui&({HRiIjUhT(Kw%S~}YDb~$)F*$a;w9ruDHJ(z z>GNk)ox{hv+}!JC*Veu-xurj;5r~W6jXOMeJK2Y!iY?*NJqC%hhhvoSud;yfMyfq* zcNdRW>MLLBe{@~=1`m(%fhU$(gA+T83W~b3-qy@TjydG%a!ycB435k=OZLPqMW&W? z82Rz6qfopJbNZa9a4$BT>ki?sAAX_WZzu0iwooY7cs88IZ|)za;DuMVQ~g4r6kp%s zjUVkjN7+E3yxI3(RIMI+T%XktdXddogyYK zu9jtS^}#`L%^)tl+J>}*goIErm$VbA@rM)@6`NCZw4&Unq;nT61hirW-#c~7$jLRy zauvipdi0Bniwm2Wn6~++Nq_VWDJHVXlcD{l@pE=0e4*_7*GzA!%@fBx>Tu2U7q z9H#mz7+MOPzhB3qagl%L-$A#))gjZoMIrj)gUZ=5M_Hb7hsTc}UkteG=a-Rv^ytwL*&xogi6}9bhD*G- z6~|%6^y@KFE(2}GQT&{ooC8B~ zQMCgg6gc-MInj5o(C`v5+rC;bTv3R`8a`N+^7Xw1XTx@m4GQ@(k zCX(_$y^^I(+3J8#LCgz4L7kZxyLC+3H0{%4t$`~JC$x!!)rHC&4W*j!95OqFJr1pFPp34 zD%vM)w7R@-RW}ybAb*6r2Rh%`Jz!~kYaqpE-7PlV_e62Ow9%Kp?h0xPh={ab;x(*# z#*Y;)uo~mx;fcl5vz|Ozoxd{IF)-VrC-~t$ouE_aO9{>s3rk7=cS zwLcn{7p5Pu>}F()7&EMklC)+C$(tzb@H-$_TOB4?H#6K^m6MYb94j9d+SH`#yO56E znRT$L4y==t9w@%GzAdHb`60@{Ex$tUl>TH&@NORt3M;lp`IStFhOWy|M1o**lE&X3 zub_~X{a_!@`&J8q-Fx;pbQ$_H2w(5)?5y^tT0d%uvmSf%YiX=p2=4&GuSKWOX`;K- zC_N-NI5^Dhmj#yWZVeYxoH^_Xui?Fdu^z8Aetms?)+0y0+^JFwJymguS8{DRy@2Dy ziOQVZ+|b?<-yWNVTkALPmmX++Yar}K5&S$#aYj(8q}`BiOwkebolDWl+iz`cjeQ6i z;q)46IO?X?89!$n=~C+K;u3g1a#huxx*tvXa@N%NJ0M zP#U#tb4f{?7EzRu;U#O$EcLM3ab@F1D(>A<4;gq0TxWTysHjT2y1G-xd!v_(#bLjx@SZay6b(>3i|BXCTiG{x1D%n;# zU%!4exVMMh$m~3_ZgNQpEhjXe89=cN#*VJ3w%b^!S;ur zlOuHT!G5D3<6W_)jj^RwRaLid-zKN?mD@>@4(P>QGZM7lrG1~A2kP}_#vD7{Jd+R=fPpNu1C&-mT8F3zOy%vY!29VkK4<9~U zT2@wOKic{+T-2#nJJ-HqJ7V!Er%t=Gn_F0bzpR2}-G7gMWlPp_zcQ-la z0}F#uiY;YQ{s;M$l$2Q6*oJcbz3`|wBK+ph&Mo0QVDH`sVQC@E+ZtB(tQRf57+jeGP#B{ zC?9x2jiHdCp|5{QxaKLFiro40x7n0l-u^x_bIM`lQ)Y`ZzqC=YH4eG3-?c2ACrH=D&H{M+??9{wah<5;^POUO`TTt`RRw6@_7 zhtH%c+M0H5dV#h>g+AGz%o9YJT2%2R-lkhQP1- zni(c9a8%6epP$*bbt`V&BW8A_MbBvk0rV+c*lwnpukIVp?#!gevN){&bP9xOlqVhwIyCDnwE(vl}#m9$YNx)E#0guH}jxN-!6yo&Z|*gtM7DO zhmJNUs>xcmXFO(k6%+IA*5-rtIOt6%87N6t2J50)kSk`&n7odkJSi8@l&U9*vUwWW zp#e8Q%gFdTgwNQl>>*=&hUpVzMP3hNVC!c!MvW0p-66d*TX(aTS5?W9qD8|X^qij0 zsJx~|9@R|5Z7x~YV^(ctrcqJz&9z%V3!iPY%dzt*kC~+s?FMUIhZExLr~0H(zR!`v z?Ks+c7|=kvq2ZOR1M;ExbT!|Jt7$Qws|&{DHHfm0_!Z-@w6wHRlKN}fg9lRtyPeOO zx1=^Tm*d37+))Z}D6j(3J>BdfH= z;x+taKfl@2#ksMh(gz1*1IFJC9+cL2xH8<58b$s;Irp1)9&Gx3r$qd(M`dE%s=M-> zT7L9+aT!!Qm{*R9tEB3Rzst!n!JQ9(daaekG?KZxFvt`kbn{4;V+^~;T8&`O4Yvs9 zi+-22wZk7W2*1qkc9jjl$z~p^G|xvp5V85}y@`e)e%qcCCP;O|ZRv5f5u$O3jGoCQ z5;p`lq6WNr{`}BbcY)UQKuzO#SKe%+qGW|px}jzVDmCgmS51_p9uRKzoWcDE4+2?` zBP+X~H4mORmO71h4*}Guy1TpQ{!oAMsAubpdQ+a0^=<}+@NuqCPMzF+l9G~3@ikZ$ zKbC1gw6;O%OP9EDjojWp{;b*lwx`+=%V2?xNzLmRcg^!oWqiw98tl2U7=o-N;`i{? z%AQ5UeS62RyLZc9zWivY^=)*)7?zDYpt>nRrSY}kn2}G+Yn&hTKzfbg7=RX1N+{5)Xof8B`1m*tb7#@YT-f6qxR=LP-N~b+$x6`|bI0~g zdqo!^5!BeXWz@Yi*Rv~K?)f%>v^7+}T5&8&$d9HuRj+5XJ#*xagHhk>>~7wG>WU}F z_#%#_D`mvQu2#vZ#f|_*W5!9UTDy0i;}h|Nfl^e@cT#yd({0u0ES{ZOyqE zUBD5&t2iTTbhNTav+5bggVfa2l0j+psHE`x+QvCsi>_RLsoCyfA?o@4rTaLvvgF1( zvV(Xo#wkRk^`Kb1LPfRgDb#5Q{P^JW=W7nbO%;XiA75T*&bICmd;Q%gYh+>~70|~b zg3+0ui;HV%4RALV`6L0LGuQe1%hly+MPC;WcXx(W8Am54jo&xzV5rT}%(4iP2|V%2 zILbqyE308O0I()ud3o8s$K!{v7}M-<^Amr6|0B%Isan%0accR_-{s-}T)Xr)@Sq(l z4Yv1UyW{K2&dOTqIWUUq{{lD~HKXlC3PMT^&BBY6q$Q5ShtJ?a8pMBeG8@Rz0XO|0hP;z(+9mZjFx zB@+eGfnh)9mVjpRmRDBPUTgCE`1*EqG_FQ{(x`Qp)LUVQ5_eO-dGn@OXU$(DT|}Z*xK2toIU$~$4K?1 zcJr?!w$04UvYxG=*Kqjq*VdmeadOJe{xRC7;_2x*G|=-#zhsBhS`54YkKtJz4)%$V zVNV+3|8oD1BXGiuhF->rDaB|l6dIFv{G+zONML$*Lxt3bA+JNh&9J#>j#Wvnk zq=&WLwq;AXjcfb#P4SkcdM>T3($aoS93Fe$?T1OuF0fV zv9Tu(9XgE%th8Co)F0A3D*G&doX*_Hp*wyRu?x22wPxmt5|_q@d4NYB$vW{F*M55y zq^k`4*FQG)+G}+|iWEcJqJC8yn>Q>dM+QUd(7l(9mfh8{UMMfWL~p<#vBjTJly#`` z?QQz2877aBQ)Y_S@9dOY2(TI#4wzb9SuWWr`R&imOiiGQlG@UZ*@c9zAV@+by$bho z>4uNxrjwt5>QT&{J0LCwj(WSm+M|n$UBU3Bo@DR-N(q zY!3vgW`S#_#LA2czje1ZGMP?F5mVX`T#b#5P2}d6+a*kqcs3D_#hD#T%%UBSpPKf> zi3?^202`PC0=RzvK&KX`z=%9^9OwA8R(9>&7Ay7StLYaJEE+&wr<}$+?Hj=s^!GVL z&K2C8+JK}cl-aTcc|FJQx zY%7N=Yt4X_u~^;N(KbH4qT=vbPhFG_G>o%I3nF6OIFEz*^A?Dhho}~fjLDt_gU27B zvIBv>((_v3@5;bBCSLz=k7jeJF^gr(RRt+;ZEfu)G%dLCvxu~2V=CJ6T$d| zc(MkvYbZ1X7Hp^q6U@#&cI+6th=?Yb9;t`J?-s=yQ4QRdkeCvinwm!Dj2$~2J1mh# z?C*H2P1i{dI*Yc9qyji26*cSAa*s_$WUN+OEC7O_w{dVC%WPp#xYJSO<@Inl|1s`u z7!XT+WpQ>E5X~?r?l{T0x`SAG+nYDvjSsq=Ddq8Qv2k!v102n?XiMAfwP|2DDLvb= zLmdrz^I(*hm31Ddz}URd6!i=fC25U|&1oY>S#H=#-z79MbGsg8ytrv?J+a+fc1d=5 zSm_?1M@^)-Xovq{d0Z5!iLEAFn6E1%V+{SnRUFp+qK+21E}VLW?j;g_fBf-?=VC{e zW!23HdY8V(Qhme2i7mQr99`uPcuKol=x4)tE?NU0$y=@=A{Vg+*7MZc`OTUx<8UAn z#L%|2b_`aBFo)F)*i>r)GyenyTvIfRB51a1dR+ucO~yOl<&I@_j2av0T2=%cm64aP z3G6BoEV(*itf8TS^x(U?f&!9>oeWD{1AuD+5oIl;hx(3V|K+mz_3JMO>Y`F7CvAKm z_n@DR6V=Ve?k*y)f>;1Sk%X>x1nq5RHfrCA5~WI?a$w~OVaVWj?$yxy_0!*`~`BWP)1z{pA3}oN4d$)l#j&KCB;b8rbw%)O^2>pH59a*oh8yL(2b+&h4d6P-IiB1^p z^W&!rKU6M883v>+Co$7|6y89yFM}ee0>UE`;Vic{Usr1v*xce0YT1#M01D)th>Z=q z{s!*#yN@ZT-nbF-zNDl%PLWA1RaeKjHvA(#m{jtm&FWDS9;fUE^Ho6!dw^>qJsNpD0Fp$2c0uhqqpT%Tm#{Wwc=XDrY|$X@OiTUf6A^4H zLXSiohRd6q)j(}1V@t)v#G3M5?7Hmg#82qv=eW*%dx4$~-ONz5zeMGCEB2wvt^3d{ zytqTdmCdw66 zi1BQ7CVIP=fTOZzSR0z?4T9{*M_)@HG@|vKMGrUf!uOyuQXy&l`>TSIta}Pwdwu9s zGtJa|e0+v{=ydDBUNNI$0qdSDD|35QF>H8^#T1X{I{D{jhqaX-Nx*oEU;QP?nu*Ug zsn7&DRea;!VOaU}BZ#w8hLyBOj~xTEG~NKxa0b=H1bCQ!{G}Gk&DpbOQlc5KzLC+7{ScReammmz>IGJOF`t08!G^bev7- zk6z<>di1;fIDHzBE((^$^UPXOB6OX5H@LUkVO*-x2Or+?k5mLv-Rh8CLlsY zyjJrISLU={qjVtGJ2`u;AQLBEe(@XjB7VpQ|t)slHv`^~#mtBS(&mW*W`D zs~jHD&gH$gj3rG10dm#VVx#iGC`0XQjFO(6q7$VxU_fwZyb?3J=C27><Ecw;qKaqCt+^klr?1#C@Z^R`!K(v8UHss4!u z`b)h8j>~o~jkgwUyp8cn%wNaGEb|Kr5JxjNpt6d3?K&rgOKrr%Dsx5m%G55M+>3PZ>TBZ z$@ke=eyDZ>pv$5_pIW(K*mzKhyu!Od@ zw%X6$m=(7}XiWdLTX7$PHhMefhAuEiD(J1jUlbi%GPCb<)D!rpC{Gop)D$kSPp;9)hd3*@=#7IDC!8jI=|vpsIlUI7EE>+_qA zAEG%Z7LOm|qH1PHdY(ZBE^c$9N)Pt)^X6aT-R288ZQlAuA3bx?sPT5@LFYQ5mUBDB zS-?Ld84o(Kx!lnyesddnGZC;f6oeYp3lYy*Q=?kqO(Xdhm)-U-ICvijSXPLjbQ|#QxpZMDgEW%GG!%-J4(Nhy zJ9cDx`D^W>nq-X`o12}TJqi*3a`BagNe=Vzc{mx~aR2WV@7V%+HXgDgtt~WBTaN;O zZe=v9b{+*3%JXZ7DBUk#Pj*`dHoWrMc*!*(AjZ;CVAqhl7vA7|G^P~ug|E5Wpuw5+Z zlIOsWNx<4@MjDVdXFNaM&;Z3&*nAnTSmSXo)=5^-8>Qr5X_zos7JQ$%nZ>h|16N$G9dwl&<{eT-qC$bBJ= zto8Hf&*fT%dV3G=-03ftL=UeC{kHxs*EHwRq7)Hu2CO9_=l{na^{CVkLj~MAP>})= zH$d9PLDbk5)s40rCg1x|6V`jh)80PAalA7D%=&S3zJS^d1c_n;NE_fk!VXanIwaxF zGOfBa(F=bBa0$f25QPfRE1YUOj# zlrS7J322+!aa>Bo(0C}dr0DdIjIg5^&5n0z(Q#_qe2FG2%=Dk5mp5F(W$K1}l!Oji zt!C&fgyR9l1gI}N=aTCJb`yi)#vL-D051kNoz{+Y) zsHVpS=_TBAzI^#Inho!ScFIAdg^4}6lq_PSy=_DJ1PyKyB>P*DT`L_qbw}(Xw+b>3Mbtil&`AB#>T>G$WW(jKA$E$4S0-U z$WIU&$7jk&EwI29k-Jq(gXQ8nECdMuXi(}`VIPUaV}f4XMF}fX%<;OOaXsS0dxZJF zx|{C*P?QIsJwQ=WRz79hcS$Y&@)=!S-EGv=jUaLi{od9Y_oDJGlwv7mQT-gOP3-R$U_Ka}@(-pj;0DE31YHSh7fiM7#Ikff^!s ztZE+O0F)!GEQ^YuP`o$%Mg2oc;qp|~J%4`<;Msn#T2G(S7T}Z)w`V5(EX4t|`}Tc% zqkZk#bC32+h@gQmDHSm>F-1u=uSXh+I4int9;_XyK15d}NCdL|@B_i_dXz)E{90lz zfVsqzddH5M5BJEYM%QmDhFqmY{07hzULtOW@XSr1O$hICDd4x?eq)h_Z^P-zIC#l9 z)0X}J7UuaLz+@Dc=eP*WHkBVfeD8gin|lIeae(T-#r+VFC}C{Tu<~lh`TF`2@!bmV z+$jG-eA64CAXkvp#1q*ZuXIk-akL&J)@;<;vQ)s0^tMws%6}o9K3XkxFuCMpAWeP) zu~OOdg{=BzY(0EX^&oDWK};Yooke9J@;XX_VU#gC0GUgdF836AK#>y$5qJp}tY>j? z#uDnl$&a$PSyE0C(|~*!@lF|v_`{45%wurMsCwVv9%P|Mb=$`NF3Q1;>(Tmejwq1E z&1-)cqI!c4ljIPI)p-L?J38b%CLwW(6$DCn2|gjO?4hNN0a9!8io!1`E_HZ|LWBt6 zjvT-;c!1&&aU2aMSq8As4BecF%T!FdEHcWQr|zO#V=g^D;tGzFXmZ5UvYCdVo|-HF zIqZgnJ_B1?mD+#{Z_9M2_AXjoOd=FIiXh>5xbi1|Bi%M3UBJw$;apE8ox)i^TQk%` z6)xhCNchPDw;zc>kc%+u{L~fevPjrb?>%`L$3H@6KVE+nk5mhSac!Z_t0~=BapUIA z#zKmxPkVQJt$7^NzzSX4p|H~%u~^hA26=iIF0S1qDBm26V_GldwdxMu(-X>ll>=~oG#9uzJ)|NabzR)%)9}_23KFbyvggv6AVCxW0z}bDmnDzI61Tbd z6o@e;kQ7tq)3~!|{jdx|?SOU41eK2PSIR0XjIpVDMcr*j@t!O9aCC5XGOzu)jyPU{ zBcIk7**R_{ZyuPTY zbDCqtjoIwM9vzp45&0ABE3>tt6W`z!4TvF12$IHv5<Am2_}_{dJzW z9EiVxRX#KZe+4<$^3MmAqQJ+I3{*}GR8vz^Y1xv?lO>uUpVOMlA zPsrF_6~vVaHLa|TxZ$rnY?Pv4O9EjdNLfKllTZ2Q8#fmd(H7_`y(hcPBG;z(w~PJ< zYwAjHS)p5vPE0LPJnO&_{@EV3SKE>VI#U{wC+Avj2ZVIvwg3whaRCqJUS zvj=5C&>?gZ2O%~r8^xt$i0X0xv3i4M=`V}LC!9p|;${_4t+ojvT_CJbYga_Gfr;{|3@nDptC_|>j0IQ2xaYs9(kn9fQP_|mfI;d z>5Y@P2Mh|tjh>k-;#0yEiR`sMC1_|1ih45%pf{Wc=$HJGizc1Oa0q1%p!9Q!H8%ARZOw$O7?5H%XAOSJ(9*9=k{cI)7p=SRJ@T1m5N_c2;Xl7hSc5Q;V z2?Yy2O0upH!LhDTh0P|q3ur8H4prXS_Os{+A7(jz{1UNoD8i-Ff>vnHu3eAK9k7m_ zW~e4nZsWp`D29j~5RY51^0w??BR0>@K~!>O=1HkPQ!2Uxg3{q2iQKe(-=U#OpIG1R z^e$m$V9+8{p27X{b|5s?2WmV-o(0nyLz zK5s*M0a$43(I7TunrkeHL92$d<~QO9M6a2ENFYKzL}a_yB-?5|Xk32{zb`4*3xy<6 z4kwUC_AFcj#B5q_pZg#1Bv`wP69w~i^-!Mo0YR(eos!G-p2!7|dexpBQ&h+7LAVaN zVn6b?4G=I$QXfyS7#OWE1uAMUaLXo~3(lBu9s^dZ>gM8>YiMzaMow@NJlL0UBg6nZ z2G=JD;eTdOk#&Uc!AhHtcfQ*XhM$iVE$_9u1ZHm(Ae53IK0y>WZ*w7JjBKl}891Ir z-O}+P^K0Rrzm}tKoQ3Pb1hN^DwQgDu$pJ%`y?MO;FmTMag6It{FGok>IeBG4J;fG>A5&$ zZ%pL{`$6pqJWFvv_+E$=%9jiZK{&>iWKgW-vGuWA`0pXF|FZK_ zCwEbfDTE&fLyq>kwF5~nhHCw$cmOS;W)XQF`CyqHWusVnn+IhXa`p^B15xg1#_Gg> zgs;M1Z3@z45gSf0F3E-qszyC{YKRER1n6AF4F@z3y}nt z!s;**7v))5M!FO*pqOq69XpiU+`w{KVdNn*Uup zVE`eK8717WfH(m*;1ZZlaYlC}0 zX4Qb{Eg8L+J*v;3R0Td+LFM@RYwBe31Jn`ORCJQ@sHmoMqiqX10Gp>H)8#0gZ}fjJ z@qh&p89^4gq%!I^ymw0ah1gbtkV5`1Qy9Tu-6&I1s;a7y$=R3^!8x#qOa_6i?f-(}yOFC`d3yH?sEt z55|FA@aok7pER++bKf&?>7)pnZRKf;btJMo02j( z4Xdg&2@bea`Z3HveDCPg7gTw0Sncq}d7LhDjH;lciOMjl*-zX8h)oS7c8^Gq6$%(n zTq6x8Il@HrK)QeIo{a7S1YX6&x~WQ(O2%<6Off(UXwR|?BU&{aRztXO%a;%Lo8hyN zZa~*SNOUrukPByvAK14q9x~(hpV9^3($WfK z-k&nTq0}qaH`tKJM8(KJx(d^*yS;UH-{7Eek!C0bE{!`x-lO26 z-vLb-rYgR3#PSF)KtK#}`Tv851Q`T;#SRx$M@1m|3n2|~BBj9xYQGK*C151V6ebUa zc;NrfpDvsDlScP+>9(;gho&HEs-~o-|08@-Qo&NNAE6v+`B(U)q;~u(d=An5LV0Yi zyqoJ%X?)-Mx~kVV#&uXk7wr8MQr)4ZM;rmG`G|Upcew-PyZ&9OY(-ThQ+#G2{muRV ze@qb5;!JxlJI+t)6W>)!ix!&0Kn^W|mKJJCam^9R`-cCONLkF`OW7g|7sLOfNNH1r z=M4BaknNvRoTmK6!Nvdm zw;OUPdP#}_tK#!XUp(i9G+MR-`aPulv5uK5r8&r5gjJ)(>2-elhw8fOm8kqhHp#o5 zn=U)=qTCXD)ROmlA>S&uvj&U@Ruy$uo<^OQTz|ZCi}&e+ zl~qx}RrkB9HaY|ylkDG2X~T&3w&48Ef@)4h%9~B=4qYzpYaTarbXn)^@C)VEx)Z66 zZUfCBRq!ce?4`Wf!S=8NFLAm zN<&7FK-ZK{q*3DRN;hWPxpirmAE4NlHc(9xb1<4zey4G5qIrOMC0#uUFK=(}WzL=D zbS^G`j&FVJv+t`DaIS)w^(hK^X66`p{MMj!I6xU`94n6951#HkqU-giTt&$`} zSj>W&4<9a|^J+o@;?-v5;Zed<%q_vM9T0=GqMbSc0z@N=-tBaPbdarOfhjZVyMpd~ zZ9#rfoutSP&g}RXJb9A z4;a{bbZzvhxia&SN!(R6yUQ_5z`8pbI*8j6ye~5(O{0eydgg*T@!59sRAJO$$|EV@ z1yzj8E^{*LU2)vNeT)dJFz>Wa#Kp!A!wpBw<(L#tv0|5y&_y?k$*ISwsn#he2iAN& z9cdQZgufXNK97oE=2~jo7Y*VET?~sxi`Jtn0+eSb)MQdpQbatLbIvVcPLYq%YHo7! zk`p7Jm)BbU38jF(k&&v+e!J`IYR^cDZeAH>m`S(mQJML@f1Lp zzXcvRr@^k*>IZQn!{{5d?K&;JU&KD1xbM(PCc%M+AyQ%zYqn_j9BDgIz zc=#~oadQust~QQHey`o{>}gA(3hChKU>gue`E}8{cKC6Zrvy|Yw2aS*^yusy4=a_s z=Z!OtOD;IHxtPuF@9%#Fx)w7Zc9?v3@`}oR)Arw65HQ(n(fZ~LYDgg1am;4sH%eHV@(6f6(9%K%j$l*A*C*V~KapNsWh&_pE&wQslzYC)lasa$ zb$TANfxru=t`$?D*AlPmeo4=K7>mNy9B(~&;#o-aBz)kK)faRo!#Ji&$X^l4>1vZY z!FeZFWDhO$0=a_mmi}SRjQI}Y{^iqV6%Y_8Suij#m`<*Jf9RNp$LkwMR^b4O`nZu&iT#Y1H9ZSu&%E zQcqY5-qLS?LaqRovps98E%ZmVE*katkSQh5>p1s;ih+CN{3z=}f`x9rBrO?s1xhMk zQ5D;%T8w2Ki*i>cjUZwsu&1Ba3~VKaY3eb1h6#FcaU=!8EP!$f>v^^31dd)kvBtx+ zMkc`Y+`s!2t}dECU=$T7l3maHVaxL;V2u7#I)nQ1w3io2xGj`Pqc0}4G5>q%0oc4= z5+ey2O*RbKsOz}2K*oBA!4=Msjt&s_N2M>^qx}dw;hw#FpJiX5J1y)Y4kD`xV=u^- zb6F@x#2SMow{>!=YYe4+iG=|jI*YGtzz}X^01FO`EKcrlM|SE(zkM))lxo_Z!{QVc z7W&~!Us7d`*hYX07cKs&)ffd*JddE~Vk zv{n(iPWt+McfOkfN3-|tJN^F!PW#k?{w5r8Jgj6uYnNd`B{VtczkVlEijC}X*!L)> ztwvyMAXyGtom%o0B{H;+8Cp!@42iYrViFp+HGQLt@`!F9)rQOQ_yPb0=sUj?AqdHf z*tgA)zX?jgnXH#kEb?4|ZhaiJm$Y*jP|HX)fGH4K$Q>Lt)k1fd0?_?59mn+w^GhEx zCV^+I0%-yLA&_Nd{;NNkU4&1VcK`lF>=hfvX#x7; z8;O(`D}Mks9BHV5Z$W27!(CPh9ph)xhW>jMYW+t-=fbw7hC!ny9Ce0I)1?Qjf7Q@lH2HRhciYLp_rUa$>G zSOzkbnW&SORk$=Rnh8@7;60`pWO5BbUnL@xhwDkaRrY!5&~ba8?FY~yf)>nm z3e^c-J`q#~b!Y^QK=J(e>X!o#84BTUB&s8sg@^fwEXX2kSi)O36m#6&b(-P>^MyZ# z3qXun4$+V`_OrqgvkgQJiG5R0z)C?->t8Ui2ZlfhDajZp*A z^d1I=WFCXkcr@uPT~@gFSpx0<`ePS4$$mV<&tl*&{;XvuJtghIPyBikHe^Kf|EBNL c8d#%ZJ^EL=)p>z-G)>b?I~z5i65;{o>m_P5uXYtAvp81wt{lC0#qH9OZ(D3o>7 z3uhH56h;pUg`s`*O8kcZYjY?55wq z&G>kZ9^pB9__rH2HWt>xyu7CWdx7{`C=EY@{7dZ>Lki_m zIQ8r)C5Pa_7JCO~<+%@|B4T?V|4uR7^y}$i#(Ssku`8@lV&2GnX*tuuTl>PVGDQCJ z$8R;NuNlMD9JXvZcQ)lWskA%$emk|7`R2RjtM@v(xsQ|-9HvHp`f$AX^qAROC@T>J_ zD9b4ni|vfRQ79g7f2A-`D7)|c|Nbq^IqB)g&z(JcqczX*oc5Y~f#R;Sy>FO!ZH&jq z#wvZ}#MRj*+J6*Ge`j)ZbnLs&TX3*nVRxM15^K(nO6iG;GaI(p?qFwRs9-qn#gcID z{#N`r=ZD0diCm6W@hFH{Vzd=eU*_0c?-8zFjZ$; zUR$wpW$bCsHT_LlNt1(3J(*?AIo4fWU4Fw4N3^pyZQfjys4e2hqpk7n+c(-ot!9S( za4TP<05|u=sW$h*Ln`sn9v&3DMw3s!^rLl4q;x(4INjI!|C0Zv@)#GMom-)eqH3@fi@W_!%>vkTm*kzvg#5n)T z+dGDh>4s9WvOFW#taqRJbKCCSCi%`&JpISUY<~30$6b1H>miF64NG~fCD;ChuCA{0 z$m><6wejj(Sy{~s3JNA`RCPn_n)<4up0VZj$Htl9asn>X7F<{a{u>oV>siGIG|Wr< zxhETqle%d9X3h03E-of@iQ1lye|Wyaf~w?L>1AcUy}LnEUtj+#v+vzKB6eysV_n=l zJY}Koi;m9Sfd=s=4XL%sSI(4t{3zI!S^Q{ETKAV{&k73**RhD!zmf|x!zv!7PL}eO zXD7_PUd8$}D=UlNw6U7Je;2=zuOF{&D^GSiV^f-e+>1TBxPZ0T%BWPA87rUbB>}t= zQ40&>J;FbNjdZguzB%s6ZD{ba z*I{eeEGKFy+hjM`SQ8!|KG5nsoUu*U#kjpVWfBp{QPd!d0;joBEQ_Uh-Gqz`l zEzT?no8HHMoQfY33ugPrMgG|5t4f*A%9$I=VawcYyRbgx32y^*vBu%BJeEfJJcZfzm z%*90yHWE{(uJ&ki8NZlV_v7L-0d?%d#6Uw;!h%fLocrQT)XdCGg9pPhEz^dEhT2p; z={U8NaI9ZxLfg${t2DYk`?csySxm0>IM?|nN9Yl1re()@dP7>2=kLFhz#B{{W=R_u zFp-v$N-YZ!(F{DMclg+`$^$Y_AIEv^mwZ&;;U!^aX=&L#JZ!#&OC|lqnY)co_DgEf zFMPHYIl2}ZD9oRGAmrLBN2wQVO;V2=9dtWI@1u- zrp%l-rPk$Q2Fh_N{O0E7nptMr1sxS}s=T=)Ze)p5MLtG}b@<}!gB6~B=$PVPZVEBN6a5rMk< zGil}G#-pR7TUl7lZr;3EiyWw#ZE+=Wk&P_aK>k!~8gdBj10CP7EW_;T)$peKI&OE? z?=cb=7dJCMudn}7(o2Bq@9XP3oM~Jef0Qa<+L*3YA{D@6`sLXPiLL^&Ek3?<)@voAs>i4&D z$1BAssL*T-=@l<-FN~G&=G2z#n8Jd^X=WwnBMco^7WuR_FHU!C$i+_yI5}IlZ@+;I zZTEKFiTX^_rsm9Ndm~@o{WFdPtWYpI?FzOD(0LS7`U)7)pmfhJTL}}qah$sHPK{oaZ%;P=@rr!E+q3C*FFtYN3yWaF9@D?7^qKqOS9>_8Y$(M zx@OCvdR)g`+W+vUEw?JGsz~rp4z)Np2JsY3U(KEVe&0GH%dqOh_wV0jb4;bA?#;~3 zcJGXO@L(;MHZ6T(u&J&_;M#|^KGzEuCZ7ccMn*=4sQy~&=oaE4V!8X(#b^4q{nba^ z-QDj$dX(urlvBKZw_v-5TkL4r$B)Sy_n%km`0$XMjqM$Y6mE_5j|oV8W~K(S&0zwjLDW=T_k|kM{F(8dXAxRqcjl)Gm-wlfNL<#5+m;vY^Vv*6 zX1#u*%_Z$(n0O0+mzZHCTbz2@s}|R(76I2;o7{oa5*csTnXZ7u!L-!WcpOxfSS8Nf zrNuc?KLjk=PmWJaWbkOyvTL0OGwKfi_FIugrb*vy{V=K~HK{P<+Q zDUvP;7N6_oVpG9!UT5yES08N5_)=b8PBvNbl}w61w`LIAGf~G>sJ%~H6?j@c_tOlPnpfd;$~ZL zj3QO%iyrp@LQZpS!uVziKslUCmj$2r`ktBK_4!O)&S*1dyJL_(#mDL^RZAfW^;pCZ9#0#SVe1YQjJ%vSP?R|F3!5AY%4SK^_ZBL zs02PZdW78gJ}lknUXAy?*Mp=u`+f{DnjNJ+q7+-XghZK3E1{4Ik(?j zRa#tJjFZ!y=@o3-QyOUf`RFXl?M3g0yH4=brD~exynpldtz3@l(&ECG;$n_E$wpSU z=@-;gk>R|m(o(yjW<&3R(PGx}n&S7{bWOa=`mQaqh&eqa^?%2)cZ=g;OZC~79r21$ zvXis)q-TK*NQy=eck-u>b(Q4m=%UDq@rf+bo@}PEI`!@)5yEd+`EKX&8;`lBJ4@Gu znyDP(;woWeWE>dE9WKkj`myj)qg=$@7bfZsd}?(W|Dsi*k$&Ub*2^@zL8HBU_h#Dk zDa6UYyhBZGZ@-*NPt0vD(6M}#Y0_{N+ql=hLT8`jdrIzyJRG@@K-fvFmmV zR-o1me810|i7Xn_5L4_7m`W<%nLBG!cJS%fPmgva0Hk*J^aRkoM)E$)%IKzzd546U zZqu^3nSs<$V<%9)ATpHw7WKgTOE0POj&4Cus;01od6Ky08;h6av$<<~|*@>AmFBDR9j__0P>5-9pQ}$6aHL6LUV4 zK3erMIXPL2?;gr`8p6frdYo#KxL$(eFYDIX&Z+V6&bYD#j?+eKHO+GxQ56z2GK_k5 zQ&9Acy-hklKFRU#D&1#VpS-ti&{|(sHD2wBaRW-$4Ise8IaHuz)HZdblAl=8zSXgq ze}5D5n_{gaZ6R!a<4CBOGsypMdt<{i40jR zG!++1BDebm#+bJiBu*wLAE}8~uNpj@<;k=y7TE=v&Yv__G?Ed#BT$XQtP1sC;Z7w1 zjl2VMCcb8xYDPvx+(0__j_?wny+C@PneJ!WO>;9dIc33fKmjeYLyZ}6hm@i%8n3k# zx)+wp=#}`3y&V0t<1W?|dHPF@PMZEjfZDE|VJHN#=ork3{U1Mm9v^==K`rG?h8aSi z-nMG}?n|hwq|gX2Zf0j=I|xi{TX^dB$~C|I^2v5(cwop^Jsxx|@g5!?T7y6+iCr_C zx&?X2U#}3K=QeZ5R-<2=K;P5%>Y15$SXdipmOIk4E&3&((D8Ks&@${#bVbFbx&<6& zs<&BA*H)BrS_Z3_lM-@DKtzW@Jd3EqB~m<_u61^Fyu|BrLf_`v4OY%h&}(plvKF*l zCu(99qsk2FBBP^UN=vH&m=la@VmS{VK3ra$?Y^`)Ha2F}V&0G%nVOoK?mB0usu?Ek zdR@Q2Dyp1wDui#bXLkEk@7FUgUc8_!Og3d8XDuL$1_9BaWg5y%K$A3-H!jPqnJI%~ zQiql*L;uT8tmQ>XFBY*ux3SGzx4r@J_44wXUszz@Bk|G>RY4f3EC`^aXMFy; z!EBIi>bhM5H;OzM?s<8sB86%X8jP!jl+Z7i8H<#K4EL#?x)Q4tT<%gk6# zBg(0H-2y-=*U_U#+n#3_l!qz~q5sSmb{Kh%4lAJ{%^)n*Yp^LR&S|3Gj9126Dc8=l zZfr9EY?tqxcwk^xL#Va7&t}g%cVe%;zh`D%AvB0^vS?|`b4+ua?Tg~u8TS7EHhN!F zSS@zZ@5z&IMiP#!3PZ)x!i%SEz893AC)rordbg0(yNr1y?0F>CosQnC`t}C#gMAJo z_wL`XLYa=MjFj%lX~}aOXGZ6b!xN_1!tt3dbeNiwlH!=z^FX<5*Nrk(ZVskjetGhv zutya*#5zF{-Dg!n&K_|WwQC<9d<_<|{!ypvo+H+#pr9b@EGp#wJhVV<=^yp3zyLh%3oNmI9=$8ct4qJay zB1{6ZFPDIT%Eh4L@~9BiXjPOGG$zuU%-RaH(e{;0EJfq%8E8vB8sAf1W$OMtQ)o$i zbc1f6!@)3qV}F%It-v1ZF|AI~VmX#wfUI~of1R<3o7Lj>=Hx54kMZ$gPv?SMUU~cY z)S&0Vn)f3``6Dch8yZAadd7rr=sOmi?FNTqe70`! zX>*bD4+-J!b5Ii;P0HL1I-=dzwX?_O1-qn!LpI9dnCrDK=DW*mTu86$!nyFgNc4Sii2NXNS6cYXmt8$QQ9y<_7UySr(udPF?%6XKv{B@1qINDD6~PqMEPnjB z($d1PyPvObgv8?X86Y7Qw3#(|j@HW2mrh-|awR8|Dkb&v(u)gox%z9$i*v+fh00&o z={R3Ry345*v78)nX-4l6gyvs8n`FJF1ev|qN{`~>tqW+KWdM?fPNH~E%0TXlpSw0r%v;XVM zyLtKfSn!Or#6rB$2nmEBBv(YR(`17IC>lZ1XRR13e%tVQN7DQUMd691z$!5o4zpdG zH@`$%pbV&vf^e3^>NQnhv=Rw<2a2br;>DRQ8#nGnWq-JX@4RNF3FpWHl6y2F ziX1d7bevX)8&C{V3Ab*6sA{xG%{8Zy_SagQ*#$+G7hOHX!y}6|e3?k=kF8qV;#)U{ z#wl7R@Yv_ipA}A@e#4oSq?MD3^TwfPV37NT(X2lCb;i8b^z1BoXH%xBNYtPlJN~bs zp<$F*CaE*X;d`eJZx|cSXv-Jg-sUlB1xgsLiM2oo-ZfsBVl;K1PAPuC(U$ z?NvYX;9w(vxow+6$GSlEn=bRt;#T5PUVr={*wx`XKL44%KMUwJ)J_7dU8d92Py(tx zt?|)rq43+QHiYWEHyFOf#B<4W&6ac|6Rna0WFI~^7graT=Gq0odp6E;%li)=*m(M* zJh_g(-)2-B-yrmbB30ZLJes5VB(31+(cg`9pw*Cy!>*VaZ*H*;&#e%7C+ zQZ3nOai64|-~pv;?0ZgDUIm&2dRYF#s ze+b+5M<5;afkr49kJ}EG%Dg)I2|?fo(aa_!&FidMnx*sZ5p$}Z;AAda*T!)86{(y+ zZqmGHaufCLZeUeGZ%uczu&_uxA%Zxu9c)w{&KrO9^Q*ZAJ48;d-y?ibNJt&1^i|v- za7qx+GZ=;h)ZMUg%Xh`aJu{4!1?0LoX#HS{KV=L6l94JA(b3V~)fI_Vts6rWb>z7V z>7n(Kl9Cc0bUM*&aT~R=S^Q4<$Bz|Akk#?(d|83bpe0m$#r?pL(eY_JMI9_cOM1$~ zlF%u~fY3FtVDpT^+5jPZ?fv?)MLhBIlkwdT9z9Y4X#DY+D=FLcdM>H1ShlDZq$s(> zhTIs%D4JNe0T(;_IZ%Ng$M<*okS}_F8*lBSJtA7BphTgUFHvvJ)I+72JG4rUH592cUIuW|@zko?e%5!5@FDe*f-WwyPHC+Tb2Ro8F33 zBKAYoXqyL(M1Vs>b%lh$;t814Tbys}qkX%x>&M#-6507?Ce7LMVeX4LO%75yORRV9 z+&Q#&?}%Ts&i-yUZHwt4O+ix|4HffxG#>gBAEJtz$8o)I6yu5M`$G#|MJT( zV^dRgDDTaoalH9IyisDRk*2f6$94_}2hulZ>1G)DA33(NYKhr zK=Y`CE~tPW+GkxT9l$d-K5p%r#or@{@)E z7{Xjy;)3aIA@RT`Dj>+}7O=;JBZ4@f4zO$r#!^kYj4vUI5yG?&vRYwC5Nd`WA zCTJO+Kl)+UWQ%=UV8dt{H$c4!;begO8sgQ{OiG^~wZl$mG^FZjH}z<(`~bwv;?T}O z`Zl_glB4#@?sxZhhLvH7O=p+rdqYbEJEL(NZGn+(CB(#9a8OM^huanJv9EuygU{s6 z@4qCvS$}ZF1rK}z+BAejkQ^22^m*UDeOdvrf=?EgggqPw=Q`8kZMPa!w>We>Of>ev zw=M}fVQz*inFTC-5}Nce-pQx`fRLN7WCE{NMagA>t9Z)hj9fk5T!)k&JgjajRQBUR z={(DD@St3Q|JAaBkK& z&CGa|+P1;9?ZKm>Gdb0<%3OrQ(Dx)hJ11cZHGwd1^jRRL7Fr-%ZpR}L zs~TTx|AkL=r$xVlkkwNCRWPTVk&5^j0aR3zZ0`I&L;0G`i69Vd<)sQ}ra@>{^o zqatpmZ?dxm34=P?QQSl*Qvjl^n>YK=`Z`+~f2H+Y!^g_vKHA22FJ(>S-i1+r&b;qc+IPs$H+P!B% zpq-d&3w*sWHvsYF)-aF_2J>J9-5#T!X$y} zb6~PsQ7%VY%>fy!77NO>X3gh`m1)jXX1k7D_5cwU6c!eC?%cTsWmWAiTG7Ct^cFD9 zDg`c%XqmHsIeVPOMFJ>KpFFujNKX{Gka5AMUF$vci+TBuL6rgDA4K?Iu-plR6}s|f zdDgb-03D}Kv8qXC3=9k^aVp$|J*AHd$zNGf1bEs6Xi9p`t-t30AA{WO4uPMXOl@H^Kapkxwu7oE-y-PN-ot`mFAz%L{q;btNyYI$j1S#gptP1kMa zy=_Yw+Dk&!(`U_v#+W=%Ykx9n5CDL)UOWBhy#mDXDZ>lcjffjWHzMpX+J$sN8_XPN zAR@(@HCDMFM?M`pcFd%=;sp_wxD;PKSg~rACKldw_D7!?9ptJyN+No#!*zO&kUh*o zMC?h|qmH#R1>fhWvwGHq@p;22lS~^rqOF@YU4{OaNm_mZ^VVSF{P{&zH*|@$z<@** zA?zko4DO5(BKd|ea_`=~tEycDOQZoL{gV$9%!3X< zwq|^MJdG$JfM?cX=Vu<;L)e2p*9a1NU`;k^i}=f>{ZTvPe7bCp_HQ=l5b@~&XG47$ zJiD;4K-fF802UF;d_XTk63S|F$$lvr*j<>vp7}9?#jq?`s1dhYpj&jeNAyR<87HK~ zA0X$$CYP^ZmILSmNfQ7ysUe&v@@#gex3m(pJ8rG)WaxvVTe&qNz{=e~VN2Y6(ZXzc z{crc0QI-At{a=9uVz%Y17&X4|X#3GyBdVttz|#r4Oed3*T^;GW8bszSdQ*O@&X3_d z?>FL9L?JV5T3kcn`zdh0UHgBKwhLFdu%xD?WmxyfLcHouFDz!FP!{);$}G)~dqMtf zcI#+`Xw-8By{uVb4Lk(*ydVihqmetOeg4UBl+e3Szb*(bE=QH;{B8toJ@CTCi^?V@ z@pRYbKk?4JJ=fYEKmL_y?C&4=6c%)qZYUr^`-A^Zq21*^jRLDd9y>d`VOxRgidJ)k z8ye6^#^G&5{kLH8_V!jvQ!(8mPg@oq$QM8 z!>{4za%!nEGD<{$-nlatRaDz$^gV!-6}JSC;L5ezpIdc(j<2SrH{3e-`X%oD z;Li8QD6gWT;&67+vTKBvWSKOCBRHZ}5;SHJ)jv=d4B!2+an0JbHVJ$VBbQyFC~IU} zyeOC-lOAl&QAE?4AGID!`}`ww$uI?HPX|qTx&6#bFY#DF!y9N!6Ejf<>IxMV6yOEm zHAG`Z8~e<~PPOhXl?RP9fjZxZKXB)L$93+hr35pG`k^^=1y>gf-nKE*G!7sNVomAb zweR@nARnJ1KuuyM$TzIElm-HOqtZH5C_@ctlISK#Hw=P4N={-ysZQHI1X_X< z(Koy!AXOl(aveTgYTAVJ$NBiB`!8v*u{vUv!(eW#rM*r%yNk^Ogl+btqg~%%v|vS4)SG>v{KX9F}LNuuU`p zbfouG)w1;TKdc_UL45k8uFI$=8z;I}bbB?A_nsm21x`w!d#Fp=3Kv z7-?I+-;y{7=KBfT#88VS6pb24F;Q3zBcL(jflvaWmk-*==xeyA@w1*#XIXI8B-oI+ zt5Hyh?$l8XGfcZ7VonDo`dFG`qBXcUI*@p|y(2&XAh!ST`v^pVRP3gyJtwiS!8`xQ zTW9bs?sEDwSiHZt1+do#`K=R84_cM3+#zlI7It1<-ix844GBb1=|{Q5rZR)eLlQW| z&7EMuw2DHR-geNR;fR=+Hfm!v0FP`ANPzq(EW0cyjU0bqPDR4E`}1hWqSPCcj{@;} z#cwTzGLrO7vwB7FeK1pT*y$*(9`q2E5Rc>J#AnVTt=ecpVvS~(_V2yR>UIfn$4-Sf zqYSF(_?0)mi!R$wp`4`3A2y)Kafpa$5*7*)b1Yz5w?=_gmlU8?&YLhcu(Kc0F>YFo04uHZf5HnK7Ve3?~mfM0N(6ltZGTT9903CZ9cd@&cl1-9mMB^)1fr z$Zxl$n%A}m@f-h`A1^l^YR&<3dZ?nH_`!~sgr(&c5Xd|^hcy}bebY8d;cP$GAPzjl zbnm33Bo|lL3~;MrBSp(-sKV@2U?I|(H`&2OQUx++%a$#E=50Vqs^FFBnb(RRy%A-` z`8zHDU7WE1t|7n2kK1c&5w9&y^aR>qW)m9~3OqT#kU?J_xEOT7TYPEW8f*sSQ~jCA zJ-`0?9OUyGE*9MRGGt~MY3Zur!lfZvp=pyqf^MM>#BRcG5~!Y800aRi2{|iN1!M?f zSpq8{$7fKcq_6*qxPpdSa;;Y$L24eo*Ckm5^`Z9MHtmS}JhThm$FG0lGQ+1Rvm_ld{&MudmOjKKTVn zNyZIujilQRYCPP2^fX*vHH7s=+aHTGXh#3fXRq+@qBykAp%5Sk{3UO1?EP5@)e@cY}vGF9~JH~6?k2moKUFNY}lY` zWo1?1s(1bR!TtN++Dh>AXIR{WLG$2QcBhp`P-a_bwmyM|Q1g3Q+EDPAckb+`p|#n1 z(hv6&s(f|XW!dSb0@qw5p8lW(pNQz_gE)@GrVBV(hRA34?%n&GRSvEO)XIC?Ca?}K zYBnJJ)TBBki-f+Jk9U6|)ZWj-Au2Ae1EWVZj(#=xFKT5^n0q2{04o0BHn(MrjLg6O zdMP?%8HK`}d36V+sv zMs;CkoUmx)_`FVRggfolt1LpFaj1PAh0WUUa7?D z_q%tEfqGV`Z^a)|E?t#l#0H&g{U2-x{ns&+!CB!_iHCzDgn9Gk{U~Ozt9&%g3G73Q zplV~24m*)Alszy-xe00ROK#-6Kss8(#H5HLfrg{|Zz#&L!2=Ap*swTs)f%*Upf0(~ zyE=-!2z7Pbx?2V;xYX;{uZ@~A6Tsbm*NB2DG&@nJOHLHGPM#XM4a8h|t^hK+yNIsb za5500Q41|&c5?8i!xP-HL5n1`-5OL^Q5bg*$c4rivpCBD=RIM|2i<+|!Gr4XbN8$B zo$db*XR^o51FNuP#S+G#m1~y(gTiyT<_;b@Bn1{big3t`bayb^0%cT% zSZAk#(o%wXgKM4&mi!hdet@%B#yDi=_O_2549RHloIvxD4vA=X0I~t_G`s*kZ3xR^ zBao!yL|Aa}VJhy|#@Gx0!!7;y$+m*;6UZ;O_LKB~0kuy;mVc+y<3RuGfYyVdwu2lW zB4{b0Z^D%$5yM-Q4QB|6g*>I5RbO8p&hrayDSnGOMJwMaV+dsTIY~)l+=3JOfDFBo zt%RipooVChK5PQAgX&Fsi`oLPAa$w#NCq=0*NW4z^L4_GJH%bGK|g9`0e1u|KEwi^ z`_#c;feXg!78Za_OGM}154pAmaAgABM;vHfB6@)Gi39r(s<@;at0b?a<%&e@uRAAP+ztw;`VZxgNz(d?lbD3MediA9X7yf8MhwwambXvU#^j-x7Dk6g% z6c!dPw*}8fT9d9ONezu4x1|O9x&<&Ah6}S3Bu9dKv_T3WTA>PvdgVx|Eu<14FVkR@ zWvAl4RGw{xeU#dDhdaM_L`eB`@v2a4#-ty0I)sVO3Wj%yeEXGDEw zgIEMap^4!(UF1O}RIv2xpEhMlmz6#{AqPK5HJBuJSPls_4GWc!*M8)}gDj#f6euMx zAw$yPIJGf`h4lqGFPqQp0UduQp=K0BRDx<)v&OYK0KH&wmKc9Sfc9!H_f;0a@P~8@ zToNbIXAz2`iumdU9~CdV3^I|Rm$-bik7y^yY=rQaM;)V-?Hf0~fct_|bNR~tg0-;PW6G-w4u$xZ(;}Sk6p>RbK1x{7l*3ZS&Ygiok z5(WCZhzG93(mqP*9Ue}C?yd;xzLI>*^XI=2$q@ABqzxyzkcMNfzV=Mft;V8Z}u+)0TfEkQIS0E?d_X4Hn6e%+l9~ukN)7nomntqFD`kb zDSSl)cTmBV@E7l=Hs)AIRaREs3nvRKIXby`1q+j!Xo_<)bRE8V z;!6~YYOhBU?Qn2ec?6rHuC5MEetukzc#Z!R9PkDm2a8|limVR~t`7#?_MUIJu4qvQ zD}!9Hpgc~IGK?cOogX(&z}@idgt-cWVghmM@Xv6GhZWBa)_ZQX6^^LnpUC#j^3;Ld7RV9o_Vu@J>Z;d@W`mN-xJcfF9p%YsQJA?Cg*K0ZF% z-ip2WLLhM!2o>XTx_dOuV*@;vcG3XopsZ@b&oRl8Iy~aBZ-%?wM>$DH8=&P2zX-S) zk6Tkot34`5H=lu@O}$zVSNC;bSz6zYj5%!AR+dsrX>h)9Akw=eo{OY^a8KzAt#FNUnY+1Qu8*XPwAJGigRux)0 zn8RX#zofjOT(CjzBO?GAp#cGPj)OgrQ?t@PZc6Mdr9;S4c`M|7mQ!zNjK z%}eYR9HQXmxY)rzqF_Bi5_Hoc-P&~4*tBF3LO}r?;twD@BZARTH|+-MU%(Q5x=zdb zAqbR2aN%_L4JDEmj+?h?{Vlw)QHjW0usCa+0fNap9jz5;8?L>3UuRr|Aq;m+#KrCg zgMY7h4~q>b9J0aDgaLy8$Tq`|TT=x;q{BqDR)SiXagi%cyc;7 zQ7ci?jzb#MhW`lHMEB8;#=y=sAjxfEwh?d}TmF)`Isi7j(@|>?E4i64l4U_Q@-+y9 zHn3*(>VwHU5qkG-;S&rX*OC4o9v(s}fq{+(`x}jd57UIx&A}cR0mdfnBt)_0b+)T< zYQj;xkv~f_MDZoDWeSi1Z`P3GV%~wQ0$Rxs)H|^t5EMXFfN32rMg3bt$Bq#8!Yu);=5sa_j?xsIE5wB~ke&kU zH>Cr4ybD(0&}xiuP-q~P(hbU4(BSo!i40f6(K@8j^WrXx0wn6$so^9D3}mE7BlY?( zbrf8aJqVee7_o34q!xilsDy6{yR8g=wkl$RJGpHc57Yz#17Vx&$$f-Ma2}Q#nBKR3 z6er8KjMPUs9>BqY!1Tt~>M|;XnT(WXxd0U4c0nM(c|es3$OMF2hT9wUIcXKV zdo{MBeFqDZ2V6Ee1s%oqi+G1<`L#BN<9ri+Rg#E$?+Bor-o)IY&$e4f(52I3yz!dd zpRO?^!#rj-MZS{U;7P-4DBq+4w&0R@nEXYN&NQG@&5}w8M`oL5t~8l^4C;K z)DnUzJO;;D19;l_KXpmluDr*ccqYADSp>}2n_9x-1J+1qR`hem_2%Azmcl~G6sW7L=)W0p619;_Wksdy?e*Vn62@UQG!1J`jZG*(0S zryM5(LFH9fD`o9Mb8D%1by+(sApDqtAf=TIT@4k?nStwfCDn?J3ii)C0z02IxNrY{ z?Gjl5{nG7lj?v7Z5_>w6y6^cMYd{D9vt~7TuzwO^I6-lLb@1InQ$kQlVOPpFgM0d5 z(LHkKFYy*AW;HR2#R;U{2WItV`)6a4Sq6Fyp9OvNIf$Qs-Kk!H_A4+tvk-vQT@L z%vs5U%GGp64I|7RY&6H(79oQ81_kxok%r9-x}nFLRM=}1v~yK34SFU~4~GFB9aS>V z;S9_HBjxy@HM%n3ji@y=CETLdp(lJEh;snk+dxL5u*z;#EBtpRn~{!#SEML)okFY? zh{Ko2z$JBcs-XI2(FpQmtb{P{$BrowFE19P)82Y2Apui?`Vqo9IF>#4=~ zKs*}5oJJ5XafBN|D!<%ejcNq03QzJmxq44;iJ#P5&nf`>$Akx@i}M zNF9hmHJCNz!PO#VXb?@%G=z}3Fj51)1rvuItY$_qQ=L;fimz|rTThuLfsGf;VlpNj zq?n$*ek@otBoX`nC-K(x@5bVPRt;eRO3XyTIK;_G z^r29mZ9jv5U~M}x(sB1imjz7VDPtIw@XzbkDFe)+C#ykQiy^49P-2KZk!Wz!(+*{n zl@wn=DwW!ZKi`~s_VlS*yKDFe%7Y+^C$$orI#er;RVN_F`$w~N`QL=IGP)fij$@wN zIa>h~gem~4QEhC({<|2r=bR2Wn8Hv|Dw`|&uQTKLd>=6FhyUU{;Hu|R+ucgj;MC36 zdH&Z2qXlk;B|-V5QU>E7-?(|=AM(5X>VHC{$0rzn!vg((cyK`*<4Q`qE*n-3d4a1! zeDmhbOz9HL?xYGVvj3Z9B3k;?FOrnyc=1&*#llw8k3VaVd(3}+kHmL3D^caMYSK0} z_k)Y1BF&df|9!u7-tpl*D{8k%t^C>E|Mxy>NupBik6!Un}O^KbZ=q-An* z($#@(6zMYIbaV2T&iA7X7HCgNU*#7Z91q8h5j0~O+&lMZhbb1nzX`p_^p}l_Sz;bRy9v37^1b*j*Y(hCL=wN9gE9dA|2k ze3d+f=*nt*I-X*Rs@u~HDPuiAG#IQ&P)|FL!epP}=0O&OUKGuKk!pym zW^Jv(n0YuleguKSw;C6lo&%9fnOZwO6>`n>K4GM*dLNrw;-qSz1cFFf6W@kYDy8tP zdVYzOVVg~xXN~!l$&m4B&{bQe=S*bX^;MoPY~|4Du6Wr7nvFbnz!jbZkjeHL5&&SN za1w4_t4Wk7K`6(f>IV=3Y$(N-kWKE>9_;&u)MtGZSdkPo71iYn=hf z+8AlejP)0Q5Z~jELKu7_P?V$57!jCHhzHmZ)S-JOrrko#mD(q{s_2;+jS7mR1`Syv zc8lMr`U1hA*RNlf4|wI_K@l5$w>d}L>ip}g!D6>9C)a*x;M;CcemR&a;95D3ImO~o zvD+$O`J$Nr2pjKzgqgtxldOmEqze+Ps08kxC${luMMJsa#4iznb(m%9d?9teFl`4N zrbcro6T^mcn{`ptesJH z^lN(Y4#6hNPz;sA!yf}W8DgdppuDhE#ASEutO)i0%~uuG(v4bt2@}^_&@>#zdhtxTQg6Dg_@rfSsPS z=z3q?@W>XPMe+!aOtThYX&ISx$PsE_vlj470+z%)2cag*vJ~`Mx$U=@;z3r_MoMoh zXV!cWGAL+p{kj^A8VQ)IoH1vh!*$h{| ze*G!{3)co!ojjKxUnR*?F&fb%8+#9+5{Y!pLVe2svC{xA&;Yq(@Ge1!LYPS3@vR6r z0wb)&C-wkhjMSO}#eVp}fe#>}e0pZl|D*w$(=v!h3sY)krxViH1e;RC8=(BeW6)s| zzI$esKiS{pJQydxf^}gsY-zq2vltgCqRR|K%826{tXCuKmSW0R|Fp>L5WQSP$p!9i zX}q&`$JGFBJM|Z}>fXUric9Z{yYa*xX?mi?#s_p}=C52_In%4_-#)@v1o4M~7p&%$Dj-Bev%@`!s5jVeJWXhVrVKI$`%U721m==9)2CXz z!qZ{aJBU9=gFr4a3UYIe^d0<)@G2k}RnZtu;L!_xI45;tQ_U~&4agA;J(ycB6ks`o zaUt@7V1YiC+6>emCG+x5m?{^QPrcSTrSmi>C>Dtsv~u(xhqywYGAtiEYB!)pi1*?A zsZcP&_i4eKIv}2Xg#Z#u4jVENZ@-mPI@)lM05{N8fcN>6*vZ6ZdtX9G*Ow1a35VBk z+fFg3bO825sA2u+@@vq2MT5}ZVrzJ^;3>n8WQ=#0VEPo&rJZpbs7lgyG#@y0s6W28 z*s|wXtq`8s@f^X?k8u@6n6%fVVG|I@NM5r@Kg8OhstGBF?Gsv>T_?Ts@P&?auGNADOdX@1XK?T0Q8vVK`1*P9t%unlv0Ern{Vr2#2OwZ-DDcm%5*~ z?R^}y9o&IZBroDjipd+L*kyKTpTW5PJmozlJh?K& zu*Y>-x)MGU<~@5ZPL~wph=B05^2`gaFxB@|jek3$qgHfp_AoPtM)KvGaLm^?=UD&D zDBwag(J`WgVu~IcW<;Fs#tURV97ZTpOf`^aXUK_;E)TdVbjCoW7WxK1N`f;AoGBja zlh_1#Nm1$UrNhC4f?*HRJjzQO9$Ay|0jNCiBHh3$r^8OBiEe;!w&!eDzkk#4XdF+) zxk8NO;Q!wucNswjjCue5y*i#^bg%!{d!A%w8`O^hGzC5?dB_10ONeP(K}%^Vi`(>- zTx^6k9djG{@N<(p#Hw{1JJNHLqIQ-%#{n?u816C`pl1N-$(q|33O-ziO{NRv0X9hE zSI9FwAa~3Gk%(zoj@k-b(}o8WPcoT&w^`1iVB~7INmv!fK1yf~ssMRZ6YzpQam<7I zC66e=g#9&^MB22_Gt08l(sU%pN{mY!#`FTVXAn=INCEOE*SQYb*s-9s`++Hh+XT?5 zIhlr-p(3_?2+yYQV6!CUMvNT0<_1!+hzc9^{|jvFvB>~er-4%d|HFV@3WH=(DEX2Mw2APeF-Qsao#4;1-MP~5uMU_!8dGbb1 zUkn!fYRU?`(hH*2;#K51SeTXwU~|Q=Fp=hoc(rR+9By9@;2~7Rfg*SYSe{)9g8|BT zHVphA7tqk>_#4192Pf2Q_cQY!V6o|_F;&=zu8xlVKk*uw5d(umM}@4#oLCGVgK`7K z)@(rrE-2!B#0!XmW!jK>4ivH?LU;oH3 z+roq@6#@z`t#==aq;sO}P)94=z?E^=at$zNlkGQOd znq#mty13OUq|z7QN{F42Bmq3B>Z!rBPsF=-XSMAb^vq0A-U2a+fJ$*#RTJl-rFMom z>>*Ltt-X$QBnX}WI8^!-?vLRthZX$fpUm60M`M3AhyZ;4!S)w;*2Ebc_upAuMqV@a zNeBwo54yq7K<35?@0wGa07mNOz-G#tH4MLc>_nN|3dC{J2^*q1d@d~|4i-$*MBz~$XDdg2&SC*hwV)9q z^eNE$aTGc1;#jK_om&iZhD0TURg5sM=&5^r=P)C3lQ`N*O(P*)DuYKx(V+6InR|?Q pvLxz-$J>#OKmYmfd1T*=#NzB`+4g3QehU6WJtuoM@$}_e{|h;jYTy6> literal 0 HcmV?d00001 diff --git a/docs/evaluations/Docling-DPBench/evaluation_DPBench_markdown_text_precision.png b/docs/evaluations/Docling-DPBench/evaluation_DPBench_markdown_text_precision.png new file mode 100644 index 0000000000000000000000000000000000000000..0eb2e3f6f36440fe6d83baa4787bb79edede5ecc GIT binary patch literal 20019 zcmdVC2T+x1n=QIAM`!~oAQ->^2na|{h89E#DoKedN=_0)GHt6EP!yCLRFY&RXXr)| zBuSPG5;w2`iIVSn`A^Txt#hhwotmjzGk^VEqU`;J_j|&6)>`lP_@a{B)=kWtC=|+8 z`E$QtrchS7Qz&$u8`j}pgsNM6@Q1kV*(t zVwn8zr&R9-)7qSztEQHhf6miSx0mlVFkrEf4ikJ?yZ+_dfs@-AHIGIMT=ou*eSA*d zdGh5MYIx7Zut&cgIC$`Lu&k`G*rz=P9{rY|3Z?$GG{Hp9hQC z$mZwgYks_O*22&G$ZwRNULw{#=cJZ?b#rN@o8jNM@gsvBty%5t(_Ly^e?OS&{F7mD ztg9#^)y&#Dx%fG2+$Kh@5`&bO(+@T|E>1PR_V)hqEeRJ!&NBJ)%RaZBPbx*rGxZvJ zF|Lbq%|vZ0tGH@^Wk}hF<%Im1st}3je3$vy-yd%upPEvOQ%{(j zsFIPG zW#wyHfBNR#y#Dw_P+{T8i3Tmp08W)C|8L(es$|<5+ssW;gLpgAOf|au`V0;pJXr3@ zB&bjoD)o$`Hd=X~hbxXDyw7dnW0C&S!fe#FzwSlJK4TV}nzYt#O1`Ct57KI5Vq*Hn zd&>?B2?@^IwdOOzrf3WEiITgHj)4fZzB(1b;yv9% zLqlWCt>pvN;T0hZ#?4ta(LblBh3(ty`~vv(A2NwqM|^BDi_*!tlaabm9w3O@$g(jM zG;KIG(c`1cEN}4l{fz;!Qj0_aw+YrR8Ty>3fpc)x{0!_3T7LxPqeM*yy= ztzrE0)7>Izrj5ekvd;wex&C^~BzgD4&+&1A`dBqV#E|uXQ{3ocU7c!eg!~S2ykbu# z{dTuy7nN|?t=;|ohV*+yJHOeFw&%%uU(LBgGq1Niw5HJQ{iKYly2#zJYbsF+Og_H8 zzAceGPcqsa%gzb3q?@N?oRyPfVPIeoyz%J~3vZ5ji!ghsC$rGkw=M(U-WFF>R4j~_ z@rh`=b{9XB^>*lfCStL?u_t+Gb*IcFgoQ#|wjJvdKMNM(wvCNob#=9fcDmd0QgLl< z?Qly*RlBdhztQP4XB3g0H1i`$!N^ET4c z3-?!rWn{Sg>hi#Cq?y!j^~P* zVzHNQY?;q3&tamENxrX$LB`gopnhh_FwDh(BLJ!0)UvcNp{$Z>bZO#Ox0i%@l6r!s z(x{y%JEiZYI>#)=q%&Uny^nHl02 z)71O-*Gw!-R0Sp9Dq}`1Q+fCM#@Z-FmK6Oj^iE-+Qci*Rg}B+!kIzpJ3kcM_;?-#; zXF!4Qm-jg=wl;0DKh2hUStZ3l(P^>6d7^@cRwQuitG{lZqga|Bk5)sg+w!6T9?9Wj z$I9dkfM0Cfyjf7s zbvE)$n4n>$|JySjf%q`-ma;ifwh;V%?M|aY z>lxT=2ZHdZ--U-)#Hd7D%q@)O&s*5cwAo9fWtg`#p#J@QSf&&z*>q08X?pm!(K0nU z%#&l{D|b?>)og#LQzdGC0K44t*3?Q*B+{320xTjHC;nWwReyW=%B<(=C=tbjQPGELeoij_YLG^1priwz#aUjI4O>*x#F#!WpHUrca=8kf+`E z*ZO_r=fnH=7j9t{uZxI?5S^5|{p0@Q=g)=F2=cn5n^O$EUfzsS4jcKlgpROBo}5xd zJ0GjYlrK~B=kId^C3vdOI8vNvtvy_$l|lr(6=p`-EHgvO_ljB-N{*MXlPX0n{c_~_ z095PpUp+w@n?p`D*F`+j`(*RP}E;^MWpJoyLfVtm>i`dLwHl~5Efp}tQ2c*@6xl#5x8 zQVgmqaC1XNR@`w!sScN1hmsy5X4Cg@p5`U9bkWQ#ey}kqtZM;z9pf}JGWh*t_`7%S zCL8tKV)VZ}*ZgqpkB3{B-W>_Odm4XLQ&8~CzC(*mO+7K6jL5H+nrRJ3vB^VO4U}|9 zMkSm2c80;2S`#_#>cOzD0Tp1fa5aO`*H@g5KR@rD8cHgNMVN~t+*A~I99H;N6L~=h z_w#HI|7n?}U*V3E16|(?QORR<^Rm$m7Fnd1aWYzap4L|86IOwZAFiMI{OQxc-w!qi z@P@s5#UVfb^XF|NzAGQza2S7i&KkfQ6dW91C9|xp7 zA+@fqE-qC4cGr3HG^6U%K1Y-r;;*Lcog;T%AE!}3&GM13A9=+Q-6+bz@q6S2zx1jq zh0B+{YlpbS3X1K=bxX?eF-A7FL?DVwC?z$M)Y^I!1y*V2@2>fc$kSs%6;XW^K||eM zG7b1^Jn);mx9j4RMw)qxy4%uhG+JzUd!CaisyBUs8?~lNrxoB}tLYr75CMXzGI_bV zswe?LBUR;MvE*61Q>3;emvV_OD}hB~Nf%DuHJl^ZDnOT{7#dT3KkL zVXkxXz&q+07HyL#qSPkCuzFP5`s7;|P?D$*)7l0N)eU}hcOQ~R4V+Go&z-R!X^oki z8Y*jR<4;CVUnI--$LjTan~VPb+jgW?jMkLWpFWu8*{bwr_UfEla)`K{wb5utLBY!& zs|`xt07mxA36CE=I{w#Re<}4lA;>Fsa$f!)+&(6G0|cFto%Zs+A8G9FXP@lM zqPoR5dQo__(#y{YkeYv{fLAAbkBEp!^fo#4FOPJzjL+u;@Yr&??li_<)#nIUUL18( zn)vF=Gh}Pg8LJlO$O+aVbZFxTIRV=r4_o<6&Dq;7dad{ z|LQn^fQ`{mP2^K#>th#9G20EI++kS(v{BFYe#}1n-sR{RG;^lK=dC2t0LMK&JUohK z9j`i!&d<#SMu)jJ$LP&Hq5;jbHrL9T$;=O@w8S=Y?V+>%15j2}fr0tNIqzUzHOuOE z4@~ObUk?*5Hv0MFGbxFl{NgTisf*i?eiQvwElKw@bMJm!wQAMyP=3QRi%|BBK`j`I3# z7q|McqFEu6x{JW1v|6B0Z;m#n(L64dC{Goqx#`MKsW?<3CDL&oY~DRQKM6pwiGiW< zuRqt**hPecDwgyNO-y1dgT=xz)5QQ9DQUcszYO5q+uM69%%VLvt0?^t8NvjLTPIBl z%KZ3ROfy3>GBVDDdD;$qJB8oWf4E+hdb4yp#!=NUx1}uE=X*Jom8)2I=K*YDPyVrL zusLmLt7$Idwr$%Gbf|!(d%_<*+Tav~C1yTh)cGN77msGd!Lv^WP#A3Pj(R+J5QBa_ z&8>qx3XjuBGWquA6hg1-!ETF~d{a4#?K^hZ%}p8$8Q1(??7v0MGt_1Jss|O-OCrw? zlN`5xxuTr1eWTR61yAA)DwDcI3#BAv}dx z)fjIiL133*Q<}w;AU8W|V2nXkXaWFdEMW8WbbPKlM$F2Z{%dm`o|p*@vvj!+P_hft zEFI-zuY7#h+SWD&d?JyGPqkm40~`Q|4JJ)$`=RrqlIT7>@^cto56ScM^T%qY#Y##^ zrKq-idc1=`1I`mCRJXDS*ZK47TQHZSD8;^b@!Ra|tXL{wT$}W=W}|CiG`D)Z|I*x0 zl2&Z>9-r&;Q$vc&f9_;v4!n-o0(9Zf_r%leP1Te0V&kDMPn&9L1`;%(mZ*JII7ktt zn(NZL`*%j$6AJbijy!9AgrixiC&v_S_=3-*4Ptej(mRU^w>c^La5*2IOn3z43m?PlA z*LrStyqA%QDQ0=0UuM&W4Hxg;%>?uf?Hc_&r4%9_iF}dBGw@bL_#A8W9#NW!_VV&d z)XfX*$~E;? z>?3DblnUVGPI~R-rR*!czINHbN~V;hOjGk@ddaY&!(x!=VsIM4T)^$+RsrI6gRPf% zPyy5?`YOoGZjc~QJm6s6?()zw z+9rB>-^rOF2bcZNpFUM7dbrgx`#JDHEXLF3Jg1CB?F>w&7rc>nH%2vLRXNR@Q{^$M zsX9C7BIM#Rp+2~OUv#qU2_Q5|jxGteDOyH{$yn!KPo6xf!P(veZ+0|@kA6yPrcfPVxQj^(HnEQ~+JbBjRPp+4oyw_*-1`RXJiDk^#* z>`$tjV4_t2ZJP`W_Ft$fVMa3!x35yJCf$KwSQ3L$s zy7=a#@8D)!@AA@OJz)JJ`g$y8;P$)S&lFZ)1F4i|P{FK|Y4ul^OpVL@bW^{f2l|1$ z_w&7C-aJDWkcn&MFaNP>twpw8!LJ{i5z=3m)(f>|+4Ng46n$=M)4>ptzOOQ;0Yr`( zJ{-O64^ND;1S3yV8M)Pu^6>D~_UB*wYZb`1bHHQSi$79MeSX47&I`;u`H)=%DGb2f ztfxgCy%ZJntb7s0N8GB@9kspRs`?9vw$TrBS}IApQt~y^v=-K-&JfYtr@wgZs~WrU z(eu~boOm$(M;s0#&U*l9B^T5fvwaRNIrd3Ee*D;*&vPiwq%ko8*m4LYWjzjCpK7d9 z5iF)cFkOV_au*NEgR`@uI9#Y zto<}DB!ioWr$SmrSXdoP%{0=1`O6fvoo2Re44dP?+hg@%O`RvzrELVYzUehYrX zWW1+zIvtmH)A7$fiETS}gkwPFJbwHlYHo?ueKGFQMZi5{B>njd7i=D!s7m&o`8@{-lnP+(3fcc$G5H8KhFw`jaQ1}O3XM+%l*etPVQ*ds zmOVt$;qnFFV;?g@q+K)t-!Eg34XI>k{iyE&tJ)}@L>~>_=A|K*qp~sMca`ghf=PNQA{<%LWg8Ip)Jrkd&cyBOo`taeh;S6 zxToah+~Q1!8a_f=(MqBHdo!P?_So%cav&a^Y}I0aGpR6rT#QLWd;rNqJMqomb{as|D+JOTHVlRb1#Eb$? zXCDoY$3JC?sLNl)hN*|Wx|5IBFr zrkzLM5C)<8-C0MlJN0;Aua90jjrOFXs``rUZYP}`Kr~u>9Z;uD?WJa1aLkR*PndDpu?iw4&1uHM!#ocZcE&=e6L|DGu=s@6 zK-a1N+5JVxVf-dx9WWTD%#;nxeEaf64iCK1oo+QLD_qD&OpSF%!kpiUwBJ3mw6x^C z11x@PY;oacf8@+Hc7c6{2Eu(S9deT#g- zD;e+rgx!p?Xv;3Hs;WZk3q&%Dx6QGKvz$UE{dM@?U!!^|TiRM5|(Way@kw=r8o2!5ztGK|I*JG>YKiP}6Y+CTv0e@5=L;S8w zVd@ddTU*gp{t&ma zyPmzVoDx1x6}D(K0zki!o0q5fc>AGp#C`B|=`&Xb!-~LLvFZsXN1mGOqZZ?xqAxXzMSM=$*i>?Jnrp|wb&eHqAmMX3RAPnO}nher^yt*vJC z9D2*Ezbf9>?rU_DX0kCIb#E^-)kkVe7R%yz}>t-z8qh5cSrE*M{c<&hAODDO+0Ob6fy1t2Tl<{44mxO z61m?aUMoWh62RewTRs?I--DVT#B>b#dKA>O?DA}t+a%7lW66S!-CgX_Hi-XY%AYpu zC$@m&t5ysFp8vZQph)D8);~ZN6FBMW>Z)x`PdUkU-Lo33$Z*+iyxmJ6Q=!;eDBxSwbkKrtbKB4&r*B+^eR7Q z(@{z+aZt51O*@?|XU>*J8cpTm{LI-Mt87k&ZgGe1604Pw80tD_ECmHeX`&l=JE~x| zkLB#|zZ3O=o(3$d5+?03T#E%3Q#nqd%+^tBqwRXf4>7u3d%tNXLq+=3;V`i%K|SRL z{9>cK*nZ`8g}_;H2-Sc}hE1ES{g*J7p=@xgy}$U^+cp%w7(U!>_NvckZf%Zz)pDzI zSMU7OlSDP?E%W0B(lc(qzj23Z?%msMc8xk@>?OTBGch89;}8djs3g{EJWx_>l|Lg| zt4~;1Snb;%pcdjl0D*mIL<nARBr@;wQ4?t`#zV+mk=Hf zSXx?oOqZ0Lo3BDwe(C4u2XZ?4{rjU$X6aiR$uT^7fhBe_2fFl2;G~(S^Xc757ei~D z0^Qw~Y6-2IG}R8bWgT!36(77R49ZUIg=wXSadvKRA;wqkW5*OgROcCYpqrOs;zF6^ zeZPf5*+0`$x~&jEVGx|*G!?wD5da8oAVZHU^u>!eP)&?6Zzz>n0{fu_5?@14e}6QO zR=N??DaRj=j}TjipA-YKr1iy-)ty*%vdr_ga}Ex`3w5noHizV4XL!Xgi>{wpJB}$I zefgK$w`~K=l1HSeG6R2LTFD+T~(@mT78BM$+6gEvPFO3o-;{q5N?T?tkg$ z_>`DCez5v4jsE}UR-aNIIxAp5LJET$>FMi>LZuWdAFYq`Yi(`a%Lc?_Ghj14e06G| z`V8ubI&dRs)3>%J-F-%&jRvKSX(bf)L{MoU*)Og+1i%lxk zu!;@Lq^6$RVj3x|u(R~pQt{=iO2LfHnXKqQ8kn3czke}E;XHcuXpsZF3IwUS&QXsD z8GSqHO=d2FODzi)({Y{2c?Jjzwf-o6o-*d|;XC-p1!VRsj)#oR&40+(KX?u{EaCLwmeM^ZN@-+C%3DY3yo8pn&=`cULYB_k;tHNit;ZDvd-JWn2CO?QgW%Renq4B~0X$D>7HxCs zjV#bpQIQSg9+P`L+`+agUmZHa&fk7}1K1+(eed4A$^I}m@k!{x)yO76Da(uAvYvb1 z5>kF2#-_4r6gi6Zy+f-yfkN&2%9+-#Uw^?H<3&8`7f}eOrNKjm)YQ~qlnqyC)K6(r z&9u7ifr~q->vM#GPA~UvwRGSSc}!c?i8?tw;K$JFp0P@(gLzhmZwdE)0w6^#R#hGn zjCTYOcN}sV4a^HNb_~Gx*1e(^F^hF?x<;YA80|$`<9UH>9I4TNL+c?<++(If=w3R2 zw@6R^fJ!}lO0lsE>yAn7ghUxCPRWk(9?~%>7W@83`Q>oQ|9S5pikHRr52t`fR58sU zFX59An9s$t$(yXX5UZ1u4yBrL`}TK$%I5(HtggsqS@$BD`^Nl_U%&S>Y$L8>sJD~? z7u$QH^`Z?cO#u2wKpnY?DqjPMnGg|1;7W+DtkQA2yO@)gw{mF@EL+@P5l$s)=;MVn zBlv(ov4nNyLLD)J&5hn1b4@;`!}s66UwIAFLi-5H_?tIx_9ie=D4k)4m7h}#>cKws zqP75miKi<2A3p^`qLBm*KtAW8Lx=1)!6|9=cBSh9)K>$`MxZcjqu``xFtABpnHp;J zD<~)+e91O;*`r`uFxr*c>L78>pFf|he1wCe$B2gT5Idl^z7PvTJcGr1D&QNtu6y%l z#9}{UvZSo+0<2DC&NUO(dN}kHBr04AHmRhT1p8Vt&_#O`gv)uEklPA(V=@N&fZjX_ zNyD(&@YADh;%-a1i?=aPnI$r##r5}=LGT;12Qilb1c-YopbEg-<(-p5XjMTPvQdDQ z6}MKxJ${n{-u@jViko55);*6f&>un~BAw$d`0)~AV+w$n8_N36pFh1h@GA!g2V;;t z&DnNwAYBh(NFy}KnTK1dcPLA~#YEoQP>aN{dW`xD&MOf_MpJZcRsT>^%8D|cls|Xg z8|t@|M=wfWv@eg=SA!H|Og_wl2A26u1)mx)ahMx+7EnThSdRPMc5 zk`$2QRfJ=Jp_37;0}z!l7@0)g)quvyILr{(28>J^NTF|X`&=S2GewER6V<2&b{yNm zI!+BgJk4vIS$DQQijIzEbLiPdj1rc4Q@(hz5!Z_z5)n&TYU&lZUF*0LIIlBLNsUWz&(TXuWJ^s*P*y3Ib=KjiNLOgF*en0*_BjDv;qzIRC`92 z?04~?SM^nf#6gV9%ErMd7pxp8!SFf~jEkVXfI*mPdLyI?kYzgBOgOzzrX+<;^6pra z3r4iM6hr0Rs<_*0QoJ(jtU4Ry1W=HXGqTJG#8!%?Pt znH&Xvj!8gMW0;eOm4{eo{r&g2fDaqTv>It=_PDn=PRjb>d`}7gWId(oYn?j1ZI<4g_L?n|nCn$sZ_g7pvR$Po2@tg3f>TOrVeE^2!&m~0?y z9>|7`UALn_u)O>A>n?bt;&yegO_1x)fo*!pK`IQ8d76ieyh&};Gw8W#cti?~AwWK` zmHi7=bmO=v%!3eeNn~2zTqGqc!u!w#@l?NKS8JsCq@{H+2hdMQt$-)XQJJAgTa zgSZjqRtef!8zR8uB(|HS@8Q33zcNh5(rvHAonsjnNT(O@TVDvtd9cmC<2C)lrY-FKd7mksFgrXG#7yz{}J|dV~K-=&1=x;CM!fb-ec5<)Z|{YfF*3L zo&)cn1Z^!rCx^JSeaO&d1g^vp>=}nVruYWPM`Rd%0G@7c{)Z1naIDY3^K?uiW6J&G zlq+yYNG*&N*{44LKg>58@Rv`5JRU$f*T7R!Lz#u1e;({DnQJ-ZVSCa*lcSYJG!TjyV2Cyyb2Y_) z0A&y}w6bGYKAc>=o?aDq(>pSfSQ#QQ1SelkcQX=fiHiCbSx}XsRp6S3;!~IBWDjkf z3hkX_0bKO9wY`V|ha#H|n7mI|w2ERacx$D>WzIoIv5a#?qUV+5Z}0 zfw)!1^z1m4kU_?-071j-Y!008LMeDI5ny+>#VJ3?SHm<=-5(J1L;n;8_LXT9^DK-V zz29`PWndANB9RwhqtIGgdX>MgZxmRw=xH#mWtgk~DHN-(p=b-)4XVzk!so-QoBJNT zK>zo~BEZPb<;;cO#(GM9h_j5S2B`mo?Rh!$Y?ANY-6@MOIoa8X;qrXD@gQM(U&QpT znQ8TwSOV}gxd2(E94D=L`g2_S|GIK+1L2g z6`!)zok)h!;~_BXp_`WiXTW90LCRLoaX-Da3(x?>B*~R7&d_5>N!pWyn@SO0A{s%-2cB9=BUWXEJq(4DYICnZ|_O+!$Xu(c6N5ltWmTz>kr%U zZ>H7`6in37cVJ=zd?Le?w{2%39bu@^C#PL4EG!avu>ytL5lHY$9R(n%2oh1oz~eJH zQ}Gt0a{`{FDU9ifyr@5gm&tXLC&9>gi^0vCO!IkgZfLIsJFs%=8#itQD0`(c4^Ogf z*WYXZz5S3#A@T_lb^xNjxa&eDY|`;4(_~`%Cqi&|xK@bW(r&a}2YKYSyNeow55crn z`fsa)Z3=6r;1M>0zwycA$1gcxTDpXKoyfZu7wK(Hll=za{0S6#Bx~=`5VyX9#cMpB zYn*sGhLypWkWFfl=NU;a#s|G4Z~_ZDG|H8G030Kd;7(WyK} zp^*)~lw+$#bx?yA|Iebc(kUn@siQ8DmiBC~Sk3Dolwa2L3Ymmb-)X%i7b z(MsO4JYRMA&><{XsdUJT7cXdOq)wdhTU!VbUR*)u=cgTs&8fiRQKZ1*VAJU%Z8>om zj?vt!@#9p@1wLa;Y9d?S0H_hYZ!;GjR*4wy0P@=H`l#UYP$@0Ks$g8qOMp3lIt&h> zgwD=u1Y2SG0$h*$e6)>yClix#n9{#E;Yvn8xfiRgX$ODP5JQYY%&@6FQ?}yZgDQWt zmSU9BH0<#KYcH}o$jwpwge}01JsLkH7 z?xcL|)s!fVE5lwO%9{Ey68;W{&@x1{gD%U9GsN4DIp?Kz{4Uf3AxWDY_uwA1L_nzM zX>>wUR2>2k>;Q+)dy~hFYD8GoI#_GF2QQP;9Mq|rZ~cQWArwG0n~M#Ci3~q7g&HV0Ik|x zA)7OvczIRXhWtl9Ky?-mU+i$l3Sr;kmWaU*h#&$Wnm|`SeBL}k@PD!AiQ93V)@Spj z*ElC&8j3ytnoHbv;02(j?bM(esItnpRREuYl7`3IkT^8?`dnF2_qV1@X~sQ!-s9O1 z=q=Cr<>djUphH;OIUr`i6hEivj>7-FLIinClPw$j_n#y@1PMM!WD1Qh|KmvJV<|^T zdJVk1TlelMq%q)I+Aysx$6f=GMTkNCI27rpTLWz;4au^lamT-?>FiiDRGq!oA5%oH z6;d!iT4F->Ipc&Gl(xcYgHbSP%T7aNs{$=VPkF-f@$%)%mECd=9zN{T?jlLHzhgZh z#`;0*hiQP5g-DPr)8?ShIH-_+nAV*PsyG}};mdA^PM3K;f2(`(9%#bnfR*b31~4Mc z2JxntHuB0}wz9IK*T4jT24YUxd;FR^X&5fEy?)~p6Dn{T%x2(#v;|Bqgh_k=Fi-G&r{$WHxZ|h)*}dDSqd3fq3Dk!FC5qbG+NLM%5c5Lfos`~< z?iX7hJM^{pLYB^^r30je_1Hu!hc&z|^HW{;<1N=k$rDlk5-;^z#xo-C#IMM&VyUhP}b7OA*+?5;oj=j~x2Pb2y0k;+;hQ4NMf{aFeQV~YVm9-IVGf8ke)LPYl6xr!4Xt*kxcdW}l? zg*4_wWyP4S3ikU1X2}O09;Sd)u=$qL#wq*DtW9p7_r{J0)hLAkDAu}hV$uJcgGv#- z^>ybIwN?&B0$&a?#Di04B!nRhU}Ao8tKH*EsV_M#!|pW~@QG`!R@03a)43BCedKl` zoA#Gvx-oRfX1L*?^jx>JvA5R%TagOcZHphsY!J@gE^{RJ!MNPt8^aGW1^3dFuM1x@ zCV^Y?jyBe9b;E&nDmLx9Gf3zaxjKJ-o5N1dmkC2kQf}ur%J5WUQvh)wnrT_;CS1j4 zCxhk^ zIoBZtVw0KWK8zP689;W`8)J8Mu@PEv;xSNKJujd>$Dk%<>BuI27~arHBa{vpR$48H zEpf~>AOm?tL^90U)K8zjhwa98!{7PgA`{hWe70QVdS~l&is^?ioj(_cnZ>fgp7Cc} zb=@NppIx5A<6En*!7Bhg&UU;<-eVfZBrYNklFhRC{~0Zakv6fv%U=eoJ?nm(?%$6z zY}i;5v#1?$?xmTYq3(~+mbE>*uqO(Wi818K>2#1fAb=XWUZ9ztbk%R#c9Q5=s@TI` z4@M0$A7JKHo-#JA{+%VfT)}zF6Lty@%hoUJldHTfdEPWvQ95_ zJ{UJYi-yCDYI^U_KW_nqV4jthu{fnZGXYdhz!lkm3Fexp3$MAG=0kF3E08sHANA$n zK`*JOP=peZLlqcmOwxdYLJkpg`4H$RkuO8#8C$e1@Bqk;EN3vWmNp`L6I(Fq6`7M-^TUdw+`2DrD9omzzsDgOyL<)8cv6mjx$)0?; z2tmONHe4gbaRqh>RDfY6cR-rA>847<4enWQK=vD8N7pnJ9Mz8P+izgsnZPi4dx8y6 zvqUN$VaeoBBcI$*IYV-GQj(vy_l<>xh4Ng73G3KT&Rut=ZO)&6LNMk|uLTO52Zr1A9m&{EfgjKUb3OIj z^ml!)fU&W$z46DEfBg30sZO4w8aSGZC@jO~fPsX}h5f<1O8^w#?EyX_^#)9T>98== z(<6?)^GZs-1qE(OuLJtqMqJt8UWI-InZN6az(j6KQy^&K>HNWOUl4gn2O z?_t@eFJBD5%fNT`Ww4xKq7C;Ws*N#Vzyj^JnX7!Cg`o^8(G?;e zkRbx&Iq0S{#P93v<0FK(dT6`~>2Ixc9RcMBlX(KDU+nC9^=hXSHpe+)qoDSsbS{)B->Qxre6)HH(!d2 z$qOgoianWvb)Ep9zw*`;Xgd83DkVFTv%?@EnUkExmKr^qUt`xNusbIZ1vwB-i$3xh zOj*~!s2Y?1I9A@ZLh2hD1+%^vuo<=l=o5n| zae4v(Q;}M;K=s;sciEC*?A;<`SvzBn3QzXs2K$1Y3MDT=A@zzdPY!Lf4}$nTm4TSA zZ2VWV`@em?QHed-rAD?$;@)3Y7^uLbf*p-hcxgz_9b?LvS7$)WTFS{gLjdjAwib(5 zijbH85PgUUYJhY-CDZr?7aw2J-m4;%o;9)*x*l$_-xAaF6yA9g10@Wv7m6pI1(*rm zqtj1I6JG$bmAtfv5T>~0^dx<7CWIUT)IeX3#{4f~7OtIT%?zEAYzQP)cnEf{Impf} zaN(d7ynt_n@H!XdRrW~f>O!7_lvO1fP*);wjm8+p;8w5>48x=bJKGS5+HZs?L=>B# z{EUx{z2KmI|9A%ZvErb|RhSS*zmUl*$i71#Lcw5afgY^~d5ivX6t50=PrN!<3Zh@& zwK@t!_`)cr^D~uh|Akgh{=*qhp(>;T;}O7$nx~FCg%fr&4x$+S`2fI6Yw+?yH%IHM7U=`}jndPOK0-8BadKvNX z%HRJeO6OkZV#*64I9f~WvHde1P;wwHDgQFKYW<64bjEP%{!Z@I3#$js| zy*a9VJ*oeEf+OpM=N>&c{jvdW5H4+o|h?(DGmga5C16?g3I0~YzHn9 z5t8g9!0b!}c(MUOgjLKLTMEhRK#1`Qt5$<0nS$fhdfNtfH;$EPM($olEuXBE7(IIU zuq?C>?95byla<)m&dtpqe2IcVg_=gBLO8J^u%G7KoD;_=no^@_QeG6?WZa^nSBU== zG`2-%8y@)psME%hm+bY}NyDU1#(gQ^3bDmkboZWL*Ox8rQ_6dr WceY3LG6qzN{8^>n6Hi~e_x}LT;fLn{ literal 0 HcmV?d00001 diff --git a/docs/evaluations/Docling-DPBench/evaluation_DPBench_markdown_text_recall.png b/docs/evaluations/Docling-DPBench/evaluation_DPBench_markdown_text_recall.png new file mode 100644 index 0000000000000000000000000000000000000000..400a5d789719f97dddfd3e909964591eb6b436f0 GIT binary patch literal 20435 zcmdtKcUY8Zn>|=)tKH@-2x0;wK|n;ZSwVt=2uM=Np~yLR+iI%>DU^(;AW=ZcPy{1E zlH^b%B}pidEP2o4*WWiYJ3ITk_S(62XZH`=Zknok-{-x*}}lq;fA$7MgE4ZrJ04TneokC_WIT~ z#uny6d;&-K1bB8C+1gs#i1G7N|MeSu7S@LRHWmxrc$eQSFR0s4DC=&JKP<7*F~$^% zqO{DJlUE$Wn5_=BSLzlPM)u#na^aNc(cgANIegfqvPb0??bEWm!-xD;?`uXfqI?KJ;_~b5@Up8*s^{3wALPy}VAlI{fn`mS1iNdA=vxBLX zh81!KoReWTg(a-Izu+oLEi3w#;O`E#Uryk!2fHcwFSXSy>nW5sH-GiSkG7ts;1^%9 zF4;w)cz#?@VWCiX?ofWEP_7I7kAKP6jIvM>MFj-~G3&0s5)-XD3r$+GMyQDq4J<)1M zB-I}m7Y}VtHw|4UJ@?sbyYMISjuPMBEbXa{Q*N64aV^t4d`L7aSlG0lRR%F4?2w+f zY8zirihgMb0#wMn<+zY>HP@!?+fDGmO)A1fZAvE`hFXjzBqZd#w+ok*1q+qGy7AtN zPNy4JJwF>gKrY+S$%)Ov!a^BOh#t>i)!oXk?W*Y88YJG{&gQ3g&WqJbb>Q5YGeJ+E z?y8|XSd3Mj-x+*4@?7wnHzzuoOsf0hyaPSoVYJz!(_6%gt0vFI0U^dwlVfMPx5Rbi z<2L(wB_$=fOP3z9NxA;es`F)>rj(uCxN)Pk8DnZAC6Gt`CRcF5!pz~>mASsurj(C3 zkb=ehL}vAii_x}y)%$pO2BQb|adWdtOH0?hx^db+B*esYagt%-%_XG&X|o2SQhL#h zE7-Uuy2*Q_WPb#Xri8|=KK>WS;Yf;Ofzw2tsy8x5)OkEMk%|yKM?+vt$~Vl9mhyXZ zjg1uTWTc-zbB1-@x^$l7C`I1v zhFrlq9`kkeTeob{*VNQ3`~2**MYels0I$5I<*R!S9;BwbjaQvNN1H~%3u?xkzPq*u zM<$H(PjRKpP4`CB+|Hbzonl8m*2F3Ar%y86n2YmM?fZ1{U84;vpJZgz)?TT3dUC09 zn$cyOiGCHjmCHoGJ~G~XD*x`wvy@y#Z&rbK7X&>PremBYn+&}pA|m3n^Kz)uh7oQC zCiO3WKJT}GER(;xc88p;t!<8#`*ioVniTzuoxQz=xw*M=j}NJob(Q#2OFd?pBAeLQ zjAa9Oj9Xe;<*#3V+J^|tS=4X8NEvL&PCa<}`I(%_;^GVBS^IEGnji?Nxp{f=7cTt8 z`)05$k8#wv`U7Gp_~ys^!HD=M_D&>fv_*SympA!M@2<7PPpJ!<<-=J zMy94jI5|0^5yl2croKXdJ?F_m;{ZM_lS3-;dMzz2av$$+DW7Whh)A-ya^+ELsJ6lK zHQV)Y;)7CNUIGT=->S|j-Ko?FVKOq=WejG<`}zX9DSbD0{Vas_|*aE3mawo3q6rRycII5+0&r-!3> zWbP&V_)ze}hYvOCgZ+N|aIDF+&{58HwCe2XF}T0^;LUgM-nlq@TBm_`2NCifD=O z@l|Byi*gS#OWW@Di_E=ZJ;(W57p8a5>Rl~&Bp5x2EOHGvtRet;W@xgXU z=d>J$VZOd*t3%=94p*B}^o3CmCk7f66RsvIp$FJ4%sDY~4yz_z!GEM1R&q`-8LFH1 zD_rs4C!gjzZB3U#$|yA?X$jl)U$Gl(y!>D*uM%FX=T$;0dSzU*ukNL~(0l!Z2M?Mw zrMx(G`n0h7oFl-%sXXV2>RhL>+UI9i)g@_ZbQXFm|Gs2dZMKc!4#Zx@(8mYc>S8bN zbz7M3?MGHc%LP9_&*^jq#i7oVW$9qX7(oZ;yti*hFrM?}&X=6Iab4=S(PUmmCnJw94(F6Orxl2BNJZo-lZqL-# z)}}?FXfS4E*b25H6>PtLxJMT==zpxLU4;(0Wy6MNxM!n|H@}aLk6%IU;WBP~bps$K zXXQ4&mz7epauIHm!OS0N7X$g?&~|3<>`-<1*RUUxV?SnA{GGCZ2BEU$(3Q_=#x>^G z6j5aChugyY79$!%ubk86HFXEBADymHH-3$!-(S?a`Klk)D}yF-p!J>Rou&rz32Gvt$8 zNl6LKJinl`vvXqDV^IT9Q;&x*j;Av)-dAJNmOCxqe!jY8$mQxeFM<2s-YQ?--KJ8} zx=G}qrc-CX&bS+wCaWe~<&~*MTXj{P_gwv<(W8`s-$?DDFKTVq9gME!*Uk+xXXv<&-1gI}Pu8(8r8C&xbam-<;*Yk+ z{Df^OP?R?pU3Y0^xVRxIvbVp#zj~UHU!vP^TVCJ7d2y541El3l4!2u~46$3}-XN9V zpx)dyOf^YU0I=_vuhdMe7M+WfyD7T~1cw1vFrIz%fQz`$Q=Fh|<_=x2>9I;_00qn^h!3qh>nhTo1GN1>G5K2#fSl9Yl22V zOPaUn|NiAY8D-|?<^~Hbt*uk$XJ?xJEDhuzK+ZY?a&yaw+Ye^-&Qv~;Jgn=raw|`e zaYUfDbIGIyfTQG(j$35>n4M%&nd_u}JSA^au|zZ1@%8qjMo<4(wyK%=0FUNHRFu*D z%mkZM1#avhAD_aS?MI8h3*5PTHvxSxV-_{(v9GUUj?j8_eE(x`Ws9}5i&dkK_yUjNf0zlaE(JC#d75+MS*cUp5QDk_|( zy91FFht}*6yRu>}$7kGd9Y#8>Z0qg-Mu`{DXogwSb9B{~k(_q7sqjB;-I_r3Gn!Fr zYa`G5+PblenDgkC@$$=CC(n*u9ndKl0_KX$Y;(Q{XrKyYVt%b54>M_mZTF`wqQ%?a)%O#Cbs6j9GH?L#=cnx6b^95gwg< zO<=|(FaY|qhvb!Vs27y*9IDf|@Th$@XE?m8si`p!HAcxX({LBUhvDD@kD_Zb8WK=y zb7ceh61gu1kP)61g@B2Z@!vOtV?#UIEn~vSNR)ka;29>)Z-BO{m^Oy{1zNHct501s zGBUcuG@k8$^ZQa-)cKumN8el3dU}J# zx#Ujvor=vKmxg*o6qZY&X=R{zCR~PInFf zf=aNUe)aqgl7tSL=R;gtBQmco&e*lPj>%8Y&L#j|IA{$sWnH}6vn+u8uf$(DNMCeq zk!$u?J~h;8I@_!!yQuD`a`WcRTGW&2nVJ5CNIWQ$Mju}KwuOZS@E4!P zZwW^hS*A`7NH;1gD^E1)`szEs{cPR+NrraJtnumoCyu9(&9z=^LYp}_Ub;??8V|K( zUqjc>qRuLiK4S18$pOhIqjcdyv3$eS#6+FP;;hcu^XCUpb`NRiW)a%Pd7?iK{Uw8~ z&1tNsZ?wDAd2XbLS?__-gG)<1pvQb^N2`>gaq=^QG*FcLX0)VT9QmE6h7z-FdgFBp z+*htz<)56K3>;PaTDNGmPLcXF$BGpzF3uG6jydmFeDbKEz~he4&IZ%QSI36+fLt&GSN?^&<@fD%-Snetj8>ti1E>Y-b^Bi(7^ zz?OQ>-xqO0X(&y3V-@1g1BsbtOyksJ>Tm1nRFG^L;!b-L$O}P=WlCg!8AvV&K-qTx z@nr>l-B-NQz)7Bs4AS*4)gv)a(Knq$A-k#JE{++=AT(RdV<9)gqTM{S&p)cMQH@Ra z5sh{;Q@fexEkbIz`$o&-w1yiePP~zih`}&&@zam=OiaNV=Di43B|W{UeF|YlrgY$k zR+-C9-M_EkBm|$JgnD;_jz=D|x8dW*k4eLd((_z0Y(tsosmqoxm(8)?_#V@uN~U=$ zJ%r^^lH2iu$!1e$W_kmuQJ6-Q346p#F*vr0z^{2csXnP%i(b)ELR_Hn(!MOTr074u zr1)aVvXy!{?wC-!`}^amjEtro=MdXl&^k)%?NJs;^Hq$H%Ge~5qzX8P(P+wtzt}a^ zja!Q$(omVCoo#*g?z){BE>DEmJIOtPF-k+zrc3QT=am(?-J++92 zh{g@C*Wq#3QVrzsbd~(~D=JLw4$8nO7khW=O>Fu0{@pwC0l5ls(sHP6x$4q$-%sc0 zlh|0s2}C6-D%!LxlC&&Le^Y7mm>b&a`OU~ME-NEb1xn0}iV-$M+#w0|%XCWE#MU-# za;Vj%Q~;x2e1d9nyvO1~Mg~r%gf)`CV6HmJqWSTpX>F`ksa5JVV7czj&d93B^9(nw z$NQKKOVHg^@|^9;Bc$7h4l{sDO#nbLTYMfoxR}#00t8ia`-}L7ZQEjgUdgld7~HPq zyJnM7+*BjoU*2^k7GD$YUb0m62)n2KfABzgpgxgVSZw5BIQ}G0zjP}e z&vb?fy>;p(zy{dklc6FO^+=5^Tek)nw}bY|q%y8fk9Mc-V1!H_M=N=;g46NDM1Nf> zy9je|ik;zio5L>Sw;{P^^*3xkYfyodabUKF4_iy^CkzxguldQWiVx@NE?Kff4wP@Xx5tv1#Rir#BL>=H-%Q&On_Z>Em>08I51s!QZt*88 z-I`<9$2sdQ?9C=rhXl46;J9=RDxXlb-KX_Czka<* zC@6=?4(1a-r+GU{a$v@ScLRrUEGwFl@*llHX8zT6$-==1Vj{a>h{VeNei|* zhxyLzg|c#n;K@&mRXGc&rgn(gLn z-R;kxFDbveelpc%%AD>_W@zJvr10RQ#x$I%k8yi`o}Zq!bKeVF;TPfh9-ZmylW{Sk zWC+BpSmP~f_4SNK!ANHjyG%4v)BwY4PpCz{K;c|FTLCyXgW`v~4KM&jgQ?3XNTiFd zY78=M*vF%lllI%MzaHr<@)ZFb`u(>xYlJa6_a!w1RkeR1{218qR{q}|9UU9 z%p6X8kRv_vVUxk+U{jB`NZS#NiJ223w5I$|7uxgPyty!7v5z$wM!cYF4cqI?;^Kqf zN;;1p+37a!#|%6nm{Vwq$x04Mm$lyZ*d9t-jQLiIw?Ge?)iEGpk+!zBLI#8Jh+n5| ze6UGKtGGGKDw2SHIq*$nE{waD9^cw6x3Dm4G1ld;xE;e!Cnm%vPCaZkAUb%~c%uA8 zTYhS-4H?jPpqIL2urC9N`jOfx=5|B^g`n!UXA)JIz1E_ z7K%pMJUKFva+so>V_UMC}#6)SB6f6viYQlhaPHF_8aP{ovdPJRdKlKnh{jh}CY`LbFopcfg@b8r8X zBOsl~MG{(^P5ajwg(Xh~8u*T&DJimpuJf1na32F{5Q@Z4Z;|Y3pb}-?UF_TEV zcPI31xk!7ieqxr3LZMz&o%`>htimRbdMd&baIt#M6e$ivaE3&*8wZYQ9sOllm%{JA z|4vxWHjl;mFg|WY$8XQ>UcpE6YUdu0xmYq5z^f76`@}_I#X8QkaHBt7772J-{sl(l z@b29wLAZYEQ`%*W5oEiR>+vuAzBA}fc74^{YuE(2@aj!pL2ofE3XHpo{Y16yb9=Fi zDBz^_%;^gm{6lVLI{_j3|3sU1U-g4CDmuC!49lTQPftV_v_hXA;xA;?%CbC((qL_$ z(WA-A@|`p6xYfHM2Tc~ri6N`OMga_eM`KEhH~jL(1utWfY0y3a23u5f4TX}mV`O9m z(@(makH!YPGRloP(Ai0}@LhqgN55#)&wXWwM*i-b!FcZ9-3f|Z z4WdcusMSfL!cXlI6T7x9Ao*!&AfLf>Bj*$C=1h_M>x0&itLR9Atb8tzU+c=VA|2KI zQ_s|onU6{rjaQD|1N*X?rxDx(|x##JC6(9ydg!{`9!fMK-jh@?<<4Hv5(xH5E7KV>lB@r{jr-Jd8> zb&1q#jE=^}Ck<8<%uX_jiz&cpbDm4)opn3;ko_7Fh>8I`>^ih#LOzEdw>qULIq@ix z6UKuRJ|&NJSTfC;lz25V%50~hY?8?nvuIoWRn!ee0h;7E5V0v*g)=lcZW)j|Dkvf2 zrThioz}i;ie}zH8S}iGzB{vv3G7s8R8Iw?Lu9MBOUw(;&!fvm#2|23;YIWthb(!gp z`E|5$#5FzTp*BxVaQsCjHZ0$P@??yL;;meE0>z#@KMcpIrtJkD1prTcGVv-20eBQK zpt_m@h5;AxcrkI7+M1!pgMVpk#mlWjHK(^&0xT}f4Dd7NLEE4VYr%VEKJx1NUapRu-`}ZS5cxv*Kb4TwJ^~Gsak3IU!5jl1P(SQE==O8Z!PC@G7;icDa-h7Twi0-S2X-{j17(aoeK_TF3O;HFJ>qC2} z#h8!|(JWD?KpPOIIEjX28e&fp)q2y4>FWi`g;R|b6d6^t+ zZOu`E1Qfs}50;4xFMs~FNpQ6no|w~Fl8=v(k9ptUtpZp15J2~l`80B_8%=zb2W>Ma zCm$X+aSQmpz+*eE`STSJi%O6UP)tfJo#f=a0LBNKHMO+5hli6)v_Jm%<0o%80bM5^hkNM2fde(K$xsRF4ugr}&@_Ds zx0iDE$?VD>x3_6ua&*kn(!fMH@op7=Eb@mI1?Dnb+Cv-Miz<$UO9qqLRj5aVY3pl* z>TK=cQ2DcFaqL1=|M}-H5ao0-W-*o5!>fRVN(TOlyY%do{rnhG7`1?KZnP)}N3k## z^|$5G$(RKZ1|!JD9pd&~tFKci4|}^mC3}*z12-KD6_OUkvUEB57s8>KyLbT!#e(iK zfpZ2x=f%v#fGIjolZqgEn%iF03d%9_7PZ14#*hb^vMm*K8d$L#Tdyv|dy98{`q_|M zB>%tY3HvYnZ9yng|EV_yQ`I3$VAgq#cqMZ(umpiY3Sq}@h!kR&Lap780$N;IsfZFt zM(tZWe}hl1^FMaPz5uOU$IB2iK11?I*DYGhtDbTOFHDr~b9LFUB)WEo_Yq1r<2PBTR%2qAJPxJdY0Zr;CtzZ#@gn%%%v zB3ILnLhc5oFS}`<+@%W_E@Viaka&pnS7hu}T1$8o4_y@J!~P|F@=eZ%nEx&Svy zJenlKV3F(Kfddy}6v9&mJfKB}OS)VIZyM~=H#EfO;4o#Dz=k;8Fp5~e_CwsV!_$4H zZ$H%8Vsj@LQ3&5i)Zpf;$=ZpiIdzE|4OQKws9Zr&kwaUDn5$3sfbR_8leS+Kt<^6uZz)5>mq z`8sgxur;>Uup$kvK$8*DH1MhdZ7>01f-tPKDA2iAFbyFggT?FzV?lu?s=Yo1?~Dq9 ziCYG_jB80eib)$r&=zGkT;!C{-AYQpLFIE`XR2WFFc;YysIqpa)Jvd9kadh^S2s8C z!CHj;q@_8H8B9-4H@%DC!v8A;dym^Fu^o(x;`&%r#P|u-*zc}*gIFbid;5{=p6FBK zP>2UGt(B*Y=-#}^C4(!pXW}0j`T_z1D}Vjs8YO;Gl_X7dSZS&O8g^(^Q`!F=?CiFc!Oc1F6e%pRva_pG zR+M}JQw(7_G1nImA?)^@JMjSMVosx7THfrNH*?8gf=D;@#XopgL7hVH9SqRPb5_9< zPM;|G3p@b&_~1VPw9hh!`3cY+X>#jg@m!3>i2FCTjB zF+>7lAc5q=hd^vzJn84(&3R>CX`%f6 z>5Bqmv&Jaa@el(8ybq5|^LD=x@Fb^%ajhmD9uT#$-bx2sQM_?L=Xc6+VD~M2n#Z+V zPMq+B%Zd5T8vlo<=nKmcOsWt+6QD?+iHRMD4t5un(xfC$&Qh9mTWz_?+Mo7=^2$M3E|*oukcg z1)vgh7%CI+#7C^{_)huO(-XZo*DDS1^T-{3NLZ!f!GI9GF7hJzFfM%9QRpUQG_*Du z!^;A?zf*$nnHWrraVp@I9$|Q62B26sS8?MtYWLqSy$-^UJPKlnf-VpMhfCx|y!h2o zydTh59{}|gWIsn9%>0xCEBKazUS{{%_6p&dHuTF~Iy~vAbTdy|XU2%dq9;UneecE{NngwZm9#}guXMg-x zmd|@%mPb)&OZ_ns?$v`$sdiu9Et4VI!6SGHH<4`vT)iO$pn?db2k>aX7&;SQ0ay>1 ziWG=`B5WG)&yTC5Aa7KGgf_>%+GK*=5ZkwKND??4w7IY?&a8GFNUs(DARpc!kgkk#I_3=Hv9htsTAj;a7J~ zJlZ!b91B~%gTkr9tgiA>?J~g};P7I=O2vX{BZL&k-!ULT!F^mQn4b*A-%+h}2SPLv z1co*-sIU@eePYtmk@AR(xb9%O5PeM^NU4n@Wex0fb?k`}|DUFL)#!6wy(QHzJ3iR22H`2sO7S ztyz6I|5wlI^i8KxocwC^@slp6wbIMKgVKIYx9}nrRhZg*RC2U`TUlfph7)SG;}%|fP_R# zh7aaCj8h5V2n<;%{xllmPLB>6!b4FQkE-hzFdl07 zH;h}AG-mEdA^Uw_Usa&L?yp}-<$PsSgiW;av z>4YALdE$JiG{a`(>se88hO`6@ABwRo4VLOS44Rl%KV?Y#tEs95)R7uiVD)2l2cd1y z-zBG7Y~TvWzxroF0(n1pyQNDrh#*2v@6gqkL8O|&4rhXxamMBVx0ZNG6#Nd(Q>_jv z;2uf)CI%r)C^-*QaGYJ-fFhvQi8Tpiv)c~h<+%dt55`u*KKp2e@c1w9?+ySIz-b=? zD&%>fj=Q==-b6rRC7{7$u#lXR&fnm}hOHtWj@I4l*7@=zVr=bj|B4FV11twP5(N$G zfTE%zo9@tbHGjk5OHV!bDL#2g7G;`?kc?29Yhmz6HKQYYro`{=T>1m_NGv2Gh;LzJ zVup+@OT#?}=m&a|a??ZV$kcE$N;g(`uHhCkCgIdDLW}-qvZJ~|d5PWW{^KNiMY<_J zss|>X{(qK6Ay2e-dm#_wiYJn#!jt%9jyGReUY)$PoVbRlQ{iD@k5E>jmnC4>_;>G$ z%3Ft4h-{RHld=luFaea=k47+q<%ECsP1ohGAb~f76eQFvAjTm9fy;z?wI6DU0~_?u z4*hNQb-aIQ?YD=-`vmj-Y0OlQxL_}6Pg1

WOWYudy_| zhNt8P%Mn4r>Y)XU;={4>2*wVvwW|pDv8#>7FIa98wckkI-Fo?d=XzXMAH@+?4br6u z7=m$D<%R4PJj?Qo7S^8*I+T54TL!nZmgD%tTU&5tdIb{f6!>goh~*qLs|J~uj2##w z6rw|Fubw7W{+DA=?{$I)no6NH6!Q^RyOB{Wv>#K`!?4mZ+Y4D?_)#ZzaI6&^mig&d zV`YuSl3E;?=C#TH7`z8pcMxY8d@KX7I9>+$GpUVz2K#Pt#vCY)`}gi$#^olKgRH=~ zR!6w?=X&_qyTz$2-?3gpy3nTp-UN~h$@w$!;B6yO7BAAE<^5xL8Gu)&IiQu0zYt>V zMF9O;_Eows3quCrV@-7;T1g#}uvx5MII3$$t5fhX^ZyLR?`M2GF zj~Y}bSJzx(iG20y)#@T!gp{wZZ!0FLU701RIxJJe?K*(tHINF4r)3`FPzdz!p0eP* zGGwa`OB&;75%FeCDa5}BX(S2055O%x!w00-o^v`zsRxUXaIM2(30rqv_}e*i75K)? zZTULRmza4czEz#ZjH8MnghF@cy5D^9$Y3xt^o zkToXQSR%`KfNsx7SG-HWp=FrF4$7vD8v&5^lMS$+nI#De@lKDq*JL1tHU}I3^rifcMYg-bs*R4#5kdqXbq2RP!7~SVXAJR{$aCS{hJ?RPtS&AmW~R_Uswl z&=Ig{y!h*{UuCK>Qg7a|BN5plS7e9mpwnIc!3VaE>QQ*mY>9=Xp@+##Bqo$*I-mn4 z-QxCk9muc!t@O-y@7|4rzwbR$f}*k3fB(`BCRp!vUEnA20o>f&L_h*YIHwSZ*8Y&m zvx$PsF?DEv|Nb=KQv;SGY@&CeW^mBbAa~sZXMXBoN?u*-OayGZ@C&NgW75IWG{6yv3NWlN;jR2ECPl*pP8W;L=JO;&kG z^&7C69@QtiDIheI;ISpEG>wgo9Pxxk5g0Y2Xd-EdLmOE4EZM>WbigDTjq>zgNxfIr zUPV|woF5&${gimT96Z3ZfTH^8I>$CB3_5)P6!lO-C%#4c%F>840dmJTAiXMRPGOt$ zNe84$h%a@Fgb*V`D<~-7BCG|V=vC5o6wwheKnj3Q6MZj$wd~D1cUm~$Ma5K}4HOuG zAu&pmF)Us)JLM<7#~RV-*jVZSJ6audrPW1P7xDrtN6q~F`Q^OiSsdc|S7#_dR51%b zI@f^@{`RDh?cW;5kNMNMrP;0~T}sV3h#1g~4IpS#uYmR`{c=!(TNFSF+Fsl`9lBXOR+7tKIt_%mMn6Vro4VAVo8wP|5IOQpe9AZ z8m1Pv?4yTp|BzV~1DJ3O;dY9z2CVg5oUZgB4+=}br_cm0p`{SNLb#aSp_q##Y>(lW zK*B0PgQ!b4RVCyISVdZ3V4xH@p)fvzJQ!I$wk)FHv<^4tyV38g-C<&CV{cFL;h05R z*cY*X^^XS+{0Wn6c+nfwFpyfws8x7)ILz}Q-JJ%#jkSY8uj53geC5l1*u?3Dt(g`P zTn7*K@W0T`4%6@4w`jwoQCME_+P`vvR|Xb|;?c^cv=J`>bun}En@tqQo-#IU+#EP^ zbeDvrQ4AlPE+X?XTUVQIvzj`uN_f^D> zTl0ggP1zCN0_5#L#-`Y|w0uZmc?J3i#`MdWlC@m$4-n>pnnkUxT9}sYsW9|4sXXT@ zDJr@v!n)$RwD|JbCV|FEpSSMzUW3h*SHq_->NfeGAZrZBM=}-H2jHSW^MhsPS#g*= zA?<*0P;h8s2LuI`2yu3Lon#co$-|hjuo79`SB;EK@qV%NI^<*9nQ>#Ha}byeoCxrC z-**3CTkywBT&_;6s zm1~l+YZ`1rFr~sJLpG60+m?*|CEi*|;3h0VTR1p)aIv63eWrXFmNBy+_)DC>zg^Ld zbyRplCjfv0`3;#CLp{R($+a*TFz`}~ew7S<;QK#4Iv9ywuJ01=6!80d6xQM_*wqxJ zrazFyda{pK2$Ri+!a{?_MGUpaV2d-r$_h-glElV#6s2B>;E`L0E8T`{b)>;a5Lc}M z$R)CgjTNttFdl~BR~T;}_wqU9MAVSoaq=Au(U`2GF=+W5JhqKqGam-p$fO~OiiuSp z>bDUD8Z&6{reOTLJ3iS2;`X859YM$#faB@Gn9~EjE}cdw)CJ(#bUd-u4NbtXS{^b)D2bl8DsLSeU?(y+ zLI#R(Ws;yx9=){^Ov)9EhhR#2{vtnVs>F_vH}G9;s-Yrgym)XDWXl^a4h|LjBpZ?0 zY%Bqj62>(B3XZG~3X3wK)Rj_IzN-hN^UPn|CHe(QGDKRLDbu~sxX9w;oq-RqU^Pnb z+qaL=U$$zx5Z%2J;XZvsPRQ5@{)Sm%_6l+7NNS;A3h~ zm0vVyyU61XwhP?+bML1LyCDN<5>jM+5v&Qnt{1U_sA@Z%U7Tqs;JBQkuQ#bqI6Ja6 zSW_c(_fTng5X&!u0?@V*%NOwD$9M0ZR|xbs#4hSd{@m|>NX>lPOC*z+(vlh|3|nCO z7q#h$1$OTL{OlH7qS2sTxkI8(#I@sg+VbV(iHmjQY#OOsB@4t3*|A+&Vt*6%oXqT{VCQhGm{}y7j`**^9argop;56XS zp>hhtPjLi(NrMl6ZO~labF8&5A0yaH^1TQ;ZWE71K`k=vVNMKl{x$nu_)ssq)za&Y?N>pW^Wu=2^OTSoS&wH?1K(^yO^eX|-Xu4m4k z4TflI!xrRnf>O4d)E(3uAw0jHXLmUT3+6^2P}LDja@b?iS|n>Rj}EI=km5_c!iz9Y zu1z{VmE-S*oVMC~u$78C_TiTZL}wVE_|-FWSYc`mLV_Gr7vi@+2t$_>HkZm!iFRn( z4cY9UC5t<`^OFqneG`T6-@pI!&YcjXS9ur|m66-MdqD1$e4jr?Z9jMJTnMa5FEnCs zJU_o*1K(Wd<{%jJ|MHhk?Wz<7laAeFVyTov@m{%dCBd~5 zVEKA&jVYEomSh;)l>n#VH-INX(&GSLU=BvG%7T?s%rGNgu4HXvQyTD^WtaGorN8b{ z6vgHk*&u{LjW|DuvPjH0WPw&uV&u=nN+^LZ!B~=SKthEh`$1GMh{)HlNI}DK5&9EK zPctm)cGS{md)VWZC{0OQI2CBDJqQBcsgtQG4hnfEC8@RA7Wg3;c} zWRP`@=|)wjfc}YPcVR4I(Fj!&TkwH*}$nC_RtnPD?7M)W=>v zjUt&o;nepcunWoyOjV3#v|r*?I|%Ru7}NCQFP4r9a_p7&{`%`PS&2gDC+;`mKgGA> z5HbLZCt8OfJl_QMj;9jJ2^kEVKp z7cxVD*(OTLN|yDW+%OYfhH*q0BJu=SSyK>9Y!yqc!bIKLebaqm+7PIz!anT!Idb9X z#<5rtQr8xz=n3a9S^h8vbhBxZ>^4wtpC$AT*gJTpM(F23`s3b{(Y(teIG->~HV2)|_+9F~;0aWu!#+?mD`QLZR#x zyLnBHLfQP7LfO={b36WqySA+t|Khj2e#cVYM90$Rp7{fc_@#9Wu%!eqSR3 zTok)@S;01Ntlci~h1EvM)WW?l=Q-1xIjoZzR3m@LFw*vrJJotqfl02ctvT!u`d888 zSGeWcG9#Gw%Dy@kR?5GJhjU(;CB@0vd9gC<1ZUCRemB37xkrLk@2VD-H9AKu)GhdC zf(-;NYPtP{dtiBY)9DEQO&K^v*-W9_*|X_K3Z?M=&d2zx!&fO=DU|orn@>_Gk3aoL z*@Tb(PT5JJ+`X{%5`O;t=l>?Z!c-F}6%>1vTc1ftNT@*k@+R}anxC5uFII+#baq^H zTD5I9OkX{>tLf0;!;#Mpo$I%u@#c13e}qq*;J0Kd5YM`oV?E~i{Q2|s)tTy5w!ZYs zl!^xvc?-dY7XyCz_1Ba!t(4~D8u*`nM{ZXB`tpc`Bj@^`4{`+s1rJhFN2eRL>i1Xp z-MV?Rw7fAZ;ZsS8!RI$;?U(v|6vO}Ew&{Cuj3-vFJ~F$&In`6rS+go8CPu#O5H&TEw6yfL9Xkvv{kfIr^Sm5~bks6UFZYyq zG*OAHzSJ!*{fx(Nz!%xs+D1QGnm)>794IF*Z^30^Z=X%p@Rk>ISy9ndtC25_10l{% zm3DG+atd*`S+P`-0T;(2M>H}`Lv<>A&FW_cn&$KCk_y&i|NM1F+`XdT&8@5|=cy9o zoF~-Rp8fI1`-RHIqOoBgS69~}-!%P3xf1si+}5LC1=UM_`HP`oHPR@+`qoI zJd)H>;9L-UJHh`!!9}B%Xp_!-j^e?|?jnCqU3Y$)@p#3!+c8G1>3+-7zUMD{F)Pa5 zy7lb-{rj^;R3Z--C%gTH9kXx8-wDmO8tI>lZMPO)U(||-jEvQ-4qj5fmH##0F-IlC zIKiLWK!165h~IuTI#Mzq(s5;8uQ67!DUe54SOX8IXlNM!``%+`oSmH=v6{mjc{#Yl z*~K1@)|JFNAFin9SS#MV`Nvw(d^vl8LTA3C)o{aEp^eoUyS!Mb3-+@QUmWFTE+{uW zk3TN0ti(C3EiccvnwZ$Gzkh$8RXO=ukt>xF7J>^W&uXyd^2nD2`T9u7u{Gldn%T-n z*wkv`lJc3w^lQVe2eb6($K6hlYf4ma47%n0-hW63d+jL-thF}G0|}H zF_*-D6jN(S1q#yYLOwCZwwXIc%k^A)9Z@=-*eh;Pg3jVo~ zrfP*~neIp*Wdwyqwa|j^n>Pm?TF)ir-bb!z&U0Ac9uUF-GqbX?S{#*e?ed1ZI-u{>AAjuh zVo{bmAzY~c%zLKq$8)N$KKCmdfbyuwm4vo={oHUEwm zL7e^B>l`B$6;+yXyUNPKM0JDn)VELiU*Cr!F)2n$(9Dgut8T1KIGXnoz? zp@&&jdIy(1qobL|zJ9nG8XCH9jHbyN8zz~eTYVeLWL0v)X7I@l+L~9VMYY{(|9W~r zF;R`r&fY#&+=q?dqF*9%N-04lLZ{L{=FW%v%(7v>@mr5Z7pyOrhgQ3v5IBegk0&Wd z3R_=axBvE((Q+Nz4Q+vKwQ$noW1eydx~8*4U^^Km3^ zW4DIeb24&sb4SOJAQZ6*bA3LV8W!tx&a1Ky%Du6M38O~WpBq!XkpLRcosp^x7E<+0wHS~>BDnSG z*&%tCU58RlyFW^31q#?mkqjXt^RlhIT`^5xQPPitp?>b|Kvj_Zojbm{d3k+3J@1fJ zhT5_${EEwmj~3d^^vB?arsw9oa&mI?8)LotUWFmU$Ko3irXdZPW?CfEBfyt?UrTe$ z37K|Xa^DE*{5)qY?#&{LfaZQ!bJ8IINqYMp>bKa4q0(2UJ&heTKUaCPsy3t>sjfK5 z#mF%TP5pJilWjwQRW&`HeyCANGc6=NXZqm6bYHn*l++3GB~n5BC|b(i<$O-7nd|E- zCK2J`YAL};n|jD&N4XxpR8w2GnKqd0`jeyhVVNgGQ{aZk#*zmr443WXU59{5oqEn{ z!FTUOJ@ka~U<#(V>3* z`QgUive(>Le^&Ku1uS&stT-1pw<5}#7wSJLNXG5i%~5NpH}>5wM;ekeMPAT}Xyn*T z+>Vk8;&PZbtPJFn|8)n|7jgEya9p6-Sxl|G_{A{>vAsvRO3m%BUHc7XKuBn1G=*Fs z)oIP1d^48w%Wz}d+;lm+LCxFC%foR=v)`Vx^H;3kfRF@4j%K8UUl?-9Lu{0OOKQwj zUiB267yD=hqu19~48FuGqoBAVcEk6b;MZRIxgpQNhJl{`;dE~qsgnKGAtoXWue;lA zIu#K=b0bLw8Ww9d<83hw4GP@)4W*&&m6ein!;SR-N{M{V*l=&UziisfTctq_UvQHQ)epnfI0&unnjn$2Xu9Y}>X?Z@3{Q_Rfc}-}jw}U78t) z!|~E@h<-&LfTSWohiA{8C09Fif854x(xDdOIPbSSKNjc5sT(Wy`phmWs-{1FIz77P zc44?K;<@nJtUSqVNqO`B?zBR&2(G^9Vm^!hXjG-;0g;WEj1GJ8%a<=76|{XX6s@7R zvA$;C`%1JSQu0{d+!s}5<>c^>A3t)eb=RPdvPj?S!xN^**v}2`3zhN&xMCTa*3%sN zvJep;$ZK|mN^tTqRxn)l-3>jmsEu(xPD_19#N?WjABDehctI-`QUbf;Lxmjg1)%HzkGis5B+0>eBlG!SM$UI}8Nb zxY4H|iE35!74_xk=L5O&_ka8JeBSAtQX-dU#n%s7V@7MsbDW+P(X#K7)iiBvY+UQN zPZVtAztSc+;nAbqJRzWW(J1O09<)aEe!u;;)pg!Y-T@h&8wt0ZYrHYa8IiBo*48!} z#dBc)em4&f4;o(6w?&yK(Z#K;t)W73_2VJV>sha~FB*QmYf{d89Z6D}ch+$*)Ky}r zF5*NB$8w@-#<>4T0Scx>dD-OPJCCEz&Km|*8>1=J{->@#^S$c2k1us$u(s$X2mY9r z!Eok`|9E@OF(gBJ8nJ$Nf${U@KI{ek9El8C&VF2ao_S-V%_+t8_4Qv3FQz=KJ>{^1 zZXsx>p%JB0^27a46&-mFCyGxXb)Hp8jehFxezBN3<BbQZtg}dgD))g zXS>{toFn3$ig94Mz0N6FKa7(Jy%HHx^7H_s8;Xe?QV7RfUahv~eU$gR0ICS|t)9Hd=0cNRN@#`LnZ@^(a;KDZ0{H6&u1n)87tGy3ugD z(a$vNhBUW&zJ4tUk0oE)l;dVx$mglCV`8GD`P-S$!zQ0%e>C-24F_qn(zwg<7gbgR8zcRoiy5r*?rypn0pE`8~)j_{G zIU>9~YCEmN=mkW>O+7t5zh&pyT2W?{yho0XjnT5Fuiv<#-&^`>pD4R|EVh)#ipLZg zp|8JR$GH>HC{_m)%%CppVnHest>D*g_gH|E*c&g7DtDxP!$MFF3l9r zR?RR5fVmbq?{+lV0x$*GfmdzTAwrbSQ2`*)uXqedq-ZFltLrWUBV(%vEu!ZJK?Bvn zLZtUtiF;TZwrk(MFUUQkGa7bV$Lb=)`K^ZEV3CEz>TueFViGCL zGD9}4UGGnh>q`U6IK2&64y4Xjr)vW`LQB1`=QkFnrfSt~+9U3NdKTlbFg`z@GoCGu z6tHjQFN7w^RzRn9-2(@Y9*s>+Uwd3>UVb`AzW3MY^9hKJIdcJ+L z#PgB-y@QIj!tygPm&&~J)w;fSY&PvyN|BCBHKA9f?A(e!JwKAtF2WJ1xzriZ?vwnj zr{}bo3UZ5;bfw1F;#5xyTEFGFk%38*OjOKR9O20InSpAb3;o$q%D^`MQBW`)opDiD`mza@fa;bRthx#}aqHDx!!kBj_;HR!c_r@of=~mfa*V*)i4panB}FZNW8Gof z)~yc_c|2+k+tF66Sc%a9zyix>NkmoPPbbpr^Bq@GQ7I)v-nkiE;dGL6rJ}oi>C&Z0 zk%P%CQ7O|i)gtG(xD-&zUL*5<_GXPkgB#+>vEi1qzI4mSxi}-6y^}>+y4rBT!)-`u zeL)9RmJ3G)C9@u#bS(OwXfzG}X!g9zGgXQQZUJ+1NR|&(v!2Vy4(4kQo|>8>fB<;v z&Ctvl_T$2j?x0U8pUpYlqcGPPQm>pS{Et6mV&o!#KUR*HY^nt@ca0%CQ z&1^)a+VwMfJSjV5(3IE+HYN=z8QFuWeikKCXtHq2J|k4fl#S%!n$TZSao>erb&bVx zH*55~^a3n;nht(U3#xRmin9i-?LMa7bC$m!BuY2Dv!ATvR;KdUerk z+xG1mHg2M{cGswc7vF*-po)lyh`Idh_Icgk6q7X6J>LWd#(M}a-#jhqZn@Z;ot+&E z#B0CMZsQdgIBH7I7iIh)2!9co8uWRy0RCJL;4p3B++=(vKOp|CFL*Q>U3I32)6z9 z0|8UZ=(CcA%tL$<)vDuuZN_4bHBGAdUUpRU5$y-{cI;~EL0v6u{wC4+EXJTS1Keo!Te)61rq(oxl9hsjf z7MFpSk=9dk&p$$~M(5(!Ulrs~S#UnYe)w!2NQR?A_Nlb?gQ3SatURaU!{dWb*v2s{ zM4uFENzwHvY7e6^qN$&`3etyOJ^vUlPskJBDMKbr%}xm$YRw_U=QB6A2Asp;$-A{~ zr~j}^07fx4Gpj|#JTB#PhLO<#VY$z!>05t)q{BpB)4FgEd%v5Nh~si97`c>Ox1Sjp zt`7|jDNB_bfM=1tfBze|^NqSbM_vvNInamuLbXr_n$(L)@P&udC*1k)v3sz?%V$$t z6@oAd%XU^+xTE{VOOP6^QQ}z{><{`7{Pm6t?NLZB>L`Q?Ng8>nII$#wH#-$Sc$BC; zXL6QNBx6)NeuCWS#|~6#%h^F$*PMuyfDO%!WgqK_jxkf_<>4`!6DJbTwZ{S*_!avC znw*jJ4+olPUKwaeOK(EqdlMWyJ}KG)fMOKll(00jk}_&9ys>IQqm`Vu!gH>e8tm&$ zLaA`EGBe*oPhI9EvVS+f)TYj`=SSG}u`K}muTgbgNhD#DnScKI^10jbnf<&7n(r+R z%FOZ3uzc$=70EzeSrCKSq0YzoNTps{Ue+IP%i0$ujM5PSHYpCBPF8+J;I1Ejcoohv zu`Blw>AZY=d~nxq`qIQ1rGwwh7Q7QuqHE9cVw@A_bKc_y5Paq8)j8B9KDCV{v`fqv zF32KGqVvX(4M}gA8*cPK^%~-X4ifi=PrJ}>4*t1=uEKnooL^KjX23QN zq4}@eCA)U-t_R}he(?FA)mTg8y%yB@*@aH$b7-$WeE6V}W1WC*>)ozH%nwmb{PO3| z@MLCZ>mwTvcNP?&J|qSUIWU8ZpU9nYXTBZpo772>%`$(`7$Yx=v+L+id+fD5$bZ5{ zgFk+^eecm|R62qS!`{7{10@lM){QhtRPrdlHCNN`3SQ*ln6IQyzeuVXV&7gY(Mk|g;0H7uL1a%W|i0uJI74sk2 z5!gT&S;g2lNLU6F@~IDH^A^87C+Ug+zqzqiC7h&X^co1(yt4D3fLNf>K=nHSIM^2b9YiE zH)n2PA&<|`HONb!MK};rmZVWZb$@vpkH=+YFHVLyEysXkG^DaVcG0i~r1hDh>CJEi~UFWW#Qz7$>_Jnh=oYpLnUMkfZHWr@2$&JaJd zhYmAabCoIv(gYNi8do|eCU$mq!l2jn8T5nniU8$ur7=!P2~}DFWE9yqK-b$492!&g zQE$mCE!*iEFvM_NFb3DjJ`k-Y6uYuL^f7a(5&Ap zxYn=Ix^i2%Jh|m|GvA}d_iOQr#dH%iD;8i4M?JX9Ckd&K$At3L!0qu^c>b$a_R4q; zVOp^XVj#>2Hkid$@y$w#iPZuMp9t=Is3(;_l34>9F=c9U^87$`NOih2P6Ie=Za`Iz zV(m4_lQhL1bQ6=*FWpo}IW>%pjf*=wJ5%-Qi*skH1onBzl*PD}6tZaKehn4MF%iY_ zW0yd`{Q`~SSsctEfIvPR=&K=MRQ%8oJjlw8|H6|~a!OlC^>(Cmh%s`)_~g!CkDb6@ zj>bb~NHuExV(Z{=3}8_{;?|)9hY#xzGGM|2#b*ZFPlK%513G}#aaPu1^y>!+50%n! zu{mWSxu2tezb1jd{gIG6ZEbCRksbVhzWTd%3nRyuwc!#GghA2PZQE!7P!3R2KMZzS z);n!yf9%xmI+)+1xUi6J}u?s)(*ZkmYl|tLhVO~AIxPhcRsz~LpMTShOIr_ zdLiV|w3s{;7WuGiKlBU_>q8sV*8o?>;>?q%lJ@x~2e4C)al-ApoM%E3Wwsm3s;LDs zD<#Iq$VcfHKHjvX?%s!OiiUmziRAa+_t1ED_MN191awLe1p&bAC?iYiQcp4rn%Ei> z)m1?pRWym~$AgX{jS0%OX|C( zOR=2|??kCSYO*P37zY8V?{t=W@33H;pq3>E60>4L&iL5zO)S^IVWi}at=&Rnzhqu3 zOrc~tnjX0Mb23`(6`ohxjEqNyH&Y&ed~y1*OKEe8ZW`d5!8}f0`r4Cd2=@N}5bTR2 zA=Oh37jZw%I4LBgj?kBXx}Rb3K8yvsc51op-@hM7*mun4>k;d%g{r0`PBfYK;fKEi z*+*k9m(*|MpQLpB;+FK4&%F1KBA z>2Ca^C2wx&M_XVWp#HyU9mwppf!*Nk-RJJ^U$ATDj%}p!K$6Z9W+Ttqbr{OYNa9g$ ztD$c$YILndGyBGt9C&^{PeptRg^~~e->|9YzO4(=cxf>7R_z#1-|SC88cW`Q+1Du* zTZnvF$;P*jzTS0St2&~sMN|4tYSp8jl z_Ec~s;MYL&7NqJ_9wkSQRNl9jf1QrihP9*hcX6j9rks#qlfX-8rc^s`tY{LQfaueW zAGAbW^&m5RR|%l}#(zqq$TJqoQXWsAzJlCw zH2`J{6!ZvRUthx#$y>L8mOR%SA3b_BesB+-_L|H1&Ms8MIS|kf*H@Piky-rnV8OvP z8#zM;=o?J=_b1kNKGwwIA^-_S#f9|1SIZw8KlVXzj{ z3k%iOqe$T31e-NQ-H(@*E8--U9&i~yXo_47dTO}l3xuz^p?Xo`&Y*Ys?YBWlZY|kX zaR5wAVgyju)z=?7emrpOD{>@+zv`TCxJKaHOWO;<)Qoi&bgm1drD@GD`3_2uj=k8_ zTS5!s;^II*TQ#ZMhH0?XLolH&HHyDxH+xWYU~`9SxEa zYAb`-=^Jh`_yaLf9QOdb8H%zWjXXi91?*ESdi#?}x8P#RG&)#%`Y~vTmlw{ItWlld ze-o2{RH%|^dXMckHb&+=`8n1v)9$~Q&j+{sU-K;O(x$Y%zxONgkQ5ePf{6uKtxcE} zoi!MUlZl0t-5#pE77jDB|TY#1Pg zu7DK63?%6Z4cx*u$xy1tAo%K_RK}wN`dnTv?s@RQfzz6rnyyZ0u}O;omre2s0|FdItjmq3;k_CvxY_ro zbwZ(nX48le_p$b+-M>E!ZOK3M!Zd>>X5=BjwJJ7v+X`cL_5Sql;_l#&KfWcTSHZ@5 z&h13C7_iPOR*oP#Js}qS@566P7ob03YFAc%I(_4&C(v^gSSbTEk)GjTF8L}ljNAFa z_+9Zlh-f6t9-@(1F8mLGf=HYr{sbX<_7y>)RC)9RL|CQ zBdei028R$02$@7v1_6<3-gk?r4TwJTFU+{*-eW$iY+X93CY>5%tr^wk{ugdQG^!b_ zeb*QvenhMeQphP05fReYk$w<*T$=ryJ}5yGA~n;0*KUz>iDr+i5|ywUyPR?eDYa?; zp+hKX)To3lt-$^$5%JI+#5_Uy(4&EISbQB`QzLWetlYH#9usT(O59A9!buN`hPZ@; z8+}=Ib#-Vv{!f_o%$Xa|HTTs+9Z`r?IO`A+PtU+`_UzfRb--%jjL*pOrv6uKww`B~ z3>H)dsGNs!M8?SnY09){mpWy30fp;RaWQ{tNpW#Y)Zaz0US3|_s=tB;*Zb{1hN}`4 z7I*=_7q1ZGi3)XyhDOiixrav+*oi~?_n#62ZC0CQF_3CCa@!%~f5u<4``Gw68`^PK z#NnmO@+XrvNk~XEpj{rDco!Cy>bPPJR_+#GYt!aK)3jBVg{-1gdQS zC?b7y5d1S(>qZBe$78*9$1`ee^P>tpK>DCq-d zBx-*9;Xv}3Jl2|3E%QA4XC$J`ghSZ|#1w04^nX?+d4(vng0?ly&q=X36i7>fZWW80 z9!{wiu16XR%(KOpms{9VR>AxzW*Do#z{3!u(r`@FEYKT&4jz|MZf;MBhd|$9H*f-G zE(th~KDg6~B;vfq0^Jg~BlPm2QBQo9g`v}UGdwA^utLUzRUpPwq9|!Rnz{~{6azm5 zKm4ywUN|r^(S53vw&VxTw^Px^KYmO>XR1!zD*KM}zD1211cwGwg!n%{o5kz!1Z`|n zuH8(kZZ$2bIq1rs63^GE>ukEUKyP|LX2gq;B97I1zcp${A*>UFPZ@({z$R?oyg7Na zXJjM}glTD+7nbqPuk~BHz?LRL@TimolCikLjYs&>GiyK@s0@h0?d$4m=GuNZ%yOq| z)SNtB$ai$_ShTFps1avggeK)TWf1Evpw2 z0}2jdL$;McR;HT9Blu(CBPr*8j1Ne4;VVGf6W)n@4jc3{jJKqvjFk!fB}E($`<@=l zv4^ly=%i;1rpTT3%~E{5y(4j8&w;~76_mkMi)L~3`E%&;0dX^X%JGYazA)4fFhVvP zz_*GGmn*EpaXJDA4seScXmiu~F#PS?N49(1P|MIB508F=y}N6JPl%E~`Ej>BdQ@#U zvwwbyciU>cw1^&>kE&F(5vvhKynE2CW0#h_g~vR#rbavijM&~U~TSLJ2)o1yuH0+ z)^JIpx+k*gB2);a+u^vNKGu<^iZfQ*5F?LBU~vc`L;_$90p=G?I^v0@4ZQ%oxLjf* z9>FvRTlJ{x$qv%Dt-!h#3D^&xHk(obtufFWF&7bIBD{FRuS98Uw@?Yb1r$_-g6;et zN=;*esuBq*Y<47ayB?V<&|fSs90|qT3O86^7cAU;*3BN={@Qrm(AJ zB9|R!H)DZS1|1!P7>bo=K|#T()2GLtu;VJI(qG_7eBI~!&pyv<84o)B9El7Emop9U zLAqLS#!bV1C_+tDitys-!sF1268!?KUnHz4Bglv^Etkf72ydL$qygrz5$zbm3mVQF zWHk^m|2$M>J_eK|(CyrLQ^m)K4;41b2{jrE$QQ)8OY%1I4HLXx(n51@irr5bOmG5i z2Z=aq0Nm7}KsSK!PSD6xp{1qGlEbbSjt!>od|cOe>?(HK+n~pbN%jg3ps8(tS0j3b zGZMaUe>iZax5XTX+Sms95dxd3t%+MLfbJaR1Y(-2V%vcWnInjj^7E^7Qeq67lZn@q z$QA!HvC`OdBLSUs=3R^H!Wz9S2z1&y`FY1#&1%TQGw=7~?+j4b+R}{z$hyEb-Htxk zGaQ!bSom1Ig0Vgou+GTn6BOKcOQ?`u?*|Lz^bC zN|1)q;LJ;e4l@Yzif1@Nb8;s&ew)!0=?2c5zTx3mBv*2N=jTgk9VFtElKxfp3@8Qg zb~r+Q2NfE4)d9!Ma&hwB-&-WI*>-)^&h@l24dkQKaK!!n1FFaH3f2Sk-gh=hX@+Q< zi0ns93V(0B(=J=krsVEG$;0=e{o**Eob%e84=(f$YDx3oi{L>4mnRBURuSzf!P4Yp z5-TIwag^%6>+A4Oh1h=r`Owof!aV=2;yOuD8+>y+--06n!_i%FozydCDUoe$m`!km z`cl#5YQ+SEED6Pqp%Me4voKw^i~aE>=qSZvf{mE6VDG z^F|y@ZgIFg+NK6@1qqOvzeXL?;<6sS1J6wWZ_cd>HQop0&)OiSDfK)pAq1V#FuR=mXyxBDq35?3q!9(aWrHWFNFqQ)yaR{0p zu{k62C*eHQ2n&H719aH6cW+t-81{3#ymvrZd^WEDP=GC4AQuy;Cc43_fnXwP(~IE>p7iL{T$fOoM(YzVqNLzzt(wUGb~*cT4ZF8B+Y4bYM2B$c z^%+Sy!k?j8*n6B;21hS%=t5CZ5e(wG&L{ZH>5rTPN;Y-Jw{F?InM^&orlWmi=I2-O z#M~CU7~HK^gGA5>vF1knfFk1i*L?4gDX@}|PHr*w!}0SW*2AK@f}Ww0fx1d|1j z>(NA+=_lZ>suUhhY<5CCTWzc@7}SM}5gWDB+MJRjM!S}~ex;%pLstTOR}&feL9T_e zd5cs4G|=z?H?6YLBGpnIYN6Od9ffo80r4@RgCS+5()YquRP5vh8b0&zst}PRG`?hH zfEc@PCuy)!QF%Oy2hKfdg5PHnBT_IlqU0cBu+aBDZk6;s|L0S}i1sF3aoBqjcbj~J zf`;b*fiXcm%YSAo?wq3JqtXIo9~FL-nMeGSM>(|rxaGr^0Ld*522EmUE;xnN2n?^; zQ3$(|BvMX3rin&s$gJQ93_f-EXd2A+q*cS&HWgMxVd@fe;isHAa|Y>LV;NI6ipkny z4h|sbh|EnIF9f0-SP^&}1&!J=WeZlP#s3aiE6Q=)1c8s>3S8$ggvHFj@F_G8am1{d zp9?Obca($N(r1j(m7XI|4mXvEn`5m{1^bYdl?CdugfI!xh>&J?zc+8f5mzfKi;IhD zafC5b-qrV0A)04m(Zj6XOIKIX#1{hpA$qbdUL@G*;AcuET7LKiufR?965P6)ni`ob zAlVWW;q=T5F=SN*2@IUq#QYqngOn3If`V#jN#~m|!!vCwiB&h(3>DnghE{5 z0Y#UXPax<$K#yA1*r@pG)hjZgMXv`-6FRxuH*eld@5#=|DQ|BVk~RSRC^c~5pm+lf zz-D9i5Fo`Dn`C4{cmD2UqNyW1%0ORYswo~4oMoruLZ4_EzF110Y0F)yTu?WE({fmc zG^5tr2pf!mq0@Oez;b}%yx6_Nbpe>kU&tX{r#d)DWE%;9*H`QgkX)ZE`u}PG9BiTR zV%OUtD3eYNdNCO@7;8=`39Uv6;L^qk2EDUz5My= zEa>Bh01yC1-`gDcNm^F~F&0UDPpP(3_Y*V=G$2&ggP{oYP{eQd3W0t*nG~;)*V4VV zx-`>=ZXJhwm~Lay6@z}L^mKaNA6*EH2oH$A~@_xAUn z0W0T9hJ&6xeOhK7h|___wYd}FGoVb^IlE8$MJcX(Xu{Bp$s^LTBh1c72eW{!UpWU? zp${|pg5WE36!^GeVWTJR9)iB|+QDIu zM93DdWeelxFr4ujQ*KZ%YB26X7(^7SAsE>(!Mf9i0?qeI0tzDv>rGFU)Q8>QCo_Rv z#5_BkR*VQgiRX|wlmZ6dSc zCk_4$aM}475Bi@I?|+&#cbN~kgq*ZN1Yb1ZS;FIBTrC_WdWANhq)=ScK=X{p%F6q& zX_QXLnFgH17s-^OY#{?^$P7ubm!?_DB)U6Jbshe+wy2!w{xp}wYvHMXioc2Oe&)+J z6WwV>5`QyYT`hUV`F!Lc31mmP`ZwfRwGytv_7%X*a9^yZFf6 zXFLT-J%qnuZXeo~<-PB7ni)H7TUaKYNO{_n%p}p+Gb=N__j%?xH>w?D@0)$4c5JuH z7M>>*@Ff4GS^e(IHK01)<}T)IJe?SInYklOsR|ZSCEW;)x+Rf~(MIaGWtx$^mnk{r zn8&n-I=a3 zAzE5vlXDPBV67zN1j~Tf-|{cYBR%K`VQGE0r%z=;Bqw^Pg1UvTd0;uyj_)G3fUcoF z+oG2I%mz-MD7OD)Qyx`Zv`sVQ|D~@QL7iz*Vi#7tf*+l2D)w4GLa}fd%{|J)ULwV3 zq2LLw!*`@Wm$R6&xOX^n+tbQBmSY|uUIZp`<&W_gD`3JT&k;@!l>(E&4KiN&1jn??0}o1v+rQ`rXi>UVfwB<)pl+C3DHC7$u3dE~f4zNu zUU6DI&R?sl{{+QJkCDZh^ujCOZe+}W{~#5Mv_a^>aNh$nr*{}z3RD9TV{XI; zd?9@5;V{i<_^|bj+l(K+2%m%uSR3-{1dw>(;52Q5VQIWf&<-9K#44FG9Iy*=@qSVd zMZJ``p5Qq6x{C9z1L?jPrQ)yRVx8s}Au)@&5cTf8x#{2;4J11ZgX+m3K-Eqqji zxhzgB>dFe#Q{tXO`3roofeal0Mu|TaZ<`2+3btpK9%0#IY^kcr3NP~;CfScHcizK{ zz;e)7M<6OP^1C%}bwqQ60}a?|N`E2l7sbaHZpr}g3A$c)nA!%dX;I7YN(Qg1tteU& zXxFAwt3J{*ZZbtw636mBBN%>t^`P<5-o9;Xw(DydTvc#N-Ttn(5s@2!nbiGqI|7nx zG)egiFmhJ2=4@To#~T)8KmnqE^-_9p*p??Jh*84#Bt!+w5Es&rzex!q?dr>yKklXB zxCTssF=}^WYR8l7dZ~A{Raj%PWrcji1&(?uXp z-hgQyYQ2G`l=zT!`J|D{yw|ku-=B`J`q-tDKG*mi`)=Q^vt0A%2ClmfbNP4N17Dj| z^n24xzp+Q4FcQ>rZW9*)h^BY2qL9&P0<$vBdIDvwpIC6VE`b+r!WR_MMcnJldaR=R_w*VuF_T=ec>-~aH(FTY1ujBXD z-@Z=X(b5R(0RBTAWJDE9L1CufS{7D}sB=Ph#p^)hfWE%8SYBLdH~@}ZdzT$eXe;$= zn@=ix3U*s4J9WC;$1bsQfO*z|(58b#rvp#d_k{D;U({M|pq4$zjC|fkwaA7c8nX(l zX9Xg&5(mU~kHo(A!E~pd7d%8EQii>o%;m3ny?psnGfw6t!Yv;x=|BY{5sV@+T5`gE zgn#?n?*TsC)OOK?RAI1MY1y=JX`rGhK5M%r?I7u=h@VKFd^P1IbfB)g$*xkX$nya9s>#QzLMIsp#g8_m$*bk$d z0eE`;g2~O=IE%_o9Xf`s#J&s(T-QGeeuCo4%1VVqwdM-&bwgH14qw#b6$XcQmqG1& z#rvyM7ukc1;cJGMs5jShYD!z75#uQA|}$%O*BcPG%748m$BwD$f(eC$^eWkoS!n=9rTNyqy9Ax+jFORqmYES`Aaq<=jm~%!jK9Ra$ z2dTLiFAstRnfT;kdrkG20-hA@$l?Y75lJ^M^A`p~=1XQ>BMhxz+fh!?Xqp5J) z5^aOT?f0&Y5MS^vheEyA$A%D&3+m^@`6?Xc@0b+}^8KA9(5SKZF`#5xK4`h-1GIu& zl|G7Bj&R;SMlt@Ql3HszXD6jpa$|j=fV^e_?{ASs6_)|;)(C?xk=j5D1Q8P?83%%Y z$FF}3RuhN=F=S%7<06z4$F;d8Vxy#27v6lQz{wFSSkW<1#u%aiIcyGMOlzyt@(?4L zOhr1uyNyI?6E9HOsNE@>nJkIx2e?TOJ`_l-K~5kQ2<-@f)dX`3nSEq|LP4ZP$k=8N z!TUWW9w$^62UW}m9bd+5A)UX7bN<>+ZM=7+5gtT9-*e2&%nM5UDaM=76fnI3a3XIX z0CK*eRqR#^;+4#Z48a{<;WhOfYQv;3rZR{;6znjbCEql%8A#JSb2b~CThLELvfPef)(zaYO?m2k=@s&QJ zQJ_8xuZ(h%_qPCqs6ehK?~P(p&yL31NR~@P)N{$NcgnbsvMLh4uB*3Dh%fBkfc;&KaXL5wsRoldGd z_8liR9A}RUTfBxBcD%7%I`is?<|A1m>;tNi7pIUDhAEP=mFaSJxFzq9Z-vpw|NOmQ z@KS@sEHF_c`T!#KeO^ipkdtV%P+1OPmPpGk?=$=}nY*F66yE3E1_J&6)~nFgsh-)k W6tpCtN<(Ky5xXvRE%D0TU;YQg;my(j literal 0 HcmV?d00001 diff --git a/docs/evaluations/Docling-DPBench/evaluation_DPBench_reading_order_ARD_norm.txt b/docs/evaluations/Docling-DPBench/evaluation_DPBench_reading_order_ARD_norm.txt new file mode 100644 index 0000000..54592db --- /dev/null +++ b/docs/evaluations/Docling-DPBench/evaluation_DPBench_reading_order_ARD_norm.txt @@ -0,0 +1,26 @@ +DPBench reading_order ARD_norm: mean=0.98 median=1.00 std=0.04 + +| ARD_norm | prob [%] | acc [%] | 1-acc [%] | total | +|----------------|------------|-----------|-------------|---------| +| (0.000, 0.050] | 0 | 0 | 100 | 0 | +| (0.050, 0.100] | 0 | 0 | 100 | 0 | +| (0.100, 0.150] | 0 | 0 | 100 | 0 | +| (0.150, 0.200] | 0 | 0 | 100 | 0 | +| (0.200, 0.250] | 0 | 0 | 100 | 0 | +| (0.250, 0.300] | 0 | 0 | 100 | 0 | +| (0.300, 0.350] | 0 | 0 | 100 | 0 | +| (0.350, 0.400] | 0 | 0 | 100 | 0 | +| (0.400, 0.450] | 0 | 0 | 100 | 0 | +| (0.450, 0.500] | 0 | 0 | 100 | 0 | +| (0.500, 0.550] | 0 | 0 | 100 | 0 | +| (0.550, 0.600] | 0 | 0 | 100 | 0 | +| (0.600, 0.650] | 0 | 0 | 100 | 0 | +| (0.650, 0.700] | 0 | 0 | 100 | 0 | +| (0.700, 0.750] | 0 | 0 | 100 | 0 | +| (0.750, 0.800] | 0.57 | 0 | 100 | 1 | +| (0.800, 0.850] | 1.14 | 0.57 | 99.43 | 2 | +| (0.850, 0.900] | 4 | 1.71 | 98.29 | 7 | +| (0.900, 0.950] | 7.43 | 5.71 | 94.29 | 13 | +| (0.950, 1.000] | 86.86 | 13.14 | 86.86 | 152 | + + diff --git a/docs/evaluations/Docling-DPBench/evaluation_DPBench_reading_order_weighted_ARD.png b/docs/evaluations/Docling-DPBench/evaluation_DPBench_reading_order_weighted_ARD.png new file mode 100644 index 0000000000000000000000000000000000000000..2b8b368a9f51e9b6f1ac9b2f29f80278504d61d1 GIT binary patch literal 20281 zcmdUXbyU^sy6&`F*D?_S3j?GPX>eIIC?TMPN_ThZaw$tCbfR>qbW2HtNeGBC32Bs^ zgot$C=UZp(eb2pjpE2$oAI}}Md7-=wYinOxyg+Kjs~`NCRUdGT)aG7 zyc`FN?d`4YM7X&v{`m(g*R$2Vma?tToMrcpvC@3@m}yYu z(b=AdkKZa+*!Y-#tiHR~ZE5L8>-><#zJ0B3FPlbO^NYCmjP?)Bx|Xn{H_g8&Y@%E6 zmocoMP+sx1WftIZD0#bA{DQyUSbr0L*?pF>nnHQOu<`(fa`XK!6nv25E@eH1a*cQO zY5eq@UHCZP{MHk$jMUacO?^ z=fp()qr*~}1v~Zmx9mHm`u6S?=GtxhB{;%r;$FOX!I8WnH;7hRn|b@&`v=KV{w$i> zW!m&l&u+KmT17P&=rgRmww2d&!I|^wOOE6!>Av#)UXN=GRjjRFY~Qg%Wmu)i*{0e2 zRQcQTa;Y5ChPwVJugDaux;@TyZ!L3^t-ro8{qo{jng9NDm3YP2wt@|ay(|D(bo1+bDLfMoe#ePmtt2d=-tGI8$ z{RCWo>OVPt>5Pt!PLfuBX34_Os?Aq+#2EH|crw_K9E8grcXD#t#jPXOuIE18?v$ic zTu@Y0q?&Df$(Ko38P}0(99@_lAEd?mZ`{Ssx_|$E0n>Wv)?BOFn>o_$dvx8Onb6`4 z2Kg)}M@yHJJm#l-Lv4txvc#ty*5pC}3Co3(@6U41slI$Yo z$7I49f&p#MNcZ>mr*{<3YjgHHHl}Htv4-TBHFY#BM2I->y??MrkXoIjmaIIilA^(@ zl4GjIZ`C38*B^|d%1d+2!VWND&u?EQQG{!WAQePLFNGI36$9flvnh2spW0xwW!e^znxJEv;J z|8{$A9RB$A-@A9O?`LmSUrUaerg}YRfzzlZCQ%;mB>(8pdHIYR@9PAX{Uw)-v0hB- z6CQ=|8~M4=V}pFc!ouiH=}{`BLrv*nA`XKLm|uR=`s4h1Wt&x=omta8G-Nh4(i(Kj ztS1HcreBG7Ta+kt9(&HF66dpF{d%5&qY8W&SAl~E55`G9I@C8or#XdtmHRQ5RaI4~ zCSR^`pZV%-IP>FcnPqMm7O3##gFpY|p~`=LeqI6l%9oYj>3dsYl5pjIuLYah?#&E5 zuiDJak7KoqFQ!uUSFT#6K#Nnr$838lcFuEZ7u1gl9-;F(4r?5)oEYttY!1e2IcF|; zEiXD{+L$#Y85B5-QFgT?I!QfBCb5Fj@hMU8py@J z3^=9~D(3Pt7^4@jQTzP-1FG*178c8uD_8nNMMVjFF3d0w?K^4wknKu#f{eVpeEF4L zCZmtxy!cNUgI=6`2=zN<=Gy~qMk^~TyPk?)rCPCkFz=tG_6xUQ2^iY;elS?OnW4*A zPG0`xYMmZX|^WBc~)Ocrmw zw+WsKs~K+2D*rsypssslUMo_}rEX?+HUt;$Xq9cdo4l$kMkWBan;$I|Z9xstYin<> zd2#*icOkFwz8Zy#7a!(ZfBijr1bgWewOUDXvB$X3w)g(M`}g_E^yRIq5^TPEQ-UX$ zQBD$D*S>mOH2zWa0<}5YBtWH*HGVX{)de+yj9o7Tb7>d!1Y2e`@7%K| zzE)zsI%8%{yFe9%pfRgfB6%ybh;b$LNP%syY-k^+YQn=0Pfo<&UbijI#YHVi z^>elf&3Mi#0$QREg5=JnHkHo{rgc>6qlBke*XNaToWn~(P2cjpBmQO!Yrctgk4P;6Zu%s zN=E?IrTOcfn719eUW++pzWd@2oxfjERVC-UPq6XUs&(OWItBJJcv+>;kNYO4r-Rbd zN15ki4adH{cl}v;Obu09>MFLNDk_u5;$)NGORLhw%NsUq80;)9jjq8Yz2Bo-pP^Tt z?K0kH&pqE~!RI<*EdJxws-KgSYK4v?_Fvzud0$>W^h8~UjM>kg%IHXOx40RdNJ*~} zRNsSm(b@U=e+aNj`A>v0!__{XB86C$hYk+&?|jB+);eB&}sB? zl!Qmm*RNlVNAd7JymP30OctH)1?$O@+JvFvTF1(0N z)ylUuZOhA`xk((sZpPa@DX^zMFLD_dX(@CVibGWps*Up>a{W!2WaT^3qUn2j0IXTv&xVevrmGS~8!8Jr%uj&Blt>)~o25 z)~^0-A79^IM_bVL5{&0%oki79eSU63L!!#D_jJ;yx;qo_0$MgUHh7o@?IxSj7xt7C zjg03D3k%tvi>^|wcem{8>FbM=TwYMYBU3HIODE7^k~|mZ{5-MWRjL9xpBz(SQ3yRL zbNGVKr|<2BYPpu}qLET$ZWN*_EA+q!F@o9civh8}c0wqcYYp(8(4Lj_=w+Ul}WH;^R z_|*0Kc0;V(9$yymE98sgdMYuTCpOifQ^w%q5VbbmL-C13HiWz^8+g z!_6wRcqMi3t<2fhU(e(ie?C(dtiL!qp&1mjm*0>)+T7xF=cr|L?4`OP+aWV1CZ-hB z9acAAOf8q7V80|A0j9BXLD^QFCHCBVO( zx{@;}?y>cI3?7D&-_VCO5pdjZ|sVIf(R9mQ^DDA!3>bJM!BKIK~J ziZ9J{Q+tBF05SaxiZBC7ms3N_eD+3n5w0sZzVcgf75_ny1hRUhI9C^L^kdJ8UG9{s)A6afAt2Eta% ziWcV!ljmYRMU!WrC*p8(C4u<+aEC}F<#9dcly>!MT*y(qIg>AO zpRLB&`|y^@hn=dC<|C~kT1iVQoK3r6LYtXS*yh_GV?TcI05N4mrz%E>^d-he$~NX| zWZdYxcPx@8UNM4?{P9|x459-O-Y@L6a#}AWgT5obv@mORwG&&`mvy{zc}eiSt9@1e zEy=}+fJcvqANEV__jqhRf6}t;iz>Ps3gr9Gfx*GXC}esc978T$#9`{L#mFw0A3;|1LU&(j}NT7e|({$WM1P)hbERjJy zA{Qv%$j!y|QJN3C)3Cr)pl`V`3((l_Y(Rs?~^ zDy5}Mb5+jH&O$BeI>n|yYVuh55ib|Rh3$M;<8jM=i+1-vIqBmLS=^-yb<%vmwocix z{aKOqg2hpl6D(W`u1e=FTsRswg+(2PlJ;(_t-So0=13XC_}gug3^IoA@9&}~H8AL5 zDxJ#T+9sx|xR%t`&4tK_^f-?j@e>Y!0mOEW5m_dfvG}`8q{o!z4hDwcZ|)%;>xJmz zIvF?4$B#FlGq@eDEJFS8d)dmovShN@b8=8k1^+nyMbBS2C*FR29wv@1A%3vf`=+kP zq}t(1^h43+Y@qRAEV>YM^hN@r33a$Y4G}QAd|&Sg=haiqPV$cWbn%;1)pQ+ki_Q{l z^bKkZfJ4&x2fMMqlDc_n(J#PlRdX%1WdxVhy#Q<+hQC~jSBi2J15n(>$e09}5?4^P z>%?VgD(x$ej4Mciazs2fe5I*`U7wHMHUm>1-YZ4U8E8pI^n6rO6~az6+}X5;$y>#9 zVN7bxnl*{o)PA$zH>naHb8U`$6E*X!pS88M)uUQST?G@!ai6sXWaQFZesV%R>Eg39 z*mV!t)skX>NfM1}BSZNNWD9~Kjwwa9E-7hIyMNI#bz?Yzor$$88hMO)$!j>HJbbIm z(VS!2Yn%b178k~A{ltZ3TIVqjrY+ftQ=^@uLMbun|nf%P5-2PdFWih6K?ny8^jmm4_`HKvNV{ER_~?#~`^n;3Y8 zqSl@v0q9g-ny!`q9L0F(krd0Ln4P-a+3t-}zKj@28mBSe+X$WRFx}j0XWvD1c=V&4 z(|0>%{$6UeZcCPZr9ri?i=|!tr$;+GGgzqgfYKO5E(NzAto_q-`F8yTz?FA4i3+>V zSQm_*e#inUcm)%HUJy7e{RgW>OSTMlK_+3(XgemYFD;UQD3->fmSbiLdTOI2<*aH0 zFNu9`J7R?DADO0I7*`V}c!HJn9417y*mW}7tcfe0q5(1pASg%0SaZ`aFi2O{^gFc; zRy;avYiFl&Z~Kw(7PQNVK{3H~Tj|SZs_cOkWcxzfLzEBiU{G`^F6$F8MMF)IUGJ$ziCGQ=wHny0kWC zqPTz5aLH|!sqI0`w6kn$94_60_7z$-3zQ`QwtL6hTQs%1yZuLI)+Vaxk z9JnOwk>$5+?LvY-8YvntuKgxpQg@D8pKYRowW+8iF&oq2wd}EFFCS}7m7=utC%_z4 zND!@4avL=g@5I-?oQ=?V5Mv=>KX6gNqUGfK5}E5>09}l{H~xt9T5>4{2ZbOM1Z_gc zrT59s(Gcg{(c9I(?A*yJbN~K**;AbC?6RO%e&Wo({r20HLPv{C?wS~v+qZ9z7Wlcj zL_p&R%^Qxj3m_Jk?@G>LMXI@7|pgFl*$vT3nX9b}%8~c)4)6 zjE9OFx3tqVW3E-F4%jOV04N76Q6YryJSbKjHh(?HfxW*|I<L{R-fF(d8OV)BNd?gzQ2DHe#!pZyAAQ1PRl-FKO(bn{d(&^0<)pk1w+%@_3N)+ zC)O5Bntq&+rF$K|vvC(SJv|+T&$C?Q!LAc==sBXz`{~7Q({Y7?P?wIA#{t{Slr-Ka zXj#8*zx&gQiVA%jymJs+M7`L!;DS(RLZPB#JcER+M_u7jIYe=6W>S+z~<>6tqlO;8alT(l*???BrqRVN68E^?13`F_rG+`}?MGkW~Z%@gd2 zT{E-C!e2}^MdK0+3rkCZLozL0MyuFW4ZJjDY159}*2{Ii7Is6IQZc$B?W~eHyt}z| zIJmjFAv`E4dgq}$xuqL)W?_eM>J(`}K9*OnP!!3DL*G*z0h*zGj+SbG=DazT|AfX8 z(`ovJFRd62NRdP9Z(FNhMoV&L~2-{sC0$ga|JlOePq#8(3;M%L3ty3Odx7O|4zMc9l zXtsF$`t=t<`6r=n#m-1BhUMlL7P@7Mi$2)RXArUzZJekD@))KG@NppztKs%Sb#U%1nww66&54AH3hDB3^!P-XfTN+%Noc|! z2>CWQH_x2qzf$M*+{&X9v!t?O#fngIx0ke++9mG!c-d}y@B8&-)=o}AHHLPsQA`r< zLN%p|Pa<1i#WX?1Oc?L4Jyxi9hWdC(N4LoN5_C@$@G7eda(TN?(PoYhy@msGHF1VoeIX8&#CVV?mP)XJ(o3^s2a{8g#hH@9kGH&-M3q9E0X%$Smqu zZ4*^`EXrLOROM1_q_|qVA~TdS-7Dx@p&xBzYp_hyFcKy`6_1?65*J}H_$@oLw%1() zsf3pi02!EO7Ht3EZkSU2QmxlgG~jh2%Af0OZ_qAS7*N6!P^XAs8LbPoeFA*X7;Fjd z!zauEk3n^9 zmBp=m({GNLg1qG(h>8RI`^1cR|j+FR3O*}H>1A5!AyMY-Ht(+ zS_F5I8Rahe@jI<@0zxkBu)o-IsD!4|-`m+WUOXj$0u<|5p|1kO^@(PqD7Hg~4mG$H zfcIa5+IS8rv26w!`QXr4`XxYd46$FR>0Ca0g&ulPlL;m6w#0muA_43{>V zHQ{bJH$9q;U7>+mZeJuBrx=lhsTqO}?l5?@#N7pL>lNL}yBwM+m{J045Moi0E{kg` zDT$5!0ad2&_W{QQyGAx=rv7${&$NO`9`g$pc~} z60u)MpMhJG2moP*evsQ9r~ILlJ23=enje$!x0!99PA>1Y=QV3l^eZfLMYSI~ zXTg`jwG3boJ-NS|GYP)dg2)ZAOVePtu-vCFP4EG|Mu za;b)FXEF7?9d;OY&e5>xM%|SmKroqC9YMAv)8zb@{o3JxHEB^SnYmvy;hy~VHEfv z&F8f=8_#>RS9hh%_T9Vt?1z4jxq6CKoY*C(D*A&>=~lR^5#+UyI1%63($;zNFE3sv zis!gKeq&)mwOr&i_^>m63d(&bC|d~RumJ3najgGf?Djf$fEYgs03Y7;lDQj*^trETz-p3R$sgZ7_iilVJj8fozwa|B=x;5{ETo2;^9)6aQGpA4YG8QKU;5 z(UVZ>LX4t38exixHC&a5?ahirr(ms#s_X6>{WGRTrQk_DR+s|7l`oVSY&2f0;>kc- zsuML_+^un9u%T+_@r$o|uv|_We>%`Sbj^nsC_#XX112c3k$L}qE27G%U~dwi56fR3 zh>Y4Ec=*(*M*G3~N<799x6Zi9d5+>-Md<$tCg`Mb#jdyjHLDg>vaywn%FaelM@(!UCuL6pA(T%4@qdP*gAY zv!tTJpEz~uG8&i~kb!{v%#Y2b-q$Y3iZstS*4f?=p-?KWtUev1sjB+eaiqneRf1Uk zll=`ygDl?qkI+ZkdPb5B`QEwXW9BWeD8Njul9CePZ{E*VP+qLm^WoAdO6cmk79Yi( zy?teuxRB?9P_Fq3%7HMlerVhtnj2R9U}0VSF^!jbB52jYk%mA`%z1fvAAnpYukzGC zDbCxdTjvQS#Vt>t((9$=rnq$N!)@REi%5!o6(tYe2q2y8>({R{o6WDhG!LQKht&jr zBbue31WbLUQy$;m-Q&{Pfxt>rRp1I9E-po4_CY>U31`{0t9N1s=pA5@Xjbu_Knga4 z({*y(`On>vdf|A&X|;+5ZLZYkE5E>00oM;cJW-gJ6$QlW7Vg>oRiD$tEW z>qdbj_ESBYojwiWganvC321B(y6$6|9a}TLb$9RDv7>te&r@NU*~{{3VrDq2wg;`B z_*^;L7*9%LxdfG#mJ;GhNW`T}mjH}Rlk^mVd47YI{Bf9yC=TC~8V0@0fkiu#@ufm?$cgij}3SS7MZZ z-LWGY6-RS!)XA@7RUS6;2tY6PX9M&6-Me=O(NIL(XI>D^X6O8uktx-;2Pu@(j*p^i zS1j}dD5(N10)F-SdAxo5wik5w=1s~TRm|)|2l9GfzGyHDS-%<+J9Ov_m=sBL}BVe|tNF>(Kl2$S&ufYQ5~gP_K6%4Hr)0?<|c6Bg@*uvx6k0UWr$!9XL1js!mgTZQXGx z-8GwW=bnGDcK_|;RR2{pteKAjIYI|^oiexs5o9>8_MI#STfqiZ^heJVo)mVQN)d7T zo&sd^0T>KQqq4bq%BCj7$qtxdS2Mu%2QfE>aF{q{(wGwR{P}Yxf1>D{xB^u{xaPw3 zW#PvaH>T^RLwweEq+>&AemtDKDQ_P{Om&oCtJZSM+!H)JJnv25-G08m>jdF+33vi( z4CXQ!wa^t22NeA9uL7$mls0a%Ub8h|sli#8Kx$-=Sm*#|Ote#|R+5sEQZoTDM-@UV zYHAd&zBHF{sK#!3|L&cTIBuR9#3XQwW(MWUbR^*5^_M%Rw5->{*wev(v$&CNf5`4Is-@AcRyB%nm#2cl8{UBv?y zbx!7iQqUeAJj+z8cjLzTTNty}%13S#E4560nO$(D*KK8b$O^Z0u%|LW(SxYE0J(9l z9Z&7Mi{47S zz)V};Gj~}p1Za>Us6++A-^s$l<>loCRoHA|;4@*8ki;F}_Z2VA4MF404n1YTF2k-{ zQUoL(2N}@}9GI{o$dwJx&aO+Npgsueylh*Q=P-1cv|vD)09NRR=kR}MKnk3Qrj4}a zs}OY!uoxwgyCLDy*?p(XAGTXM`D9#sy^<)7Ai|fSKoFaw2H~va%Nx)u49Gl!Kz?8d z7@UamoTK~c&CJgaJ>V{>sK_KUb;mq^9deEoC;xxnIOBh{|Nifyng6nN>RmZ==69kA zp+U+>NJwnly7e;rXt?4w@x1rona@@r|_kW_IXFis?yvJG)>SP)ke!LLy zj60!#Dy`sch>vlezi=VR`15l=M_pZAEN4U0R7}})80LODY{~d3u$CmK z`NRxHDC;@!W4mMc|FSX3uZX9?juGr|J_}ty6%I1<2+1Nr?Cl*LO^0!*hFzrs^aC;% zI;cUcdx(zOmrkM8#Z`V@!2?;6c+lJKxs8#nm+lR(UAT_1}jb z-R^gzV^^D$1P3$R`Qs0L#W6f`2y6M;S1~`F5J@5Sm7?3S@^)}dJm+7;pEy;+dT7sE zyz$}Y^M0Fe%6zqg+75X;b>q(C*5xihoBwRA$ls!|V&r{&8^Gw9YGKFzy&eB{0fwE#mEjo%c!JsY&acMu6^jb=}!il>FRRpfEhJs6A zX17b7iwj_56PoEnyHlqO6(c-&vJ-Hi;l=f{%_EMST6vI)e3jkZi-<;qfQx8E=Q=!} zia{0S3qG?~PdUP~(1Z#v$$`Wu_G<6OaK5~DZyp_$(zX0=-UPdUQk)UGQ0Lx`p%b)m zp&)FQjp2;}7au~lEKNrY-M=?qjA99o7UilDReL7`LjqipLAW?h4tw#7fxSN|KV)aQr)MJ0Kc2#IizU)IvIBpA_T)BoZfjWm z5SG#)1tjn0yfOgpua45%)8}~jK_GN`L?_sFit>`>&?;Z74qEpLQL2l?CWUVM?CH~| zJLgxgUOfgi=PkbS6Sh1pP9JRtX6 zDfKL#o1oh{omhkQ?3=gni!~Hftnx=G#qjJQ%gECh{>B+9;f6jWJUG#K)k-*xbqRZ1 zEzkNo#4CBQxKEJFvTwZKxHwk6A9?3ERufzoZ-1x5V_JL7x^V( zCefM3B`EQvq)m%?o;&x)^XTY?7?~YXkiKS#NQK+VQU4_%$?pe4qkK2w>RbgdEVTt( z0t78dx3q+n4ngKK&`1j6Pi^Bfq%OpO)QaZp;nMoT%lS7np^1x$^^A^=4g^q10WB{V zjVyonuFb9(mrd*YNtyX5IywMSOu}H3C=!#1Vtt^|#v{wOZQlHtHA2YdP2wVmlb;Fr6%o*A z6QnjmT=tV-+k4797{$KJnMvGrB60|`VJ#3ex^0k^mmiBN(H4clMBd{OwkF>I_Z&dv!i z?=&-v@uMnC;Q~sEm1r@lIad-vUZGknf4y#eKk~jvu2RLs#mNzil?O0?N9|_SUSSaI zT;G+InT~c8BS6Qs=N2xNcRFxk_m5OaBhf_ZZ572WZ~8dtu0BmKyK}JIe}C;DA3kO) z78JE=1|45mT(Imo40nFR>QB-B5`+JIk002n$e8d-GmuUkQ3y%{y8AETyt{g*7rl$W40_`vQDRmI~4B7?8hO zx=#D&TEndTf|K~9oV5$Z9J)9H<&(r>K$wYY*~V0Wa1Ti}=pZ?z9)xkZ#>OSN?gu0V z8iJxsUzMmo`)kvYfVSPmgBRVDCq^lI_ig~QQ`w>vArgxWaX|r@!z#3H_r4IL`YahI6OTmtUo2EWH>rHwx2uHarlB0zzFq{w3OGzE2eemxpUNa<{1evkiXSA{g!1|!kdw4w zGX7N3=PXxXb9Svn*&wJ5yKU#*y@}XQqAgKw--S~(GTt=&wQKyUYF9ucjRoyFwu?>q zU1G6(2%m^;&!5Rv*q1MuP&@(J?OMizdQf3ATSwBk0NGyIVo41(B*&wnw~e(KX0f5N zBe8QC`0gJQyYR~ol*I;QQ!G%VNK%KCC5T$2RAYAp{Led(|Czg+eSwFn(od-C*pmRjs7>p+O)-EkYyhwL+e&bUkZj>MfouCKO-d+HK-4 zvTpT!_Usuo20bjM#R%BG0B1|OZ9x7vr|!b?*`@Vz9S{BTiQJb4FM;mZfn7TY&cgtabKLb~r+%ECH-!IRIfaAqg8t(z*??3E4_is8~@ z;jdYVnnzN#{*d+!ZDF`j_SW*P;DK2~4mIEy7s&WCEO!$WNH%c_#J&g;i+GKATj{`>;!oxwDTrJG<&~)PlJsODbAfh4b-JWKzk%y#9cf=#bs)C zy~1miS7ogD0oVf(JsO7%qmkqy1axjBx0CdEnY!hL+GQep0$-{DdO_bKhc^vu$n%D9 zL*T$GbT}5ka3=OQG}D3v48hI$+Xf$35XiFS|4x_sH&+Dy>l4%1M~bj-$nhh<7;<1K zKFae7X47GiM${A-oli@ zJzUaM1OOsh_8=VwKO_n98%DylvsLq8#1UHCaKDILOoYr+8yHf%~Z+32u z9UPd7FpLCFAq@ew&Eb$1$?EPpevz>9di+QY!>NY4@*o+IsX`3bAjeIBMTch%p-QMB z1JgsNCt{16p~T&XHYzRsC&|O*cDSC{l!WeT9l^AZ;>(*G!Loh(b1;_o=g9_YljN}5 zf13QC)0F?l+Wr6Smcy}myljO#`m~70oc&$@LZIr+`*(y8J^u98JwUeWs-84GsS3hD z{I+AnKI^f@(S=6*M766@{Ymm^>AQX&v!5<&f8milTYj!D%bkMgUmVeY2L_B`N1i9g zpkNYZ7H|L-X{D_^n7Jfa*6uGXM1JYjt=wg2L`h7)^59P18t`LKw79p;sRQD%O3IYNQ4mLcF-x8bqCEjP)) zK)N8BQ~VS<=nzB>8JozzTTx}p*)x1oe_X*Ryv>g?)Gy5V>r8XEY#rS3Cpda2zi-*o zu$kg42zDGtWbMdfM~@Byk&_k=PTa(RBkXpOC6dH_8!{ ziiZc0Q!j+pk_gOi^7p>ObY&6lcvD`FPMK-5tEGeXx#G+gPHCm$fP^V(SS8oy%8h8D zh%U)HR_xvT;GZwFwzXN{%g2x^!xZ-|BZ-$)Q)aY^l#u zG3$TNL^Lz^Yr@pTJ)w7{AvlDyo*cBH>Ws}=mdK)%K@K5F0Yd~s$hQrY)z;RA3)vic zzl7)|-}mp|4Y4B(q2TtV=63KQ7XH`qvIi$>J?jcnMI8q9(eX_jhre8h$EOD|_&Ugk z5im#Jw+~rC6PEp~NG$f;3l^BjD?1an6a zl*Qh3)0Qm)$XnLbi`e}>VL5aa1h-~pZZ4GQ)LF9CzkW9q)y0pnp2`h?XqUmkUbKO-~z!P!?_j`(e$ECrn8waL!U&nHpi79h_1gB8M&&*3B~pYCC~L9^@PuS7I~!%+=rwjzzfxnqP5x8z#1ErtUsA9KHv+ zirU$K&(BXW!^_^@KCU5TM_bLyl7@X&MbEmry7YBLW+a`mA_Ohl5^yHu9C9CA2|Px5 zVI%JFONa?ZAUoL-h=!bAoS#O@{r3xgO!WJiSrLWJAAv3E;Lzg$v-Vy+`;is_Jgd_3 zY^|5prA&jEsazfuBpf;p%*}P$trjSrvF;b?_aGreRwGRxTTWA3(a#2wqQgjw8gh9V z8+UUAH}v)x6YUq!2)%;DR>Y^@Y%*Jb^r+k0XM@eMD=F2=rwWS}gpG*Zc(Zv?vVZ)fZ8h_=fP{ZJkt?ne`7ZwH(*tjG*~u;`uxHb(klWZ^7F>Oz@dnn? z>&!X!ohL|K2T{JG930g+?%0ZzRZc>W=-YZeyx465)@+fDt2QJTD@0$WfIUa!nAj<34nGDSXT{j zX$`=q6;d75;5b>PyE5;fErjko2wuXdfn9$Su6k0`ahQkv zp;lwR*Py6X<}(KkLmo?I&g#$VOYM>g}a7hcfdUdalP zfW-gv7eSr&v%@j#VtgwKJH9jp&xwuEfUqPiMDP;_D0`^%I3UJn#}^J95TsurW0ZP$n74A!+I< zR4GWfmH4g(pd4s^55Il;rj`4X^6&x>w6HAs!VY}10`c>}iz?#)kCH&UiHOlx83UpqQpAdJO zB1ilx`(P?$p_s>FDT}n|cr7|C;v@`aJ}73N8zosSIS%<(k-4HDZ#Lm^$+?jp<5u`M zA!fnx zU3BtrmytXmEqOp((9zr5^P-ZJl-vLKgrtYpd8vyY>*H{hEuPxu7cq?OEczcV^GJph zhMhlmT>a=N-{jfPe!j*gZ;s4++>&!E&)vqDuJOC5UQKLYQddaCzA1&*F(Ook%v1#v z&DNBmn3#yD*CtjsN5s;ojPI$!7QH2$j&ZvCPRAGA;he18`*lg{)`??W_ZJ4mKP>%H z_H%T1|EMCLQ2gcL#TLc=LSYy?{8PqKlqN<}hkt(8W7_bav2)y5IF(8rU?vE{`*BHU zv0cK#!d^1W6!a$scnZsP}OY%BWu4muLND=c68|9|=a3xly~nN{E2!AzHY_%iGI z`ntihQ!zQLunt8OB0fDmof;JtW-q`U zr7yv;4^^%_zOwXvd7)jO-qRT_e60EDUQ)u^YT3&f~buw9SWOVdG zi^`JVp+g;QCL-X|Mp2dxqft@J3$y)P{D&`vrKAYlx_vwUY%QIZStPYFJ6Sc5r}hTh zjoe(RJ^Bx4TFalLrytGgd~B9$v%J1C$8W)<<@dYyd|Qt7VaG!}H#&#M-ra@;Yy9{z zzBrmkmJx@;ajW}$47PzF`s_L@E;zT>3#V-Q;A*i)oW$)sOD(Zlg!v7>)Z{o0u9A( z{dtaU!cJE#EO@S7y*gu&u!UpK5Ew-(*QR2A^;b*(V$a8+=jm(f z%%7hv`C?mpou#MZioEK^>P0QHhVe3QOk7Gz%hgrf$k>=SFmSx-uJ-ZcHLHJq<$HWl zt$X{{c44xu)c42Z2QE75k`CXD1eJJ77fna2z66X;^1OTe^y%9+6UUCSi(frQqvdUf zszVrB?_$VG)reWg8zKSsfi6@tx`&Z^y=Q?N4zNXFk#qq=Qur%=Q}p z@arkPAo6pjwLrmZO#6MZB8UEwwL@>-ypi@_aKx%7J|xB6yopIkNqJAU;x10SKl1cBC$xfq|jA z(5XGoseMo8sr}owZ$IvkI34tJ?6^}uth2OhkMyLs672TjXBiLG{QUevmHhnuby7n3 z-e=$2_|p>HnxQY?>*qJT@U6YnYn*Y9UNUuPhzvFoR8-udsHlhuMk{YT9TX6}aN)va z>!Z;6Kk(^Ck00aJ7+x=YIx#s=CiD9BYvPU_;TaG8o1XjMKlZJtn;^}4_3wNwTU>Io zIPLgfADPSjIaXy`MKq%*B_;BvFHYkywDHgo`}@a$~# z(NH=eF|n+7N=<=o19wl*S5LscC(jL5XzS?QDt^}KyleOFv)~IDB`l1FPuZ7vzU85Y zx%rNu)gM~l=JC z={Vp&olT92c?*9f`|Rk_Q?Cg4-qDcZ{M8q-N2-$3mw_W@Za3oTzy5Txg(*ii)ANwOgm3mA-}s z?V$g(M8Mx~84BK$#J(bz8#e@kxUc-4sR>^HB``Bk#-YYA01t%1rB3>r`Qh61mi4B) zV&K*c`^Ch5tjtvrq-`Q=Lg{NE?0yMakr=!(rxLKxrq%fFUCn)2m)5G_;H;_BCMLTd z1l!pJ{LmP<{3Q@sW@o_s(bblXRkj#DMce$AWW|fpE?we$ie5BnTCr~zW=A{VnCZCV1dSE8Oj-*0GTC9d)@ zSy2R|j*Qrvw_Kz0eYMiOo=FJHT-p~M!%1IIpzh$1PE?~76dc4=3&j=`6mX_AQ`tr8 zV38z>RW??CJqCArLzdR=hp=`nB_*YGdQpu*$o=&X%^$z}J(z^TxQ_(A_FY@-zGj-W z9rCNNl$1V3Y>43p>$ZV`e!xEMUG+;*^3j1X6S1+hv3@4B8^ZV=Jb}RL8XFVCIoc3g z(o=k4R1-(CD)Z9li@$q!H%1*8uo4O*8b3YB4qN}F@0YG@%WD?d@j2P%nwr-!!LAqP zcKzY9UXqylrAx(yvpxBCcuG%Y;H$#yygUQg>OU5{&%o{ZZmcb7e5nk)MpS!Y{!(+- zo;?w-<|-`%mWOY^bKJUhtL|6J5vefZ$mh#nwyXFr5M^a$cvaW5?OQm4l;xH1>#O7*SYW-V7r~a!M5RWVm$ajC|QxFR3p!I}>QayyKucgHRnr zwq0t+j!ERyfYa^6s4rf;pj-4E3E!(8yHiCyC@ARVS7UQ?QC0!ffuTM|A@vIv^7}W| z-Cs&jBz(H9FAu}xC1^4Nfh*nH>Z7m@@Jp5B}YpcP+Os}!C3GV@08ZbrF)vNet<33}JaS%N?oLyWX z<*`1K{S8n-%fVr{d+9J8N?~Xhl>g2>Umvj@5YF=7>D(lK@&WtCn{+L?Znzi0ZcWY2 z7e?x~oV<6ie&nM6Y*Du-{O#LCZfP>5{%gSOg{dp!cg<2eodKQq+UcyH zjDUaufH(6+1xS9O;ZCwVEHOJ(oJ|x}RaH$N#OCJacBa%c#)%+X&wQy8c=+()WM@{w zHYGWa!6BlPRkFy}iAk>|iM%A|36{v6lG#0YVe_cVxHCmF(V?HaZp_d&5EN z%bz>PDwQ65oO)NcvNRWa^Cl4%zE>D?hvmO_?;b?J`qoFsa2TLlc#wUn%R>a%LCQbs zIJi{>2f!&CAt&7wI(C0H$SwSD_tO_In6deHds9va=+U$7tjn=}<+?xk$4LFY zefz4fv+&M0iRBhP&-wo1105;bG;(>__en;^iGJ{tPwnkVYt2`w1D25ViCp{=DszOP z#j=YM_*9+rt6(w{*dqQzV3YU>({{&0%adubt!Y|pSxru}&buHi#($TU>8}htyt=m5 z_RM%MW;RC%n5$TNl6Rh!k|gJL%Od&xTvHswhTige)WMRs!Cs+b%Qkh%p97367*-8| zB6biUpP$tOz4L}KRaL=}Q%u~s8rIhQm*xf~++KNq+X|~6p&IxXlU_f=E9c7Hm17-w z>;~&_yrp{$1xG+BFjn<1Lns|f0fUv&1J=>E(Y26gVqtTl9IoIwRP8f8GB+0s_EP>c zcDtv4s3?cU?bW4O=ik4e?j+gU?^*|!p-b^gkhFf%GFUeX)DEOxD>W>CP8=JwY za7|-l<2Han%sZ7!i!w;+dQS@qwwgSBCXJB@U%Qh=f`gAj{tmbfMko{@{=b|w6h&+YHDETVE}f7u2q0t?I0Q}T$)ML zU>a)IKN6i1^mhx~8a%0hoclnJtDs{PST-$0WX~Qt2L}gjeSNL7XNC6f-+x~`&9cOu z8WHj4gj7XXax&iw%M#YP%9WgYRVYgMF3t3Wz&!{bKFkYf?ewclZfT*U=UG|9qoZk) zKZ8HNy4(OEKKhw)hK@uI0-de-c3K00OP@alK}@PkI1?0JTB@-8=g-k>Q@iE);nVrH z*XoNP#?^jGKXGAx=qQ*;0P?g{YWp{$^&H#%=>zmVPw_*~O|s^D&!?b>op>Zz_29vS zJZq~{SN@DA^&iVYkTW=%SE zm6-$VJUsg`>Yra7=W-ab@qI(_9I zG)jTXLJ#Q;(%O$&nuot1tQcOJEtv$*c}c)G%1OPR$D8I@g$J(uR;+I^3az0)+Ob0$ z-giDVEz8afJa)1zjULrWQT&IyMr7~mTjuV5n*Ct z()i(n+rmnGRh9K?rbMc?o*F{{;2Z`y4{jMcthM4XzDTb;LeoO&??PG{l z`JH%dy1=%c5fwR+Qk5hforq)8BMfL`s4NYH0?_7w6anS!&E({$`FT%N9_9wGWu;|} z{^ZeH)a|}||9(VA$ElC%dL3^!3yqJ-M-werxS3m71!6{griWNk%L_Z&IxmXCk8xMR zUeP=Q`)vLC?i2CaEdad3;Q>aVB96`>oz=hm*@r7e;7ANBztSOY?vdxk?w=#Z#>V!X zk&o?=UHSfM&fwIku;v6QF>Bc@RAx9Am3PYc-DD@EvYdOL7_vj=oVISGdWRf(aEy{8 zBt$wZMBI9HBxtSm`={O9Cr`K;HoyMaB&K_*O}1Y?PYJ-Yh2p^IdfgHeFHju; z`wuvyuq@+FVrRn@%&ppqOznGkB0=)_bQpb7#ySJ*_3yv?aoC{rpn{%y2d6aFM-dAl z7O6~k=iaCxlOSOaJ5_#>+C{^yK=iSY=)m?zM(_=?XsYj`M|_f1#fHZ~Pd>>o?t-1C z^w760b@ez**~P`s-27W2<5u=}RA3kfY#{G+<(^@~xaGZek)^rMFv#gKjKuz6gv zyZtWqI0@h>8)pF;6Erk57`AMw8OpJa=B=oRX2Ynl^w{znZmgH(quJ(!P2E8 z(-7fP@LJG$eKEF~s#i(3?G`>+3Oma6UV=Y_Hv_>z*^sqq+YFxgQ-b+<^r zzX<^G@MbyO@?C6{%1Vlp5&1|SZ>j9VE8i6dyqYBt2ENY9zu!$j(QDVkpubM9y2krG zs_Fq?6^^!_)h3sP4;Xah-XJ%X>grG1JlM`c!c-fNpq@+mTppK=rk%56R zBapSJ<1sciSQKs^ioF@S;x^!1HaR6NEz62e*NJ!^AD@EG-MeGs@UQ`E<>)!%-Q!WH{2`#oTH(qmmkD4XfF}|)-E&U zL493Jm>9?L%JXcPB-W6tk8J$W_nz)z0Zb2+!AlLX<`4osx{_K(yU?LS+o0HS$rC>v z_$A;M%ipCrso9h}ckCCx3Who64fNNt9f|soBtLp)<1g2#{oCPRl#CIwPd6HF5C~Qe z2v7zRBG0YASo4)OWLBprKE+QX16;rXKyW0%Psqt}!liZ7e10x0-0^YJH~TYyvumDy ze^2JX#fg(NbacYv;>=Q&%a7{?6b>9X;Jesqb^xLX4Gj&suh0pjz@^VZtq;H(WnyD} z6$iuING>Vcfzk``8bmxMNc3&?Yh#sk-hr^6#=71wZG86o`rs|g#P`oGgw$uW2J|Em znjB!3#>cxT1r@9(gSnMyf$i@T#M7Rzt4_s8k)ZX9UJ!|$) zVJW_UX^u-+#(2!N*LI+R9g2I-2bRZ=htL1~xiux==kc`MV1P8j3JRz1bc%U=F85Ib zoX*50$siP#c*yBiDcmR&%`=~0L8ZG_|6$>JD5n}XGhM#-HzOPgBK0BPz&a|st!YJz z6@w5!H>h%DUAj)Y=AMyy2>>J1uBfz-=6`6+ed^Le3X047xp9(Oa zy1E)Ltv|z)vs||H{j()p->Dy^ee|>{{~-r_zuXZut^;yUyx8q?xk+ZHGekmBfZ6~> z5Z}890FD70saP6xF^J8rTDS1^^*ufF#p2VFn}b7WH*kiZ*O%li|BX7w{QZ@jot<0L zPjD?vbvBpgKa(U^D{#|Ls{EcQ?7all^C>g4(c*r0m!FFO36}4F+#-D`l$V)~PsP8m zW<5VYzjU^^ACW%;!5eGKfO$RZSIw;(Af*qFk6%NarboqMeCPBh*3DWu7Lr^<1S3Ku z^S^%G-kB1-t_UPn8+pUb#*fw_n-j{=MAbn5+vBQ;Wf+%zyZy*gl6(1<=l&ZRH^5&3 z>PAZTy5!cZudT$z$DjM*7qwG`hb|Ncuo*E(%F4<_kNav{F#pA=YtRwUoSy%BWMc=% zo|7aswKtt}8b+sTdGXM%QeXRi<{KCJBCfT8zSN60q_VQ7#DNK@+g1pH@%%G0=BbzG zvTmOwP&5gAs?6RqJwJ6;mJ1!_ALw=eJZMQuYa9BS68siO^*o^X0hDqhU<0*JGz6DJ zD}#-V+V&0(LJ|@zurCkp(Y?RRC%jmFEsq9+)*+0+0h-S2AM|LLpEm!j@es59F$^cx`3bdtY(d~kRafVSz~ zD%jrvfU!@Pdy4|Qan_jsosugK!u_tWyyW%z7tai20p4Q*iLllAIf!0Ii?egEh7R3sOy780Ww)pz1?WVyW;_2w@d{kIiI8OiIAgzMm zCk4jkvE!o`uK|=FE&KYg_Q?7VwrA59fm%R@#)cp?g#mF5OB)tS&t?k|%lGHIy+44v zkmb3G96^52mw+f%*bG0Q+FrmyH-ys@8>bpdt~-ktq19{Jz}s^740*)-Gjz zUs<_dMHm|yc`i&EZaT$l-vj49-0yU~_rEO+vkx&m0alY+<{a}={gfD3d^@1Se1X_a zauvG!i^rfj4?y+6t+GWaQPC}f6T81qo@(I;Z@c7=92sib9tO03>0^9?)EP|@iPX$C z@a*=Fv-FTMg~1mPvx_>9(7&s;i&de+=70RCfd)w!M8vZ|oTI-&&3WS*6PKKYDT#y? zyY^nyh~#_(y+vTzP@@TY1+FF!I-GsdhQC8$S^wntZPc4MWoinnW68UNnz1{hQEJ>+ zZa89KVsZ@%G#ZY*dUb#!o&?$iE}B67W><5S7O_giZQH_lJzlf79N7p6{kh;D_%hqN z%PYNwA4C0(rY46Dc24nTZ}nc}kX#X)~7l!&)}b*VmV3&)c{U~I3W z&<3pw=!Mgu#uCbE(eZniWK*Y&!(BX{q;iUhiXMeNAq3p)6ubF@EE!pqH+blT#wJsSrON znbdJ%ys5KBz{KR!SPQ4b78@JVCArk@(Ev~Bk2Ymj^YWyjI_+6gPS&`%j42;Vj!%;xU(}RCt{)gP`{=&(I%yQ^an zDna=$sscOx63>hEv*&hVBcmeMs_0JbA9WnKwUy?I|A=^$3*qwjfnK*)A=9zd6El1448{HsNqke!R#E#nS zd+zh%tWus$>zbHeC%0E7e3;uU?1e+Y$#hD8eIT{@w~_?qlSKQ(mU}KP?9Pm#P_TUX zXsrd33nA7IsG(kBTe*>YbV;h!9y`>cZ)$98H;`(2BJ1ihl%LK;(%j7Zb5*hszxTM9auXYusr`ZBgpYFtwFlc zBVhQTAg>2p8O19m7V<_dHIt$9?Ad+d;&td^#>S(?X(8ds#|;c_HtYScV?)m^cORvx zdq)K!lRyNj>SM98Vjj3GAV^c!l?~0TJwJc`M9pbvoN0znaTqc|x3XY-P6=wKoB!0| zAPLx%3uHp*uq<}Pwbrc#SNqV1WOhzvaWwf~;mrtD^u&O)l_(jwS~BR*Bxd@e8k*S- zIV$XNuY2^*hC&TJ1Z}Uz_wT*EO{kgIg9HCK2G4JOVP}KN1`-HA}n=E^(_p zv;LC{d+R;{$;7lPo~mt*o`Ms=y=?Cq(;-EMC&wq!i;Lasfx`yoR35_B2m4leTE7~sTTo2f%7?uyNZl`13!w$W#LRLEi=L~iU#bD?y4jpNw}+9Dkw_#` zEZ=`Fj?S0EE=|#>YZ2%Oc`gatFaWJuAGc_$>qSEm+^2kT`b*0C!d}OszPFb@O-GAd z35lmC4LOSq6}wG#=L!Ihj0VlaDgYuUjp%Ud9VFp#8}!Kt#!((W&N^esfqjsOuq**J z!ST?eJx~pQj6&Bfgqlwn^8Eh0jPKsP)17ntzP`9f`@&qQX#ij9!ewAA;zRK0$-|kw zci76Vh$KGF?Lo`K^cX-cJa7cv3o%_WnB4SdUIrG~9OhehuK{zM7QC^JxOSki++N7C z#}z=~n5pa1wjf1WB#_>p+CMar6n)0yGb3gsHq?D4*v;A;tnvokF%&V|gVz6?$GY#W z(>h+-Q9Auxo=8LS78pV$5QN3)W)Wed#D4nfE8olLfpT5`m9jB2`J5Q{9tCv3pU`~+ z<}2sV12;DujkqaINDbxt=T~;nj)j)qGVm=gq++SsXWa#`^B?YCgM=~A)+s1Vo2az` zc#m`c-RFNEfUzK#w0@(UNT8%?;V8G&IO2>iu3$(Au)MNz7<4qdWQ5e4p9UXz=|g0| z9E(V{kAJwd2T~^7hoDLoI&$R8--F@QZDSoc&*pduZh3b;P#i!*KM&G1-O7`vPlbW? z9UL6gK6w)E_xhTcj=J7g;HVY+e(r+8*0bNO#Cn4z+X?p3QcH zr4X%qMj-q7DcaiE#ofQ(H#rT?rLLpHIMI?43+x*ze^Cz>c655+$jC@*?wKk&RB-lA zAEGwm-e(7=X+}d2Vs7z^pc!_Eg$B3j63w-3+a2{Fh`i(|k+do$LRYXJ=x-c`+Ts9b z5dnM_8m$-w2#DS(W1#t`rkua|LAB-Tz!CS_LU}FH+(i>dsMeLoIm>$f+;ua7-g_zx zsDFVLJ?&2-h^F%UnXsy^)+OQlu=9I>#{Gcwxxy=o%}cpkRNj%UU%hcA2|DBC9~ z8Nc)V$K?F7b@eC7h^_ap*a@Zt&CSW?!HzzR;q(h8*SGFuVJ}TdUThoyaut##AvE0F zT#GtJ1KrKdOrYiAWE1vMUCan}%I}K8%ejoHA~vK2nDCymi+lA_RNb!hs;eLL>czM@ zIq9M`+sXddpgb(}_+ry$m+|!JWBn9WZe`!R#|=Wl)87{TS0QC$FG@FZb6EFD$OcJh zF%5+fNQUf~;K74if%2ipE)h85+C6(Qs2JG=1Y#E!yq%JOe*>KVGTYZH@qis2l+ID_ z-=EZoJ*M8(U*y6i37K_hXefR|T>6?UMl!fLOCL+c$}iK0VJDXa@n&CX?B%7|6`3+v zDbSoi{S?ig6B85Ab+DY$JpR9JtVl*Pnlf5q;mHYQE0^CV9+;nZ99E~FV6R?ZC!P4x zen+!BWbW0RD%_*}xpO0+O>(sU17`4?!b$-XGxI4B3A!TFMn*=`okm^*tz7_3T?kN*ujy|#D&Cvhc=>#V>!v*R{8aQH zJva0jbgf!y1l7kJnJIEZfNnl!fY_#L{jV4%_H~0`QN=t^#8bvbEL9R+05}{1Eh(fu zcsR<=ZXS+8c-WcXwe3LLf}}+gT29$lUM5RWsyjKWXZvv#Hhs!LtG*yQgQ9*LBuBBw zX=#Yr@QXRFZcj@`;)N2R@lVH@wPWc<=1b?w2tyRyTTurc9ZD#D`iJrH@z!ppFo-)) z&z(OZ$ll8jv~&;c$uX-iVzKUq?Vg5%o{N6I*u?S7X&-{T-14#9?or6SKlu_M&mz8X zd<}XGut7pW-30+ITZ;6Zla$&PB8W0?tJ1SJ$5zOqe_~Fkci>T`{Ry!N{0lg?F0NE{ z?8Omc-J#FU&_SK=FxEf*3Z4WhQ()WP4rtgd?2LnqnA%wbnC^>9v-fPkRt z;IdD}3RYWNi{myEMq+?~1`*)`JUq4nLs+BVi!)U_)fnpP>wCNIT39mON#L!Dm%;GR znswcUb%9Xv$rJTKSehat$QjED|F&24)D8Tl$BrJ&&NT>(W~OQT<_+MB94$gitg^j6a;b&MLdUvNA_H8dBIhz>XDe4NnEzO>`abO!YCZ^2>9qMSVXT z51h3{VSY?~`S3bV}NrLj=Fu~A}WX(^$ zpZj0ey7lzry{lwnN6V_9idnc9ek_FUb2a*v&7Zmj6+d)EXfUK?n57j}VW4r8CWcv8 z|L1)WN(zNhbkfyMpH7^m?R`n>*)5q`9FOkR6Fu`GY4w=%tok6&YArm6KYxX$S~W0g zJRyP5+iS!`hL=Oz-h<4!8gnG*K{)Z9z_{7j+sd!r*-6U`6`AL+4{viz+-~E^YwoVX z!dGkomnYmwNnsnBqA(`)rqL2h`&eH4h!t8$Ts=9>`b!mjkOn#8@d-{+!WPOzh+uy) zPE9+T@|x`?l`jN#snxAA+B_5M>dsD*#&3s^3W{HwryBL%mqOPcExSTUv`;W zU>~xB_irb1?>m-#e|VgB*Dmg=!ScJtHN?1Mn~Q(Tfk;)v=xBG)JMYUL3~NPXkk;l6 z?rIh!UirLn-qb_Qu^9YBhM{g} zb9E;d5%Zp+!AeCWSTZo?zN#;(4SR&2-+-`XwI!@EU+2UL5S+18*IQt}eo|!4Kgr<1 zM73ghN;ueiQLCzSD?^;!y;r(xtE)q!qjp!0{$`~pDpH7?Nry1c)zZKieRX>T@FcZ< zn~?5U(dATr{~T@nvBQ|2^d&v=zus;bH^m*8?%0y3V(D)|PgbHdXFJvJ5-z|Y^YEF| zQLx`^6l{Jn?QO7+5;HU7oAX1bq}A<%H$Bwe1y_iofa90JWeMhiko*r*`USd46T<_tE2-y^*8Ki+tM5+~i)%!DBy zU*Gic#)Aw`o;NLHe4Lm_=Uagqv;hBHNMu^6+F5pJPU=|p3o;{x{ps;Xy?AZ)-xFQ9 zdFVTELu&^K-f#pua_4iCn4GQb)>^u{EIEm9pa+Z5)9yHys&Wrcc65`#&$35tW6(PcW?fGf>4W>U6kO_*8VQK%mW3GT-XAf7iz==0B0YO!ogk?)2xpp zXyXh5e+YN%-;Sl`<+ZLa3JRBucUV_L$)c^N=l*jMaGmW`$1_ApVB~kVcAqYB=T}{s zkpZ*QV>cf>_&jp}BoYT86{1(5j;;7!3B8R^zqT4Lv9R~Xx=v$hom>eK8IuMX+7#ZCv)Xg>(0nY63;4`!Mj2oQsWfTapGrBJC1R;8YgG#rrn z6hM3(7@f$)bptdSdTvBpsO-Z)tPEB|_OhVI;- zKYt>=XrToLDG!pnM(n&n<#Fb)Mx1ibyV~W;3b1*4Z#VyMq)nhhBWlE|*u7=uQw`t- zanc1qv>{BSBXF51~H^RYO=Q9`nsYwk^K*g=Nsy=_cit#aiz8CtNPegScXRl~P$ z#>Za=B|AdlfDvoK36SL$r@Kok9fp)Z$O)wpV}(ubdQt{m&NRhjvWBac?E^A0UF2q0l0l#k~&{E1~k>+6?>uRy$5YgEP5({QV+7Up8S z_bjlAlSktrhuQ0+6t#~m++bjjQJqBPUdhmsKHL?X#|+@tp@?tjxJDgdMJbsvvGG9P z`)$$~k%!UokinJ^Rvs2?kRDJ1546P=K;Qe>fE0Ga*6j zYJJVeH%EW~brvhc&Ea-=G55Zr@E5V>-TS6mEvdx+PoP14d!UfOSpmVjB}!W&ipZO{MSYsr*aw%Afjm|t{r$hL>jt%Ap6C_e_G zKTg~Xy*G~!k&&UTrbIAeknH~JQx2>eFcLApuz+mZNBcrd6jv=t7A{~zfW*Im1;Zsv zVfQ~CH#WN=xM@3dDHapk{WeCV1*!9L1whL(OpW@aYI&#%mG`k1=g zPFQnAm0n1q$+>`jly($;Th~NY=O?ss-$Xyr&WNW`aH7~H*=*hSP$L`h{PPXE5QbqW z^4kUnH|^7e5^Dw+oNT@$wVSsJNS_|VG#n%kVWWS4gDxb|a--0Kl;HwvVGn6k4I@JY z5RLs8id5S3(ijEPu)B4S?oyXw>YFK-H2dEpfYR3Y&}~kT6k< z&eRnD?=|4j@0Ts=XSCO}wB-6+Qyj&JBu0uV)HxI|3tR&Tx9oL{XyWq9?SU+I{W_k9 z<&>Hr9|~vV2v8Z{_SsH6&qCp0Ax{muv!T`fiEQxfT{$CFMv?&VOR*G6ONMqTSy*^C zT6kk)`|ZTNY!pa1FgoDl8c5C5-L%7={{4cKUom*PoE*Bxo%o?Ziux0Av~14byYP+U zL)6Bdo0l%D7eseyxN|24fB%P(?6Bvj8NuW7D!z%nqCgII=f@%=B2YYq;^fhpsvy*` z<+I)`Tz>t91VvJsQQ<`nE80@p<~wybZBC!2LIDQVnjM6FH0E*Z#>@LK2?lK%Jkt6m zB&5MLN|y$&fD&$M@;N&@Voa^dy|+-9B`94pYE$YeFm2hpviIolL2ZIdS!YuL6 z`VkM0(c0eS6#_&=K~#pq&*RB}wLsS@rm_T+Im@_s!2j7!L0R$p|8K{iI2&ZYe7QV3 zI#6~Iqok$^B1JUxu$PyrHYC>G2njb_OcEtyq;%&z9tQR}Bl~Im`nQY}X~?B8Hzp)2 zyM5ztca`+nH#9(9LE{pHnj#VsAD$n)efu^v%?8G%LA_81#8tZcI7rUOAZb{hdD((S zjWE)sOP3;JwgVjkEqzyLO~4FD3h(EP@^YR{yaR&(oDa9K_r7s^N;wco#k?-I?W}f3 zUVIhMb5wZvCe4>nwMvT`V3#h*A?X@)@^`>8FadQHefWFnv7FjR|5sQDAjK!I6CfJw zDR5x&hp~;ZbtX=65}BO3fZ*ci<3z)k1Lye3GCA}o)h`LJ9Fk^Jp2rT#$m9oa1c8WW zD{u{*=EDU5YRHYxZ_Z39Wvb$B*rUO8q^JMRMR(3i1wKolA< zTCku1+McYBA=LXB+FW@uKul+GVbf4d+V3nn37oCD7H$JbKO`mq;`9d?-4=iY>b!wn zqmWENsh#EHM;JAERBv5}K_iVxV7WxD~phMZ@qu_LEs3=tK=dWLFm4PdJ26xAT z$PlKm=%K@Gx}c!`{bM#=2xgWL`fGuvI<8v%2_N_%Oza`LJpklM(7vccqY*d^>Q2=E z41gXW^vmVPl=$Ek7+Q>b_)yYKhMO6*#0|hl+8@@lLlQ!u2;tt0+eV3Hhf8>AY#Ay1 zr}m>H0>4mB2@DL3184?9Y?$A9TB8yi_;;ZO<{7g0zXVAFj1ML}{JKf}@j{wG(_e_8 zbNaLk1|&dgx8*bpGXcHuz`4LlxZuGnR{WOkX`JVNZegPsSDYubXA>1J2tV|n5(RxG z86-s6k{4SZs&LCX-&s(GiAxM!3cUX8o3JfNQUBWSm*?;sT8Lq1p(glQ$azQv_5)DF ztw}-J9C52*ME8DLKG*y15hzG(yLt#W!C?q5VC|zh> zQ_`}Ibv)h1#dYF{7TjsN1FxH$BQfu?)co2~bx(eFZtB+O&!2RXuHuvz#%`lI8h*>Y zkL9RURSr98`FMEx{w^woQ$33on;~T^4zqbibCNtf1otuQ+po5nJnx9CC{w;u(kS6~ z_D*BOYFWHsH45I62w)git2f>D!KgJ@;$D+WkFZ9R>KIXS+-*)z*~Uef=zp*S#;j4G zR}?K^q{gxh&1C)QV4_^Rb`9EnG1+YFMaK~&0BW%5;ww?|`zcydXbq)#;tvy0=yj0w z1_|0vluQ0aB7WR^orXw6VI%y%KN3<(#-chLv3Z~=jOaFBygY|@KVQBSi);wQ1@HV$ zlJNLDC5F1~2#U~Fr}kN7)}R}u0~G4ZCKy)$EoTG!`z88aoYgmPb|s`2=1G&XuhWqf zP(s>)s&DW2j~Za$rN*nUugr)mPm_?SCS^tt5>7`cu>AmiHRwb#I}ffVdZl9wo(CO; zjrh>L@RwNA_e4ztgano5rd+b=mJcI$brI|ZgdOYu(%wH6GEQgR`atx$Ae-#5O*VR5+LMjmt^8;Qe?S4Q?$qbQcLE^<_ zf|Vq8>5v3{Lx`aIE_sx7l_FjcW~P~Z{fjG=uSnBYGYDn_rFznxVG=MAVBVqRFw@?=_IbtS@QaU0? zL^$!uK6KCfBeL-$IzV8c5dIiB(X)NUtkRI@sk@MkP9YmPa%^fqGoL-Xun9gkt?=fk z;GRXiAO`Apr5s81j)_gEa+WLaohfIAgz!rFWHj2HBgfd&&&$AwuSWJfg6@sy?%lilWNGDq)QXJA336jJmb`|D zEH!kSPKn22OtUmdB+@j~p)VU>=h%FG3&HL(*U0eQTfE5T(SMDrGfzG5;+lL8G`|8D zpujL>2D%JN8684BLg*0>m9oyuBJj<)`CK(Vl-y0;2;rn?Stgs9-m&xF>h;>6hDYQpet445`4vBc%$=ZSz}i7cO}VA7Nml?{Ib` zGl#(M?&6{&VIaa&axee3N6Rd(g1YD!%({Te733-H#5$GzF67dJRaH!;&m2EV83%N$B~k;9~I!2_k@XAX>s4wAZ)}NEXQJtACK1 z2+|#rltv2NLSwz%xo1KfKns@#Q% ziifp!2i&dx_Roz5fUp!*mKnyQfpmj`PA0wpmAPplCLgmf*)FIg?2 z)TnK|7#`^MpLf17Aka=*>WYa2(zMC-m~Vzremt;f24ji;4KPFrHwbxY!9Lo`?Z~G) z{TvuSJo@Gz1s|?F6&0X%x<5BSV-IvPO=i~{SuSk9xd+uu+lC4xC{UZVIvz4Ai8Buh zQte9$BJbY~`3j}bp_$pN`4d&R(Vftv#e*%QQ@T@)f2(Fba&|AWb9EQv&q`Bh5De zX9@y42Xg*|IWBFE|A|o<2)CWaUQHeJJO={p*?vNX-ti^eL2Rp`i=qz(Gr+sTh#$Vo z03L(U`F1_ogZH5eiO|SD{kXox9(A|vFsm|pA7WsKh2#}BLG~hk=rw+-aC=Rk$Ii!~ zrhd>4SPLy2$Ghf%=jrX=hNh>*^dGq$!V19pV~D(u!SA_cwuTeje^=+BTEF8zkrL7E ze0sleg4LZz1>MB|^J#5$*1anjambk|H7gcRhqB0}f79yETY=k&6rg6*Kal_7Np2;~ zNI)}>;#VI9`#{R(Jo|@)2}GJb?zltmsRT5M`GL3atAsUVdtktx2`AE_PGK3P8kX z5s7}iY2g0$&5p7Nm9_#Ew51Xfg!8DKdJD1m zKsafuM&uCQ!wZi!QMr5U0{;-_LE7p|7-Jb9AA1oxr9<(~W9w=+QSADSW0{?;SmrCw znzkY#VJ=iWv!*u+-Yn+EWO=xuA%#6mki4Km7IxmgdD|KmSpSfbCzd+Zio=K&W;cM> zL}!jz?by8P49u}Tk77%8E5h&wNq9ksk%lh+rqFlC7=$(;Z7=0ouq>}M8$W$1ea&8v z_LzD$%0G2K z@-*~y(Q8?h)1vzr==Ak5%J8Zmz1e>riP;j%eq!`UQk)q}odEpS!H7Igy;c3Efj3dx zl9@v3V41N#MH5;xG}J_l%d!RU+k17-ts9R+Nqr;G$=Jz%7?1VsU@o;)H4_#VLn~6d z?-RZ!{uKq6DI&57w4M2paj780fLa#^Fwt}vDB)cM^eV%}>AN>>&;!006YJ5%lW9<% zEJozRjm+SU6bp&=*vCyespWmkKD=xcz$}luP(Hk2V9&_?i-8fk?b z1YlI1-Dk=^K3Vri4Lkcyhh)ehxQZB)f8z}NzbD&GtEo$F`s39sN>5`0y)0DK)ksk$ zXN#^qq*Ps;#lk|1Wu8nCVvj3Q=Cj6WMTUnTC6R#0l)V}nnh!Jbp!{!?$fM2ME-XxW zt?c+A)P_oZG4>lr#9#mtcjU+slybB8m9u*CTGM?z*`6-~h(6%NiAUYjaO?~sKD-W~ zD1HNCZ@<&w%XH*q=D#qf7L&~keKv@FASJ6Qg_AgT?AQWBXrSmn)Uy{tH_U=S)uw4{ zD-08kFO;rB8wAb?kcRof7pL>pp5Es$ya@WP!U({Xi={iI*^ADxfc4*KQ0LC+9BlarHEb44CxXdoqb?Js75LuKkmtAdF$4`ARr; zh8aUY2bJ3mKnu)pA_)$&KYw2SN)!;&csQ|Yk^;3m0eV3$2QgE~#eJ{eZ==AbanZX$ zJqh|YS~$2QtFy^0kC(1`6v0n(-ZP+J}yeSzB2u@ePJj9cKPSPm2gI zr8e?W)JpX+X0{uYAN0NsHV@NWz;ccM)&7X5d5JVP@@qaX?)+1SXY_Wd2|M410C4IQ zR#fyhx8EOIm?x#f@kpsLu+3eE3UDBk7gkl>`R&`cPjXO$fzSq>gV1Kzl6ZgsBRrbt z&OHUyj&s&FYuY-*C*RGr+G}E0-V)bb1NkGloT>ExMNOSSMKRAE}D4jN{&r$o{~LrI3Qo_(9j=Tz%h-44!ueY68za~m=Tr0BT}%)|ToEH?G1 zn(AXZR=*+5d09s=<1f6#CiyuC)e2l>p;sZ{jXEd-Li3~FBGfJvRbCmn z@{{G=aLhQ*Zv%=AUDv2qsDpxP*V$6mypvb28f3_=Gvq4!eaiO{5>BZ7r);)AtnrkB z8;}Aq&OR9ldb2ANQK4eT)N|0$W-~f*z|t)-QLyK_U-y4Zx$LP+ZR|uSWg5ouK7Q`j|ak`q^2h9SKoj;3~cP! zPK(XU>b}rb?=V7P)C{uji%z`KgMe`*Me77fjB@in56tHrE|mjwq9=D>@UJLz&j!%d ztXNifTZW#bN2ja)&%>9&L_VB202Ks~nh@z(=qVT_0u|NKn_vAG*3Zx1%q3sMywh-k zn0Plq?z81_^^UFoS&)6jj8s3Gyrd%RO-J;1LMrw4QI5Fn|8f|`!+ey3zZNyppY1MM zjXlC=wL{nojgi{RThXcqr3VFJl)m;1ek+duyzGj*bH>Dgh*f(+Mi7IHa1EJMGtPvB zm9SEL9zBB7^UaAJW)zcn=*$JZ=6Bnfb(j-g!v8DnVhKhsi^X__RCimJnbdG)6k}`1NZm z#(f3F7$Xj(`Bj%{vBpJ8cq1oz2(F3mXMpvF!I362PbbmM&IEXf4VwbKx}J%P&k~*% zksS715xO|Low%7H7= z6felc&O-1<0TU}`s23i~hNb@P5Z&BuwCEeO85Bc32eKe0Nw1|Yfnd}ns9?uC?wgVf z@2SbLw=HgZ-M&iJyu;985I&jm>}&c7)|@TuWsLt8>C>X{CT|J${O~0jL_TGXqlLRm zqP}R5B#--QX%Vp1=H|`y*}nN8d^C^wQh41ljHbJT#4tS62i8RyzP_`oYoy>=7Kmwn zGR^dIU_>FBrd~#EbqSQgBgDkTX^_k??E3X4O9et|2cDZTRZyxW|1NB*%}>Z&-nf3_ z#)5h2!a&b2dtPu;Lj(UUWD-0wv5wEEu_;7ZyT zB69JM+i^BFuspWT?{SF`c@3FL1F`p@T!&f*jhMg?e~R%wq9`ffgjC9$Tu1)vs-i2B z6HdK9Iy}j!&{qqCig2(93yg-;f*fIZd?K4cUB}=BQ(|-V@s7AS4DXi*NaG!YJh~EcYtjk7<3BZM14|@Mrw=)%UlLLMc4I@!*Za%a6ZBE#E0|V8eBY zL;mrGNs0>RNa$^zuT<>221NB%i)_;XU^qrVD1rnO!u$4V!%)VF6*$N(0$6G}-()z2 z>4im+4)pD*z-~vvaZBi=D>(b^I%juoY3XSU9TDL5pa4!+lDNGmTxqmN(~uthUqzoX zo%jz*{TZ&ir(gSxs5>;RFBl7*+AoBSR;2{(*DsTF9!)&(w&m)^+%}0MpJM%9oi1~R z@u1*WTkjHc^1Vs{dVzu*i8h=p0gnMJh(i1~Ee6w*=zudgl%)=+ha5{di03dme`z=) zB?ynb2E`#pfnhfJ@?bdO4RjyrUxDy}_Ji7m3kUEgjin2_{=A2Fz^>E(M{VaG4fVeF z@fqFJrbtnW(B55+wAD@*my~qfrJUqiv`MxYDs7jYF}h>26{Y=1IHjGEnOuup%Jqp5 ziQJ{)xK6_va~NaJ`%`-HI6OsXzDKgU}6IR8tbTRtXf{Y+Q+vemBn7m2|S95!{hldBD7@h=qW(xf!r?S5s78RvMnh{x)a!L`z4JhNhh$Z)vj_b;gC`jhnU5GR83!z5Ya6XF-9i?e=PT zwLW1n9R}$`$*f~9XN<@&RZGP2Mt&>N$xrNtWNky{qO}S?Cp>o=x(SHkgSqQ+xVkp_ z&K1#9IrlTO{#bE~dk=zWA>ZH9IC7R77Hs2WFk!F%`p00eR?lx=Usx{kne?NBOknFa zT7Dp{zqCAMlG2Ic%C$Fkh95sA$6Ed@ZOSJM;J~g*b-7d$1lxfsrU$R?94Tk)M}s^W44_8>LqA;iYd46KbAFO@ev;igs?2Yn6uUP^_utPkV$36y}L! zBwjK2RqPt~*~CEQYyDpsfYMtI4>u5+yNy z9>1^CJ+t1(=e;{L{^vZ*@DKG|C-MfWZY2OL3dO|?1|F=FNR2>(wTBN1;*L|N3>Kcn z!Oj8$fF@&yKrrk2u5UaVzdVG6@Ub_2kWM!?ZD^>kzS`YA)-<+ZXxww@`AzQb%gH8T z{j|IFxF^$d3Pjpqc`pAMlAPeRqv5R5VIMY2x-cM!$4`l0TwT3(uxZGxD=zg0eRw#E zV>&Zm2F)zzX*kP&`9mc_lAv6afH*KTGnqn9F3~ieP07S`RR>f_ym7If-SG82>je4 z_*6DxBW}K5s^$~t zvpw3sCxzVpARf&FEEZ-pjnark5wM7?mBhgpJGG&_xw+ZGAgc|`p}K_pyq`tKMof4o zrNO0~ID{<7`R~+754{^>{m*{2i!et?juiS(8^R@c1r=;;Y;0e&$AEYh8e4`~j*4Wq zFb2vgJfGLlx5~HQmdK7U%G>4#Zm6xTg_ktrXAP&AwX@@t#8J=JBJrhf;M?g!-kU;-M`b;Ja>DZ8uTalkM-oU10m;9Lk}(PdOb z<2`#hMBz*qd?hdc=!rFi>}mVMS+8NG=8gdgv;Dva- zM5O!Qga*=h+4#~nj*Pv+E^YEtKfy(yX2~G?;K73>6&WLvUwf55?1R-v1!GUeEE;nE zPC3}oO#Dh^B_%Bc!UcML@WIIiW0ZU?i=3FSgAuHz+)-SQ>5wv5x}CU;?!E=T)0G+R#smdo0yaU1(nDGMM9ZTOirDozvNB;EJ`vr!Z2n)h!Xb_qtt8k1^%s=A=GmBxcZ1ub$g>A{w!GW8Y__ezO z9ysd7gSYuw^9}1dTAN6j4*Hq8f8 z)o*xCY{*N{PdQs+FhfDX9JJV|hX0Hfxa5?>(3Mb%!Oe4>apsjcCn!x{h%-DIs+@14 zEr{_!Okl(vq7Gpv*oL(k(<%T8iA*9;`i8=6rby0mg1sSetg$P5n&IiAz^QwormE_iVKfe49dt+wMkebK$%X?H0A z0_u*yz4vr&BmX)A1tHQ1$Vf`ylLY|Lb<$*sSaJ|Wk_!kMObPBEPr~k%$6O47atmT7 zZ40?j2*FZ3Z>FZPJclEjDtX!~niPzVUP4lm{4CS)<57|)O`@mKqlUIV;hv)#e@mXn za9snnOfDq5m4iOmlKa|{`geJ5=+4bG9tYHfSc`mzAgM%+FoH0-cL0-76sofH6}Y3# zx@~6lu};8BWw6kw!wF+C8)7=wk@5Zf#*@;jI1?^`Td~cQ(S(Z4zr$a|B)%dLt2gsZ zXEF%RqP^wfj%w_gVOT@&v8@#%8-H-hASD_EUAD>LXADv>85*j?++zT@MCY@Z7v4R! zcOiG)cI3#B&u6r_%!P5S2I`a7{Z2nJjsu>Yk@Rq6pn2WA7e_QTnn+~Q?lTb12j>Ub z<;f_4yPiW;Kx3LKVhkmpt|B2y59U0Fgnnf$d!IcjDk_L&mxx#ZY=EMpcYD4G&Hb&c zj?~QfH9nsIR+JN+*$sd7hRc9f(+vTO$5SNiXG>#d|88X~{>djER;huMruO-m{Ong( zILgw!0xCCosIVM#t&9TDY5WU%p?m5Z=JFRYVhw+q0d*1gO-dKfSNgI6?#V^K3S7zR zhC3Wi&KmLCAw1Ko80E_VFvJ#>cFh3d<6Jnsh%;T6A_1^)v6dDCQ#bb^6F91AcnCQR zlR+*jbZh^{wehNtOIMP^23ds6`yEKUA|d=_%x9g`&PU?Z(*bL?PF`@T;$}%%SrNcD z=F2i8Sc;8|k;Ba~BGQwppI@>yGt#YiEf?9$^?|LoNp7Bb6(R zQeG`ugE;;48ZjD#lW{bEP%iJXOl9Ngt-R|7=aq;jk%}nm*_1IIkv%m>Ae7NET&BTTj_#p?F(MPdu=GR!+f=Y<@X%wwdX>*UO}xo7d?;QJ7Q9Yo zDLCXu+}GS5VRpg>5A&B9_1fzK!mfk3a zF%I^UdqlrNF^GH3g`#RJGNkPvy1K&P6%JeA^<=XGP_S)p@_A@^hamLsMUR>btV0OE z5@sMeaB4TD-mCn@4b46Czb5iC*YI~#h+;(=&|H1 zvdIZ@VvDX*JOHAg6Vc8YI43G=)dOU>mZ2Mbx^_sgvJZis6M}tZJ4Th}FXK&}P{fDf z`mgVu6Z1DjMlM*=sYu~8hkuCNAvMuyecp=p~| z9DK5mZVc?U(?Rkbu3WbjF$!qM6p~aHgD;|&bE-!Nu;#m~z4AoXss2Leo>39*v%5m@ z%+*j(F%zDfq{X=K<~LIKZq?!bFEO9%Isy{J8W`;0Mt?i zkx&&~2)F=30O8ol51+TnY7tS&w93gOT0mQHd`BEwnkVXwpTw`*q@f=5uWRHv! zDXk|R*}sxs%ZW<_5tZt;n_+;hgbhRqM+Mr)a`&(Ln?E}cTCelt8^I{~nh`>YuYcuS ze*CCkedes;0CW_DKwgHyjRCgNH(pN%vZi)xBJqKr+B-ERTi|jmW|zEne{CJT7{lD+ z%KVJ*?PUkwd@h>*E#|n`0ZfUz3Jp`V)M~M9PIG7QYZB zCo!j1_ zIPyUZ$Mg;BlkWV!X%Fd5(QV zu!jd-|5)8JZ1C^ln>WYnBUD!&3?W(v622nj+mBG4p1!4{e02{*2vpBklE&u$>-91d ZrlT$KInNKz-^9Sr&aD>aJhQ`R{{gU(cCr8f literal 0 HcmV?d00001 diff --git a/docs/evaluations/Docling-DPBench/evaluation_DPBench_table_structure.json b/docs/evaluations/Docling-DPBench/evaluation_DPBench_table_structure.json new file mode 100644 index 0000000..62a01f2 --- /dev/null +++ b/docs/evaluations/Docling-DPBench/evaluation_DPBench_table_structure.json @@ -0,0 +1,718 @@ +{ + "TEDS": { + "bins": [ + 0.0, + 0.05, + 0.1, + 0.15000000000000002, + 0.2, + 0.25, + 0.30000000000000004, + 0.35000000000000003, + 0.4, + 0.45, + 0.5, + 0.55, + 0.6000000000000001, + 0.65, + 0.7000000000000001, + 0.75, + 0.8, + 0.8500000000000001, + 0.9, + 0.9500000000000001, + 1.0 + ], + "hist": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 0.0, + 6.0, + 6.0, + 7.0, + 15.0, + 8.0, + 5.0 + ], + "mean": 0.8315098039215686, + "median": 0.861, + "std": 0.11816401695085008, + "total": 51 + }, + "TEDS_complex": { + "bins": [ + 0.0, + 0.05, + 0.1, + 0.15000000000000002, + 0.2, + 0.25, + 0.30000000000000004, + 0.35000000000000003, + 0.4, + 0.45, + 0.5, + 0.55, + 0.6000000000000001, + 0.65, + 0.7000000000000001, + 0.75, + 0.8, + 0.8500000000000001, + 0.9, + 0.9500000000000001, + 1.0 + ], + "hist": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 2.0, + 3.0, + 0.0, + 1.0 + ], + "mean": 0.8462857142857143, + "median": 0.874, + "std": 0.11662291696705568, + "total": 7 + }, + "TEDS_simple": { + "bins": [ + 0.0, + 0.05, + 0.1, + 0.15000000000000002, + 0.2, + 0.25, + 0.30000000000000004, + 0.35000000000000003, + 0.4, + 0.45, + 0.5, + 0.55, + 0.6000000000000001, + 0.65, + 0.7000000000000001, + 0.75, + 0.8, + 0.8500000000000001, + 0.9, + 0.9500000000000001, + 1.0 + ], + "hist": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 1.0, + 1.0, + 0.0, + 0.0, + 6.0, + 6.0, + 5.0, + 12.0, + 8.0, + 4.0 + ], + "mean": 0.8291590909090909, + "median": 0.859, + "std": 0.119568982363071, + "total": 44 + }, + "TEDS_struct": { + "bins": [ + 0.0, + 0.05, + 0.1, + 0.15000000000000002, + 0.2, + 0.25, + 0.30000000000000004, + 0.35000000000000003, + 0.4, + 0.45, + 0.5, + 0.55, + 0.6000000000000001, + 0.65, + 0.7000000000000001, + 0.75, + 0.8, + 0.8500000000000001, + 0.9, + 0.9500000000000001, + 1.0 + ], + "hist": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 1.0, + 1.0, + 4.0, + 6.0, + 7.0, + 17.0, + 9.0, + 5.0 + ], + "mean": 0.8471176470588235, + "median": 0.871, + "std": 0.10569534465790319, + "total": 51 + }, + "evaluations": [ + { + "TEDS": 0.919, + "filename": "doc_00f0adaaa8358a28b4b4e83bc97dcd83a01f7283605b140c2be8e8d47bba8b6b_page_000001.png", + "is_complex": false, + "pred_ncols": 3, + "pred_nrows": 9, + "table_id": 0, + "true_ncols": 3, + "true_nrows": 9 + }, + { + "TEDS": 0.874, + "filename": "doc_d276ca9a5ecb8d6d11359f515e50c8f78395548de4e3e2c49e38f5500ee40ebe_page_000001.png", + "is_complex": true, + "pred_ncols": 7, + "pred_nrows": 12, + "table_id": 0, + "true_ncols": 7, + "true_nrows": 12 + }, + { + "TEDS": 0.848, + "filename": "doc_934fbf534914863f6431eef38f5bf66fa91afd439ddf20fb1af0cf3225159ac1_page_000001.png", + "is_complex": true, + "pred_ncols": 7, + "pred_nrows": 10, + "table_id": 0, + "true_ncols": 7, + "true_nrows": 10 + }, + { + "TEDS": 0.913, + "filename": "doc_4682941b1a9a3ec96599d8188b673e8c6d1c4f2a555b5dc0d739c6c706815553_page_000001.png", + "is_complex": false, + "pred_ncols": 4, + "pred_nrows": 9, + "table_id": 0, + "true_ncols": 4, + "true_nrows": 9 + }, + { + "TEDS": 0.932, + "filename": "doc_f4dea73d2b75fbc7d590e2085b14e9bee7e766da73e14f723a2580fb0f36c707_page_000001.png", + "is_complex": false, + "pred_ncols": 4, + "pred_nrows": 12, + "table_id": 0, + "true_ncols": 4, + "true_nrows": 12 + }, + { + "TEDS": 0.933, + "filename": "doc_0ce1ded16152ad18c45203c1dcba0287235b888266fbcd112396fdf7cd094fab_page_000001.png", + "is_complex": false, + "pred_ncols": 4, + "pred_nrows": 9, + "table_id": 0, + "true_ncols": 4, + "true_nrows": 9 + }, + { + "TEDS": 0.896, + "filename": "doc_bdf0e09acee7a07f7e60c6055128967b36b44564842983ad3b0f273f1c1c5914_page_000001.png", + "is_complex": false, + "pred_ncols": 3, + "pred_nrows": 12, + "table_id": 0, + "true_ncols": 3, + "true_nrows": 11 + }, + { + "TEDS": 0.893, + "filename": "doc_c4416f86c1db7fee5dccc1a10e85b1aade1cd77803f4ee626a090e8bc26aa9a9_page_000001.png", + "is_complex": true, + "pred_ncols": 7, + "pred_nrows": 10, + "table_id": 0, + "true_ncols": 7, + "true_nrows": 10 + }, + { + "TEDS": 0.769, + "filename": "doc_24ab9416df440de9d792a4034e177eb71933a6eed72d4cf1ead17d22a787702f_page_000001.png", + "is_complex": false, + "pred_ncols": 3, + "pred_nrows": 3, + "table_id": 0, + "true_ncols": 3, + "true_nrows": 3 + }, + { + "TEDS": 0.897, + "filename": "doc_24ab9416df440de9d792a4034e177eb71933a6eed72d4cf1ead17d22a787702f_page_000001.png", + "is_complex": false, + "pred_ncols": 3, + "pred_nrows": 7, + "table_id": 1, + "true_ncols": 3, + "true_nrows": 7 + }, + { + "TEDS": 0.889, + "filename": "doc_0ec2813813e8e7496d97aa4fa15dd414aa66cae2d6f62db818d30d9856ff0439_page_000001.png", + "is_complex": false, + "pred_ncols": 4, + "pred_nrows": 7, + "table_id": 0, + "true_ncols": 4, + "true_nrows": 7 + }, + { + "TEDS": 0.871, + "filename": "doc_0ec2813813e8e7496d97aa4fa15dd414aa66cae2d6f62db818d30d9856ff0439_page_000001.png", + "is_complex": false, + "pred_ncols": 4, + "pred_nrows": 6, + "table_id": 1, + "true_ncols": 4, + "true_nrows": 6 + }, + { + "TEDS": 0.897, + "filename": "doc_970804c04c81dd9cc3ba80f0063b9551d290e83934c4e2c258e376845b3d180f_page_000001.png", + "is_complex": false, + "pred_ncols": 5, + "pred_nrows": 8, + "table_id": 0, + "true_ncols": 5, + "true_nrows": 8 + }, + { + "TEDS": 0.857, + "filename": "doc_970804c04c81dd9cc3ba80f0063b9551d290e83934c4e2c258e376845b3d180f_page_000001.png", + "is_complex": false, + "pred_ncols": 4, + "pred_nrows": 4, + "table_id": 1, + "true_ncols": 4, + "true_nrows": 4 + }, + { + "TEDS": 0.903, + "filename": "doc_970804c04c81dd9cc3ba80f0063b9551d290e83934c4e2c258e376845b3d180f_page_000001.png", + "is_complex": false, + "pred_ncols": 4, + "pred_nrows": 6, + "table_id": 2, + "true_ncols": 4, + "true_nrows": 6 + }, + { + "TEDS": 0.857, + "filename": "doc_39a0d95abd6dc977dfc7986b57fbbd9a8331af6b5b603e023f41dfcbc67003c7_page_000001.png", + "is_complex": false, + "pred_ncols": 4, + "pred_nrows": 4, + "table_id": 0, + "true_ncols": 4, + "true_nrows": 4 + }, + { + "TEDS": 0.871, + "filename": "doc_39a0d95abd6dc977dfc7986b57fbbd9a8331af6b5b603e023f41dfcbc67003c7_page_000001.png", + "is_complex": false, + "pred_ncols": 4, + "pred_nrows": 6, + "table_id": 1, + "true_ncols": 4, + "true_nrows": 6 + }, + { + "TEDS": 0.865, + "filename": "doc_1039bce8b948fac48b0b9a42f8266700a104aca355c3681a544d65e6b136b8d5_page_000001.png", + "is_complex": false, + "pred_ncols": 5, + "pred_nrows": 6, + "table_id": 0, + "true_ncols": 5, + "true_nrows": 6 + }, + { + "TEDS": 0.861, + "filename": "doc_1cf54980192667fd7a6977b301a175e98be02b11651bbf78b6cae338858db2aa_page_000001.png", + "is_complex": false, + "pred_ncols": 5, + "pred_nrows": 6, + "table_id": 0, + "true_ncols": 5, + "true_nrows": 6 + }, + { + "TEDS": 0.884, + "filename": "doc_d1437ae1c6bedfa2bbf72e4d04dee3a01896dd045024438ee7c4e775e0ab6e9c_page_000001.png", + "is_complex": false, + "pred_ncols": 5, + "pred_nrows": 7, + "table_id": 0, + "true_ncols": 5, + "true_nrows": 7 + }, + { + "TEDS": 0.846, + "filename": "doc_a5528f4dd6f5ba0d52e6bf97a0aae99ede782fe97f4a0b4f96029e5ed7fe18b5_page_000001.png", + "is_complex": false, + "pred_ncols": 4, + "pred_nrows": 5, + "table_id": 0, + "true_ncols": 4, + "true_nrows": 5 + }, + { + "TEDS": 0.571, + "filename": "doc_10f75857bb705a5441260399c3fca1080262cc0c271270825caf141cf7b31770_page_000001.png", + "is_complex": false, + "pred_ncols": 4, + "pred_nrows": 3, + "table_id": 0, + "true_ncols": 4, + "true_nrows": 4 + }, + { + "TEDS": 0.92, + "filename": "doc_b9e5879b4dcd812e854cc06c6c3cb4f54231af28c919be63e27bef4c639551c9_page_000001.png", + "is_complex": false, + "pred_ncols": 3, + "pred_nrows": 6, + "table_id": 0, + "true_ncols": 3, + "true_nrows": 6 + }, + { + "TEDS": 0.734, + "filename": "doc_6916893bac49ba4fefa0a83eaf039b73c4adf00ed81ef97d4aad8f1a5cb90994_page_000001.png", + "is_complex": false, + "pred_ncols": 5, + "pred_nrows": 3, + "table_id": 0, + "true_ncols": 5, + "true_nrows": 3 + }, + { + "TEDS": 0.71, + "filename": "doc_f151ca1888000160dc6ce2e46c3438bc13f9a034200b1ec5d81b03c77606eed6_page_000001.png", + "is_complex": false, + "pred_ncols": 2, + "pred_nrows": 2, + "table_id": 0, + "true_ncols": 2, + "true_nrows": 2 + }, + { + "TEDS": 0.913, + "filename": "doc_603d71e12f52d9801a9d82995babf69681d923f493b8d49abfbab8662a88b376_page_000001.png", + "is_complex": false, + "pred_ncols": 4, + "pred_nrows": 9, + "table_id": 0, + "true_ncols": 4, + "true_nrows": 9 + }, + { + "TEDS": 0.839, + "filename": "doc_603d71e12f52d9801a9d82995babf69681d923f493b8d49abfbab8662a88b376_page_000001.png", + "is_complex": false, + "pred_ncols": 5, + "pred_nrows": 5, + "table_id": 1, + "true_ncols": 5, + "true_nrows": 5 + }, + { + "TEDS": 0.839, + "filename": "doc_603d71e12f52d9801a9d82995babf69681d923f493b8d49abfbab8662a88b376_page_000001.png", + "is_complex": false, + "pred_ncols": 5, + "pred_nrows": 5, + "table_id": 2, + "true_ncols": 5, + "true_nrows": 5 + }, + { + "TEDS": 0.956, + "filename": "doc_a0edae1fa147c7bb78ebc493743a68ba4372b5ead31f2a2b146c35119462379e_page_000001.png", + "is_complex": false, + "pred_ncols": 6, + "pred_nrows": 16, + "table_id": 0, + "true_ncols": 6, + "true_nrows": 16 + }, + { + "TEDS": 0.88, + "filename": "doc_95f808815969717e4ef6390b2b789a8ea552a000a2b936c3c9371f695a71bd9e_page_000001.png", + "is_complex": false, + "pred_ncols": 3, + "pred_nrows": 6, + "table_id": 0, + "true_ncols": 3, + "true_nrows": 6 + }, + { + "TEDS": 1.0, + "filename": "doc_ba0e025d53c091e8d4bb87499ff69ed3428dcee325c8895ecdb40e973b4c835c_page_000001.png", + "is_complex": true, + "pred_ncols": 2, + "pred_nrows": 9, + "table_id": 0, + "true_ncols": 2, + "true_nrows": 9 + }, + { + "TEDS": 0.81, + "filename": "doc_b862cd0d6f06c06ee5ab7729ed4e8ce58e6964eb0f1ab98b3865b57a4808216f_page_000001.png", + "is_complex": false, + "pred_ncols": 4, + "pred_nrows": 4, + "table_id": 0, + "true_ncols": 4, + "true_nrows": 4 + }, + { + "TEDS": 0.998, + "filename": "doc_750091cec2c31ca61ffaa40c282148258f40bf5db7356a7ed3f918a07c81ef0f_page_000001.png", + "is_complex": false, + "pred_ncols": 1, + "pred_nrows": 8, + "table_id": 0, + "true_ncols": 1, + "true_nrows": 8 + }, + { + "TEDS": 0.886, + "filename": "doc_a46e40b2b04dc83e4014fca656dae6df20af1fe7a420df306406409ea8c0db31_page_000001.png", + "is_complex": true, + "pred_ncols": 2, + "pred_nrows": 6, + "table_id": 0, + "true_ncols": 2, + "true_nrows": 6 + }, + { + "TEDS": 1.0, + "filename": "doc_ed046862e68a27a7260da60f3984484577f83500dea61020d043c7af8b76731f_page_000001.png", + "is_complex": false, + "pred_ncols": 2, + "pred_nrows": 6, + "table_id": 0, + "true_ncols": 2, + "true_nrows": 6 + }, + { + "TEDS": 0.773, + "filename": "doc_2e4f2561d8624cab6140564d53aed11862cf5979572fc3d1205af105486e0b1f_page_000001.png", + "is_complex": false, + "pred_ncols": 2, + "pred_nrows": 6, + "table_id": 0, + "true_ncols": 2, + "true_nrows": 7 + }, + { + "TEDS": 0.763, + "filename": "doc_4d5b682effaf9580928a7073980e3916687d168238bc96b6c17222f747eacde6_page_000001.png", + "is_complex": false, + "pred_ncols": 6, + "pred_nrows": 7, + "table_id": 0, + "true_ncols": 6, + "true_nrows": 8 + }, + { + "TEDS": 0.764, + "filename": "doc_4d5b682effaf9580928a7073980e3916687d168238bc96b6c17222f747eacde6_page_000001.png", + "is_complex": false, + "pred_ncols": 5, + "pred_nrows": 9, + "table_id": 1, + "true_ncols": 5, + "true_nrows": 9 + }, + { + "TEDS": 0.549, + "filename": "doc_62c4b0340519f5963a37534d3abeb0546766cb53bc90504f663eb7cc321a6a6d_page_000001.png", + "is_complex": false, + "pred_ncols": 3, + "pred_nrows": 7, + "table_id": 0, + "true_ncols": 3, + "true_nrows": 7 + }, + { + "TEDS": 0.749, + "filename": "doc_4b5f23f97fd273cacb8de95fc1799ed4dc4d5c15e9ed86d707cdf4be53eb7c7c_page_000001.png", + "is_complex": false, + "pred_ncols": 4, + "pred_nrows": 3, + "table_id": 0, + "true_ncols": 4, + "true_nrows": 3 + }, + { + "TEDS": 0.742, + "filename": "doc_c31d842c665085cb1379d441ef0b65869f29537b9252956d6105e6b4cc972311_page_000001.png", + "is_complex": false, + "pred_ncols": 4, + "pred_nrows": 4, + "table_id": 0, + "true_ncols": 4, + "true_nrows": 4 + }, + { + "TEDS": 0.619, + "filename": "doc_29d883eb26e7b9d08250b4bab1c51092d74e310e6acfa955f8fe28f1008accf5_page_000001.png", + "is_complex": true, + "pred_ncols": 7, + "pred_nrows": 6, + "table_id": 0, + "true_ncols": 7, + "true_nrows": 6 + }, + { + "TEDS": 0.916, + "filename": "doc_61070c2fe64a690c7c9cc97d12b76380e7a134feda3c5315f779306699a89a1b_page_000001.png", + "is_complex": false, + "pred_ncols": 10, + "pred_nrows": 14, + "table_id": 0, + "true_ncols": 10, + "true_nrows": 14 + }, + { + "TEDS": 0.795, + "filename": "doc_2f972daaa81cb3949db12a39cf6910c4f9952135f7eb3850e1f60ef7d8b5aaf9_page_000001.png", + "is_complex": false, + "pred_ncols": 11, + "pred_nrows": 6, + "table_id": 0, + "true_ncols": 11, + "true_nrows": 6 + }, + { + "TEDS": 0.756, + "filename": "doc_2f972daaa81cb3949db12a39cf6910c4f9952135f7eb3850e1f60ef7d8b5aaf9_page_000001.png", + "is_complex": false, + "pred_ncols": 10, + "pred_nrows": 4, + "table_id": 1, + "true_ncols": 10, + "true_nrows": 4 + }, + { + "TEDS": 0.71, + "filename": "doc_2f972daaa81cb3949db12a39cf6910c4f9952135f7eb3850e1f60ef7d8b5aaf9_page_000001.png", + "is_complex": false, + "pred_ncols": 9, + "pred_nrows": 3, + "table_id": 2, + "true_ncols": 9, + "true_nrows": 3 + }, + { + "TEDS": 0.714, + "filename": "doc_eac01901729786f15ba498716c592370d50f00658c291495086ff49a8538c5c6_page_000001.png", + "is_complex": false, + "pred_ncols": 8, + "pred_nrows": 3, + "table_id": 0, + "true_ncols": 8, + "true_nrows": 3 + }, + { + "TEDS": 0.822, + "filename": "doc_eac01901729786f15ba498716c592370d50f00658c291495086ff49a8538c5c6_page_000001.png", + "is_complex": false, + "pred_ncols": 9, + "pred_nrows": 5, + "table_id": 1, + "true_ncols": 9, + "true_nrows": 5 + }, + { + "TEDS": 0.4, + "filename": "doc_39f32d6a01b8e434ad2fb16ff3896931153cdf4f85bca09194b71e1514711355_page_000001.png", + "is_complex": false, + "pred_ncols": 1, + "pred_nrows": 2, + "table_id": 0, + "true_ncols": 2, + "true_nrows": 1 + }, + { + "TEDS": 1.0, + "filename": "doc_39f32d6a01b8e434ad2fb16ff3896931153cdf4f85bca09194b71e1514711355_page_000001.png", + "is_complex": false, + "pred_ncols": 6, + "pred_nrows": 2, + "table_id": 1, + "true_ncols": 6, + "true_nrows": 2 + }, + { + "TEDS": 0.804, + "filename": "doc_646f014ab615ac22266a43be2691010032d9fd62368737d488ffcb912cd68fd2_page_000001.png", + "is_complex": true, + "pred_ncols": 4, + "pred_nrows": 10, + "table_id": 0, + "true_ncols": 4, + "true_nrows": 10 + } + ] +} \ No newline at end of file diff --git a/docs/evaluations/Docling-DPBench/evaluation_DPBench_table_structure_TEDS_struct-only.png b/docs/evaluations/Docling-DPBench/evaluation_DPBench_table_structure_TEDS_struct-only.png new file mode 100644 index 0000000000000000000000000000000000000000..975844deb060930a0dbf03c6a0657dc8c49d80cd GIT binary patch literal 19818 zcmdVCcTkks-Zk3jm;*Cr42X&#Dj*0V8M93kNs?891OdsC(HR3uvdLK`XC-Hh1WA%5 zD^YR=C4Or=bIyI=``z!4TXpKzeNWYt$F}?F=h^!g)>^-{+vSS%rA_O1ucuHbo2Zx1 zU!_o1I8!KeU)Qe2E4l7kcgYrkd+kmaBQp4_g^$@{gLUj}BTUnO9nd zn&&4E3Y^k%{R200{E$cP2K*`3aFDWsLb(g*bV(k3~f} zx!xO7B`Il5xBGJvRWmEcStR;TnRcGPR45h5W17}i>UF80pdd~@mS#IM6k8v!#Hw1B zZrI3U)XY;KryzlUJ1QUJ8K)2*<+wPfk1ynKXwS7CZSd#PYMqOu?K3FZI6gkUTi7;D z7VX{wlV#v^3C^8eJJG9F$~}tdSr0>eUH_L=}af6Fs%+ z)@_ljp?A+ZEl&H@TDZJDlHC$37tP(J|BYq8)52@q)j<;z6NQ*7XKJHmWM=c$Y~Fjt zk3(HPMVs1F<|pHGOsOHuv}@betzG^siNnLSGNK_2G7?-|T)R)|x;7-L`mrrWcMm2P zDh@ZN_Y4mwUX2vjpZZ>%(;IT)#{K3@ljNHppG3~l4;MJh^V}V~&a3xXti8RR z<7e^Bt|B*b9S*guJ1q`*%S-ddn@kv*e2=STT@kjOy7FNCcJ0)rx(KS5_WV#=Zf25t zZWO)952g*F_Y8Y1t?86|QmaJFbC&y~!wjYe>Z#4~^06~uTCcY}!1n9CHED*896lWC&H8huT}83Cn^L5WunK0=wPr!s@i+-Tj+B<+5sr`F z{-hWAjHi!x-zR8L;(1{1XI~iaov&gaKYY;pG4?IC(r)UzEMiT@(b3U%>1SWh=xDN3 z&?&Mv%{i9wOYqwbF}%jR`I^Z0@JRGWa0AdW($U^+U7?Kli$1%n;vPKcOT5&+TY|`9>^=9nrU3` zFxwVw-d7p<#V<26^X$2EygHw@<&VAFkkWKJK-R4H#17u;Zzx_hwW9VvUOm%pcbp&Q znP0bV9T&-@8sv{siLfr#tZyf>mES z$kX%s&6_vXau!@0JOh{ms>3hkCKS#Lw**~u+nt?I7~GrP7kW}eM5MRm*&+7~Eao0+ zFP8nXWji*RfvWrc^9$)=(WM{Z&$Oc?13LABPTq2VrtP@AXx%76$7eo0UtN#uONxm( z>(@rP$=S`0v`H;3{H!QkUJ4pm*vcY#f>w??(_hyfuf%zXt>Wi?+wTXu1_uq3)UwU& zWSBPHstTY4u_Vq1eNr-?AdlZDa2;DQFqC+BHh#gVY`_K{^f7)AGvYq zH#&~A@^9U~UB{KHcQ~`-6m9QhJBQMgT%ro^)-7AiS|(W?`Kkg=_GR=|m1Hm>H1|iaiEnqeHK{Y&oo**`knfTUmIv)f~yI6382& zJKp&*PA=Nr-rj!Ou3a&V!nWUkjCV=bPK~swu3EqC^LSTr{fl#JNCmv`WLx#;7w5L` z-ye3f!>}>Qr$V>3CQ`ynXa;G4a*p+vVykqT9#BYB%f5=RkeTquTPCYSlMxW{2K8|h zB=_0`RaPQ(sqfdUEVQ0^tO}p`ab+o?wUjCO$i}UfQ zclf9*_BixzrwjAq;mvlpzq~qg?%_t+X9q8Mi#j<9&wSCMJ~ZUi$csVr3$!Q4{v3!; zuFEh|^*O5Wsk$N0##|xcn(WgZ?BbT~$Zsm`$@XKtb&T7#F;j7As%b%;M&lGIlW0CO zqaDduh?1%*8S>0AvfCyvn&&iIpLO(O3$>boyN<#Hy>S=Vn?= zbjk!%_&WOKFcoc%;MHBmiR(L;183yWKuz{{_iaa$H)w|&>HBm4L7_hDC6 zlo9zE5wYieCGPDRGpa?yYxNwPmWS!5p8u_*1i(DDPZ@hm?F@3=y=BQdaF;+g$3!dU7hbrrGzmHEWK^gic+E5U?C5>FDTq ztt1%DM2R}c!gBH1vu9M}eqoLG@86$89aWH&banjMvy*DJ(Xb*+RJ3g|LL2Km(3~E> zekVr))r&>H^Mi{)O@zy#BSJsE{<-_~o#)K`^-7xcx56Bvszl}`0Y6NqUYm4?SPnNU z(2Fj-ijoS73gk7^jd_TM5`(o;L5>Pok0hXE$e>Krq2EOK`})f%)da2}JKt0s77*+@>HM-C`5|; zY%3hvvQN}e?N(*5Y+H_{mylwzhOkjbfpGoLGK4zqu!O-xcj;Vjkcq|67cQgrJY}qK z0s4#*2* zp>hV3a(VP?n0+`kHO@WSTt7EGxNq_>Gjoi?{ETg1h;?0(x*+MN`vfdL=Z+S{BZmap z!%aF00tGDNRMPZb;F_OmYvquQR3(RR-@YBa^4$61MuwNTkyz;v0b7LU^rvIV%mFiK z+^AnZBl)u$QlUc11hI{Md!K&u6gp~me---iQ@#f*04uhnfA#)}VAJDisD1+0dS0AmA(T7Lh0#$j%% z+Lv9GcHHp8<3Hk1F=PCv}8W16d+Z&))&z_#NB<+TyTbab`WA+<#tj-^)f7u=wtYFNn%8YB4|-?>=?g zpWrQgBILB`!TM(R{h~=n72@>p_5F?_0k7F(@l`aZnp9!y(fbyDQyb}`En3p_y|kLW zA3ASkSI?=t%#FxAS=72qcN4$`P4e~4kCKf^>J?aukcZVZhM%9GDQawNR4a58{-}R- zk0U;l_DowbNv*#wwAt9IjeF=bdb$^@OsKYXEGsi}gz`Bv088B`JJ=24l#)ky7E&(tK!Cs z<%K@ci?pV6LnT@(@a8UR6i}G^i+~TiEoxthpFMj&_R5R<`b{a3T8j+k zTBnl$C40X;+HYl$TaP;7$~=kMCeXa)k#HcN$w*z_*SCMVJ$m%0c2XwzRN?_&Y@%?f z*_KUvPL*HgR!Y{8O84k+oUiacrc~LVst_%`&m09kx*w6FXFEG$)LR*1=*yvQpkX)6 z>&2-lvUS%ky@Ll2URA#+e$j)`h&cf1!QY&uS!j&?)<+);IBohed=a5rR9syA*TaVb z%_8$dC$V0PlgL}aX+gW0+lNZgN#E|$DhI@MWhS^TxUKJoEbCg(v>V<%p(YfCx7a z506?yA+zohp1ydb#k%$D%{IQ(k?*wPOMg*N_fXcbAwmEC{rkZpdq_Ax^VVK_+H{Y1 z!)LwD-wn)0n-HT_txn5}v6W%%&2lQnx|`@&3yKl3gLNWk6NgF{esp=bXLvun3OW|p zoh|~j`nqLUx@D%jAZSaA@sfR1w^ioheukV8&<3zOYU;zGzl5D*zvHhcO(ETi038;y z&+c03b$a>%ssK$-740#Jx%2JaT<23x9-v^*1Qm3k%Fhgnpa@qjEG(Yf+7o@LDMhC~ zMW?JKkng~OGbrP7sK>Ontrod!-WxZiTp$&2d1*1^p;L33emMF(tKihsP->Mxn@MZd z>&^QFya&usHX?x&qF?wu*syZ|wQB%SB5sm}aT|!*H|_mS-(NfmerO2F!`WZ7PK=Mf zf;oe=AxEotGb*DP>T7MbdEZ{Uknr&E>Cflr^%v)+#V=p>>=u&Ty>lml_pyK_g!Cd| zeIp3%<v84+U6mp+s42UTjaLq7z+za=?UX|D^>-!XKH9@AiXtcRc{fg+OAadCZ!}b zs;PfNON(Gdqf!pn&@9mJzEKM)UL7#UnEeP_5T*+T;Ub8NP^2mbPly&&bH3B^mfWRo_9kCRHRg{KM=I&jaUz zn>B6w4^zb_uh8`u2~h15ySln8K@12|#?y&gsamCS>wGHN<&~8B^rX4!)vP-=)U(YF zH&y!$ed4}F*Wtc?`%z{)SEY;08v93jIURD_lAay9WB|e;U^5ZnIM>e#Dk%lTN{fz; z?yU&&ODe#B&wu~TRUWiuJic=WJW<=-W$u~s5+^cceh$U zysxL{u#=P1C8k*U*sC-f;IN@x(lXt{!^69tT$aaOg@`(d)c0Tsm1oR~m2)}Ol@Z$03&!4yX*^GByC@x|-eq7SL zx4as$5Mn?b*;SxwD84aSKQX$Sd9_z}l7r8#_NxN!oUaS8m4;tl-Eikz-(NI#&5K2f z=T_xjLPyvtJ7Q!H)hBCeiY|FR<23I(=m_co=0{) zv*sf6n>Lf+C}Qat*yumeI)BqfllxT;(p>9mfFB{M(x>#rCP zQo`BW9cI;e^=r-pR$R$6ZmSV06EK~ZdvW#wvqvPWX9RO%m88nx$y{-JkB*#*;~IHc z*y!#ldR_4?Tec8ADjyPz0BTk3Ya0e(4v}!Rbara%Qg1}6^E&5%p(@d3?}%+HFI>6e znPbua5@k6K?Fwk+88oqw&0D0Y)h{okqY98+hj8#2L}u@DTX~2epZfIS%xY@?rlBW) zj+&g(`^*$AaPI6`^@|Z@SU3Wvh+e@&I=0*L@tVy*m+5mLD}jeb7;b| z)_aX;!rq02PH~!Pnhw)G(qSTr;P`^+kN*0r1dM6VsoR%%DdK$ZrXWvotIb!?R9X9# zWw6w7vf&TG>uEQQtOwpb*x)|RW!!oi@Jbp?q&dSV=pp_7&~9F|SG-rC|t!xX72 z@)4JxY12RH0BH_n-5Mg|pmMdw>8LB$6AurW#&uVrREe9?tLZ|}3bq<)jeW#ud!)YV zT7OL>!RgRlTJ)1OZ3}Lr%91vuUrIY{JN9rmVL#-{>x)EwfU8YIL zHqFcrcxu+wt<=j|HV*B3XQYEqzd(}e%n!OmWfUOYEWW;>i}rd-!g0e+j?+fTasyqU z=<>jw)8oba1=BUN?i?s>|55(+iBT!komxvX1*>;l&H>CKA{r~#Y_0>N;(_4zb8b#N z=e4ZMN^m*J5Dh%4mDlS*kyUV5sD3}dOgJMkcSdP9VKeh{KijHYd3C` zA8bhEg?dU@Aj$Z~)j;L~6B85SGd3MA0I{S360$>Ew8;+QZ0Y>=?V0vy6Cp{0M&Rd*K8tvy6zk91%&T90FcL>XFduH?ShZ?ZEC55GpmkDh-*p6S zbH1H5O($}e`vf3}9BA@1nzn$$#ak7Dhrx}O=L?t3({#Gc{S~e@yR$Bt-x;(MFdh9- zG3o_a%sQ{`-I{S53YdPVGP}+Y_lbaP_9hga-8_1CE;(rP&QPLmLRKqf^spX0e3nBa z@4G(-3ro?s1QTFsCMnNoc!E)F+2+x>=_mwzJOt&5+1W({N5Ep%Lzj}+e?0{I81s)4g`;pN!}>vghLqu!8~EbPC{Z+ z2orHYcMs-~`ORj!UMc8R?deq9>PyH*BLh^g#d*| zVHx`&2DHqf>?&y`rbZQr(l|VmEf&-gP=<_6 zKufCM{<)@B?IP}#0Hg;G9~ywRLmi*Pb&TyU^U0!>PC?I!Yy!veDDv2!Ot*6NcNf;s zE8qgtAXkR%c{YI<%%D^3tF6S{qb$`oryK6`b7BB-CP@w`G>CXtf~x7ushNaajDa*k zedviUmwxBlS+dDg_kg^B3SEiGJI}~NK_P3@@(_*!DMz)HAUWARI5@cWI0RpyggBxx z01kY@unj`G67pmFjvY~_jGBq&WIIvHN+Qr;ZmJ*UM}_88e$jQ;vys_Wlw8uS<57f& z_Jf`&Llztz)Uw&myQXgpBam8P)4gZxKr)B|!`yaPhGoPk_wM-1F4p-ioKoCcMH{yt zmA_2JC^-vqwxTCpmY;j<<-e+)WAP056^|Z`d4IYASH)Ls(XHBB{on3MWM0Hx+7#@&YRXz5W#$? zg@&{cw7MTGY^Mla)ZKCs*k54Sr-KY&7H2huu}b<4L5bSK)Z(V%5Vh64y|oovIb%KV z9)sFL_zX@V0iA|IOc$7jimiuWs2E6DS>+#bJM5g#7 zsCIL5a>`*k33^<`ob6h4X}^Q^NWr{u7nCFmeqU@5YxXCnm;U~{CZPi{h>D8Jy}Ev; zHdc-mP&q)GQ}|AI$se^)FytaG|Dl!HQi10(TD@`C^_yB+wFw7!(Xn_C>fBbjZ4g+4 zM%r?N5Jmh_TQRDb=&K6TZvRkb-+DZ|*)7{N+ZEtxkEm$j#>Ig*&~021G-f|3RNXhu zbkas_i#g@k0XODUBw;kRIx`4(xb?T+e%qDXW{k0i5gJLiMdVMcZjgw>oIl1!a?i!+ zYEyJ1QjsfiITrfSvwGr6g)(Y070DNDIahjpM>pm{5}m*R^dz>O8=aXCy-9o~yTdl% zA%eGYBcsq4`GjlLUxuxSR)KnX6V$hxrwy4ch0JpU%Nn1W;sGt>t`eSAqxJ04k^})G z!U4b2l6{;dYk?>Ru7h=gtT zLfht{qoZSicp8Nd^g^!e&)Hc1aMPZ0)YlmiCbCbP6|ONo9n`O-zRzn=>A ziwE?L8yqtVowXNxVwn@{$+fH+mg$(}gh53uI-^|PUDK}qe*L4033@;#U#BE?xj>iF zXAXc0%=u9T8i+P?q|LfO)TaPQQh}hpkdcguG6hs?2)VtJrwtWADn!82)z43f*4P!F zT)+*rnG4%3pQuuQTseigX*!_2>O~na`SJax7P=$%K4rIu#so<{8&hjs>wWOd}xFHe2-%cI6=z7_VEe1;$ zfZE^>IJKMVE)lPmZ(E-?RU^T2s&QQl*cSQ2`?Sx-50u7rBD8FXb{0zWjukv)teER{A z08^D7_D+tCeajf*&Fv$omQ8;um?$E`Gu1vZ@%;qbnLXRYOS8w$m2!wg0D91-s(40mScK(nf~e7FsEj>Nz1 z9Pg=W@1Yf>gvR9ef27A@e$Vv6YnTIz_v1n^aJkE1^OStvfGL@>b#I7azSoVbGIHM1!!2_TQF zz4ebj{)m&$ev+i&VrtXTF7h+YWR&qN6EnGsl2c$lmeb;{E1CAE8Nzk!d6zBd=qeb8()4GB zn!-^Q%K&6h0~^VF{PE+*37k796s8S5&6HjH_9c?`g^9f6%s42HDwxz&z|C~yHi1Pj ziQT*g-Wa*1X2HMYoZ!>gwo2QVpeHCbY$76N-Is+YN)Fi$CT8w6!l%lf)Zoclk2a3b zNlHOQE?AsVCF4oxZSrV+3V?Omk6d8=G20`t`VGGHJK%F5)LekENVISb58)YqMyCbS zMSu{t%AFWP#>5)5v}R)22_L!t76VnTDcRb z0FD7&@Uo5&XJ-Flc=>^KD)E zy{9JQO_=v0XI5z=XIm8))*E1X)kl0qqw&pt)bfF)xMJ)=b{@eo)IeK`OL zipXLZZp&e!GmR!4hcSyvwpk>>lEl*EZ~-6IzK1O$ybxegV1yo2mbPv8?l?@+$cQ&k z*e>JUyLW_=p;S_d)PR-mPNHG0QqCuBat7#q#>G;vA8J2AC2U<;9)>Hind_xqVtCX9 z*Z+x5D%|)#uCV{-@O@GXSw9uAjIox1Z~Yo<6bE+T;-yQ27~|Drnj#ls47^ALABC6a zU5>|XM19Ji$jNmsuR${-F$QxISB>dkxoM{KZ9-F1H^5aXEPx>Fz#=t8vuxqdA_}agIqAjsZ)xOHr?lCWo4CR8F6;EU!!LR0%0}W% zNm@+@zJ3_9hNMJH7J!)gl|bz}nJuZ$1@KMfUWO~bg1ivETDBYE1vy&2GzafCHU-#{vvc8Y zbyj1?->?+%=m40Eh9+s`4~&?RCkVN{laguF5@k1%eTmq6pv_E!Ug-gIjc~v-%hXt6 zMfVEZNQwxN`(S4|5A9h#{GywDzOAKH0QUt<2`sH$a5a9Fj1A5%6TKDjWbjP{5-gqt zwrA5m0eL_t?6seCs8IjN7VLmo#O=Qu4|LdLM~+-ZnIUecX^4-8AkqS+oqsVpOvzgN zAt*f$5o2U#Ah*sFB8`J6_dmPDSREbaNc@oJZ6;bv;!uYa?%vG;mhEakORgVtJ9Et) z*!iI31QGQIZ-_;DxR3e8=vBX*!z@#7_!jszyow>>-bW5VzQSAhg^gXS0dJF@KWF;+ z<3|jtArX<|!#1N-W_S&+AcGMev}#CWk*D4K{QThOlN)9g5KsXXN!G0l4jAe($)ENm zjt&9<3KwUBz#fw^$~Wt0z_SeMCCB0$>O53Ky9V0>Eacvf=?f-E?w=osnCB(2_M8X?n8xKYN2Xl#8IN zVoJqE%5!xuG#iXM4H4%5cib|(;`;gXf58nB6p?SyU-RVg<2$9?pbm$rU|Ifhb|4RX zJr1*C(gd5ByDAhPi2;0ViFgwS1&mzYX<_^lZ3ZKOqfAV1Hf>+5`me3UIFYnPEA3p$ z7Yy|`?LB=3qK^Xj3DltJE)P)ycnvrNPbFi#Pm6$5CYTP*p?Un887xbvEFs1uoTLy= zFw-evlgPxiw7B>Zs049sKt<|J(V{^7iSPXQ)DU>fJp+ggJjaCiP1RD+ah->bwuG!2 z0(^piYyhg!4A~HD6rsBbQ)Yp4i)#M|%*0vn<7NBZu1&;1jF5moDx4H(xIW%~tU@?4 z*_B$+osj+bWV-?usU_42_hHbS7TdLiIVCd{n#+26`bH4mgzw+=pq9wM?r;SOlV;W< zZewTHVLFVbYf{G3VMtEiipQ})_l1?_wD1yECn`RSMJf=Z%+4+pVJI@SU|%CK8hEH+ zlGw2BvIz;P!AC<}Y-sGQ>RS+=Ni>8f*kzht^+AGv!TE5rN}sH5(b?J9kn!{g2Ou9) z<0mV+u}D9S8p!Hbg&pZ}H<_F*o9LD&$~aZveIsfafJbRsrtFO=qC*AM(XE>js+QcHatUNVP+-_lsXRt zE)PM7PaY-FGna%W=D83A>P2iJMU(N`yWsN%u$mA?jlPH3x-qE) z7&A;mKF`V(!qX%`@6;AAym#*&X`oS4ZspIAR%pP4Vza|=gd}-{hfjzOwH1e8V=cgj zFOQH}=$WKdM8Wf@Ex!CO)^>d}w(mJ}4;(SI?GZSpAa!;~r*zk}AGg!1=~gUu*|E)CC{%NOxENR;H1cms=)w$`g_B{LG)hBgG#-vcac^ zwSm)fKRNjDb+b-xs&iGC=%_`m{m&H8uxKC$(=7;W=M7jD|6t5d^&o>&XEvj`F?E@p zr+6C&?=tKBu+G|#6rL`fTrEoB(wt%Re9};rQ+kwL%3CxQ2id8}Y_N;PxLtbQwA)EnZ1Ypyl&7|I{f;cN-977V` z8VNs!xF^bJns(_HGSUUws3mp>vbDqkTIRbINCUzzF}M@^Y_?eskK@8!1}(LWU*+fs z`EFdBbVJDLazIHDdqh{xLagkEB4X8KhNO=mp0$zIY(!lkRI8Uz2Z=8hBAZ1M>DVbj z=-87zWzl2;Yf`wF@wbN=-$iV>ejK;(yo88iIDmmc0JqK*f|ay$4t3JOth|O{1|xe$ znmFLyuTRleV0!&ahetC$OeLm6GTJ~h%C+Ffmo`adw5+@fvjuTKkckAmJ7|Lzm|OjF zIzzKnw_+JPmYnwzRagPe1`NP4y=?qz-PStzgSYj$xVb4nY3g!>Swh0#?>1sg{}qGB zp8o#W$B)+n$W>LhXPe)Gm$^o#Od0ccI3tqD2$JBq)@<|XO2JV*tPES_)*U-opzap4 zpFaIXXBzjX?L(>xMwgA4Wsp-I(KCqQ?x%M9RxtJ7o0fA=yD$SQrRF$9&>Xf;-xX6YIRSJp;3khLx?Aa-V zmm_)cy;0SmLC0>asRAB&+r5eZ{@$jl;-R4JwY(x{PgIf%yxr@-R zKA)@-3Wt#pw}?MHtmarUCQnw$T+hC~Mc>d(8nES&Jt6L)tjIkNKE zO1N@n4)5=HjyJq1Qskf#)qT9nUljk?SazzIvRkVz*V+WUm0AOiGmSwK3;%|@D9s~B z(C~n3Ptl#=;i>C)^k5WjmH$<5e2rFo4dyp*RmVl04{HZ$QAoApO{riM!{Y)Vt-WyZ zVoqKMihfp}3==;={oG6t=X{PTRo5TmLch&2`ujz0?yE}4AC5d9mYCSM)=IxaX|qxF z$FRQaw0}Nk{O2vPS3@7ma?`4^%rB`o@Aupy{fo{1M-QTbW(heiWZ6!Af8p#*A#fIz zz8PMC+%i03iM&mlx^r%@sFMc5jLhhXzm`;Na$X8K-KM@9B_k^3HYJ!xpGFMQPL9Lpi%nGWZ$Y{^qX0e^{38C(lZDS;OnrLqhTJw zz6i+zQET;t?gStz1~nC*xalaFZ2xULO016Wq5OIOzI?iYLUaat^mnNK;-;a09J;87 zZu5(`;ut|?+HQQ5^S^G|owkg*PcPVWeoPAaP5*rd=y2&UrblO>2gL(1v;9LyPv-KV zbK3mfe9PvZ3R>6cIM9vUZ1NJrfaiG8G?1hU1TR?Y<4pmdQqH}h5vMr@bPa94DhFHH2pyortE2yU_2VDe z*{tAKoL6Bs1zG4*&*FU|=8cH`8@puXnV5vxH)mSYA2_j zrY|~+MInN5=4GtMxVdQ*|8;Af0Y2RSSto!Lfxj`wfB4Yl*m4krTexvG?8h%y`ytLf zW7j&0UE5VNw*8&NY34GCleFq7>r{K z+ViqO2HfPM-Ttp0Ca>$ahLWuT{sfI(y?qq66}5I&Pt|Jz6k;h(0sM6Z1o$69fS#~KZG5`j0}cG`e|Q_S*7bNV}9Sx5jy!&N8H z2A2t$uBTfLDhlJki*X!6kI7uwmi}uZ{4GbQ5Ui}t50j`XuCb$-_`=aeoPVSsqZD|B zu>4nrO@SzEWv|wZE$@K^ocIR7F_=E;|5xev#sgd|=?ukt3RXNr42ub)BL@;XE&iLv zcEg9f??(TD>sqel2rdC95R2td-Q_~sjWf%PGjGXSk{f$pLwAQfEv`4+fx}dUtKqmU zGZnf3aIUHOKkqvC26Ouwhl*1DjqO||ySOc{~k zmfT0b$CRsA(U?^kOPRH$J18mJ=zbTAdI}>II{bRQc^W2W>J28X+2I12_9;mJbVxwRkT{6ZqJ@<60gfOtWqe`~+hD3$sM|7gj8 zmyp&}2z!Fdix-py=v*eue!?YJ0z9dN=NE2J%e9gsr=w7^_8QqtbjyNdq(flT{x{fP zLM$Anf>8xr?LxpO}=3{-$@Kj_T zLizWk+>shg`={Tmq(9xP>G)HJ4DRMYvS{Vts=IIu@6@p}w9kNUj6sM!EVt~=1{AWq zeI!Sq;_RJ;CuIsX9O9N3z-naKlP6EELIZPW6h4Z%uiuQ@@(tlzn;Og;uK)cYhmoi@uF^W;imO_S9 z$TU_bdOp-{1q=wtNak;l;TE~_I);FjGegSxIGck@c|+iiMS+4X@a71I6mAKvNSeEQ zW8=;L_EfkXjHkx=uCAzFlVDcv8lJ9l`)o3`x@(kQq&VItL z&!WnjZ7`A#rytv5;b~YU3;CGm_NU1f!d7Zm= z@A_x9+m;Qr=eOj|G#eH{8XIa(Klb!kl?``JQm*BoE|gz=7%Yu3+SVechv3^ThvJly zH4%I`e2~8ys%!;uiQu7cL)flPU^}&@?7OYIn;S2N*1R4<6OZ8z)+{IQLpLSogZ}xf z4UShA8D|Ee{RW{k2LWOSU?oo89s7DbN$A$kk=)U8aM}uDLt;(^FqMxM#o4h^oa|*l zV35MAenfQujG49K|O>_DXdflsQAhF4{G168Yan0FpZQuEluu+OG=y~`N%b? z4+=&AoF*+O5bW{d<=eS)Lf61$MKs&GQvf%$VC3YgR2hmr9$5;WtKeoFXCdBz02iMg zjg54-UPEOdC4iu>aCVtcA@YAI%a5Kc z&TX3(f~g&@b$J=#X^gc3y4osj5%=Hj)ULfCQ_oVC8NuIG7DtmHdo= zmj4e7BIdBHQzeHyw5b(}Z>?V#D$n_ewH^RPAP1WqjUtdZ%^;X5lNfa!ZR$rT`siEs zuFgzNGU2uVamVL~?7`BDYnE}{pRlr3gEo^vsP>r$+3$xF`Y@X5$4l}!^E(a86LDY< zbQBhLc6RcBttA{0EY1#@ZcS9>1NMVu{~~wMcc41knWkIQIA^VUq15+Tjr9$PDa3S+ zlg4}sz9(O=`Tg?S4=!8NufJVEx_Nth`v?bXW0pn#uixk(7#Bh%UQ&{{e3@`H;@u@? zp@G)yRFMDJU&?EmnhwM%a#(d59z8=qS(OxR7vh+3+b`PQ7D>l;lm7aS4I4Ll<76F- z-otf{aPe*D)DUX2A@%J=zbPJY@H{!k#3CIWLv#V0H<8ix4d~Lb5QguwV-8XPIV*Ze z6{Lyb^W7FZ)8DSmvbT=Jo5zAsitgp}j`v;?<;i;P`=!jg5{$1%EZxkH~D8 zI}xWYb1XUN#?9 zHg-JV=p9MDlAuQl4-y+@i&UIMuT9nzm5aG@z#M&2%asf4izhj6;)&KAF6*CXGZ~0; z%f;|y6}vJD2bR5wkh|D8jqN5gsGrb`DsWzVUnW$p>_N8fwG8T2nVrYKPL-m>Jbruv zn2iVIGp`8?tz2NA%?4a+17m6H^ft4sI z5L>AcOkv8JcQ<=g&rVgWb?*18p3Z~4Mh^ z6BMH(B#$D&XUxEGpgwGH0@9nCt@!(hoTvT*_^gbTlRyB zT{AL(JJ;3~Q@|qsU;k=Qi6yL*@t~X1a6Zt`bTC`R8Ab^s^b@3!ScWHDOBCmL%b_o_ za2FDZ(~=0PI7&cHf#dMBA6pp$E-)`Ik2}(YQnGP~?l;eP99BFdy1bA|v|@O6;!y8V z%0gi+eu2Faj6%u21rsP222Jr;Y!z&FxCe9>aX>jQ+yx|I9thykI^Jw@7s+%D)@qoV z<8dmBN+W0Yd5v4IVO~OhUE$5WmD~623$c`8q|~@j)-oxv!NR+K+mYv(im{zKb*i)r zRuzM`ob*$-zdXmJR3GJx*rt%U%v59!&eVpGlS{-%OwP+g!`#NmI8s+7IGPtu{9P@X zCPdhrf|wnTwl;uYt09^rJANvsaAD$UC`LO8cbh4m2k=Wyrq+K@zKRnIQmEscK92TX zKr>e)ZhM>yAoFW-sFj!;kT@}jCu;9-dwwq21HicU>KP)d!g*bf=sVnB#P|l@E`7Lk z-1X;XhL3@!5p}>l17Pt2JX@$IzL}ZUEFxNzl3g>D6(yE9eH%ldC8SwJcy`i=i6=>2 z72ab~1PH;p{lhDQd^dgqK-!3u0!ZZB_U|9ni^s9dS{|tC@)%HS(QD&WWgP+mDEK>U z^GQxL%AGZ**Zn4zghPUfWEw_(_kg%LaCplAvAwC5(-Az0M~+_y?fV>`$% zU*N1^dr$gG=XkE)HZm#k<2)&{APU>qk8#Fl;1F_zfT-DIp0It(mX~Yx+h0KstHT`M z^2gYX#A~n50#JA}6PpkTS>nrwHf9KsK9Ec6VxZk-oIpy#5!eA}b&aT-?m~INAbfGSIB%9-F5)`M1G(M1$7#5eGoqj;bYOo{0umd92p=b9=MGcesL<_ zOgF61mjTvt$d{tf;#;IKX^^(eMzirCr#ao=hR3VETo+Nti3jfFJljgKho(DD5Pjmr jZv@l+hkoO0X+Mwmt0~i;8(#q#Q>YiD&nKO|e*b>~9bE5s literal 0 HcmV?d00001 diff --git a/docs/evaluations/Docling-DPBench/evaluation_DPBench_table_structure_TEDS_struct-only.txt b/docs/evaluations/Docling-DPBench/evaluation_DPBench_table_structure_TEDS_struct-only.txt new file mode 100644 index 0000000..bd8a0d4 --- /dev/null +++ b/docs/evaluations/Docling-DPBench/evaluation_DPBench_table_structure_TEDS_struct-only.txt @@ -0,0 +1,26 @@ +DPBench table_structure TEDS_struct-only: mean=0.85 median=0.87 std=0.11 + +| TEDS_struct-only | prob [%] | acc [%] | 1-acc [%] | total | +|--------------------|------------|-----------|-------------|---------| +| (0.000, 0.050] | 0 | 0 | 100 | 0 | +| (0.050, 0.100] | 0 | 0 | 100 | 0 | +| (0.100, 0.150] | 0 | 0 | 100 | 0 | +| (0.150, 0.200] | 0 | 0 | 100 | 0 | +| (0.200, 0.250] | 0 | 0 | 100 | 0 | +| (0.250, 0.300] | 0 | 0 | 100 | 0 | +| (0.300, 0.350] | 0 | 0 | 100 | 0 | +| (0.350, 0.400] | 0 | 0 | 100 | 0 | +| (0.400, 0.450] | 1.96 | 0 | 100 | 1 | +| (0.450, 0.500] | 0 | 1.96 | 98.04 | 0 | +| (0.500, 0.550] | 0 | 1.96 | 98.04 | 0 | +| (0.550, 0.600] | 0 | 1.96 | 98.04 | 0 | +| (0.600, 0.650] | 1.96 | 1.96 | 98.04 | 1 | +| (0.650, 0.700] | 1.96 | 3.92 | 96.08 | 1 | +| (0.700, 0.750] | 7.84 | 5.88 | 94.12 | 4 | +| (0.750, 0.800] | 11.76 | 13.73 | 86.27 | 6 | +| (0.800, 0.850] | 13.73 | 25.49 | 74.51 | 7 | +| (0.850, 0.900] | 33.33 | 39.22 | 60.78 | 17 | +| (0.900, 0.950] | 17.65 | 72.55 | 27.45 | 9 | +| (0.950, 1.000] | 9.8 | 90.2 | 9.8 | 5 | + + diff --git a/docs/evaluations/Docling-DPBench/evaluation_DPBench_table_structure_TEDS_struct-with-text.png b/docs/evaluations/Docling-DPBench/evaluation_DPBench_table_structure_TEDS_struct-with-text.png new file mode 100644 index 0000000000000000000000000000000000000000..7bfb5b0f12dc03cc32f19e7a45576c81fc4ef6bc GIT binary patch literal 20196 zcmdtKcT|+GJ+Be7{~%5Stv0P1SLvVNs@C;T9#5Of|4^LIV(9s6-q{w zC>bSZL~`u8t$Tm>z3#q$bdPu681L2?mU1}fJNw&vg}LUM>w9uT=IZX9hj&sal-&~7 zF3C|So9|O7n|goVhF5q$b@t;gLF>!6tmV!0tnKbt>Qb)Xu{Jk0vo9f(#pWh zl!t@sEC(0c(Rp5v>bnRZXodE_G1D}JF4=We`a>6 z-hb@Lvcg`M&^>-u1J2=er#9VpM<(<N1{3XBBjNL7X(hIjlWPnwwpaVJCz*ET=jHFJ*-1s^6A}{Q$tXHGDmQ*nT)a6_+}&7B z({uNIQ&ZECWwip^6#UY)!r+ul=)I{N37=E*^TG}@@wfl_J%6frr76!!KSec*<-O|H zuV2r(EL%4xDoSmoI;aqN@zMUhdtYl7IT~!Nuc<8OE=+bMeTuqTFnf8V`f%~;jMVjz z^L5NFZi`>su?Q>4o}BhVhm0#;3@J4MoN-R!`1Qz8gj>s-J7uBDx2P9J;vy0y{a9vZ zX5`*KJ;1A7?qOZ;H2m?{qsNaE6%ypC12`jClv9)Kx*e{5`t*sq@j0WY<{pvR7f%ne z8)|(heXzym%ioWZUHNmb8yhD--hCu??@7tZjt;fC(Pk-|un_O&B;_Q7#@LyK1!b&V zA;(n1<=gv1%rZ`sodR={U1}$#{Cxug0?6Jmh&bvri%XOr7^{8v?$Y&eVU>YuKOg+k zuqi(B5!Insqi!Dk55v#-x^FbbULWY|^BFz#ZFu-zT`;edm*{%#<}F+BVw7r@p~3uk zo8+}?mDUq>1>s(vo-M+gD2~t46#< ze%$Kwn>(uc)(N&#-4p$zX4N6=QdNI+^n8wRThCKUQT0nM!d~UOuH`c&Xjga|;#AU$ z@&5ty0_;`r**m{8F|)8py6<6Z;p=gZTKYbeB<@ZfEb3M)EG#_Um}AzX$*Pomqr#KE zIYVFeeMN;44sm&XWomA^Y){o`_{1P_AH}-nW6frara<$ z^4iH$r}Fc6ZSTsrX>8SUOL?$md*a>qPhpDkNlGc1dUc0IW-3qEg)S6K6*tPuip1dF zjAn;FkrPnXa*5&4$Uk%Z_{FrM`Cym!>1lJ`dmqKTXRBG$;&rOL1znaiT~{Xf;zZ6X zC+A%Cpox64?{uPmeYoTFU)$#_mCkn0;u z^=@?S*dm|n3oUA9%tw!kWf?X%;G^;rX|Rk$`8dyRw+$DU^`#*dO*8I6hEF+WT0W|I zIBOp7T3=@QhVeLyzVcH{OiU^(&yVm)M_h36VB)bJzTvcDRNOTLYfe5`bk&V*4*GM9I)_em-4F1u^j z{xs;!wHSOR6~`oEB`+(gnn;}Y^N5HDgRcB6@s($s z#rJS>W>|5A&<9b|PW}^BtN1tdghswKmv=+x214F>@7li~QSmHL)Y(RgzC2wpa$H*; zORH-6)S9Nn_t#&4#k#I78+_RKtnC#9#}5n)q;XPnb}rmOO?{r%u*p0yTerqP7Vf4} zY+CG7o6J+=%dDMmI}w~P5GH6-ZEtTMSnRUW-m!PzzU#2b>j`o(wb-0Vem5PR=sf#x zuek<4WNGD^A8T8wrqRx~n=)kzf=B5lt~a%OA3^BZdZNnXFfU7^OKq4CpZM5Fhs{KL zbANq=s7B){_7fsDB@5k7r`*<82S=Nef)ZIzSdMe_jt@>wrtZ10D5jXC)La|HV>3{F zK%(L0iK|Ri>(6KA9adLYxidpAKRcL!@G$f3+s(gz->O*TXmdT{f<{A~_TF%(NdXBt zgdlobD=9uNT7>s|ui!v1 zxFotd{ZJ;9|Blw=L{IF#efxNETz)KylA4;D#LuZ2FF3~?%kk>MM`LWpTH02M=R*1h zY61`{#I|nTTBVnfaPiU3;ByAvF04 zADJ1b8EuGpw(}52GEBy3_+un@In$9e><%*%(kdI9d5h?UN_ zUNSd-Q|?J0V$3J&bLzT6iYo6!SAjBFVX^a)Lb{FwGJ!x_pP<8xPCoU^TN!%KIW!C7 zFSso8XD&>1DBrw!bNP!q<3A?DM4~zM#DfP9s(ns-l^q=PWhyG?POn*4lF&}quV=uv z-CCL%kbkuEz<5Pcs#=Z=@{~_bPR?Y()P)qyB0&{Kw~e(NO~;Xw2RJp=;rPdn9a|pH zsQ12ZH`>H3GS?(u9U)rs^Z;vYtESWSw{PE8#Y&CPl)}nn_lT~(f$8*5Or#(v#@S1! zAQBfU6)7VbQ&jnO0-VPhD%#{6t6EEAg`dH@R-`GGFklqB`PZFz^}i)9G@zGeqjeYUdUF#+B);RY3CUU z7Wr5YCIdL)$mD5WWD+^VRmEKMJJ*BHN$fiP0sSFD&+Mv6<{t z8EZ+Ax3;#l|$I`%t}d0EfoxU)veuk)ID=aL(Pf`!LhUVD(%^{RMexX%9_@fdft@QCFTf5Esw0 z8e+l8I1UEsH@(%qD$!7~v1YcC?yYo-xeZtVW;mXf)lsxOnj~mF{DMFOgffEglGOlGVvb&k;p8`>#8?;5m*PYYU2bmb%!RT842S>yd^5 z&$JLGA-hRmrsf3sipV0YgSh6}dK1&uB_O_gt*IJ2_v~qfrSLk<8zZiWwwY>ZdmKN0 z+SJp9;MBuDp4zTM zVRq+SSM3MtLX_)87Sx|^a_TF8KH3=P1!%#GNCel5cUhUQx^O8;A>lPpOLX3Nn2=qf zT#O{ktv9d4odAGNN)*^m2AkjXyh&i*t8ZYEBQf_&J?`98yRL4(T-l?AW&oMrAlnx-R(y0lpcr_*!BM zRoz+=eL*cwgQj63PD%}R_XBU*sn%2P2^WM_mzP`T>m##}z$*2XW<6foNwu|QW&Vy|B6?Nb3+Uc}nrD*|A5S*wd zB)r@vJM*drKbeGt6j+bko$0UAc7@^TG@d$jDpo~Y5~35R%1lq5jBJG z@W=J?BW5&qc6L@X{Sr%PRDPqp~+Dg#%RQL9W#c;>U$ z*x&gfEMc@Y&8)Mtvz9fj@UB^bMc-p)#YEq-z7AO2OPzykw|YNL$%?M|CX2*~??8ey z!9fqb>0lozng$%}=(ww`eO*lKzCllMQQ4|QV{D9BPjPuym&p4SQiQ=E8^^WBKff)< zo_}!>z5DLZ{c$2zgC1qpF_M1V&kk|QSkWd(IEHGuF2xj?py0UTT?G$pLsTkqL4{S- zoh0Hodlosf%2ws^Ncud;Qov3mKRGv3@{ra^uGNyNt7lCf$F4iknbcYV< z4tjh3+k-KYm(Vixza)0BkjD1wMO!|q|O9z!4+?ZzUO=m<0n$u z<@X&qVw9+qVh9AL|Fy5Lu_#n6COcf#@j0+!PxnFG4mlY+n_C7 zH<-6GMLlcl8$=^yf5r3?>{o1o*XFG|uX(Q>$p<(f(Z{aGMH#hi3m{W7e8`Vk{t6P2 z^JCj12Urw_Q~4(RS(Rmwo+_IPpFDY@TI`&M@Eu}o^vwGETfHkVWQA8^n^Lq&G#l#J zU2x!XNF$lr71WonUag^tf^#L_DSI5Gdc_+7U1+>PYIF(}P|&1RmG?=*&u=1u*2doq zf*N`H!*Jly0FMNM>j795xVdY$GOoenm%)- zsDgrm(WZpHv(!@lY$UcYIE{0;S110P;7 zckaItClfZFBO)S#9NeW-{}*sm<7zoObK4WK9552;5&br&5p5Ka_Ri^lpkvp_SKus~ z^)B~fObg=CKNXUF?d5Uot>5}e&&Ft&UH8k9k_`7EOCMrfMAQ8cLp4~^n4LkeMk5!rMm_!4rJq6DQ6oenXtc*kIWlYkNGjVS8jphJu(16bQa#fPSYCpk>i(OlW6zY- z)k8I1<`mr*%neVA(uukXYpAhI*$PixDREuPw42hb3l)%e_^vnM5G}gC7zN-p?sSu> zs;o?WZDpYqp(qoiz`2#&IU9z3`=b8*^JkJ1y?wtLttlZz`EZ@mIK5LQ@a%5{M7dMb zlcf6Q%~{xuOvrhY{IMFplP(7Z)I3F~_<2~Ak^{&26FnqVA1p2|e(pd$a|Jc{uP3ey zql!|^F%2WYA+2~>l7WE%q=#$%@+2~5@$x8ZmT{XLn4YFSFVR$Ldcol*r^Bz=k>FGF zc}xC#?9YB$+63eoUBh*3G=K6fVc10UwO9nD+Y2;^?O+@$lT1$7}mgz3kc6D`4<*q2^JM4HV;xr#E*p*VTD#6F8yqAtH z8T)O3LZ6y7v^HPN8vxNWRx zrT3|riZaB4>!~Vc+s~nX%V~aGVveix2`UH>b#jHA$Ut@9G&1kKgnPt1+`fl!Xx^p_ z0l|TR#C!;AL|rlJ=;+wqy3eFN^Ja!#UDK=Iwj5wplE#l!P1@HCq`@-CJ9|cNI2CE; zn*SESsnrrHU+715?0Nm_5sCWK+2>C(hFMO0Cj*=`w9T1IWhuw z|2fVl8rf0?u%ZPO-`q?Mr+)l=fr?oKzs1Fdy82kP1MI4^o?0k6a{7W)04P44j&Oec zR=XnB{{8a;o3HP7fN}ZsCC^8vr#b9gxZMoS=pHhHYUaJ0={9y(JSS>ANbz*ys%I0b z@5EjLET4ve$8W{=sgnMSe5sdE26u4`}QlsoEEz{J^JI1;iR-;xm&kxKmRQh!|iXi-9eZ7k6Ru$5ZO+9fQBX>m^@6>t@~JVIjU*k zVQ>cT=!Z5yvVo{zcUqKcNYxN}^yrbn(1$R))w_%M1?$eJ>Ra#SOE3KH;dTyP3 zZi1NF1Si-ZmvS@eN~~diGrpz?p9CLsjnQ>6lF+1IzC@ygh|N{3rLF(gM|+`xpdCmAH*nymoHxYov_)cCpqiJWWg>-ZB=CyEu+Q-9It z{G6;&pmc~+vx(5cxRJIFFM1*W0p1v$%+CsA4UauTQN>YH9t4h7_ zcMrX~V4K#pE`vfq?kF($?r$QjX(Su$W6y6O<<8b{mXI*u2ts*c0+-Go3&tr;2h|pT zhz=P`0@(|z<)C7fQb`ruHf;*z81J^RPPpdfg()*F2?VP%?`!DP;ZB4k0W@n|)9G>D zrV%deFy1rilYQo7*xSID+Ej1|!i&3WH%Eu4T`|?_COq@T`a%hilUll~cX3yok~pY) zlE_!I8mg7@9gKb4-Q7=J4~{;5`RQj)`j2n!h!Z*(G=NlVOUdfe_nzkM7T5}U!@@a_ zKKvZVpNjQ-B%E@g*buCge)FaJa1lSm3P5=UtHIivAmy8*uI_PXKSAq4J=U6*hBNAF zw(wvc`55_d@7}%VY{x(AEOg~slxb=;w1lfMimR)ur?<*piCN%2>vdr{{^@?^yP_GK zRVdN?IJGWhnRc~zAN=*#NF;Wl)xWO->}6UF$S&NP)LG@_<%Y1QMCG*bvecOlEk49HdC-2D#%hZ$EZ^09T7sKkNQS8P)zx?{m9*$03Fp{dCcQ=AVP_TVn>gZKZx|{dz#g}a#1caXZ zn~F0Ykd$C&oX=}yUp3rVBiIFABya`0fX>N*0Ie1zu_#nK3OJ+&)U)SMO{3+|=;|Sk z7&rj_86Ce7B(U^0W5?M~SLH=K$6AU{%Y?=gXR;Y>`h?Wt<8oAz&u!-z+I31`AJD_# zR7{Y&3?T3ctbZn0Sd<87Hy>YijX+jCLOT?T8>BwY8m?VEoj6f0Z+sO@Q{`8>Qw$hSZG6#+F%J{0V`P1(sEn< z=1%BJVkDpngN5u;T^GM>M>cA+X^?yi7nI1>=3M(~=wlampu-_eeoG(ya0-TQPASn(r#lyKzkV}2xg zfu{TfhHgB-gqB<9Po?Brk%<0@AV+4te{Tf8k_JpMjC0GLq837q3Fb3P#ja~2oGT%^ zWSVrGuZs}X!nrr!-?W8pR5IT-Aj7TO=)DOn^&nFBVEb zZz@>8DmLGGre(1B6p*ka4}PTLEPMq95T%kzBHDMcUR}PX3?VjS zkw+5};CAG;b~9d3N8zgs%yQ1GTbLXJ$`J+yuU(8F6kU<|4vmG0=V@@IuUTqk2PETF&-8 z49Pq8or(wJiM#T%o$O4&7De0d@Qaozjh+i;1%t7Tnl2vQCQ#6t#J2o7wNg>t-2={{ zj*uvG-_u&?@ZEa+Gnf0=CjwBtbxKZs0S9%bWL2esqZ90ufaDq& zBgd~#PS5uH$O=ktyg#Hh3|4WT^A)0Zj+Yx?7ybNEl-!cY1T{?oe!Hxe9m-YuO*yt$ zjU~)W1Q=giW<}|laO>^sk){NnC9pR>b1vW^c`UwOYRt9J-bX`oZ)UL02vxT+VNSl! zwQA`Pb2+cgwGP4sNND9D)CS1TSi7n2L0~fKMwgkY)3MG=GoXCBzRa$2>sB+ZWc|T} zZhS1by7Jc6R#anwaGgNdP@H}B65@?Cz_2u^yXSR!JV;J_pek*03ndMCJF8wC69@Z- zzNq_rbd3U?faSYLk(ioN^#a?v!s)-MVqt(+yw~J5C9LhO4C8oURe&H7jL>p~kugFi zAT8H%E|C`WH2TcHf18~>hhQ*>_RD#KnV&p+#)D=uSrTy>l@g@F%qsLt1RQ7Mi%j69 zvFPHS0%<;y&qN@+G|C^kc5e;|^OciMI5SK>hB6TsOd|Bqd>f2(XW)hf#9n%`R}ammc`5(S<<)3g+8AFBA@A;*(oWSO1TuY8 zlGo8;T3RUCm@pslP-9$~NUI>#4iZ?yPe1)cvuDp9`2@L>JS1AwaH;2-|1~?>T%!Pr z9(}|fpv+*jdxFtSK6OM&+vD590w44bH-c^A&Ywys(mE-y`ujsJzE&P=G4F4`{Z?L9 zR)(tv0t^M=7Wk#@P!Kf&e)}MW_f1Wi5ni(-SynaR!4lZa>o{_9NQgA&QN(0<>8sA2 z6bbL(RI^-J8J0=aDly%*hrt+Sy$9X}H@lrKrAq6wJf;UXi$!Tq&WkXAAU8RiZKlD{ z)U7S_n4vajM!RjKH7#Oo?{Nu2ki?GdJ!e=yKe;-ErhNNEyK?S|-ZzmRL1g+x_a;V267RA|19MIV{6;zM|j43i* zx&`&uRe#2cpuL`(ezLreD~-YdWxt(nezAry9EJ4z{Wt$-DJmIPP^U7j>i*izV`qkV zPx4a6$yJHHYLpBy6e(qYwKFrr>DP`oLiUvEn;=SCI% z&fGjM<=WM+d5Ka;MR42VjuRA$`P9#Q_T+4&`!xSTnY!3e-|&aKyU=Q>03=y?;4csC z->d${Lk%b9dxpe7|yQx=e z((1-$rm3paUHdp6k9}R3Vm*HRcgoKaL@{Pbx2NQ2+h&&B{9XNbLw|za?7v(|cB*w- zDL6gSs3Glp(AAkRQH3~{0YByb!NFK`wsqB9aO0`oMq6$xDEN}*DB#_gXCu2>wltxr zp6orDp1_T_v{sLeZVrth$gwSbfIe<%WvXOuG%3vx9GD@9xbuoIh+fq^TdLctq?!fy zx|^2!60A9uV>^X%w4=YB{k}mE`uE5mN$YDXkaD4G@xHvg-R$a>D}qty@Fj8S{Rc~d zGmL|mRLX` z41LRd%YmnBoB(kvPqDblsi-9P}&*75ZWL%xUNKa7e6L8(#`UVJ|TSy7Hy@3#wR0|Dz) zN16bJ5PcZ&VB*tPpJ4krUK-_4^qP>ih$nzb%gf(SEU-FDjly&M)Tt|EZKOy4=JsC- zanGJVSA;xp%~nT8$JQ72{33>}Wb0Yn0*dGtTvq@wiu*G6osxcm{=pjOnKNfX5^=Vb zg8tO^CB94ngn{L6MkbJGKm(QNbyl>X9XhWF(d`QxwgM`C=2cwj8W8l$hzPCV@1Kp& zw;5*}$5Ufd9V|^Kh3H#BBV%0R0z8VEND88do8TA2UFKEDrhot{LP<_K&JuEHXZDYb zj3lPR_(C}l1P}lc;Oz4q=<#?nL1CmAb=9i{otx7Vi7FYfWzP8Co2Y!#lcb?`H4Z>D zz)w0J5_9W9UF2>LZ9ha?km7Ng7TOV_nI<$?MLEp%yaNJ~qfS0?9O zG?SlC(MN|ZMimX|uP)^@C{nJQ70vmFJI_c+G@t@Y1$$})kiK*G?z#_hIL36v)UM6; zr(uwdZ5jH6_;Ct>sgc4B#-96zgy^9f*zQz6idgSxd|oIMd|jujmL%G*2hcn$p{J!C z2_*?Tw9*?!ZuIvbJJ9^j+JL+&FP{nbvMWP}vg_xcUoe3>{}8!`SUJBU$L+jmu90W? ze*F*w!(@{n`S@Sno8CYYxAW(pKXR6=-ZgvX;n9TtXd+>g1taS8VZb6y%E~QFciBo{5YLcdWL#yoEy1d8mGb zl7nI&1yj4(9xwpj;Dwb5feg6(6cj<4W|8&Mj4n}RLYWlNKQJ(m=uLL(^1mW(^Ga7| z=PYRJVYJt($`i3F^?L7$A7)JZ|6A{EsZJ3_NsuIk%T+exr({$U!KgsdQ)fl5qM4}O z`vnC!D3fqb5Z6cw55fArf=Fcyl%ti63f_=+#gpnj_)SvD+APl+xB!|7f;M7s=>HXr z`E?-AX7FO_8pmOwBJ34gOPSE;Z04yKjv^#OL$rjT^Wq!8>*QCjj@pbg5N5@~hD5i8 zXu5y44l1)VDB8)DdjEQ8(_i^3DgOY&T6|UZkA5O%7i6?QXeHD6$SH! zja4gzEJ=r%&e}xeczGyjpm82+zeqlE6L$vgN{uu#3CU7gTDm`u8SS+TYqO2Usk@LA z?J6xcO@E9OH;@6aPQbD2iSv?smh}|cM)+M-2;K>EIT@oHmj%I)yk+xO9%(a>NZTgzDywpG}6{1#y)fPdt>3;AlK zq@@*5gz|4g+&fCupGz6WZGfvITupnCQx1rKqNzl`ySzDuEO956cq#$CbJAoPB@SsB zMt6v^Z|~my2M@-gDSDakAJD>0UN9859{; zJn0f3$E}u-k>${17FA2PpN_HqXMxumA zzzxUxNl9)q)vZxnTnz2En?F0jm2t#okgLg%04inrcF2#CN;cMoYWy?MRTf$5_W?Z3 zD%yY}jl%P4(tq6PC30A&PO$)=s;q!y>W3I#x5fcO2fXOhgOa{Mdf%uFv%peb4-@2V zcK9E{Xprt4z^?Ji{DhTc2?zuN4ds^~Tfs^eLBSZcXBq_a=zB6%)zw`mP!dw~^x2^= z-S&OEFJHNmh@G&svs1>tptTnVMv1H0fe?vY+T}VbyAk-@|1b(*;^PC2TE)EPaV(8|X636cE&GfHnb<(KyZyM-oLAC=49cG4aPg1{=VdG9{pQENJod58TrWLrP|w zF-T$+B}DYLsxr=N+4py*a%>S0gVB2X@|kJInZ(NMrKe93cA!i9y${SXD#`nzDC;C#LHN0LL`vTJ$O8TN|LGdJlY7u%JyM#I7-HBaFlts$mSJ_cu}JfN?oM zPtPtZD@z)1HJf`qXnD?A_FtPDX}n6@5Ooca{gSM%(jW>)l<|WjqRZo*MYeNexwYh+ zPUQKma_0W-GaP1GvUZE`Vwe7b`&piPlo?|zoBlQIM|(uAHF~7 zAYgiE8W3y9yn`kRV;}c`LWwrRjoTJ)h$cORN-sWQ0r@PX|QWXu;Km@X5j*{H?!#{Rpp9Cb9&`1PXF5;+jrs z2+-4@U|Rp!v2Xl~tT)j^<30j1?jx?GlVf}N@@2mHmy7B1;JSw4`2P{Tr;&w~AikR; z;2dlR)FQtA92$wxNe&0;hg;Y<;7%3_>Fjp(qPY+aed!n`9&SKz9~~mPr4`*KP&e=2 zH{+W+SFlC3nBKXtIxUV@FQbj|k_oyMR%it|{yoJM-1mXD;0h8SX<3s(f>v8y1A8Nu zN+K)bH5rspOjM9S-fMv7hDicivEIF?FeoZV5q(@IvRV-sBYjTGk};zBc|w~5CV!Q~ zTLOd}W}?wjOfT#KJErRpAtp&o5^Pfz*j7Y{>c}DthY(vQ$J!fX39 z1o`)ECS^6q)_~+SjNrVi)3f^RGKI1l0UgVGbuWLdkk)}OhkRpPv_$i-E1$2eZ@BQl z7f}RLJmO>$1CcU0I;3PN1Z_EJ-pC*lM=CYteu(hu?>bcRFJfHEwRY7vtlw|Khxj4u zB&tfBvOHR$AR7}))|aI=R=<0lH)-!5oxwfBEGOA|k>E!F2ORrfUL{m93(73g|0E=N zZ|`m80N9;*ea%fZ3fyS!-9Fi2w6+?c_EpzFJtCIs8aT;*Tp zfg|@HI56yCD*2pb#?4*T&{HANjWoT37;ybK~D!||8?*E z5EdzGI-G1%DeZCo{+zX%d@(P+cpQD3vI)?8dGSBojU5XHM#|5}RVh-HTt8ssQKx^V z1IrsWO;43Tut||o= zid?NjQjG&623$_hQFYd^9ieR5vPBVEL-%ZesCy#~F$o{EGuqOfl{ZtwK&*MQzq;r@ zPwU3j$^37hK%*BrbHW2qo1o>R0OD6&;a8mb{2OD6GkiPKW=x3)#mPt;T9XN&wBsQ^ zP31tIElt2T!kB&|d;wya0st?X+P6qK##y}hI!OwKV9rv*u;q*d(K?~Uq^^NO63b2> zKPZM%N&2rYhyURvb1^28!&$R3iy1`hNk*X-vA9x=Yq_=(mR7*_Vd;ftoKwz%Z`>G< zecVC#q@#`hLq`9<6OzL}{e;z52R|& zviVgsP>lu|J3@>6%cvZ7HuS^HrfCEWN3=`lfGgwCMg2h}mX5G7QUJV?2wFxpWFT|o z$dRPV%4_b(;`g1dYbqb=l_a2{>i zU6N|B4dyIT+e;Cq6X3FB3I$bA%;KWG$ogU*>VHcc1MHxTPiynm+y-Qxesdu}Ila=^sKUFd5or>ei*c;xP`ul)$?QF*9Q z>P@H+Bm+$}UX?KuC>QoYF9S1dy4t$r9rX>HKGH;?tW!j_(ugz1JdJ>I0L-A*8jA(i zXeP0WOg*8WBIq=q%v9Cdstlw}vOgifU3T??=OIKQ8LmSsY`z3ory&0ai~e@@pcttG z@l}rGP|vnAgx3da6i-EW#Fk1d*hnahL=G;v!*!ickS709ljl zNK!1BYrn8E&hrmyI7g`Oq<8P%8z3~2;j1EcG^q`{3-hv#+q}ODW6tmC^XFMT#l#Dk zA*W;J1!_=2m0dUE`n|sjYPWZ6Q>{TR@>cV+6UnZ7==hIHJQZ=e^~8cprNwMjG>%E4 zBgc%$vD1^asvLNw)*9Gke0|W zz@3ffU}ltvr7P>}WsyEq@{>4FMjd5EbS4>A`06vbi0Y1&<3J)j=wOJnmY)2cASI!|B>J)^&%=({h z>!F1$M@{%vFl@IK{Y3~lRhmc={{Q2F6&gBd1Cu8^6c!dLq-lzv(lxUSb;UUlr5kw> z>=hA=v?+`fah}j#BUu&-Zx;33H>Bi)bUJRSvp3eGIM#;^(}mb{M-;4~YPUIFezL%X zO8PRgFs2RKjL+l5p3Gep)UghYL#Yg@j4VPGP+x;lT2X9ZVDegBSEt_E@*I)6g5o^} z9TdA(v2dM3OR7fFb_RPfpm5MxH^4DAj@4%EO7Ftx5ozErVRE$=S_fUn`EdiZ*mb~8 z1$>{i=c}XjGg%2Zj|=p;(#)rJZQN7cK%++hw&Tu zAbp%D_~I%!E$z8`FWR8o3*yp#Y++}|hclO--yd4@3q|7`7w>$J*jTNw zo@n=@1X9z|520ref+{p5H#c{jg-f||m>-a`Rd{(girNI#2si2gbHlJa3CGR7plq*0 zQZL?tip&f-?#sJsOHSKO`Wd;o{3boc8XymHat5k=uHqm6Jd!{kNsPKtN)D(?E{uUs zS1;M@5cnMAX*`PIor-SvcW$8%Q-$9>$<#4 zA$rTN!iZ|uEtb)e6HLKA@nE0G-~v=ORZ4=3qCvDnphZWCxQrZV0`Yhi`%v z-e|7DBONqH`k?DCjqEmk+}>9MMT9le&6WtNdTK z>)*l9KNK$$f2Q2=2*!SUNDtf?c;93u5v7^{%@e|^f}Ib_KJEupyjanY9*^P&whL<`3I)g zaCVR@*ad2OlSh?|&gL@nXtPUzQ)$7C&f&xokOLqEM`MzO$1){B7s`OmpNu(sTV zE!|DMK=mH_bX6V5j&@K{@i;9^@)kMHaYL=h4Rhixr)h+6;5&vWKgacC8u(N}a^L{P zSS^&~wP1I1cS#HCS|>yQdJnmw4i647!BJ{&6hLg2#PGMsv8X<0p%0s7B0N051NJ-z zsYsMf0>HUTApNetz5Biv)%{qaySXf!)9WyAHPOU@qzeW?5)A674RZVtXykH7k9F=# z)hsduSz|!{Fa?272}Vi_hW>#@+k;FDO2Ysq8c%C%)5Act9w>P|*!9v^uUy&hx=~ z#yM9Rd_J|#ru=+=B;b<;d3*~IdcJKlCPF_w@+1#!`kT(6;)s#>q6qj+6rw1X;BtlI_rmqp_PfE1s zT0|W_Z+r&91UM(bZDSoJPj&J3J(Q74l%HNC?ZkeR zn1jVArRCQX+wObZp*nhBJr4DeAfBXCyfP^y^6lL|RXmG|!M=ABX$-@L!5k?(1#eBh z9NZ9pCMoF1BnbK~MJb3N)Q#iaMQV5M+?i*8fepNYyyc7di$t!w_VB0s1*H;4DV8Sz z)y&vO?VA#9c_RvzeJ$|$l*dA+?!lVBeJ#cMsW7C!45w(pijnE)c PC=!=tE~Q?){rmp~LC%|v literal 0 HcmV?d00001 diff --git a/docs/evaluations/Docling-DPBench/evaluation_DPBench_table_structure_TEDS_struct-with-text.txt b/docs/evaluations/Docling-DPBench/evaluation_DPBench_table_structure_TEDS_struct-with-text.txt new file mode 100644 index 0000000..41774e8 --- /dev/null +++ b/docs/evaluations/Docling-DPBench/evaluation_DPBench_table_structure_TEDS_struct-with-text.txt @@ -0,0 +1,26 @@ +DPBench table_structure TEDS_struct-with-text: mean=0.83 median=0.86 std=0.12 + +| TEDS_struct-with-text | prob [%] | acc [%] | 1-acc [%] | total | +|-------------------------|------------|-----------|-------------|---------| +| (0.000, 0.050] | 0 | 0 | 100 | 0 | +| (0.050, 0.100] | 0 | 0 | 100 | 0 | +| (0.100, 0.150] | 0 | 0 | 100 | 0 | +| (0.150, 0.200] | 0 | 0 | 100 | 0 | +| (0.200, 0.250] | 0 | 0 | 100 | 0 | +| (0.250, 0.300] | 0 | 0 | 100 | 0 | +| (0.300, 0.350] | 0 | 0 | 100 | 0 | +| (0.350, 0.400] | 0 | 0 | 100 | 0 | +| (0.400, 0.450] | 1.96 | 0 | 100 | 1 | +| (0.450, 0.500] | 0 | 1.96 | 98.04 | 0 | +| (0.500, 0.550] | 1.96 | 1.96 | 98.04 | 1 | +| (0.550, 0.600] | 1.96 | 3.92 | 96.08 | 1 | +| (0.600, 0.650] | 1.96 | 5.88 | 94.12 | 1 | +| (0.650, 0.700] | 0 | 7.84 | 92.16 | 0 | +| (0.700, 0.750] | 11.76 | 7.84 | 92.16 | 6 | +| (0.750, 0.800] | 11.76 | 19.61 | 80.39 | 6 | +| (0.800, 0.850] | 13.73 | 31.37 | 68.63 | 7 | +| (0.850, 0.900] | 29.41 | 45.1 | 54.9 | 15 | +| (0.900, 0.950] | 15.69 | 74.51 | 25.49 | 8 | +| (0.950, 1.000] | 9.8 | 90.2 | 9.8 | 5 | + +