UpstageAI · ts-kim · May 6, 2025 · Apr 29, 2025 · Apr 29, 2025 · Apr 29, 2025
diff --git a/.gitignore b/.gitignore
@@ -43,4 +43,5 @@ lmms_eval/tasks/vatex/__pycache__/utils.cpython-310.pyc
 lmms_eval/tasks/mlvu/__pycache__/utils.cpython-310.pyc
 
 scripts/
-.env
+.env
+UpScore_results/
diff --git a/README.md b/README.md
@@ -41,4 +41,19 @@ scripts/run_eval.sh \
     --tasks docvqa \
     --gpu_ids 0,1,2,3 \
     --port 35001
-```
+```
+
+
+### (Optional) KIE_bench 사용을 위한 추가 설정
+- `pdf2image` 파이썬 라이브러리 사용을 위해 `poppler-utils` 설치가 필요합니다.
+    ```
+    apt-get install poppler-utils
+    ```
+
+- Dataset 전처리 (KIE-bench -> KIE-bench huggingface version)
+    - 주의: 이를 통해 생성된 KIE-bench huggingface version에는 일부 정보가 실제 Upstage huggingface uploaded version과 다를 수 있습니다. (다른 부분은 코드 참고) 하지만 사용되지 않는 부분이기에 최신 버전의 KIE bench를 빠르게 적용하기 위해 전처리 코드를 작성하였습니다. 최신 버전의 KIE-bench가 huggingface에 업로드 되어 있는 경우, 해당 데이터를 다운로드 받아 사용하시기 바랍니다.
+    ```
+    # 1. 코드 내 base_path 변수 수정
+    # 2. 아래 코드 실행
+    python preprocessor/KIE_bench_to_HF_dataset.py
+    ```
diff --git a/lmms_eval/api/task.py b/lmms_eval/api/task.py
@@ -1037,10 +1037,9 @@ def concat_tar_parts(tar_parts, output_tar):
                 dataset_kwargs.pop("create_link")
 
         if dataset_kwargs is not None and "load_from_disk" in dataset_kwargs and dataset_kwargs["load_from_disk"]:
-            dataset_kwargs.pop("load_from_disk")
             # using local task in offline environment, need to process the online dataset into local format via
             # `ds = load_datasets("lmms-lab/MMMU")`
-            self.dataset = datasets.load_from_disk(path=self.DATASET_PATH, name=self.DATASET_NAME)
+            self.dataset = datasets.load_from_disk(dataset_path=self.DATASET_PATH)
         else:
             self.dataset = datasets.load_dataset(
                 path=self.DATASET_PATH,

diff --git a/lmms_eval/models/docvision.py b/lmms_eval/models/docvision.py
@@ -54,8 +54,6 @@ def get_config(pretrained):
 
         self._batch_size_per_gpu = int(batch_size)
         self._image_token = self.config.components.lm.image_token
-        self._eos_token = self.config.test.get("stop_token", None)
-        assert self._eos_token is not None, "test.stop_token (e.g., <|im_end|>) in training_config.yaml is not set!"
 
         # 2. accelerator 초기화
         accelerator_kwargs = InitProcessGroupKwargs(timeout=timedelta(weeks=52))
@@ -74,6 +72,9 @@ def get_config(pretrained):
             test=True,
         )
         self._model.eval()
+        self._eos_token = self.config.test.get("stop_token", None)
+        assert self._eos_token is not None, "test.stop_token (e.g., <|im_end|>) in training_config.yaml is not set!"
+
         self._eos_token_id = self.tokenizer.convert_tokens_to_ids(self._eos_token)
 
         # 4. DeepSpeed 설정 초기화 및 accelerator 적용

diff --git a/lmms_eval/tasks/KIE_bench/KIE_bench_test.yaml b/lmms_eval/tasks/KIE_bench/KIE_bench_test.yaml
@@ -0,0 +1,40 @@
+task: "KIE_bench"
+test_split: test
+
+# Dataset configuration options
+dataset_path: "/app/docfm/datasets/benchmark/key_information_extraction/v3.1_HuggingFace"
+dataset_name: "KIE_bench"
+dataset_kwargs: 
+  load_from_disk: true
+process_docs: null
+
+# Prompting / in-context formatting options
+doc_to_visual: !function utils.KIE_bench_doc_to_visual
+doc_to_text: !function utils.KIE_bench_doc_to_text
+doc_to_target: !function utils.KIE_bench_doc_to_target
+doc_to_choice: null
+
+# Generation settings
+output_type: generate_until
+generation_kwargs: {}
+
+# Post-processing and metrics
+process_results: !function utils.KIE_bench_process_results
+metric_list: 
+  - metric: UpScore
+    aggregation: !function utils.KIE_bench_aggregate_results
+    higher_is_better: true
+
+
+# Model-specific prompt configurations
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: "Extract information from the given image based on this schema: "
+    post_prompt: "\n1. If you cannot find the information or the value is not mentioned, return nothing.\n2. If you can find more than one value for a key, return all the values in an array.\n3. Return the value only if the given key’s value exists in the provided content. If it does not exist, return empty string."
+metadata:
+  - version: 0.0
+
+
+# Additional metadata
+    metadata: 
+  - version: 0.0
diff --git a/lmms_eval/tasks/KIE_bench/UpScore/__init__.py b/lmms_eval/tasks/KIE_bench/UpScore/__init__.py