eole-nlp · vince62s · Mar 12, 2025 · Mar 9, 2025 · Mar 10, 2025 · Mar 10, 2025
diff --git a/eole/bin/model/average_models.py b/eole/bin/model/average_models.py
@@ -54,15 +54,15 @@ def run(cls, args):
 
         # this maybe better implemented using model_saver classes
         # config
-        with open(os.path.join(args.output, "config.json"), "w") as f:
+        with open(os.path.join(args.output, "config.json"), "w", encoding="utf-8") as f:
             json.dump(
                 recursive_model_fields_set(final["config"]),
                 f,
                 indent=2,
                 ensure_ascii=False,
             )
         # vocab
-        with open(os.path.join(args.output, "vocab.json"), "w") as f:
+        with open(os.path.join(args.output, "vocab.json"), "w", encoding="utf-8") as f:
             json.dump(final["vocab"], f, indent=2, ensure_ascii=False)
         # optimizer
         torch.save(final["optim"], os.path.join(args.output, "optimizer.pt"))

diff --git a/eole/bin/run/build_vocab.py b/eole/bin/run/build_vocab.py
@@ -228,9 +228,9 @@ def build_vocab_main(config):
 
     def save_counter(counter, save_path):
         check_path(save_path, exist_ok=config.overwrite, log=logger.warning)
-        with open(save_path, "w", encoding="utf8") as fo:
+        with open(save_path, "wb") as fo:
             for tok, count in counter.most_common():
-                fo.write(tok + "\t" + str(count) + "\n")
+                fo.write(f"{tok}\t{str(count)}\n".encode("utf-8"))
 
     if config.share_vocab:
         src_counter += tgt_counter

diff --git a/eole/bin/tools/embeddings_to_torch.py b/eole/bin/tools/embeddings_to_torch.py
@@ -13,7 +13,7 @@
 def get_vocabs(model_path):
     # model = torch.load(dict_path, map_location=torch.device("cpu"))
     vocabs_path = os.path.join(model_path, "vocab.json")
-    with open(vocabs_path) as f:
+    with open(vocabs_path, encoding="utf-8") as f:
         vocabs_dict = json.load(f)
     vocabs = dict_to_vocabs(vocabs_dict)
 

diff --git a/eole/bin/tools/run_mmlu.py b/eole/bin/tools/run_mmlu.py
@@ -184,7 +184,7 @@ def evaluate(args, data_dir):
 
     run_results = compute_metric(run_results)
 
-    with open(output_filename, "w") as f:
+    with open(output_filename, "w", encoding="utf-8") as f:
         json.dump(run_results, f, ensure_ascii=False, indent=2)
 
     end_time = time.time()

diff --git a/eole/inputters/inputter.py b/eole/inputters/inputter.py
@@ -94,7 +94,7 @@ def _read_vocab_file(vocab_path, min_count):
             lines = lines[:-1]
 
             first_line = lines[0].split(None, 1)
-            has_count = len(first_line) == 2 and first_line[-1].isdigit()
+            has_count = len(first_line) == 2 and first_line[-1].strip().isdigit()
             if has_count:
                 vocab = []
                 for line in lines:

diff --git a/eole/models/model_saver.py b/eole/models/model_saver.py
@@ -58,7 +58,7 @@ def load_checkpoint(model_path):
             raise FileNotFoundError(f"{model_path} does not contain config.json")
         vocab_path = os.path.join(model_path, "vocab.json")
         if os.path.exists(vocab_path):
-            with open(vocab_path) as f:
+            with open(vocab_path, encoding="utf-8") as f:
                 checkpoint["vocab"] = json.load(f)
             # use default specials if not specified
             if "specials" not in checkpoint["vocab"].keys():
@@ -241,14 +241,14 @@ def _save_weights(self, model_state_dict):
     def _save_vocab(self):
         vocab_data = vocabs_to_dict(self.vocabs)
         vocab_path = os.path.join(self.model_path, self.step_dir, "vocab.json")
-        with open(vocab_path, "w") as f:
+        with open(vocab_path, "w", encoding="utf-8") as f:
             json.dump(vocab_data, f, indent=2, ensure_ascii=False)
         self._make_symlink("vocab.json")
 
     def _save_config(self):
         config_data = recursive_model_fields_set(self.config)
         config_path = os.path.join(self.model_path, self.step_dir, "config.json")
-        with open(config_path, "w") as f:
+        with open(config_path, "w", encoding="utf-8") as f:
             json.dump(config_data, f, indent=2, ensure_ascii=False)
         self._make_symlink("config.json")