remove extra code

Signed-off-by: Sukriti-Sharma4 <[email protected]>

tuning/sft_trainer.py

@@ -669,7 +669,7 @@ def main():
             checkpoint_dir = training_args.save_model_dir
             if checkpoint_dir:
                 print(f"Post processing LoRA adapters in {checkpoint_dir}")
-                post_process_vLLM_adapters_new_tokens(path_to_checkpoint=checkpoint_dir)
+                post_process_vLLM_adapters_new_tokens(path_to_checkpoint=checkpoint_dir, num_added_tokens=num_added_tokens)
         except Exception as e:  # pylint: disable=broad-except
             logging.error(traceback.format_exc())
             write_termination_log(

tuning/utils/merge_model_utils.py

@@ -126,15 +126,6 @@ def post_process_vLLM_adapters_new_tokens(
     if not modified_checkpoint_path:
         modified_checkpoint_path = path_to_checkpoint
 
-    # Get all values of new token indexes
-    sorted_token_indexes = []
-    if os.path.isfile(os.path.join(path_to_checkpoint, "added_tokens.json")):
-        with open(
-            os.path.join(path_to_checkpoint, "added_tokens.json"), "r", encoding="utf-8"
-        ) as fp:
-            added_tokens = json.load(fp)
-            sorted_token_indexes = sorted(added_tokens.values())
-
     with safe_open(
         os.path.join(path_to_checkpoint, "adapter_model.safetensors"), framework="pt"
     ) as f: