From 84b86e88179f292d51fadc1efdb3ebf1e839c906 Mon Sep 17 00:00:00 2001 From: Shi Yu Date: Mon, 19 Aug 2024 18:20:41 +0800 Subject: [PATCH] Update tokenize_shuffle.py --- open_lm/datapreprocess/ray/tokenize_shuffle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/open_lm/datapreprocess/ray/tokenize_shuffle.py b/open_lm/datapreprocess/ray/tokenize_shuffle.py index f0e350f2..b3eb0ff3 100644 --- a/open_lm/datapreprocess/ray/tokenize_shuffle.py +++ b/open_lm/datapreprocess/ray/tokenize_shuffle.py @@ -496,7 +496,7 @@ def glob_files(path, suffixes): def write_manifest(jsonl_lines, args): "Write manifest to provided output path." - output_path = os.path.join(args.output.strip("/"), "manifest.jsonl") + output_path = os.path.join(args.output.rstrip("/"), "manifest.jsonl") if output_path.startswith("s3://"): # Use boto3 for S3 paths