Merge pull request #1436 from vespa-engine/kkraune/fix

trust_remote_code
vespa-engine · Jul 4, 2024 · 5b29ae5 · 5b29ae5
2 parents a7b8ffb + 635e76d
commit 5b29ae5
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/colbert-long/scripts/convert.py b/colbert-long/scripts/convert.py
@@ -14,7 +14,7 @@ def remove_control_characters(s):
     is_separator_regex = False,
 )
 
-documents = load_dataset('Shitao/MLDR', "corpus-en", split='corpus')    
+documents = load_dataset('Shitao/MLDR', "corpus-en", split='corpus', trust_remote_code=True)
 feed_file = "/tmp/vespa_feed_file_en.json"
 with open(feed_file, "w") as f:
     for doc in documents: