From 6859ca998ac841b4a6312d31815a18d23e54f7e1 Mon Sep 17 00:00:00 2001 From: Helena Rasche Date: Wed, 26 Jun 2024 08:52:40 +0200 Subject: [PATCH] more failure tolerant --- bin/wfh-upload.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/bin/wfh-upload.py b/bin/wfh-upload.py index dfe08133764e67..d1f9a4113dd715 100755 --- a/bin/wfh-upload.py +++ b/bin/wfh-upload.py @@ -17,6 +17,17 @@ "_site/training-material/api/workflows/**/rocrate.zip", recursive=True ) +def doUploadWrap(crate_path): + try: + return doUpload(crate_path) + except requests.exceptions.SSLError as e: + try: + time.sleep(5) + return doUpload(crate_path) + except requests.exceptions.SSLError as e: + print(f"Error uploading {crate_path}: {e}") + return None + def doUpload(crate_path): payload = { "ro_crate": (crate_path, open(crate_path, "rb")), @@ -84,12 +95,18 @@ def doUpload(crate_path): return (topic, tutorial, workflow, wfid) + with multiprocessing.Pool(4) as p: - results = p.map(doUpload, crates) + results = p.map(doUploadWrap, crates) results = [x for x in results if x is not None] -data = {} +if os.path.exists('metadata/workflowhub.yml'): + with open('metadata/workflowhub.yml') as handle: + data = yaml.safe_load(handle) +else: + data = {} + for (topic, tutorial, workflow, wf_id) in results: data['/'.join([topic, tutorial, workflow])] = wf_id