XLS data transfer to Kobo

run.py

@@ -1,5 +1,3 @@
-#!/usr/bin/env python3
-
 import argparse
 import json
 import sys
@@ -12,9 +10,12 @@
     print_stats,
     transfer_submissions,
 )
-
+from xls_transfer.xlsx_kobo import general_xls_to_xml
 
 def main(
+    warnings,
+    xtransfer,
+    excel_file,
     limit,
     last_failed=False,
     keep_media=False,
@@ -23,40 +24,50 @@ def main(
     validate=True,
     config_file=None,
 ):
+
     config = Config(config_file=config_file, validate=validate)
     config_src = config.src
 
-    print('📸 Getting all submission media', end=' ', flush=True)
-    get_media()
+    if not xtransfer:
+        print('📸 Getting all submission media', end=' ', flush=True)
+        get_media()
 
     xml_url_src = config_src['xml_url'] + f'?limit={limit}'
 
     if last_failed and config.last_failed_uuids:
         xml_url_src += f'&query={json.dumps(config.data_query)}'
 
     all_results = []
+
     submission_edit_data = get_submission_edit_data()
 
     print('📨 Transferring submission data')
 
     def transfer(all_results, url=None):
-        parsed_xml = get_src_submissions_xml(xml_url=url)
+        if (xtransfer):
+            parsed_xml = general_xls_to_xml(excel_file, submission_edit_data, warnings)
+        else:
+            parsed_xml = get_src_submissions_xml(xml_url=url)
+
         submissions = parsed_xml.findall(f'results/{config_src["asset_uid"]}')
         next_ = parsed_xml.find('next').text
-        results = transfer_submissions(
+
+        """results = transfer_submissions(
             submissions,
             submission_edit_data,
             quiet=quiet,
             regenerate=regenerate,
         )
-        all_results += results
+        all_results += results"""
+
         if next_ != 'None' and next_ is not None:
             transfer(all_results, next_)
-
+        
     transfer(all_results, xml_url_src)
-
-    if not keep_media:
-        del_media()
+
+    if not xtransfer:
+        if not keep_media:
+            del_media()
 
     print('✨ Done')
     print_stats(all_results)
@@ -66,6 +77,34 @@ def transfer(all_results, url=None):
     parser = argparse.ArgumentParser(
         description='A CLI tool to transfer submissions between projects with identical XLSForms.'
     )
+
+    parser.add_argument( 
+        '--print-warnings',
+        '-w',
+        default = False,
+        action = 'store_true', 
+        help='Print warnings if questions in Kobo form do not match XLS form.', 
+    )
+
+    parser.add_argument( 
+        '--google-transfer',
+        '-gt',
+        default = False,
+        action = 'store_true', 
+        help='Complete transfer from Google Form data to Kobo project.', 
+    )
+    parser.add_argument( 
+        '--excel-transfer',
+        '-xt',
+        default = False,
+        action = 'store_true', 
+        help='Complete transfer from any xlsx form to Kobo project.', 
+    )
+    parser.add_argument( 
+        '--excel-file',
+        '-ef', 
+        help='Excel file path for data to upload', 
+    )
     parser.add_argument(
         '--limit',
         '-l',
@@ -117,8 +156,14 @@ def transfer(all_results, url=None):
     )
     args = parser.parse_args()
 
+    if (args.excel_transfer and not args.excel_file) or (args.google_transfer and not args.excel_file):
+        parser.error("If --excel-transfer (-xt) or --google-transfer (-gt) is passed, --excel-file (-xt) is required.")
+
     try:
         main(
+            warnings = args.print_warnings,
+            xtransfer = args.excel_transfer,
+            excel_file=args.excel_file,
             limit=args.limit,
             last_failed=args.last_failed,
             regenerate=args.regenerate_uuids,
@@ -127,7 +172,10 @@ def transfer(all_results, url=None):
             validate=not args.no_validate,
             config_file=args.config_file,
         )
+
     except KeyboardInterrupt:
         print('🛑 Stopping run')
         # Do something here so we can pick up again where this leaves off
         sys.exit()
+
+

submission.xml

@@ -0,0 +1 @@
+<root><next /><previous /><results><aRxMkyoPa6dj5wqMnT4vnz id="aRxMkyoPa6dj5wqMnT4vnz" version="1 (2024-03-14 10:48:27)"><formhub><uuid>8f5f9a0801cb4c0c85bc340dde5c3dc4</uuid></formhub><start>2024-03-19T18:03:42.398000</start><end>2024-03-19T18:07:04.622000</end><Quest_name>Quest for Crystal</Quest_name><Quest_starting_date>2024-03-05 00:00:00</Quest_starting_date><character_selection><Character_name>Percy</Character_name><character_objective><Objective_title>Retrieval of Crystal</Objective_title><Objective_description>locate the cave and get it from the guardian</Objective_description></character_objective><character_tasks><group_yj2jm99><Task_difficulty>3</Task_difficulty><Task_description>navigate through forest, and avoid traps</Task_description></group_yj2jm99><Task_title>forest navigation</Task_title></character_tasks></character_selection><__version__>v5NMWz6Pcy8Wq9XzTHLViC</__version__><meta><instanceID>uuid:77de2884-b34e-48ed-84bf-e172680fb2e2</instanceID><deprecatedID>uuid:63047222-c474-4e5f-bd16-f37b6eab7891</deprecatedID></meta></aRxMkyoPa6dj5wqMnT4vnz><aRxMkyoPa6dj5wqMnT4vnz id="aRxMkyoPa6dj5wqMnT4vnz" version="1 (2024-03-14 10:48:27)"><formhub><uuid>8f5f9a0801cb4c0c85bc340dde5c3dc4</uuid></formhub><start>2024-03-19T18:07:31.300000</start><end>2024-03-19T18:15:06.317000</end><Quest_name>Mountain expedition</Quest_name><Quest_starting_date>2024-03-03 00:00:00</Quest_starting_date><character_selection><Character_name>aria</Character_name><character_objective><Objective_title>reach the summit</Objective_title><Objective_description>go through extreme weather and terrain to reach highest peak</Objective_description></character_objective><character_tasks><group_yj2jm99><Task_difficulty>1</Task_difficulty><Task_description>visit base camp to get essential gear</Task_description></group_yj2jm99><Task_title>climbing gear</Task_title></character_tasks><character_tasks><group_yj2jm99><Task_difficulty>3</Task_difficulty><Task_description>icefall</Task_description></group_yj2jm99><Task_title>navigate the icefall</Task_title></character_tasks><character_tasks><group_yj2jm99><Task_difficulty>4</Task_difficulty><Task_description>summit attempt</Task_description></group_yj2jm99><Task_title>final push to the summit!</Task_title></character_tasks></character_selection><character_selection><Character_name>Horace</Character_name><character_objective><Objective_title>guidance</Objective_title><Objective_description>guide the team safely</Objective_description></character_objective><character_tasks><group_yj2jm99><Task_difficulty>3</Task_difficulty><Task_description>assess conditions to plan the safest and most efficient route</Task_description></group_yj2jm99><Task_title>planning</Task_title></character_tasks><character_tasks><group_yj2jm99><Task_difficulty>3</Task_difficulty><Task_description>establish emergency response protocols</Task_description></group_yj2jm99><Task_title>emergency planning</Task_title></character_tasks><character_tasks><group_yj2jm99><Task_difficulty /><Task_description>provide constant support and encouragement to team, and help with physical and mental well-being</Task_description></group_yj2jm99><Task_title>support</Task_title></character_tasks></character_selection><__version__>v5NMWz6Pcy8Wq9XzTHLViC</__version__><meta><instanceID>uuid:61c7dc7e-7a25-4af2-abfa-136f8d0941eb</instanceID><deprecatedID>uuid:a148ed1a-64d4-44e2-90de-80b80d715227</deprecatedID></meta></aRxMkyoPa6dj5wqMnT4vnz><aRxMkyoPa6dj5wqMnT4vnz id="aRxMkyoPa6dj5wqMnT4vnz" version="1 (2024-03-14 10:48:27)"><formhub><uuid>8f5f9a0801cb4c0c85bc340dde5c3dc4</uuid></formhub><start>2024-03-19T18:15:06.346000</start><end>2024-03-19T18:16:26.712000</end><Quest_name>The Lost City</Quest_name><Quest_starting_date>2024-03-03 00:00:00</Quest_starting_date><character_selection><Character_name>Elara</Character_name><character_objective><Objective_title>find the entrance</Objective_title><Objective_description>uncover hidden entrance</Objective_description></character_objective><character_tasks><group_yj2jm99><Task_difficulty>3</Task_difficulty><Task_description>use knowledge of ancient languages to decipher inscriptions</Task_description></group_yj2jm99><Task_title>decode inscriptions</Task_title></character_tasks></character_selection><__version__>v5NMWz6Pcy8Wq9XzTHLViC</__version__><meta><instanceID>uuid:c74ddab2-40e7-4a08-b1e6-e859f5629637</instanceID><deprecatedID>uuid:2faf5ce3-49d7-4109-9819-18d4bc9d8292</deprecatedID></meta></aRxMkyoPa6dj5wqMnT4vnz><aRxMkyoPa6dj5wqMnT4vnz id="aRxMkyoPa6dj5wqMnT4vnz" version="1 (2024-03-14 10:48:27)"><formhub><uuid>8f5f9a0801cb4c0c85bc340dde5c3dc4</uuid></formhub><start>2024-03-19T18:16:26.736000</start><end>2024-03-19T18:19:58.767000</end><Quest_name>The forbidden ruins</Quest_name><Quest_starting_date /><character_selection><Character_name>Thalia</Character_name><character_objective><Objective_title>Uncover lost civilization</Objective_title><Objective_description>piece together clues to unravel mysteries of forgotten civilization</Objective_description></character_objective></character_selection><__version__>v5NMWz6Pcy8Wq9XzTHLViC</__version__><meta><instanceID>uuid:dff86a4f-2b46-4a06-b0ad-a0fe08d1ce49</instanceID><deprecatedID>uuid:cbd2b4d0-f98d-48d0-8988-52cd94ed5861</deprecatedID></meta></aRxMkyoPa6dj5wqMnT4vnz></results></root>

transfer/media.py

@@ -10,6 +10,20 @@
 
 from helpers.config import Config
 
+def rename_media_folder(submission_data, uuid, row_num):
+    current_attachments_path = os.path.join(
+        Config.ATTACHMENTS_DIR, submission_data['asset_uid'], str(row_num)
+    )
+    if os.path.exists(current_attachments_path):   
+        new_attachments_path = os.path.join(
+            Config.ATTACHMENTS_DIR, submission_data['asset_uid'], str(uuid)
+        )
+        try:
+            # Move the folder to the new path
+            shutil.move(current_attachments_path, new_attachments_path)
+
+        except Exception as e: #TODO
+            print(f"Error: {e}")
 
 def del_media():
     config = Config().src
@@ -18,7 +32,6 @@ def del_media():
         print('🧹 Cleaning up media (pass `--keep-media` to prevent cleanup).')
         shutil.rmtree(media_path)
 
-
 def get_media(verbosity=0, chunk_size=1024, throttle=0.1, limit=1000, query=''):
     config = Config().src
     config.update(
@@ -29,9 +42,10 @@ def get_media(verbosity=0, chunk_size=1024, throttle=0.1, limit=1000, query=''):
             'throttle': throttle,
         }
     )
+
     stats = download_all_media(
-        data_url=config['data_url'],
-        stats=get_clean_stats(),
+       data_url=config['data_url'],
+       stats=get_clean_stats(),
     )
 
 

transfer/xml.py

@@ -11,7 +11,6 @@
 from utils.text import get_valid_filename
 from helpers.config import Config
 
-
 def get_submission_edit_data():
     config = Config().dest
     _v_, v = get_info_from_deployed_versions()
@@ -116,6 +115,7 @@ def generate_new_instance_id() -> (str, str):
 def transfer_submissions(all_submissions_xml, asset_data, quiet, regenerate):
     results = []
     for submission_xml in all_submissions_xml:
+
         # Use the same UUID so that duplicates are rejected
         original_uuid = submission_xml.find('meta/instanceID').text.replace(
             'uuid:', ''