updating db load function

scrape/dapp_scraper.py

@@ -1,11 +1,24 @@
+import os
+import argparse
+
+# Change the current working directory to the parent directory of the script
+os.chdir(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
 import requests
 import json
-import os
 from typing import List
 
+from scrape.models import (
+    db_session,
+    Dapp
+)
+
 BROWSERLESS_API_KEY = os.getenv('BROWSERLESS_API_KEY', '')
 SCRAPE_API_URL = f'https://chrome.browserless.io/scrape?token={BROWSERLESS_API_KEY}'
 
+# Construct the path to the dapps_ranked.json file
+dapps_json_path = os.path.join(os.getcwd(), "scrape", "dapps_ranked_unique.json")
+
 # scrape a URL for IPFS links, return 
 def get_ipfs_links_from_url(url: str) -> List[str]:
 
@@ -160,7 +173,8 @@ def clean_payload_data(original_data):
     return reduced_data
 
 
-def load_data_from_json_to_db(session, json_path):
+def load_data_from_json_to_db(session=db_session, json_path=dapps_json_path):
+    print("Loading data from JSON to DB")
     # 1. Setup
     # If the table doesn't exist, create it
     # Base.metadata.create_all(session.bind) Dont need this - jacob b
@@ -173,7 +187,8 @@ def load_data_from_json_to_db(session, json_path):
 
     # Loop through the JSON data and insert each entry into the database
     for dapp in dapps_data:
-        dapp_instance = DApp(
+        print(f'adding {dapp["name"]}')
+        dapp_instance = Dapp(
             description=dapp["description"],
             name=dapp["name"],
             url=dapp["url"],
@@ -191,3 +206,21 @@ def load_data_from_json_to_db(session, json_path):
     # Commit the transactions
     session.commit()
 
+    print("Finished loading data from JSON to DB")
+
+
+
+if __name__ == "__main__":
+
+    # create an ArgumentParser instance
+    parser = argparse.ArgumentParser(description='Run functions in the dapp_scraper module.')
+
+     # Add an argument for the load_data_from_json_to_db function
+    parser.add_argument('--load', action='store_true', help='Run the load_data_from_json_to_db function.')
+
+    # Parse the command line arguments
+    args = parser.parse_args()
+
+    # If the --load argument is present, call the load_data_from_json_to_db function
+    if args.load:
+        load_data_from_json_to_db()