Merge pull request #1229 from amadolid/bugfix/webtool

[BUGFIX]: Webtool optional headers
Jaseci-Labs · Oct 24, 2023 · b81dd03 · b81dd03
2 parents 279fe14 + b3db6cd
commit b81dd03
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 6 deletions.
diff --git a/jaseci_core/jaseci/extens/act_lib/tests/fixtures/webtool.jac b/jaseci_core/jaseci/extens/act_lib/tests/fixtures/webtool.jac
@@ -5,7 +5,7 @@ walker get_meta_valid {
 }
 
 walker get_meta_need_auth {
-    has url = "https://docs.google.com/presentation/d/1lIYEuzzhZZ9PJaG_u3XgrFXX5Y6xd0zHV-aB2F8bXXU/edit";
+    has url = "https://github.com/settings/profile";
     can webtool.get_page_meta;
     report webtool.get_page_meta(url);
 }
@@ -26,5 +26,7 @@ walker get_meta_timeout {
 walker get_meta_need_header {
     has url = "https://www.invaluable.com/blog/what-is-a-mandala/";
     can webtool.get_page_meta;
-    report webtool.get_page_meta(url);
+    report webtool.get_page_meta(url, headers = {
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0"
+    });
 }
diff --git a/jaseci_core/jaseci/extens/act_lib/webtool.py b/jaseci_core/jaseci/extens/act_lib/webtool.py
@@ -5,7 +5,7 @@
 
 
 @jaseci_action()
-def get_page_meta(url: str, timeout: int = 3, parser: str = "lxml"):
+def get_page_meta(url: str, timeout: int = 3, parser: str = "lxml", headers: dict = {}):
     """
     Util to parse metadata out of urls and html documents
     Parser option: lxml (default), html5lib, html.parser
@@ -15,9 +15,7 @@ def get_page_meta(url: str, timeout: int = 3, parser: str = "lxml"):
         webpage = requests.get(
             url,
             timeout=timeout,
-            headers={
-                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0"
-            },
+            headers=headers,
         )
         soup = BeautifulSoup(webpage.content, features=parser)
         meta = soup.find_all("meta")