diff --git a/dev-documentation.md b/dev-documentation.md index 0a30421f..42286249 100644 --- a/dev-documentation.md +++ b/dev-documentation.md @@ -2019,4 +2019,92 @@ cwars.get_profile() | ----------------- | ---------------------------------------------------------------------------------- | | `.get_profile()` | Returns the user data in json format. | +# Chrome Web Store + +```python +from scrape_up import chromewebstore.ChromeWebStore +``` +### Scraping Details about Chrome extensions + +First create a object for class `ChromeWebStore` + +```python +store = ChromeWebStore() +``` +### Methods Details +| Methods | Details | +|----|----| +| `.search(keywords,filter(optional))` | returns a list of dictionaries containing extension details based on keyword and optional search filter and statues message | +| `.get_details(id,overview(optional))` | returns a dict containing details about given extension id | + +### Notes +### **.search()**
+`.search()` returns message "success" if scraping is successful. +`.search()` uses an optional argument `filter` to filter the extension results +```python +filter = { + 'itemTypes': 'EXTENSION' | 'THEME', + 'filterBy': 'featured' | 'establishedPublisher' | 'featured,establishedPublisher', + 'minimalRating': rating out of 5 +} +``` +the list of dictionaries returned by `.search()` has the following structure +```python +{ + "title": "Elephantastic Theme", + "id": "abcdefg1234567890", + "link": "https://chromewebstore.google.com/webstore/detail/elephantastic-theme/abcdefg1234567890", + "thumbnail": "https://...", + "rating": "4.5", + "no_of_rating": "1000", + "description": "A delightful theme featuring playful elephant illustrations.", + "featured": True, + "verified_publisher": False +} +``` + +### **.get_details()** +`.get_details()` returns message "success" if scraping is sucessfull. +`.get_details()` uses optional argument `overview` which if `True` returns the overview or descripton of the extension along with the details. +the dicionary returned by `.get_details()` has following structure +```python +{ + "title": "Video Speed Controller", + "thumbnail": "https:/...", + "rating": "4.6", + "featured": True, + "verified_publisher": False, + "tags": ["Extension", "Functionality & UI"], + "no_of_users": "3,000,000", + "overview": "Speed up, slow down, advance and rewind HTML5 audio/video with shortcuts", + "Updated": "May 24, 2024", + "Offered by": "igrigorik", + "Size": "72.43KiB", + "Languages": [""], + "Developer": "N/A708 Long Bridge Street\nSan Francisco, CA 94158-2512\nUS Email igrigorik@gmail.com", + "Non-trader": True, +} + +``` +## Example Usage +`.search()` without `filter` +```python +from scrape_up import chromewebstore.ChromeWebStore +store = ChromeWebStore() +extensions,message=store.search("video downloader") +``` +`.search()` with `filter` +```python +from scrape_up import chromewebstore.ChromeWebStore +store = ChromeWebStore() +filter = { 'itemTypes': 'THEME','filterBy': 'featured','minimalRating': 3 } +extensions,message=store.search( "elephant", filter) +``` +`.get_details()` +```python +store = ChromeWebStore() +extensions,message=store.get_details("nffaoalbilbmmfgbnbgppjihopabppdk", overview=True) +``` + + --- diff --git a/docs/modules/chromewebstore.md b/docs/modules/chromewebstore.md new file mode 100644 index 00000000..abb003f7 --- /dev/null +++ b/docs/modules/chromewebstore.md @@ -0,0 +1,86 @@ +## Chrome Web Store + +```python +from scrape_up import chromewebstore.ChromeWebStore +``` +### Scraping Details about Chrome extensions + +First create a object for class `ChromeWebStore` + +```python +store = ChromeWebStore() +``` +### Methods Details +| Methods | Details | +|----|----| +| `.search(keywords,filter(optional))` | returns a list of dictionaries containing extension details based on keyword and optional search filter and statues message | +| `.get_details(id,overview(optional))` | returns a dict containing details about given extension id | + +### Notes +### **.search()**
+`.search()` returns message "success" if scraping is successful. +`.search()` uses an optional argument `filter` to filter the extension results +```python +filter = { + 'itemTypes': 'EXTENSION' | 'THEME', + 'filterBy': 'featured' | 'establishedPublisher' | 'featured,establishedPublisher', + 'minimalRating': rating out of 5 +} +``` +the list of dictionaries returned by `.search()` has the following structure +```python +{ + "title": "Elephantastic Theme", + "id": "abcdefg1234567890", + "link": "https://chromewebstore.google.com/webstore/detail/elephantastic-theme/abcdefg1234567890", + "thumbnail": "https://...", + "rating": "4.5", + "no_of_rating": "1000", + "description": "A delightful theme featuring playful elephant illustrations.", + "featured": True, + "verified_publisher": False +} +``` + +### **.get_details()** +`.get_details()` returns message "success" if scraping is sucessfull. +`.get_details()` uses optional argument `overview` which if `True` returns the overview or descripton of the extension along with the details. +the dicionary returned by `.get_details()` has following structure +```python +{ + "title": "Video Speed Controller", + "thumbnail": "https:/...", + "rating": "4.6", + "featured": True, + "verified_publisher": False, + "tags": ["Extension", "Functionality & UI"], + "no_of_users": "3,000,000", + "overview": "Speed up, slow down, advance and rewind HTML5 audio/video with shortcuts", + "Updated": "May 24, 2024", + "Offered by": "igrigorik", + "Size": "72.43KiB", + "Languages": [""], + "Developer": "N/A708 Long Bridge Street\nSan Francisco, CA 94158-2512\nUS Email igrigorik@gmail.com", + "Non-trader": True, +} + +``` +## Example Usage +`.search()` without `filter` +```python +from scrape_up import chromewebstore.ChromeWebStore +store = ChromeWebStore() +extensions,message=store.search("video downloader") +``` +`.search()` with `filter` +```python +from scrape_up import chromewebstore.ChromeWebStore +store = ChromeWebStore() +filter = { 'itemTypes': 'THEME','filterBy': 'featured','minimalRating': 3 } +extensions,message=store.search( "elephant", filter) +``` +`.get_details()` +```python +store = ChromeWebStore() +extensions,message=store.get_details("nffaoalbilbmmfgbnbgppjihopabppdk", overview=True) +``` \ No newline at end of file diff --git a/documentation.md b/documentation.md index cec75950..632556ae 100644 --- a/documentation.md +++ b/documentation.md @@ -733,3 +733,88 @@ boxoffice = imdb.BoxOffice() | Methods | Details | | --------------- | ------------------------------------------------------------------------------- | | `.top_movies()` | Returns the top box office movies, weekend and total gross, and weeks released. | + + +### Chrome Web Store + +```python +from scrape_up import chromewebstore.ChromeWebStore +``` +#### Scraping Details about Chrome extensions + +First create a object for class `ChromeWebStore` + +```python +store = ChromeWebStore() +``` +#### Methods Details +| Methods | Details | +|----|----| +| `.search(keywords,filter(optional))` | returns a list of dictionaries containing extension details based on keyword and optional search filter and statues message | +| `.get_details(id,overview(optional))` | returns a dict containing details about given extension id | + +#### Notes +`.search()` returns message "success" if scraping is successful. +`.search()` uses an optional argument `filter` to filter the extension results +```python +filter = { + 'itemTypes': 'EXTENSION' | 'THEME', + 'filterBy': 'featured' | 'establishedPublisher' | 'featured,establishedPublisher', + 'minimalRating': rating out of 5 +} +``` +the list of dictionaries returned by `.search()` has the following structure +```python +{ + "title": "Elephantastic Theme", + "id": "abcdefg1234567890", + "link": "https://chromewebstore.google.com/webstore/detail/elephantastic-theme/abcdefg1234567890", + "thumbnail": "https://...", + "rating": "4.5", + "no_of_rating": "1000", + "description": "A delightful theme featuring playful elephant illustrations.", + "featured": True, + "verified_publisher": False +} +``` +`.get_details()` returns message "success" if scraping is sucessfull. +`.get_details()` uses optional argument `overview` which if `True` returns the overview or descripton of the extension along with the details. +the dicionary returned by `.get_details()` has following structure +```python +{ + "title": "Video Speed Controller", + "thumbnail": "https:/...", + "rating": "4.6", + "featured": True, + "verified_publisher": False, + "tags": ["Extension", "Functionality & UI"], + "no_of_users": "3,000,000", + "overview": "Speed up, slow down, advance and rewind HTML5 audio/video with shortcuts", + "Updated": "May 24, 2024", + "Offered by": "igrigorik", + "Size": "72.43KiB", + "Languages": [""], + "Developer": "N/A708 Long Bridge Street\nSan Francisco, CA 94158-2512\nUS Email igrigorik@gmail.com", + "Non-trader": True, +} + +``` +#### Example Usage +`.search()` without `filter` +```python +from scrape_up import chromewebstore.ChromeWebStore +store = ChromeWebStore() +extensions,message=store.search("video downloader") +``` +`.search()` with `filter` +```python +from scrape_up import chromewebstore.ChromeWebStore +store = ChromeWebStore() +filter = { 'itemTypes': 'THEME','filterBy': 'featured','minimalRating': 3 } +extensions,message=store.search( "elephant", filter) +``` +`.get_details()` +```python +store = ChromeWebStore() +extensions,message=store.get_details("nffaoalbilbmmfgbnbgppjihopabppdk", overview=True) +``` \ No newline at end of file diff --git a/src/scrape_up/chromewebstore/__init__.py b/src/scrape_up/chromewebstore/__init__.py new file mode 100644 index 00000000..31e2c426 --- /dev/null +++ b/src/scrape_up/chromewebstore/__init__.py @@ -0,0 +1,3 @@ +from .chromewebstore import ChromeWebStore + +__all__ = ['ChromeWebStore'] \ No newline at end of file diff --git a/src/scrape_up/chromewebstore/chromewebstore.py b/src/scrape_up/chromewebstore/chromewebstore.py new file mode 100644 index 00000000..e46a08c7 --- /dev/null +++ b/src/scrape_up/chromewebstore/chromewebstore.py @@ -0,0 +1,254 @@ +import requests +from bs4 import BeautifulSoup +from urllib.parse import quote_from_bytes + + +class ChromeWebStore: + """ + A class to interact with the Chrome Web Store and retrieve extension details. + + Attributes: + base_url (str): Base URL for the Chrome Web Store. + """ + + base_url = "https://chromewebstore.google.com/" + + def search(self, keyword: str, filter: dict = None): + """ + Scrapes a list of extensions from the Chrome Web Store based on a keyword search and optional filters. + + Args: + keyword (str): The keyword(s) to search for (URL encoded). + filter (dict, optional): Search filters. + { + 'itemTypes': 'EXTENSION' or 'THEME', + 'filterBy': 'featured' or 'establishedPublisher' or 'featured,establishedPublisher', + 'minimalRating': Rating out of 5 + } + + Returns: + tuple: A tuple containing two elements: + - list: A list of dictionaries containing extension details + { + "title": "Elephantastic Theme", + "id": "abcdefg1234567890", + "link": "https://chromewebstore.google.com/webstore/detail/elephantastic-theme/abcdefg1234567890", + "thumbnail": "https://...", # URL to the thumbnail image + "rating": "4.5", + "no_of_rating": "1000", + "description": "A delightful theme featuring playful elephant illustrations.", + "featured": True, + "verified_publisher": False + } + - str: A message indicating success or errors ("success" or "No results found for given Criteria"). + + Raises: + requests.exceptions.RequestException: If there's an error fetching search results. + """ + keyword = quote_from_bytes(keyword.encode("utf-8")) + url = f"{ChromeWebStore.base_url}search/{keyword}" + + try: + params = filter or {} + response = requests.get(url, params=params) + response.raise_for_status() # Raise an exception for non-200 status codes + + soup = BeautifulSoup(response.content, "html.parser") + except requests.exceptions.RequestException as e: + return [], f"Error fetching search results : {e}" + finally: + # to check for no results + error = soup.find("h1", class_="SCmIo") + if error: + return [], "No results found for given criteria" + + extension_elements = soup.find_all("div", class_="Cb7Kte") + + extensions = [] + for element in extension_elements: + extension = {} + + try: + # title + title_element = element.find("div", class_="IcZnBc") + if title_element: + extension["title"] = title_element.text.strip() + + # extension id + extension["id"] = element["data-item-id"] + + # description + desc_element = element.find("p", class_="g3IrHd") + if desc_element: + extension["description"] = desc_element.text.strip() + + # thumbnail + thumb_nail = element.find("img", class_="fzxcm") + if thumb_nail: + extension["thumbnail"] = thumb_nail["src"] + + # hyperlink + link_element = element.find("a", class_="q6LNgd") + if link_element: + extension["link"] = ( + ChromeWebStore.base_url + link_element["href"][2:] + ) + + # average rating + rating_element = element.find("span", class_="Vq0ZA") + if rating_element: + extension["rating"] = rating_element.text.strip() + + # number of user ratings + no_of_rating = element.find("span", class_="Y30PE") + if no_of_rating: + extension["no_of_rating"] = no_of_rating.text.strip() + + # is the extension or theme featured in chrome store + featured_tag = element.find("span", class_="OmOMFc") + extension["featured"] = True if featured_tag else False + + # is the publisher established + verified_publisher = element.find( + "svg", class_="BFBOHf z088xd RVihJ NMm5M" + ) + extension["verified_publisher"] = True if verified_publisher else False + + except AttributeError as e: + print(f"Error parsing search element {e}") + continue + + extensions.append(extension) + + if extensions: + return extensions, "success" + else: + return extensions, "No results found for given criteria" + + def get_details(self, id: str, overview: bool = False): + """ + Fetches details about a specific extension from the Chrome Web Store. + + Args: + id (str): The ID of the extension to fetch details for. Example: "gjdmanggfaalgnpinolamlefhcjimmam". + overview (bool, optional): Whether to include the extension's overview description. Defaults to False. + + Returns: + tuple[dict, str]: A tuple containing a dictionary with the fetched details and a status message + The dictionary will be empty if there's an error, and the message will indicate the error type. + EXAMPLE FOR DICT + { + title: Minesweeper Original + thumbnail: https:/.... + rating: 4.8 + tags: ['Extension', 'Workflow & Planning'] + no_of_users: 2,000,000 + Updated: July 21, 2023 + Offered by: Davi Hendrix + Size: 133KiB + Languages: Bahasa Indonesia, Bahasa Melayu, Deutsch, English, English (UK), English (United States), Filipino, Français, Kiswahili, Nederlands, Norsk, Tiếng Việt, Türkçe, català, dansk, eesti, español, español (Latinoamérica), hrvatski, italiano, latviešu, lietuvių, magyar, polski, português (Brasil), português (Portugal), română, slovenský, slovenščina, suomi, svenska, čeština, Ελληνικά, Српски, български, русский, українська, עברית, فارسی‎, मराठी, हिन्दी, বাংলা, ગુજરાતી, தமிழ், తెలుగు, ಕನ್ನಡ, മലയാളം, ไทย, አማርኛ, ‫العربية, 中文 (简体), 中, suomi文 (繁體), 日本語, 한국어 + 文 (繁體), 日本語, 한국어 + Developer: Email davihendrixm3602@gmail.com + Non-trader: True + } + + Raises: + requests.exceptions.RequestException: If there's an error fetching the details from the web. + + Notes: + - This method fetches details from the Chrome Web Store using the provided extension ID. + - If overview is set to True, the returned dictionary will include the extension's overview description. + - The fetched details include information like name, version, size, languages, and other metadata. + + Example: + >>> chrome_extension_id = "gjdmanggfaalgnpinolamlefhcjimmam" + >>> details, status = get_details(chrome_extension_id, overview=True) + >>> print(details) + {'Name': 'Example Extension', 'Version': '1.0', 'Size': '2.1 MB', 'Languages': ['English'], ...} + >>> print(status) + 'success' + """ + id = quote_from_bytes(id.encode("utf-8")) + url = f"{ChromeWebStore.base_url}detail/{id}" + + try: + response = requests.get(url) + response.raise_for_status() + + soup = BeautifulSoup(response.content, "html.parser") + except requests.exceptions.RequestException as e: + return {}, f"Error fetching search results : {e}" + + # to check for invalid id + error = soup.find("h1", class_="eFHPDf") + if error: + return {}, "No results found" + + extension = {} + + try: + + # title + title = soup.find("h1", class_="Pa2dE") + extension["title"] = title.text.strip() + + # thumbnail + thumbnail = soup.find("img", class_="rBxtY") + extension["thumbnail"] = thumbnail["src"] + + # user rating out of 5 + rate = soup.find("span", class_="Vq0ZA") + extension["rating"] = rate.text.strip() + + # is the extension or theme featured in chrome store + featured_tag = soup.find("span", class_="OmOMFc") + extension["featured"] = True if featured_tag else False + + # is the publisher established + verified_publisher = soup.find("svg", class_="cJI8ee") + extension["verified_publisher"] = True if verified_publisher else False + + div_element = soup.find("div", class_="F9iKBc") + if div_element: + # tags + tag_elements = div_element.find_all("a", class_="gqpEIe") + extension["tags"] = [tag.text for tag in tag_elements] + + # number of users + user_text = div_element.get_text(strip=True, separator=" ") + extension["no_of_users"] = user_text.split()[-2] + + # Description / overview of extension + if overview: + overview_ele = soup.find("div", class_="JJ3H1e") + extension["overview"] = overview_ele.text.strip() + + # extracting details from