From d7a5b79997f190f1580dc1141e83d01ff110538d Mon Sep 17 00:00:00 2001 From: Tybo Verslype Date: Sat, 21 Dec 2024 01:21:46 +0100 Subject: [PATCH 1/5] chore: startup of sync button --- run_sync.py | 13 ++++++++++++- website/app.py | 16 ++++++++++++++++ website/templates/index.html | 19 +++++++++++++++++++ 3 files changed, 47 insertions(+), 1 deletion(-) diff --git a/run_sync.py b/run_sync.py index 2efc6fd..dad33a4 100644 --- a/run_sync.py +++ b/run_sync.py @@ -9,7 +9,7 @@ def sync_files_to_gitea(): repo, api_handler = sync.init_sync() - print(db.get_files().items()) + # print(db.get_files().items()) for file_id, file_info in db.get_files().items(): # print(file_id, file_info) try: @@ -19,6 +19,17 @@ def sync_files_to_gitea(): traceback.print_exc() +def sync_gitmate(): + print() + print("================================================") + print("== Syncing files to git ==") + sync_files_to_gitea() + print() + return { + "synced": "success" + } + + if __name__ == "__main__": print() print("================================================") diff --git a/website/app.py b/website/app.py index d4491a6..11e8ddc 100644 --- a/website/app.py +++ b/website/app.py @@ -10,6 +10,7 @@ # Add the parent directory to the system path to allow imports from the higher-level directory sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) from main import run_scrapers +from run_sync import sync_gitmate app = Flask(__name__) @@ -198,6 +199,21 @@ def home(): return render_template('index.html', scraper_info=scraper_info) +@app.route("/sync-all", methods=["POST"]) +def sync_all_files(): + """ + Sync all files to GitMate. + """ + try: + # Call the `sync_gitmate` function without arguments to sync all files + print("Syncing all files to GitMate...") + sync_gitmate() + print("Synced all files to GitMate") + return jsonify({"message": "All files synced successfully."}), 200 + except Exception as e: + return jsonify({"error": str(e)}), 500 + + if __name__ == "__main__": # Initialize the database when the app starts init_db() diff --git a/website/templates/index.html b/website/templates/index.html index 1076e25..9defe9d 100644 --- a/website/templates/index.html +++ b/website/templates/index.html @@ -122,6 +122,23 @@ }); } + function syncAll() { + fetch('/sync-all', { + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + }) + .then(response => response.json()) + .then(data => { + {#alert(data.message); // Show a success message from the backend#} + console.log(data.message) + }) + .catch(error => { + alert('Error: ' + error); + }); + } + // Set interval to update scraper table every 5 seconds setInterval(updateScraperTable, 5000); setInterval(scrapeAllRestaurants, 30 * 60 * 1000); // 30 minutes in milliseconds @@ -133,6 +150,8 @@

Restaurant Scraper

+ +
From a39128593b7b72ff83eb67a3c8fb6b8352a18669 Mon Sep 17 00:00:00 2001 From: Tybo Verslype Date: Sat, 21 Dec 2024 14:40:06 +0100 Subject: [PATCH 2/5] chore: startup of sync button --- db.json | 36 +++++++++---------------- db.py => db_file.py | 46 ++++++++++++++++++++++++++------ hlds_files/simpizza.hlds | 49 ++++++++++++++++++++++++++++++++++ run_sync.py | 2 +- scraper_data.db | Bin 16384 -> 16384 bytes sync_gitmate.py | 55 +++++++++++++++++++++++++++++++++------ 6 files changed, 147 insertions(+), 41 deletions(-) rename db.py => db_file.py (69%) diff --git a/db.json b/db.json index b54b0fe..0cddaa1 100644 --- a/db.json +++ b/db.json @@ -1,39 +1,27 @@ { "files": { - "hlds_files\\bicyclette.hlds": { - "local_file_path": "hlds_files\\bicyclette.hlds", + "hlds_files/bicyclette.hlds": { + "local_file_path": "hlds_files/bicyclette.hlds", "metadata": { - "sync-to": "menus\\la_bicylette.hlds" + "sync-to": "menus/la_bicylette.hlds" } }, - "hlds_files\\bocca_ovp.hlds": { - "local_file_path": "hlds_files\\bocca_ovp.hlds", + "hlds_files/bocca_ovp.hlds": { + "local_file_path": "hlds_files/bocca_ovp.hlds", "metadata": { - "sync-to": "menus\\bocca_ovp.hlds" + "sync-to": "menus/bocca_ovp.hlds" } }, - "hlds_files\\metropol.hlds": { - "local_file_path": "hlds_files\\metropol.hlds", + "hlds_files/pizza_donna.hlds": { + "local_file_path": "hlds_files/pizza_donna.hlds", "metadata": { - "sync-to": "menus\\pitta_metropol.hlds" + "sync-to": "menus/prima_donna.hlds" } }, - "hlds_files\\pizza_donna.hlds": { - "local_file_path": "hlds_files\\pizza_donna.hlds", + "hlds_files/simpizza.hlds": { + "local_file_path": "hlds_files/simpizza.hlds", "metadata": { - "sync-to": "menus\\prima_donna.hlds" - } - }, - "hlds_files\\s5.hlds": { - "local_file_path": "hlds_files\\s5.hlds", - "metadata": { - "sync-to": "menus\\s5.hlds" - } - }, - "hlds_files\\simpizza.hlds": { - "local_file_path": "hlds_files\\simpizza.hlds", - "metadata": { - "sync-to": "menus\\simpizza.hlds" + "sync-to": "menus/simpizza.hlds" } } } diff --git a/db.py b/db_file.py similarity index 69% rename from db.py rename to db_file.py index 66fb38f..9c4701c 100644 --- a/db.py +++ b/db_file.py @@ -54,6 +54,15 @@ def is_file_different(hlds_file: str, menu_file: str) -> bool: return hlds_hash != menu_hash +def dos2unix(path: str) -> str: + """ + Converts Windows-style backslashes in a file path to Unix-style forward slashes. + :param path: The input file path (string) + :return: The normalized file path with forward slashes + """ + return path.replace("\\", "/") + + def get_manual_file_mapping() -> Dict[str, str]: """ Creates a manual mapping of file names between the hlds_menus directory and the menus directory. @@ -64,25 +73,46 @@ def get_manual_file_mapping() -> Dict[str, str]: # Manual mapping of file names file_mapping = { - os.path.join(hlds_dir, "bicyclette.hlds"): os.path.join(menu_dir, "la_bicylette.hlds"), - os.path.join(hlds_dir, "bocca_ovp.hlds"): os.path.join(menu_dir, "bocca_ovp.hlds"), - os.path.join(hlds_dir, "metropol.hlds"): os.path.join(menu_dir, "pitta_metropol.hlds"), - os.path.join(hlds_dir, "pizza_donna.hlds"): os.path.join(menu_dir, "prima_donna.hlds"), - os.path.join(hlds_dir, "s5.hlds"): os.path.join(menu_dir, "s5.hlds"), - os.path.join(hlds_dir, "simpizza.hlds"): os.path.join(menu_dir, "simpizza.hlds"), + dos2unix(os.path.join(hlds_dir, "bicyclette.hlds")): dos2unix(os.path.join(menu_dir, "la_bicylette.hlds")), + dos2unix(os.path.join(hlds_dir, "bocca_ovp.hlds")): dos2unix(os.path.join(menu_dir, "bocca_ovp.hlds")), + dos2unix(os.path.join(hlds_dir, "metropol.hlds")): dos2unix(os.path.join(menu_dir, "pitta_metropol.hlds")), + dos2unix(os.path.join(hlds_dir, "pizza_donna.hlds")): dos2unix(os.path.join(menu_dir, "prima_donna.hlds")), + dos2unix(os.path.join(hlds_dir, "s5.hlds")): dos2unix(os.path.join(menu_dir, "s5.hlds")), + dos2unix(os.path.join(hlds_dir, "simpizza.hlds")): dos2unix(os.path.join(menu_dir, "simpizza.hlds")), # Add more mappings here as needed } return file_mapping +def get_mapped_path(file_path: str) -> str: + """ + Maps a given file path using the manual mapping or returns the same path if not in the mapping. + Ensures the returned path uses forward slashes. + :param file_path: The input file path + :return: The mapped file path or the original path + """ + file_mapping = get_manual_file_mapping() + normalized_path = dos2unix(file_path) + return file_mapping.get(normalized_path, normalized_path) + + def test_file_comparison(): """ Compares all files based on the manual mapping and prints whether they are different or identical. """ - file_mapping = get_manual_file_mapping() + hlds_dir = "hlds_files" + menu_dir = "menus" + + # Get a list of files in the `hlds_files` directory + hlds_files = [ + dos2unix(os.path.join(hlds_dir, f)) + for f in os.listdir(hlds_dir) if os.path.isfile(os.path.join(hlds_dir, f)) + ] + + for hlds_file in hlds_files: + menu_file = get_mapped_path(hlds_file) - for hlds_file, menu_file in file_mapping.items(): if not os.path.exists(hlds_file): print(f"{hlds_file} does not exist. Skipping...") continue diff --git a/hlds_files/simpizza.hlds b/hlds_files/simpizza.hlds index 10bfb48..2b0da21 100644 --- a/hlds_files/simpizza.hlds +++ b/hlds_files/simpizza.hlds @@ -2355,6 +2355,55 @@ dish pasta_chicken_picanto: Pasta Chicken picanto -- Roomsaus, rode pesto, kip, zakje_parmezaanse_kaas: Zakje Parmezaanse kaas € 1.0 bestek: bestek € 0.35 dish pasta_deal: Pasta Deal -- Kleine portie pasta, lookbrood naar keuze, frisdrank € 18.95 + single_choice Spaghetti/penne: Welke Spaghetti/penne + spaghetti: spaghetti € 0.0 + penne: penne € 0.0 + single_choice Drank?: Welke Drank? + cola: Cola € 0.0 + cola_zero: Cola Zero € 0.0 + fanta: Fanta € 0.0 + fanta_exotic: Fanta Exotic € 0.0 + fanta_cassis: Fanta Cassis € 0.0 + fanta_strawberry_&_kiwi: Fanta Strawberry & Kiwi € 0.0 + tropico: Tropico € 0.0 + ice_tea: Ice Tea € 0.0 + ice_tea_peach: Ice Tea Peach € 0.0 + sprite: Sprite € 0.0 + spa_plat: Spa Plat € 0.0 + spa_bruis: Spa Bruis € 0.0 + uludag: Uludag € 0.0 + oasis_tropical: Oasis Tropical € 0.0 + single_choice Garnering: Welke Garnering + scampi: Scampi € 1.5 + kip: kip € 1.5 + ham: Ham € 1.5 + extra_kaas: extra kaas € 1.5 + extra_saus: extra saus € 1.5 + zakje_parmezaanse_kaas: Zakje Parmezaanse kaas € 1.0 + bestek: bestek € 0.35 + single_choice Pasta: Welke Pasta + pasta_bolognese: Pasta Bolognese € 0.0 + pasta_kaassaus: Pasta kaassaus € 0.0 + pasta_kip_en_kaassaus: Pasta kip en kaassaus € 0.0 + pasta_ham_en_kaassaus: Pasta ham en kaassaus € 0.0 + pasta_milano: Pasta Milano € 0.0 + pasta_scampi: Pasta scampi € 0.0 + pasta_multi_cheese: Pasta Multi cheese € 0.0 + pasta_veggie: Pasta Veggie € 0.0 + pasta_pesto_chicken: Pasta Pesto Chicken € 0.0 + pasta_veggie: Pasta Veggie € 0.0 + pasta_exotique: Pasta Exotique € 0.0 + pasta_chicken_pesto_spinazie: Pasta Chicken pesto spinazie € 0.0 + pasta_gambaretti_spinazie: Pasta Gambaretti spinazie € 0.0 + pasta_chicken_arabiata: Pasta Chicken arabiata € 0.0 + pasta_chicken_picanto: Pasta Chicken picanto € 0.0 + single_choice Lookbrood: Welke Lookbrood + lookbrood_natuur: Lookbrood natuur € 0.0 + lookbrood_kaas: Lookbrood kaas € 0.0 + lookbrood_kaas_&_ham: Lookbrood kaas & ham € 0.0 + lookbrood_kaas_&_tomaat: Lookbrood kaas & tomaat € 0.0 + lookbrood_kaas_&_salami: Lookbrood kaas & salami € 0.0 + lookbrood_kaas_&_kip: Lookbrood kaas & kip € 0.0 dish pasta_exotique: Pasta Exotique -- Tomatenroomsaus, ananas, champignons, chili, cherrytomaat, paprika € 13.95 single_choice Spaghetti/penne: Welke Spaghetti/penne spaghetti: spaghetti € 0.0 diff --git a/run_sync.py b/run_sync.py index dad33a4..81fae92 100644 --- a/run_sync.py +++ b/run_sync.py @@ -1,6 +1,6 @@ import traceback -import db +import db_file as db # import dir_utils # import mattermost_client # import mattermost_communication diff --git a/scraper_data.db b/scraper_data.db index 77684b6ce473acc3b6305175867d5fa5c5feccc7..92784f94305e110b53dd9981f1f97b8ac501f4dd 100644 GIT binary patch delta 376 zcmZo@U~Fh$oFL8kV4{pO$gHYL zWN0ulFf!3KG}1LPR4_ELGBU9;G}N;&G&D0aa?8xiEY3(xG4Yz*D<`c8mNzuDGBVIJ zu`n<*GJ(nSPh>C#nkhXwUQVSxs5CDxGcVo9&z>% delta 365 zcmZo@U~Fh$oFL73aiWYf)$L_(#Aj{+^JNcxX3YS}E zUS@GdYKn>Tr;Ug^_`g ziLRlMt`Sg+p_PG|m5GI(iIIhofmu*#US4KidWw-N8-qNfqij-sa&lsPep!JOvL3KC zhDLhEhUO;522g95Q;c0E$H_~DAi2ZB($Lb_1n!Pg48}l1rEwZ+W@TuqXJBGzXT%cTAosugn26XY*lsCIM?VUS@elxKXbB%*u>l-^Hio=jA0j^DxUY!aVH3 Z$E?Vhl$l(aoReBolIpU_;6Fd32LK$ Date: Mon, 23 Dec 2024 21:07:17 +0100 Subject: [PATCH 3/5] chore: startup mattermost_comunication --- config.py | 8 ++ mattermost_client.py | 178 +++++++++++++++++++++++++++++++++++++ mattermost_comunication.py | 60 +++++++++++++ mattermost_objects.py | 137 ++++++++++++++++++++++++++++ requirements.txt | 3 +- scraper_data.db | Bin 16384 -> 16384 bytes sync_gitmate.py | 2 +- utils.py | 14 +++ 8 files changed, 400 insertions(+), 2 deletions(-) create mode 100644 config.py create mode 100644 mattermost_client.py create mode 100644 mattermost_comunication.py create mode 100644 mattermost_objects.py diff --git a/config.py b/config.py new file mode 100644 index 0000000..5af306c --- /dev/null +++ b/config.py @@ -0,0 +1,8 @@ +import tomllib +from pprint import pprint + +with open("config.toml", mode="rb") as config_toml: + config = tomllib.load(config_toml) + + +# pprint(config) diff --git a/mattermost_client.py b/mattermost_client.py new file mode 100644 index 0000000..b7c0ad3 --- /dev/null +++ b/mattermost_client.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +import pprint as pp +from abc import ABC, abstractmethod +from enum import Enum +from typing import Dict +from config import config + +from colored import Style +from mattermostdriver import Driver + +from mattermost_objects import MMChannelPosts +from utils import timer + +pp = pp.PrettyPrinter(indent=2) + + +class LogLevel(Enum): + INFO = "INFO" + ERROR = "ERROR" + + +class User(ABC): + @abstractmethod + def credentials_dict(self) -> dict: + pass + +class TokenUser(User): + def __init__(self, token): + self.token = token + + def credentials_dict(self) -> dict: + return {"token": self.token} + + def __repr__(self): + return "TokenUser".format(self.token) + + +users: {str: [User]} = {} + + +def loadusers(): + usr = None + for name, data in config["mattermost"]["users"].items(): + if "token" in data: + usr = TokenUser(token=data["token"]) + else: + print("Invalid user '{}' in toml file".format(name)) + exit(1) + users[name] = usr + + +loadusers() + + +def merge_dict(a: dict, b: dict) -> dict: + return {**a, **b} + + +class MMApi(Driver): + def __init__(self, user: User = users["tyboro"]): + print(f"Initializing MMApi client for user {user}") + Driver.__init__( + self, + merge_dict( + { + "url": "mattermost.zeus.gent", + "port": 443, + "debug": False, + }, + user.credentials_dict(), + ), + ) + self.login() + self.user_id = self.users.get_user(user_id="me")["id"] + self.team_id = self.teams.get_team_by_name("zeus")["id"] + print(" = Creating mattermost client") + print(f" = - User: {self.user_id}") + print(f" = - Team: {self.team_id}") + + @staticmethod + def print_response(resp, title="Response"): + print("--------") + print(Style.BOLD + title + Style.RESET) + pp.pprint(resp) + + def log(self, text: str, log_level: LogLevel = LogLevel.INFO): + print(f"{Style.BOLD}[{log_level.value}]{Style.RESET} {text}") + + def get_channel_id(self, channel_name): + resp = self.channels.get_channel_by_name(self.team_id, channel_name) + id = resp["id"] + self.log(f"Fetching channel id for {channel_name}: {id}") + return id + + @timer + def get_posts_for_channel(self, channel_id, since): + print(f"Fetching posts for {channel_id} since {since}") + page_size = 200 + page_i = 0 + data = {} + more = True + while more: + resp = self.posts.get_posts_for_channel( + channel_id, + params={"page": page_i, "per_page": page_size, "since": since}, + ) + page_i += 1 + print(f"Fetching page {page_i}") + # print("-", end=" ") + + paged_data = resp["posts"] + paged_count = len(paged_data) + + if since != 0: + # The mattermost api is absolutely retarted + # If you add the since parameter and it's different then 0 it will give you 1000 posts max. + # It will not respect you page_index or page_size. + more = False + else: + if paged_count < page_size: + more = False + + # Transform the data into something more sensible or practical + if type(paged_data) is list: + paged_data = {item["id"]: item for item in paged_data} + + # Append the paged_data to our global data variable + data = {**data, **paged_data} + print() + + self.log(f"Post count: {len(data)}") + return data + + +class ChannelApi(MMApi): + def __init__(self, channel_name=None, channel_id=None, user=None): + MMApi.__init__(self, user) + assert channel_name is not None or channel_id != None + + if channel_name is not None: + self.channel_id = self.get_channel_id(channel_name) + if channel_id is not None: + self.channel_id = channel_id + + def create_post(self, message: str, props: Dict = None) -> None: + resp = self.posts.create_post( + options={"channel_id": self.channel_id, "message": message, "props": props} + ) + self.log(f'Message successfully created: "{message}"') + + def create_threaded_post( + self, post_id: str, message: str, props: Dict = None + ) -> None: + resp = self.posts.create_post( + options={ + "channel_id": self.channel_id, + "message": message, + "root_id": post_id, + "props": props, + } + ) + self.log(f'Message successfully created: "{message}"') + # print_response("Create post", resp) + + +if __name__ == "__main__": + foo = MMApi(user=users["flynn"]) + + # all_posts = foo.get_all_posts() + + channel = foo.channels.get_channel_by_name( + foo.team_id, + "bestuur", + ) + channel_id = channel["id"] + resp = foo.posts.get_posts_for_channel(channel_id, params={"per_page": 200}) + channel_posts: MMChannelPosts = MMChannelPosts.load(resp) diff --git a/mattermost_comunication.py b/mattermost_comunication.py new file mode 100644 index 0000000..78ffc4e --- /dev/null +++ b/mattermost_comunication.py @@ -0,0 +1,60 @@ +import mattermostdriver.exceptions + +import mattermost_client +from config import config +from mattermost_client import ChannelApi, MMApi + + +def send_message(file_info, message): + channel_id = file_info["originating_mm_post_channel_id"] + post_id = file_info["originating_mm_post_id"] + + # TODO Comment below line, this is for testing purposes + # channel_id = MMApi().get_channel_id("bestuur-dev") + channel = ChannelApi( + channel_id=channel_id, + user=mattermost_client.users[config["mattermost"]["selected_user"]], + ) + + try: + channel.create_threaded_post( + post_id, + f"{message}", + ) + except mattermostdriver.exceptions.InvalidOrMissingParameters as e: + # This will occur when we try to react to a file in a channel that is not the same as the originating channel. + unique_post_url = f"{config['mattermost']['server_url']}/pl/{post_id}" + channel.create_post( + f"{unique_post_url}\n\n{message}", + ) + + +def report_newly_found_file(file_info): + git_url = f"https://{config['gitea']['server_url']}/{config['gitea']['remote_org']}/{config['gitea']['remote_repo']}" + message = f"I found a new CodiMD file in this post! Making work of putting it on git :)\n - Requested location in the [drive]({git_url}): {file_info['metadata']['sync-to']}" + send_message(file_info, message) + + +def report_newly_found_but_invalid_file(file_info): + message = """Hi there! :wave: +I'm your friendly neighbourhood document sync bot. +I could synchronize this CodiMD file automatically to our Git DRIVE for safekeeping, but the necessary metadata block is not present. +You can easily add the correct info and I will do the rest of the work for you! + +Just add the following lines to your file, the location in your file is not important but at the top would be my recommendation. + +``` +:::spoiler git drive sync +- sync-to: +::: +```""" + send_message(file_info, message) + + +send_message( + { + "originating_mm_post_channel_id": "dm1abp4wfidezmig1yqyu53mmy", + "originating_mm_post_id": "dm1abp4wfidezmig1yqyu53mmy" + }, + "this is a test message" +) diff --git a/mattermost_objects.py b/mattermost_objects.py new file mode 100644 index 0000000..588d1ff --- /dev/null +++ b/mattermost_objects.py @@ -0,0 +1,137 @@ +from typing import Dict, List, NamedTuple + + +class MMUser(NamedTuple): + id: str + create_at: int + update_at: int + delete_at: int + username: str + first_name: str + last_name: str + nickname: str + email: str + auth_data: str + auth_service: str + roles: str + locale: str + timezone: dict + position: any + + is_bot: bool = None + bot_description: str = None + email_verified: bool = None + notify_props: dict = None + last_password_update: int = None + failed_attempts: int = None + mfa_active: bool = False + terms_of_service_id: str = None + terms_of_service_create_at: int = None + props: dict = {} + last_picture_update: int = None + + @staticmethod + def load(data): + try: + return MMUser(**data) + except TypeError as e: + print("[ERROR] Could not load dict into MMUser namedtuple") + print(str(e)) + + +class MMPostProps(NamedTuple): + from_webhook: str = False + override_icon_url: str = None + override_username: str = None + webhook_display_name: str = None + + channel_mentions: Dict = None + matterircd_krcggydky38kdcuubsc7fddc7w: str = None + matterircd_s4ptwhx7wfnx7qwexp1khorh7e: str = None + username: str = None + userId: str = None + old_header: str = None + new_header: str = None + old_purpose: str = None + new_purpose: str = None + old_displayname: str = None + new_displayname: str = None + remove_link_preview: str = None + removedUserId: str = None + addedUserId: str = None + removedUsername: str = None + addedUsername: str = None + message: str = None + attachments: str = None + from_bot: str = False + disable_group_highlight: str = None + + +class MMPost(NamedTuple): + channel_id: str + create_at: int + delete_at: int + edit_at: int + hashtags: str + id: str + is_pinned: bool + message: str + metadata: Dict + original_id: str + pending_post_id: str + root_id: str + type: str + update_at: int + user_id: str + parent_id: str = None + message_source: str = None + has_reactions: bool = None + file_ids: List[str] = None + props: MMPostProps = None + reply_count: int = None + last_reply_at: str = None + participants: any = None + + def from_human(self): + return self.props is None or ( + self.props.from_webhook is False and self.props.from_bot is False + ) + + @staticmethod + def load(data): + try: + props = None + if "props" in data: + try: + props: MMPostProps = MMPostProps(**data["props"]) + except TypeError as e: + print("[ERROR] Could not load dict into MMPostProps namedtuple") + print(str(e)) + del data["props"] + return MMPost(props=props, **data) + except TypeError as e: + print("[ERROR] Could not load dict into MMPost namedtuple") + print(str(e)) + + +class MMChannelPosts(NamedTuple): + prev_post_id: str + next_post_id: str + order: List[str] + posts: Dict[str, MMPost] + has_next: any + first_inaccessible_post_time: any + reply_count: any = None + disable_group_highlight: any = None + + @staticmethod + def load(data): + try: + posts: Dict[str, MMPost] = { + k: MMPost.load(v) for (k, v) in data["posts"].items() + } + del data["posts"] + return MMChannelPosts(posts=posts, **data) + except TypeError as e: + print("[ERROR] Could not load dict into MMUser namedtuple") + print(str(e)) diff --git a/requirements.txt b/requirements.txt index 20d86e3..b000791 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,4 +6,5 @@ PyPDF2==3.0.1 pdfplumber==0.11.4 requests==2.32.3 selenium==4.27.1 -seleniumbase==4.33.11 \ No newline at end of file +seleniumbase==4.33.11 +mattermostdriver \ No newline at end of file diff --git a/scraper_data.db b/scraper_data.db index 92784f94305e110b53dd9981f1f97b8ac501f4dd..a5c0cc374a1930e7a082c6f020001a1e67a75436 100644 GIT binary patch delta 396 zcma)$yH3JT7{@s@I_aq{#0$L;69*HTo*Sn1LlXd6HhnL4tNX!-;` zg0j1~`8qy;R!E)Am;8tCm;WV6K$3u`gCZzBmm8VB-o41X@^N7U&LMnJGijj_(pteBBF!}FY z_)~wRUq6Jjm1U`W8C|7uZqpiBrQQ1;X{L^*T>U{(t6`o)v~E-W(Ux1}&jMD8^~1EQ zmP$-uE+{rQL6%9fDPcgclisT3eI^(Yj2jGFnA^GA!|n7$E3daw)37X#OqSEoTYhN8 zozu>?q=A0qy1i~RjoY$x1ma*e>vg>-4129DNde float: return float(inp.replace(',', '.')) From fc2593b019e13a828fec62b4b4b1ced0c8a0b8bf Mon Sep 17 00:00:00 2001 From: Tybo Verslype Date: Tue, 24 Dec 2024 02:39:39 +0100 Subject: [PATCH 4/5] chore: added a favicon --- website/static/favicon.ico | Bin 0 -> 15406 bytes website/templates/index.html | 4 +++- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 website/static/favicon.ico diff --git a/website/static/favicon.ico b/website/static/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..d39f97f24a74088819d05702e54ede4b07c6d2bb GIT binary patch literal 15406 zcmeHOON?Ac6>USpB4Lp%fJG883Q7Jud|G($c-}x3p?14bE@jrtM~eKPuqbN8R_X& z*Q>hs+*|kFs=8GZi|vYiJoeB-F}!!jesNzc_VrjSwtKh!{l$l4u_n|#^_2PkL@YM+ zsaWibXoD{3q3iLg`q$YK|9xjNzS?{6od-A8vQJ+v=i*n_^QrA}_9h>eS+}{GdzwT# zS`uiVK>LFa?$W=L9m#~3Uzt6zRnFZ(TkmQ)>!EC)tLs_R$)6^aLceONN42gmK(H?!oh6GDjYopRN+virMu}?v zpp|cHiGP!0Ff(%An;AW?_-__7u-zqPzZ%34c8kCJa0B`$^4&+@{04pE^0~G) z(hyngSV9+D#(x&4FL!8t#l80A-bXr`6Qga-@vq|D)Rm0?e5mjE>*xo&Dx^5J+v_>& zzqEDjzwtkVZ)#xR6-15?_s2N18_ym*j88lLjV9vvgZBwM|LSOtH>hq-r%WdE&~~}- z!>i@|KFBe$UCwVI{`{K{>bF?OZ*0T1PFGbNsY!oVti3t@Jb2qNgdcx}F=%6T^$FNr z6X@T-qsCC%z1R`oVoSCTZL>|AYpaO5ls~5F8+$fS4L0=9o+cW<>)S7N;&L+8P7(-=Ouc*&F>E z`!oDiY&ZBf)>ag+Oln;5E@2K_FD$EvWn$nrVq#mtF8W)XCV!uPSpOpzyIyy{XH%0_ zmST2Z(Iy|ECvVG#I1!83CX*V2KIQ`$wEUVs^=tWK05;dsPcBVex<&qa47C2qqp$)! zFHLz{C3k$?vgXUSqHDIzFHL)ez|Vl-uX4_#4|0Eb zSlNT7uWg$=vWw$4{Fn;+4B5y0_(hE6l9jWne~B)tV4*$GCd2fzixZv`&nv#qja{^3 zCvw?0>`Y{|b%Miv8%*?zg~=h`{+0anZ_Fj@oHr`vUz)iblpVA*Pq96|cfz6D(lv=&r&=vpAaDD;*Yg|Ww zuGPPZK}!EMXtGZ|Y{ z-+Y_AbT+_OFr@YTs@6;^b0Z%1N;dW~KAbuAwwGJ<=YX(IBV{*x8OtA7F7_rhe`joo zLC^2`eG8MrKkjbbhc;&I;k4^L@s_u~y6o%O${6i;K}=dlr%%3Z%9(wIjJ?Z$OvPWUbw>kyDZitG!86BHoN;w3W8tn#q|SYF{t3`_ zzs0vH|Or2WBWrGbbOQf7y7^5g}n~; zsPQWoI#t_=OMPmMMZ2IKat=)+hDWVG!g|r`k8wNi4s;x@;7{3P{iVt+i6O3$$A?bO z@Hb-t>5}&u*bmFq)?a1BHpbWR`b+%slG;P-dDr!CS^sD~$A`EUfPMZtz#qJYzsRZA zp9|nOgL!odxs_?X9>kiI-`LLegW0hQ0ejMWbbtLtejP1_fAgEaGW=DphP(s(?Kp?x zQK(+{^Ue6DdC&*U`U7uQeyoYK{D%K<-^rKJcSiF+pkz|=hxk;lGx5>1|J&)-+r;2L zK7V3z_`?r6j_!Sd_JFv(q2)L9#NJh$i;h9Q{P*~A6kNs!*N%G(yY%ihZSNv~w z9Nvozp~|sMcQz*)S%$pNquoDxjwPPa-{RdEvu>9CwW72CF3bK~&7o>8QTy+7aQ{tN zT9PfFfS%t1h8u`^zmt6zY5za}F;9XNZR|xnweuv5J;{Av=xRwMwLTN0?ML4D z7Dz$%*+>-nX}k*oa<)*Y{6?b8JxTijmHUx&@!hMnFzmnUfa|xw z(*~Ty>ONmJCKcJ2G&YO5XGt?Gg2HojCp>?r2Or!I zzb!zwhyxBj_%LH*pl-+CpaELg4>pmu@ZnqV5QF=49Bur*V>7nP$s>P#4DDt$ey7io zh8N#2l|F@>(Ql9+{)fzA3g?b-r&6A3GAi2OQ7$ik8so8xc(C?YbFYgZG5SXqv{TUd z%mxCeOJS9ce;0EGkQ=wz-zOV*`x3$>%ebJD2U%*bIp(cjWOPw zlQ=x9+D34mt4pTfID%KUoLicDr8C(u7q$n{zi;_E1O@T?_A@^rdH#B^v(1ObuY3sk zxI6tFhiojNEl`#qeqW0J7JlYtc<#fq6F%x3DUheCPY=u|0)N1FDTCm5OkWiRe|Wq^ z|Fvv>%j7SaA7fsG`5~T9=?7)z5b=$e*tQ;X^HrJn8vSwKnG3y_T)j)W2^wb{NjLz4Q5= zWBbzK_)~!&zQ%JRwKssCvrCf-=f*1XvlVs%UogJOZ*0T0a}IqtMyr`w(8769L@W^e zVS5n!35$?k9Dk|5^Uw!7;ordCn|fw0ggj(_5rv^zo#e|&gS1F9+*a^gJZAqZ_^skm z*e;cJLoe?Ge}%ue`e)9GIUerwH!ubPp1~Nn!(6Om6#bjDNRzZ(dV=58zluMt`+nli zzat|f_j6q??XA4&1368LI$ob}B~;r?MgkOz4&wkrG)c;=CFCuZ&+xQ_r$wSJFkPkS35K7)8S z<=c(xr?wo@v#qZoS411m)r_r-%~kCI_=#EXo3h^#okZcM{f`axs+cY9_zvsM{U+sl z+UT}H**QOOj^n%+84opn+*jb9ukfny6Z~$xBK`^JpYx9871CS%IO`h=^5Q&Ae~w!J z(0+&oHiZ3Zp-bjR!B5QUG4sXb<+Bm@He5p?7wjQNW#&^IKWQ7Hf9OH-1Zq4g^l$KE|6**X z#`rt>b8*~jc@?&wnjH4}4qSC5}APRL@%!mW49uDov9Z^L;vtb6LvJ)^VuO+J$P=ei8* zI-FO&WyZ~4?+x~~;?Bp6*qrl@#-Qc}R~EM%>AyaH+N!JTaM|@==37-R75j$EI1j?! zF|sf3AfD?!iaqU|gWcyT_Q&^*F)D;V+#YudBQdygNc<%Be17rqv7yuD?iT+nTaT9? zv!w6&o@T7!qW7w<3{nnbtEji*qHT}u&vhQLao)8V!N9kfztsAzC%OL%I7`X9-z|PK zU%21w{gaR$1>1*xpSJev>(Am*exUKI^)qt!oz3wd0H2@ZioopRKi|zUaQ3}chtc+= zYVG9aQ^Co+-otu8l_7&e_A7n0r3X{ciry`h^*nVEz#K9_9}* z-!eBNV={8F+tI&2hbo-L7ZiSzJhb>*5(ju@v$=%#vUT` z=gg-|{v7Lin?H~67s~0c148;YHWk!=C4Su7=&KulMC_rRR6W12n0;Rm+u@u4!@zl7 z>>}zN)W04hSN{(F+pr&X?iFo6HRh}cxm=vp%3JuK4d!b!?i%`+_#cd~B zU0Gky?Qy^D{q0irC7zYHw>0hnKYv~3%m?j9;@@>*jk2|FWj{Koiv8pMkHXKkxFhpZ z Restaurant Scraper +