Sync

config.py

@@ -0,0 +1,8 @@
+import tomllib
+from pprint import pprint
+
+with open("config.toml", mode="rb") as config_toml:
+    config = tomllib.load(config_toml)
+
+
+# pprint(config)

db.json

@@ -1,39 +1,27 @@
 {
     "files": {
-        "hlds_files\\bicyclette.hlds": {
-            "local_file_path": "hlds_files\\bicyclette.hlds",
+        "hlds_files/bicyclette.hlds": {
+            "local_file_path": "hlds_files/bicyclette.hlds",
             "metadata": {
-                "sync-to": "menus\\la_bicylette.hlds"
+                "sync-to": "menus/la_bicylette.hlds"
             }
         },
-        "hlds_files\\bocca_ovp.hlds": {
-            "local_file_path": "hlds_files\\bocca_ovp.hlds",
+        "hlds_files/bocca_ovp.hlds": {
+            "local_file_path": "hlds_files/bocca_ovp.hlds",
             "metadata": {
-                "sync-to": "menus\\bocca_ovp.hlds"
+                "sync-to": "menus/bocca_ovp.hlds"
             }
         },
-        "hlds_files\\metropol.hlds": {
-            "local_file_path": "hlds_files\\metropol.hlds",
+        "hlds_files/pizza_donna.hlds": {
+            "local_file_path": "hlds_files/pizza_donna.hlds",
             "metadata": {
-                "sync-to": "menus\\pitta_metropol.hlds"
+                "sync-to": "menus/prima_donna.hlds"
             }
         },
-        "hlds_files\\pizza_donna.hlds": {
-            "local_file_path": "hlds_files\\pizza_donna.hlds",
+        "hlds_files/simpizza.hlds": {
+            "local_file_path": "hlds_files/simpizza.hlds",
             "metadata": {
-                "sync-to": "menus\\prima_donna.hlds"
-            }
-        },
-        "hlds_files\\s5.hlds": {
-            "local_file_path": "hlds_files\\s5.hlds",
-            "metadata": {
-                "sync-to": "menus\\s5.hlds"
-            }
-        },
-        "hlds_files\\simpizza.hlds": {
-            "local_file_path": "hlds_files\\simpizza.hlds",
-            "metadata": {
-                "sync-to": "menus\\simpizza.hlds"
+                "sync-to": "menus/simpizza.hlds"
             }
         }
     }

db.py → db_file.py

@@ -54,6 +54,15 @@ def is_file_different(hlds_file: str, menu_file: str) -> bool:
     return hlds_hash != menu_hash
 
 
+def dos2unix(path: str) -> str:
+    """
+    Converts Windows-style backslashes in a file path to Unix-style forward slashes.
+    :param path: The input file path (string)
+    :return: The normalized file path with forward slashes
+    """
+    return path.replace("\\", "/")
+
+
 def get_manual_file_mapping() -> Dict[str, str]:
     """
     Creates a manual mapping of file names between the hlds_menus directory and the menus directory.
@@ -64,25 +73,46 @@ def get_manual_file_mapping() -> Dict[str, str]:
 
     # Manual mapping of file names
     file_mapping = {
-        os.path.join(hlds_dir, "bicyclette.hlds"): os.path.join(menu_dir, "la_bicylette.hlds"),
-        os.path.join(hlds_dir, "bocca_ovp.hlds"): os.path.join(menu_dir, "bocca_ovp.hlds"),
-        os.path.join(hlds_dir, "metropol.hlds"): os.path.join(menu_dir, "pitta_metropol.hlds"),
-        os.path.join(hlds_dir, "pizza_donna.hlds"): os.path.join(menu_dir, "prima_donna.hlds"),
-        os.path.join(hlds_dir, "s5.hlds"): os.path.join(menu_dir, "s5.hlds"),
-        os.path.join(hlds_dir, "simpizza.hlds"): os.path.join(menu_dir, "simpizza.hlds"),
+        dos2unix(os.path.join(hlds_dir, "bicyclette.hlds")): dos2unix(os.path.join(menu_dir, "la_bicylette.hlds")),
+        dos2unix(os.path.join(hlds_dir, "bocca_ovp.hlds")): dos2unix(os.path.join(menu_dir, "bocca_ovp.hlds")),
+        dos2unix(os.path.join(hlds_dir, "metropol.hlds")): dos2unix(os.path.join(menu_dir, "pitta_metropol.hlds")),
+        dos2unix(os.path.join(hlds_dir, "pizza_donna.hlds")): dos2unix(os.path.join(menu_dir, "prima_donna.hlds")),
+        dos2unix(os.path.join(hlds_dir, "s5.hlds")): dos2unix(os.path.join(menu_dir, "s5.hlds")),
+        dos2unix(os.path.join(hlds_dir, "simpizza.hlds")): dos2unix(os.path.join(menu_dir, "simpizza.hlds")),
         # Add more mappings here as needed
     }
 
     return file_mapping
 
 
+def get_mapped_path(file_path: str) -> str:
+    """
+    Maps a given file path using the manual mapping or returns the same path if not in the mapping.
+    Ensures the returned path uses forward slashes.
+    :param file_path: The input file path
+    :return: The mapped file path or the original path
+    """
+    file_mapping = get_manual_file_mapping()
+    normalized_path = dos2unix(file_path)
+    return file_mapping.get(normalized_path, normalized_path)
+
+
 def test_file_comparison():
     """
     Compares all files based on the manual mapping and prints whether they are different or identical.
     """
-    file_mapping = get_manual_file_mapping()
+    hlds_dir = "hlds_files"
+    menu_dir = "menus"
+
+    # Get a list of files in the `hlds_files` directory
+    hlds_files = [
+        dos2unix(os.path.join(hlds_dir, f))
+        for f in os.listdir(hlds_dir) if os.path.isfile(os.path.join(hlds_dir, f))
+    ]
+
+    for hlds_file in hlds_files:
+        menu_file = get_mapped_path(hlds_file)
 
-    for hlds_file, menu_file in file_mapping.items():
         if not os.path.exists(hlds_file):
             print(f"{hlds_file} does not exist. Skipping...")
             continue

hlds_files/simpizza.hlds

@@ -2355,6 +2355,55 @@ dish pasta_chicken_picanto: Pasta Chicken picanto -- Roomsaus, rode pesto, kip,
 		zakje_parmezaanse_kaas: Zakje Parmezaanse kaas  € 1.0
 		bestek: bestek  € 0.35
 dish pasta_deal: Pasta Deal -- Kleine portie pasta, lookbrood naar keuze, frisdrank € 18.95
+	single_choice Spaghetti/penne: Welke Spaghetti/penne
+		spaghetti: spaghetti  € 0.0
+		penne: penne  € 0.0
+	single_choice Drank?: Welke Drank?
+		cola: Cola  € 0.0
+		cola_zero: Cola Zero  € 0.0
+		fanta: Fanta  € 0.0
+		fanta_exotic: Fanta Exotic  € 0.0
+		fanta_cassis: Fanta Cassis  € 0.0
+		fanta_strawberry_&_kiwi: Fanta Strawberry & Kiwi  € 0.0
+		tropico: Tropico  € 0.0
+		ice_tea: Ice Tea  € 0.0
+		ice_tea_peach: Ice Tea Peach  € 0.0
+		sprite: Sprite  € 0.0
+		spa_plat: Spa Plat  € 0.0
+		spa_bruis: Spa Bruis  € 0.0
+		uludag: Uludag  € 0.0
+		oasis_tropical: Oasis Tropical  € 0.0
+	single_choice Garnering: Welke Garnering
+		scampi: Scampi  € 1.5
+		kip: kip  € 1.5
+		ham: Ham  € 1.5
+		extra_kaas: extra kaas  € 1.5
+		extra_saus: extra saus  € 1.5
+		zakje_parmezaanse_kaas: Zakje Parmezaanse kaas  € 1.0
+		bestek: bestek  € 0.35
+	single_choice Pasta: Welke Pasta
+		pasta_bolognese: Pasta Bolognese  € 0.0
+		pasta_kaassaus: Pasta kaassaus  € 0.0
+		pasta_kip_en_kaassaus: Pasta kip en kaassaus  € 0.0
+		pasta_ham_en_kaassaus: Pasta ham en kaassaus  € 0.0
+		pasta_milano: Pasta Milano  € 0.0
+		pasta_scampi: Pasta scampi  € 0.0
+		pasta_multi_cheese: Pasta Multi cheese  € 0.0
+		pasta_veggie: Pasta Veggie  € 0.0
+		pasta_pesto_chicken: Pasta Pesto Chicken  € 0.0
+		pasta_veggie: Pasta Veggie  € 0.0
+		pasta_exotique: Pasta Exotique  € 0.0
+		pasta_chicken_pesto_spinazie: Pasta Chicken pesto spinazie  € 0.0
+		pasta_gambaretti_spinazie: Pasta Gambaretti spinazie  € 0.0
+		pasta_chicken_arabiata: Pasta Chicken arabiata  € 0.0
+		pasta_chicken_picanto: Pasta Chicken picanto  € 0.0
+	single_choice Lookbrood: Welke Lookbrood
+		lookbrood_natuur: Lookbrood natuur  € 0.0
+		lookbrood_kaas: Lookbrood kaas  € 0.0
+		lookbrood_kaas_&_ham: Lookbrood kaas & ham  € 0.0
+		lookbrood_kaas_&_tomaat: Lookbrood kaas & tomaat  € 0.0
+		lookbrood_kaas_&_salami: Lookbrood kaas & salami  € 0.0
+		lookbrood_kaas_&_kip: Lookbrood kaas & kip  € 0.0
 dish pasta_exotique: Pasta Exotique -- Tomatenroomsaus, ananas, champignons, chili, cherrytomaat, paprika € 13.95
 	single_choice Spaghetti/penne: Welke Spaghetti/penne
 		spaghetti: spaghetti  € 0.0

main.py

@@ -149,7 +149,7 @@ def run_scrapers(
     # Default values
     default_run_everything: bool = False
     default_use_parallelism: bool = False
-    default_restaurant_names = ["bocca_ovp"]
+    default_restaurant_names = ["s5"]
 
     # Parse command-line arguments
     args = parse_arguments()

mattermost_client.py

@@ -0,0 +1,178 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+import pprint as pp
+from abc import ABC, abstractmethod
+from enum import Enum
+from typing import Dict
+from config import config
+
+from colored import Style
+from mattermostdriver import Driver
+
+from mattermost_objects import MMChannelPosts
+from utils import timer
+
+pp = pp.PrettyPrinter(indent=2)
+
+
+class LogLevel(Enum):
+    INFO = "INFO"
+    ERROR = "ERROR"
+
+
+class User(ABC):
+    @abstractmethod
+    def credentials_dict(self) -> dict:
+        pass
+
+class TokenUser(User):
+    def __init__(self, token):
+        self.token = token
+
+    def credentials_dict(self) -> dict:
+        return {"token": self.token}
+
+    def __repr__(self):
+        return "TokenUser<token: {}>".format(self.token)
+
+
+users: {str: [User]} = {}
+
+
+def loadusers():
+    usr = None
+    for name, data in config["mattermost"]["users"].items():
+        if "token" in data:
+            usr = TokenUser(token=data["token"])
+        else:
+            print("Invalid user '{}' in toml file".format(name))
+            exit(1)
+        users[name] = usr
+
+
+loadusers()
+
+
+def merge_dict(a: dict, b: dict) -> dict:
+    return {**a, **b}
+
+
+class MMApi(Driver):
+    def __init__(self, user: User = users["tyboro"]):
+        print(f"Initializing MMApi client for user {user}")
+        Driver.__init__(
+            self,
+            merge_dict(
+                {
+                    "url": "mattermost.zeus.gent",
+                    "port": 443,
+                    "debug": False,
+                },
+                user.credentials_dict(),
+            ),
+        )
+        self.login()
+        self.user_id = self.users.get_user(user_id="me")["id"]
+        self.team_id = self.teams.get_team_by_name("zeus")["id"]
+        print(" = Creating mattermost client")
+        print(f" =  - User: {self.user_id}")
+        print(f" =  - Team: {self.team_id}")
+
+    @staticmethod
+    def print_response(resp, title="Response"):
+        print("--------")
+        print(Style.BOLD + title + Style.RESET)
+        pp.pprint(resp)
+
+    def log(self, text: str, log_level: LogLevel = LogLevel.INFO):
+        print(f"{Style.BOLD}[{log_level.value}]{Style.RESET} {text}")
+
+    def get_channel_id(self, channel_name):
+        resp = self.channels.get_channel_by_name(self.team_id, channel_name)
+        id = resp["id"]
+        self.log(f"Fetching channel id for {channel_name}: {id}")
+        return id
+
+    @timer
+    def get_posts_for_channel(self, channel_id, since):
+        print(f"Fetching posts for {channel_id} since {since}")
+        page_size = 200
+        page_i = 0
+        data = {}
+        more = True
+        while more:
+            resp = self.posts.get_posts_for_channel(
+                channel_id,
+                params={"page": page_i, "per_page": page_size, "since": since},
+            )
+            page_i += 1
+            print(f"Fetching page {page_i}")
+            # print("-", end=" ")
+
+            paged_data = resp["posts"]
+            paged_count = len(paged_data)
+
+            if since != 0:
+                # The mattermost api is absolutely retarted
+                # If you add the since parameter and it's different then 0 it will give you 1000 posts max.
+                # It will not respect you page_index or page_size.
+                more = False
+            else:
+                if paged_count < page_size:
+                    more = False
+
+            # Transform the data into something more sensible or practical
+            if type(paged_data) is list:
+                paged_data = {item["id"]: item for item in paged_data}
+
+            # Append the paged_data to our global data variable
+            data = {**data, **paged_data}
+        print()
+
+        self.log(f"Post count: {len(data)}")
+        return data
+
+
+class ChannelApi(MMApi):
+    def __init__(self, channel_name=None, channel_id=None, user=None):
+        MMApi.__init__(self, user)
+        assert channel_name is not None or channel_id != None
+
+        if channel_name is not None:
+            self.channel_id = self.get_channel_id(channel_name)
+        if channel_id is not None:
+            self.channel_id = channel_id
+
+    def create_post(self, message: str, props: Dict = None) -> None:
+        resp = self.posts.create_post(
+            options={"channel_id": self.channel_id, "message": message, "props": props}
+        )
+        self.log(f'Message successfully created: "{message}"')
+
+    def create_threaded_post(
+        self, post_id: str, message: str, props: Dict = None
+    ) -> None:
+        resp = self.posts.create_post(
+            options={
+                "channel_id": self.channel_id,
+                "message": message,
+                "root_id": post_id,
+                "props": props,
+            }
+        )
+        self.log(f'Message successfully created: "{message}"')
+        # print_response("Create post", resp)
+
+
+if __name__ == "__main__":
+    foo = MMApi(user=users["flynn"])
+
+    # all_posts = foo.get_all_posts()
+
+    channel = foo.channels.get_channel_by_name(
+        foo.team_id,
+        "bestuur",
+    )
+    channel_id = channel["id"]
+    resp = foo.posts.get_posts_for_channel(channel_id, params={"per_page": 200})
+    channel_posts: MMChannelPosts = MMChannelPosts.load(resp)