From 769dc04ce4ef05481577c83602156ba2ff2c495a Mon Sep 17 00:00:00 2001 From: Belissimo_T Date: Sat, 28 Sep 2024 20:05:49 +0200 Subject: [PATCH] bump stundenplan24-wrapper version and modify executors (cherry picked from commit 8c76d3d6b1e552b9e3787afcb69257324efd672c) --- backend/load_plans.py | 6 +++++- requirements.txt | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/backend/load_plans.py b/backend/load_plans.py index e00a01d..d47dc0a 100644 --- a/backend/load_plans.py +++ b/backend/load_plans.py @@ -28,7 +28,7 @@ def __init__(self, school_number: str, plan_downloader: PlanDownloader, plan_pro self.school_number = school_number self.plan_downloader = plan_downloader self.plan_processor = plan_processor - self._plan_compute_executor = concurrent.futures.ThreadPoolExecutor(max_workers=3) + self._plan_compute_executor = concurrent.futures.ProcessPoolExecutor(max_workers=1) async def check_infinite(self, interval: int = 60, *, once: bool = False, ignore_exceptions: bool = False): try: @@ -83,6 +83,8 @@ async def get_crawlers(session: requests.Session | None = None, creds_provider = get_creds_provider(Path("creds.json")) _creds = creds_provider.get_creds() + request_executor = concurrent.futures.ThreadPoolExecutor(max_workers=5) + crawlers = {} for school_name, data in _creds.items(): @@ -108,10 +110,12 @@ async def get_crawlers(session: requests.Session | None = None, for plan_client in client.substitution_plan_clients: plan_client.proxy_provider = proxy_provider plan_client.no_delay = True + plan_client.request_executor = request_executor for plan_client in client.indiware_mobil_clients: # plan_client.proxy_provider = proxy_provider plan_client.no_delay = True + plan_client.request_executor = request_executor else: client = None diff --git a/requirements.txt b/requirements.txt index a9a19e4..1998b6d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ Flask-WTF~=1.1.1 Flask-Compress~=1.13 pymongo~=4.3.3 python-dotenv~=1.0.0 -stundenplan24-wrapper@ git+https://github.com/Belissimo-T/stundenplan24-wrapper@bec25b2f +stundenplan24-wrapper@ git+https://github.com/Belissimo-T/stundenplan24-wrapper@4187258a Werkzeug~=2.3.6 discord-webhook~=1.3.0 brotlicffi~=1.1.0.0