From 90a04a952de8d8b421b959a867ef0e9416a21d3e Mon Sep 17 00:00:00 2001 From: yoori Date: Sun, 17 Nov 2024 09:46:45 +0000 Subject: [PATCH 1/9] Adapt for parse cookie partitionKey when it is str --- zendriver/cdp/network.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/zendriver/cdp/network.py b/zendriver/cdp/network.py index 557480c..edf8138 100644 --- a/zendriver/cdp/network.py +++ b/zendriver/cdp/network.py @@ -1491,6 +1491,9 @@ def to_json(self) -> T_JSON_DICT: @classmethod def from_json(cls, json: T_JSON_DICT) -> CookiePartitionKey: + if isinstance(json, str): + # Some chrome versions return partitionKey as string + return cls(json, False) return cls( top_level_site=str(json["topLevelSite"]), has_cross_site_ancestor=bool(json["hasCrossSiteAncestor"]), From 1d6ea84f233848a84965ff7b78b4cd3b917de22f Mon Sep 17 00:00:00 2001 From: yoori Date: Sun, 17 Nov 2024 09:49:00 +0000 Subject: [PATCH 2/9] Added chrome stderr output to Exception when chrome start failed (help to debug specific chrome installations) --- zendriver/core/browser.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/zendriver/core/browser.py b/zendriver/core/browser.py index 8c8cd93..aaa7ee7 100644 --- a/zendriver/core/browser.py +++ b/zendriver/core/browser.py @@ -358,6 +358,12 @@ async def start(self) -> Browser: break if not self.info: + stderr = None + try : + _, stderr_bytes = await self._process.communicate() + stderr = stderr_bytes.decode()[:1000] + except Exception : + pass raise Exception( ( """ @@ -366,7 +372,8 @@ async def start(self) -> Browser: --------------------- One of the causes could be when you are running as root. In that case you need to pass no_sandbox=True - """ + """ + + ("Browser error output:" + stderr if stderr else '') ) ) From 12806e4a10c79b2a5986b289390816124b72ea02 Mon Sep 17 00:00:00 2001 From: yoori Date: Sun, 17 Nov 2024 09:51:56 +0000 Subject: [PATCH 3/9] adapt for websockets 14.0 : websocket.closed attribute removed --- zendriver/core/browser.py | 2 -- zendriver/core/connection.py | 9 +++++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/zendriver/core/browser.py b/zendriver/core/browser.py index aaa7ee7..c2a9015 100644 --- a/zendriver/core/browser.py +++ b/zendriver/core/browser.py @@ -709,8 +709,6 @@ async def save(self, file: PathLike = ".session.dat", pattern: str = ".*"): # return # if not connection.websocket: # return - # if connection.websocket.closed: - # return cookies = await self.get_all(requests_cookie_format=False) included_cookies = [] for cookie in cookies: diff --git a/zendriver/core/connection.py b/zendriver/core/connection.py index a0ee46c..e73c36f 100644 --- a/zendriver/core/connection.py +++ b/zendriver/core/connection.py @@ -230,7 +230,7 @@ def target(self, target: cdp.target.TargetInfo): def closed(self): if not self.websocket: return True - return self.websocket.closed + return (not self.websocket) def add_handler( self, @@ -282,7 +282,7 @@ async def aopen(self, **kw): :return: """ - if not self.websocket or self.websocket.closed: + if not self.websocket: try: self.websocket = await websockets.connect( self.websocket_url, @@ -308,11 +308,12 @@ async def aclose(self): """ closes the websocket connection. should not be called manually by users. """ - if self.websocket and not self.websocket.closed: + if self.websocket: if self.listener and self.listener.running: self.listener.cancel() self.enabled_domains.clear() await self.websocket.close() + self.websocket = None logger.debug("\n❌ closed websocket connection to %s", self.websocket_url) async def sleep(self, t: Union[int, float] = 0.25): @@ -411,7 +412,7 @@ async def send( :return: """ await self.aopen() - if not self.websocket or self.closed: + if not self.websocket: return if self._owner: browser = self._owner From 05588318d453e10562a6c199b2437c68f4e232c7 Mon Sep 17 00:00:00 2001 From: yoori Date: Sun, 17 Nov 2024 09:54:16 +0000 Subject: [PATCH 4/9] Added getter for get browser process output (help to debug chrome installations) --- zendriver/core/browser.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/zendriver/core/browser.py b/zendriver/core/browser.py index c2a9015..a9fe5b2 100644 --- a/zendriver/core/browser.py +++ b/zendriver/core/browser.py @@ -276,6 +276,12 @@ async def get( await connection.sleep(0.25) return connection + async def get_outputs(self): + if self._process: + stdout_bytes, stderr_bytes = await self._process.communicate() + return [stdout_bytes, stderr_bytes] + return None + async def start(self) -> Browser: """launches the actual browser""" if not self: From 570df122f03d8d8e12dc7616339c1070b05a1c3d Mon Sep 17 00:00:00 2001 From: yoori Date: Sun, 17 Nov 2024 10:00:43 +0000 Subject: [PATCH 5/9] Fix: don't drop CDP message id counter (this can be reason for id repeatance and message responses losing) --- zendriver/core/connection.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/zendriver/core/connection.py b/zendriver/core/connection.py index e73c36f..77d17e5 100644 --- a/zendriver/core/connection.py +++ b/zendriver/core/connection.py @@ -201,7 +201,7 @@ def __init__( ): super().__init__() self._target = target - self.__count__ = itertools.count(0) + self._cdp_id_generator = itertools.count(0) self._owner = _owner self.websocket_url: str = websocket_url self.websocket = None @@ -426,9 +426,7 @@ async def send( try: tx = Transaction(cdp_obj) tx.connection = self - if not self.mapper: - self.__count__ = itertools.count(0) - tx.id = next(self.__count__) + tx.id = next(self._cdp_id_generator) self.mapper.update({tx.id: tx}) if not _is_update: await self._register_handlers() @@ -647,9 +645,7 @@ async def listener_loop(self): try: event = cdp.util.parse_json_event(message) event_tx = EventTransaction(event) - if not self.connection.mapper: - self.connection.__count__ = itertools.count(0) - event_tx.id = next(self.connection.__count__) + event_tx.id = next(self.connection._cdp_id_generator) self.connection.mapper[event_tx.id] = event_tx except Exception as e: logger.info( From 97390c18fa333420cdf4c6c70a36e20e0ccfebd6 Mon Sep 17 00:00:00 2001 From: yoori Date: Sun, 17 Nov 2024 21:12:34 +0000 Subject: [PATCH 6/9] More accurate handling of exceptions, that can interrupt websocket reading loop --- zendriver/core/connection.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/zendriver/core/connection.py b/zendriver/core/connection.py index 77d17e5..891e5d8 100644 --- a/zendriver/core/connection.py +++ b/zendriver/core/connection.py @@ -602,15 +602,17 @@ async def listener_loop(self): # breathe # await asyncio.sleep(self.time_before_considered_idle / 10) continue - except (Exception,) as e: - # break on any other exception - # which is mostly socket is closed or does not exist - # or is not allowed - + except (websockets.exceptions.ConnectionClosedError,) as e: logger.debug( "connection listener exception while reading websocket:\n%s", e ) break + except (Exception,) as e: + # we don't expect here other exceptions, need to debug if it will appear. + logger.exception( + "connection listener exception while reading websocket" + ) + break if not self.running: # if we have been cancelled or otherwise stopped running From b9254fb01716ba6262159748af465f379e6eda26 Mon Sep 17 00:00:00 2001 From: yoori Date: Sun, 17 Nov 2024 21:30:29 +0000 Subject: [PATCH 7/9] adapt for websockets 14.0 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c4dc455..927cf17 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ dependencies = [ "asyncio-atexit>=1.0.1", "deprecated>=1.2.14", "mss>=9.0.2", - "websockets>=13.1,<14", + "websockets>=13.1", ] [build-system] From 8f5ae76f70cf37bf951b74006ce0d42d20639772 Mon Sep 17 00:00:00 2001 From: yoori Date: Sun, 17 Nov 2024 21:37:38 +0000 Subject: [PATCH 8/9] Added lost dependency of package on requests module --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 927cf17..c010d8d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,7 @@ classifiers = [ ] requires-python = ">=3.10" dependencies = [ + "requests", "asyncio-atexit>=1.0.1", "deprecated>=1.2.14", "mss>=9.0.2", From ab058aa223ac53d5e31587e1c6a683da795e9578 Mon Sep 17 00:00:00 2001 From: Yuri Date: Mon, 18 Nov 2024 03:14:08 +0300 Subject: [PATCH 9/9] Update zendriver/core/browser.py Co-authored-by: Stephan Lensky <8302875+stephanlensky@users.noreply.github.com> --- zendriver/core/browser.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/zendriver/core/browser.py b/zendriver/core/browser.py index a9fe5b2..cebdf9e 100644 --- a/zendriver/core/browser.py +++ b/zendriver/core/browser.py @@ -276,11 +276,10 @@ async def get( await connection.sleep(0.25) return connection - async def get_outputs(self): - if self._process: - stdout_bytes, stderr_bytes = await self._process.communicate() - return [stdout_bytes, stderr_bytes] - return None + async def communicate(self) -> tuple[bytes, bytes]: + if self._process is None: + raise ValueError("Browser process not running") + return await self._process.communicate() async def start(self) -> Browser: """launches the actual browser"""