From 6f5bb0212097f4f1ce24798012e3ad5c124ac8c0 Mon Sep 17 00:00:00 2001 From: Frank Villaro-Dixon Date: Mon, 6 May 2024 23:43:37 +0200 Subject: [PATCH] http: remove `chunk` capability There is no need to support chunks in python3 as the `read()` interface does that for us. Furthermore, this improves fetch performance in large queries (c.f. #111). Signed-off-by: Frank Villaro-Dixon --- overpy/__init__.py | 17 ++--------------- tests/test_request.py | 9 --------- 2 files changed, 2 insertions(+), 24 deletions(-) diff --git a/overpy/__init__.py b/overpy/__init__.py index 3de3c1b..2d79ce6 100644 --- a/overpy/__init__.py +++ b/overpy/__init__.py @@ -1,7 +1,6 @@ from datetime import datetime from decimal import Decimal -from functools import partial -from urllib.request import urlopen +from urllib.request import urlopen, Request from urllib.error import HTTPError from xml.sax import handler, make_parser import xml.etree.ElementTree @@ -60,7 +59,6 @@ class Overpass: """ Class to access the Overpass API - :param read_chunk_size: Max size of each chunk read from the server response :param url: Optional URL of the Overpass server. Defaults to http://overpass-api.de/api/interpreter :param xml_parser: The xml parser to use :param max_retry_count: Max number of retries (Default: default_max_retry_count) @@ -70,9 +68,6 @@ class Overpass: #: Global max number of retries (Default: 0) default_max_retry_count: ClassVar[int] = 0 - #: Max size of each chunk read from the server response - default_read_chunk_size: ClassVar[int] = 4096 - #: Global time to wait between tries (Default: 1.0s) default_retry_timeout: ClassVar[float] = 1.0 @@ -81,7 +76,6 @@ class Overpass: def __init__( self, - read_chunk_size: Optional[int] = None, url: Optional[str] = None, xml_parser: int = XML_PARSER_SAX, max_retry_count: int = None, @@ -94,11 +88,6 @@ def __init__( self._regex_extract_error_msg = re.compile(br"\(?P\") self._regex_remove_tag = re.compile(b"<[^>]*?>") - if read_chunk_size is None: - read_chunk_size = self.default_read_chunk_size - - #: The chunk size for this instance - self.read_chunk_size = read_chunk_size if max_retry_count is None: max_retry_count = self.default_max_retry_count @@ -151,9 +140,7 @@ def query(self, query: Union[bytes, str]) -> "Result": response = b"" try: with urlopen(self.url, query) as f: - f_read = partial(f.read, self.read_chunk_size) - for data in iter(f_read, b""): - response += data + response = f.read() except HTTPError as exc: f = exc diff --git a/tests/test_request.py b/tests/test_request.py index a545035..c7257bf 100644 --- a/tests/test_request.py +++ b/tests/test_request.py @@ -128,15 +128,6 @@ def do_POST(self): class TestQuery: - def test_chunk_size(self): - url, server = new_server_thread(HandleResponseJSON) - - api = overpy.Overpass(read_chunk_size=128) - api.url = url - result = api.query("[out:json];node(50.745,7.17,50.75,7.18);out;") - stop_server_thread(server) - assert len(result.nodes) > 0 - def test_overpass_syntax_error(self): url, server = new_server_thread(HandleOverpassBadRequest)