http: remove chunk capability

There is no need to support chunks in python3 as the `read()` interface does that for us. Furthermore, this improves fetch performance in large queries (c.f. DinoTools#111). Signed-off-by: Frank Villaro-Dixon <[email protected]>
Frankkkkk · May 7, 2024 · 6f5bb02 · 6f5bb02
1 parent 5ab8845
commit 6f5bb02
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 24 deletions.
diff --git a/overpy/__init__.py b/overpy/__init__.py
@@ -1,7 +1,6 @@
 from datetime import datetime
 from decimal import Decimal
-from functools import partial
-from urllib.request import urlopen
+from urllib.request import urlopen, Request
 from urllib.error import HTTPError
 from xml.sax import handler, make_parser
 import xml.etree.ElementTree
@@ -60,7 +59,6 @@ class Overpass:
     """
     Class to access the Overpass API
 
-    :param read_chunk_size: Max size of each chunk read from the server response
     :param url: Optional URL of the Overpass server. Defaults to http://overpass-api.de/api/interpreter
     :param xml_parser: The xml parser to use
     :param max_retry_count: Max number of retries (Default: default_max_retry_count)
@@ -70,9 +68,6 @@ class Overpass:
     #: Global max number of retries (Default: 0)
     default_max_retry_count: ClassVar[int] = 0
 
-    #: Max size of each chunk read from the server response
-    default_read_chunk_size: ClassVar[int] = 4096
-
     #: Global time to wait between tries (Default: 1.0s)
     default_retry_timeout: ClassVar[float] = 1.0
 
@@ -81,7 +76,6 @@ class Overpass:
 
     def __init__(
             self,
-            read_chunk_size: Optional[int] = None,
             url: Optional[str] = None,
             xml_parser: int = XML_PARSER_SAX,
             max_retry_count: int = None,
@@ -94,11 +88,6 @@ def __init__(
 
         self._regex_extract_error_msg = re.compile(br"\<p\>(?P<msg>\<strong\s.*?)\</p\>")
         self._regex_remove_tag = re.compile(b"<[^>]*?>")
-        if read_chunk_size is None:
-            read_chunk_size = self.default_read_chunk_size
-
-        #: The chunk size for this instance
-        self.read_chunk_size = read_chunk_size
 
         if max_retry_count is None:
             max_retry_count = self.default_max_retry_count
@@ -151,9 +140,7 @@ def query(self, query: Union[bytes, str]) -> "Result":
             response = b""
             try:
                 with urlopen(self.url, query) as f:
-                    f_read = partial(f.read, self.read_chunk_size)
-                    for data in iter(f_read, b""):
-                        response += data
+                    response = f.read()
             except HTTPError as exc:
                 f = exc
 

diff --git a/tests/test_request.py b/tests/test_request.py
@@ -128,15 +128,6 @@ def do_POST(self):
 
 
 class TestQuery:
-    def test_chunk_size(self):
-        url, server = new_server_thread(HandleResponseJSON)
-
-        api = overpy.Overpass(read_chunk_size=128)
-        api.url = url
-        result = api.query("[out:json];node(50.745,7.17,50.75,7.18);out;")
-        stop_server_thread(server)
-        assert len(result.nodes) > 0
-
     def test_overpass_syntax_error(self):
         url, server = new_server_thread(HandleOverpassBadRequest)