From b966ab43ac91d576ddc0759e6160a5b4bc56bb42 Mon Sep 17 00:00:00 2001 From: AnonTester <40003252+AnonTester@users.noreply.github.com> Date: Mon, 5 Oct 2020 20:11:16 +0100 Subject: [PATCH 1/6] v11.0.2: Remove Python requirement for Kodi 18/19 compatibility --- addon.xml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/addon.xml b/addon.xml index b9d8f35..43a0865 100644 --- a/addon.xml +++ b/addon.xml @@ -1,10 +1,10 @@ - + @@ -17,5 +17,6 @@ BSD https://github.com/marcelveldt/script.module.cherrypy all + v11.0.2: Remove Python requirement for Kodi18/19 compatibility From 5023bfb9baf009459a22983f13be788f36f26316 Mon Sep 17 00:00:00 2001 From: AnonTester <40003252+AnonTester@users.noreply.github.com> Date: Tue, 6 Oct 2020 21:49:00 +0100 Subject: [PATCH 2/6] v17.4.2: - update CherryPy to last Py2/3 compatible version 17.4.2 - update cheroot to version 8.4.5 - add more-itertools-5.0.0 dependency - add contextlib2-0.6.0.post1 dependency - add zc.lockfile-2.0 dependency - add jaraco.functools-2.0 dependency --- addon.xml | 16 +- lib/cheroot/LICENSE.md | 4 +- lib/cheroot/PKG-INFO | 148 ++ lib/cheroot/__init__.py | 5 +- lib/cheroot/__main__.py | 6 + lib/cheroot/_compat.py | 390 +-- lib/cheroot/cli.py | 241 ++ lib/cheroot/connections.py | 292 +++ lib/cheroot/errors.py | 25 +- lib/cheroot/makefile.py | 103 +- lib/cheroot/server.py | 1464 +++++++---- lib/cheroot/ssl/__init__.py | 16 +- lib/cheroot/ssl/builtin.py | 455 +++- lib/cheroot/ssl/pyopenssl.py | 293 ++- lib/cheroot/test/conftest.py | 69 + lib/cheroot/test/helper.py | 194 +- lib/cheroot/test/test.pem | 38 - lib/cheroot/test/test__compat.py | 66 + lib/cheroot/test/test_cli.py | 92 + lib/cheroot/test/test_compat.py | 31 - lib/cheroot/test/test_config_server.py | 129 - lib/cheroot/test/test_conn.py | 1775 ++++++++----- lib/cheroot/test/test_core.py | 568 +++- lib/cheroot/test/test_dispatch.py | 55 + lib/cheroot/test/test_errors.py | 30 + lib/cheroot/test/test_http.py | 260 -- lib/cheroot/test/test_makefile.py | 52 + lib/cheroot/test/test_server.py | 342 +++ lib/cheroot/test/test_ssl.py | 731 ++++++ lib/cheroot/test/test_wsgi.py | 58 + lib/cheroot/test/test_wsgiapps.py | 99 - lib/cheroot/test/webtest.py | 488 ++-- lib/cheroot/testing.py | 153 ++ lib/cheroot/workers/threadpool.py | 213 +- lib/cheroot/wsgi.py | 147 +- lib/cherrypy/PKG-INFO | 141 + lib/cherrypy/__init__.py | 116 +- lib/cherrypy/__main__.py | 4 +- lib/cherrypy/_cpchecker.py | 47 +- lib/cherrypy/_cpcompat.py | 261 +- lib/cherrypy/_cpconfig.py | 73 +- lib/cherrypy/_cpdispatch.py | 45 +- lib/cherrypy/_cperror.py | 129 +- lib/cherrypy/_cplogging.py | 94 +- lib/cherrypy/_cpmodpy.py | 20 +- lib/cherrypy/_cpnative_server.py | 37 +- lib/cherrypy/_cpreqbody.py | 83 +- lib/cherrypy/_cprequest.py | 280 +- lib/cherrypy/_cpserver.py | 73 +- lib/cherrypy/_cptools.py | 79 +- lib/cherrypy/_cptree.py | 52 +- lib/cherrypy/_cpwsgi.py | 10 +- lib/cherrypy/_cpwsgi_server.py | 63 +- lib/cherrypy/_helper.py | 57 +- lib/cherrypy/daemon.py | 3 +- lib/cherrypy/lib/__init__.py | 29 +- lib/cherrypy/lib/auth.py | 97 - lib/cherrypy/lib/auth_basic.py | 56 +- lib/cherrypy/lib/auth_digest.py | 194 +- lib/cherrypy/lib/caching.py | 32 +- lib/cherrypy/lib/covercp.py | 11 +- lib/cherrypy/lib/cpstats.py | 25 +- lib/cherrypy/lib/cptools.py | 42 +- lib/cherrypy/lib/encoding.py | 51 +- lib/cherrypy/lib/gctools.py | 4 +- lib/cherrypy/lib/httpauth.py | 376 --- lib/cherrypy/lib/httputil.py | 224 +- lib/cherrypy/lib/jsontools.py | 6 - lib/cherrypy/lib/lockfile.py | 142 - lib/cherrypy/lib/profiler.py | 6 +- lib/cherrypy/lib/reprconf.py | 72 +- lib/cherrypy/lib/sessions.py | 80 +- lib/cherrypy/lib/static.py | 59 +- lib/cherrypy/lib/xmlrpcutil.py | 38 +- lib/cherrypy/process/__init__.py | 7 +- lib/cherrypy/process/plugins.py | 168 +- lib/cherrypy/process/servers.py | 23 +- lib/cherrypy/process/win32.py | 10 +- lib/cherrypy/process/wspbus.py | 90 +- lib/cherrypy/scaffold/__init__.py | 4 + lib/cherrypy/scaffold/apache-fcgi.conf | 2 +- lib/cherrypy/scaffold/example.conf | 2 +- .../static/made_with_cherrypy_small.png | Bin 7455 -> 6347 bytes lib/cherrypy/test/_test_decorators.py | 3 +- lib/cherrypy/test/_test_states_demo.py | 6 +- lib/cherrypy/test/benchmark.py | 64 +- lib/cherrypy/test/helper.py | 21 +- lib/cherrypy/test/logtest.py | 30 +- lib/cherrypy/test/modpy.py | 3 +- lib/cherrypy/test/modwsgi.py | 8 +- lib/cherrypy/test/sessiondemo.py | 8 +- lib/cherrypy/test/static/dirback.jpg | Bin 18238 -> 16585 bytes lib/cherrypy/test/static/index.html | 2 +- lib/cherrypy/test/test_auth_basic.py | 46 +- lib/cherrypy/test/test_auth_digest.py | 149 +- lib/cherrypy/test/test_caching.py | 68 +- lib/cherrypy/test/test_compat.py | 5 +- lib/cherrypy/test/test_config.py | 20 +- lib/cherrypy/test/test_conn.py | 71 +- lib/cherrypy/test/test_core.py | 43 +- .../test/test_dynamicobjectmapping.py | 1 - lib/cherrypy/test/test_encoding.py | 73 +- lib/cherrypy/test/test_http.py | 85 +- lib/cherrypy/test/test_httpauth.py | 195 -- lib/cherrypy/test/test_httplib.py | 30 - lib/cherrypy/test/test_httputil.py | 80 + lib/cherrypy/test/test_iterator.py | 3 +- lib/cherrypy/test/test_logging.py | 57 +- lib/cherrypy/test/test_mime.py | 44 +- lib/cherrypy/test/test_misc_tools.py | 9 +- lib/cherrypy/test/test_native.py | 38 + lib/cherrypy/test/test_objectmapping.py | 2 +- lib/cherrypy/test/test_params.py | 5 +- lib/cherrypy/test/test_plugins.py | 14 + lib/cherrypy/test/test_proxy.py | 16 + lib/cherrypy/test/test_refleaks.py | 4 +- lib/cherrypy/test/test_request_obj.py | 93 +- lib/cherrypy/test/test_routes.py | 3 +- lib/cherrypy/test/test_session.py | 38 +- lib/cherrypy/test/test_states.py | 66 +- lib/cherrypy/test/test_static.py | 39 +- lib/cherrypy/test/test_tools.py | 61 +- lib/cherrypy/test/test_tutorials.py | 9 +- lib/cherrypy/test/test_wsgi_ns.py | 5 +- lib/cherrypy/test/test_wsgi_unix_socket.py | 6 +- lib/cherrypy/test/test_wsgiapps.py | 5 +- lib/cherrypy/test/test_xmlrpc.py | 10 +- lib/cherrypy/test/webtest.py | 617 +---- lib/cherrypy/tutorial/README.rst | 2 +- lib/cherrypy/tutorial/tutorial.conf | 8 +- lib/contextlib2.py | 518 ++++ lib/jaraco/LICENSE | 7 + lib/jaraco/PKG-INFO | 32 + lib/jaraco/__init__.py | 1 + lib/jaraco/functools.py | 467 ++++ lib/more_itertools/LICENSE | 19 + lib/more_itertools/PKG-INFO | 457 ++++ lib/more_itertools/__init__.py | 2 + lib/more_itertools/more.py | 2333 +++++++++++++++++ lib/more_itertools/recipes.py | 577 ++++ lib/more_itertools/tests/__init__.py | 0 lib/more_itertools/tests/test_more.py | 2313 ++++++++++++++++ lib/more_itertools/tests/test_recipes.py | 616 +++++ lib/zc/LICENSE.txt | 44 + lib/zc/PKG-INFO | 207 ++ lib/zc/__init__.py | 1 + lib/zc/lockfile/README.txt | 70 + lib/zc/lockfile/__init__.py | 125 + lib/zc/lockfile/tests.py | 201 ++ 149 files changed, 16645 insertions(+), 6075 deletions(-) create mode 100644 lib/cheroot/PKG-INFO create mode 100644 lib/cheroot/__main__.py create mode 100644 lib/cheroot/cli.py create mode 100644 lib/cheroot/connections.py create mode 100644 lib/cheroot/test/conftest.py delete mode 100644 lib/cheroot/test/test.pem create mode 100644 lib/cheroot/test/test__compat.py create mode 100644 lib/cheroot/test/test_cli.py delete mode 100644 lib/cheroot/test/test_compat.py delete mode 100644 lib/cheroot/test/test_config_server.py create mode 100644 lib/cheroot/test/test_dispatch.py create mode 100644 lib/cheroot/test/test_errors.py delete mode 100644 lib/cheroot/test/test_http.py create mode 100644 lib/cheroot/test/test_makefile.py create mode 100644 lib/cheroot/test/test_server.py create mode 100644 lib/cheroot/test/test_ssl.py create mode 100644 lib/cheroot/test/test_wsgi.py delete mode 100644 lib/cheroot/test/test_wsgiapps.py create mode 100644 lib/cheroot/testing.py create mode 100644 lib/cherrypy/PKG-INFO mode change 100644 => 100755 lib/cherrypy/__main__.py mode change 100644 => 100755 lib/cherrypy/daemon.py delete mode 100644 lib/cherrypy/lib/auth.py delete mode 100644 lib/cherrypy/lib/httpauth.py delete mode 100644 lib/cherrypy/lib/lockfile.py mode change 100644 => 100755 lib/cherrypy/test/sessiondemo.py delete mode 100644 lib/cherrypy/test/test_httpauth.py delete mode 100644 lib/cherrypy/test/test_httplib.py create mode 100644 lib/cherrypy/test/test_httputil.py create mode 100644 lib/cherrypy/test/test_native.py create mode 100644 lib/cherrypy/test/test_plugins.py mode change 100644 => 100755 lib/cherrypy/test/test_session.py create mode 100644 lib/contextlib2.py create mode 100644 lib/jaraco/LICENSE create mode 100644 lib/jaraco/PKG-INFO create mode 100644 lib/jaraco/__init__.py create mode 100644 lib/jaraco/functools.py create mode 100644 lib/more_itertools/LICENSE create mode 100644 lib/more_itertools/PKG-INFO create mode 100644 lib/more_itertools/__init__.py create mode 100644 lib/more_itertools/more.py create mode 100644 lib/more_itertools/recipes.py create mode 100644 lib/more_itertools/tests/__init__.py create mode 100644 lib/more_itertools/tests/test_more.py create mode 100644 lib/more_itertools/tests/test_recipes.py create mode 100644 lib/zc/LICENSE.txt create mode 100644 lib/zc/PKG-INFO create mode 100644 lib/zc/__init__.py create mode 100644 lib/zc/lockfile/README.txt create mode 100644 lib/zc/lockfile/__init__.py create mode 100644 lib/zc/lockfile/tests.py diff --git a/addon.xml b/addon.xml index 43a0865..b3adc84 100644 --- a/addon.xml +++ b/addon.xml @@ -1,8 +1,8 @@ + version="17.4.2" + provider-name="marcelveldt, CherryPy Team, wuff"> @@ -17,6 +17,16 @@ BSD https://github.com/marcelveldt/script.module.cherrypy all - v11.0.2: Remove Python requirement for Kodi18/19 compatibility + + v17.4.2: + - update CherryPy to last Py2/3 compatible version 17.4.2 + - update cheroot to version 8.4.5 + - add more-itertools-5.0.0 dependency + - add contextlib2-0.6.0.post1 dependency + - add zc.lockfile-2.0 dependency + - add jaraco.functools-2.0 dependency + v11.0.2: + - Remove Python requirement for Kodi18/19 compatibility + diff --git a/lib/cheroot/LICENSE.md b/lib/cheroot/LICENSE.md index ed321f5..aadc86c 100644 --- a/lib/cheroot/LICENSE.md +++ b/lib/cheroot/LICENSE.md @@ -1,6 +1,6 @@ -**Copyright © 2004-2016, CherryPy Team (team@cherrypy.org)** +Copyright © 2004-2020, CherryPy Team (team@cherrypy.org) -**All rights reserved.** +All rights reserved. * * * diff --git a/lib/cheroot/PKG-INFO b/lib/cheroot/PKG-INFO new file mode 100644 index 0000000..bbce17d --- /dev/null +++ b/lib/cheroot/PKG-INFO @@ -0,0 +1,148 @@ +Metadata-Version: 2.1 +Name: cheroot +Version: 8.4.5 +Summary: Highly-optimized, pure-python HTTP server +Home-page: https://cheroot.cherrypy.org +Author: CherryPy Team +Author-email: team@cherrypy.org +License: UNKNOWN +Project-URL: CI: AppVeyor, https://ci.appveyor.com/project/cherrypy/cheroot +Project-URL: CI: Circle, https://circleci.com/gh/cherrypy/cheroot +Project-URL: CI: GitHub, https://github.com/cherrypy/cheroot/actions +Project-URL: CI: Travis, https://travis-ci.com/cherrypy/cheroot +Project-URL: Docs: RTD, https://cheroot.cherrypy.org +Project-URL: GitHub: issues, https://github.com/cherrypy/cheroot/issues +Project-URL: GitHub: repo, https://github.com/cherrypy/cheroot +Project-URL: Tidelift: funding, https://tidelift.com/subscription/pkg/pypi-cheroot?utm_source=pypi-cheroot&utm_medium=referral&utm_campaign=pypi +Description: .. image:: https://img.shields.io/pypi/v/cheroot.svg + :target: https://pypi.org/project/cheroot + + .. image:: https://tidelift.com/badges/package/pypi/cheroot + :target: https://tidelift.com/subscription/pkg/pypi-cheroot?utm_source=pypi-cheroot&utm_medium=readme + :alt: Cheroot is available as part of the Tidelift Subscription + + .. image:: https://img.shields.io/travis/com/cherrypy/cheroot/master.svg?logo=travis&label=Linux%20build%20%40%20Travis%20CI + :target: https://travis-ci.com/cherrypy/cheroot + + .. image:: https://circleci.com/gh/cherrypy/cheroot/tree/master.svg?style=svg + :target: https://circleci.com/gh/cherrypy/cheroot/tree/master + + .. image:: https://img.shields.io/appveyor/ci/cherrypy/cheroot/master.svg?label=Windows%20build%20%40%20Appveyor + :target: https://ci.appveyor.com/project/cherrypy/cheroot/branch/master + + .. image:: https://github.com/cherrypy/cheroot/workflows/Test%20suite/badge.svg + :target: https://github.com/cherrypy/cheroot/actions?query=workflow%3A%22Test+suite%22+branch%3Amaster + :alt: GitHub Actions Workflow — Test suite + + .. image:: https://github.com/cherrypy/cheroot/workflows/Code%20quality/badge.svg + :target: https://github.com/cherrypy/cheroot/actions?query=workflow%3A%22Code+quality%22+branch%3Amaster + :alt: GitHub Actions Workflow — Code quality + + .. image:: https://img.shields.io/badge/license-BSD-blue.svg?maxAge=3600 + :target: https://pypi.org/project/cheroot + + .. image:: https://img.shields.io/pypi/pyversions/cheroot.svg + :target: https://pypi.org/project/cheroot + + .. image:: https://codecov.io/gh/cherrypy/cheroot/branch/master/graph/badge.svg + :target: https://codecov.io/gh/cherrypy/cheroot + :alt: codecov + + .. image:: https://readthedocs.org/projects/cheroot/badge/?version=latest + :target: https://cheroot.cherrypy.org/en/latest/?badge=latest + + .. image:: https://img.shields.io/badge/StackOverflow-Cheroot-blue.svg + :target: https://stackoverflow.com/questions/tagged/cheroot+or+cherrypy + + .. image:: https://img.shields.io/gitter/room/cherrypy/cherrypy.svg + :target: https://gitter.im/cherrypy/cherrypy + + .. image:: https://img.shields.io/badge/PRs-welcome-brightgreen.svg?style=flat-square + :target: http://makeapullrequest.com/ + + .. image:: https://app.fossa.io/api/projects/git%2Bgithub.com%2Fcherrypy%2Fcheroot.svg?type=shield + :target: https://app.fossa.io/projects/git%2Bgithub.com%2Fcherrypy%2Fcheroot?ref=badge_shield + :alt: FOSSA Status + + Cheroot is the high-performance, pure-Python HTTP server used by CherryPy. + + Status + ====== + + The test suite currently relies on pytest. It's being run via Travis CI. + + For Enterprise + ============== + + .. list-table:: + :widths: 10 100 + + * - |tideliftlogo| + - Professional support for Cheroot is available as part of the + `Tidelift Subscription`_. The CherryPy maintainers and the + maintainers of thousands of other packages are working with + Tidelift to deliver one enterprise subscription that covers all + of the open source you use. + + Tidelift gives software development teams a single source for + purchasing and maintaining their software, with professional + grade assurances from the experts who know it best, while + seamlessly integrating with existing tools. + + `Learn more `_. + + .. _Tidelift Subscription: https://tidelift.com/subscription/pkg/pypi-cheroot?utm_source=pypi-cheroot&utm_medium=referral&utm_campaign=readme + + .. |tideliftlogo| image:: https://cdn2.hubspot.net/hubfs/4008838/website/logos/logos_for_download/Tidelift_primary-shorthand-logo.png + :target: https://tidelift.com/subscription/pkg/pypi-cheroot?utm_source=pypi-cheroot&utm_medium=readme + :width: 75 + :alt: Tidelift + + Contribute Cheroot + ================== + **Want to add something to upstream?** Feel free to submit a PR or file an issue + if unsure. Please follow `CherryPy's common contribution guidelines + `_. + Note that PR is more likely to be accepted if it includes tests and detailed + description helping maintainers to understand it better 🎉 + + Oh, and be pythonic, please 🐍 + + **Don't know how?** Check out `How to Contribute to Open Source + `_ article by GitHub 🚀 + + + License + ======= + .. image:: https://app.fossa.io/api/projects/git%2Bgithub.com%2Fcherrypy%2Fcheroot.svg?type=large + :target: https://app.fossa.io/projects/git%2Bgithub.com%2Fcherrypy%2Fcheroot?ref=badge_large + :alt: FOSSA Status + +Keywords: http,server,ssl,wsgi +Platform: UNKNOWN +Classifier: Development Status :: 5 - Production/Stable +Classifier: Environment :: Web Environment +Classifier: Intended Audience :: Developers +Classifier: Operating System :: OS Independent +Classifier: Framework :: CherryPy +Classifier: License :: OSI Approved :: BSD License +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 2 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.4 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: Implementation +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python :: Implementation :: Jython +Classifier: Programming Language :: Python :: Implementation :: PyPy +Classifier: Topic :: Internet :: WWW/HTTP +Classifier: Topic :: Internet :: WWW/HTTP :: HTTP Servers +Classifier: Topic :: Internet :: WWW/HTTP :: WSGI +Classifier: Topic :: Internet :: WWW/HTTP :: WSGI :: Server +Requires-Python: !=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7 +Provides-Extra: docs +Provides-Extra: testing diff --git a/lib/cheroot/__init__.py b/lib/cheroot/__init__.py index 69523d2..30d38ca 100644 --- a/lib/cheroot/__init__.py +++ b/lib/cheroot/__init__.py @@ -1,4 +1,7 @@ -"""Cheroot is the high-performance, pure-Python HTTP server used by CherryPy.""" +"""High-performance, pure-Python HTTP server used by CherryPy.""" + +from __future__ import absolute_import, division, print_function +__metaclass__ = type try: import pkg_resources diff --git a/lib/cheroot/__main__.py b/lib/cheroot/__main__.py new file mode 100644 index 0000000..d2e27c1 --- /dev/null +++ b/lib/cheroot/__main__.py @@ -0,0 +1,6 @@ +"""Stub for accessing the Cheroot CLI tool.""" + +from .cli import main + +if __name__ == '__main__': + main() diff --git a/lib/cheroot/_compat.py b/lib/cheroot/_compat.py index 2834087..3ebbf2a 100644 --- a/lib/cheroot/_compat.py +++ b/lib/cheroot/_compat.py @@ -1,55 +1,73 @@ -"""Compatibility code for using Cheroot with various versions of Python. +# pylint: disable=unused-import +"""Compatibility code for using Cheroot with various versions of Python.""" -Cheroot is compatible with Python versions 2.6+. This module provides a -useful abstraction over the differences between Python versions, sometimes by -preferring a newer idiom, sometimes an older one, and sometimes a custom one. -In particular, Python 2 uses str and '' for byte strings, while Python 3 -uses str and '' for unicode strings. Refer to each of these the 'native -string' type for each version. Because of this major difference, this module -provides -two functions: 'ntob', which translates native strings (of type 'str') into -byte strings regardless of Python version, and 'ntou', which translates native -strings to unicode strings. This also provides a 'BytesIO' name for dealing -specifically with bytes, and a 'StringIO' name for dealing with native strings. -It also provides a 'base64_decode' function with native strings as input and -output. -""" +from __future__ import absolute_import, division, print_function +__metaclass__ = type -import binascii -import os +import platform import re -import sys -import threading import six -if six.PY3: +try: + import selectors # lgtm [py/unused-import] +except ImportError: + import selectors2 as selectors # noqa: F401 # lgtm [py/unused-import] + +try: + import ssl + IS_ABOVE_OPENSSL10 = ssl.OPENSSL_VERSION_INFO >= (1, 1) + del ssl +except ImportError: + IS_ABOVE_OPENSSL10 = None + +# contextlib.suppress was added in Python 3.4 +try: + from contextlib import suppress +except ImportError: + from contextlib import contextmanager + + @contextmanager + def suppress(*exceptions): + """Return a context manager that suppresses the `exceptions`.""" + try: + yield + except exceptions: + pass + + +IS_PYPY = platform.python_implementation() == 'PyPy' + + +SYS_PLATFORM = platform.system() +IS_WINDOWS = SYS_PLATFORM == 'Windows' +IS_LINUX = SYS_PLATFORM == 'Linux' +IS_MACOS = SYS_PLATFORM == 'Darwin' + +PLATFORM_ARCH = platform.machine() +IS_PPC = PLATFORM_ARCH.startswith('ppc') + + +if not six.PY2: def ntob(n, encoding='ISO-8859-1'): - """Return the given native string as a byte string in the given encoding.""" + """Return the native string as bytes in the given encoding.""" assert_native(n) # In Python 3, the native string type is unicode return n.encode(encoding) def ntou(n, encoding='ISO-8859-1'): - """Return the given native string as a unicode string with the given encoding.""" + """Return the native string as Unicode with the given encoding.""" assert_native(n) # In Python 3, the native string type is unicode return n - def tonative(n, encoding='ISO-8859-1'): - """Return the given string as a native string in the given encoding.""" - # In Python 3, the native string type is unicode - if isinstance(n, bytes): - return n.decode(encoding) - return n - def bton(b, encoding='ISO-8859-1'): - """Return the given byte string as a native string in the given encoding.""" + """Return the byte string as native string in the given encoding.""" return b.decode(encoding) else: # Python 2 def ntob(n, encoding='ISO-8859-1'): - """Return the given native string as a byte string in the given encoding.""" + """Return the native string as bytes in the given encoding.""" assert_native(n) # In Python 2, the native string type is bytes. Assume it's already # in the given encoding, which for ISO-8859-1 is almost always what @@ -57,7 +75,7 @@ def ntob(n, encoding='ISO-8859-1'): return n def ntou(n, encoding='ISO-8859-1'): - """Return the given native string as a unicode string with the given encoding.""" + """Return the native string as Unicode with the given encoding.""" assert_native(n) # In Python 2, the native string type is bytes. # First, check for the special encoding 'escape'. The test suite uses @@ -65,303 +83,61 @@ def ntou(n, encoding='ISO-8859-1'): # escapes, but without having to prefix it with u'' for Python 2, # but no prefix for Python 3. if encoding == 'escape': - return six.u( - re.sub(r'\\u([0-9a-zA-Z]{4})', - lambda m: six.unichr(int(m.group(1), 16)), - n.decode('ISO-8859-1'))) + return re.sub( + r'\\u([0-9a-zA-Z]{4})', + lambda m: six.unichr(int(m.group(1), 16)), + n.decode('ISO-8859-1'), + ) # Assume it's already in the given encoding, which for ISO-8859-1 # is almost always what was intended. return n.decode(encoding) - def tonative(n, encoding='ISO-8859-1'): - """Return the given string as a native string in the given encoding.""" - # In Python 2, the native string type is bytes. - if isinstance(n, six.text_type): - return n.encode(encoding) - return n - def bton(b, encoding='ISO-8859-1'): - """Return the given byte string as a native string in the given encoding.""" + """Return the byte string as native string in the given encoding.""" return b def assert_native(n): - """Check whether the input is of nativ ``str`` type. + """Check whether the input is of native :py:class:`str` type. Raises: TypeError: in case of failed check + """ if not isinstance(n, str): raise TypeError('n must be a native str (got %s)' % type(n).__name__) -try: - # Python 3.1+ - from base64 import decodebytes as _base64_decodebytes -except ImportError: - # Python 3.0- - # since Cheroot claims compability with Python 2.6+, we must use - # the legacy API of base64 - from base64 import decodestring as _base64_decodebytes - - -def base64_decode(n, encoding='ISO-8859-1'): - """Return the native string base64-decoded (as a native string).""" - if isinstance(n, six.text_type): - b = n.encode(encoding) - else: - b = n - b = _base64_decodebytes(b) - if str is six.text_type: - return b.decode(encoding) - else: - return b - - -try: - sorted = sorted -except NameError: - def sorted(i): - """Return sorted sequence.""" - i = i[:] - i.sort() - return i - -try: - reversed = reversed -except NameError: - def reversed(x): - """Return sequence in reversed order.""" - i = len(x) - while i > 0: - i -= 1 - yield x[i] - -try: - # Python 3 - from urllib.parse import urljoin, urlencode - from urllib.parse import quote, quote_plus - from urllib.request import unquote, urlopen - from urllib.request import parse_http_list, parse_keqv_list -except ImportError: - # Python 2 - from urlparse import urljoin # noqa - from urllib import urlencode, urlopen # noqa - from urllib import quote, quote_plus # noqa - from urllib import unquote # noqa - from urllib2 import parse_http_list, parse_keqv_list # noqa - -try: - dict.iteritems - # Python 2 - iteritems = lambda d: d.iteritems() - copyitems = lambda d: d.items() -except AttributeError: - # Python 3 - iteritems = lambda d: d.items() - copyitems = lambda d: list(d.items()) - -try: - dict.iterkeys - # Python 2 - iterkeys = lambda d: d.iterkeys() - copykeys = lambda d: d.keys() -except AttributeError: - # Python 3 - iterkeys = lambda d: d.keys() - copykeys = lambda d: list(d.keys()) - -try: - dict.itervalues - # Python 2 - itervalues = lambda d: d.itervalues() - copyvalues = lambda d: d.values() -except AttributeError: - # Python 3 - itervalues = lambda d: d.values() - copyvalues = lambda d: list(d.values()) - -try: - # Python 3 - import builtins -except ImportError: - # Python 2 - import __builtin__ as builtins # noqa - -try: - # Python 2. We try Python 2 first clients on Python 2 - # don't try to import the 'http' module from cheroot submodules directly - from Cookie import SimpleCookie, CookieError - from httplib import BadStatusLine, HTTPConnection, IncompleteRead - from httplib import NotConnected - from BaseHTTPServer import BaseHTTPRequestHandler -except ImportError: - # Python 3 - from http.cookies import SimpleCookie, CookieError # noqa - from http.client import BadStatusLine, HTTPConnection, IncompleteRead # noqa - from http.client import NotConnected # noqa - from http.server import BaseHTTPRequestHandler # noqa - -# Some platforms don't expose HTTPSConnection, so handle it separately -if six.PY3: - try: - from http.client import HTTPSConnection - except ImportError: - # Some platforms which don't have SSL don't expose HTTPSConnection - HTTPSConnection = None -else: - try: - from httplib import HTTPSConnection - except ImportError: - HTTPSConnection = None - -try: - # Python 2 - xrange = xrange -except NameError: - # Python 3 - xrange = range - -try: - # Python 3 - from urllib.parse import unquote as parse_unquote - - def unquote_qs(atom, encoding, errors='strict'): - """Return urldecoded query string.""" - return parse_unquote( - atom.replace('+', ' '), - encoding=encoding, - errors=errors) -except ImportError: - # Python 2 - from urllib import unquote as parse_unquote - - def unquote_qs(atom, encoding, errors='strict'): - """Return urldecoded query string.""" - return parse_unquote(atom.replace('+', ' ')).decode(encoding, errors) - -try: - # Prefer simplejson, which is usually more advanced than the builtin - # module. - import simplejson as json - json_decode = json.JSONDecoder().decode - _json_encode = json.JSONEncoder().iterencode -except ImportError: - if sys.version_info >= (2, 6): - # Python >=2.6 : json is part of the standard library - import json - json_decode = json.JSONDecoder().decode - _json_encode = json.JSONEncoder().iterencode - else: - json = None - - def json_decode(s): - """Alert that decoding JSON is not supported because of missing package.""" - raise ValueError('No JSON library is available') - - def _json_encode(s): - """Alert that encoding JSON is not supported because of missing package.""" - raise ValueError('No JSON library is available') -finally: - if json and six.PY3: - # The two Python 3 implementations (simplejson/json) - # outputs str. We need bytes. - def json_encode(value): - """Return JSON string as bytes.""" - for chunk in _json_encode(value): - yield chunk.encode('utf8') - else: - json_encode = _json_encode - -text_or_bytes = six.text_type, six.binary_type - -try: - import cPickle as pickle -except ImportError: - # In Python 2, pickle is a Python version. - # In Python 3, pickle is the sped-up C version. - import pickle # noqa - - -def random20(): - """Return a random string of 20 bytes.""" - return binascii.hexlify(os.urandom(20)).decode('ascii') - - -try: - from _thread import get_ident as get_thread_ident -except ImportError: - from thread import get_ident as get_thread_ident # noqa - -try: - # Python 3 - next = next -except NameError: - # Python 2 - def next(i): - """Return the next value of an iterator.""" - return i.next() - -if sys.version_info >= (3, 3): - Timer = threading.Timer - Event = threading.Event +if not six.PY2: + """Python 3 has :py:class:`memoryview` builtin.""" + # Python 2.7 has it backported, but socket.write() does + # str(memoryview(b'0' * 100)) -> + # instead of accessing it correctly. + memoryview = memoryview else: - # Python 3.2 and earlier - Timer = threading._Timer - Event = threading._Event + """Link :py:class:`memoryview` to buffer under Python 2.""" + memoryview = buffer # noqa: F821 -try: - # Python 2.7+ - from subprocess import _args_from_interpreter_flags -except ImportError: - def _args_from_interpreter_flags(): - """Try to reconstruct original interpreter args from sys.flags for Python 2.6. - Backported from Python 3.5. Aims to return a list of - command-line arguments reproducing the current - settings in sys.flags and sys.warnoptions. - """ - flag_opt_map = { - 'debug': 'd', - # 'inspect': 'i', - # 'interactive': 'i', - 'optimize': 'O', - 'dont_write_bytecode': 'B', - 'no_user_site': 's', - 'no_site': 'S', - 'ignore_environment': 'E', - 'verbose': 'v', - 'bytes_warning': 'b', - 'quiet': 'q', - 'hash_randomization': 'R', - 'py3k_warning': '3', - } +def extract_bytes(mv): + r"""Retrieve bytes out of the given input buffer. - args = [] - for flag, opt in flag_opt_map.items(): - v = getattr(sys.flags, flag) - if v > 0: - if flag == 'hash_randomization': - v = 1 # Handle specification of an exact seed - args.append('-' + opt * v) - for opt in sys.warnoptions: - args.append('-W' + opt) + :param mv: input :py:func:`buffer` + :type mv: memoryview or bytes - return args + :return: unwrapped bytes + :rtype: bytes -# html module come in 3.2 version -try: - from html import escape -except ImportError: - from cgi import escape - - -# html module needed the argument quote=False because in cgi the default -# is False. With quote=True the results differ. + :raises ValueError: if the input is not one of \ + :py:class:`memoryview`/:py:func:`buffer` \ + or :py:class:`bytes` + """ + if isinstance(mv, memoryview): + return bytes(mv) if six.PY2 else mv.tobytes() -def escape_html(s, escape_quote=False): - """Replace special characters "&", "<" and ">" to HTML-safe sequences. + if isinstance(mv, bytes): + return mv - When escape_quote=True, escape (') and (") chars. - """ - return escape(s, quote=escape_quote) + raise ValueError( + 'extract_bytes() only accepts bytes and memoryview/buffer', + ) diff --git a/lib/cheroot/cli.py b/lib/cheroot/cli.py new file mode 100644 index 0000000..f3c0504 --- /dev/null +++ b/lib/cheroot/cli.py @@ -0,0 +1,241 @@ +"""Command line tool for starting a Cheroot WSGI/HTTP server instance. + +Basic usage:: + + # Start a server on 127.0.0.1:8000 with the default settings + # for the WSGI app myapp/wsgi.py:application() + cheroot myapp.wsgi + + # Start a server on 0.0.0.0:9000 with 8 threads + # for the WSGI app myapp/wsgi.py:main_app() + cheroot myapp.wsgi:main_app --bind 0.0.0.0:9000 --threads 8 + + # Start a server for the cheroot.server.Gateway subclass + # myapp/gateway.py:HTTPGateway + cheroot myapp.gateway:HTTPGateway + + # Start a server on the UNIX socket /var/spool/myapp.sock + cheroot myapp.wsgi --bind /var/spool/myapp.sock + + # Start a server on the abstract UNIX socket CherootServer + cheroot myapp.wsgi --bind @CherootServer +""" + +import argparse +from importlib import import_module +import os +import sys + +import six + +from . import server +from . import wsgi +from ._compat import suppress + + +__metaclass__ = type + + +class BindLocation: + """A class for storing the bind location for a Cheroot instance.""" + + +class TCPSocket(BindLocation): + """TCPSocket.""" + + def __init__(self, address, port): + """Initialize. + + Args: + address (str): Host name or IP address + port (int): TCP port number + + """ + self.bind_addr = address, port + + +class UnixSocket(BindLocation): + """UnixSocket.""" + + def __init__(self, path): + """Initialize.""" + self.bind_addr = path + + +class AbstractSocket(BindLocation): + """AbstractSocket.""" + + def __init__(self, abstract_socket): + """Initialize.""" + self.bind_addr = '\x00{sock_path}'.format(sock_path=abstract_socket) + + +class Application: + """Application.""" + + @classmethod + def resolve(cls, full_path): + """Read WSGI app/Gateway path string and import application module.""" + mod_path, _, app_path = full_path.partition(':') + app = getattr(import_module(mod_path), app_path or 'application') + # suppress the `TypeError` exception, just in case `app` is not a class + with suppress(TypeError): + if issubclass(app, server.Gateway): + return GatewayYo(app) + + return cls(app) + + def __init__(self, wsgi_app): + """Initialize.""" + if not callable(wsgi_app): + raise TypeError( + 'Application must be a callable object or ' + 'cheroot.server.Gateway subclass', + ) + self.wsgi_app = wsgi_app + + def server_args(self, parsed_args): + """Return keyword args for Server class.""" + args = { + arg: value + for arg, value in vars(parsed_args).items() + if not arg.startswith('_') and value is not None + } + args.update(vars(self)) + return args + + def server(self, parsed_args): + """Server.""" + return wsgi.Server(**self.server_args(parsed_args)) + + +class GatewayYo: + """Gateway.""" + + def __init__(self, gateway): + """Init.""" + self.gateway = gateway + + def server(self, parsed_args): + """Server.""" + server_args = vars(self) + server_args['bind_addr'] = parsed_args['bind_addr'] + if parsed_args.max is not None: + server_args['maxthreads'] = parsed_args.max + if parsed_args.numthreads is not None: + server_args['minthreads'] = parsed_args.numthreads + return server.HTTPServer(**server_args) + + +def parse_wsgi_bind_location(bind_addr_string): + """Convert bind address string to a BindLocation.""" + # if the string begins with an @ symbol, use an abstract socket, + # this is the first condition to verify, otherwise the urlparse + # validation would detect //@ as a valid url with a hostname + # with value: "" and port: None + if bind_addr_string.startswith('@'): + return AbstractSocket(bind_addr_string[1:]) + + # try and match for an IP/hostname and port + match = six.moves.urllib.parse.urlparse( + '//{addr}'.format(addr=bind_addr_string), + ) + try: + addr = match.hostname + port = match.port + if addr is not None or port is not None: + return TCPSocket(addr, port) + except ValueError: + pass + + # else, assume a UNIX socket path + return UnixSocket(path=bind_addr_string) + + +def parse_wsgi_bind_addr(bind_addr_string): + """Convert bind address string to bind address parameter.""" + return parse_wsgi_bind_location(bind_addr_string).bind_addr + + +_arg_spec = { + '_wsgi_app': dict( + metavar='APP_MODULE', + type=Application.resolve, + help='WSGI application callable or cheroot.server.Gateway subclass', + ), + '--bind': dict( + metavar='ADDRESS', + dest='bind_addr', + type=parse_wsgi_bind_addr, + default='[::1]:8000', + help='Network interface to listen on (default: [::1]:8000)', + ), + '--chdir': dict( + metavar='PATH', + type=os.chdir, + help='Set the working directory', + ), + '--server-name': dict( + dest='server_name', + type=str, + help='Web server name to be advertised via Server HTTP header', + ), + '--threads': dict( + metavar='INT', + dest='numthreads', + type=int, + help='Minimum number of worker threads', + ), + '--max-threads': dict( + metavar='INT', + dest='max', + type=int, + help='Maximum number of worker threads', + ), + '--timeout': dict( + metavar='INT', + dest='timeout', + type=int, + help='Timeout in seconds for accepted connections', + ), + '--shutdown-timeout': dict( + metavar='INT', + dest='shutdown_timeout', + type=int, + help='Time in seconds to wait for worker threads to cleanly exit', + ), + '--request-queue-size': dict( + metavar='INT', + dest='request_queue_size', + type=int, + help='Maximum number of queued connections', + ), + '--accepted-queue-size': dict( + metavar='INT', + dest='accepted_queue_size', + type=int, + help='Maximum number of active requests in queue', + ), + '--accepted-queue-timeout': dict( + metavar='INT', + dest='accepted_queue_timeout', + type=int, + help='Timeout in seconds for putting requests into queue', + ), +} + + +def main(): + """Create a new Cheroot instance with arguments from the command line.""" + parser = argparse.ArgumentParser( + description='Start an instance of the Cheroot WSGI/HTTP server.', + ) + for arg, spec in _arg_spec.items(): + parser.add_argument(arg, **spec) + raw_args = parser.parse_args() + + # ensure cwd in sys.path + '' in sys.path or sys.path.insert(0, '') + + # create a server based on the arguments provided + raw_args._wsgi_app.server(raw_args).safe_start() diff --git a/lib/cheroot/connections.py b/lib/cheroot/connections.py new file mode 100644 index 0000000..b230307 --- /dev/null +++ b/lib/cheroot/connections.py @@ -0,0 +1,292 @@ +"""Utilities to manage open connections.""" + +from __future__ import absolute_import, division, print_function +__metaclass__ = type + +import collections +import io +import os +import socket +import time + +from . import errors +from ._compat import selectors +from ._compat import suppress +from .makefile import MakeFile + +import six + +try: + import fcntl +except ImportError: + try: + from ctypes import windll, WinError + import ctypes.wintypes + _SetHandleInformation = windll.kernel32.SetHandleInformation + _SetHandleInformation.argtypes = [ + ctypes.wintypes.HANDLE, + ctypes.wintypes.DWORD, + ctypes.wintypes.DWORD, + ] + _SetHandleInformation.restype = ctypes.wintypes.BOOL + except ImportError: + def prevent_socket_inheritance(sock): + """Stub inheritance prevention. + + Dummy function, since neither fcntl nor ctypes are available. + """ + pass + else: + def prevent_socket_inheritance(sock): + """Mark the given socket fd as non-inheritable (Windows).""" + if not _SetHandleInformation(sock.fileno(), 1, 0): + raise WinError() +else: + def prevent_socket_inheritance(sock): + """Mark the given socket fd as non-inheritable (POSIX).""" + fd = sock.fileno() + old_flags = fcntl.fcntl(fd, fcntl.F_GETFD) + fcntl.fcntl(fd, fcntl.F_SETFD, old_flags | fcntl.FD_CLOEXEC) + + +class ConnectionManager: + """Class which manages HTTPConnection objects. + + This is for connections which are being kept-alive for follow-up requests. + """ + + def __init__(self, server): + """Initialize ConnectionManager object. + + Args: + server (cheroot.server.HTTPServer): web server object + that uses this ConnectionManager instance. + """ + self.server = server + self._readable_conns = collections.deque() + self._selector = selectors.DefaultSelector() + + self._selector.register( + server.socket.fileno(), + selectors.EVENT_READ, data=server, + ) + + def put(self, conn): + """Put idle connection into the ConnectionManager to be managed. + + Args: + conn (cheroot.server.HTTPConnection): HTTP connection + to be managed. + """ + conn.last_used = time.time() + # if this conn doesn't have any more data waiting to be read, + # register it with the selector. + if conn.rfile.has_data(): + self._readable_conns.append(conn) + else: + self._selector.register( + conn.socket.fileno(), selectors.EVENT_READ, data=conn, + ) + + def expire(self): + """Expire least recently used connections. + + This happens if there are either too many open connections, or if the + connections have been timed out. + + This should be called periodically. + """ + # find any connections still registered with the selector + # that have not been active recently enough. + threshold = time.time() - self.server.timeout + timed_out_connections = [ + (sock_fd, conn) + for _, (_, sock_fd, _, conn) + in self._selector.get_map().items() + if conn != self.server and conn.last_used < threshold + ] + for sock_fd, conn in timed_out_connections: + self._selector.unregister(sock_fd) + conn.close() + + def get_conn(self): # noqa: C901 # FIXME + """Return a HTTPConnection object which is ready to be handled. + + A connection returned by this method should be ready for a worker + to handle it. If there are no connections ready, None will be + returned. + + Any connection returned by this method will need to be `put` + back if it should be examined again for another request. + + Returns: + cheroot.server.HTTPConnection instance, or None. + + """ + # return a readable connection if any exist + with suppress(IndexError): + return self._readable_conns.popleft() + + # Will require a select call. + try: + # The timeout value impacts performance and should be carefully + # chosen. Ref: + # github.com/cherrypy/cheroot/issues/305#issuecomment-663985165 + rlist = [ + key for key, _ + in self._selector.select(timeout=0.01) + ] + except OSError: + # Mark any connection which no longer appears valid + invalid_entries = [] + for _, key in self._selector.get_map().items(): + # If the server socket is invalid, we'll just ignore it and + # wait to be shutdown. + if key.data == self.server: + continue + + try: + os.fstat(key.fd) + except OSError: + invalid_entries.append((key.fd, key.data)) + + for sock_fd, conn in invalid_entries: + self._selector.unregister(sock_fd) + conn.close() + + # Wait for the next tick to occur. + return None + + for key in rlist: + if key.data is self.server: + # New connection + return self._from_server_socket(self.server.socket) + + conn = key.data + # unregister connection from the selector until the server + # has read from it and returned it via put() + self._selector.unregister(key.fd) + self._readable_conns.append(conn) + + try: + return self._readable_conns.popleft() + except IndexError: + return None + + def _from_server_socket(self, server_socket): # noqa: C901 # FIXME + try: + s, addr = server_socket.accept() + if self.server.stats['Enabled']: + self.server.stats['Accepts'] += 1 + prevent_socket_inheritance(s) + if hasattr(s, 'settimeout'): + s.settimeout(self.server.timeout) + + mf = MakeFile + ssl_env = {} + # if ssl cert and key are set, we try to be a secure HTTP server + if self.server.ssl_adapter is not None: + try: + s, ssl_env = self.server.ssl_adapter.wrap(s) + except errors.NoSSLError: + msg = ( + 'The client sent a plain HTTP request, but ' + 'this server only speaks HTTPS on this port.' + ) + buf = [ + '%s 400 Bad Request\r\n' % self.server.protocol, + 'Content-Length: %s\r\n' % len(msg), + 'Content-Type: text/plain\r\n\r\n', + msg, + ] + + sock_to_make = s if not six.PY2 else s._sock + wfile = mf(sock_to_make, 'wb', io.DEFAULT_BUFFER_SIZE) + try: + wfile.write(''.join(buf).encode('ISO-8859-1')) + except socket.error as ex: + if ex.args[0] not in errors.socket_errors_to_ignore: + raise + return + if not s: + return + mf = self.server.ssl_adapter.makefile + # Re-apply our timeout since we may have a new socket object + if hasattr(s, 'settimeout'): + s.settimeout(self.server.timeout) + + conn = self.server.ConnectionClass(self.server, s, mf) + + if not isinstance( + self.server.bind_addr, + (six.text_type, six.binary_type), + ): + # optional values + # Until we do DNS lookups, omit REMOTE_HOST + if addr is None: # sometimes this can happen + # figure out if AF_INET or AF_INET6. + if len(s.getsockname()) == 2: + # AF_INET + addr = ('0.0.0.0', 0) + else: + # AF_INET6 + addr = ('::', 0) + conn.remote_addr = addr[0] + conn.remote_port = addr[1] + + conn.ssl_env = ssl_env + return conn + + except socket.timeout: + # The only reason for the timeout in start() is so we can + # notice keyboard interrupts on Win32, which don't interrupt + # accept() by default + return + except socket.error as ex: + if self.server.stats['Enabled']: + self.server.stats['Socket Errors'] += 1 + if ex.args[0] in errors.socket_error_eintr: + # I *think* this is right. EINTR should occur when a signal + # is received during the accept() call; all docs say retry + # the call, and I *think* I'm reading it right that Python + # will then go ahead and poll for and handle the signal + # elsewhere. See + # https://github.com/cherrypy/cherrypy/issues/707. + return + if ex.args[0] in errors.socket_errors_nonblocking: + # Just try again. See + # https://github.com/cherrypy/cherrypy/issues/479. + return + if ex.args[0] in errors.socket_errors_to_ignore: + # Our socket was closed. + # See https://github.com/cherrypy/cherrypy/issues/686. + return + raise + + def close(self): + """Close all monitored connections.""" + for conn in self._readable_conns: + conn.close() + self._readable_conns.clear() + + for _, key in self._selector.get_map().items(): + if key.data != self.server: # server closes its own socket + key.data.socket.close() + + self._selector.close() + + @property + def _num_connections(self): + """Return the current number of connections. + + Includes any in the readable list or registered with the selector, + minus one for the server socket, which is always registered + with the selector. + """ + return len(self._readable_conns) + len(self._selector.get_map()) - 1 + + @property + def can_add_keepalive_connection(self): + """Flag whether it is allowed to add a new keep-alive connection.""" + ka_limit = self.server.keep_alive_conn_limit + return ka_limit is None or self._num_connections < ka_limit diff --git a/lib/cheroot/errors.py b/lib/cheroot/errors.py index e7bbb00..4395e56 100644 --- a/lib/cheroot/errors.py +++ b/lib/cheroot/errors.py @@ -1,5 +1,8 @@ """Collection of exceptions raised and/or processed by Cheroot.""" +from __future__ import absolute_import, division, print_function +__metaclass__ = type + import errno import sys @@ -20,16 +23,15 @@ class FatalSSLAlert(Exception): def plat_specific_errors(*errnames): - """Return error numbers for all errors in errnames on this platform. + """Return error numbers for all errors in ``errnames`` on this platform. - The 'errno' module contains different global constants depending on - the specific platform (OS). This function will return the list of - numeric values for a given list of potential names. + The :py:mod:`errno` module contains different global constants + depending on the specific platform (OS). This function will return + the list of numeric values for a given list of potential names. """ - errno_names = dir(errno) - nums = [getattr(errno, k) for k in errnames if k in errno_names] - # de-dupe the list - return list(dict.fromkeys(nums).keys()) + missing_attr = {None} + unique_nums = {getattr(errno, k, None) for k in errnames} + return list(unique_nums - missing_attr) socket_error_eintr = plat_specific_errors('EINTR', 'WSAEINTR') @@ -48,8 +50,9 @@ def plat_specific_errors(*errnames): socket_errors_to_ignore.append('timed out') socket_errors_to_ignore.append('The read operation timed out') socket_errors_nonblocking = plat_specific_errors( - 'EAGAIN', 'EWOULDBLOCK', 'WSAEWOULDBLOCK') + 'EAGAIN', 'EWOULDBLOCK', 'WSAEWOULDBLOCK', +) if sys.platform == 'darwin': - socket_errors_to_ignore.append(plat_specific_errors('EPROTOTYPE')) - socket_errors_nonblocking.append(plat_specific_errors('EPROTOTYPE')) + socket_errors_to_ignore.extend(plat_specific_errors('EPROTOTYPE')) + socket_errors_nonblocking.extend(plat_specific_errors('EPROTOTYPE')) diff --git a/lib/cheroot/makefile.py b/lib/cheroot/makefile.py index a59af3a..1383c65 100644 --- a/lib/cheroot/makefile.py +++ b/lib/cheroot/makefile.py @@ -1,5 +1,8 @@ """Socket file object.""" +from __future__ import absolute_import, division, print_function +__metaclass__ = type + import socket try: @@ -12,6 +15,11 @@ import six from . import errors +from ._compat import extract_bytes, memoryview + + +# Write only 16K at a time to sockets +SOCK_WRITE_BLOCKSIZE = 16384 class BufferedWriter(io.BufferedWriter): @@ -40,14 +48,6 @@ def _flush_unlocked(self): del self._write_buf[:n] -def MakeFile_PY3(sock, mode='r', bufsize=io.DEFAULT_BUFFER_SIZE): - """File object attached to a socket object.""" - if 'r' in mode: - return io.BufferedReader(socket.SocketIO(sock, mode), bufsize) - else: - return BufferedWriter(socket.SocketIO(sock, mode), bufsize) - - class MakeFile_PY2(getattr(socket, '_fileobject', object)): """Faux file object attached to a socket object.""" @@ -56,20 +56,34 @@ def __init__(self, *args, **kwargs): self.bytes_read = 0 self.bytes_written = 0 socket._fileobject.__init__(self, *args, **kwargs) + self._refcount = 0 + + def _reuse(self): + self._refcount += 1 + + def _drop(self): + if self._refcount < 0: + self.close() + else: + self._refcount -= 1 def write(self, data): - """Sendall for non-blocking sockets.""" - while data: + """Send entire data contents for non-blocking sockets.""" + bytes_sent = 0 + data_mv = memoryview(data) + payload_size = len(data_mv) + while bytes_sent < payload_size: try: - bytes_sent = self.send(data) - data = data[bytes_sent:] + bytes_sent += self.send( + data_mv[bytes_sent:bytes_sent + SOCK_WRITE_BLOCKSIZE], + ) except socket.error as e: if e.args[0] not in errors.socket_errors_nonblocking: raise def send(self, data): """Send some part of message to the socket.""" - bytes_sent = self._sock.send(data) + bytes_sent = self._sock.send(extract_bytes(data)) self.bytes_written += bytes_sent return bytes_sent @@ -88,22 +102,27 @@ def recv(self, size): self.bytes_read += len(data) return data except socket.error as e: - if e.args[0] not in errors.socket_errors_nonblocking and e.args[0] not in errors.socket_error_eintr: + what = ( + e.args[0] not in errors.socket_errors_nonblocking + and e.args[0] not in errors.socket_error_eintr + ) + if what: raise - class FauxSocket(object): + class FauxSocket: """Faux socket with the minimal interface required by pypy.""" def _reuse(self): pass _fileobject_uses_str_type = six.PY2 and isinstance( - socket._fileobject(FauxSocket())._rbuf, six.string_types) + socket._fileobject(FauxSocket())._rbuf, six.string_types, + ) # FauxSocket is no longer needed del FauxSocket - if not _fileobject_uses_str_type: + if not _fileobject_uses_str_type: # noqa: C901 # FIXME def read(self, size=-1): """Read data from the socket to buffer.""" # Use max, disallow tiny reads in a loop as they are very @@ -216,6 +235,7 @@ def readline(self, size=-1): break buf.write(data) return buf.getvalue() + else: # Read until size bytes or \n or EOF seen, whichever comes # first @@ -260,6 +280,11 @@ def readline(self, size=-1): buf_len += n # assert buf_len == buf.tell() return buf.getvalue() + + def has_data(self): + """Return true if there is buffered data to read.""" + return bool(self._rbuf.getvalue()) + else: def read(self, size=-1): """Read data from the socket to buffer.""" @@ -376,5 +401,47 @@ def readline(self, size=-1): buf_len += n return ''.join(buffers) + def has_data(self): + """Return true if there is buffered data to read.""" + return bool(self._rbuf) + + +if not six.PY2: + class StreamReader(io.BufferedReader): + """Socket stream reader.""" + + def __init__(self, sock, mode='r', bufsize=io.DEFAULT_BUFFER_SIZE): + """Initialize socket stream reader.""" + super().__init__(socket.SocketIO(sock, mode), bufsize) + self.bytes_read = 0 + + def read(self, *args, **kwargs): + """Capture bytes read.""" + val = super().read(*args, **kwargs) + self.bytes_read += len(val) + return val + + def has_data(self): + """Return true if there is buffered data to read.""" + return len(self._read_buf) > self._read_pos + + class StreamWriter(BufferedWriter): + """Socket stream writer.""" + + def __init__(self, sock, mode='w', bufsize=io.DEFAULT_BUFFER_SIZE): + """Initialize socket stream writer.""" + super().__init__(socket.SocketIO(sock, mode), bufsize) + self.bytes_written = 0 + + def write(self, val, *args, **kwargs): + """Capture bytes written.""" + res = super().write(val, *args, **kwargs) + self.bytes_written += len(val) + return res -MakeFile = MakeFile_PY2 if six.PY2 else MakeFile_PY3 + def MakeFile(sock, mode='r', bufsize=io.DEFAULT_BUFFER_SIZE): + """File object attached to a socket object.""" + cls = StreamReader if 'r' in mode else StreamWriter + return cls(sock, mode, bufsize) +else: + StreamReader = StreamWriter = MakeFile = MakeFile_PY2 diff --git a/lib/cheroot/server.py b/lib/cheroot/server.py index 28b3848..5405b2c 100644 --- a/lib/cheroot/server.py +++ b/lib/cheroot/server.py @@ -7,12 +7,12 @@ server = HTTPServer(...) server.start() - while True: - tick() - # This blocks until a request comes in: - child = socket.accept() - conn = HTTPConnection(child, ...) - server.requests.put(conn) + -> while True: + tick() + # This blocks until a request comes in: + child = socket.accept() + conn = HTTPConnection(child, ...) + server.requests.put(conn) Worker threads are kept in a pool and poll the Queue, popping off and then handling each connection in turn. Each connection can consist of an arbitrary @@ -39,13 +39,30 @@ if req.close_connection: return +For running a server you can invoke :func:`start() ` (it +will run the server forever) or use invoking :func:`prepare() +` and :func:`serve() ` like this:: + + server = HTTPServer(...) + server.prepare() + try: + threading.Thread(target=server.serve).start() + + # waiting/detecting some appropriate stop condition here + ... + + finally: + server.stop() + And now for a trivial doctest to exercise the test suite >>> 'HTTPServer' in globals() True - """ +from __future__ import absolute_import, division, print_function +__metaclass__ = type + import os import io import re @@ -55,58 +72,102 @@ import time import traceback as traceback_ import logging +import platform +import contextlib +import threading try: - from urllib.parse import unquote_to_bytes + from functools import lru_cache except ImportError: - from urllib import unquote as unquote_to_bytes + from backports.functools_lru_cache import lru_cache import six from six.moves import queue from six.moves import urllib -from . import errors, __version__ -from ._compat import ntob +from . import connections, errors, __version__ +from ._compat import bton, ntou +from ._compat import IS_PPC from .workers import threadpool -from .makefile import MakeFile -from urllib import quote +from .makefile import MakeFile, StreamWriter + -__all__ = ('HTTPRequest', 'HTTPConnection', 'HTTPServer', - 'SizeCheckWrapper', 'KnownLengthRFile', 'ChunkedRFile', - 'Gateway', 'get_ssl_adapter_class') +__all__ = ( + 'HTTPRequest', 'HTTPConnection', 'HTTPServer', + 'HeaderReader', 'DropUnderscoreHeaderReader', + 'SizeCheckWrapper', 'KnownLengthRFile', 'ChunkedRFile', + 'Gateway', 'get_ssl_adapter_class', +) -if 'win' in sys.platform and hasattr(socket, 'AF_INET6'): +IS_WINDOWS = platform.system() == 'Windows' +"""Flag indicating whether the app is running under Windows.""" + + +IS_GAE = os.getenv('SERVER_SOFTWARE', '').startswith('Google App Engine/') +"""Flag indicating whether the app is running in GAE env. + +Ref: +https://cloud.google.com/appengine/docs/standard/python/tools +/using-local-server#detecting_application_runtime_environment +""" + + +IS_UID_GID_RESOLVABLE = not IS_WINDOWS and not IS_GAE +"""Indicates whether UID/GID resolution's available under current platform.""" + + +if IS_UID_GID_RESOLVABLE: + try: + import grp + import pwd + except ImportError: + """Unavailable in the current env. + + This shouldn't be happening normally. + All of the known cases are excluded via the if clause. + """ + IS_UID_GID_RESOLVABLE = False + grp, pwd = None, None + import struct + + +if IS_WINDOWS and hasattr(socket, 'AF_INET6'): if not hasattr(socket, 'IPPROTO_IPV6'): socket.IPPROTO_IPV6 = 41 if not hasattr(socket, 'IPV6_V6ONLY'): socket.IPV6_V6ONLY = 27 +if not hasattr(socket, 'SO_PEERCRED'): + """ + NOTE: the value for SO_PEERCRED can be architecture specific, in + which case the getsockopt() will hopefully fail. The arch + specific value could be derived from platform.processor() + """ + socket.SO_PEERCRED = 21 if IS_PPC else 17 + + LF = b'\n' CRLF = b'\r\n' TAB = b'\t' SPACE = b' ' -AMPERSAND = b'&' -QUOTE = b'"' COLON = b':' SEMICOLON = b';' EMPTY = b'' -NUMBER_SIGN = b'#' -QUESTION_MARK = b'?' ASTERISK = b'*' FORWARD_SLASH = b'/' -quoted_slash = re.compile(b'(?i)%2F') +QUOTED_SLASH = b'%2F' +QUOTED_SLASH_REGEX = re.compile(b''.join((b'(?i)', QUOTED_SLASH))) comma_separated_headers = [ - ntob(h) for h in - ['Accept', 'Accept-Charset', 'Accept-Encoding', - 'Accept-Language', 'Accept-Ranges', 'Allow', 'Cache-Control', - 'Connection', 'Content-Encoding', 'Content-Language', 'Expect', - 'If-Match', 'If-None-Match', 'Pragma', 'Proxy-Authenticate', 'TE', - 'Trailer', 'Transfer-Encoding', 'Upgrade', 'Vary', 'Via', 'Warning', - 'WWW-Authenticate'] + b'Accept', b'Accept-Charset', b'Accept-Encoding', + b'Accept-Language', b'Accept-Ranges', b'Allow', b'Cache-Control', + b'Connection', b'Content-Encoding', b'Content-Language', b'Expect', + b'If-Match', b'If-None-Match', b'Pragma', b'Proxy-Authenticate', b'TE', + b'Trailer', b'Transfer-Encoding', b'Upgrade', b'Vary', b'Via', b'Warning', + b'WWW-Authenticate', ] @@ -114,13 +175,13 @@ logging.statistics = {} -class HeaderReader(object): +class HeaderReader: """Object for reading headers from an HTTP request. Interface and default implementation. """ - def __call__(self, rfile, hdict=None): + def __call__(self, rfile, hdict=None): # noqa: C901 # FIXME """ Read headers from the given stream into the given header dict. @@ -130,7 +191,8 @@ def __call__(self, rfile, hdict=None): Headers which are repeated are folded together using a comma if their specification so dictates. - This function raises ValueError when the read bytes violate the HTTP spec. + This function raises ValueError when the read bytes violate the HTTP + spec. You should probably return "400 Bad Request" if this happens. """ if hdict is None: @@ -187,16 +249,15 @@ def _allow_header(self, key_name): return orig and '_' not in key_name -class SizeCheckWrapper(object): - """Wraps a file-like object, raising MaxSizeExceeded if too large.""" +class SizeCheckWrapper: + """Wraps a file-like object, raising MaxSizeExceeded if too large. - def __init__(self, rfile, maxlen): - """Initialize SizeCheckWrapper instance. + :param rfile: ``file`` of a limited size + :param int maxlen: maximum length of the file being read + """ - Args: - rfile (file): file of a limited size - maxlen (int): maximum length of the file being read - """ + def __init__(self, rfile, maxlen): + """Initialize SizeCheckWrapper instance.""" self.rfile = rfile self.maxlen = maxlen self.bytes_read = 0 @@ -206,13 +267,12 @@ def _check_length(self): raise errors.MaxSizeExceeded() def read(self, size=None): - """Read a chunk from rfile buffer and return it. + """Read a chunk from ``rfile`` buffer and return it. - Args: - size (int): amount of data to read + :param int size: amount of data to read - Returns: - bytes: Chunk from rfile, limited by size if specified. + :returns: chunk from ``rfile``, limited by size if specified + :rtype: bytes """ data = self.rfile.read(size) self.bytes_read += len(data) @@ -220,13 +280,12 @@ def read(self, size=None): return data def readline(self, size=None): - """Read a single line from rfile buffer and return it. + """Read a single line from ``rfile`` buffer and return it. - Args: - size (int): minimum amount of data to read + :param int size: minimum amount of data to read - Returns: - bytes: One line from rfile. + :returns: one line from ``rfile`` + :rtype: bytes """ if size is not None: data = self.rfile.readline(size) @@ -247,13 +306,12 @@ def readline(self, size=None): return EMPTY.join(res) def readlines(self, sizehint=0): - """Read all lines from rfile buffer and return them. + """Read all lines from ``rfile`` buffer and return them. - Args: - sizehint (int): hint of minimum amount of data to read + :param int sizehint: hint of minimum amount of data to read - Returns: - list[bytes]: Lines of bytes read from rfile. + :returns: lines of bytes read from ``rfile`` + :rtype: list[bytes] """ # Shamelessly stolen from StringIO total = 0 @@ -268,7 +326,7 @@ def readlines(self, sizehint=0): return lines def close(self): - """Release resources allocated for rfile.""" + """Release resources allocated for ``rfile``.""" self.rfile.close() def __iter__(self): @@ -282,35 +340,28 @@ def __next__(self): self._check_length() return data - def next(self): - """Generate next file chunk.""" - data = self.rfile.next() - self.bytes_read += len(data) - self._check_length() - return data + next = __next__ + +class KnownLengthRFile: + """Wraps a file-like object, returning an empty string when exhausted. -class KnownLengthRFile(object): - """Wraps a file-like object, returning an empty string when exhausted.""" + :param rfile: ``file`` of a known size + :param int content_length: length of the file being read + """ def __init__(self, rfile, content_length): - """Initialize KnownLengthRFile instance. - - Args: - rfile (file): file of a known size - content_length (int): length of the file being read - """ + """Initialize KnownLengthRFile instance.""" self.rfile = rfile self.remaining = content_length def read(self, size=None): - """Read a chunk from rfile buffer and return it. + """Read a chunk from ``rfile`` buffer and return it. - Args: - size (int): amount of data to read + :param int size: amount of data to read - Returns: - bytes: Chunk from rfile, limited by size if specified. + :rtype: bytes + :returns: chunk from ``rfile``, limited by size if specified """ if self.remaining == 0: return b'' @@ -324,13 +375,12 @@ def read(self, size=None): return data def readline(self, size=None): - """Read a single line from rfile buffer and return it. + """Read a single line from ``rfile`` buffer and return it. - Args: - size (int): minimum amount of data to read + :param int size: minimum amount of data to read - Returns: - bytes: One line from rfile. + :returns: one line from ``rfile`` + :rtype: bytes """ if self.remaining == 0: return b'' @@ -344,13 +394,12 @@ def readline(self, size=None): return data def readlines(self, sizehint=0): - """Read all lines from rfile buffer and return them. + """Read all lines from ``rfile`` buffer and return them. - Args: - sizehint (int): hint of minimum amount of data to read + :param int sizehint: hint of minimum amount of data to read - Returns: - list[bytes]: Lines of bytes read from rfile. + :returns: lines of bytes read from ``rfile`` + :rtype: list[bytes] """ # Shamelessly stolen from StringIO total = 0 @@ -365,7 +414,7 @@ def readlines(self, sizehint=0): return lines def close(self): - """Release resources allocated for rfile.""" + """Release resources allocated for ``rfile``.""" self.rfile.close() def __iter__(self): @@ -378,23 +427,23 @@ def __next__(self): self.remaining -= len(data) return data + next = __next__ + -class ChunkedRFile(object): +class ChunkedRFile: """Wraps a file-like object, returning an empty string when exhausted. This class is intended to provide a conforming wsgi.input value for request entities that have been encoded with the 'chunked' transfer encoding. + + :param rfile: file encoded with the 'chunked' transfer encoding + :param int maxlen: maximum length of the file being read + :param int bufsize: size of the buffer used to read the file """ def __init__(self, rfile, maxlen, bufsize=8192): - """Initialize ChunkedRFile instance. - - Args: - rfile (file): file encoded with the 'chunked' transfer encoding - maxlen (int): maximum length of the file being read - bufsize (int): size of the buffer used to read the file - """ + """Initialize ChunkedRFile instance.""" self.rfile = rfile self.maxlen = maxlen self.bytes_read = 0 @@ -410,7 +459,9 @@ def _fetch(self): self.bytes_read += len(line) if self.maxlen and self.bytes_read > self.maxlen: - raise errors.MaxSizeExceeded('Request Entity Too Large', self.maxlen) + raise errors.MaxSizeExceeded( + 'Request Entity Too Large', self.maxlen, + ) line = line.strip().split(SEMICOLON, 1) @@ -418,7 +469,10 @@ def _fetch(self): chunk_size = line.pop(0) chunk_size = int(chunk_size, 16) except ValueError: - raise ValueError('Bad chunked transfer size: ' + repr(chunk_size)) + raise ValueError( + 'Bad chunked transfer size: {chunk_size!r}'. + format(chunk_size=chunk_size), + ) if chunk_size <= 0: self.closed = True @@ -437,16 +491,16 @@ def _fetch(self): if crlf != CRLF: raise ValueError( "Bad chunked transfer coding (expected '\\r\\n', " - 'got ' + repr(crlf) + ')') + 'got ' + repr(crlf) + ')', + ) def read(self, size=None): - """Read a chunk from rfile buffer and return it. + """Read a chunk from ``rfile`` buffer and return it. - Args: - size (int): amount of data to read + :param int size: amount of data to read - Returns: - bytes: Chunk from rfile, limited by size if specified. + :returns: chunk from ``rfile``, limited by size if specified + :rtype: bytes """ data = EMPTY @@ -472,13 +526,12 @@ def read(self, size=None): self.buffer = EMPTY def readline(self, size=None): - """Read a single line from rfile buffer and return it. + """Read a single line from ``rfile`` buffer and return it. - Args: - size (int): minimum amount of data to read + :param int size: minimum amount of data to read - Returns: - bytes: One line from rfile. + :returns: one line from ``rfile`` + :rtype: bytes """ data = EMPTY @@ -514,13 +567,12 @@ def readline(self, size=None): self.buffer = self.buffer[newline_pos:] def readlines(self, sizehint=0): - """Read all lines from rfile buffer and return them. + """Read all lines from ``rfile`` buffer and return them. - Args: - sizehint (int): hint of minimum amount of data to read + :param int sizehint: hint of minimum amount of data to read - Returns: - list[bytes]: Lines of bytes read from rfile. + :returns: lines of bytes read from ``rfile`` + :rtype: list[bytes] """ # Shamelessly stolen from StringIO total = 0 @@ -539,10 +591,12 @@ def read_trailer_lines(self): Returns: Generator: yields CRLF separated lines. + """ if not self.closed: raise ValueError( - 'Cannot read trailers until the request body has been read.') + 'Cannot read trailers until the request body has been read.', + ) while True: line = self.rfile.readline() @@ -563,11 +617,11 @@ def read_trailer_lines(self): yield line def close(self): - """Release resources allocated for rfile.""" + """Release resources allocated for ``rfile``.""" self.rfile.close() -class HTTPRequest(object): +class HTTPRequest: """An HTTP Request (and response). A single HTTP connection may consist of multiple request/response pairs. @@ -605,12 +659,17 @@ class HTTPRequest(object): A HeaderReader instance or compatible reader. """ - def __init__(self, server, conn): + def __init__(self, server, conn, proxy_mode=False, strict_mode=True): """Initialize HTTP request container instance. Args: server (HTTPServer): web server object receiving this request conn (HTTPConnection): HTTP connection object for this request + proxy_mode (bool): whether this HTTPServer should behave as a PROXY + server for certain requests + strict_mode (bool): whether we should return a 400 Bad Request when + we encounter a request that a HTTP compliant client should not be + making """ self.server = server self.conn = conn @@ -630,18 +689,23 @@ def __init__(self, server, conn): self.close_connection = self.__class__.close_connection self.chunked_read = False self.chunked_write = self.__class__.chunked_write + self.proxy_mode = proxy_mode + self.strict_mode = strict_mode def parse_request(self): """Parse the next HTTP request start-line and message-headers.""" - self.rfile = SizeCheckWrapper(self.conn.rfile, - self.server.max_request_header_size) + self.rfile = SizeCheckWrapper( + self.conn.rfile, + self.server.max_request_header_size, + ) try: success = self.read_request_line() except errors.MaxSizeExceeded: self.simple_response( '414 Request-URI Too Long', 'The Request-URI sent with the request exceeds the maximum ' - 'allowed bytes.') + 'allowed bytes.', + ) return else: if not success: @@ -653,7 +717,8 @@ def parse_request(self): self.simple_response( '413 Request Entity Too Large', 'The headers sent with the request exceed the maximum ' - 'allowed bytes.') + 'allowed bytes.', + ) return else: if not success: @@ -661,11 +726,12 @@ def parse_request(self): self.ready = True - def read_request_line(self): + def read_request_line(self): # noqa: C901 # FIXME """Read and parse first line of the HTTP request. Returns: bool: True if the request line is valid or False if it's malformed. + """ # HTTP/1.1 connections are persistent by default. If a client # requests a page, then idles (leaves the connection open), @@ -693,74 +759,175 @@ def read_request_line(self): if not request_line.endswith(CRLF): self.simple_response( - '400 Bad Request', 'HTTP requires CRLF terminators') + '400 Bad Request', 'HTTP requires CRLF terminators', + ) return False - - # TEMP: Workaround for Kodi which is using non urlencoded urls - try: - splitter = "HEAD " if "HEAD " in request_line else "GET " - request_path = request_line.split(splitter)[1].split(" HTTP/")[0] - if QUOTE in request_path: - for part in request_path.split('"')[1::2]: - bad_part = "%s%s%s" %(QUOTE, part, QUOTE) - request_line = request_line.replace(bad_part, quote(part)) - elif SPACE in request_path: - corrected_request_path = request_path.replace(SPACE, '%20') - request_line = request_line.replace(request_path, corrected_request_path) - if not QUESTION_MARK in request_path and AMPERSAND in request_path: - action = request_path.split(AMPERSAND)[0] - wrong_action = "%s%s" %(action, AMPERSAND) - correct_action = "%s%s" %(action, QUESTION_MARK) - request_line = request_line.replace(wrong_action, correct_action) - except Exception: - pass - # end of kodi workaround - + try: method, uri, req_protocol = request_line.strip().split(SPACE, 2) - req_protocol_str = req_protocol.decode('ascii') - rp = int(req_protocol_str[5]), int(req_protocol_str[7]) + if not req_protocol.startswith(b'HTTP/'): + self.simple_response( + '400 Bad Request', 'Malformed Request-Line: bad protocol', + ) + return False + rp = req_protocol[5:].split(b'.', 1) + if len(rp) != 2: + self.simple_response( + '400 Bad Request', 'Malformed Request-Line: bad version', + ) + return False + rp = tuple(map(int, rp)) # Minor.Major must be threat as integers + if rp > (1, 1): + self.simple_response( + '505 HTTP Version Not Supported', 'Cannot fulfill request', + ) + return False except (ValueError, IndexError): self.simple_response('400 Bad Request', 'Malformed Request-Line') return False self.uri = uri - self.method = method - - # uri may be an abs_path (including "http://host.domain.tld"); - scheme, authority, path = self.parse_request_uri(uri) - if path is None: - self.simple_response('400 Bad Request', - 'Invalid path in Request-URI.') + self.method = method.upper() + + if self.strict_mode and method != self.method: + resp = ( + 'Malformed method name: According to RFC 2616 ' + '(section 5.1.1) and its successors ' + 'RFC 7230 (section 3.1.1) and RFC 7231 (section 4.1) ' + 'method names are case-sensitive and uppercase.' + ) + self.simple_response('400 Bad Request', resp) return False - if NUMBER_SIGN in path: - self.simple_response('400 Bad Request', - 'Illegal #fragment in Request-URI.') + + try: + if six.PY2: # FIXME: Figure out better way to do this + # Ref: https://stackoverflow.com/a/196392/595220 (like this?) + """This is a dummy check for unicode in URI.""" + ntou(bton(uri, 'ascii'), 'ascii') + scheme, authority, path, qs, fragment = urllib.parse.urlsplit(uri) + except UnicodeError: + self.simple_response('400 Bad Request', 'Malformed Request-URI') return False - if scheme: - self.scheme = scheme + uri_is_absolute_form = (scheme or authority) + + if self.method == b'OPTIONS': + # TODO: cover this branch with tests + path = ( + uri + # https://tools.ietf.org/html/rfc7230#section-5.3.4 + if (self.proxy_mode and uri_is_absolute_form) + else path + ) + elif self.method == b'CONNECT': + # TODO: cover this branch with tests + if not self.proxy_mode: + self.simple_response('405 Method Not Allowed') + return False - qs = EMPTY - if QUESTION_MARK in path: - path, qs = path.split(QUESTION_MARK, 1) + # `urlsplit()` above parses "example.com:3128" as path part of URI. + # this is a workaround, which makes it detect netloc correctly + uri_split = urllib.parse.urlsplit(b''.join((b'//', uri))) + _scheme, _authority, _path, _qs, _fragment = uri_split + _port = EMPTY + try: + _port = uri_split.port + except ValueError: + pass - # Unquote the path+params (e.g. "/this%20path" -> "/this path"). - # http://www.w3.org/Protocols/rfc2616/rfc2616-sec5.html#sec5.1.2 - # - # But note that "...a URI must be separated into its components - # before the escaped characters within those components can be - # safely decoded." http://www.ietf.org/rfc/rfc2396.txt, sec 2.4.2 - # Therefore, "/this%2Fpath" becomes "/this%2Fpath", not "/this/path". - try: - atoms = [ - unquote_to_bytes(x) - for x in quoted_slash.split(path) - ] - except ValueError as ex: - self.simple_response('400 Bad Request', ex.args[0]) - return False - path = b'%2F'.join(atoms) + # FIXME: use third-party validation to make checks against RFC + # the validation doesn't take into account, that urllib parses + # invalid URIs without raising errors + # https://tools.ietf.org/html/rfc7230#section-5.3.3 + invalid_path = ( + _authority != uri + or not _port + or any((_scheme, _path, _qs, _fragment)) + ) + if invalid_path: + self.simple_response( + '400 Bad Request', + 'Invalid path in Request-URI: request-' + 'target must match authority-form.', + ) + return False + + authority = path = _authority + scheme = qs = fragment = EMPTY + else: + disallowed_absolute = ( + self.strict_mode + and not self.proxy_mode + and uri_is_absolute_form + ) + if disallowed_absolute: + # https://tools.ietf.org/html/rfc7230#section-5.3.2 + # (absolute form) + """Absolute URI is only allowed within proxies.""" + self.simple_response( + '400 Bad Request', + 'Absolute URI not allowed if server is not a proxy.', + ) + return False + + invalid_path = ( + self.strict_mode + and not uri.startswith(FORWARD_SLASH) + and not uri_is_absolute_form + ) + if invalid_path: + # https://tools.ietf.org/html/rfc7230#section-5.3.1 + # (origin_form) and + """Path should start with a forward slash.""" + resp = ( + 'Invalid path in Request-URI: request-target must contain ' + 'origin-form which starts with absolute-path (URI ' + 'starting with a slash "/").' + ) + self.simple_response('400 Bad Request', resp) + return False + + if fragment: + self.simple_response( + '400 Bad Request', + 'Illegal #fragment in Request-URI.', + ) + return False + + if path is None: + # FIXME: It looks like this case cannot happen + self.simple_response( + '400 Bad Request', + 'Invalid path in Request-URI.', + ) + return False + + # Unquote the path+params (e.g. "/this%20path" -> "/this path"). + # https://www.w3.org/Protocols/rfc2616/rfc2616-sec5.html#sec5.1.2 + # + # But note that "...a URI must be separated into its components + # before the escaped characters within those components can be + # safely decoded." https://www.ietf.org/rfc/rfc2396.txt, sec 2.4.2 + # Therefore, "/this%2Fpath" becomes "/this%2Fpath", not + # "/this/path". + try: + # TODO: Figure out whether exception can really happen here. + # It looks like it's caught on urlsplit() call above. + atoms = [ + urllib.parse.unquote_to_bytes(x) + for x in QUOTED_SLASH_REGEX.split(path) + ] + except ValueError as ex: + self.simple_response('400 Bad Request', ex.args[0]) + return False + path = QUOTED_SLASH.join(atoms) + + if not path.startswith(FORWARD_SLASH): + path = FORWARD_SLASH + path + + if scheme is not EMPTY: + self.scheme = scheme + self.authority = authority self.path = path # Note that, like wsgiref and most other HTTP servers, @@ -790,8 +957,14 @@ def read_request_line(self): return True - def read_request_headers(self): - """Read self.rfile into self.inheaders. Return success.""" + def read_request_headers(self): # noqa: C901 # FIXME + """Read ``self.rfile`` into ``self.inheaders``. + + Ref: :py:attr:`self.inheaders `. + + :returns: success status + :rtype: bool + """ # then all the http headers try: self.header_reader(self.rfile, self.inheaders) @@ -800,11 +973,22 @@ def read_request_headers(self): return False mrbs = self.server.max_request_body_size - if mrbs and int(self.inheaders.get(b'Content-Length', 0)) > mrbs: + + try: + cl = int(self.inheaders.get(b'Content-Length', 0)) + except ValueError: + self.simple_response( + '400 Bad Request', + 'Malformed Content-Length Header.', + ) + return False + + if mrbs and cl > mrbs: self.simple_response( '413 Request Entity Too Large', 'The entity sent with the request exceeds the maximum ' - 'allowed bytes.') + 'allowed bytes.', + ) return False # Persistent connection support @@ -858,8 +1042,10 @@ def read_request_headers(self): # Don't use simple_response here, because it emits headers # we don't want. See # https://github.com/cherrypy/cherrypy/issues/951 - msg = self.server.protocol.encode('ascii') - msg += b' 100 Continue\r\n\r\n' + msg = b''.join(( + self.server.protocol.encode('ascii'), SPACE, b'100 Continue', + CRLF, CRLF, + )) try: self.conn.wfile.write(msg) except socket.error as ex: @@ -867,44 +1053,6 @@ def read_request_headers(self): raise return True - def parse_request_uri(self, uri): - """Parse a Request-URI into (scheme, authority, path). - - Note that Request-URI's must be one of:: - - Request-URI = "*" | absoluteURI | abs_path | authority - - Therefore, a Request-URI which starts with a double forward-slash - cannot be a "net_path":: - - net_path = "//" authority [ abs_path ] - - Instead, it must be interpreted as an "abs_path" with an empty first - path segment:: - - abs_path = "/" path_segments - path_segments = segment *( "/" segment ) - segment = *pchar *( ";" param ) - param = *pchar - """ - if uri == ASTERISK: - return None, None, uri - - parsed = urllib.parse.urlparse(uri) - if parsed.scheme and QUESTION_MARK not in parsed.scheme: - # An absoluteURI. - # If there's a scheme (and it must be http or https), then: - # http_URL = "http:" "//" host [ ":" port ] [ abs_path [ "?" query - # ]] - return parsed.scheme, parsed.netloc, parsed.path - - if uri.startswith(FORWARD_SLASH): - # An abs_path. - return None, None, uri - else: - # An authority. - return None, uri, None - def respond(self): """Call the gateway and write its iterable output.""" mrbs = self.server.max_request_body_size @@ -917,15 +1065,14 @@ def respond(self): self.simple_response( '413 Request Entity Too Large', 'The entity sent with the request exceeds the ' - 'maximum allowed bytes.') + 'maximum allowed bytes.', + ) return self.rfile = KnownLengthRFile(self.conn.rfile, cl) self.server.gateway(self).respond() + self.ready and self.ensure_headers_sent() - if (self.ready and not self.sent_headers): - self.sent_headers = True - self.send_headers() if self.chunked_write: self.conn.wfile.write(b'0\r\n\r\n') @@ -966,6 +1113,12 @@ def simple_response(self, status, msg=''): if ex.args[0] not in errors.socket_errors_to_ignore: raise + def ensure_headers_sent(self): + """Ensure headers are sent to the client if not already sent.""" + if not self.sent_headers: + self.sent_headers = True + self.send_headers() + def write(self, chunk): """Write unbuffered data to the client.""" if self.chunked_write and chunk: @@ -975,10 +1128,11 @@ def write(self, chunk): else: self.conn.wfile.write(chunk) - def send_headers(self): + def send_headers(self): # noqa: C901 # FIXME """Assert, process, and send the HTTP response message-headers. - You must set self.status, and self.outheaders before calling this. + You must set ``self.status``, and :py:attr:`self.outheaders + ` before calling this. """ hkeys = [key.lower() for key, value in self.outheaders] status = int(self.status[:3]) @@ -993,7 +1147,11 @@ def send_headers(self): if status < 200 or status in (204, 205, 304): pass else: - if self.response_protocol == 'HTTP/1.1' and self.method != b'HEAD': + needs_chunked = ( + self.response_protocol == 'HTTP/1.1' + and self.method != b'HEAD' + ) + if needs_chunked: # Use the chunked transfer-coding self.chunked_write = True self.outheaders.append((b'Transfer-Encoding', b'chunked')) @@ -1001,6 +1159,12 @@ def send_headers(self): # Closing the conn is the only way to determine len. self.close_connection = True + # Override the decision to not close the connection if the connection + # manager doesn't have space for it. + if not self.close_connection: + can_keep = self.server.can_add_keepalive_connection + self.close_connection = not can_keep + if b'connection' not in hkeys: if self.response_protocol == 'HTTP/1.1': # Both server and client are HTTP/1.1 or better @@ -1011,6 +1175,14 @@ def send_headers(self): if not self.close_connection: self.outheaders.append((b'Connection', b'Keep-Alive')) + if (b'Connection', b'Keep-Alive') in self.outheaders: + self.outheaders.append(( + b'Keep-Alive', + u'timeout={connection_timeout}'. + format(connection_timeout=self.server.timeout). + encode('ISO-8859-1'), + )) + if (not self.close_connection) and (not self.chunked_read): # Read any remaining request body data on the socket. # "If an origin server receives a request that does not include an @@ -1048,7 +1220,7 @@ def send_headers(self): self.conn.wfile.write(EMPTY.join(buf)) -class HTTPConnection(object): +class HTTPConnection: """An HTTP connection (active socket).""" remote_addr = None @@ -1057,13 +1229,19 @@ class HTTPConnection(object): rbufsize = io.DEFAULT_BUFFER_SIZE wbufsize = io.DEFAULT_BUFFER_SIZE RequestHandlerClass = HTTPRequest + peercreds_enabled = False + peercreds_resolve_enabled = False + + # Fields set by ConnectionManager. + last_used = None def __init__(self, server, sock, makefile=MakeFile): """Initialize HTTPConnection instance. Args: server (HTTPServer): web server object receiving this request - socket (socket._socketobject): the raw socket object (usually TCP) for this connection + sock (socket._socketobject): the raw socket object (usually + TCP) for this connection makefile (file): a fileobject class for reading from the socket """ self.server = server @@ -1072,80 +1250,68 @@ def __init__(self, server, sock, makefile=MakeFile): self.wfile = makefile(sock, 'wb', self.wbufsize) self.requests_seen = 0 - def communicate(self): - """Read each request and respond appropriately.""" + self.peercreds_enabled = self.server.peercreds_enabled + self.peercreds_resolve_enabled = self.server.peercreds_resolve_enabled + + # LRU cached methods: + # Ref: https://stackoverflow.com/a/14946506/595220 + self.resolve_peer_creds = ( + lru_cache(maxsize=1)(self.resolve_peer_creds) + ) + self.get_peer_creds = ( + lru_cache(maxsize=1)(self.get_peer_creds) + ) + + def communicate(self): # noqa: C901 # FIXME + """Read each request and respond appropriately. + + Returns true if the connection should be kept open. + """ request_seen = False try: - while True: - # (re)set req to None so that if something goes wrong in - # the RequestHandlerClass constructor, the error doesn't - # get written to the previous request. - req = None - req = self.RequestHandlerClass(self.server, self) - - # This order of operations should guarantee correct pipelining. - req.parse_request() - if self.server.stats['Enabled']: - self.requests_seen += 1 - if not req.ready: - # Something went wrong in the parsing (and the server has - # probably already made a simple_response). Return and - # let the conn close. - return + req = self.RequestHandlerClass(self.server, self) + req.parse_request() + if self.server.stats['Enabled']: + self.requests_seen += 1 + if not req.ready: + # Something went wrong in the parsing (and the server has + # probably already made a simple_response). Return and + # let the conn close. + return False - request_seen = True - req.respond() - if req.close_connection: - return + request_seen = True + req.respond() + if not req.close_connection: + return True except socket.error as ex: errnum = ex.args[0] # sadly SSL sockets return a different (longer) time out string - if ( - errnum == 'timed out' or - errnum == 'The read operation timed out' - ): + timeout_errs = 'timed out', 'The read operation timed out' + if errnum in timeout_errs: # Don't error if we're between requests; only error # if 1) no request has been started at all, or 2) we're # in the middle of a request. # See https://github.com/cherrypy/cherrypy/issues/853 if (not request_seen) or (req and req.started_request): - # Don't bother writing the 408 if the response - # has already started being written. - if req and not req.sent_headers: - try: - req.simple_response('408 Request Timeout') - except errors.FatalSSLAlert: - # Close the connection. - return - except errors.NoSSLError: - self._handle_no_ssl(req) + self._conditional_error(req, '408 Request Timeout') elif errnum not in errors.socket_errors_to_ignore: - self.server.error_log('socket.error %s' % repr(errnum), - level=logging.WARNING, traceback=True) - if req and not req.sent_headers: - try: - req.simple_response('500 Internal Server Error') - except errors.FatalSSLAlert: - # Close the connection. - return - except errors.NoSSLError: - self._handle_no_ssl(req) - return + self.server.error_log( + 'socket.error %s' % repr(errnum), + level=logging.WARNING, traceback=True, + ) + self._conditional_error(req, '500 Internal Server Error') except (KeyboardInterrupt, SystemExit): raise except errors.FatalSSLAlert: - # Close the connection. - return + pass except errors.NoSSLError: self._handle_no_ssl(req) except Exception as ex: - self.server.error_log(repr(ex), level=logging.ERROR, traceback=True) - if req and not req.sent_headers: - try: - req.simple_response('500 Internal Server Error') - except errors.FatalSSLAlert: - # Close the connection. - return + self.server.error_log( + repr(ex), level=logging.ERROR, traceback=True, + ) + self._conditional_error(req, '500 Internal Server Error') + return False linger = False @@ -1153,7 +1319,12 @@ def _handle_no_ssl(self, req): if not req or req.sent_headers: return # Unwrap wfile - self.wfile = MakeFile(self.socket._sock, 'wb', self.wbufsize) + try: + resp_sock = self.socket._sock + except AttributeError: + # self.socket is of OpenSSL.SSL.Connection type + resp_sock = self.socket._socket + self.wfile = StreamWriter(resp_sock, 'wb', self.wbufsize) msg = ( 'The client sent a plain HTTP request, but ' 'this server only speaks HTTPS on this port.' @@ -1161,6 +1332,22 @@ def _handle_no_ssl(self, req): req.simple_response('400 Bad Request', msg) self.linger = True + def _conditional_error(self, req, response): + """Respond with an error. + + Don't bother writing if a response + has already started being written. + """ + if not req or req.sent_headers: + return + + try: + req.simple_response(response) + except errors.FatalSSLAlert: + pass + except errors.NoSSLError: + self._handle_no_ssl(req) + def close(self): """Close the socket underlying this connection.""" self.rfile.close() @@ -1177,6 +1364,106 @@ def close(self): # Apache does, but not today. pass + def get_peer_creds(self): # LRU cached on per-instance basis, see __init__ + """Return the PID/UID/GID tuple of the peer socket for UNIX sockets. + + This function uses SO_PEERCRED to query the UNIX PID, UID, GID + of the peer, which is only available if the bind address is + a UNIX domain socket. + + Raises: + NotImplementedError: in case of unsupported socket type + RuntimeError: in case of SO_PEERCRED lookup unsupported or disabled + + """ + PEERCRED_STRUCT_DEF = '3i' + + if IS_WINDOWS or self.socket.family != socket.AF_UNIX: + raise NotImplementedError( + 'SO_PEERCRED is only supported in Linux kernel and WSL', + ) + elif not self.peercreds_enabled: + raise RuntimeError( + 'Peer creds lookup is disabled within this server', + ) + + try: + peer_creds = self.socket.getsockopt( + # FIXME: Use LOCAL_CREDS for BSD-like OSs + # Ref: https://gist.github.com/LucaFilipozzi/e4f1e118202aff27af6aadebda1b5d91 # noqa + socket.SOL_SOCKET, socket.SO_PEERCRED, + struct.calcsize(PEERCRED_STRUCT_DEF), + ) + except socket.error as socket_err: + """Non-Linux kernels don't support SO_PEERCRED. + + Refs: + http://welz.org.za/notes/on-peer-cred.html + https://github.com/daveti/tcpSockHack + msdn.microsoft.com/en-us/commandline/wsl/release_notes#build-15025 + """ + six.raise_from( # 3.6+: raise RuntimeError from socket_err + RuntimeError, + socket_err, + ) + else: + pid, uid, gid = struct.unpack(PEERCRED_STRUCT_DEF, peer_creds) + return pid, uid, gid + + @property + def peer_pid(self): + """Return the id of the connected peer process.""" + pid, _, _ = self.get_peer_creds() + return pid + + @property + def peer_uid(self): + """Return the user id of the connected peer process.""" + _, uid, _ = self.get_peer_creds() + return uid + + @property + def peer_gid(self): + """Return the group id of the connected peer process.""" + _, _, gid = self.get_peer_creds() + return gid + + def resolve_peer_creds(self): # LRU cached on per-instance basis + """Look up the username and group tuple of the ``PEERCREDS``. + + :returns: the username and group tuple of the ``PEERCREDS`` + + :raises NotImplementedError: if the OS is unsupported + :raises RuntimeError: if UID/GID lookup is unsupported or disabled + """ + if not IS_UID_GID_RESOLVABLE: + raise NotImplementedError( + 'UID/GID lookup is unavailable under current platform. ' + 'It can only be done under UNIX-like OS ' + 'but not under the Google App Engine', + ) + elif not self.peercreds_resolve_enabled: + raise RuntimeError( + 'UID/GID lookup is disabled within this server', + ) + + user = pwd.getpwuid(self.peer_uid).pw_name # [0] + group = grp.getgrgid(self.peer_gid).gr_name # [0] + + return user, group + + @property + def peer_user(self): + """Return the username of the connected peer process.""" + user, _ = self.resolve_peer_creds() + return user + + @property + def peer_group(self): + """Return the group of the connected peer process.""" + _, group = self.resolve_peer_creds() + return group + def _close_kernel_socket(self): """Close kernel socket in outdated Python versions. @@ -1192,37 +1479,7 @@ def _close_kernel_socket(self): self.socket._sock.close() -try: - import fcntl -except ImportError: - try: - from ctypes import windll, WinError - import ctypes.wintypes - _SetHandleInformation = windll.kernel32.SetHandleInformation - _SetHandleInformation.argtypes = [ - ctypes.wintypes.HANDLE, - ctypes.wintypes.DWORD, - ctypes.wintypes.DWORD, - ] - _SetHandleInformation.restype = ctypes.wintypes.BOOL - except ImportError: - def prevent_socket_inheritance(sock): - """Dummy function, since neither fcntl nor ctypes are available.""" - pass - else: - def prevent_socket_inheritance(sock): - """Mark the given socket fd as non-inheritable (Windows).""" - if not _SetHandleInformation(sock.fileno(), 1, 0): - raise WinError() -else: - def prevent_socket_inheritance(sock): - """Mark the given socket fd as non-inheritable (POSIX).""" - fd = sock.fileno() - old_flags = fcntl.fcntl(fd, fcntl.F_GETFD) - fcntl.fcntl(fd, fcntl.F_SETFD, old_flags | fcntl.FD_CLOEXEC) - - -class HTTPServer(object): +class HTTPServer: """An HTTP server.""" _bind_addr = '127.0.0.1' @@ -1235,7 +1492,9 @@ class HTTPServer(object): """The minimum number of worker threads to create (default 10).""" maxthreads = None - """The maximum number of worker threads to create (default -1 = no limit).""" + """The maximum number of worker threads to create. + + (default -1 = no limit)""" server_name = None """The name of the server; defaults to ``self.version``.""" @@ -1247,15 +1506,19 @@ class HTTPServer(object): features used in the response.""" request_queue_size = 5 - """The 'backlog' arg to socket.listen(); max queued connections (default 5).""" + """The 'backlog' arg to socket.listen(); max queued connections. + + (default 5).""" shutdown_timeout = 5 - """The total time, in seconds, to wait for worker threads to cleanly exit.""" + """The total time to wait for worker threads to cleanly exit. + + Specified in seconds.""" timeout = 10 """The timeout in seconds for accepted connections (default 10).""" - version = 'Cheroot/' + __version__ + version = 'Cheroot/{version!s}'.format(version=__version__) """A version string for the HTTPServer.""" software = None @@ -1265,7 +1528,7 @@ class HTTPServer(object): """ ready = False - """An internal flag which marks whether the socket is accepting connections.""" + """Internal flag which indicating the socket is accepting connections.""" max_request_header_size = 0 """The maximum size, in bytes, for request headers, or 0 for no limit.""" @@ -1280,12 +1543,34 @@ class HTTPServer(object): """The class to use for handling HTTP connections.""" ssl_adapter = None - """An instance of ssl.Adapter (or a subclass). + """An instance of ``ssl.Adapter`` (or a subclass). - You must have the corresponding SSL driver library installed. + Ref: :py:class:`ssl.Adapter `. + + You must have the corresponding TLS driver library installed. + """ + + peercreds_enabled = False + """ + If :py:data:`True`, peer creds will be looked up via UNIX domain socket. """ - def __init__(self, bind_addr, gateway, minthreads=10, maxthreads=-1, server_name=None): + peercreds_resolve_enabled = False + """ + If :py:data:`True`, username/group will be looked up in the OS from + ``PEERCREDS``-provided IDs. + """ + + keep_alive_conn_limit = 10 + """The maximum number of waiting keep-alive connections that will be kept open. + + Default is 10. Set to None to have unlimited connections.""" + + def __init__( + self, bind_addr, gateway, + minthreads=10, maxthreads=-1, server_name=None, + peercreds_enabled=False, peercreds_resolve_enabled=False, + ): """Initialize HTTPServer instance. Args: @@ -1293,23 +1578,24 @@ def __init__(self, bind_addr, gateway, minthreads=10, maxthreads=-1, server_name gateway (Gateway): gateway for processing HTTP requests minthreads (int): minimum number of threads for HTTP thread pool maxthreads (int): maximum number of threads for HTTP thread pool - server_name (str): web server name to be advertised via Server HTTP header + server_name (str): web server name to be advertised via Server + HTTP header """ self.bind_addr = bind_addr self.gateway = gateway - self.requests = threadpool.ThreadPool(self, min=minthreads or 1, max=maxthreads) + self.requests = threadpool.ThreadPool( + self, min=minthreads or 1, max=maxthreads, + ) + self.serving = False if not server_name: server_name = self.version - """ - Default server name. It is used to fall back to ``socket.gethostname()``. - Ref: cherrypy/cheroot#33. If the previous logic needed the following should restore it: - - >>> from cheroot._compat import import ntou, tonative - >>> ntou(tonative(ntou(tonative(hn, 'utf-8'), 'utf-8').encode('idna'))) - """ self.server_name = server_name + self.peercreds_enabled = peercreds_enabled + self.peercreds_resolve_enabled = ( + peercreds_resolve_enabled and peercreds_enabled + ) self.clear_stats() def clear_stats(self): @@ -1327,20 +1613,30 @@ def clear_stats(self): 'Threads Idle': lambda s: getattr(self.requests, 'idle', None), 'Socket Errors': 0, 'Requests': lambda s: (not s['Enabled']) and -1 or sum( - [w['Requests'](w) for w in s['Worker Threads'].values()], 0), + (w['Requests'](w) for w in s['Worker Threads'].values()), 0, + ), 'Bytes Read': lambda s: (not s['Enabled']) and -1 or sum( - [w['Bytes Read'](w) for w in s['Worker Threads'].values()], 0), + (w['Bytes Read'](w) for w in s['Worker Threads'].values()), 0, + ), 'Bytes Written': lambda s: (not s['Enabled']) and -1 or sum( - [w['Bytes Written'](w) for w in s['Worker Threads'].values()], - 0), + (w['Bytes Written'](w) for w in s['Worker Threads'].values()), + 0, + ), 'Work Time': lambda s: (not s['Enabled']) and -1 or sum( - [w['Work Time'](w) for w in s['Worker Threads'].values()], 0), + (w['Work Time'](w) for w in s['Worker Threads'].values()), 0, + ), 'Read Throughput': lambda s: (not s['Enabled']) and -1 or sum( - [w['Bytes Read'](w) / (w['Work Time'](w) or 1e-6) - for w in s['Worker Threads'].values()], 0), + ( + w['Bytes Read'](w) / (w['Work Time'](w) or 1e-6) + for w in s['Worker Threads'].values() + ), 0, + ), 'Write Throughput': lambda s: (not s['Enabled']) and -1 or sum( - [w['Bytes Written'](w) / (w['Work Time'](w) or 1e-6) - for w in s['Worker Threads'].values()], 0), + ( + w['Bytes Written'](w) / (w['Work Time'](w) or 1e-6) + for w in s['Worker Threads'].values() + ), 0, + ), 'Worker Threads': {}, } logging.statistics['Cheroot HTTPServer %d' % id(self)] = self.stats @@ -1354,13 +1650,42 @@ def runtime(self): def __str__(self): """Render Server instance representing bind address.""" - return '%s.%s(%r)' % (self.__module__, self.__class__.__name__, - self.bind_addr) + return '%s.%s(%r)' % ( + self.__module__, self.__class__.__name__, + self.bind_addr, + ) + + @property + def bind_addr(self): + """Return the interface on which to listen for connections. + + For TCP sockets, a (host, port) tuple. Host values may be any + :term:`IPv4` or :term:`IPv6` address, or any valid hostname. + The string 'localhost' is a synonym for '127.0.0.1' (or '::1', + if your hosts file prefers :term:`IPv6`). + The string '0.0.0.0' is a special :term:`IPv4` entry meaning + "any active interface" (INADDR_ANY), and '::' is the similar + IN6ADDR_ANY for :term:`IPv6`. + The empty string or :py:data:`None` are not allowed. - def _get_bind_addr(self): + For UNIX sockets, supply the file name as a string. + + Systemd socket activation is automatic and doesn't require tempering + with this variable. + + .. glossary:: + + :abbr:`IPv4 (Internet Protocol version 4)` + Internet Protocol version 4 + + :abbr:`IPv6 (Internet Protocol version 6)` + Internet Protocol version 6 + """ return self._bind_addr - def _set_bind_addr(self, value): + @bind_addr.setter + def bind_addr(self, value): + """Set the interface on which to listen for connections.""" if isinstance(value, tuple) and value[0] in ('', None): # Despite the socket module docs, using '' does not # allow AI_PASSIVE to work. Passing None instead @@ -1372,26 +1697,12 @@ def _set_bind_addr(self, value): # None N 127.0.0.1 # But since you can get the same effect with an explicit # '0.0.0.0', we deny both the empty string and None as values. - raise ValueError("Host values of '' or None are not allowed. " - "Use '0.0.0.0' (IPv4) or '::' (IPv6) instead " - 'to listen on all active interfaces.') + raise ValueError( + "Host values of '' or None are not allowed. " + "Use '0.0.0.0' (IPv4) or '::' (IPv6) instead " + 'to listen on all active interfaces.', + ) self._bind_addr = value - bind_addr = property( - _get_bind_addr, - _set_bind_addr, - doc="""The interface on which to listen for connections. - - For TCP sockets, a (host, port) tuple. Host values may be any IPv4 - or IPv6 address, or any valid hostname. The string 'localhost' is a - synonym for '127.0.0.1' (or '::1', if your hosts file prefers IPv6). - The string '0.0.0.0' is a special IPv4 entry meaning "any active - interface" (INADDR_ANY), and '::' is the similar IN6ADDR_ANY for - IPv6. The empty string or None are not allowed. - - For UNIX sockets, supply the filename as a string. - - Systemd socket activation is automatic and doesn't require tempering - with this variable""") def safe_start(self): """Run the server forever, and stop it cleanly on exit.""" @@ -1408,12 +1719,12 @@ def safe_start(self): self.stop() raise - def start(self): - """Run the server forever.""" - # We don't have to trap KeyboardInterrupt or SystemExit here, - # because cherrpy.server already does so, calling self.stop() for us. - # If you're using this server with another framework, you should - # trap those exceptions in whatever code block calls start(). + def prepare(self): # noqa: C901 # FIXME + """Prepare server to serving requests. + + It binds a socket's port, setups the socket to ``listen()`` and does + other preparing things. + """ self._interrupt = None if self.software is None: @@ -1421,26 +1732,17 @@ def start(self): # Select the appropriate socket self.socket = None + msg = 'No socket could be created' if os.getenv('LISTEN_PID', None): # systemd socket activation self.socket = socket.fromfd(3, socket.AF_INET, socket.SOCK_STREAM) - elif isinstance(self.bind_addr, six.string_types): + elif isinstance(self.bind_addr, (six.text_type, six.binary_type)): # AF_UNIX socket - - # So we can reuse the socket... - try: - os.unlink(self.bind_addr) - except: - pass - - # So everyone can access the socket... try: - os.chmod(self.bind_addr, 0o777) - except: - pass - - info = [ - (socket.AF_UNIX, socket.SOCK_STREAM, 0, '', self.bind_addr)] + self.bind_unix_socket(self.bind_addr) + except socket.error as serr: + msg = '%s -- (%s: %s)' % (msg, self.bind_addr, serr) + six.raise_from(socket.error(msg), serr) else: # AF_INET or AF_INET6 socket # Get the correct address family for our host (allows IPv6 @@ -1449,17 +1751,18 @@ def start(self): try: info = socket.getaddrinfo( host, port, socket.AF_UNSPEC, - socket.SOCK_STREAM, 0, socket.AI_PASSIVE) + socket.SOCK_STREAM, 0, socket.AI_PASSIVE, + ) except socket.gaierror: - if ':' in self.bind_addr[0]: - info = [(socket.AF_INET6, socket.SOCK_STREAM, - 0, '', self.bind_addr + (0, 0))] - else: - info = [(socket.AF_INET, socket.SOCK_STREAM, - 0, '', self.bind_addr)] + sock_type = socket.AF_INET + bind_addr = self.bind_addr + + if ':' in host: + sock_type = socket.AF_INET6 + bind_addr = bind_addr + (0, 0) + + info = [(sock_type, socket.SOCK_STREAM, 0, '', bind_addr)] - if not self.socket: - msg = 'No socket could be created' for res in info: af, socktype, proto, canonname, sa = res try: @@ -1471,33 +1774,74 @@ def start(self): self.socket.close() self.socket = None - if not self.socket: - raise socket.error(msg) + if not self.socket: + raise socket.error(msg) # Timeout so KeyboardInterrupt can be caught on Win32 self.socket.settimeout(1) self.socket.listen(self.request_queue_size) + # must not be accessed once stop() has been called + self._connections = connections.ConnectionManager(self) + # Create worker threads self.requests.start() self.ready = True self._start_time = time.time() + + def serve(self): + """Serve requests, after invoking :func:`prepare()`.""" + self.serving = True while self.ready: try: self.tick() except (KeyboardInterrupt, SystemExit): raise - except: - self.error_log('Error in HTTPServer.tick', level=logging.ERROR, - traceback=True) + except Exception: + self.error_log( + 'Error in HTTPServer.tick', level=logging.ERROR, + traceback=True, + ) - if self.interrupt: - while self.interrupt is True: - # Wait for self.stop() to complete. See _set_interrupt. - time.sleep(0.1) - if self.interrupt: - raise self.interrupt + self.serving = False + + def start(self): + """Run the server forever. + + It is shortcut for invoking :func:`prepare()` then :func:`serve()`. + """ + # We don't have to trap KeyboardInterrupt or SystemExit here, + # because cherrypy.server already does so, calling self.stop() for us. + # If you're using this server with another framework, you should + # trap those exceptions in whatever code block calls start(). + self.prepare() + self.serve() + + @contextlib.contextmanager + def _run_in_thread(self): + """Context manager for running this server in a thread.""" + self.prepare() + thread = threading.Thread(target=self.serve) + thread.setDaemon(True) + thread.start() + try: + yield thread + finally: + self.stop() + + @property + def can_add_keepalive_connection(self): + """Flag whether it is allowed to add a new keep-alive connection.""" + return self.ready and self._connections.can_add_keepalive_connection + + def put_conn(self, conn): + """Put an idle connection back into the ConnectionManager.""" + if self.ready: + self._connections.put(conn) + else: + # server is shutting down, just close it + conn.close() def error_log(self, msg='', level=20, traceback=False): """Write error message to log. @@ -1508,7 +1852,7 @@ def error_log(self, msg='', level=20, traceback=False): traceback (bool): add traceback to output or not """ # Override this in subclasses as desired - sys.stderr.write(msg + '\n') + sys.stderr.write('{msg!s}\n'.format(msg=msg)) sys.stderr.flush() if traceback: tblines = traceback_.format_exc() @@ -1517,144 +1861,221 @@ def error_log(self, msg='', level=20, traceback=False): def bind(self, family, type, proto=0): """Create (or recreate) the actual socket object.""" - self.socket = socket.socket(family, type, proto) - prevent_socket_inheritance(self.socket) - self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - if self.nodelay and not isinstance(self.bind_addr, str): - self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + sock = self.prepare_socket( + self.bind_addr, + family, type, proto, + self.nodelay, self.ssl_adapter, + ) + sock = self.socket = self.bind_socket(sock, self.bind_addr) + self.bind_addr = self.resolve_real_bind_addr(sock) + return sock + + def bind_unix_socket(self, bind_addr): # noqa: C901 # FIXME + """Create (or recreate) a UNIX socket object.""" + if IS_WINDOWS: + """ + Trying to access socket.AF_UNIX under Windows + causes an AttributeError. + """ + raise ValueError( # or RuntimeError? + 'AF_UNIX sockets are not supported under Windows.', + ) + + fs_permissions = 0o777 # TODO: allow changing mode + + try: + # Make possible reusing the socket... + os.unlink(self.bind_addr) + except OSError: + """ + File does not exist, which is the primary goal anyway. + """ + except TypeError as typ_err: + err_msg = str(typ_err) + if ( + 'remove() argument 1 must be encoded ' + 'string without null bytes, not unicode' + not in err_msg + and 'embedded NUL character' not in err_msg # py34 + and 'argument must be a ' + 'string without NUL characters' not in err_msg # pypy2 + ): + raise + except ValueError as val_err: + err_msg = str(val_err) + if ( + 'unlink: embedded null ' + 'character in path' not in err_msg + and 'embedded null byte' not in err_msg + and 'argument must be a ' + 'string without NUL characters' not in err_msg # pypy3 + ): + raise + + sock = self.prepare_socket( + bind_addr=bind_addr, + family=socket.AF_UNIX, type=socket.SOCK_STREAM, proto=0, + nodelay=self.nodelay, ssl_adapter=self.ssl_adapter, + ) + + try: + """Linux way of pre-populating fs mode permissions.""" + # Allow everyone access the socket... + os.fchmod(sock.fileno(), fs_permissions) + FS_PERMS_SET = True + except OSError: + FS_PERMS_SET = False + + try: + sock = self.bind_socket(sock, bind_addr) + except socket.error: + sock.close() + raise + + bind_addr = self.resolve_real_bind_addr(sock) + + try: + """FreeBSD/macOS pre-populating fs mode permissions.""" + if not FS_PERMS_SET: + try: + os.lchmod(bind_addr, fs_permissions) + except AttributeError: + os.chmod(bind_addr, fs_permissions, follow_symlinks=False) + FS_PERMS_SET = True + except OSError: + pass + + if not FS_PERMS_SET: + self.error_log( + 'Failed to set socket fs mode permissions', + level=logging.WARNING, + ) + + self.bind_addr = bind_addr + self.socket = sock + return sock - if self.ssl_adapter is not None: - self.socket = self.ssl_adapter.bind(self.socket) + @staticmethod + def prepare_socket(bind_addr, family, type, proto, nodelay, ssl_adapter): + """Create and prepare the socket object.""" + sock = socket.socket(family, type, proto) + connections.prevent_socket_inheritance(sock) - host, port = self.bind_addr[:2] + host, port = bind_addr[:2] + IS_EPHEMERAL_PORT = port == 0 + + if not (IS_WINDOWS or IS_EPHEMERAL_PORT): + """Enable SO_REUSEADDR for the current socket. + + Skip for Windows (has different semantics) + or ephemeral ports (can steal ports from others). + + Refs: + * https://msdn.microsoft.com/en-us/library/ms740621(v=vs.85).aspx + * https://github.com/cherrypy/cheroot/issues/114 + * https://gavv.github.io/blog/ephemeral-port-reuse/ + """ + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + if nodelay and not isinstance( + bind_addr, + (six.text_type, six.binary_type), + ): + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + + if ssl_adapter is not None: + sock = ssl_adapter.bind(sock) # If listening on the IPV6 any address ('::' = IN6ADDR_ANY), # activate dual-stack. See # https://github.com/cherrypy/cherrypy/issues/871. - if hasattr(socket, 'AF_INET6') and family == socket.AF_INET6 and host in ('::', '::0', '::0.0.0.0'): + listening_ipv6 = ( + hasattr(socket, 'AF_INET6') + and family == socket.AF_INET6 + and host in ('::', '::0', '::0.0.0.0') + ) + if listening_ipv6: try: - self.socket.setsockopt( - socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0) + sock.setsockopt( + socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0, + ) except (AttributeError, socket.error): # Apparently, the socket option is not available in # this machine's TCP stack pass - self.socket.bind(self.bind_addr) - - def tick(self): - """Accept a new connection and put it on the Queue.""" - try: - s, addr = self.socket.accept() - if self.stats['Enabled']: - self.stats['Accepts'] += 1 - if not self.ready: - return + return sock + + @staticmethod + def bind_socket(socket_, bind_addr): + """Bind the socket to given interface.""" + socket_.bind(bind_addr) + return socket_ + + @staticmethod + def resolve_real_bind_addr(socket_): + """Retrieve actual bind address from bound socket.""" + # FIXME: keep requested bind_addr separate real bound_addr (port + # is different in case of ephemeral port 0) + bind_addr = socket_.getsockname() + if socket_.family in ( + # Windows doesn't have socket.AF_UNIX, so not using it in check + socket.AF_INET, + socket.AF_INET6, + ): + """UNIX domain sockets are strings or bytes. + + In case of bytes with a leading null-byte it's an abstract socket. + """ + return bind_addr[:2] - prevent_socket_inheritance(s) - if hasattr(s, 'settimeout'): - s.settimeout(self.timeout) + if isinstance(bind_addr, six.binary_type): + bind_addr = bton(bind_addr) - mf = MakeFile - ssl_env = {} - # if ssl cert and key are set, we try to be a secure HTTP server - if self.ssl_adapter is not None: - try: - s, ssl_env = self.ssl_adapter.wrap(s) - except errors.NoSSLError: - msg = ('The client sent a plain HTTP request, but ' - 'this server only speaks HTTPS on this port.') - buf = ['%s 400 Bad Request\r\n' % self.protocol, - 'Content-Length: %s\r\n' % len(msg), - 'Content-Type: text/plain\r\n\r\n', - msg] - - sock_to_make = s if six.PY3 else s._sock - wfile = mf(sock_to_make, 'wb', io.DEFAULT_BUFFER_SIZE) - try: - wfile.write(''.join(buf).encode('ISO-8859-1')) - except socket.error as ex: - if ex.args[0] not in errors.socket_errors_to_ignore: - raise - return - if not s: - return - mf = self.ssl_adapter.makefile - # Re-apply our timeout since we may have a new socket object - if hasattr(s, 'settimeout'): - s.settimeout(self.timeout) - - conn = self.ConnectionClass(self, s, mf) - - if not isinstance(self.bind_addr, six.string_types): - # optional values - # Until we do DNS lookups, omit REMOTE_HOST - if addr is None: # sometimes this can happen - # figure out if AF_INET or AF_INET6. - if len(s.getsockname()) == 2: - # AF_INET - addr = ('0.0.0.0', 0) - else: - # AF_INET6 - addr = ('::', 0) - conn.remote_addr = addr[0] - conn.remote_port = addr[1] - - conn.ssl_env = ssl_env + return bind_addr + def tick(self): + """Accept a new connection and put it on the Queue.""" + conn = self._connections.get_conn() + if conn: try: self.requests.put(conn) except queue.Full: # Just drop the conn. TODO: write 503 back? conn.close() - return - except socket.timeout: - # The only reason for the timeout in start() is so we can - # notice keyboard interrupts on Win32, which don't interrupt - # accept() by default - return - except socket.error as ex: - if self.stats['Enabled']: - self.stats['Socket Errors'] += 1 - if ex.args[0] in errors.socket_error_eintr: - # I *think* this is right. EINTR should occur when a signal - # is received during the accept() call; all docs say retry - # the call, and I *think* I'm reading it right that Python - # will then go ahead and poll for and handle the signal - # elsewhere. See - # https://github.com/cherrypy/cherrypy/issues/707. - return - if ex.args[0] in errors.socket_errors_nonblocking: - # Just try again. See - # https://github.com/cherrypy/cherrypy/issues/479. - return - if ex.args[0] in errors.socket_errors_to_ignore: - # Our socket was closed. - # See https://github.com/cherrypy/cherrypy/issues/686. - return - raise - def _get_interrupt(self): + self._connections.expire() + + @property + def interrupt(self): + """Flag interrupt of the server.""" return self._interrupt - def _set_interrupt(self, interrupt): + @interrupt.setter + def interrupt(self, interrupt): + """Perform the shutdown of this server and save the exception.""" self._interrupt = True self.stop() self._interrupt = interrupt - interrupt = property(_get_interrupt, _set_interrupt, - doc='Set this to an Exception instance to ' - 'interrupt the server.') + if self._interrupt: + raise self.interrupt - def stop(self): + def stop(self): # noqa: C901 # FIXME """Gracefully shutdown a server that is serving forever.""" self.ready = False if self._start_time is not None: self._run_time += (time.time() - self._start_time) self._start_time = None + # ensure serve is no longer accessing socket, connections + while self.serving: + time.sleep(0.1) + sock = getattr(self, 'socket', None) if sock: - if not isinstance(self.bind_addr, six.string_types): + if not isinstance( + self.bind_addr, + (six.text_type, six.binary_type), + ): # Touch our own socket to make accept() return immediately. try: host, port = sock.getsockname()[:2] @@ -1669,14 +2090,16 @@ def stop(self): # here, because we want an actual IP to touch. # localhost won't work if we've bound to a public IP, # but it will if we bound to '0.0.0.0' (INADDR_ANY). - for res in socket.getaddrinfo(host, port, socket.AF_UNSPEC, - socket.SOCK_STREAM): + for res in socket.getaddrinfo( + host, port, socket.AF_UNSPEC, + socket.SOCK_STREAM, + ): af, socktype, proto, canonname, sa = res s = None try: s = socket.socket(af, socktype, proto) # See - # http://groups.google.com/group/cherrypy-users/ + # https://groups.google.com/group/cherrypy-users/ # browse_frm/thread/bbfe5eb39c904fe0 s.settimeout(1.0) s.connect((host, port)) @@ -1688,11 +2111,12 @@ def stop(self): sock.close() self.socket = None + self._connections.close() self.requests.stop(self.shutdown_timeout) -class Gateway(object): - """A base class to interface HTTPServer with other systems, such as WSGI.""" +class Gateway: + """Base class to interface HTTPServer with other systems, such as WSGI.""" def __init__(self, req): """Initialize Gateway instance with request. @@ -1704,7 +2128,7 @@ def __init__(self, req): def respond(self): """Process the current request. Must be overridden in a subclass.""" - raise NotImplemented + raise NotImplementedError # pragma: no cover # These may either be ssl.Adapter subclasses or the string names @@ -1735,7 +2159,9 @@ def get_ssl_adapter_class(name='builtin'): try: adapter = getattr(mod, attr_name) except AttributeError: - raise AttributeError("'%s' object has no attribute '%s'" - % (mod_path, attr_name)) + raise AttributeError( + "'%s' object has no attribute '%s'" + % (mod_path, attr_name), + ) return adapter diff --git a/lib/cheroot/ssl/__init__.py b/lib/cheroot/ssl/__init__.py index fa0e177..d45fd7f 100644 --- a/lib/cheroot/ssl/__init__.py +++ b/lib/cheroot/ssl/__init__.py @@ -1,12 +1,15 @@ """Implementation of the SSL adapter base interface.""" +from __future__ import absolute_import, division, print_function +__metaclass__ = type + from abc import ABCMeta, abstractmethod from six import add_metaclass @add_metaclass(ABCMeta) -class Adapter(object): +class Adapter: """Base class for SSL driver library adapters. Required methods: @@ -17,7 +20,10 @@ class Adapter(object): """ @abstractmethod - def __init__(self, certificate, private_key, certificate_chain=None, ciphers=None): + def __init__( + self, certificate, private_key, certificate_chain=None, + ciphers=None, + ): """Set up certificates, private key ciphers and reset context.""" self.certificate = certificate self.private_key = private_key @@ -33,14 +39,14 @@ def bind(self, sock): @abstractmethod def wrap(self, sock): """Wrap and return the given socket, plus WSGI environ entries.""" - raise NotImplementedError + raise NotImplementedError # pragma: no cover @abstractmethod def get_environ(self): """Return WSGI environ entries to be merged into each request.""" - raise NotImplementedError + raise NotImplementedError # pragma: no cover @abstractmethod def makefile(self, sock, mode='r', bufsize=-1): """Return socket file object.""" - raise NotImplementedError + raise NotImplementedError # pragma: no cover diff --git a/lib/cheroot/ssl/builtin.py b/lib/cheroot/ssl/builtin.py index b9e8904..14ef17d 100644 --- a/lib/cheroot/ssl/builtin.py +++ b/lib/cheroot/ssl/builtin.py @@ -1,12 +1,19 @@ """ -A library for integrating Python's builtin ``ssl`` library with Cheroot. +A library for integrating Python's builtin :py:mod:`ssl` library with Cheroot. -The ssl module must be importable for SSL functionality. +The :py:mod:`ssl` module must be importable for SSL functionality. To use this module, set ``HTTPServer.ssl_adapter`` to an instance of ``BuiltinSSLAdapter``. """ +from __future__ import absolute_import, division, print_function +__metaclass__ = type + +import socket +import sys +import threading + try: import ssl except ImportError: @@ -20,46 +27,244 @@ except ImportError: DEFAULT_BUFFER_SIZE = -1 +import six + from . import Adapter from .. import errors -from ..makefile import MakeFile +from .._compat import IS_ABOVE_OPENSSL10, suppress +from ..makefile import StreamReader, StreamWriter +from ..server import HTTPServer + +if six.PY2: + generic_socket_error = socket.error +else: + generic_socket_error = OSError + + +def _assert_ssl_exc_contains(exc, *msgs): + """Check whether SSL exception contains either of messages provided.""" + if len(msgs) < 1: + raise TypeError( + '_assert_ssl_exc_contains() requires ' + 'at least one message to be passed.', + ) + err_msg_lower = str(exc).lower() + return any(m.lower() in err_msg_lower for m in msgs) + + +def _loopback_for_cert_thread(context, server): + """Wrap a socket in ssl and perform the server-side handshake.""" + # As we only care about parsing the certificate, the failure of + # which will cause an exception in ``_loopback_for_cert``, + # we can safely ignore connection and ssl related exceptions. Ref: + # https://github.com/cherrypy/cheroot/issues/302#issuecomment-662592030 + with suppress(ssl.SSLError, OSError): + with context.wrap_socket( + server, do_handshake_on_connect=True, server_side=True, + ) as ssl_sock: + # in TLS 1.3 (Python 3.7+, OpenSSL 1.1.1+), the server + # sends the client session tickets that can be used to + # resume the TLS session on a new connection without + # performing the full handshake again. session tickets are + # sent as a post-handshake message at some _unspecified_ + # time and thus a successful connection may be closed + # without the client having received the tickets. + # Unfortunately, on Windows (Python 3.8+), this is treated + # as an incomplete handshake on the server side and a + # ``ConnectionAbortedError`` is raised. + # TLS 1.3 support is still incomplete in Python 3.8; + # there is no way for the client to wait for tickets. + # While not necessary for retrieving the parsed certificate, + # we send a tiny bit of data over the connection in an + # attempt to give the server a chance to send the session + # tickets and close the connection cleanly. + # Note that, as this is essentially a race condition, + # the error may still occur ocasionally. + ssl_sock.send(b'0000') + + +def _loopback_for_cert(certificate, private_key, certificate_chain): + """Create a loopback connection to parse a cert with a private key.""" + context = ssl.create_default_context(cafile=certificate_chain) + context.load_cert_chain(certificate, private_key) + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + + # Python 3+ Unix, Python 3.5+ Windows + client, server = socket.socketpair() + try: + # `wrap_socket` will block until the ssl handshake is complete. + # it must be called on both ends at the same time -> thread + # openssl will cache the peer's cert during a successful handshake + # and return it via `getpeercert` even after the socket is closed. + # when `close` is called, the SSL shutdown notice will be sent + # and then python will wait to receive the corollary shutdown. + thread = threading.Thread( + target=_loopback_for_cert_thread, args=(context, server), + ) + try: + thread.start() + with context.wrap_socket( + client, do_handshake_on_connect=True, + server_side=False, + ) as ssl_sock: + ssl_sock.recv(4) + return ssl_sock.getpeercert() + finally: + thread.join() + finally: + client.close() + server.close() + + +def _parse_cert(certificate, private_key, certificate_chain): + """Parse a certificate.""" + # loopback_for_cert uses socket.socketpair which was only + # introduced in Python 3.0 for *nix and 3.5 for Windows + # and requires OS support (AttributeError, OSError) + # it also requires a private key either in its own file + # or combined with the cert (SSLError) + with suppress(AttributeError, ssl.SSLError, OSError): + return _loopback_for_cert(certificate, private_key, certificate_chain) + + # KLUDGE: using an undocumented, private, test method to parse a cert + # unfortunately, it is the only built-in way without a connection + # as a private, undocumented method, it may change at any time + # so be tolerant of *any* possible errors it may raise + with suppress(Exception): + return ssl._ssl._test_decode_cert(certificate) + + return {} + + +def _sni_callback(sock, sni, context): + """Handle the SNI callback to tag the socket with the SNI.""" + sock.sni = sni + # return None to allow the TLS negotiation to continue class BuiltinSSLAdapter(Adapter): - """A wrapper for integrating Python's builtin ssl module with Cheroot.""" + """Wrapper for integrating Python's builtin :py:mod:`ssl` with Cheroot.""" certificate = None - """The filename of the server SSL certificate.""" + """The file name of the server SSL certificate.""" private_key = None - """The filename of the server's private key file.""" + """The file name of the server's private key file.""" certificate_chain = None - """The filename of the certificate chain file.""" - - context = None - """The ssl.SSLContext that will be used to wrap sockets where available - (on Python > 2.7.9 / 3.3) - """ + """The file name of the certificate chain file.""" ciphers = None """The ciphers list of SSL.""" - def __init__(self, certificate, private_key, certificate_chain=None, ciphers=None): + # from mod_ssl/pkg.sslmod/ssl_engine_vars.c ssl_var_lookup_ssl_cert + CERT_KEY_TO_ENV = { + 'version': 'M_VERSION', + 'serialNumber': 'M_SERIAL', + 'notBefore': 'V_START', + 'notAfter': 'V_END', + 'subject': 'S_DN', + 'issuer': 'I_DN', + 'subjectAltName': 'SAN', + # not parsed by the Python standard library + # - A_SIG + # - A_KEY + # not provided by mod_ssl + # - OCSP + # - caIssuers + # - crlDistributionPoints + } + + # from mod_ssl/pkg.sslmod/ssl_engine_vars.c ssl_var_lookup_ssl_cert_dn_rec + CERT_KEY_TO_LDAP_CODE = { + 'countryName': 'C', + 'stateOrProvinceName': 'ST', + # NOTE: mod_ssl also provides 'stateOrProvinceName' as 'SP' + # for compatibility with SSLeay + 'localityName': 'L', + 'organizationName': 'O', + 'organizationalUnitName': 'OU', + 'commonName': 'CN', + 'title': 'T', + 'initials': 'I', + 'givenName': 'G', + 'surname': 'S', + 'description': 'D', + 'userid': 'UID', + 'emailAddress': 'Email', + # not provided by mod_ssl + # - dnQualifier: DNQ + # - domainComponent: DC + # - postalCode: PC + # - streetAddress: STREET + # - serialNumber + # - generationQualifier + # - pseudonym + # - jurisdictionCountryName + # - jurisdictionLocalityName + # - jurisdictionStateOrProvince + # - businessCategory + } + + def __init__( + self, certificate, private_key, certificate_chain=None, + ciphers=None, + ): """Set up context in addition to base class properties if available.""" if ssl is None: raise ImportError('You must install the ssl module to use HTTPS.') - super(BuiltinSSLAdapter, self).__init__(certificate, private_key, certificate_chain, ciphers) + super(BuiltinSSLAdapter, self).__init__( + certificate, private_key, certificate_chain, ciphers, + ) - if hasattr(ssl, 'create_default_context'): - self.context = ssl.create_default_context( - purpose=ssl.Purpose.CLIENT_AUTH, - cafile=certificate_chain - ) - self.context.load_cert_chain(certificate, private_key) - if self.ciphers is not None: - self.context.set_ciphers(ciphers) + self.context = ssl.create_default_context( + purpose=ssl.Purpose.CLIENT_AUTH, + cafile=certificate_chain, + ) + self.context.load_cert_chain(certificate, private_key) + if self.ciphers is not None: + self.context.set_ciphers(ciphers) + + self._server_env = self._make_env_cert_dict( + 'SSL_SERVER', + _parse_cert(certificate, private_key, self.certificate_chain), + ) + if not self._server_env: + return + cert = None + with open(certificate, mode='rt') as f: + cert = f.read() + + # strip off any keys by only taking the first certificate + cert_start = cert.find(ssl.PEM_HEADER) + if cert_start == -1: + return + cert_end = cert.find(ssl.PEM_FOOTER, cert_start) + if cert_end == -1: + return + cert_end += len(ssl.PEM_FOOTER) + self._server_env['SSL_SERVER_CERT'] = cert[cert_start:cert_end] + + @property + def context(self): + """:py:class:`~ssl.SSLContext` that will be used to wrap sockets.""" + return self._context + + @context.setter + def context(self, context): + """Set the ssl ``context`` to use.""" + self._context = context + # Python 3.7+ + # if a context is provided via `cherrypy.config.update` then + # `self.context` will be set after `__init__` + # use a property to intercept it to add an SNI callback + # but don't override the user's callback + # TODO: chain callbacks + with suppress(AttributeError): + if ssl.HAS_SNI and context.sni_callback is None: + context.sni_callback = _sni_callback def bind(self, sock): """Wrap and return the given socket.""" @@ -67,45 +272,62 @@ def bind(self, sock): def wrap(self, sock): """Wrap and return the given socket, plus WSGI environ entries.""" + EMPTY_RESULT = None, {} try: - if self.context is not None: - s = self.context.wrap_socket(sock, do_handshake_on_connect=True, - server_side=True) - else: - s = ssl.wrap_socket(sock, do_handshake_on_connect=True, - server_side=True, certfile=self.certificate, - keyfile=self.private_key, - ssl_version=ssl.PROTOCOL_SSLv23, - ca_certs=self.certificate_chain) + s = self.context.wrap_socket( + sock, do_handshake_on_connect=True, server_side=True, + ) except ssl.SSLError as ex: if ex.errno == ssl.SSL_ERROR_EOF: # This is almost certainly due to the cherrypy engine # 'pinging' the socket to assert it's connectable; # the 'ping' isn't SSL. - return None, {} + return EMPTY_RESULT elif ex.errno == ssl.SSL_ERROR_SSL: - if 'http request' in ex.args[1]: + if _assert_ssl_exc_contains(ex, 'http request'): # The client is speaking HTTP to an HTTPS server. raise errors.NoSSLError # Check if it's one of the known errors - # Errors that are caught by PyOpenSSL, but thrown by built-in ssl - _block_errors = ('unknown protocol', 'unknown ca', 'unknown_ca', 'unknown error', - 'https proxy request', 'inappropriate fallback', 'wrong version number', - 'no shared cipher', 'certificate unknown', 'ccs received early') - for error_text in _block_errors: - if error_text in ex.args[1].lower(): - # Accepted error, let's pass - return None, {} - elif 'handshake operation timed out' in ex.args[0]: + # Errors that are caught by PyOpenSSL, but thrown by + # built-in ssl + _block_errors = ( + 'unknown protocol', 'unknown ca', 'unknown_ca', + 'unknown error', + 'https proxy request', 'inappropriate fallback', + 'wrong version number', + 'no shared cipher', 'certificate unknown', + 'ccs received early', + 'certificate verify failed', # client cert w/o trusted CA + ) + if _assert_ssl_exc_contains(ex, *_block_errors): + # Accepted error, let's pass + return EMPTY_RESULT + elif _assert_ssl_exc_contains(ex, 'handshake operation timed out'): # This error is thrown by builtin SSL after a timeout # when client is speaking HTTP to an HTTPS server. # The connection can safely be dropped. - return None, {} + return EMPTY_RESULT + raise + except generic_socket_error as exc: + """It is unclear why exactly this happens. + + It's reproducible only with openssl>1.0 and stdlib + :py:mod:`ssl` wrapper. + In CherryPy it's triggered by Checker plugin, which connects + to the app listening to the socket port in TLS mode via plain + HTTP during startup (from the same process). + + + Ref: https://github.com/cherrypy/cherrypy/issues/1618 + """ + is_error0 = exc.args == (0, 'Error') + + if is_error0 and IS_ABOVE_OPENSSL10: + return EMPTY_RESULT raise return s, self.get_environ(s) - # TODO: fill this out more with mod ssl env def get_environ(self, sock): """Create WSGI environ entries to be merged into each request.""" cipher = sock.cipher() @@ -113,12 +335,149 @@ def get_environ(self, sock): 'wsgi.url_scheme': 'https', 'HTTPS': 'on', 'SSL_PROTOCOL': cipher[1], - 'SSL_CIPHER': cipher[0] - # SSL_VERSION_INTERFACE string The mod_ssl program version - # SSL_VERSION_LIBRARY string The OpenSSL program version + 'SSL_CIPHER': cipher[0], + 'SSL_CIPHER_EXPORT': '', + 'SSL_CIPHER_USEKEYSIZE': cipher[2], + 'SSL_VERSION_INTERFACE': '%s Python/%s' % ( + HTTPServer.version, sys.version, + ), + 'SSL_VERSION_LIBRARY': ssl.OPENSSL_VERSION, + 'SSL_CLIENT_VERIFY': 'NONE', + # 'NONE' - client did not provide a cert (overriden below) } + + # Python 3.3+ + with suppress(AttributeError): + compression = sock.compression() + if compression is not None: + ssl_environ['SSL_COMPRESS_METHOD'] = compression + + # Python 3.6+ + with suppress(AttributeError): + ssl_environ['SSL_SESSION_ID'] = sock.session.id.hex() + with suppress(AttributeError): + target_cipher = cipher[:2] + for cip in sock.context.get_ciphers(): + if target_cipher == (cip['name'], cip['protocol']): + ssl_environ['SSL_CIPHER_ALGKEYSIZE'] = cip['alg_bits'] + break + + # Python 3.7+ sni_callback + with suppress(AttributeError): + ssl_environ['SSL_TLS_SNI'] = sock.sni + + if self.context and self.context.verify_mode != ssl.CERT_NONE: + client_cert = sock.getpeercert() + if client_cert: + # builtin ssl **ALWAYS** validates client certificates + # and terminates the connection on failure + ssl_environ['SSL_CLIENT_VERIFY'] = 'SUCCESS' + ssl_environ.update( + self._make_env_cert_dict('SSL_CLIENT', client_cert), + ) + ssl_environ['SSL_CLIENT_CERT'] = ssl.DER_cert_to_PEM_cert( + sock.getpeercert(binary_form=True), + ).strip() + + ssl_environ.update(self._server_env) + + # not supplied by the Python standard library (as of 3.8) + # - SSL_SESSION_RESUMED + # - SSL_SECURE_RENEG + # - SSL_CLIENT_CERT_CHAIN_n + # - SRP_USER + # - SRP_USERINFO + return ssl_environ + def _make_env_cert_dict(self, env_prefix, parsed_cert): + """Return a dict of WSGI environment variables for a certificate. + + E.g. SSL_CLIENT_M_VERSION, SSL_CLIENT_M_SERIAL, etc. + See https://httpd.apache.org/docs/2.4/mod/mod_ssl.html#envvars. + """ + if not parsed_cert: + return {} + + env = {} + for cert_key, env_var in self.CERT_KEY_TO_ENV.items(): + key = '%s_%s' % (env_prefix, env_var) + value = parsed_cert.get(cert_key) + if env_var == 'SAN': + env.update(self._make_env_san_dict(key, value)) + elif env_var.endswith('_DN'): + env.update(self._make_env_dn_dict(key, value)) + else: + env[key] = str(value) + + # mod_ssl 2.1+; Python 3.2+ + # number of days until the certificate expires + if 'notBefore' in parsed_cert: + remain = ssl.cert_time_to_seconds(parsed_cert['notAfter']) + remain -= ssl.cert_time_to_seconds(parsed_cert['notBefore']) + remain /= 60 * 60 * 24 + env['%s_V_REMAIN' % (env_prefix,)] = str(int(remain)) + + return env + + def _make_env_san_dict(self, env_prefix, cert_value): + """Return a dict of WSGI environment variables for a certificate DN. + + E.g. SSL_CLIENT_SAN_Email_0, SSL_CLIENT_SAN_DNS_0, etc. + See SSL_CLIENT_SAN_* at + https://httpd.apache.org/docs/2.4/mod/mod_ssl.html#envvars. + """ + if not cert_value: + return {} + + env = {} + dns_count = 0 + email_count = 0 + for attr_name, val in cert_value: + if attr_name == 'DNS': + env['%s_DNS_%i' % (env_prefix, dns_count)] = val + dns_count += 1 + elif attr_name == 'Email': + env['%s_Email_%i' % (env_prefix, email_count)] = val + email_count += 1 + + # other mod_ssl SAN vars: + # - SAN_OTHER_msUPN_n + return env + + def _make_env_dn_dict(self, env_prefix, cert_value): + """Return a dict of WSGI environment variables for a certificate DN. + + E.g. SSL_CLIENT_S_DN_CN, SSL_CLIENT_S_DN_C, etc. + See SSL_CLIENT_S_DN_x509 at + https://httpd.apache.org/docs/2.4/mod/mod_ssl.html#envvars. + """ + if not cert_value: + return {} + + dn = [] + dn_attrs = {} + for rdn in cert_value: + for attr_name, val in rdn: + attr_code = self.CERT_KEY_TO_LDAP_CODE.get(attr_name) + dn.append('%s=%s' % (attr_code or attr_name, val)) + if not attr_code: + continue + dn_attrs.setdefault(attr_code, []) + dn_attrs[attr_code].append(val) + + env = { + env_prefix: ','.join(dn), + } + for attr_code, values in dn_attrs.items(): + env['%s_%s' % (env_prefix, attr_code)] = ','.join(values) + if len(values) == 1: + continue + for i, val in enumerate(values): + env['%s_%s_%i' % (env_prefix, attr_code, i)] = val + return env + def makefile(self, sock, mode='r', bufsize=DEFAULT_BUFFER_SIZE): """Return socket file object.""" - return MakeFile(sock, mode, bufsize) + cls = StreamReader if 'r' in mode else StreamWriter + return cls(sock, mode, bufsize) diff --git a/lib/cheroot/ssl/pyopenssl.py b/lib/cheroot/ssl/pyopenssl.py index 2f98864..14d238e 100644 --- a/lib/cheroot/ssl/pyopenssl.py +++ b/lib/cheroot/ssl/pyopenssl.py @@ -1,59 +1,87 @@ """ -A library for integrating pyOpenSSL with Cheroot. +A library for integrating :doc:`pyOpenSSL ` with Cheroot. -The OpenSSL module must be importable for SSL functionality. -You can obtain it from `here `_. +The :py:mod:`OpenSSL ` module must be importable +for SSL/TLS/HTTPS functionality. +You can obtain it from `here `_. -To use this module, set HTTPServer.ssl_adapter to an instance of -ssl.Adapter. There are two ways to use SSL: +To use this module, set :py:attr:`HTTPServer.ssl_adapter +` to an instance of +:py:class:`ssl.Adapter `. +There are two ways to use :abbr:`TLS (Transport-Level Security)`: Method One ---------- - * ``ssl_adapter.context``: an instance of SSL.Context. - -If this is not None, it is assumed to be an SSL.Context instance, -and will be passed to SSL.Connection on bind(). The developer is -responsible for forming a valid Context object. This approach is -to be preferred for more flexibility, e.g. if the cert and key are -streams instead of files, or need decryption, or SSL.SSLv3_METHOD -is desired instead of the default SSL.SSLv23_METHOD, etc. Consult -the pyOpenSSL documentation for complete options. + * :py:attr:`ssl_adapter.context + `: an instance of + :py:class:`SSL.Context `. + +If this is not None, it is assumed to be an :py:class:`SSL.Context +` instance, and will be passed to +:py:class:`SSL.Connection ` on bind(). +The developer is responsible for forming a valid :py:class:`Context +` object. This +approach is to be preferred for more flexibility, e.g. if the cert and +key are streams instead of files, or need decryption, or +:py:data:`SSL.SSLv3_METHOD ` +is desired instead of the default :py:data:`SSL.SSLv23_METHOD +`, etc. Consult +the :doc:`pyOpenSSL ` documentation for +complete options. Method Two (shortcut) --------------------- - * ``ssl_adapter.certificate``: the filename of the server SSL certificate. - * ``ssl_adapter.private_key``: the filename of the server's private key file. - -Both are None by default. If ssl_adapter.context is None, but .private_key -and .certificate are both given and valid, they will be read, and the -context will be automatically created from them. + * :py:attr:`ssl_adapter.certificate + `: the file name + of the server's TLS certificate. + * :py:attr:`ssl_adapter.private_key + `: the file name + of the server's private key file. + +Both are :py:data:`None` by default. If :py:attr:`ssl_adapter.context +` is :py:data:`None`, +but ``.private_key`` and ``.certificate`` are both given and valid, they +will be read, and the context will be automatically created from them. """ +from __future__ import absolute_import, division, print_function +__metaclass__ = type + import socket +import sys import threading import time +import six + try: + import OpenSSL.version from OpenSSL import SSL from OpenSSL import crypto + + try: + ssl_conn_type = SSL.Connection + except AttributeError: + ssl_conn_type = SSL.ConnectionType except ImportError: SSL = None from . import Adapter from .. import errors, server as cheroot_server -from ..makefile import MakeFile +from ..makefile import StreamReader, StreamWriter -class SSL_fileobject(MakeFile): - """SSL file object attached to a socket object.""" +class SSLFileobjectMixin: + """Base mixin for a TLS socket stream.""" ssl_timeout = 3 ssl_retry = .01 - def _safe_call(self, is_reader, call, *args, **kwargs): - """Wrap the given call with SSL error-trapping. + # FIXME: + def _safe_call(self, is_reader, call, *args, **kwargs): # noqa: C901 + """Wrap the given call with TLS error-trapping. is_reader: if False EOF errors will be raised. If True, EOF errors will return "" (to emulate normal sockets). @@ -67,20 +95,21 @@ def _safe_call(self, is_reader, call, *args, **kwargs): # the rest of the stack has no way of differentiating # between a "new handshake" error and "client dropped". # Note this isn't an endless loop: there's a timeout below. + # Ref: https://stackoverflow.com/a/5133568/595220 time.sleep(self.ssl_retry) except SSL.WantWriteError: time.sleep(self.ssl_retry) except SSL.SysCallError as e: if is_reader and e.args == (-1, 'Unexpected EOF'): - return '' + return b'' errnum = e.args[0] if is_reader and errnum in errors.socket_errors_to_ignore: - return '' + return b'' raise socket.error(errnum) except SSL.Error as e: if is_reader and e.args == (-1, 'Unexpected EOF'): - return '' + return b'' thirdarg = None try: @@ -93,31 +122,116 @@ def _safe_call(self, is_reader, call, *args, **kwargs): raise errors.NoSSLError() raise errors.FatalSSLAlert(*e.args) - except: - raise if time.time() - start > self.ssl_timeout: raise socket.timeout('timed out') def recv(self, size): """Receive message of a size from the socket.""" - return self._safe_call(True, super(SSL_fileobject, self).recv, size) + return self._safe_call( + True, + super(SSLFileobjectMixin, self).recv, + size, + ) + + def readline(self, size=-1): + """Receive message of a size from the socket. + + Matches the following interface: + https://docs.python.org/3/library/io.html#io.IOBase.readline + """ + return self._safe_call( + True, + super(SSLFileobjectMixin, self).readline, + size, + ) def sendall(self, *args, **kwargs): """Send whole message to the socket.""" - return self._safe_call(False, super(SSL_fileobject, self).sendall, - *args, **kwargs) + return self._safe_call( + False, + super(SSLFileobjectMixin, self).sendall, + *args, **kwargs + ) def send(self, *args, **kwargs): """Send some part of message to the socket.""" - return self._safe_call(False, super(SSL_fileobject, self).send, - *args, **kwargs) + return self._safe_call( + False, + super(SSLFileobjectMixin, self).send, + *args, **kwargs + ) + + +class SSLFileobjectStreamReader(SSLFileobjectMixin, StreamReader): + """SSL file object attached to a socket object.""" +class SSLFileobjectStreamWriter(SSLFileobjectMixin, StreamWriter): + """SSL file object attached to a socket object.""" + + +class SSLConnectionProxyMeta: + """Metaclass for generating a bunch of proxy methods.""" + + def __new__(mcl, name, bases, nmspc): + """Attach a list of proxy methods to a new class.""" + proxy_methods = ( + 'get_context', 'pending', 'send', 'write', 'recv', 'read', + 'renegotiate', 'bind', 'listen', 'connect', 'accept', + 'setblocking', 'fileno', 'close', 'get_cipher_list', + 'getpeername', 'getsockname', 'getsockopt', 'setsockopt', + 'makefile', 'get_app_data', 'set_app_data', 'state_string', + 'sock_shutdown', 'get_peer_certificate', 'want_read', + 'want_write', 'set_connect_state', 'set_accept_state', + 'connect_ex', 'sendall', 'settimeout', 'gettimeout', + 'shutdown', + ) + proxy_methods_no_args = ( + 'shutdown', + ) + + proxy_props = ( + 'family', + ) + + def lock_decorator(method): + """Create a proxy method for a new class.""" + def proxy_wrapper(self, *args): + self._lock.acquire() + try: + new_args = ( + args[:] if method not in proxy_methods_no_args else [] + ) + return getattr(self._ssl_conn, method)(*new_args) + finally: + self._lock.release() + return proxy_wrapper + for m in proxy_methods: + nmspc[m] = lock_decorator(m) + nmspc[m].__name__ = m + + def make_property(property_): + """Create a proxy method for a new class.""" + def proxy_prop_wrapper(self): + return getattr(self._ssl_conn, property_) + proxy_prop_wrapper.__name__ = property_ + return property(proxy_prop_wrapper) + for p in proxy_props: + nmspc[p] = make_property(p) + + # Doesn't work via super() for some reason. + # Falling back to type() instead: + return type(name, bases, nmspc) + + +@six.add_metaclass(SSLConnectionProxyMeta) class SSLConnection: - """A thread-safe wrapper for an SSL.Connection. + r"""A thread-safe wrapper for an ``SSL.Connection``. - ``*args``: the arguments to create the wrapped ``SSL.Connection(*args)``. + :param tuple args: the arguments to create the wrapped \ + :py:class:`SSL.Connection(*args) \ + ` """ def __init__(self, *args): @@ -125,61 +239,41 @@ def __init__(self, *args): self._ssl_conn = SSL.Connection(*args) self._lock = threading.RLock() - for f in ('get_context', 'pending', 'send', 'write', 'recv', 'read', - 'renegotiate', 'bind', 'listen', 'connect', 'accept', - 'setblocking', 'fileno', 'close', 'get_cipher_list', - 'getpeername', 'getsockname', 'getsockopt', 'setsockopt', - 'makefile', 'get_app_data', 'set_app_data', 'state_string', - 'sock_shutdown', 'get_peer_certificate', 'want_read', - 'want_write', 'set_connect_state', 'set_accept_state', - 'connect_ex', 'sendall', 'settimeout', 'gettimeout'): - exec("""def %s(self, *args): - self._lock.acquire() - try: - return self._ssl_conn.%s(*args) - finally: - self._lock.release() -""" % (f, f)) - - def shutdown(self, *args): - """Shutdown the SSL connection. - - Ignore all incoming args since pyOpenSSL.socket.shutdown takes no args. - """ - self._lock.acquire() - try: - return self._ssl_conn.shutdown() - finally: - self._lock.release() - class pyOpenSSLAdapter(Adapter): """A wrapper for integrating pyOpenSSL with Cheroot.""" certificate = None - """The filename of the server SSL certificate.""" + """The file name of the server's TLS certificate.""" private_key = None - """The filename of the server's private key file.""" + """The file name of the server's private key file.""" certificate_chain = None - """Optional. The filename of CA's intermediate certificate bundle. + """Optional. The file name of CA's intermediate certificate bundle. - This is needed for cheaper "chained root" SSL certificates, and should be - left as None if not required.""" + This is needed for cheaper "chained root" TLS certificates, + and should be left as :py:data:`None` if not required.""" context = None - """An instance of SSL.Context.""" + """ + An instance of :py:class:`SSL.Context `. + """ ciphers = None - """The ciphers list of SSL.""" + """The ciphers list of TLS.""" - def __init__(self, certificate, private_key, certificate_chain=None, ciphers=None): + def __init__( + self, certificate, private_key, certificate_chain=None, + ciphers=None, + ): """Initialize OpenSSL Adapter instance.""" if SSL is None: raise ImportError('You must install pyOpenSSL to use HTTPS.') - super(pyOpenSSLAdapter, self).__init__(certificate, private_key, certificate_chain, ciphers) + super(pyOpenSSLAdapter, self).__init__( + certificate, private_key, certificate_chain, ciphers, + ) self._environ = None @@ -193,11 +287,17 @@ def bind(self, sock): def wrap(self, sock): """Wrap and return the given socket, plus WSGI environ entries.""" + # pyOpenSSL doesn't perform the handshake until the first read/write + # forcing the handshake to complete tends to result in the connection + # closing so we can't reliably access protocol/client cert for the env return sock, self._environ.copy() def get_context(self): - """Return an SSL.Context from self attributes.""" - # See http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/442473 + """Return an ``SSL.Context`` from self attributes. + + Ref: :py:class:`SSL.Context ` + """ + # See https://code.activestate.com/recipes/442473/ c = SSL.Context(SSL.SSLv23_METHOD) c.use_privatekey_file(self.private_key) if self.certificate_chain: @@ -208,18 +308,25 @@ def get_context(self): def get_environ(self): """Return WSGI environ entries to be merged into each request.""" ssl_environ = { + 'wsgi.url_scheme': 'https', 'HTTPS': 'on', - # pyOpenSSL doesn't provide access to any of these AFAICT - # 'SSL_PROTOCOL': 'SSLv2', - # SSL_CIPHER string The cipher specification name - # SSL_VERSION_INTERFACE string The mod_ssl program version - # SSL_VERSION_LIBRARY string The OpenSSL program version + 'SSL_VERSION_INTERFACE': '%s %s/%s Python/%s' % ( + cheroot_server.HTTPServer.version, + OpenSSL.version.__title__, OpenSSL.version.__version__, + sys.version, + ), + 'SSL_VERSION_LIBRARY': SSL.SSLeay_version( + SSL.SSLEAY_VERSION, + ).decode(), } if self.certificate: # Server certificate attributes - cert = open(self.certificate, 'rb').read() - cert = crypto.load_certificate(crypto.FILETYPE_PEM, cert) + with open(self.certificate, 'rb') as cert_file: + cert = crypto.load_certificate( + crypto.FILETYPE_PEM, cert_file.read(), + ) + ssl_environ.update({ 'SSL_SERVER_M_VERSION': cert.get_version(), 'SSL_SERVER_M_SERIAL': cert.get_serial_number(), @@ -229,8 +336,10 @@ def get_environ(self): # Validity of server's certificate (end time), }) - for prefix, dn in [('I', cert.get_issuer()), - ('S', cert.get_subject())]: + for prefix, dn in [ + ('I', cert.get_issuer()), + ('S', cert.get_subject()), + ]: # X509Name objects don't seem to have a way to get the # complete DN string. Use str() and slice it instead, # because str(dn) == "" @@ -254,10 +363,16 @@ def get_environ(self): def makefile(self, sock, mode='r', bufsize=-1): """Return socket file object.""" - if SSL and isinstance(sock, SSL.ConnectionType): - timeout = sock.gettimeout() - f = SSL_fileobject(sock, mode, bufsize) - f.ssl_timeout = timeout - return f + cls = ( + SSLFileobjectStreamReader + if 'r' in mode else + SSLFileobjectStreamWriter + ) + if SSL and isinstance(sock, ssl_conn_type): + wrapped_socket = cls(sock, mode, bufsize) + wrapped_socket.ssl_timeout = sock.gettimeout() + return wrapped_socket + # This is from past: + # TODO: figure out what it's meant for else: return cheroot_server.CP_fileobject(sock, mode, bufsize) diff --git a/lib/cheroot/test/conftest.py b/lib/cheroot/test/conftest.py new file mode 100644 index 0000000..0004f0f --- /dev/null +++ b/lib/cheroot/test/conftest.py @@ -0,0 +1,69 @@ +"""Pytest configuration module. + +Contains fixtures, which are tightly bound to the Cheroot framework +itself, useless for end-users' app testing. +""" + +from __future__ import absolute_import, division, print_function +__metaclass__ = type + +import threading +import time + +import pytest + +from ..server import Gateway, HTTPServer +from ..testing import ( # noqa: F401 + native_server, wsgi_server, +) +from ..testing import get_server_client + + +@pytest.fixture +def wsgi_server_client(wsgi_server): # noqa: F811 + """Create a test client out of given WSGI server.""" + return get_server_client(wsgi_server) + + +@pytest.fixture +def native_server_client(native_server): # noqa: F811 + """Create a test client out of given HTTP server.""" + return get_server_client(native_server) + + +@pytest.fixture +def http_server(): + """Provision a server creator as a fixture.""" + def start_srv(): + bind_addr = yield + if bind_addr is None: + return + httpserver = make_http_server(bind_addr) + yield httpserver + yield httpserver + + srv_creator = iter(start_srv()) + next(srv_creator) + yield srv_creator + try: + while True: + httpserver = next(srv_creator) + if httpserver is not None: + httpserver.stop() + except StopIteration: + pass + + +def make_http_server(bind_addr): + """Create and start an HTTP server bound to ``bind_addr``.""" + httpserver = HTTPServer( + bind_addr=bind_addr, + gateway=Gateway, + ) + + threading.Thread(target=httpserver.safe_start).start() + + while not httpserver.ready: + time.sleep(0.1) + + return httpserver diff --git a/lib/cheroot/test/helper.py b/lib/cheroot/test/helper.py index 5048fa6..bdf2975 100644 --- a/lib/cheroot/test/helper.py +++ b/lib/cheroot/test/helper.py @@ -1,28 +1,27 @@ """A library of helper functions for the Cheroot test suite.""" +from __future__ import absolute_import, division, print_function +__metaclass__ = type + import datetime -import io import logging import os -import subprocess import sys import time import threading import types -import portend -import pytest +from six.moves import http_client + import six import cheroot.server import cheroot.wsgi -from cheroot._compat import HTTPSConnection, ntob from cheroot.test import webtest log = logging.getLogger(__name__) thisdir = os.path.abspath(os.path.dirname(__file__)) -serverpem = os.path.join(os.getcwd(), thisdir, 'test.pem') config = { @@ -33,6 +32,7 @@ class CherootWebCase(webtest.WebCase): + """Helper class for a web app test suite.""" script_name = '' scheme = 'http' @@ -60,7 +60,7 @@ def setup_class(cls): cls.scheme = 'http' else: ssl = ' (ssl)' - cls.HTTP_CONN = HTTPSConnection + cls.HTTP_CONN = http_client.HTTPSConnection cls.scheme = 'https' v = sys.version.split()[0] @@ -77,7 +77,7 @@ def setup_class(cls): @classmethod def teardown_class(cls): - '' + """Cleanup HTTP server.""" if hasattr(cls, 'setup_server'): cls.stop() @@ -90,31 +90,16 @@ def start(cls): @classmethod def stop(cls): + """Terminate HTTP server.""" cls.httpserver.stop() td = getattr(cls, 'teardown', None) if td: td() - def base(self): - if ((self.scheme == 'http' and self.PORT == 80) or - (self.scheme == 'https' and self.PORT == 443)): - port = '' - else: - port = ':%s' % self.PORT - - return '%s://%s%s%s' % (self.scheme, self.HOST, port, - self.script_name.rstrip('/')) - - def exit(self): - sys.exit() - - def skip(self, msg='skipped '): - pytest.skip(msg) - date_tolerance = 2 def assertEqualDates(self, dt1, dt2, seconds=None): - """Assert abs(dt1 - dt2) is within Y seconds.""" + """Assert ``abs(dt1 - dt2)`` is within ``Y`` seconds.""" if seconds is None: seconds = self.date_tolerance @@ -123,47 +108,62 @@ def assertEqualDates(self, dt1, dt2, seconds=None): else: diff = dt2 - dt1 if not diff < datetime.timedelta(seconds=seconds): - raise AssertionError('%r and %r are not within %r seconds.' % - (dt1, dt2, seconds)) + raise AssertionError( + '%r and %r are not within %r seconds.' % + (dt1, dt2, seconds), + ) -class Request(object): +class Request: + """HTTP request container.""" def __init__(self, environ): + """Initialize HTTP request.""" self.environ = environ -class Response(object): +class Response: + """HTTP response container.""" def __init__(self): + """Initialize HTTP response.""" self.status = '200 OK' self.headers = {'Content-Type': 'text/html'} self.body = None def output(self): + """Generate iterable response body object.""" if self.body is None: return [] elif isinstance(self.body, six.text_type): - return [ntob(self.body)] + return [self.body.encode('iso-8859-1')] elif isinstance(self.body, six.binary_type): return [self.body] else: - return [ntob(x) for x in self.body] + return [x.encode('iso-8859-1') for x in self.body] -class Controller(object): +class Controller: + """WSGI app for tests.""" def __call__(self, environ, start_response): + """WSGI request handler.""" req, resp = Request(environ), Response() try: - handler = getattr(self, environ['PATH_INFO'].lstrip('/').replace('/', '_')) - except AttributeError: + # Python 3 supports unicode attribute names + # Python 2 encodes them + handler = self.handlers[environ['PATH_INFO']] + except KeyError: resp.status = '404 Not Found' else: output = handler(req, resp) - if (output is not None and - not any(resp.status.startswith(status_code) - for status_code in ('204', '304'))): + if ( + output is not None + and not any( + resp.status.startswith(status_code) + for status_code in ('204', '304') + ) + ): resp.body = output try: resp.headers.setdefault('Content-Length', str(len(output))) @@ -172,123 +172,3 @@ def __call__(self, environ, start_response): raise start_response(resp.status, resp.headers.items()) return resp.output() - - -# --------------------------- Spawning helpers --------------------------- # - - -class CherootProcess(object): - - pid_file = os.path.join(thisdir, 'test.pid') - config_file = os.path.join(thisdir, 'test.conf') - config_template = """[global] -server.socket_host: '%(host)s' -server.socket_port: %(port)s -checker.on: False -log.screen: False -log.error_file: r'%(error_log)s' -log.access_file: r'%(access_log)s' -%(ssl)s -%(extra)s -""" - error_log = os.path.join(thisdir, 'test.error.log') - access_log = os.path.join(thisdir, 'test.access.log') - - def __init__(self, wait=False, daemonize=False, ssl=False, - socket_host=None, socket_port=None): - self.wait = wait - self.daemonize = daemonize - self.ssl = ssl - self.host = socket_host - self.port = socket_port - - def write_conf(self, extra=''): - if self.ssl: - serverpem = os.path.join(thisdir, 'test.pem') - ssl = """ -server.ssl_certificate: r'%s' -server.ssl_private_key: r'%s' -""" % (serverpem, serverpem) - else: - ssl = '' - - conf = self.config_template % { - 'host': self.host, - 'port': self.port, - 'error_log': self.error_log, - 'access_log': self.access_log, - 'ssl': ssl, - 'extra': extra, - } - with io.open(self.config_file, 'w', encoding='utf-8') as f: - f.write(six.text_type(conf)) - - def start(self, imports=None): - """Start cherryd in a subprocess.""" - portend.free(self.host, self.port, timeout=1) - - args = [ - os.path.join(thisdir, '..', 'cherryd'), - '-c', self.config_file, - '-p', self.pid_file, - ] - - if not isinstance(imports, (list, tuple)): - imports = [imports] - for i in imports: - if i: - args.append('-i') - args.append(i) - - if self.daemonize: - args.append('-d') - - env = os.environ.copy() - # Make sure we import the cheroot package in which this module is - # defined. - grandparentdir = os.path.abspath(os.path.join(thisdir, '..', '..')) - if env.get('PYTHONPATH', ''): - env['PYTHONPATH'] = os.pathsep.join( - (grandparentdir, env['PYTHONPATH'])) - else: - env['PYTHONPATH'] = grandparentdir - self._proc = subprocess.Popen([sys.executable] + args, env=env) - if self.wait: - self.exit_code = self._proc.wait() - else: - portend.occupied(self.host, self.port, timeout=5) - - # Give the engine a wee bit more time to finish STARTING - if self.daemonize: - time.sleep(2) - else: - time.sleep(1) - - def get_pid(self): - if self.daemonize: - return int(open(self.pid_file, 'rb').read()) - return self._proc.pid - - def join(self): - """Wait for the process to exit.""" - if self.daemonize: - return self._join_daemon() - self._proc.wait() - - def _join_daemon(self): - try: - try: - # Mac, UNIX - os.wait() - except AttributeError: - # Windows - try: - pid = self.get_pid() - except IOError: - # Assume the subprocess deleted the pidfile on shutdown. - pass - else: - os.waitpid(pid, 0) - except OSError as ex: - if ex.args != (10, 'No child processes'): - raise diff --git a/lib/cheroot/test/test.pem b/lib/cheroot/test/test.pem deleted file mode 100644 index 47a4704..0000000 --- a/lib/cheroot/test/test.pem +++ /dev/null @@ -1,38 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIICXAIBAAKBgQDBKo554mzIMY+AByUNpaUOP9bJnQ7ZLQe9XgHwoLJR4VzpyZZZ -R9L4WtImEew05FY3Izerfm3MN3+MC0tJ6yQU9sOiU3vBW6RrLIMlfKsnRwBRZ0Kn -da+O6xldVSosu8Ev3z9VZ94iC/ZgKzrH7Mjj/U8/MQO7RBS/LAqee8bFNQIDAQAB -AoGAWOCF0ZrWxn3XMucWq2LNwPKqlvVGwbIwX3cDmX22zmnM4Fy6arXbYh4XlyCj -9+ofqRrxIFz5k/7tFriTmZ0xag5+Jdx+Kwg0/twiP7XCNKipFogwe1Hznw8OFAoT -enKBdj2+/n2o0Bvo/tDB59m9L/538d46JGQUmJlzMyqYikECQQDyoq+8CtMNvE18 -8VgHcR/KtApxWAjj4HpaHYL637ATjThetUZkW92mgDgowyplthusxdNqhHWyv7E8 -tWNdYErZAkEAy85ShTR0M5aWmrE7o0r0SpWInAkNBH9aXQRRARFYsdBtNfRu6I0i -0lvU9wiu3eF57FMEC86yViZ5UBnQfTu7vQJAVesj/Zt7pwaCDfdMa740OsxMUlyR -MVhhGx4OLpYdPJ8qUecxGQKq13XZ7R1HGyNEY4bd2X80Smq08UFuATfC6QJAH8UB -yBHtKz2GLIcELOg6PIYizW/7v3+6rlVF60yw7sb2vzpjL40QqIn4IKoR2DSVtOkb -8FtAIX3N21aq0VrGYQJBAIPiaEc2AZ8Bq2GC4F3wOz/BxJ/izvnkiotR12QK4fh5 -yjZMhTjWCas5zwHR5PDjlD88AWGDMsZ1PicD4348xJQ= ------END RSA PRIVATE KEY----- ------BEGIN CERTIFICATE----- -MIIDxTCCAy6gAwIBAgIJAI18BD7eQxlGMA0GCSqGSIb3DQEBBAUAMIGeMQswCQYD -VQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTESMBAGA1UEBxMJU2FuIERpZWdv -MRkwFwYDVQQKExBDaGVycnlQeSBQcm9qZWN0MREwDwYDVQQLEwhkZXYtdGVzdDEW -MBQGA1UEAxMNQ2hlcnJ5UHkgVGVhbTEgMB4GCSqGSIb3DQEJARYRcmVtaUBjaGVy -cnlweS5vcmcwHhcNMDYwOTA5MTkyMDIwWhcNMzQwMTI0MTkyMDIwWjCBnjELMAkG -A1UEBhMCVVMxEzARBgNVBAgTCkNhbGlmb3JuaWExEjAQBgNVBAcTCVNhbiBEaWVn -bzEZMBcGA1UEChMQQ2hlcnJ5UHkgUHJvamVjdDERMA8GA1UECxMIZGV2LXRlc3Qx -FjAUBgNVBAMTDUNoZXJyeVB5IFRlYW0xIDAeBgkqhkiG9w0BCQEWEXJlbWlAY2hl -cnJ5cHkub3JnMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDBKo554mzIMY+A -ByUNpaUOP9bJnQ7ZLQe9XgHwoLJR4VzpyZZZR9L4WtImEew05FY3Izerfm3MN3+M -C0tJ6yQU9sOiU3vBW6RrLIMlfKsnRwBRZ0Knda+O6xldVSosu8Ev3z9VZ94iC/Zg -KzrH7Mjj/U8/MQO7RBS/LAqee8bFNQIDAQABo4IBBzCCAQMwHQYDVR0OBBYEFDIQ -2feb71tVZCWpU0qJ/Tw+wdtoMIHTBgNVHSMEgcswgciAFDIQ2feb71tVZCWpU0qJ -/Tw+wdtooYGkpIGhMIGeMQswCQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5p -YTESMBAGA1UEBxMJU2FuIERpZWdvMRkwFwYDVQQKExBDaGVycnlQeSBQcm9qZWN0 -MREwDwYDVQQLEwhkZXYtdGVzdDEWMBQGA1UEAxMNQ2hlcnJ5UHkgVGVhbTEgMB4G -CSqGSIb3DQEJARYRcmVtaUBjaGVycnlweS5vcmeCCQCNfAQ+3kMZRjAMBgNVHRME -BTADAQH/MA0GCSqGSIb3DQEBBAUAA4GBAL7AAQz7IePV48ZTAFHKr88ntPALsL5S -8vHCZPNMevNkLTj3DYUw2BcnENxMjm1kou2F2BkvheBPNZKIhc6z4hAml3ed1xa2 -D7w6e6OTcstdK/+KrPDDHeOP1dhMWNs2JE1bNlfF1LiXzYKSXpe88eCKjCXsCT/T -NluCaWQys3MS ------END CERTIFICATE----- diff --git a/lib/cheroot/test/test__compat.py b/lib/cheroot/test/test__compat.py new file mode 100644 index 0000000..35c6280 --- /dev/null +++ b/lib/cheroot/test/test__compat.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- +"""Test suite for cross-python compatibility helpers.""" + +from __future__ import absolute_import, division, print_function +__metaclass__ = type + +import pytest +import six + +from cheroot._compat import extract_bytes, memoryview, ntob, ntou, bton + + +@pytest.mark.parametrize( + ('func', 'inp', 'out'), + ( + (ntob, 'bar', b'bar'), + (ntou, 'bar', u'bar'), + (bton, b'bar', 'bar'), + ), +) +def test_compat_functions_positive(func, inp, out): + """Check that compatibility functions work with correct input.""" + assert func(inp, encoding='utf-8') == out + + +@pytest.mark.parametrize( + 'func', + ( + ntob, + ntou, + ), +) +def test_compat_functions_negative_nonnative(func): + """Check that compatibility functions fail loudly for incorrect input.""" + non_native_test_str = u'bar' if six.PY2 else b'bar' + with pytest.raises(TypeError): + func(non_native_test_str, encoding='utf-8') + + +def test_ntou_escape(): + """Check that ``ntou`` supports escape-encoding under Python 2.""" + expected = u'hišřії' + actual = ntou('hi\u0161\u0159\u0456\u0457', encoding='escape') + assert actual == expected + + +@pytest.mark.parametrize( + ('input_argument', 'expected_result'), + ( + (b'qwerty', b'qwerty'), + (memoryview(b'asdfgh'), b'asdfgh'), + ), +) +def test_extract_bytes(input_argument, expected_result): + """Check that legitimate inputs produce bytes.""" + assert extract_bytes(input_argument) == expected_result + + +def test_extract_bytes_invalid(): + """Ensure that invalid input causes exception to be raised.""" + with pytest.raises( + ValueError, + match=r'^extract_bytes\(\) only accepts bytes ' + 'and memoryview/buffer$', + ): + extract_bytes(u'some юнікод їїї') diff --git a/lib/cheroot/test/test_cli.py b/lib/cheroot/test/test_cli.py new file mode 100644 index 0000000..62c35af --- /dev/null +++ b/lib/cheroot/test/test_cli.py @@ -0,0 +1,92 @@ +"""Tests to verify the command line interface.""" +# -*- coding: utf-8 -*- +# vim: set fileencoding=utf-8 : +import sys + +import six +import pytest + +from cheroot.cli import ( + Application, + parse_wsgi_bind_addr, +) + + +@pytest.mark.parametrize( + ('raw_bind_addr', 'expected_bind_addr'), + ( + # tcp/ip + ('192.168.1.1:80', ('192.168.1.1', 80)), + # ipv6 ips has to be enclosed in brakets when specified in url form + ('[::1]:8000', ('::1', 8000)), + ('localhost:5000', ('localhost', 5000)), + # this is a valid input, but foo gets discarted + ('foo@bar:5000', ('bar', 5000)), + ('foo', ('foo', None)), + ('123456789', ('123456789', None)), + # unix sockets + ('/tmp/cheroot.sock', '/tmp/cheroot.sock'), + ('/tmp/some-random-file-name', '/tmp/some-random-file-name'), + # abstract sockets + ('@cheroot', '\x00cheroot'), + ), +) +def test_parse_wsgi_bind_addr(raw_bind_addr, expected_bind_addr): + """Check the parsing of the --bind option. + + Verify some of the supported addresses and the excpected return value. + """ + assert parse_wsgi_bind_addr(raw_bind_addr) == expected_bind_addr + + +@pytest.fixture +def wsgi_app(monkeypatch): + """Return a WSGI app stub.""" + class WSGIAppMock: + """Mock of a wsgi module.""" + + def application(self): + """Empty application method. + + Default method to be called when no specific callable + is defined in the wsgi application identifier. + + It has an empty body because we are expecting to verify that + the same method is return no the actual execution of it. + """ + + def main(self): + """Empty custom method (callable) inside the mocked WSGI app. + + It has an empty body because we are expecting to verify that + the same method is return no the actual execution of it. + """ + app = WSGIAppMock() + # patch sys.modules, to include the an instance of WSGIAppMock + # under a specific namespace + if six.PY2: + # python2 requires the previous namespaces to be part of sys.modules + # (e.g. for 'a.b.c' we need to insert 'a', 'a.b' and 'a.b.c') + # otherwise it fails, we're setting the same instance on each level, + # we don't really care about those, just the last one. + monkeypatch.setitem(sys.modules, 'mypkg', app) + monkeypatch.setitem(sys.modules, 'mypkg.wsgi', app) + return app + + +@pytest.mark.parametrize( + ('app_name', 'app_method'), + ( + (None, 'application'), + ('application', 'application'), + ('main', 'main'), + ), +) +def test_Aplication_resolve(app_name, app_method, wsgi_app): + """Check the wsgi application name conversion.""" + if app_name is None: + wsgi_app_spec = 'mypkg.wsgi' + else: + wsgi_app_spec = 'mypkg.wsgi:{app_name}'.format(**locals()) + expected_app = getattr(wsgi_app, app_method) + assert Application.resolve(wsgi_app_spec).wsgi_app == expected_app diff --git a/lib/cheroot/test/test_compat.py b/lib/cheroot/test/test_compat.py deleted file mode 100644 index 52f3186..0000000 --- a/lib/cheroot/test/test_compat.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Test Python 2/3 compatibility module.""" -from __future__ import unicode_literals - -import unittest - -import pytest -import six - -from cheroot import _compat as compat - - -class StringTester(unittest.TestCase): - """Tests for string conversion.""" - - @pytest.mark.skipif(six.PY3, reason='Only useful on Python 2') - def test_ntob_non_native(self): - """ntob should raise an Exception on unicode. - - (Python 2 only) - - See #1132 for discussion. - """ - self.assertRaises(TypeError, compat.ntob, 'fight') - - -class EscapeTester(unittest.TestCase): - """Class to test escape_html function from _cpcompat.""" - - def test_escape_quote(self): - """Verify the output for &<>"' chars.""" - self.assertEqual("""xx&<>"aa'""", compat.escape_html("""xx&<>"aa'""")) diff --git a/lib/cheroot/test/test_config_server.py b/lib/cheroot/test/test_config_server.py deleted file mode 100644 index b081009..0000000 --- a/lib/cheroot/test/test_config_server.py +++ /dev/null @@ -1,129 +0,0 @@ -"""Tests for the CherryPy configuration system.""" - -import os - -import pytest - -from cheroot.test import helper - - -pytestmark = pytest.mark.skip(reason='Depends on CherryPy') - - -localDir = os.path.join(os.getcwd(), os.path.dirname(__file__)) - - -# Client-side code # - - -class ServerConfigTests(helper.CherootWebCase): - - @staticmethod - def setup_server(): - - class Root: - - def index(self): - return cherrypy.request.wsgi_environ['SERVER_PORT'] - - @cherrypy.expose - def upload(self, file): - return 'Size: %s' % len(file.file.read()) - - @cherrypy.expose - @cherrypy.config(**{'request.body.maxbytes': 100}) - def tinyupload(self): - return cherrypy.request.body.read() - - cherrypy.tree.mount(Root()) - - cherrypy.config.update({ - 'server.socket_host': '0.0.0.0', - 'server.socket_port': 9876, - 'server.max_request_body_size': 200, - 'server.max_request_header_size': 500, - 'server.socket_timeout': 0.5, - - # Test explicit server.instance - 'server.2.instance': 'cherrypy._cpwsgi_server.CPWSGIServer', - 'server.2.socket_port': 9877, - - # Test non-numeric - # Also test default server.instance = builtin server - 'server.yetanother.socket_port': 9878, - }) - - PORT = 9876 - - def testBasicConfig(self): - self.getPage('/') - self.assertBody(str(self.PORT)) - - def testAdditionalServers(self): - if self.scheme == 'https': - return self.skip('not available under ssl') - self.PORT = 9877 - self.getPage('/') - self.assertBody(str(self.PORT)) - self.PORT = 9878 - self.getPage('/') - self.assertBody(str(self.PORT)) - - def testMaxRequestSizePerHandler(self): - if getattr(cherrypy.server, 'using_apache', False): - return self.skip('skipped due to known Apache differences... ') - - self.getPage('/tinyupload', method='POST', - headers=[('Content-Type', 'text/plain'), - ('Content-Length', '100')], - body='x' * 100) - self.assertStatus(200) - self.assertBody('x' * 100) - - self.getPage('/tinyupload', method='POST', - headers=[('Content-Type', 'text/plain'), - ('Content-Length', '101')], - body='x' * 101) - self.assertStatus(413) - - def testMaxRequestSize(self): - if getattr(cherrypy.server, 'using_apache', False): - return self.skip('skipped due to known Apache differences... ') - - for size in (500, 5000, 50000): - self.getPage('/', headers=[('From', 'x' * 500)]) - self.assertStatus(413) - - # Test for https://github.com/cherrypy/cherrypy/issues/421 - # (Incorrect border condition in readline of SizeCheckWrapper). - # This hangs in rev 891 and earlier. - lines256 = 'x' * 248 - self.getPage('/', - headers=[('Host', '%s:%s' % (self.HOST, self.PORT)), - ('From', lines256)]) - - # Test upload - cd = ( - 'Content-Disposition: form-data; ' - 'name="file"; ' - 'filename="hello.txt"' - ) - body = '\r\n'.join([ - '--x', - cd, - 'Content-Type: text/plain', - '', - '%s', - '--x--']) - partlen = 200 - len(body) - b = body % ('x' * partlen) - h = [('Content-type', 'multipart/form-data; boundary=x'), - ('Content-Length', '%s' % len(b))] - self.getPage('/upload', h, 'POST', b) - self.assertBody('Size: %d' % partlen) - - b = body % ('x' * 200) - h = [('Content-type', 'multipart/form-data; boundary=x'), - ('Content-Length', '%s' % len(b))] - self.getPage('/upload', h, 'POST', b) - self.assertStatus(413) diff --git a/lib/cheroot/test/test_conn.py b/lib/cheroot/test/test_conn.py index c448709..51a9ee5 100644 --- a/lib/cheroot/test/test_conn.py +++ b/lib/cheroot/test/test_conn.py @@ -1,710 +1,1149 @@ """Tests for TCP connection handling, including proper and timely close.""" +from __future__ import absolute_import, division, print_function +__metaclass__ = type + import socket import time +import logging +import traceback as traceback_ +from collections import namedtuple + +from six.moves import range, http_client, urllib import six import pytest +from jaraco.text import trim, unwrap -from cheroot._compat import HTTPConnection, HTTPSConnection, NotConnected, BadStatusLine -from cheroot._compat import ntob, urlopen from cheroot.test import helper, webtest - - -# avpytestmark = pytest.mark.skip(reason="incomplete") +from cheroot._compat import IS_PYPY +import cheroot.server timeout = 1 pov = 'pPeErRsSiIsStTeEnNcCeE oOfF vViIsSiIoOnN' -class ConnectionCloseTests(helper.CherootWebCase): - - @classmethod - def setup_server(cls): - - class Root(helper.Controller): - - def pov(self, req, resp): - return pov - page1 = pov - page2 = pov - page3 = pov - - def hello(self, req, resp): - return 'Hello, world!' - - def timeout(self, req, resp): - return str(cls.httpserver.timeout) - - def stream(self, req, resp): - if 'set_cl' in req.environ['QUERY_STRING']: - resp.headers['Content-Length'] = str(10) - - def content(): - for x in range(10): - yield str(x) - - return content() - - def upload(self, req, resp): - if not req.environ['REQUEST_METHOD'] == 'POST': - raise AssertionError("'POST' != request.method %r" % - req.environ['REQUEST_METHOD']) - return "thanks for '%s'" % req.environ['wsgi.input'].read() - - def custom_204(self, req, resp): - resp.status = '204' - return 'Code = 204' - - def custom_304(self, req, resp): - resp.status = '304' - return 'Code = 304' - - def err_before_read(self, req, resp): - resp.status = '500 Internal Server Error' - return 'ok' - - def one_megabyte_of_a(self, req, resp): - return ['a' * 1024] * 1024 - - def wrong_cl_buffered(self, req, resp): - resp.headers['Content-Length'] = '5' - return 'I have too many bytes' - - def wrong_cl_unbuffered(self, req, resp): - resp.headers['Content-Length'] = '5' - return ['I too', ' have too many bytes'] +class Controller(helper.Controller): + """Controller for serving WSGI apps.""" + + def hello(req, resp): + """Render Hello world.""" + return 'Hello, world!' + + def pov(req, resp): + """Render ``pov`` value.""" + return pov + + def stream(req, resp): + """Render streaming response.""" + if 'set_cl' in req.environ['QUERY_STRING']: + resp.headers['Content-Length'] = str(10) + + def content(): + for x in range(10): + yield str(x) + + return content() + + def upload(req, resp): + """Process file upload and render thank.""" + if not req.environ['REQUEST_METHOD'] == 'POST': + raise AssertionError( + "'POST' != request.method %r" % + req.environ['REQUEST_METHOD'], + ) + return "thanks for '%s'" % req.environ['wsgi.input'].read() + + def custom_204(req, resp): + """Render response with status 204.""" + resp.status = '204' + return 'Code = 204' + + def custom_304(req, resp): + """Render response with status 304.""" + resp.status = '304' + return 'Code = 304' + + def err_before_read(req, resp): + """Render response with status 500.""" + resp.status = '500 Internal Server Error' + return 'ok' + + def one_megabyte_of_a(req, resp): + """Render 1MB response.""" + return ['a' * 1024] * 1024 + + def wrong_cl_buffered(req, resp): + """Render buffered response with invalid length value.""" + resp.headers['Content-Length'] = '5' + return 'I have too many bytes' + + def wrong_cl_unbuffered(req, resp): + """Render unbuffered response with invalid length value.""" + resp.headers['Content-Length'] = '5' + return ['I too', ' have too many bytes'] + + def _munge(string): + """Encode PATH_INFO correctly depending on Python version. + + WSGI 1.0 is a mess around unicode. Create endpoints + that match the PATH_INFO that it produces. + """ + if six.PY2: + return string + return string.encode('utf-8').decode('latin-1') + + handlers = { + '/hello': hello, + '/pov': pov, + '/page1': pov, + '/page2': pov, + '/page3': pov, + '/stream': stream, + '/upload': upload, + '/custom/204': custom_204, + '/custom/304': custom_304, + '/err_before_read': err_before_read, + '/one_megabyte_of_a': one_megabyte_of_a, + '/wrong_cl_buffered': wrong_cl_buffered, + '/wrong_cl_unbuffered': wrong_cl_unbuffered, + } + + +class ErrorLogMonitor: + """Mock class to access the server error_log calls made by the server.""" + + ErrorLogCall = namedtuple('ErrorLogCall', ['msg', 'level', 'traceback']) + + def __init__(self): + """Initialize the server error log monitor/interceptor. + + If you need to ignore a particular error message use the property + ``ignored_msgs` by appending to the list the expected error messages. + """ + self.calls = [] + # to be used the the teardown validation + self.ignored_msgs = [] + + def __call__(self, msg='', level=logging.INFO, traceback=False): + """Intercept the call to the server error_log method.""" + if traceback: + tblines = traceback_.format_exc() + else: + tblines = '' + self.calls.append(ErrorLogMonitor.ErrorLogCall(msg, level, tblines)) + + +@pytest.fixture +def raw_testing_server(wsgi_server_client): + """Attach a WSGI app to the given server and preconfigure it.""" + app = Controller() + + def _timeout(req, resp): + return str(wsgi_server.timeout) + app.handlers['/timeout'] = _timeout + wsgi_server = wsgi_server_client.server_instance + wsgi_server.wsgi_app = app + wsgi_server.max_request_body_size = 1001 + wsgi_server.timeout = timeout + wsgi_server.server_client = wsgi_server_client + wsgi_server.keep_alive_conn_limit = 2 + + return wsgi_server + + +@pytest.fixture +def testing_server(raw_testing_server, monkeypatch): + """Modify the "raw" base server to monitor the error_log messages. + + If you need to ignore a particular error message use the property + ``testing_server.error_log.ignored_msgs`` by appending to the list + the expected error messages. + """ + # patch the error_log calls of the server instance + monkeypatch.setattr(raw_testing_server, 'error_log', ErrorLogMonitor()) + + yield raw_testing_server + + # Teardown verification, in case that the server logged an + # error that wasn't notified to the client or we just made a mistake. + for c in raw_testing_server.error_log.calls: + if c.level <= logging.WARNING: + continue + + assert c.msg in raw_testing_server.error_log.ignored_msgs, ( + 'Found error in the error log: ' + "message = '{c.msg}', level = '{c.level}'\n" + '{c.traceback}'.format(**locals()), + ) + + +@pytest.fixture +def test_client(testing_server): + """Get and return a test client out of the given server.""" + return testing_server.server_client + + +def header_exists(header_name, headers): + """Check that a header is present.""" + return header_name.lower() in (k.lower() for (k, _) in headers) + + +def header_has_value(header_name, header_value, headers): + """Check that a header with a given value is present.""" + return header_name.lower() in ( + k.lower() for (k, v) in headers + if v == header_value + ) + + +def test_HTTP11_persistent_connections(test_client): + """Test persistent HTTP/1.1 connections.""" + # Initialize a persistent HTTP connection + http_connection = test_client.get_connection() + http_connection.auto_open = False + http_connection.connect() + + # Make the first request and assert there's no "Connection: close". + status_line, actual_headers, actual_resp_body = test_client.get( + '/pov', http_conn=http_connection, + ) + actual_status = int(status_line[:3]) + assert actual_status == 200 + assert status_line[4:] == 'OK' + assert actual_resp_body == pov.encode() + assert not header_exists('Connection', actual_headers) + + # Make another request on the same connection. + status_line, actual_headers, actual_resp_body = test_client.get( + '/page1', http_conn=http_connection, + ) + actual_status = int(status_line[:3]) + assert actual_status == 200 + assert status_line[4:] == 'OK' + assert actual_resp_body == pov.encode() + assert not header_exists('Connection', actual_headers) + + # Test client-side close. + status_line, actual_headers, actual_resp_body = test_client.get( + '/page2', http_conn=http_connection, + headers=[('Connection', 'close')], + ) + actual_status = int(status_line[:3]) + assert actual_status == 200 + assert status_line[4:] == 'OK' + assert actual_resp_body == pov.encode() + assert header_has_value('Connection', 'close', actual_headers) + + # Make another request on the same connection, which should error. + with pytest.raises(http_client.NotConnected): + test_client.get('/pov', http_conn=http_connection) + + +@pytest.mark.parametrize( + 'set_cl', + ( + False, # Without Content-Length + True, # With Content-Length + ), +) +def test_streaming_11(test_client, set_cl): + """Test serving of streaming responses with HTTP/1.1 protocol.""" + # Initialize a persistent HTTP connection + http_connection = test_client.get_connection() + http_connection.auto_open = False + http_connection.connect() + + # Make the first request and assert there's no "Connection: close". + status_line, actual_headers, actual_resp_body = test_client.get( + '/pov', http_conn=http_connection, + ) + actual_status = int(status_line[:3]) + assert actual_status == 200 + assert status_line[4:] == 'OK' + assert actual_resp_body == pov.encode() + assert not header_exists('Connection', actual_headers) + + # Make another, streamed request on the same connection. + if set_cl: + # When a Content-Length is provided, the content should stream + # without closing the connection. + status_line, actual_headers, actual_resp_body = test_client.get( + '/stream?set_cl=Yes', http_conn=http_connection, + ) + assert header_exists('Content-Length', actual_headers) + assert not header_has_value('Connection', 'close', actual_headers) + assert not header_exists('Transfer-Encoding', actual_headers) + + assert actual_status == 200 + assert status_line[4:] == 'OK' + assert actual_resp_body == b'0123456789' + else: + # When no Content-Length response header is provided, + # streamed output will either close the connection, or use + # chunked encoding, to determine transfer-length. + status_line, actual_headers, actual_resp_body = test_client.get( + '/stream', http_conn=http_connection, + ) + assert not header_exists('Content-Length', actual_headers) + assert actual_status == 200 + assert status_line[4:] == 'OK' + assert actual_resp_body == b'0123456789' + + chunked_response = False + for k, v in actual_headers: + if k.lower() == 'transfer-encoding': + if str(v) == 'chunked': + chunked_response = True + + if chunked_response: + assert not header_has_value('Connection', 'close', actual_headers) + else: + assert header_has_value('Connection', 'close', actual_headers) + + # Make another request on the same connection, which should + # error. + with pytest.raises(http_client.NotConnected): + test_client.get('/pov', http_conn=http_connection) + + # Try HEAD. + # See https://www.bitbucket.org/cherrypy/cherrypy/issue/864. + # TODO: figure out how can this be possible on an closed connection + # (chunked_response case) + status_line, actual_headers, actual_resp_body = test_client.head( + '/stream', http_conn=http_connection, + ) + assert actual_status == 200 + assert status_line[4:] == 'OK' + assert actual_resp_body == b'' + assert not header_exists('Transfer-Encoding', actual_headers) + + +@pytest.mark.parametrize( + 'set_cl', + ( + False, # Without Content-Length + True, # With Content-Length + ), +) +def test_streaming_10(test_client, set_cl): + """Test serving of streaming responses with HTTP/1.0 protocol.""" + original_server_protocol = test_client.server_instance.protocol + test_client.server_instance.protocol = 'HTTP/1.0' + + # Initialize a persistent HTTP connection + http_connection = test_client.get_connection() + http_connection.auto_open = False + http_connection.connect() + + # Make the first request and assert Keep-Alive. + status_line, actual_headers, actual_resp_body = test_client.get( + '/pov', http_conn=http_connection, + headers=[('Connection', 'Keep-Alive')], + protocol='HTTP/1.0', + ) + actual_status = int(status_line[:3]) + assert actual_status == 200 + assert status_line[4:] == 'OK' + assert actual_resp_body == pov.encode() + assert header_has_value('Connection', 'Keep-Alive', actual_headers) + + # Make another, streamed request on the same connection. + if set_cl: + # When a Content-Length is provided, the content should + # stream without closing the connection. + status_line, actual_headers, actual_resp_body = test_client.get( + '/stream?set_cl=Yes', http_conn=http_connection, + headers=[('Connection', 'Keep-Alive')], + protocol='HTTP/1.0', + ) + actual_status = int(status_line[:3]) + assert actual_status == 200 + assert status_line[4:] == 'OK' + assert actual_resp_body == b'0123456789' + + assert header_exists('Content-Length', actual_headers) + assert header_has_value('Connection', 'Keep-Alive', actual_headers) + assert not header_exists('Transfer-Encoding', actual_headers) + else: + # When a Content-Length is not provided, + # the server should close the connection. + status_line, actual_headers, actual_resp_body = test_client.get( + '/stream', http_conn=http_connection, + headers=[('Connection', 'Keep-Alive')], + protocol='HTTP/1.0', + ) + actual_status = int(status_line[:3]) + assert actual_status == 200 + assert status_line[4:] == 'OK' + assert actual_resp_body == b'0123456789' + + assert not header_exists('Content-Length', actual_headers) + assert not header_has_value('Connection', 'Keep-Alive', actual_headers) + assert not header_exists('Transfer-Encoding', actual_headers) - cls.httpserver.wsgi_app = Root() - cls.httpserver.max_request_body_size = 1001 - cls.httpserver.timeout = timeout + # Make another request on the same connection, which should error. + with pytest.raises(http_client.NotConnected): + test_client.get( + '/pov', http_conn=http_connection, + protocol='HTTP/1.0', + ) + + test_client.server_instance.protocol = original_server_protocol + + +@pytest.mark.parametrize( + 'http_server_protocol', + ( + 'HTTP/1.0', + pytest.param( + 'HTTP/1.1', + marks=pytest.mark.xfail( + IS_PYPY and not six.PY2, + reason='Fails under PyPy for unknown reason', + ), + ), + ), +) +def test_keepalive(test_client, http_server_protocol): + """Test Keep-Alive enabled connections.""" + original_server_protocol = test_client.server_instance.protocol + test_client.server_instance.protocol = http_server_protocol + + http_client_protocol = 'HTTP/1.0' + + # Initialize a persistent HTTP connection + http_connection = test_client.get_connection() + http_connection.auto_open = False + http_connection.connect() + + # Test a normal HTTP/1.0 request. + status_line, actual_headers, actual_resp_body = test_client.get( + '/page2', + protocol=http_client_protocol, + ) + actual_status = int(status_line[:3]) + assert actual_status == 200 + assert status_line[4:] == 'OK' + assert actual_resp_body == pov.encode() + assert not header_exists('Connection', actual_headers) + + # Test a keep-alive HTTP/1.0 request. + + status_line, actual_headers, actual_resp_body = test_client.get( + '/page3', headers=[('Connection', 'Keep-Alive')], + http_conn=http_connection, protocol=http_client_protocol, + ) + actual_status = int(status_line[:3]) + assert actual_status == 200 + assert status_line[4:] == 'OK' + assert actual_resp_body == pov.encode() + assert header_has_value('Connection', 'Keep-Alive', actual_headers) + assert header_has_value( + 'Keep-Alive', + 'timeout={test_client.server_instance.timeout}'.format(**locals()), + actual_headers, + ) + + # Remove the keep-alive header again. + status_line, actual_headers, actual_resp_body = test_client.get( + '/page3', http_conn=http_connection, + protocol=http_client_protocol, + ) + actual_status = int(status_line[:3]) + assert actual_status == 200 + assert status_line[4:] == 'OK' + assert actual_resp_body == pov.encode() + assert not header_exists('Connection', actual_headers) + assert not header_exists('Keep-Alive', actual_headers) + + test_client.server_instance.protocol = original_server_protocol + + +def test_keepalive_conn_management(test_client): + """Test management of Keep-Alive connections.""" + test_client.server_instance.timeout = 2 + + def connection(): + # Initialize a persistent HTTP connection + http_connection = test_client.get_connection() + http_connection.auto_open = False + http_connection.connect() + return http_connection + + def request(conn, keepalive=True): + status_line, actual_headers, actual_resp_body = test_client.get( + '/page3', headers=[('Connection', 'Keep-Alive')], + http_conn=conn, protocol='HTTP/1.0', + ) + actual_status = int(status_line[:3]) + assert actual_status == 200 + assert status_line[4:] == 'OK' + assert actual_resp_body == pov.encode() + if keepalive: + assert header_has_value('Connection', 'Keep-Alive', actual_headers) + assert header_has_value( + 'Keep-Alive', + 'timeout={test_client.server_instance.timeout}'. + format(**locals()), + actual_headers, + ) + else: + assert not header_exists('Connection', actual_headers) + assert not header_exists('Keep-Alive', actual_headers) - def test_HTTP11_persistent_connections(self): - self.httpserver.protocol = 'HTTP/1.1' - self.PROTOCOL = 'HTTP/1.1' + disconnect_errors = ( + http_client.BadStatusLine, + http_client.CannotSendRequest, + http_client.NotConnected, + ) - self.persistent = True + # Make a new connection. + c1 = connection() + request(c1) - # Make the first request and assert there's no "Connection: close". - self.getPage('/pov') - self.assertStatus('200 OK') - self.assertBody(pov) - self.assertNoHeader('Connection') + # Make a second one. + c2 = connection() + request(c2) - # Make another request on the same connection. - self.getPage('/page1') - self.assertStatus('200 OK') - self.assertBody(pov) - self.assertNoHeader('Connection') + # Reusing the first connection should still work. + request(c1) - # Test client-side close. - self.getPage('/page2', headers=[('Connection', 'close')]) - self.assertStatus('200 OK') - self.assertBody(pov) - self.assertHeader('Connection', 'close') + # Creating a new connection should still work, but we should + # have run out of available connections to keep alive, so the + # server should tell us to close. + c3 = connection() + request(c3, keepalive=False) - # Make another request on the same connection, which should error. - self.assertRaises(NotConnected, self.getPage, '/pov') + # Show that the third connection was closed. + with pytest.raises(disconnect_errors): + request(c3) - def test_Streaming_no_len_11(self): - self._streaming_11(set_cl=False) + # Wait for some of our timeout. + time.sleep(1.2) - def test_Streaming_with_len_11(self): - self._streaming_11(set_cl=True) + # Refresh the second connection. + request(c2) - def test_Streaming_no_len_10(self): - self._streaming_10(set_cl=False) + # Wait for the remainder of our timeout, plus one tick. + time.sleep(1.2) - def test_Streaming_with_len_10(self): - self._streaming_10(set_cl=True) + # First connection should now be expired. + with pytest.raises(disconnect_errors): + request(c1) - def _streaming_11(self, set_cl): - self.httpserver.protocol = 'HTTP/1.1' - self.PROTOCOL = 'HTTP/1.1' + # But the second one should still be valid. + request(c2) - self.persistent = True + # Restore original timeout. + test_client.server_instance.timeout = timeout - # Make the first request and assert there's no "Connection: close". - self.getPage('/pov') - self.assertStatus('200 OK') - self.assertBody(pov) - self.assertNoHeader('Connection') - # Make another, streamed request on the same connection. - if set_cl: - # When a Content-Length is provided, the content should stream - # without closing the connection. - self.getPage('/stream?set_cl=Yes') - self.assertHeader('Content-Length') - self.assertNoHeader('Connection', 'close') - self.assertNoHeader('Transfer-Encoding') +@pytest.mark.parametrize( + 'timeout_before_headers', + ( + True, + False, + ), +) +def test_HTTP11_Timeout(test_client, timeout_before_headers): + """Check timeout without sending any data. - self.assertStatus('200 OK') - self.assertBody('0123456789') - else: - # When no Content-Length response header is provided, - # streamed output will either close the connection, or use - # chunked encoding, to determine transfer-length. - self.getPage('/stream') - self.assertNoHeader('Content-Length') - self.assertStatus('200 OK') - self.assertBody('0123456789') - - chunked_response = False - for k, v in self.headers: - if k.lower() == 'transfer-encoding': - if str(v) == 'chunked': - chunked_response = True - - if chunked_response: - self.assertNoHeader('Connection', 'close') - else: - self.assertHeader('Connection', 'close') - - # Make another request on the same connection, which should - # error. - self.assertRaises(NotConnected, self.getPage, '/pov') - - # Try HEAD. - # See http://www.bitbucket.org/cherrypy/cherrypy/issue/864. - self.getPage('/stream', method='HEAD') - self.assertStatus('200 OK') - self.assertBody('') - self.assertNoHeader('Transfer-Encoding') - - def _streaming_10(self, set_cl): - self.httpserver.protocol = 'HTTP/1.0' - self.PROTOCOL = 'HTTP/1.0' - - self.persistent = True - - # Make the first request and assert Keep-Alive. - self.getPage('/pov', headers=[('Connection', 'Keep-Alive')]) - self.assertStatus('200 OK') - self.assertBody(pov) - self.assertHeader('Connection', 'Keep-Alive') - - # Make another, streamed request on the same connection. - if set_cl: - # When a Content-Length is provided, the content should - # stream without closing the connection. - self.getPage('/stream?set_cl=Yes', - headers=[('Connection', 'Keep-Alive')]) - self.assertHeader('Content-Length') - self.assertHeader('Connection', 'Keep-Alive') - self.assertNoHeader('Transfer-Encoding') - self.assertStatus('200 OK') - self.assertBody('0123456789') - else: - # When a Content-Length is not provided, - # the server should close the connection. - self.getPage('/stream', headers=[('Connection', 'Keep-Alive')]) - self.assertStatus('200 OK') - self.assertBody('0123456789') - - self.assertNoHeader('Content-Length') - self.assertNoHeader('Connection', 'Keep-Alive') - self.assertNoHeader('Transfer-Encoding') - - # Make another request on the same connection, which should error. - self.assertRaises(NotConnected, self.getPage, '/pov') - - def test_HTTP10_to_10_KeepAlive(self): - self.httpserver.protocol = 'HTTP/1.0' - self._keepalive() - - def test_HTTP10_to_11_KeepAlive(self): - self.httpserver.protocol = 'HTTP/1.1' - self._keepalive() - - def _keepalive(self): - self.PROTOCOL = 'HTTP/1.0' - if self.scheme == 'https': - self.HTTP_CONN = HTTPSConnection - else: - self.HTTP_CONN = HTTPConnection - - # Test a normal HTTP/1.0 request. - self.getPage('/page2') - self.assertStatus('200 OK') - self.assertBody(pov) - self.assertNoHeader('Connection') - - # Test a keep-alive HTTP/1.0 request. - self.persistent = True - - self.getPage('/page3', headers=[('Connection', 'Keep-Alive')]) - self.assertStatus('200 OK') - self.assertBody(pov) - self.assertHeader('Connection', 'Keep-Alive') - - # Remove the keep-alive header again. - self.getPage('/page3') - self.assertStatus('200 OK') - self.assertBody(pov) - self.assertNoHeader('Connection') - - def test_HTTP11_Timeout(self): - # If we timeout without sending any data, - # the server will close the conn with a 408. - self.httpserver.protocol = 'HTTP/1.1' - self.PROTOCOL = 'HTTP/1.1' - - # Connect but send nothing. - self.persistent = True - conn = self.HTTP_CONN - conn.auto_open = False - conn.connect() - - # Wait for our socket timeout - time.sleep(timeout * 2) - - # The request should have returned 408 already. - response = conn.response_class(conn.sock, method='GET') - response.begin() - self.assertEqual(response.status, 408) - conn.close() + The server will close the connection with a 408. + """ + conn = test_client.get_connection() + conn.auto_open = False + conn.connect() + if not timeout_before_headers: # Connect but send half the headers only. - self.persistent = True - conn = self.HTTP_CONN - conn.auto_open = False - conn.connect() conn.send(b'GET /hello HTTP/1.1') - conn.send(('Host: %s' % self.HOST).encode('ascii')) - - # Wait for our socket timeout - time.sleep(timeout * 2) - - # The conn should have already sent 408. - response = conn.response_class(conn.sock, method='GET') + conn.send(('Host: %s' % conn.host).encode('ascii')) + # else: Connect but send nothing. + + # Wait for our socket timeout + time.sleep(timeout * 2) + + # The request should have returned 408 already. + response = conn.response_class(conn.sock, method='GET') + response.begin() + assert response.status == 408 + conn.close() + + +def test_HTTP11_Timeout_after_request(test_client): + """Check timeout after at least one request has succeeded. + + The server should close the connection without 408. + """ + fail_msg = "Writing to timed out socket didn't fail as it should have: %s" + + # Make an initial request + conn = test_client.get_connection() + conn.putrequest('GET', '/timeout?t=%s' % timeout, skip_host=True) + conn.putheader('Host', conn.host) + conn.endheaders() + response = conn.response_class(conn.sock, method='GET') + response.begin() + assert response.status == 200 + actual_body = response.read() + expected_body = str(timeout).encode() + assert actual_body == expected_body + + # Make a second request on the same socket + conn._output(b'GET /hello HTTP/1.1') + conn._output(('Host: %s' % conn.host).encode('ascii')) + conn._send_output() + response = conn.response_class(conn.sock, method='GET') + response.begin() + assert response.status == 200 + actual_body = response.read() + expected_body = b'Hello, world!' + assert actual_body == expected_body + + # Wait for our socket timeout + time.sleep(timeout * 2) + + # Make another request on the same socket, which should error + conn._output(b'GET /hello HTTP/1.1') + conn._output(('Host: %s' % conn.host).encode('ascii')) + conn._send_output() + response = conn.response_class(conn.sock, method='GET') + try: response.begin() - self.assertEqual(response.status, 408) - conn.close() - - def test_HTTP11_Timeout_after_request(self): - # If we timeout after at least one request has succeeded, - # the server will close the conn without 408. - self.httpserver.protocol = 'HTTP/1.1' - self.PROTOCOL = 'HTTP/1.1' - - # Make an initial request - self.persistent = True - conn = self.HTTP_CONN - conn.putrequest('GET', '/timeout?t=%s' % timeout, skip_host=True) - conn.putheader('Host', self.HOST) - conn.endheaders() - response = conn.response_class(conn.sock, method='GET') - response.begin() - self.assertEqual(response.status, 200) - self.body = response.read() - self.assertBody(str(timeout)) - - # Make a second request on the same socket - conn._output(b'GET /hello HTTP/1.1') - conn._output(ntob('Host: %s' % self.HOST, 'ascii')) - conn._send_output() - response = conn.response_class(conn.sock, method='GET') + except (socket.error, http_client.BadStatusLine): + pass + except Exception as ex: + pytest.fail(fail_msg % ex) + else: + if response.status != 408: + pytest.fail(fail_msg % response.read()) + + conn.close() + + # Make another request on a new socket, which should work + conn = test_client.get_connection() + conn.putrequest('GET', '/pov', skip_host=True) + conn.putheader('Host', conn.host) + conn.endheaders() + response = conn.response_class(conn.sock, method='GET') + response.begin() + assert response.status == 200 + actual_body = response.read() + expected_body = pov.encode() + assert actual_body == expected_body + + # Make another request on the same socket, + # but timeout on the headers + conn.send(b'GET /hello HTTP/1.1') + # Wait for our socket timeout + time.sleep(timeout * 2) + response = conn.response_class(conn.sock, method='GET') + try: response.begin() - self.assertEqual(response.status, 200) - self.body = response.read() - self.assertBody('Hello, world!') - - # Wait for our socket timeout - time.sleep(timeout * 2) - - # Make another request on the same socket, which should error - conn._output(b'GET /hello HTTP/1.1') - conn._output(ntob('Host: %s' % self.HOST, 'ascii')) + except (socket.error, http_client.BadStatusLine): + pass + except Exception as ex: + pytest.fail(fail_msg % ex) + else: + if response.status != 408: + pytest.fail(fail_msg % response.read()) + + conn.close() + + # Retry the request on a new connection, which should work + conn = test_client.get_connection() + conn.putrequest('GET', '/pov', skip_host=True) + conn.putheader('Host', conn.host) + conn.endheaders() + response = conn.response_class(conn.sock, method='GET') + response.begin() + assert response.status == 200 + actual_body = response.read() + expected_body = pov.encode() + assert actual_body == expected_body + conn.close() + + +def test_HTTP11_pipelining(test_client): + """Test HTTP/1.1 pipelining. + + :py:mod:`http.client` doesn't support this directly. + """ + conn = test_client.get_connection() + + # Put request 1 + conn.putrequest('GET', '/hello', skip_host=True) + conn.putheader('Host', conn.host) + conn.endheaders() + + for trial in range(5): + # Put next request + conn._output( + ('GET /hello?%s HTTP/1.1' % trial).encode('iso-8859-1'), + ) + conn._output(('Host: %s' % conn.host).encode('ascii')) conn._send_output() - response = conn.response_class(conn.sock, method='GET') - try: - response.begin() - except Exception as ex: - if not isinstance(ex, - (socket.error, BadStatusLine)): - self.fail("Writing to timed out socket didn't fail" - ' as it should have: %s' % ex) - else: - if response.status != 408: - self.fail("Writing to timed out socket didn't fail" - ' as it should have: %s' % - response.read()) - - conn.close() - - # Make another request on a new socket, which should work - self.persistent = True - conn = self.HTTP_CONN - conn.putrequest('GET', '/pov', skip_host=True) - conn.putheader('Host', self.HOST) - conn.endheaders() - response = conn.response_class(conn.sock, method='GET') - response.begin() - self.assertEqual(response.status, 200) - self.body = response.read() - self.assertBody(pov) - # Make another request on the same socket, - # but timeout on the headers - conn.send(b'GET /hello HTTP/1.1') - # Wait for our socket timeout - time.sleep(timeout * 2) - response = conn.response_class(conn.sock, method='GET') - try: - response.begin() - except Exception as ex: - if not isinstance(ex, - (socket.error, BadStatusLine)): - self.fail("Writing to timed out socket didn't fail" - ' as it should have: %s' % ex) - else: - self.fail("Writing to timed out socket didn't fail" - ' as it should have: %s' % - response.read()) - - conn.close() - - # Retry the request on a new connection, which should work - self.persistent = True - conn = self.HTTP_CONN - conn.putrequest('GET', '/pov', skip_host=True) - conn.putheader('Host', self.HOST) - conn.endheaders() + # Retrieve previous response response = conn.response_class(conn.sock, method='GET') + # there is a bug in python3 regarding the buffering of + # ``conn.sock``. Until that bug get's fixed we will + # monkey patch the ``response`` instance. + # https://bugs.python.org/issue23377 + if not six.PY2: + response.fp = conn.sock.makefile('rb', 0) response.begin() - self.assertEqual(response.status, 200) - self.body = response.read() - self.assertBody(pov) - conn.close() - - def test_HTTP11_pipelining(self): - self.httpserver.protocol = 'HTTP/1.1' - self.PROTOCOL = 'HTTP/1.1' - - # Test pipelining. httplib doesn't support this directly. - self.persistent = True - conn = self.HTTP_CONN - - # Put request 1 - conn.putrequest('GET', '/hello', skip_host=True) - conn.putheader('Host', self.HOST) - conn.endheaders() - - for trial in range(5): - # Put next request - conn._output(ntob('GET /hello?%s HTTP/1.1' % trial)) - conn._output(ntob('Host: %s' % self.HOST, 'ascii')) - conn._send_output() - - # Retrieve previous response - response = conn.response_class(conn.sock, method='GET') - # there is a bug in python3 regarding the buffering of - # ``conn.sock``. Until that bug get's fixed we will - # monkey patch the ``reponse`` instance. - # https://bugs.python.org/issue23377 - if six.PY3: - response.fp = conn.sock.makefile('rb', 0) - response.begin() - body = response.read(13) - self.assertEqual(response.status, 200) - self.assertEqual(body, b'Hello, world!') - - # Retrieve final response - response = conn.response_class(conn.sock, method='GET') - response.begin() - body = response.read() - self.assertEqual(response.status, 200) - self.assertEqual(body, b'Hello, world!') - - conn.close() - - def test_100_Continue(self): - self.httpserver.protocol = 'HTTP/1.1' - self.PROTOCOL = 'HTTP/1.1' - - self.persistent = True - conn = self.HTTP_CONN - - # Try a page without an Expect request header first. - # Note that httplib's response.begin automatically ignores - # 100 Continue responses, so we must manually check for it. - conn.putrequest('POST', '/upload', skip_host=True) - conn.putheader('Host', self.HOST) - conn.putheader('Content-Type', 'text/plain') - conn.putheader('Content-Length', '4') - conn.endheaders() - conn.send(b"d'oh") - response = conn.response_class(conn.sock, method='POST') - version, status, reason = response._read_status() - self.assertNotEqual(status, 100) - conn.close() - - # Now try a page with an Expect header... - conn.connect() - conn.putrequest('POST', '/upload', skip_host=True) - conn.putheader('Host', self.HOST) - conn.putheader('Content-Type', 'text/plain') - conn.putheader('Content-Length', '17') - conn.putheader('Expect', '100-continue') - conn.endheaders() - response = conn.response_class(conn.sock, method='POST') - - # ...assert and then skip the 100 response - version, status, reason = response._read_status() - self.assertEqual(status, 100) - while True: - line = response.fp.readline().strip() - if line: - self.fail( - '100 Continue should not output any headers. Got %r' % - line) - else: - break - - # ...send the body - body = b'I am a small file' - conn.send(body) - - # ...get the final response - response.begin() - self.status, self.headers, self.body = webtest.shb(response) - self.assertStatus(200) - self.assertBody("thanks for '%s'" % body) - conn.close() - - def test_readall_or_close(self): - self.httpserver.protocol = 'HTTP/1.1' - self.PROTOCOL = 'HTTP/1.1' - - if self.scheme == 'https': - self.HTTP_CONN = HTTPSConnection + body = response.read(13) + assert response.status == 200 + assert body == b'Hello, world!' + + # Retrieve final response + response = conn.response_class(conn.sock, method='GET') + response.begin() + body = response.read() + assert response.status == 200 + assert body == b'Hello, world!' + + conn.close() + + +def test_100_Continue(test_client): + """Test 100-continue header processing.""" + conn = test_client.get_connection() + + # Try a page without an Expect request header first. + # Note that http.client's response.begin automatically ignores + # 100 Continue responses, so we must manually check for it. + conn.putrequest('POST', '/upload', skip_host=True) + conn.putheader('Host', conn.host) + conn.putheader('Content-Type', 'text/plain') + conn.putheader('Content-Length', '4') + conn.endheaders() + conn.send(b"d'oh") + response = conn.response_class(conn.sock, method='POST') + version, status, reason = response._read_status() + assert status != 100 + conn.close() + + # Now try a page with an Expect header... + conn.connect() + conn.putrequest('POST', '/upload', skip_host=True) + conn.putheader('Host', conn.host) + conn.putheader('Content-Type', 'text/plain') + conn.putheader('Content-Length', '17') + conn.putheader('Expect', '100-continue') + conn.endheaders() + response = conn.response_class(conn.sock, method='POST') + + # ...assert and then skip the 100 response + version, status, reason = response._read_status() + assert status == 100 + while True: + line = response.fp.readline().strip() + if line: + pytest.fail( + '100 Continue should not output any headers. Got %r' % + line, + ) else: - self.HTTP_CONN = HTTPConnection - - # Test a max of 0 (the default) and then reset to what it was above. - old_max = self.httpserver.max_request_body_size - for new_max in (0, old_max): - self.httpserver.max_request_body_size = new_max - - self.persistent = True - conn = self.HTTP_CONN - - # Get a POST page with an error - conn.putrequest('POST', '/err_before_read', skip_host=True) - conn.putheader('Host', self.HOST) - conn.putheader('Content-Type', 'text/plain') - conn.putheader('Content-Length', '1000') - conn.putheader('Expect', '100-continue') - conn.endheaders() - response = conn.response_class(conn.sock, method='POST') - - # ...assert and then skip the 100 response - version, status, reason = response._read_status() - self.assertEqual(status, 100) - while True: - skip = response.fp.readline().strip() - if not skip: - break - - # ...send the body - conn.send(ntob('x' * 1000)) - - # ...get the final response - response.begin() - self.status, self.headers, self.body = webtest.shb(response) - self.assertStatus(500) - - # Now try a working page with an Expect header... - conn._output(b'POST /upload HTTP/1.1') - conn._output(ntob('Host: %s' % self.HOST, 'ascii')) - conn._output(b'Content-Type: text/plain') - conn._output(b'Content-Length: 17') - conn._output(b'Expect: 100-continue') - conn._send_output() - response = conn.response_class(conn.sock, method='POST') - - # ...assert and then skip the 100 response - version, status, reason = response._read_status() - self.assertEqual(status, 100) - while True: - skip = response.fp.readline().strip() - if not skip: - break - - # ...send the body - body = b'I am a small file' - conn.send(body) - - # ...get the final response - response.begin() - self.status, self.headers, self.body = webtest.shb(response) - self.assertStatus(200) - self.assertBody("thanks for '%s'" % body) + break + + # ...send the body + body = b'I am a small file' + conn.send(body) + + # ...get the final response + response.begin() + status_line, actual_headers, actual_resp_body = webtest.shb(response) + actual_status = int(status_line[:3]) + assert actual_status == 200 + expected_resp_body = ("thanks for '%s'" % body).encode() + assert actual_resp_body == expected_resp_body + conn.close() + + +@pytest.mark.parametrize( + 'max_request_body_size', + ( + 0, + 1001, + ), +) +def test_readall_or_close(test_client, max_request_body_size): + """Test a max_request_body_size of 0 (the default) and 1001.""" + old_max = test_client.server_instance.max_request_body_size + + test_client.server_instance.max_request_body_size = max_request_body_size + + conn = test_client.get_connection() + + # Get a POST page with an error + conn.putrequest('POST', '/err_before_read', skip_host=True) + conn.putheader('Host', conn.host) + conn.putheader('Content-Type', 'text/plain') + conn.putheader('Content-Length', '1000') + conn.putheader('Expect', '100-continue') + conn.endheaders() + response = conn.response_class(conn.sock, method='POST') + + # ...assert and then skip the 100 response + version, status, reason = response._read_status() + assert status == 100 + skip = True + while skip: + skip = response.fp.readline().strip() + + # ...send the body + conn.send(b'x' * 1000) + + # ...get the final response + response.begin() + status_line, actual_headers, actual_resp_body = webtest.shb(response) + actual_status = int(status_line[:3]) + assert actual_status == 500 + + # Now try a working page with an Expect header... + conn._output(b'POST /upload HTTP/1.1') + conn._output(('Host: %s' % conn.host).encode('ascii')) + conn._output(b'Content-Type: text/plain') + conn._output(b'Content-Length: 17') + conn._output(b'Expect: 100-continue') + conn._send_output() + response = conn.response_class(conn.sock, method='POST') + + # ...assert and then skip the 100 response + version, status, reason = response._read_status() + assert status == 100 + skip = True + while skip: + skip = response.fp.readline().strip() + + # ...send the body + body = b'I am a small file' + conn.send(body) + + # ...get the final response + response.begin() + status_line, actual_headers, actual_resp_body = webtest.shb(response) + actual_status = int(status_line[:3]) + assert actual_status == 200 + expected_resp_body = ("thanks for '%s'" % body).encode() + assert actual_resp_body == expected_resp_body + conn.close() + + test_client.server_instance.max_request_body_size = old_max + + +def test_No_Message_Body(test_client): + """Test HTTP queries with an empty response body.""" + # Initialize a persistent HTTP connection + http_connection = test_client.get_connection() + http_connection.auto_open = False + http_connection.connect() + + # Make the first request and assert there's no "Connection: close". + status_line, actual_headers, actual_resp_body = test_client.get( + '/pov', http_conn=http_connection, + ) + actual_status = int(status_line[:3]) + assert actual_status == 200 + assert status_line[4:] == 'OK' + assert actual_resp_body == pov.encode() + assert not header_exists('Connection', actual_headers) + + # Make a 204 request on the same connection. + status_line, actual_headers, actual_resp_body = test_client.get( + '/custom/204', http_conn=http_connection, + ) + actual_status = int(status_line[:3]) + assert actual_status == 204 + assert not header_exists('Content-Length', actual_headers) + assert actual_resp_body == b'' + assert not header_exists('Connection', actual_headers) + + # Make a 304 request on the same connection. + status_line, actual_headers, actual_resp_body = test_client.get( + '/custom/304', http_conn=http_connection, + ) + actual_status = int(status_line[:3]) + assert actual_status == 304 + assert not header_exists('Content-Length', actual_headers) + assert actual_resp_body == b'' + assert not header_exists('Connection', actual_headers) + + +@pytest.mark.xfail( + reason=unwrap( + trim(""" + Headers from earlier request leak into the request + line for a subsequent request, resulting in 400 + instead of 413. See cherrypy/cheroot#69 for details. + """), + ), +) +def test_Chunked_Encoding(test_client): + """Test HTTP uploads with chunked transfer-encoding.""" + # Initialize a persistent HTTP connection + conn = test_client.get_connection() + + # Try a normal chunked request (with extensions) + body = ( + b'8;key=value\r\nxx\r\nxxxx\r\n5\r\nyyyyy\r\n0\r\n' + b'Content-Type: application/json\r\n' + b'\r\n' + ) + conn.putrequest('POST', '/upload', skip_host=True) + conn.putheader('Host', conn.host) + conn.putheader('Transfer-Encoding', 'chunked') + conn.putheader('Trailer', 'Content-Type') + # Note that this is somewhat malformed: + # we shouldn't be sending Content-Length. + # RFC 2616 says the server should ignore it. + conn.putheader('Content-Length', '3') + conn.endheaders() + conn.send(body) + response = conn.getresponse() + status_line, actual_headers, actual_resp_body = webtest.shb(response) + actual_status = int(status_line[:3]) + assert actual_status == 200 + assert status_line[4:] == 'OK' + expected_resp_body = ("thanks for '%s'" % b'xx\r\nxxxxyyyyy').encode() + assert actual_resp_body == expected_resp_body + + # Try a chunked request that exceeds server.max_request_body_size. + # Note that the delimiters and trailer are included. + body = b'\r\n'.join((b'3e3', b'x' * 995, b'0', b'', b'')) + conn.putrequest('POST', '/upload', skip_host=True) + conn.putheader('Host', conn.host) + conn.putheader('Transfer-Encoding', 'chunked') + conn.putheader('Content-Type', 'text/plain') + # Chunked requests don't need a content-length + # conn.putheader("Content-Length", len(body)) + conn.endheaders() + conn.send(body) + response = conn.getresponse() + status_line, actual_headers, actual_resp_body = webtest.shb(response) + actual_status = int(status_line[:3]) + assert actual_status == 413 + conn.close() + + +def test_Content_Length_in(test_client): + """Try a non-chunked request where Content-Length exceeds limit. + + (server.max_request_body_size). + Assert error before body send. + """ + # Initialize a persistent HTTP connection + conn = test_client.get_connection() + + conn.putrequest('POST', '/upload', skip_host=True) + conn.putheader('Host', conn.host) + conn.putheader('Content-Type', 'text/plain') + conn.putheader('Content-Length', '9999') + conn.endheaders() + response = conn.getresponse() + status_line, actual_headers, actual_resp_body = webtest.shb(response) + actual_status = int(status_line[:3]) + assert actual_status == 413 + expected_resp_body = ( + b'The entity sent with the request exceeds ' + b'the maximum allowed bytes.' + ) + assert actual_resp_body == expected_resp_body + conn.close() + + +def test_Content_Length_not_int(test_client): + """Test that malicious Content-Length header returns 400.""" + status_line, actual_headers, actual_resp_body = test_client.post( + '/upload', + headers=[ + ('Content-Type', 'text/plain'), + ('Content-Length', 'not-an-integer'), + ], + ) + actual_status = int(status_line[:3]) + + assert actual_status == 400 + assert actual_resp_body == b'Malformed Content-Length Header.' + + +@pytest.mark.parametrize( + ('uri', 'expected_resp_status', 'expected_resp_body'), + ( + ( + '/wrong_cl_buffered', 500, + ( + b'The requested resource returned more bytes than the ' + b'declared Content-Length.' + ), + ), + ('/wrong_cl_unbuffered', 200, b'I too'), + ), +) +def test_Content_Length_out( + test_client, + uri, expected_resp_status, expected_resp_body, +): + """Test response with Content-Length less than the response body. + + (non-chunked response) + """ + conn = test_client.get_connection() + conn.putrequest('GET', uri, skip_host=True) + conn.putheader('Host', conn.host) + conn.endheaders() + + response = conn.getresponse() + status_line, actual_headers, actual_resp_body = webtest.shb(response) + actual_status = int(status_line[:3]) + + assert actual_status == expected_resp_status + assert actual_resp_body == expected_resp_body + + conn.close() + + # the server logs the exception that we had verified from the + # client perspective. Tell the error_log verification that + # it can ignore that message. + test_client.server_instance.error_log.ignored_msgs.append( + "ValueError('Response body exceeds the declared Content-Length.')", + ) + + +@pytest.mark.xfail( + reason='Sometimes this test fails due to low timeout. ' + 'Ref: https://github.com/cherrypy/cherrypy/issues/598', +) +def test_598(test_client): + """Test serving large file with a read timeout in place.""" + # Initialize a persistent HTTP connection + conn = test_client.get_connection() + remote_data_conn = urllib.request.urlopen( + '%s://%s:%s/one_megabyte_of_a' + % ('http', conn.host, conn.port), + ) + buf = remote_data_conn.read(512) + time.sleep(timeout * 0.6) + remaining = (1024 * 1024) - 512 + while remaining: + data = remote_data_conn.read(remaining) + if not data: + break + buf += data + remaining -= len(data) + + assert len(buf) == 1024 * 1024 + assert buf == b'a' * 1024 * 1024 + assert remaining == 0 + remote_data_conn.close() + + +@pytest.mark.parametrize( + 'invalid_terminator', + ( + b'\n\n', + b'\r\n\n', + ), +) +def test_No_CRLF(test_client, invalid_terminator): + """Test HTTP queries with no valid CRLF terminators.""" + # Initialize a persistent HTTP connection + conn = test_client.get_connection() + + # (b'%s' % b'') is not supported in Python 3.4, so just use bytes.join() + conn.send(b''.join((b'GET /hello HTTP/1.1', invalid_terminator))) + response = conn.response_class(conn.sock, method='GET') + response.begin() + actual_resp_body = response.read() + expected_resp_body = b'HTTP requires CRLF terminators' + assert actual_resp_body == expected_resp_body + conn.close() + + +class FaultySelect: + """Mock class to insert errors in the selector.select method.""" + + def __init__(self, original_select): + """Initilize helper class to wrap the selector.select method.""" + self.original_select = original_select + self.request_served = False + self.os_error_triggered = False + + def __call__(self, timeout): + """Intercept the calls to selector.select.""" + if self.request_served: + self.os_error_triggered = True + raise OSError('Error while selecting the client socket.') + + return self.original_select(timeout) + + +class FaultyGetMap: + """Mock class to insert errors in the selector.get_map method.""" + + def __init__(self, original_get_map): + """Initilize helper class to wrap the selector.get_map method.""" + self.original_get_map = original_get_map + self.sabotage_conn = False + self.socket_closed = False + + def __call__(self): + """Intercept the calls to selector.get_map.""" + sabotage_targets = ( + conn for _, (*_, conn) in self.original_get_map().items() + if isinstance(conn, cheroot.server.HTTPConnection) + ) if self.sabotage_conn else () + + for conn in sabotage_targets: + # close the socket to cause OSError conn.close() - - def test_No_Message_Body(self): - self.PROTOCOL = 'HTTP/1.1' - - # Set our HTTP_CONN to an instance so it persists between requests. - self.persistent = True - - # Make the first request and assert there's no "Connection: close". - self.getPage('/pov') - self.assertStatus('200 OK') - self.assertBody(pov) - self.assertNoHeader('Connection') - - # Make a 204 request on the same connection. - self.getPage('/custom/204') - self.assertStatus(204) - self.assertNoHeader('Content-Length') - self.assertBody('') - self.assertNoHeader('Connection') - - # Make a 304 request on the same connection. - self.getPage('/custom/304') - self.assertStatus(304) - self.assertNoHeader('Content-Length') - self.assertBody('') - self.assertNoHeader('Connection') - - def _test_Chunked_Encoding(self): - self.httpserver.protocol = 'HTTP/1.1' - self.PROTOCOL = 'HTTP/1.1' - - # Set our HTTP_CONN to an instance so it persists between requests. - self.persistent = True - conn = self.HTTP_CONN - - # Try a normal chunked request (with extensions) - body = (b'8;key=value\r\nxx\r\nxxxx\r\n5\r\nyyyyy\r\n0\r\n' - b'Content-Type: application/json\r\n' - b'\r\n') - conn.putrequest('POST', '/upload', skip_host=True) - conn.putheader('Host', self.HOST) - conn.putheader('Transfer-Encoding', 'chunked') - conn.putheader('Trailer', 'Content-Type') - # Note that this is somewhat malformed: - # we shouldn't be sending Content-Length. - # RFC 2616 says the server should ignore it. - conn.putheader('Content-Length', '3') - conn.endheaders() - conn.send(body) - response = conn.getresponse() - self.status, self.headers, self.body = webtest.shb(response) - self.assertStatus('200 OK') - self.assertBody("thanks for '%s'" % b'xx\r\nxxxxyyyyy') - - # Try a chunked request that exceeds server.max_request_body_size. - # Note that the delimiters and trailer are included. - body = b'3e3\r\n' + (b'x' * 995) + b'\r\n0\r\n\r\n' - conn.putrequest('POST', '/upload', skip_host=True) - conn.putheader('Host', self.HOST) - conn.putheader('Transfer-Encoding', 'chunked') - conn.putheader('Content-Type', 'text/plain') - # Chunked requests don't need a content-length - # conn.putheader("Content-Length", len(body)) - conn.endheaders() - conn.send(body) - response = conn.getresponse() - self.status, self.headers, self.body = webtest.shb(response) - self.assertStatus(413) - conn.close() - - def test_Content_Length_in(self): - # Try a non-chunked request where Content-Length exceeds - # server.max_request_body_size. Assert error before body send. - self.httpserver.protocol = 'HTTP/1.1' - self.PROTOCOL = 'HTTP/1.1' - self.persistent = True - conn = self.HTTP_CONN - conn.putrequest('POST', '/upload', skip_host=True) - conn.putheader('Host', self.HOST) - conn.putheader('Content-Type', 'text/plain') - conn.putheader('Content-Length', '9999') - conn.endheaders() - response = conn.getresponse() - self.status, self.headers, self.body = webtest.shb(response) - self.assertStatus(413) - self.assertBody('The entity sent with the request exceeds ' - 'the maximum allowed bytes.') - conn.close() - - def test_Content_Length_out_preheaders(self): - # Try a non-chunked response where Content-Length is less than - # the actual bytes in the response body. - self.httpserver.protocol = 'HTTP/1.1' - self.PROTOCOL = 'HTTP/1.1' - self.persistent = True - conn = self.HTTP_CONN - conn.putrequest('GET', '/wrong_cl_buffered', skip_host=True) - conn.putheader('Host', self.HOST) - conn.endheaders() - response = conn.getresponse() - self.status, self.headers, self.body = webtest.shb(response) - self.assertStatus(500) - self.assertBody( - 'The requested resource returned more bytes than the ' - 'declared Content-Length.') - conn.close() - - def test_Content_Length_out_postheaders(self): - # Try a non-chunked response where Content-Length is less than - # the actual bytes in the response body. - self.httpserver.protocol = 'HTTP/1.1' - self.PROTOCOL = 'HTTP/1.1' - self.persistent = True - conn = self.HTTP_CONN - conn.putrequest('GET', '/wrong_cl_unbuffered', skip_host=True) - conn.putheader('Host', self.HOST) - conn.endheaders() - response = conn.getresponse() - self.status, self.headers, self.body = webtest.shb(response) - self.assertStatus(200) - self.assertBody('I too') - conn.close() - - @pytest.mark.xfail(reason='Sometimes this test fails due to low timeout') - def test_598(self): - self.httpserver.protocol = 'HTTP/1.1' - self.PROTOCOL = 'HTTP/1.1' - remote_data_conn = urlopen('%s://%s:%s/one_megabyte_of_a' % - (self.scheme, self.HOST, self.PORT)) - buf = remote_data_conn.read(512) - time.sleep(timeout * 0.6) - remaining = (1024 * 1024) - 512 - while remaining: - data = remote_data_conn.read(remaining) - if not data: - break - else: - buf += data - remaining -= len(data) - - self.assertEqual(len(buf), 1024 * 1024) - self.assertEqual(buf, b'a' * 1024 * 1024) - self.assertEqual(remaining, 0) - remote_data_conn.close() - - def test_No_CRLF(self): - self.httpserver.protocol = 'HTTP/1.1' - self.PROTOCOL = 'HTTP/1.1' - self.persistent = True - - conn = self.HTTP_CONN - conn.send(b'GET /hello HTTP/1.1\n\n') - response = conn.response_class(conn.sock, method='GET') - response.begin() - self.body = response.read() - self.assertBody('HTTP requires CRLF terminators') - conn.close() - - conn.connect() - conn.send(b'GET /hello HTTP/1.1\r\n\n') - response = conn.response_class(conn.sock, method='GET') - response.begin() - self.body = response.read() - self.assertBody('HTTP requires CRLF terminators') - conn.close() + self.socket_closed = True + + return self.original_get_map() + + +def test_invalid_selected_connection(test_client, monkeypatch): + """Test the error handling segment of HTTP connection selection. + + See :py:meth:`cheroot.connections.ConnectionManager.get_conn`. + """ + # patch the select method + faux_select = FaultySelect( + test_client.server_instance._connections._selector.select, + ) + monkeypatch.setattr( + test_client.server_instance._connections._selector, + 'select', + faux_select, + ) + + # patch the get_map method + faux_get_map = FaultyGetMap( + test_client.server_instance._connections._selector.get_map, + ) + + monkeypatch.setattr( + test_client.server_instance._connections._selector, + 'get_map', + faux_get_map, + ) + + # request a page with connection keep-alive to make sure + # we'll have a connection to be modified. + resp_status, resp_headers, resp_body = test_client.request( + '/page1', headers=[('Connection', 'Keep-Alive')], + ) + + assert resp_status == '200 OK' + # trigger the internal errors + faux_get_map.sabotage_conn = faux_select.request_served = True + # give time to make sure the error gets handled + time.sleep(0.2) + assert faux_select.os_error_triggered + assert faux_get_map.socket_closed + # any error in the error handling should be catched by the + # teardown verification for the error_log diff --git a/lib/cheroot/test/test_core.py b/lib/cheroot/test/test_core.py index 23368f9..e7a65c9 100644 --- a/lib/cheroot/test/test_core.py +++ b/lib/cheroot/test/test_core.py @@ -1,134 +1,458 @@ """Tests for managing HTTP issues (malformed requests, etc).""" +# -*- coding: utf-8 -*- +# vim: set fileencoding=utf-8 : + +from __future__ import absolute_import, division, print_function +__metaclass__ = type import errno +import os import socket -from cheroot._compat import HTTPConnection, HTTPSConnection +import pytest +import six +from six.moves import urllib from cheroot.test import helper -class HTTPTests(helper.CherootWebCase): - - def setup_server(cls): - class Root(helper.Controller): - - def hello(self, req, resp): - return 'Hello world!' - - def no_body(self, req, resp): - return 'Hello world!' - - def body_required(self, req, resp): - if req.environ.get('Content-Length', None) is None: - resp.status = '411 Length Required' - return - return 'Hello world!' - - cls.httpserver.wsgi_app = Root() - cls.httpserver.max_request_body_size = 30000000 - setup_server = classmethod(setup_server) - - def test_normal_request(self): - self.getPage('/hello') - self.assertStatus(200) - self.assertBody(b'Hello world!') - - def test_no_content_length(self): - # "The presence of a message-body in a request is signaled by the - # inclusion of a Content-Length or Transfer-Encoding header field in - # the request's message-headers." - # - # Send a message with neither header and no body. - if self.scheme == 'https': - c = HTTPSConnection('%s:%s' % (self.interface(), self.PORT)) - else: - c = HTTPConnection('%s:%s' % (self.interface(), self.PORT)) - c.request('POST', '/no_body') - response = c.getresponse() - self.body = response.fp.read() - self.status = str(response.status) - self.assertStatus(200) - self.assertBody(b'Hello world!') - - def test_content_length_required(self): - # Now send a message that has no Content-Length, but does send a body. - # Verify that CP times out the socket and responds - # with 411 Length Required. - - if self.scheme == 'https': - c = HTTPSConnection('%s:%s' % (self.interface(), self.PORT)) - else: - c = HTTPConnection('%s:%s' % (self.interface(), self.PORT)) - c.request('POST', '/body_required') - response = c.getresponse() - self.body = response.fp.read() - - self.status = str(response.status) - self.assertStatus(411) - - def test_malformed_request_line(self): - # Test missing version in Request-Line - - if self.scheme == 'https': - c = HTTPSConnection('%s:%s' % (self.interface(), self.PORT)) - else: - c = HTTPConnection('%s:%s' % (self.interface(), self.PORT)) - c._output(b'GET /') - c._send_output() - if hasattr(c, 'strict'): - response = c.response_class(c.sock, strict=c.strict, method='GET') - else: - # Python 3.2 removed the 'strict' feature, saying: - # "http.client now always assumes HTTP/1.x compliant servers." - response = c.response_class(c.sock, method='GET') +IS_CIRCLE_CI_ENV = 'CIRCLECI' in os.environ + + +HTTP_BAD_REQUEST = 400 +HTTP_LENGTH_REQUIRED = 411 +HTTP_NOT_FOUND = 404 +HTTP_REQUEST_ENTITY_TOO_LARGE = 413 +HTTP_OK = 200 +HTTP_VERSION_NOT_SUPPORTED = 505 + + +class HelloController(helper.Controller): + """Controller for serving WSGI apps.""" + + def hello(req, resp): + """Render Hello world.""" + return 'Hello world!' + + def body_required(req, resp): + """Render Hello world or set 411.""" + if req.environ.get('Content-Length', None) is None: + resp.status = '411 Length Required' + return + return 'Hello world!' + + def query_string(req, resp): + """Render QUERY_STRING value.""" + return req.environ.get('QUERY_STRING', '') + + def asterisk(req, resp): + """Render request method value.""" + method = req.environ.get('REQUEST_METHOD', 'NO METHOD FOUND') + tmpl = 'Got asterisk URI path with {method} method' + return tmpl.format(**locals()) + + def _munge(string): + """Encode PATH_INFO correctly depending on Python version. + + WSGI 1.0 is a mess around unicode. Create endpoints + that match the PATH_INFO that it produces. + """ + if six.PY2: + return string + return string.encode('utf-8').decode('latin-1') + + handlers = { + '/hello': hello, + '/no_body': hello, + '/body_required': body_required, + '/query_string': query_string, + _munge('/привіт'): hello, + _munge('/Юххууу'): hello, + '/\xa0Ðblah key 0 900 4 data': hello, + '/*': asterisk, + } + + +def _get_http_response(connection, method='GET'): + c = connection + kwargs = {'strict': c.strict} if hasattr(c, 'strict') else {} + # Python 3.2 removed the 'strict' feature, saying: + # "http.client now always assumes HTTP/1.x compliant servers." + return c.response_class(c.sock, method=method, **kwargs) + + +@pytest.fixture +def testing_server(wsgi_server_client): + """Attach a WSGI app to the given server and preconfigure it.""" + wsgi_server = wsgi_server_client.server_instance + wsgi_server.wsgi_app = HelloController() + wsgi_server.max_request_body_size = 30000000 + wsgi_server.server_client = wsgi_server_client + return wsgi_server + + +@pytest.fixture +def test_client(testing_server): + """Get and return a test client out of the given server.""" + return testing_server.server_client + + +@pytest.fixture +def testing_server_with_defaults(wsgi_server_client): + """Attach a WSGI app to the given server and preconfigure it.""" + wsgi_server = wsgi_server_client.server_instance + wsgi_server.wsgi_app = HelloController() + wsgi_server.server_client = wsgi_server_client + return wsgi_server + + +@pytest.fixture +def test_client_with_defaults(testing_server_with_defaults): + """Get and return a test client out of the given server.""" + return testing_server_with_defaults.server_client + + +def test_http_connect_request(test_client): + """Check that CONNECT query results in Method Not Allowed status.""" + status_line = test_client.connect('/anything')[0] + actual_status = int(status_line[:3]) + assert actual_status == 405 + + +def test_normal_request(test_client): + """Check that normal GET query succeeds.""" + status_line, _, actual_resp_body = test_client.get('/hello') + actual_status = int(status_line[:3]) + assert actual_status == HTTP_OK + assert actual_resp_body == b'Hello world!' + + +def test_query_string_request(test_client): + """Check that GET param is parsed well.""" + status_line, _, actual_resp_body = test_client.get( + '/query_string?test=True', + ) + actual_status = int(status_line[:3]) + assert actual_status == HTTP_OK + assert actual_resp_body == b'test=True' + + +@pytest.mark.parametrize( + 'uri', + ( + '/hello', # plain + '/query_string?test=True', # query + '/{0}?{1}={2}'.format( # quoted unicode + *map(urllib.parse.quote, ('Юххууу', 'ї', 'йо')) + ), + ), +) +def test_parse_acceptable_uri(test_client, uri): + """Check that server responds with OK to valid GET queries.""" + status_line = test_client.get(uri)[0] + actual_status = int(status_line[:3]) + assert actual_status == HTTP_OK + + +@pytest.mark.xfail(six.PY2, reason='Fails on Python 2') +def test_parse_uri_unsafe_uri(test_client): + """Test that malicious URI does not allow HTTP injection. + + This effectively checks that sending GET request with URL + + /%A0%D0blah%20key%200%20900%204%20data + + is not converted into + + GET / + blah key 0 900 4 data + HTTP/1.1 + + which would be a security issue otherwise. + """ + c = test_client.get_connection() + resource = '/\xa0Ðblah key 0 900 4 data'.encode('latin-1') + quoted = urllib.parse.quote(resource) + assert quoted == '/%A0%D0blah%20key%200%20900%204%20data' + request = 'GET {quoted} HTTP/1.1'.format(**locals()) + c._output(request.encode('utf-8')) + c._send_output() + response = _get_http_response(c, method='GET') + response.begin() + assert response.status == HTTP_OK + assert response.read(12) == b'Hello world!' + c.close() + + +def test_parse_uri_invalid_uri(test_client): + """Check that server responds with Bad Request to invalid GET queries. + + Invalid request line test case: it should only contain US-ASCII. + """ + c = test_client.get_connection() + c._output(u'GET /йопта! HTTP/1.1'.encode('utf-8')) + c._send_output() + response = _get_http_response(c, method='GET') + response.begin() + assert response.status == HTTP_BAD_REQUEST + assert response.read(21) == b'Malformed Request-URI' + c.close() + + +@pytest.mark.parametrize( + 'uri', + ( + 'hello', # ascii + 'привіт', # non-ascii + ), +) +def test_parse_no_leading_slash_invalid(test_client, uri): + """Check that server responds with Bad Request to invalid GET queries. + + Invalid request line test case: it should have leading slash (be absolute). + """ + status_line, _, actual_resp_body = test_client.get( + urllib.parse.quote(uri), + ) + actual_status = int(status_line[:3]) + assert actual_status == HTTP_BAD_REQUEST + assert b'starting with a slash' in actual_resp_body + + +def test_parse_uri_absolute_uri(test_client): + """Check that server responds with Bad Request to Absolute URI. + + Only proxy servers should allow this. + """ + status_line, _, actual_resp_body = test_client.get('http://google.com/') + actual_status = int(status_line[:3]) + assert actual_status == HTTP_BAD_REQUEST + expected_body = b'Absolute URI not allowed if server is not a proxy.' + assert actual_resp_body == expected_body + + +def test_parse_uri_asterisk_uri(test_client): + """Check that server responds with OK to OPTIONS with "*" Absolute URI.""" + status_line, _, actual_resp_body = test_client.options('*') + actual_status = int(status_line[:3]) + assert actual_status == HTTP_OK + expected_body = b'Got asterisk URI path with OPTIONS method' + assert actual_resp_body == expected_body + + +def test_parse_uri_fragment_uri(test_client): + """Check that server responds with Bad Request to URI with fragment.""" + status_line, _, actual_resp_body = test_client.get( + '/hello?test=something#fake', + ) + actual_status = int(status_line[:3]) + assert actual_status == HTTP_BAD_REQUEST + expected_body = b'Illegal #fragment in Request-URI.' + assert actual_resp_body == expected_body + + +def test_no_content_length(test_client): + """Test POST query with an empty body being successful.""" + # "The presence of a message-body in a request is signaled by the + # inclusion of a Content-Length or Transfer-Encoding header field in + # the request's message-headers." + # + # Send a message with neither header and no body. + c = test_client.get_connection() + c.request('POST', '/no_body') + response = c.getresponse() + actual_resp_body = response.read() + actual_status = response.status + assert actual_status == HTTP_OK + assert actual_resp_body == b'Hello world!' + + +def test_content_length_required(test_client): + """Test POST query with body failing because of missing Content-Length.""" + # Now send a message that has no Content-Length, but does send a body. + # Verify that CP times out the socket and responds + # with 411 Length Required. + + c = test_client.get_connection() + c.request('POST', '/body_required') + response = c.getresponse() + response.read() + + actual_status = response.status + assert actual_status == HTTP_LENGTH_REQUIRED + + +@pytest.mark.xfail( + IS_CIRCLE_CI_ENV, + reason='https://github.com/cherrypy/cheroot/issues/106', + strict=False, # sometimes it passes +) +def test_large_request(test_client_with_defaults): + """Test GET query with maliciously large Content-Length.""" + # If the server's max_request_body_size is not set (i.e. is set to 0) + # then this will result in an `OverflowError: Python int too large to + # convert to C ssize_t` in the server. + # We expect that this should instead return that the request is too + # large. + c = test_client_with_defaults.get_connection() + c.putrequest('GET', '/hello') + c.putheader('Content-Length', str(2**64)) + c.endheaders() + + response = c.getresponse() + actual_status = response.status + + assert actual_status == HTTP_REQUEST_ENTITY_TOO_LARGE + + +@pytest.mark.parametrize( + ('request_line', 'status_code', 'expected_body'), + ( + ( + b'GET /', # missing proto + HTTP_BAD_REQUEST, b'Malformed Request-Line', + ), + ( + b'GET / HTTPS/1.1', # invalid proto + HTTP_BAD_REQUEST, b'Malformed Request-Line: bad protocol', + ), + ( + b'GET / HTTP/1', # invalid version + HTTP_BAD_REQUEST, b'Malformed Request-Line: bad version', + ), + ( + b'GET / HTTP/2.15', # invalid ver + HTTP_VERSION_NOT_SUPPORTED, b'Cannot fulfill request', + ), + ), +) +def test_malformed_request_line( + test_client, request_line, + status_code, expected_body, +): + """Test missing or invalid HTTP version in Request-Line.""" + c = test_client.get_connection() + c._output(request_line) + c._send_output() + response = _get_http_response(c, method='GET') + response.begin() + assert response.status == status_code + assert response.read(len(expected_body)) == expected_body + c.close() + + +def test_malformed_http_method(test_client): + """Test non-uppercase HTTP method.""" + c = test_client.get_connection() + c.putrequest('GeT', '/malformed_method_case') + c.putheader('Content-Type', 'text/plain') + c.endheaders() + + response = c.getresponse() + actual_status = response.status + assert actual_status == HTTP_BAD_REQUEST + actual_resp_body = response.read(21) + assert actual_resp_body == b'Malformed method name' + + +def test_malformed_header(test_client): + """Check that broken HTTP header results in Bad Request.""" + c = test_client.get_connection() + c.putrequest('GET', '/') + c.putheader('Content-Type', 'text/plain') + # See https://www.bitbucket.org/cherrypy/cherrypy/issue/941 + c._output(b'Re, 1.2.3.4#015#012') + c.endheaders() + + response = c.getresponse() + actual_status = response.status + assert actual_status == HTTP_BAD_REQUEST + actual_resp_body = response.read(20) + assert actual_resp_body == b'Illegal header line.' + + +def test_request_line_split_issue_1220(test_client): + """Check that HTTP request line of exactly 256 chars length is OK.""" + Request_URI = ( + '/hello?' + 'intervenant-entreprise-evenement_classaction=' + 'evenement-mailremerciements' + '&_path=intervenant-entreprise-evenement' + '&intervenant-entreprise-evenement_action-id=19404' + '&intervenant-entreprise-evenement_id=19404' + '&intervenant-entreprise_id=28092' + ) + assert len('GET %s HTTP/1.1\r\n' % Request_URI) == 256 + + actual_resp_body = test_client.get(Request_URI)[2] + assert actual_resp_body == b'Hello world!' + + +def test_garbage_in(test_client): + """Test that server sends an error for garbage received over TCP.""" + # Connect without SSL regardless of server.scheme + + c = test_client.get_connection() + c._output(b'gjkgjklsgjklsgjkljklsg') + c._send_output() + response = c.response_class(c.sock, method='GET') + try: response.begin() - self.assertEqual(response.status, 400) - self.assertEqual(response.fp.read(22), b'Malformed Request-Line') + actual_status = response.status + assert actual_status == HTTP_BAD_REQUEST + actual_resp_body = response.read(22) + assert actual_resp_body == b'Malformed Request-Line' c.close() + except socket.error as ex: + # "Connection reset by peer" is also acceptable. + if ex.errno != errno.ECONNRESET: + raise + + +class CloseController: + """Controller for testing the close callback.""" + + def __call__(self, environ, start_response): + """Get the req to know header sent status.""" + self.req = start_response.__self__.req + resp = CloseResponse(self.close) + start_response(resp.status, resp.headers.items()) + return resp + + def close(self): + """Close, writing hello.""" + self.req.write(b'hello') + + +class CloseResponse: + """Dummy empty response to trigger the no body status.""" + + def __init__(self, close): + """Use some defaults to ensure we have a header.""" + self.status = '200 OK' + self.headers = {'Content-Type': 'text/html'} + self.close = close + + def __getitem__(self, index): + """Ensure we don't have a body.""" + raise IndexError() + + def output(self): + """Return self to hook the close method.""" + return self + + +@pytest.fixture +def testing_server_close(wsgi_server_client): + """Attach a WSGI app to the given server and preconfigure it.""" + wsgi_server = wsgi_server_client.server_instance + wsgi_server.wsgi_app = CloseController() + wsgi_server.max_request_body_size = 30000000 + wsgi_server.server_client = wsgi_server_client + return wsgi_server + - def test_malformed_header(self): - - if self.scheme == 'https': - c = HTTPSConnection('%s:%s' % (self.interface(), self.PORT)) - else: - c = HTTPConnection('%s:%s' % (self.interface(), self.PORT)) - c.putrequest('GET', '/') - c.putheader('Content-Type', 'text/plain') - # See http://www.bitbucket.org/cherrypy/cherrypy/issue/941 - c._output(b'Re, 1.2.3.4#015#012') - c.endheaders() - - response = c.getresponse() - self.status = str(response.status) - self.assertStatus(400) - self.body = response.fp.read(20) - self.assertBody('Illegal header line.') - - def test_request_line_split_issue_1220(self): - - Request_URI = ( - '/hello?intervenant-entreprise-evenement_classaction=evenement-mailremerciements' - '&_path=intervenant-entreprise-evenement&intervenant-entreprise-evenement_action-id=19404' - '&intervenant-entreprise-evenement_id=19404&intervenant-entreprise_id=28092' - ) - self.assertEqual(len('GET %s HTTP/1.1\r\n' % Request_URI), 256) - self.getPage(Request_URI) - self.assertBody('Hello world!') - - def test_garbage_in(self): - # Connect without SSL regardless of server.scheme - - c = HTTPConnection('%s:%s' % (self.interface(), self.PORT)) - c._output(b'gjkgjklsgjklsgjkljklsg') - c._send_output() - response = c.response_class(c.sock, method='GET') - try: - response.begin() - self.assertEqual(response.status, 400) - self.assertEqual(response.fp.read(22), b'Malformed Request-Line') - c.close() - except socket.error as ex: - # "Connection reset by peer" is also acceptable. - if ex.errno != errno.ECONNRESET: - raise +def test_send_header_before_closing(testing_server_close): + """Test we are actually sending the headers before calling 'close'.""" + _, _, resp_body = testing_server_close.server_client.get('/') + assert resp_body == b'hello' diff --git a/lib/cheroot/test/test_dispatch.py b/lib/cheroot/test/test_dispatch.py new file mode 100644 index 0000000..9974fda --- /dev/null +++ b/lib/cheroot/test/test_dispatch.py @@ -0,0 +1,55 @@ +"""Tests for the HTTP server.""" +# -*- coding: utf-8 -*- +# vim: set fileencoding=utf-8 : + +from __future__ import absolute_import, division, print_function + +from cheroot.wsgi import PathInfoDispatcher + + +def wsgi_invoke(app, environ): + """Serve 1 request from a WSGI application.""" + response = {} + + def start_response(status, headers): + response.update({ + 'status': status, + 'headers': headers, + }) + + response['body'] = b''.join( + app(environ, start_response), + ) + + return response + + +def test_dispatch_no_script_name(): + """Dispatch despite lack of ``SCRIPT_NAME`` in environ.""" + # Bare bones WSGI hello world app (from PEP 333). + def app(environ, start_response): + start_response( + '200 OK', [ + ('Content-Type', 'text/plain; charset=utf-8'), + ], + ) + return [u'Hello, world!'.encode('utf-8')] + + # Build a dispatch table. + d = PathInfoDispatcher([ + ('/', app), + ]) + + # Dispatch a request without `SCRIPT_NAME`. + response = wsgi_invoke( + d, { + 'PATH_INFO': '/foo', + }, + ) + assert response == { + 'status': '200 OK', + 'headers': [ + ('Content-Type', 'text/plain; charset=utf-8'), + ], + 'body': b'Hello, world!', + } diff --git a/lib/cheroot/test/test_errors.py b/lib/cheroot/test/test_errors.py new file mode 100644 index 0000000..469b70a --- /dev/null +++ b/lib/cheroot/test/test_errors.py @@ -0,0 +1,30 @@ +"""Test suite for ``cheroot.errors``.""" + +import pytest + +from cheroot import errors + +from .._compat import IS_LINUX, IS_MACOS, IS_WINDOWS + + +@pytest.mark.parametrize( + ('err_names', 'err_nums'), + ( + (('', 'some-nonsense-name'), []), + ( + ( + 'EPROTOTYPE', 'EAGAIN', 'EWOULDBLOCK', + 'WSAEWOULDBLOCK', 'EPIPE', + ), + (91, 11, 32) if IS_LINUX else + (32, 35, 41) if IS_MACOS else + (32, 10041, 11, 10035) if IS_WINDOWS else + (), + ), + ), +) +def test_plat_specific_errors(err_names, err_nums): + """Test that ``plat_specific_errors`` gets correct error numbers list.""" + actual_err_nums = errors.plat_specific_errors(*err_names) + assert len(actual_err_nums) == len(err_nums) + assert sorted(actual_err_nums) == sorted(err_nums) diff --git a/lib/cheroot/test/test_http.py b/lib/cheroot/test/test_http.py deleted file mode 100644 index 37bb8d9..0000000 --- a/lib/cheroot/test/test_http.py +++ /dev/null @@ -1,260 +0,0 @@ -"""Tests for managing HTTP issues (malformed requests, etc).""" - -import errno -import mimetypes -import socket -import sys -from unittest import mock - -import six - -from cheroot._compat import HTTPConnection, HTTPSConnection, ntob - -from cheroot.test import helper - -import pytest -pytestmark = pytest.mark.skip(reason='Depends on CherryPy') - - -def encode_multipart_formdata(files): - """Return (content_type, body) ready for httplib.HTTP instance. - - files: a sequence of (name, filename, value) tuples for multipart uploads. - """ - BOUNDARY = '________ThIs_Is_tHe_bouNdaRY_$' - L = [] - for key, filename, value in files: - L.append('--' + BOUNDARY) - L.append('Content-Disposition: form-data; name="%s"; filename="%s"' % - (key, filename)) - ct = mimetypes.guess_type(filename)[0] or 'application/octet-stream' - L.append('Content-Type: %s' % ct) - L.append('') - L.append(value) - L.append('--' + BOUNDARY + '--') - L.append('') - body = '\r\n'.join(L) - content_type = 'multipart/form-data; boundary=%s' % BOUNDARY - return content_type, body - - -class HTTPTests(helper.CherootWebCase): - - def make_connection(self): - if self.scheme == 'https': - return HTTPSConnection('%s:%s' % (self.interface(), self.PORT)) - else: - return HTTPConnection('%s:%s' % (self.interface(), self.PORT)) - - @staticmethod - def setup_server(): - class Root: - - @cherrypy.expose - def index(self, *args, **kwargs): - return 'Hello world!' - - @cherrypy.expose - @cherrypy.config(**{'request.process_request_body': False}) - def no_body(self, *args, **kwargs): - return 'Hello world!' - - @cherrypy.expose - def post_multipart(self, file): - r"""Return a summary ("a * 65536\nb * 65536") of the uploaded file.""" - contents = file.file.read() - summary = [] - curchar = None - count = 0 - for c in contents: - if c == curchar: - count += 1 - else: - if count: - if six.PY3: - curchar = chr(curchar) - summary.append('%s * %d' % (curchar, count)) - count = 1 - curchar = c - if count: - if six.PY3: - curchar = chr(curchar) - summary.append('%s * %d' % (curchar, count)) - return ', '.join(summary) - - @cherrypy.expose - def post_filename(self, myfile): - """Return the name of the file which was uploaded.""" - return myfile.filename - - cherrypy.tree.mount(Root()) - cherrypy.config.update({'server.max_request_body_size': 30000000}) - - def test_no_content_length(self): - # "The presence of a message-body in a request is signaled by the - # inclusion of a Content-Length or Transfer-Encoding header field in - # the request's message-headers." - # - # Send a message with neither header and no body. Even though - # the request is of method POST, this should be OK because we set - # request.process_request_body to False for our handler. - c = self.make_connection() - c.request('POST', '/no_body') - response = c.getresponse() - self.body = response.fp.read() - self.status = str(response.status) - self.assertStatus(200) - self.assertBody(ntob('Hello world!')) - - # Now send a message that has no Content-Length, but does send a body. - # Verify that CP times out the socket and responds - # with 411 Length Required. - if self.scheme == 'https': - c = HTTPSConnection('%s:%s' % (self.interface(), self.PORT)) - else: - c = HTTPConnection('%s:%s' % (self.interface(), self.PORT)) - - # `_get_content_length` is needed for Python 3.6+ - with mock.patch.object(c, '_get_content_length', lambda body, method: None, create=True): - # `_set_content_length` is needed for Python 2.7-3.5 - with mock.patch.object(c, '_set_content_length', create=True): - c.request('POST', '/') - - response = c.getresponse() - self.body = response.fp.read() - self.status = str(response.status) - self.assertStatus(411) - - def test_post_multipart(self): - alphabet = 'abcdefghijklmnopqrstuvwxyz' - # generate file contents for a large post - contents = ''.join([c * 65536 for c in alphabet]) - - # encode as multipart form data - files = [('file', 'file.txt', contents)] - content_type, body = encode_multipart_formdata(files) - body = body.encode('Latin-1') - - # post file - c = self.make_connection() - c.putrequest('POST', '/post_multipart') - c.putheader('Content-Type', content_type) - c.putheader('Content-Length', str(len(body))) - c.endheaders() - c.send(body) - - response = c.getresponse() - self.body = response.fp.read() - self.status = str(response.status) - self.assertStatus(200) - self.assertBody(', '.join(['%s * 65536' % c for c in alphabet])) - - def test_post_filename_with_special_characters(self): - """Test that we can handle filenames with special characters. - - This was reported as a bug in: - https://github.com/cherrypy/cherrypy/issues/1146 - https://github.com/cherrypy/cherrypy/issues/1397 - """ - # We'll upload a bunch of files with differing names. - fnames = ['boop.csv', 'foo, bar.csv', 'bar, xxxx.csv', 'file"name.csv', - 'file;name.csv', 'file; name.csv'] - for fname in fnames: - files = [('myfile', fname, 'yunyeenyunyue')] - content_type, body = encode_multipart_formdata(files) - body = body.encode('Latin-1') - - # post file - c = self.make_connection() - c.putrequest('POST', '/post_filename') - c.putheader('Content-Type', content_type) - c.putheader('Content-Length', str(len(body))) - c.endheaders() - c.send(body) - - response = c.getresponse() - self.body = response.fp.read() - self.status = str(response.status) - self.assertStatus(200) - self.assertBody(fname) - - def test_malformed_request_line(self): - if getattr(cherrypy.server, 'using_apache', False): - return self.skip('skipped due to known Apache differences...') - - # Test missing version in Request-Line - c = self.make_connection() - c._output(ntob('GET /')) - c._send_output() - if hasattr(c, 'strict'): - response = c.response_class(c.sock, strict=c.strict, method='GET') - else: - # Python 3.2 removed the 'strict' feature, saying: - # "http.client now always assumes HTTP/1.x compliant servers." - response = c.response_class(c.sock, method='GET') - response.begin() - self.assertEqual(response.status, 400) - self.assertEqual(response.fp.read(22), ntob('Malformed Request-Line')) - c.close() - - def test_request_line_split_issue_1220(self): - Request_URI = ( - '/index?intervenant-entreprise-evenement_classaction=evenement-mailremerciements' - '&_path=intervenant-entreprise-evenement&intervenant-entreprise-evenement_action-id=19404' - '&intervenant-entreprise-evenement_id=19404&intervenant-entreprise_id=28092' - ) - self.assertEqual(len('GET %s HTTP/1.1\r\n' % Request_URI), 256) - self.getPage(Request_URI) - self.assertBody('Hello world!') - - def test_malformed_header(self): - c = self.make_connection() - c.putrequest('GET', '/') - c.putheader('Content-Type', 'text/plain') - # See https://github.com/cherrypy/cherrypy/issues/941 - c._output(ntob('Re, 1.2.3.4#015#012')) - c.endheaders() - - response = c.getresponse() - self.status = str(response.status) - self.assertStatus(400) - self.body = response.fp.read(20) - self.assertBody('Illegal header line.') - - def test_http_over_https(self): - if self.scheme != 'https': - return self.skip('skipped (not running HTTPS)... ') - - # Try connecting without SSL. - conn = HTTPConnection('%s:%s' % (self.interface(), self.PORT)) - conn.putrequest('GET', '/', skip_host=True) - conn.putheader('Host', self.HOST) - conn.endheaders() - response = conn.response_class(conn.sock, method='GET') - try: - response.begin() - self.assertEqual(response.status, 400) - self.body = response.read() - self.assertBody('The client sent a plain HTTP request, but this ' - 'server only speaks HTTPS on this port.') - except socket.error as ex: - # "Connection reset by peer" is also acceptable. - if ex.errno != errno.ECONNRESET: - raise - - def test_garbage_in(self): - # Connect without SSL regardless of server.scheme - c = HTTPConnection('%s:%s' % (self.interface(), self.PORT)) - c._output(ntob('gjkgjklsgjklsgjkljklsg')) - c._send_output() - response = c.response_class(c.sock, method='GET') - try: - response.begin() - self.assertEqual(response.status, 400) - self.assertEqual(response.fp.read(22), - ntob('Malformed Request-Line')) - c.close() - except socket.error as ex: - # "Connection reset by peer" is also acceptable. - if ex.errno != errno.ECONNRESET: - raise diff --git a/lib/cheroot/test/test_makefile.py b/lib/cheroot/test/test_makefile.py new file mode 100644 index 0000000..cdded07 --- /dev/null +++ b/lib/cheroot/test/test_makefile.py @@ -0,0 +1,52 @@ +"""Tests for :py:mod:`cheroot.makefile`.""" + +from cheroot import makefile + + +__metaclass__ = type + + +class MockSocket: + """A mock socket.""" + + def __init__(self): + """Initialize :py:class:`MockSocket`.""" + self.messages = [] + + def recv_into(self, buf): + """Simulate ``recv_into`` for Python 3.""" + if not self.messages: + return 0 + msg = self.messages.pop(0) + for index, byte in enumerate(msg): + buf[index] = byte + return len(msg) + + def recv(self, size): + """Simulate ``recv`` for Python 2.""" + try: + return self.messages.pop(0) + except IndexError: + return '' + + def send(self, val): + """Simulate a send.""" + return len(val) + + +def test_bytes_read(): + """Reader should capture bytes read.""" + sock = MockSocket() + sock.messages.append(b'foo') + rfile = makefile.MakeFile(sock, 'r') + rfile.read() + assert rfile.bytes_read == 3 + + +def test_bytes_written(): + """Writer should capture bytes written.""" + sock = MockSocket() + sock.messages.append(b'foo') + wfile = makefile.MakeFile(sock, 'w') + wfile.write(b'bar') + assert wfile.bytes_written == 3 diff --git a/lib/cheroot/test/test_server.py b/lib/cheroot/test/test_server.py new file mode 100644 index 0000000..7b0ef91 --- /dev/null +++ b/lib/cheroot/test/test_server.py @@ -0,0 +1,342 @@ +"""Tests for the HTTP server.""" +# -*- coding: utf-8 -*- +# vim: set fileencoding=utf-8 : + +from __future__ import absolute_import, division, print_function +__metaclass__ = type + +from contextlib import closing +import os +import socket +import tempfile +import threading +import uuid + +import pytest +import requests +import requests_unixsocket +import six + +from six.moves import urllib + +from .._compat import bton, ntob +from .._compat import IS_LINUX, IS_MACOS, IS_WINDOWS, SYS_PLATFORM +from ..server import IS_UID_GID_RESOLVABLE, Gateway, HTTPServer +from ..testing import ( + ANY_INTERFACE_IPV4, + ANY_INTERFACE_IPV6, + EPHEMERAL_PORT, +) + + +unix_only_sock_test = pytest.mark.skipif( + not hasattr(socket, 'AF_UNIX'), + reason='UNIX domain sockets are only available under UNIX-based OS', +) + + +non_macos_sock_test = pytest.mark.skipif( + IS_MACOS, + reason='Peercreds lookup does not work under macOS/BSD currently.', +) + + +@pytest.fixture(params=('abstract', 'file')) +def unix_sock_file(request): + """Check that bound UNIX socket address is stored in server.""" + name = 'unix_{request.param}_sock'.format(**locals()) + return request.getfixturevalue(name) + + +@pytest.fixture +def unix_abstract_sock(): + """Return an abstract UNIX socket address.""" + if not IS_LINUX: + pytest.skip( + '{os} does not support an abstract ' + 'socket namespace'.format(os=SYS_PLATFORM), + ) + return b''.join(( + b'\x00cheroot-test-socket', + ntob(str(uuid.uuid4())), + )).decode() + + +@pytest.fixture +def unix_file_sock(): + """Yield a unix file socket.""" + tmp_sock_fh, tmp_sock_fname = tempfile.mkstemp() + + yield tmp_sock_fname + + os.close(tmp_sock_fh) + os.unlink(tmp_sock_fname) + + +def test_prepare_makes_server_ready(): + """Check that prepare() makes the server ready, and stop() clears it.""" + httpserver = HTTPServer( + bind_addr=(ANY_INTERFACE_IPV4, EPHEMERAL_PORT), + gateway=Gateway, + ) + + assert not httpserver.ready + assert not httpserver.requests._threads + + httpserver.prepare() + + assert httpserver.ready + assert httpserver.requests._threads + for thr in httpserver.requests._threads: + assert thr.ready + + httpserver.stop() + + assert not httpserver.requests._threads + assert not httpserver.ready + + +def test_stop_interrupts_serve(): + """Check that stop() interrupts running of serve().""" + httpserver = HTTPServer( + bind_addr=(ANY_INTERFACE_IPV4, EPHEMERAL_PORT), + gateway=Gateway, + ) + + httpserver.prepare() + serve_thread = threading.Thread(target=httpserver.serve) + serve_thread.start() + + serve_thread.join(0.5) + assert serve_thread.is_alive() + + httpserver.stop() + + serve_thread.join(0.5) + assert not serve_thread.is_alive() + + +@pytest.mark.parametrize( + 'ip_addr', + ( + ANY_INTERFACE_IPV4, + ANY_INTERFACE_IPV6, + ), +) +def test_bind_addr_inet(http_server, ip_addr): + """Check that bound IP address is stored in server.""" + httpserver = http_server.send((ip_addr, EPHEMERAL_PORT)) + + assert httpserver.bind_addr[0] == ip_addr + assert httpserver.bind_addr[1] != EPHEMERAL_PORT + + +@unix_only_sock_test +def test_bind_addr_unix(http_server, unix_sock_file): + """Check that bound UNIX socket address is stored in server.""" + httpserver = http_server.send(unix_sock_file) + + assert httpserver.bind_addr == unix_sock_file + + +@unix_only_sock_test +def test_bind_addr_unix_abstract(http_server, unix_abstract_sock): + """Check that bound UNIX abstract socket address is stored in server.""" + httpserver = http_server.send(unix_abstract_sock) + + assert httpserver.bind_addr == unix_abstract_sock + + +PEERCRED_IDS_URI = '/peer_creds/ids' +PEERCRED_TEXTS_URI = '/peer_creds/texts' + + +class _TestGateway(Gateway): + def respond(self): + req = self.req + conn = req.conn + req_uri = bton(req.uri) + if req_uri == PEERCRED_IDS_URI: + peer_creds = conn.peer_pid, conn.peer_uid, conn.peer_gid + self.send_payload('|'.join(map(str, peer_creds))) + return + elif req_uri == PEERCRED_TEXTS_URI: + self.send_payload('!'.join((conn.peer_user, conn.peer_group))) + return + return super(_TestGateway, self).respond() + + def send_payload(self, payload): + req = self.req + req.status = b'200 OK' + req.ensure_headers_sent() + req.write(ntob(payload)) + + +@pytest.fixture +def peercreds_enabled_server(http_server, unix_sock_file): + """Construct a test server with ``peercreds_enabled``.""" + httpserver = http_server.send(unix_sock_file) + httpserver.gateway = _TestGateway + httpserver.peercreds_enabled = True + return httpserver + + +@unix_only_sock_test +@non_macos_sock_test +def test_peercreds_unix_sock(peercreds_enabled_server): + """Check that ``PEERCRED`` lookup works when enabled.""" + httpserver = peercreds_enabled_server + bind_addr = httpserver.bind_addr + + if isinstance(bind_addr, six.binary_type): + bind_addr = bind_addr.decode() + + quoted = urllib.parse.quote(bind_addr, safe='') + unix_base_uri = 'http+unix://{quoted}'.format(**locals()) + + expected_peercreds = os.getpid(), os.getuid(), os.getgid() + expected_peercreds = '|'.join(map(str, expected_peercreds)) + + with requests_unixsocket.monkeypatch(): + peercreds_resp = requests.get(unix_base_uri + PEERCRED_IDS_URI) + peercreds_resp.raise_for_status() + assert peercreds_resp.text == expected_peercreds + + peercreds_text_resp = requests.get(unix_base_uri + PEERCRED_TEXTS_URI) + assert peercreds_text_resp.status_code == 500 + + +@pytest.mark.skipif( + not IS_UID_GID_RESOLVABLE, + reason='Modules `grp` and `pwd` are not available ' + 'under the current platform', +) +@unix_only_sock_test +@non_macos_sock_test +def test_peercreds_unix_sock_with_lookup(peercreds_enabled_server): + """Check that ``PEERCRED`` resolution works when enabled.""" + httpserver = peercreds_enabled_server + httpserver.peercreds_resolve_enabled = True + + bind_addr = httpserver.bind_addr + + if isinstance(bind_addr, six.binary_type): + bind_addr = bind_addr.decode() + + quoted = urllib.parse.quote(bind_addr, safe='') + unix_base_uri = 'http+unix://{quoted}'.format(**locals()) + + import grp + import pwd + expected_textcreds = ( + pwd.getpwuid(os.getuid()).pw_name, + grp.getgrgid(os.getgid()).gr_name, + ) + expected_textcreds = '!'.join(map(str, expected_textcreds)) + with requests_unixsocket.monkeypatch(): + peercreds_text_resp = requests.get(unix_base_uri + PEERCRED_TEXTS_URI) + peercreds_text_resp.raise_for_status() + assert peercreds_text_resp.text == expected_textcreds + + +@pytest.mark.skipif( + IS_WINDOWS, + reason='This regression test is for a Linux bug, ' + 'and the resource module is not available on Windows', +) +@pytest.mark.parametrize( + 'resource_limit', + ( + 1024, + 2048, + ), + indirect=('resource_limit',), +) +@pytest.mark.usefixtures('many_open_sockets') +def test_high_number_of_file_descriptors(resource_limit): + """Test the server does not crash with a high file-descriptor value. + + This test shouldn't cause a server crash when trying to access + file-descriptor higher than 1024. + + The earlier implementation used to rely on ``select()`` syscall that + doesn't support file descriptors with numbers higher than 1024. + """ + # We want to force the server to use a file-descriptor with + # a number above resource_limit + + # Create our server + httpserver = HTTPServer( + bind_addr=(ANY_INTERFACE_IPV4, EPHEMERAL_PORT), gateway=Gateway, + ) + httpserver.prepare() + + try: + # This will trigger a crash if select() is used in the implementation + httpserver.tick() + except: # noqa: E722 + raise # only needed for `else` to work + else: + # We use closing here for py2-compat + with closing(socket.socket()) as sock: + # Check new sockets created are still above our target number + assert sock.fileno() >= resource_limit + finally: + # Stop our server + httpserver.stop() + + +if not IS_WINDOWS: + test_high_number_of_file_descriptors = pytest.mark.forked( + test_high_number_of_file_descriptors, + ) + + +@pytest.fixture +def resource_limit(request): + """Set the resource limit two times bigger then requested.""" + resource = pytest.importorskip( + 'resource', + reason='The "resource" module is Unix-specific', + ) + + # Get current resource limits to restore them later + soft_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE) + + # We have to increase the nofile limit above 1024 + # Otherwise we see a 'Too many files open' error, instead of + # an error due to the file descriptor number being too high + resource.setrlimit( + resource.RLIMIT_NOFILE, + (request.param * 2, hard_limit), + ) + + try: # noqa: WPS501 + yield request.param + finally: + # Reset the resource limit back to the original soft limit + resource.setrlimit(resource.RLIMIT_NOFILE, (soft_limit, hard_limit)) + + +@pytest.fixture +def many_open_sockets(resource_limit): + """Allocate a lot of file descriptors by opening dummy sockets.""" + # Hoard a lot of file descriptors by opening and storing a lot of sockets + test_sockets = [] + # Open a lot of file descriptors, so the next one the server + # opens is a high number + try: + for i in range(resource_limit): + sock = socket.socket() + test_sockets.append(sock) + # If we reach a high enough number, we don't need to open more + if sock.fileno() >= resource_limit: + break + # Check we opened enough descriptors to reach a high number + the_highest_fileno = test_sockets[-1].fileno() + assert the_highest_fileno >= resource_limit + yield the_highest_fileno + finally: + # Close our open resources + for test_socket in test_sockets: + test_socket.close() diff --git a/lib/cheroot/test/test_ssl.py b/lib/cheroot/test/test_ssl.py new file mode 100644 index 0000000..8aa258f --- /dev/null +++ b/lib/cheroot/test/test_ssl.py @@ -0,0 +1,731 @@ +"""Tests for TLS support.""" +# -*- coding: utf-8 -*- +# vim: set fileencoding=utf-8 : + +from __future__ import absolute_import, division, print_function +__metaclass__ = type + +import functools +import json +import os +import ssl +import subprocess +import sys +import threading +import time +import traceback + +import OpenSSL.SSL +import pytest +import requests +import six +import trustme + +from .._compat import bton, ntob, ntou +from .._compat import IS_ABOVE_OPENSSL10, IS_PYPY +from .._compat import IS_LINUX, IS_MACOS, IS_WINDOWS +from ..server import HTTPServer, get_ssl_adapter_class +from ..testing import ( + ANY_INTERFACE_IPV4, + ANY_INTERFACE_IPV6, + EPHEMERAL_PORT, + # get_server_client, + _get_conn_data, + _probe_ipv6_sock, +) +from ..wsgi import Gateway_10 + + +IS_GITHUB_ACTIONS_WORKFLOW = bool(os.getenv('GITHUB_WORKFLOW')) +IS_WIN2016 = ( + IS_WINDOWS + # pylint: disable=unsupported-membership-test + and b'Microsoft Windows Server 2016 Datacenter' in subprocess.check_output( + ('systeminfo',), + ) +) +IS_LIBRESSL_BACKEND = ssl.OPENSSL_VERSION.startswith('LibreSSL') +IS_PYOPENSSL_SSL_VERSION_1_0 = ( + OpenSSL.SSL.SSLeay_version(OpenSSL.SSL.SSLEAY_VERSION). + startswith(b'OpenSSL 1.0.') +) +PY27 = sys.version_info[:2] == (2, 7) +PY34 = sys.version_info[:2] == (3, 4) +PY3 = not six.PY2 + + +_stdlib_to_openssl_verify = { + ssl.CERT_NONE: OpenSSL.SSL.VERIFY_NONE, + ssl.CERT_OPTIONAL: OpenSSL.SSL.VERIFY_PEER, + ssl.CERT_REQUIRED: + OpenSSL.SSL.VERIFY_PEER + OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT, +} + + +fails_under_py3 = pytest.mark.xfail( + not six.PY2, + reason='Fails under Python 3+', +) + + +fails_under_py3_in_pypy = pytest.mark.xfail( + not six.PY2 and IS_PYPY, + reason='Fails under PyPy3', +) + + +missing_ipv6 = pytest.mark.skipif( + not _probe_ipv6_sock('::1'), + reason='' + 'IPv6 is disabled ' + '(for example, under Travis CI ' + 'which runs under GCE supporting only IPv4)', +) + + +class HelloWorldGateway(Gateway_10): + """Gateway responding with Hello World to root URI.""" + + def respond(self): + """Respond with dummy content via HTTP.""" + req = self.req + req_uri = bton(req.uri) + if req_uri == '/': + req.status = b'200 OK' + req.ensure_headers_sent() + req.write(b'Hello world!') + return + if req_uri == '/env': + req.status = b'200 OK' + req.ensure_headers_sent() + env = self.get_environ() + # drop files so that it can be json dumped + env.pop('wsgi.errors') + env.pop('wsgi.input') + print(env) + req.write(json.dumps(env).encode('utf-8')) + return + return super(HelloWorldGateway, self).respond() + + +def make_tls_http_server(bind_addr, ssl_adapter, request): + """Create and start an HTTP server bound to ``bind_addr``.""" + httpserver = HTTPServer( + bind_addr=bind_addr, + gateway=HelloWorldGateway, + ) + # httpserver.gateway = HelloWorldGateway + httpserver.ssl_adapter = ssl_adapter + + threading.Thread(target=httpserver.safe_start).start() + + while not httpserver.ready: + time.sleep(0.1) + + request.addfinalizer(httpserver.stop) + + return httpserver + + +@pytest.fixture +def tls_http_server(request): + """Provision a server creator as a fixture.""" + return functools.partial(make_tls_http_server, request=request) + + +@pytest.fixture +def ca(): + """Provide a certificate authority via fixture.""" + return trustme.CA() + + +@pytest.fixture +def tls_ca_certificate_pem_path(ca): + """Provide a certificate authority certificate file via fixture.""" + with ca.cert_pem.tempfile() as ca_cert_pem: + yield ca_cert_pem + + +@pytest.fixture +def tls_certificate(ca): + """Provide a leaf certificate via fixture.""" + interface, host, port = _get_conn_data(ANY_INTERFACE_IPV4) + return ca.issue_server_cert(ntou(interface)) + + +@pytest.fixture +def tls_certificate_chain_pem_path(tls_certificate): + """Provide a certificate chain PEM file path via fixture.""" + with tls_certificate.private_key_and_cert_chain_pem.tempfile() as cert_pem: + yield cert_pem + + +@pytest.fixture +def tls_certificate_private_key_pem_path(tls_certificate): + """Provide a certificate private key PEM file path via fixture.""" + with tls_certificate.private_key_pem.tempfile() as cert_key_pem: + yield cert_key_pem + + +def _thread_except_hook(exceptions, args): + """Append uncaught exception ``args`` in threads to ``exceptions``.""" + if issubclass(args.exc_type, SystemExit): + return + # cannot store the exception, it references the thread's stack + exceptions.append(( + args.exc_type, + str(args.exc_value), + ''.join( + traceback.format_exception( + args.exc_type, args.exc_value, args.exc_traceback, + ), + ), + )) + + +@pytest.fixture +def thread_exceptions(): + """Provide a list of uncaught exceptions from threads via a fixture. + + Only catches exceptions on Python 3.8+. + The list contains: ``(type, str(value), str(traceback))`` + """ + exceptions = [] + # Python 3.8+ + orig_hook = getattr(threading, 'excepthook', None) + if orig_hook is not None: + threading.excepthook = functools.partial( + _thread_except_hook, exceptions, + ) + try: + yield exceptions + finally: + if orig_hook is not None: + threading.excepthook = orig_hook + + +@pytest.mark.parametrize( + 'adapter_type', + ( + 'builtin', + 'pyopenssl', + ), +) +def test_ssl_adapters( + tls_http_server, adapter_type, + tls_certificate, + tls_certificate_chain_pem_path, + tls_certificate_private_key_pem_path, + tls_ca_certificate_pem_path, +): + """Test ability to connect to server via HTTPS using adapters.""" + interface, _host, port = _get_conn_data(ANY_INTERFACE_IPV4) + tls_adapter_cls = get_ssl_adapter_class(name=adapter_type) + tls_adapter = tls_adapter_cls( + tls_certificate_chain_pem_path, tls_certificate_private_key_pem_path, + ) + if adapter_type == 'pyopenssl': + tls_adapter.context = tls_adapter.get_context() + + tls_certificate.configure_cert(tls_adapter.context) + + tlshttpserver = tls_http_server((interface, port), tls_adapter) + + # testclient = get_server_client(tlshttpserver) + # testclient.get('/') + + interface, _host, port = _get_conn_data( + tlshttpserver.bind_addr, + ) + + resp = requests.get( + 'https://{host!s}:{port!s}/'.format(host=interface, port=port), + verify=tls_ca_certificate_pem_path, + ) + + assert resp.status_code == 200 + assert resp.text == 'Hello world!' + + +@pytest.mark.parametrize( # noqa: C901 # FIXME + 'adapter_type', + ( + 'builtin', + 'pyopenssl', + ), +) +@pytest.mark.parametrize( + ('is_trusted_cert', 'tls_client_identity'), + ( + (True, 'localhost'), (True, '127.0.0.1'), + (True, '*.localhost'), (True, 'not_localhost'), + (False, 'localhost'), + ), +) +@pytest.mark.parametrize( + 'tls_verify_mode', + ( + ssl.CERT_NONE, # server shouldn't validate client cert + ssl.CERT_OPTIONAL, # same as CERT_REQUIRED in client mode, don't use + ssl.CERT_REQUIRED, # server should validate if client cert CA is OK + ), +) +def test_tls_client_auth( # noqa: C901 # FIXME + # FIXME: remove twisted logic, separate tests + mocker, + tls_http_server, adapter_type, + ca, + tls_certificate, + tls_certificate_chain_pem_path, + tls_certificate_private_key_pem_path, + tls_ca_certificate_pem_path, + is_trusted_cert, tls_client_identity, + tls_verify_mode, +): + """Verify that client TLS certificate auth works correctly.""" + test_cert_rejection = ( + tls_verify_mode != ssl.CERT_NONE + and not is_trusted_cert + ) + interface, _host, port = _get_conn_data(ANY_INTERFACE_IPV4) + + client_cert_root_ca = ca if is_trusted_cert else trustme.CA() + with mocker.mock_module.patch( + 'idna.core.ulabel', + return_value=ntob(tls_client_identity), + ): + client_cert = client_cert_root_ca.issue_server_cert( + # FIXME: change to issue_cert once new trustme is out + ntou(tls_client_identity), + ) + del client_cert_root_ca + + with client_cert.private_key_and_cert_chain_pem.tempfile() as cl_pem: + tls_adapter_cls = get_ssl_adapter_class(name=adapter_type) + tls_adapter = tls_adapter_cls( + tls_certificate_chain_pem_path, + tls_certificate_private_key_pem_path, + ) + if adapter_type == 'pyopenssl': + tls_adapter.context = tls_adapter.get_context() + tls_adapter.context.set_verify( + _stdlib_to_openssl_verify[tls_verify_mode], + lambda conn, cert, errno, depth, preverify_ok: preverify_ok, + ) + else: + tls_adapter.context.verify_mode = tls_verify_mode + + ca.configure_trust(tls_adapter.context) + tls_certificate.configure_cert(tls_adapter.context) + + tlshttpserver = tls_http_server((interface, port), tls_adapter) + + interface, _host, port = _get_conn_data(tlshttpserver.bind_addr) + + make_https_request = functools.partial( + requests.get, + 'https://{host!s}:{port!s}/'.format(host=interface, port=port), + + # Server TLS certificate verification: + verify=tls_ca_certificate_pem_path, + + # Client TLS certificate verification: + cert=cl_pem, + ) + + if not test_cert_rejection: + resp = make_https_request() + is_req_successful = resp.status_code == 200 + if ( + not is_req_successful + and IS_PYOPENSSL_SSL_VERSION_1_0 + and adapter_type == 'builtin' + and tls_verify_mode == ssl.CERT_REQUIRED + and tls_client_identity == 'localhost' + and is_trusted_cert + ) or PY34: + pytest.xfail( + 'OpenSSL 1.0 has problems with verifying client certs', + ) + assert is_req_successful + assert resp.text == 'Hello world!' + return + + # xfail some flaky tests + # https://github.com/cherrypy/cheroot/issues/237 + issue_237 = ( + IS_MACOS + and adapter_type == 'builtin' + and tls_verify_mode != ssl.CERT_NONE + ) + if issue_237: + pytest.xfail('Test sometimes fails') + + expected_ssl_errors = ( + requests.exceptions.SSLError, + OpenSSL.SSL.Error, + ) if PY34 else ( + requests.exceptions.SSLError, + ) + if IS_WINDOWS or IS_GITHUB_ACTIONS_WORKFLOW: + expected_ssl_errors += requests.exceptions.ConnectionError, + with pytest.raises(expected_ssl_errors) as ssl_err: + make_https_request() + + if PY34 and isinstance(ssl_err, OpenSSL.SSL.Error): + pytest.xfail( + 'OpenSSL behaves wierdly under Python 3.4 ' + 'because of an outdated urllib3', + ) + + try: + err_text = ssl_err.value.args[0].reason.args[0].args[0] + except AttributeError: + if PY34: + pytest.xfail('OpenSSL behaves wierdly under Python 3.4') + elif IS_WINDOWS or IS_GITHUB_ACTIONS_WORKFLOW: + err_text = str(ssl_err.value) + else: + raise + + if isinstance(err_text, int): + err_text = str(ssl_err.value) + + expected_substrings = ( + 'sslv3 alert bad certificate' if IS_LIBRESSL_BACKEND + else 'tlsv1 alert unknown ca', + ) + if not six.PY2: + if IS_MACOS and IS_PYPY and adapter_type == 'pyopenssl': + expected_substrings = ('tlsv1 alert unknown ca',) + if ( + tls_verify_mode in ( + ssl.CERT_REQUIRED, + ssl.CERT_OPTIONAL, + ) + and not is_trusted_cert + and tls_client_identity == 'localhost' + ): + expected_substrings += ( + 'bad handshake: ' + "SysCallError(10054, 'WSAECONNRESET')", + "('Connection aborted.', " + 'OSError("(10054, \'WSAECONNRESET\')"))', + "('Connection aborted.', " + 'OSError("(10054, \'WSAECONNRESET\')",))', + "('Connection aborted.', " + 'error("(10054, \'WSAECONNRESET\')",))', + "('Connection aborted.', " + 'ConnectionResetError(10054, ' + "'An existing connection was forcibly closed " + "by the remote host', None, 10054, None))", + ) if IS_WINDOWS else ( + "('Connection aborted.', " + 'OSError("(104, \'ECONNRESET\')"))', + "('Connection aborted.', " + 'OSError("(104, \'ECONNRESET\')",))', + "('Connection aborted.', " + 'error("(104, \'ECONNRESET\')",))', + "('Connection aborted.', " + "ConnectionResetError(104, 'Connection reset by peer'))", + "('Connection aborted.', " + "error(104, 'Connection reset by peer'))", + ) if ( + IS_GITHUB_ACTIONS_WORKFLOW + and IS_LINUX + ) else ( + "('Connection aborted.', " + "BrokenPipeError(32, 'Broken pipe'))", + ) + assert any(e in err_text for e in expected_substrings) + + +@pytest.mark.parametrize( # noqa: C901 # FIXME + 'adapter_type', + ( + 'builtin', + 'pyopenssl', + ), +) +@pytest.mark.parametrize( + ('tls_verify_mode', 'use_client_cert'), + ( + (ssl.CERT_NONE, False), + (ssl.CERT_NONE, True), + (ssl.CERT_OPTIONAL, False), + (ssl.CERT_OPTIONAL, True), + (ssl.CERT_REQUIRED, True), + ), +) +def test_ssl_env( # noqa: C901 # FIXME + thread_exceptions, + recwarn, + mocker, + tls_http_server, adapter_type, + ca, tls_verify_mode, tls_certificate, + tls_certificate_chain_pem_path, + tls_certificate_private_key_pem_path, + tls_ca_certificate_pem_path, + use_client_cert, +): + """Test the SSL environment generated by the SSL adapters.""" + interface, _host, port = _get_conn_data(ANY_INTERFACE_IPV4) + + with mocker.mock_module.patch( + 'idna.core.ulabel', + return_value=ntob('127.0.0.1'), + ): + client_cert = ca.issue_cert(ntou('127.0.0.1')) + + with client_cert.private_key_and_cert_chain_pem.tempfile() as cl_pem: + tls_adapter_cls = get_ssl_adapter_class(name=adapter_type) + tls_adapter = tls_adapter_cls( + tls_certificate_chain_pem_path, + tls_certificate_private_key_pem_path, + ) + if adapter_type == 'pyopenssl': + tls_adapter.context = tls_adapter.get_context() + tls_adapter.context.set_verify( + _stdlib_to_openssl_verify[tls_verify_mode], + lambda conn, cert, errno, depth, preverify_ok: preverify_ok, + ) + else: + tls_adapter.context.verify_mode = tls_verify_mode + + ca.configure_trust(tls_adapter.context) + tls_certificate.configure_cert(tls_adapter.context) + + tlswsgiserver = tls_http_server((interface, port), tls_adapter) + + interface, _host, port = _get_conn_data(tlswsgiserver.bind_addr) + + resp = requests.get( + 'https://' + interface + ':' + str(port) + '/env', + verify=tls_ca_certificate_pem_path, + cert=cl_pem if use_client_cert else None, + ) + if PY34 and resp.status_code != 200: + pytest.xfail( + 'Python 3.4 has problems with verifying client certs', + ) + + env = json.loads(resp.content.decode('utf-8')) + + # hard coded env + assert env['wsgi.url_scheme'] == 'https' + assert env['HTTPS'] == 'on' + + # ensure these are present + for key in {'SSL_VERSION_INTERFACE', 'SSL_VERSION_LIBRARY'}: + assert key in env + + # pyOpenSSL generates the env before the handshake completes + if adapter_type == 'pyopenssl': + return + + for key in {'SSL_PROTOCOL', 'SSL_CIPHER'}: + assert key in env + + # client certificate env + if tls_verify_mode == ssl.CERT_NONE or not use_client_cert: + assert env['SSL_CLIENT_VERIFY'] == 'NONE' + else: + assert env['SSL_CLIENT_VERIFY'] == 'SUCCESS' + + with open(cl_pem, 'rt') as f: + assert env['SSL_CLIENT_CERT'] in f.read() + + for key in { + 'SSL_CLIENT_M_VERSION', 'SSL_CLIENT_M_SERIAL', + 'SSL_CLIENT_I_DN', 'SSL_CLIENT_S_DN', + }: + assert key in env + + # builtin ssl environment generation may use a loopback socket + # ensure no ResourceWarning was raised during the test + # NOTE: python 2.7 does not emit ResourceWarning for ssl sockets + if IS_PYPY: + # NOTE: PyPy doesn't have ResourceWarning + # Ref: https://doc.pypy.org/en/latest/cpython_differences.html + return + for warn in recwarn: + if not issubclass(warn.category, ResourceWarning): + continue + + # the tests can sporadically generate resource warnings + # due to timing issues + # all of these sporadic warnings appear to be about socket.socket + # and have been observed to come from requests connection pool + msg = str(warn.message) + if 'socket.socket' in msg: + pytest.xfail( + '\n'.join(( + 'Sometimes this test fails due to ' + 'a socket.socket ResourceWarning:', + msg, + )), + ) + pytest.fail(msg) + + # to perform the ssl handshake over that loopback socket, + # the builtin ssl environment generation uses a thread + for _, _, trace in thread_exceptions: + print(trace, file=sys.stderr) + assert not thread_exceptions, ': '.join(( + thread_exceptions[0][0].__name__, + thread_exceptions[0][1], + )) + + +@pytest.mark.parametrize( + 'ip_addr', + ( + ANY_INTERFACE_IPV4, + ANY_INTERFACE_IPV6, + ), +) +def test_https_over_http_error(http_server, ip_addr): + """Ensure that connecting over HTTPS to HTTP port is handled.""" + httpserver = http_server.send((ip_addr, EPHEMERAL_PORT)) + interface, _host, port = _get_conn_data(httpserver.bind_addr) + with pytest.raises(ssl.SSLError) as ssl_err: + six.moves.http_client.HTTPSConnection( + '{interface}:{port}'.format( + interface=interface, + port=port, + ), + ).request('GET', '/') + expected_substring = ( + 'wrong version number' if IS_ABOVE_OPENSSL10 + else 'unknown protocol' + ) + assert expected_substring in ssl_err.value.args[-1] + + +@pytest.mark.parametrize( + 'adapter_type', + ( + pytest.param( + 'builtin', + marks=pytest.mark.xfail( + IS_WINDOWS and six.PY2, + raises=requests.exceptions.ConnectionError, + reason='Stdlib `ssl` module behaves weirdly ' + 'on Windows under Python 2', + strict=False, + ), + ), + 'pyopenssl', + ), +) +@pytest.mark.parametrize( + 'ip_addr', + ( + ANY_INTERFACE_IPV4, + pytest.param(ANY_INTERFACE_IPV6, marks=missing_ipv6), + ), +) +def test_http_over_https_error( + tls_http_server, adapter_type, + ca, ip_addr, + tls_certificate, + tls_certificate_chain_pem_path, + tls_certificate_private_key_pem_path, +): + """Ensure that connecting over HTTP to HTTPS port is handled.""" + # disable some flaky tests + # https://github.com/cherrypy/cheroot/issues/225 + issue_225 = ( + IS_MACOS + and adapter_type == 'builtin' + ) + if issue_225: + pytest.xfail('Test fails in Travis-CI') + + tls_adapter_cls = get_ssl_adapter_class(name=adapter_type) + tls_adapter = tls_adapter_cls( + tls_certificate_chain_pem_path, tls_certificate_private_key_pem_path, + ) + if adapter_type == 'pyopenssl': + tls_adapter.context = tls_adapter.get_context() + + tls_certificate.configure_cert(tls_adapter.context) + + interface, _host, port = _get_conn_data(ip_addr) + tlshttpserver = tls_http_server((interface, port), tls_adapter) + + interface, host, port = _get_conn_data( + tlshttpserver.bind_addr, + ) + + fqdn = interface + if ip_addr is ANY_INTERFACE_IPV6: + fqdn = '[{fqdn}]'.format(**locals()) + + expect_fallback_response_over_plain_http = ( + ( + adapter_type == 'pyopenssl' + and (IS_ABOVE_OPENSSL10 or not six.PY2) + ) + or PY27 + ) or ( + IS_GITHUB_ACTIONS_WORKFLOW + and IS_WINDOWS + and six.PY2 + and not IS_WIN2016 + ) + if ( + IS_GITHUB_ACTIONS_WORKFLOW + and IS_WINDOWS + and six.PY2 + and IS_WIN2016 + and adapter_type == 'builtin' + and ip_addr is ANY_INTERFACE_IPV6 + ): + expect_fallback_response_over_plain_http = True + if ( + IS_GITHUB_ACTIONS_WORKFLOW + and IS_WINDOWS + and six.PY2 + and not IS_WIN2016 + and adapter_type == 'builtin' + and ip_addr is not ANY_INTERFACE_IPV6 + ): + expect_fallback_response_over_plain_http = False + if expect_fallback_response_over_plain_http: + resp = requests.get( + 'http://{host!s}:{port!s}/'.format(host=fqdn, port=port), + ) + assert resp.status_code == 400 + assert resp.text == ( + 'The client sent a plain HTTP request, ' + 'but this server only speaks HTTPS on this port.' + ) + return + + with pytest.raises(requests.exceptions.ConnectionError) as ssl_err: + requests.get( # FIXME: make stdlib ssl behave like PyOpenSSL + 'http://{host!s}:{port!s}/'.format(host=fqdn, port=port), + ) + + if IS_LINUX: + expected_error_code, expected_error_text = ( + 104, 'Connection reset by peer', + ) + if IS_MACOS: + expected_error_code, expected_error_text = ( + 54, 'Connection reset by peer', + ) + if IS_WINDOWS: + expected_error_code, expected_error_text = ( + 10054, + 'An existing connection was forcibly closed by the remote host', + ) + + underlying_error = ssl_err.value.args[0].args[-1] + err_text = str(underlying_error) + assert underlying_error.errno == expected_error_code, ( + 'The underlying error is {underlying_error!r}'. + format(**locals()) + ) + assert expected_error_text in err_text diff --git a/lib/cheroot/test/test_wsgi.py b/lib/cheroot/test/test_wsgi.py new file mode 100644 index 0000000..d3c47ec --- /dev/null +++ b/lib/cheroot/test/test_wsgi.py @@ -0,0 +1,58 @@ +"""Test wsgi.""" + +from concurrent.futures.thread import ThreadPoolExecutor + +import pytest +import portend +import requests +from requests_toolbelt.sessions import BaseUrlSession as Session +from jaraco.context import ExceptionTrap + +from cheroot import wsgi +from cheroot._compat import IS_MACOS, IS_WINDOWS + + +IS_SLOW_ENV = IS_MACOS or IS_WINDOWS + + +@pytest.fixture +def simple_wsgi_server(): + """Fucking simple wsgi server fixture (duh).""" + port = portend.find_available_local_port() + + def app(environ, start_response): + status = '200 OK' + response_headers = [('Content-type', 'text/plain')] + start_response(status, response_headers) + return [b'Hello world!'] + + host = '::' + addr = host, port + server = wsgi.Server(addr, app, timeout=600 if IS_SLOW_ENV else 20) + url = 'http://localhost:{port}/'.format(**locals()) + with server._run_in_thread() as thread: + yield locals() + + +def test_connection_keepalive(simple_wsgi_server): + """Test the connection keepalive works (duh).""" + session = Session(base_url=simple_wsgi_server['url']) + pooled = requests.adapters.HTTPAdapter( + pool_connections=1, pool_maxsize=1000, + ) + session.mount('http://', pooled) + + def do_request(): + with ExceptionTrap(requests.exceptions.ConnectionError) as trap: + resp = session.get('info') + resp.raise_for_status() + return bool(trap) + + with ThreadPoolExecutor(max_workers=10 if IS_SLOW_ENV else 50) as pool: + tasks = [ + pool.submit(do_request) + for n in range(250 if IS_SLOW_ENV else 1000) + ] + failures = sum(task.result() for task in tasks) + + assert not failures diff --git a/lib/cheroot/test/test_wsgiapps.py b/lib/cheroot/test/test_wsgiapps.py deleted file mode 100644 index a5dc611..0000000 --- a/lib/cheroot/test/test_wsgiapps.py +++ /dev/null @@ -1,99 +0,0 @@ -import sys - -import pytest - -from cheroot._compat import ntob -from cheroot.test import helper - - -@pytest.mark.xfail(reason='issue 1') -class WSGIGraftTests(helper.CherootWebCase): - - @staticmethod - def setup_server(): - def test_app(environ, start_response): - status = '200 OK' - response_headers = [('Content-type', 'text/plain')] - start_response(status, response_headers) - output = ['Hello, world!\n', - 'This is a wsgi app running within CherryPy!\n\n'] - keys = list(environ.keys()) - keys.sort() - for k in keys: - output.append('%s: %s\n' % (k, environ[k])) - return [ntob(x, 'utf-8') for x in output] - - def test_empty_string_app(environ, start_response): - status = '200 OK' - response_headers = [('Content-type', 'text/plain')] - start_response(status, response_headers) - return [ - ntob('Hello'), ntob(''), ntob(' '), ntob(''), ntob('world') - ] - - class WSGIResponse(object): - - def __init__(self, appresults): - self.appresults = appresults - self.iter = iter(appresults) - - def __iter__(self): - return self - - if sys.version_info >= (3, 0): - def __next__(self): - return next(self.iter) - else: - def next(self): - return self.iter.next() - - def close(self): - if hasattr(self.appresults, 'close'): - self.appresults.close() - - class ReversingMiddleware(object): - - def __init__(self, app): - self.app = app - - def __call__(self, environ, start_response): - results = self.app(environ, start_response) - - class Reverser(WSGIResponse): - - if sys.version_info >= (3, 0): - def __next__(this): - line = list(next(this.iter)) - line.reverse() - return bytes(line) - else: - def next(this): - line = list(this.iter.next()) - line.reverse() - return ''.join(line) - - return Reverser(results) - - wsgi_output = '''Hello, world! -This is a wsgi app running within CherryPy!''' - - def test_01_standard_app(self): - self.getPage('/') - self.assertBody("I'm a regular CherryPy page handler!") - - def test_04_pure_wsgi(self): - self.getPage('/hosted/app1') - self.assertHeader('Content-Type', 'text/plain') - self.assertInBody(self.wsgi_output) - - def test_05_wrapped_cp_app(self): - self.getPage('/hosted/app2/') - body = list("I'm a regular CherryPy page handler!") - body.reverse() - body = ''.join(body) - self.assertInBody(body) - - def test_06_empty_string_app(self): - self.getPage('/hosted/app3') - self.assertHeader('Content-Type', 'text/plain') - self.assertInBody('Hello world') diff --git a/lib/cheroot/test/webtest.py b/lib/cheroot/test/webtest.py index ea6bd3c..51506ef 100644 --- a/lib/cheroot/test/webtest.py +++ b/lib/cheroot/test/webtest.py @@ -1,36 +1,40 @@ """Extensions to unittest for web frameworks. -Use the WebCase.getPage method to request a page from your HTTP server. +Use the :py:meth:`WebCase.getPage` method to request a page +from your HTTP server. + Framework Integration ===================== If you have control over your server process, you can handle errors in the server-side of the HTTP conversation a bit better. You must run -both the client (your WebCase tests) and the server in the same process -(but in separate threads, obviously). +both the client (your :py:class:`WebCase` tests) and the server in the +same process (but in separate threads, obviously). When an error occurs in the framework, call server_error. It will print the traceback to stdout, and keep any assertions you have from running (the assumption is that, if the server errors, the page output will not be of further significance to your tests). """ +from __future__ import absolute_import, division, print_function +__metaclass__ = type + import pprint import re import socket import sys import time import traceback -import types import os import json - -from imp import reload - import unittest +import warnings +import functools +from six.moves import http_client, map, urllib_parse import six -from cheroot._compat import text_or_bytes, HTTPConnection -from cheroot._compat import HTTPSConnection +from more_itertools.more import always_iterable +import jaraco.functools def interface(host): @@ -48,109 +52,11 @@ def interface(host): return host -class TerseTestResult(unittest._TextTestResult): - - def printErrors(self): - # Overridden to avoid unnecessary empty line - if self.errors or self.failures: - if self.dots or self.showAll: - self.stream.writeln() - self.printErrorList('ERROR', self.errors) - self.printErrorList('FAIL', self.failures) - - -class TerseTestRunner(unittest.TextTestRunner): - """A test runner class that displays results in textual form.""" - - def _makeResult(self): - return TerseTestResult(self.stream, self.descriptions, self.verbosity) - - def run(self, test): - """Run the given test case or test suite.""" - # Overridden to remove unnecessary empty lines and separators - result = self._makeResult() - test(result) - result.printErrors() - if not result.wasSuccessful(): - self.stream.write('FAILED (') - failed, errored = list(map(len, (result.failures, result.errors))) - if failed: - self.stream.write('failures=%d' % failed) - if errored: - if failed: - self.stream.write(', ') - self.stream.write('errors=%d' % errored) - self.stream.writeln(')') - return result - - -class ReloadingTestLoader(unittest.TestLoader): - - def loadTestsFromName(self, name, module=None): - """Return a suite of all tests cases given a string specifier. - - The name may resolve either to a module, a test case class, a - test method within a test case class, or a callable object which - returns a TestCase or TestSuite instance. - The method optionally resolves the names relative to a given module. - """ - parts = name.split('.') - unused_parts = [] - if module is None: - if not parts: - raise ValueError('incomplete test name: %s' % name) - else: - parts_copy = parts[:] - while parts_copy: - target = '.'.join(parts_copy) - if target in sys.modules: - module = reload(sys.modules[target]) - parts = unused_parts - break - else: - try: - module = __import__(target) - parts = unused_parts - break - except ImportError: - unused_parts.insert(0, parts_copy[-1]) - del parts_copy[-1] - if not parts_copy: - raise - parts = parts[1:] - obj = module - for part in parts: - obj = getattr(obj, part) - - if isinstance(obj, types.ModuleType): - return self.loadTestsFromModule(obj) - elif ( - ( - (six.PY3 and isinstance(obj, type)) or - isinstance(obj, (type, types.ClassType)) - ) and - issubclass(obj, unittest.TestCase)): - return self.loadTestsFromTestCase(obj) - elif isinstance(obj, types.UnboundMethodType): - if six.PY3: - return obj.__self__.__class__(obj.__name__) - else: - return obj.im_class(obj.__name__) - elif hasattr(obj, '__call__'): - test = obj() - if not isinstance(test, unittest.TestCase) and \ - not isinstance(test, unittest.TestSuite): - raise ValueError('calling %s returned %s, ' - 'not a test' % (obj, test)) - return test - else: - raise ValueError('do not know how to make test from: %s' % obj) - - try: # Jython support if sys.platform[:4] == 'java': def getchar(): + """Get a key press.""" # Hopefully this is enough return sys.stdin.read(1) else: @@ -158,6 +64,7 @@ def getchar(): import msvcrt def getchar(): + """Get a key press.""" return msvcrt.getch() except ImportError: # Unix getchr @@ -165,6 +72,7 @@ def getchar(): import termios def getchar(): + """Get a key press.""" fd = sys.stdin.fileno() old_settings = termios.tcgetattr(fd) try: @@ -176,26 +84,33 @@ def getchar(): # from jaraco.properties -class NonDataProperty(object): +class NonDataProperty: + """Non-data property decorator.""" + def __init__(self, fget): + """Initialize a non-data property.""" assert fget is not None, 'fget cannot be none' assert callable(fget), 'fget must be callable' self.fget = fget def __get__(self, obj, objtype=None): + """Return a class property.""" if obj is None: return self return self.fget(obj) class WebCase(unittest.TestCase): + """Helper web test suite base.""" + HOST = '127.0.0.1' PORT = 8000 - HTTP_CONN = HTTPConnection + HTTP_CONN = http_client.HTTPConnection PROTOCOL = 'HTTP/1.1' scheme = 'http' url = None + ssl_context = None status = None headers = None @@ -205,13 +120,18 @@ class WebCase(unittest.TestCase): time = None + @property + def _Conn(self): + """Return HTTPConnection or HTTPSConnection based on self.scheme. + + * from http.client. + """ + cls_name = '{scheme}Connection'.format(scheme=self.scheme.upper()) + return getattr(http_client, cls_name) + def get_conn(self, auto_open=False): """Return a connection to our HTTP server.""" - if self.scheme == 'https': - cls = HTTPSConnection - else: - cls = HTTPConnection - conn = cls(self.interface(), self.PORT) + conn = self._Conn(self.interface(), self.PORT) # Automatically re-connect? conn.auto_open = auto_open conn.connect() @@ -221,30 +141,30 @@ def set_persistent(self, on=True, auto_open=False): """Make our HTTP_CONN persistent (or not). If the 'on' argument is True (the default), then self.HTTP_CONN - will be set to an instance of HTTPConnection (or HTTPS - if self.scheme is "https"). This will then persist across requests. - We only allow for a single open connection, so if you call this - and we currently have an open connection, it will be closed. + will be set to an instance of HTTP(S)?Connection + to persist across requests. + As this class only allows for a single open connection, if + self already has an open connection, it will be closed. """ try: self.HTTP_CONN.close() except (TypeError, AttributeError): pass - if on: - self.HTTP_CONN = self.get_conn(auto_open=auto_open) - else: - if self.scheme == 'https': - self.HTTP_CONN = HTTPSConnection - else: - self.HTTP_CONN = HTTPConnection + self.HTTP_CONN = ( + self.get_conn(auto_open=auto_open) + if on + else self._Conn + ) - def _get_persistent(self): + @property + def persistent(self): + """Presence of the persistent HTTP connection.""" return hasattr(self.HTTP_CONN, '__class__') - def _set_persistent(self, on): + @persistent.setter + def persistent(self, on): self.set_persistent(on) - persistent = property(_get_persistent, _set_persistent) def interface(self): """Return an IP address for a client connection. @@ -254,14 +174,30 @@ def interface(self): """ return interface(self.HOST) - def getPage(self, url, headers=None, method='GET', body=None, - protocol=None, raise_subcls=None): - """Open the url with debugging support. Return status, headers, body. + def getPage( + self, url, headers=None, method='GET', body=None, + protocol=None, raise_subcls=(), + ): + """Open the url with debugging support. + + Return status, headers, body. + + url should be the identifier passed to the server, typically a + server-absolute path and query string (sent between method and + protocol), and should only be an absolute URI if proxy support is + enabled in the server. + + If the application under test generates absolute URIs, be sure + to wrap them first with :py:func:`strip_netloc`:: + + >>> class MyAppWebCase(WebCase): + ... def getPage(url, *args, **kwargs): + ... super(MyAppWebCase, self).getPage( + ... cheroot.test.webtest.strip_netloc(url), + ... *args, **kwargs + ... ) - `raise_subcls` must be a tuple with the exceptions classes - or a single exception class that are not going to be considered - a socket.error regardless that they were are subclass of a - socket.error and therefore not considered for a connection retry. + ``raise_subcls`` is passed through to :py:func:`openURL`. """ ServerError.on = False @@ -270,18 +206,26 @@ def getPage(self, url, headers=None, method='GET', body=None, if isinstance(body, six.text_type): body = body.encode('utf-8') + # for compatibility, support raise_subcls is None + raise_subcls = raise_subcls or () + self.url = url self.time = None start = time.time() - result = openURL(url, headers, method, body, self.HOST, self.PORT, - self.HTTP_CONN, protocol or self.PROTOCOL, - raise_subcls) + result = openURL( + url, headers, method, body, self.HOST, self.PORT, + self.HTTP_CONN, protocol or self.PROTOCOL, + raise_subcls=raise_subcls, + ssl_context=self.ssl_context, + ) self.time = time.time() - start self.status, self.headers, self.body = result # Build a list of request cookies from the previous response cookies. - self.cookies = [('Cookie', v) for k, v in self.headers - if k.lower() == 'set-cookie'] + self.cookies = [ + ('Cookie', v) for k, v in self.headers + if k.lower() == 'set-cookie' + ] if ServerError.on: raise ServerError() @@ -296,20 +240,29 @@ def interactive(self): False or 1 or 0. """ env_str = os.environ.get('WEBTEST_INTERACTIVE', 'True') - return bool(json.loads(env_str.lower())) + is_interactive = bool(json.loads(env_str.lower())) + if is_interactive: + warnings.warn( + 'Interactive test failure interceptor support via ' + 'WEBTEST_INTERACTIVE environment variable is deprecated.', + DeprecationWarning, + ) + return is_interactive console_height = 30 - def _handlewebError(self, msg): + def _handlewebError(self, msg): # noqa: C901 # FIXME print('') print(' ERROR: %s' % msg) if not self.interactive: raise self.failureException(msg) - p = (' Show: ' - '[B]ody [H]eaders [S]tatus [U]RL; ' - '[I]gnore, [R]aise, or sys.e[X]it >> ') + p = ( + ' Show: ' + '[B]ody [H]eaders [S]tatus [U]RL; ' + '[I]gnore, [R]aise, or sys.e[X]it >> ' + ) sys.stdout.write(p) sys.stdout.flush() while True: @@ -342,41 +295,36 @@ def _handlewebError(self, msg): elif i == 'R': raise self.failureException(msg) elif i == 'X': - self.exit() + sys.exit() sys.stdout.write(p) sys.stdout.flush() - def exit(self): - sys.exit() + @property + def status_code(self): # noqa: D401; irrelevant for properties + """Integer HTTP status code.""" + return int(self.status[:3]) + + def status_matches(self, expected): + """Check whether actual status matches expected.""" + actual = ( + self.status_code + if isinstance(expected, int) else + self.status + ) + return expected == actual def assertStatus(self, status, msg=None): - """Fail if self.status != status.""" - if isinstance(status, text_or_bytes): - if not self.status == status: - if msg is None: - msg = 'Status (%r) != %r' % (self.status, status) - self._handlewebError(msg) - elif isinstance(status, int): - code = int(self.status[:3]) - if code != status: - if msg is None: - msg = 'Status (%r) != %r' % (self.status, status) - self._handlewebError(msg) - else: - # status is a tuple or list. - match = False - for s in status: - if isinstance(s, text_or_bytes): - if self.status == s: - match = True - break - elif int(self.status[:3]) == s: - match = True - break - if not match: - if msg is None: - msg = 'Status (%r) not in %r' % (self.status, status) - self._handlewebError(msg) + """Fail if self.status != status. + + status may be integer code, exact string status, or + iterable of allowed possibilities. + """ + if any(map(self.status_matches, always_iterable(status))): + return + + tmpl = 'Status {self.status} does not match {status}' + msg = msg or tmpl.format(**locals()) + self._handlewebError(msg) def assertHeader(self, key, value=None, msg=None): """Fail if (key, [value]) not in self.headers.""" @@ -426,6 +374,16 @@ def assertNoHeader(self, key, msg=None): msg = '%r in headers' % key self._handlewebError(msg) + def assertNoHeaderItemValue(self, key, value, msg=None): + """Fail if the header contains the specified value.""" + lowkey = key.lower() + hdrs = self.headers + matches = [k for k, v in hdrs if k.lower() == lowkey and v == value] + if matches: + if msg is None: + msg = '%r:%r in %r' % (key, value, hdrs) + self._handlewebError(msg) + def assertBody(self, value, msg=None): """Fail if value != self.body.""" if isinstance(value, six.text_type): @@ -433,7 +391,8 @@ def assertBody(self, value, msg=None): if value != self.body: if msg is None: msg = 'expected body:\n%r\n\nactual body:\n%r' % ( - value, self.body) + value, self.body, + ) self._handlewebError(msg) def assertInBody(self, value, msg=None): @@ -494,7 +453,8 @@ def cleanHeaders(headers, method, body, host, port): break if not found: headers.append( - ('Content-Type', 'application/x-www-form-urlencoded')) + ('Content-Type', 'application/x-www-form-urlencoded'), + ) headers.append(('Content-Length', str(len(body or '')))) return headers @@ -502,82 +462,116 @@ def cleanHeaders(headers, method, body, host, port): def shb(response): """Return status, headers, body the way we like from a response.""" - if six.PY3: - h = response.getheaders() - else: - h = [] - key, value = None, None - for line in response.msg.headers: - if line: - if line[0] in ' \t': - value += line.strip() - else: - if key and value: - h.append((key, value)) - key, value = line.split(':', 1) - key = key.strip() - value = value.strip() - if key and value: - h.append((key, value)) - - return '%s %s' % (response.status, response.reason), h, response.read() - - -def openURL(url, headers=None, method='GET', body=None, - host='127.0.0.1', port=8000, http_conn=HTTPConnection, - protocol='HTTP/1.1', raise_subcls=None): - """ - Open the given HTTP resource and return status, headers, and body. + resp_status_line = '%s %s' % (response.status, response.reason) - `raise_subcls` must be a tuple with the exceptions classes - or a single exception class that are not going to be considered - a socket.error regardless that they were are subclass of a - socket.error and therefore not considered for a connection retry. - """ - headers = cleanHeaders(headers, method, body, host, port) + if not six.PY2: + return resp_status_line, response.getheaders(), response.read() - # Trying 10 times is simply in case of socket errors. - # Normal case--it should run once. - for trial in range(10): - try: - # Allow http_conn to be a class or an instance - if hasattr(http_conn, 'host'): - conn = http_conn + h = [] + key, value = None, None + for line in response.msg.headers: + if line: + if line[0] in ' \t': + value += line.strip() else: - conn = http_conn(interface(host), port) - - conn._http_vsn_str = protocol - conn._http_vsn = int(''.join([x for x in protocol if x.isdigit()])) + if key and value: + h.append((key, value)) + key, value = line.split(':', 1) + key = key.strip() + value = value.strip() + if key and value: + h.append((key, value)) - if six.PY3 and isinstance(url, bytes): - url = url.decode() - conn.putrequest(method.upper(), url, skip_host=True, - skip_accept_encoding=True) + return resp_status_line, h, response.read() - for key, value in headers: - conn.putheader(key, value.encode('Latin-1')) - conn.endheaders() - if body is not None: - conn.send(body) - - # Handle response - response = conn.getresponse() - - s, h, b = shb(response) +# def openURL(*args, raise_subcls=(), **kwargs): +# py27 compatible signature: +def openURL(*args, **kwargs): + """ + Open a URL, retrying when it fails. - if not hasattr(http_conn, 'host'): - # We made our own conn instance. Close it. - conn.close() + Specify ``raise_subcls`` (class or tuple of classes) to exclude + those socket.error subclasses from being suppressed and retried. + """ + raise_subcls = kwargs.pop('raise_subcls', ()) + opener = functools.partial(_open_url_once, *args, **kwargs) + + def on_exception(): + type_, exc = sys.exc_info()[:2] + if isinstance(exc, raise_subcls): + raise + time.sleep(0.5) + + # Try up to 10 times + return jaraco.functools.retry_call( + opener, + retries=9, + cleanup=on_exception, + trap=socket.error, + ) + + +def _open_url_once( + url, headers=None, method='GET', body=None, + host='127.0.0.1', port=8000, http_conn=http_client.HTTPConnection, + protocol='HTTP/1.1', ssl_context=None, +): + """Open the given HTTP resource and return status, headers, and body.""" + headers = cleanHeaders(headers, method, body, host, port) - return s, h, b - except socket.error as e: - if raise_subcls is not None and isinstance(e, raise_subcls): - raise - else: - time.sleep(0.5) - if trial == 9: - raise + # Allow http_conn to be a class or an instance + if hasattr(http_conn, 'host'): + conn = http_conn + else: + kw = {} + if ssl_context: + kw['context'] = ssl_context + conn = http_conn(interface(host), port, **kw) + conn._http_vsn_str = protocol + conn._http_vsn = int(''.join([x for x in protocol if x.isdigit()])) + if not six.PY2 and isinstance(url, bytes): + url = url.decode() + conn.putrequest( + method.upper(), url, skip_host=True, + skip_accept_encoding=True, + ) + for key, value in headers: + conn.putheader(key, value.encode('Latin-1')) + conn.endheaders() + if body is not None: + conn.send(body) + # Handle response + response = conn.getresponse() + s, h, b = shb(response) + if not hasattr(http_conn, 'host'): + # We made our own conn instance. Close it. + conn.close() + return s, h, b + + +def strip_netloc(url): + """Return absolute-URI path from URL. + + Strip the scheme and host from the URL, returning the + server-absolute portion. + + Useful for wrapping an absolute-URI for which only the + path is expected (such as in calls to :py:meth:`WebCase.getPage`). + + >>> strip_netloc('https://google.com/foo/bar?bing#baz') + '/foo/bar?bing' + + >>> strip_netloc('//google.com/foo/bar?bing#baz') + '/foo/bar?bing' + + >>> strip_netloc('/foo/bar?bing#baz') + '/foo/bar?bing' + """ + parsed = urllib_parse.urlparse(url) + scheme, netloc, path, params, query, fragment = parsed + stripped = '', '', path, params, query, '' + return urllib_parse.urlunparse(stripped) # Add any exceptions which your web framework handles @@ -591,6 +585,8 @@ def openURL(url, headers=None, method='GET', body=None, class ServerError(Exception): + """Exception for signalling server error.""" + on = False diff --git a/lib/cheroot/testing.py b/lib/cheroot/testing.py new file mode 100644 index 0000000..94bb773 --- /dev/null +++ b/lib/cheroot/testing.py @@ -0,0 +1,153 @@ +"""Pytest fixtures and other helpers for doing testing by end-users.""" + +from __future__ import absolute_import, division, print_function +__metaclass__ = type + +from contextlib import closing +import errno +import socket +import threading +import time + +import pytest +from six.moves import http_client + +import cheroot.server +from cheroot.test import webtest +import cheroot.wsgi + +EPHEMERAL_PORT = 0 +NO_INTERFACE = None # Using this or '' will cause an exception +ANY_INTERFACE_IPV4 = '0.0.0.0' +ANY_INTERFACE_IPV6 = '::' + +config = { + cheroot.wsgi.Server: { + 'bind_addr': (NO_INTERFACE, EPHEMERAL_PORT), + 'wsgi_app': None, + }, + cheroot.server.HTTPServer: { + 'bind_addr': (NO_INTERFACE, EPHEMERAL_PORT), + 'gateway': cheroot.server.Gateway, + }, +} + + +def cheroot_server(server_factory): + """Set up and tear down a Cheroot server instance.""" + conf = config[server_factory].copy() + bind_port = conf.pop('bind_addr')[-1] + + for interface in ANY_INTERFACE_IPV6, ANY_INTERFACE_IPV4: + try: + actual_bind_addr = (interface, bind_port) + httpserver = server_factory( # create it + bind_addr=actual_bind_addr, + **conf + ) + except OSError: + pass + else: + break + + httpserver.shutdown_timeout = 0 # Speed-up tests teardown + + threading.Thread(target=httpserver.safe_start).start() # spawn it + while not httpserver.ready: # wait until fully initialized and bound + time.sleep(0.1) + + yield httpserver + + httpserver.stop() # destroy it + + +@pytest.fixture(scope='module') +def wsgi_server(): + """Set up and tear down a Cheroot WSGI server instance.""" + for srv in cheroot_server(cheroot.wsgi.Server): + yield srv + + +@pytest.fixture(scope='module') +def native_server(): + """Set up and tear down a Cheroot HTTP server instance.""" + for srv in cheroot_server(cheroot.server.HTTPServer): + yield srv + + +class _TestClient: + def __init__(self, server): + self._interface, self._host, self._port = _get_conn_data( + server.bind_addr, + ) + self.server_instance = server + self._http_connection = self.get_connection() + + def get_connection(self): + name = '{interface}:{port}'.format( + interface=self._interface, + port=self._port, + ) + conn_cls = ( + http_client.HTTPConnection + if self.server_instance.ssl_adapter is None else + http_client.HTTPSConnection + ) + return conn_cls(name) + + def request( + self, uri, method='GET', headers=None, http_conn=None, + protocol='HTTP/1.1', + ): + return webtest.openURL( + uri, method=method, + headers=headers, + host=self._host, port=self._port, + http_conn=http_conn or self._http_connection, + protocol=protocol, + ) + + def __getattr__(self, attr_name): + def _wrapper(uri, **kwargs): + http_method = attr_name.upper() + return self.request(uri, method=http_method, **kwargs) + + return _wrapper + + +def _probe_ipv6_sock(interface): + # Alternate way is to check IPs on interfaces using glibc, like: + # github.com/Gautier/minifail/blob/master/minifail/getifaddrs.py + try: + with closing(socket.socket(family=socket.AF_INET6)) as sock: + sock.bind((interface, 0)) + except (OSError, socket.error) as sock_err: + # In Python 3 socket.error is an alias for OSError + # In Python 2 socket.error is a subclass of IOError + if sock_err.errno != errno.EADDRNOTAVAIL: + raise + else: + return True + + return False + + +def _get_conn_data(bind_addr): + if isinstance(bind_addr, tuple): + host, port = bind_addr + else: + host, port = bind_addr, 0 + + interface = webtest.interface(host) + + if ':' in interface and not _probe_ipv6_sock(interface): + interface = '127.0.0.1' + if ':' in host: + host = interface + + return interface, host, port + + +def get_server_client(server): + """Create and return a test client for the given server.""" + return _TestClient(server) diff --git a/lib/cheroot/workers/threadpool.py b/lib/cheroot/workers/threadpool.py index a4bbbb8..6e6c721 100644 --- a/lib/cheroot/workers/threadpool.py +++ b/lib/cheroot/workers/threadpool.py @@ -1,18 +1,25 @@ """A thread-based worker pool.""" +from __future__ import absolute_import, division, print_function +__metaclass__ = type + +import collections import threading import time import socket +import warnings from six.moves import queue +from jaraco.functools import pass_none + __all__ = ('WorkerThread', 'ThreadPool') -class TrueyZero(object): - """An object which equals and does math like the integer 0 but evals True.""" +class TrueyZero: + """Object which equals and does math like the integer 0 but evals True.""" def __add__(self, other): return other @@ -50,7 +57,8 @@ def __init__(self, server): """Initialize WorkerThread instance. Args: - server (cheroot.server.HTTPServer): web server object receiving this request + server (cheroot.server.HTTPServer): web server object + receiving this request """ self.ready = False self.server = server @@ -62,34 +70,39 @@ def __init__(self, server): self.work_time = 0 self.stats = { 'Requests': lambda s: self.requests_seen + ( - (self.start_time is None) and - trueyzero or - self.conn.requests_seen + self.start_time is None + and trueyzero + or self.conn.requests_seen ), 'Bytes Read': lambda s: self.bytes_read + ( - (self.start_time is None) and - trueyzero or - self.conn.rfile.bytes_read + self.start_time is None + and trueyzero + or self.conn.rfile.bytes_read ), 'Bytes Written': lambda s: self.bytes_written + ( - (self.start_time is None) and - trueyzero or - self.conn.wfile.bytes_written + self.start_time is None + and trueyzero + or self.conn.wfile.bytes_written ), 'Work Time': lambda s: self.work_time + ( - (self.start_time is None) and - trueyzero or - time.time() - self.start_time + self.start_time is None + and trueyzero + or time.time() - self.start_time ), 'Read Throughput': lambda s: s['Bytes Read'](s) / ( - s['Work Time'](s) or 1e-6), + s['Work Time'](s) or 1e-6 + ), 'Write Throughput': lambda s: s['Bytes Written'](s) / ( - s['Work Time'](s) or 1e-6), + s['Work Time'](s) or 1e-6 + ), } threading.Thread.__init__(self) def run(self): - """Process incoming HTTP connections retrieving them from thread pool.""" + """Process incoming HTTP connections. + + Retrieves incoming connections from thread pool. + """ self.server.stats['Worker Threads'][self.getName()] = self.stats try: self.ready = True @@ -99,13 +112,18 @@ def run(self): return self.conn = conn - if self.server.stats['Enabled']: + is_stats_enabled = self.server.stats['Enabled'] + if is_stats_enabled: self.start_time = time.time() + keep_conn_open = False try: - conn.communicate() + keep_conn_open = conn.communicate() finally: - conn.close() - if self.server.stats['Enabled']: + if keep_conn_open: + self.server.put_conn(conn) + else: + conn.close() + if is_stats_enabled: self.requests_seen += self.conn.requests_seen self.bytes_read += self.conn.rfile.bytes_read self.bytes_written += self.conn.wfile.bytes_written @@ -116,22 +134,28 @@ def run(self): self.server.interrupt = ex -class ThreadPool(object): +class ThreadPool: """A Request Queue for an HTTPServer which pools threads. ThreadPool objects must provide min, get(), put(obj), start() and stop(timeout) attributes. """ - def __init__(self, server, min=10, max=-1, accepted_queue_size=-1, accepted_queue_timeout=10): + def __init__( + self, server, min=10, max=-1, accepted_queue_size=-1, + accepted_queue_timeout=10, + ): """Initialize HTTP requests queue instance. Args: - server (cheroot.server.HTTPServer): web server object receiving this request + server (cheroot.server.HTTPServer): web server object + receiving this request min (int): minimum number of worker threads max (int): maximum number of worker threads - accepted_queue_size (int): maximum number of active requests in queue - accepted_queue_timeout (int): timeout for putting request into queue + accepted_queue_size (int): maximum number of active + requests in queue + accepted_queue_timeout (int): timeout for putting request + into queue """ self.server = server self.min = min @@ -140,32 +164,45 @@ def __init__(self, server, min=10, max=-1, accepted_queue_size=-1, accepted_queu self._queue = queue.Queue(maxsize=accepted_queue_size) self._queue_put_timeout = accepted_queue_timeout self.get = self._queue.get + self._pending_shutdowns = collections.deque() def start(self): """Start the pool of threads.""" for i in range(self.min): self._threads.append(WorkerThread(self.server)) for worker in self._threads: - worker.setName('CP Server ' + worker.getName()) + worker.setName( + 'CP Server {worker_name!s}'. + format(worker_name=worker.getName()), + ) worker.start() for worker in self._threads: while not worker.ready: time.sleep(.1) - def _get_idle(self): + @property + def idle(self): # noqa: D401; irrelevant for properties """Number of worker threads which are idle. Read-only.""" - return len([t for t in self._threads if t.conn is None]) - idle = property(_get_idle, doc=_get_idle.__doc__) + idles = len([t for t in self._threads if t.conn is None]) + return max(idles - len(self._pending_shutdowns), 0) def put(self, obj): """Put request into queue. Args: - obj (cheroot.server.HTTPConnection): HTTP connection waiting to be processed + obj (:py:class:`~cheroot.server.HTTPConnection`): HTTP connection + waiting to be processed """ self._queue.put(obj, block=True, timeout=self._queue_put_timeout) - if obj is _SHUTDOWNREQUEST: - return + + def _clear_dead_threads(self): + # Remove any dead threads from our list + for t in [t for t in self._threads if not t.is_alive()]: + self._threads.remove(t) + try: + self._pending_shutdowns.popleft() + except IndexError: + pass def grow(self, amount): """Spawn new worker threads (not above self.max).""" @@ -184,7 +221,10 @@ def grow(self, amount): def _spawn_worker(self): worker = WorkerThread(self.server) - worker.setName('CP Server ' + worker.getName()) + worker.setName( + 'CP Server {worker_name!s}'. + format(worker_name=worker.getName()), + ) worker.start() return worker @@ -192,10 +232,10 @@ def shrink(self, amount): """Kill off worker threads (not below self.min).""" # Grow/shrink the pool if necessary. # Remove any dead threads from our list - for t in self._threads: - if not t.isAlive(): - self._threads.remove(t) - amount -= 1 + amount -= len(self._pending_shutdowns) + self._clear_dead_threads() + if amount <= 0: + return # calculate the number of threads above the minimum n_extra = max(len(self._threads) - self.min, 0) @@ -207,6 +247,7 @@ def shrink(self, amount): # to remove. As each request is processed by a worker, that worker # will terminate and be culled from the list. for n in range(n_to_remove): + self._pending_shutdowns.append(None) self._queue.put(_SHUTDOWNREQUEST) def stop(self, timeout=5): @@ -215,43 +256,69 @@ def stop(self, timeout=5): Args: timeout (int): time to wait for threads to stop gracefully """ + # for compatability, negative timeouts are treated like None + # TODO: treat negative timeouts like already expired timeouts + if timeout is not None and timeout < 0: + timeout = None + warnings.warning( + 'In the future, negative timeouts to Server.stop() ' + 'will be equivalent to a timeout of zero.', + stacklevel=2, + ) + + if timeout is not None: + endtime = time.time() + timeout + # Must shut down threads here so the code that calls # this method can know when all threads are stopped. for worker in self._threads: self._queue.put(_SHUTDOWNREQUEST) - # Don't join currentThread (when stop is called inside a request). - current = threading.currentThread() - if timeout is not None and timeout >= 0: - endtime = time.time() + timeout - while self._threads: - worker = self._threads.pop() - if worker is not current and worker.isAlive(): - try: - if timeout is None or timeout < 0: - worker.join() - else: - remaining_time = endtime - time.time() - if remaining_time > 0: - worker.join(remaining_time) - if worker.isAlive(): - # We exhausted the timeout. - # Forcibly shut down the socket. - c = worker.conn - if c and not c.rfile.closed: - try: - c.socket.shutdown(socket.SHUT_RD) - except TypeError: - # pyOpenSSL sockets don't take an arg - c.socket.shutdown() - worker.join() - except (AssertionError, - # Ignore repeated Ctrl-C. - # See - # https://github.com/cherrypy/cherrypy/issues/691. - KeyboardInterrupt): - pass - - def _get_qsize(self): + ignored_errors = ( + # TODO: explain this exception. + AssertionError, + # Ignore repeated Ctrl-C. See cherrypy#691. + KeyboardInterrupt, + ) + + for worker in self._clear_threads(): + remaining_time = timeout and endtime - time.time() + try: + worker.join(remaining_time) + if worker.is_alive(): + # Timeout exhausted; forcibly shut down the socket. + self._force_close(worker.conn) + worker.join() + except ignored_errors: + pass + + @staticmethod + @pass_none + def _force_close(conn): + if conn.rfile.closed: + return + try: + try: + conn.socket.shutdown(socket.SHUT_RD) + except TypeError: + # pyOpenSSL sockets don't take an arg + conn.socket.shutdown() + except OSError: + # shutdown sometimes fails (race with 'closed' check?) + # ref #238 + pass + + def _clear_threads(self): + """Clear self._threads and yield all joinable threads.""" + # threads = pop_all(self._threads) + threads, self._threads[:] = self._threads[:], [] + return ( + thread + for thread in threads + if thread is not threading.currentThread() + ) + + @property + def qsize(self): + """Return the queue size.""" return self._queue.qsize() - qsize = property(_get_qsize) diff --git a/lib/cheroot/wsgi.py b/lib/cheroot/wsgi.py index 65f988c..9ed949c 100644 --- a/lib/cheroot/wsgi.py +++ b/lib/cheroot/wsgi.py @@ -8,7 +8,7 @@ def my_crazy_app(environ, start_response): status = '200 OK' response_headers = [('Content-type','text/plain')] start_response(status, response_headers) - return ['Hello world!'] + return [b'Hello world!'] addr = '0.0.0.0', 8070 server = wsgi.Server(addr, my_crazy_app) @@ -25,6 +25,9 @@ def my_crazy_app(environ, start_response): server = wsgi.Server(addr, d) """ +from __future__ import absolute_import, division, print_function +__metaclass__ = type + import sys import six @@ -41,27 +44,37 @@ class Server(server.HTTPServer): wsgi_version = (1, 0) """The version of WSGI to produce.""" - def __init__(self, bind_addr, wsgi_app, numthreads=10, server_name=None, - max=-1, request_queue_size=5, timeout=10, shutdown_timeout=5, - accepted_queue_size=-1, accepted_queue_timeout=10): + def __init__( + self, bind_addr, wsgi_app, numthreads=10, server_name=None, + max=-1, request_queue_size=5, timeout=10, shutdown_timeout=5, + accepted_queue_size=-1, accepted_queue_timeout=10, + peercreds_enabled=False, peercreds_resolve_enabled=False, + ): """Initialize WSGI Server instance. Args: bind_addr (tuple): network interface to listen to wsgi_app (callable): WSGI application callable numthreads (int): number of threads for WSGI thread pool - server_name (str): web server name to be advertised via Server HTTP header + server_name (str): web server name to be advertised via + Server HTTP header max (int): maximum number of worker threads - request_queue_size (int): the 'backlog' arg to socket.listen(); max queued connections + request_queue_size (int): the 'backlog' arg to + socket.listen(); max queued connections timeout (int): the timeout in seconds for accepted connections - shutdown_timeout (int): the total time, in seconds, to wait for worker threads to cleanly exit - accepted_queue_size (int): maximum number of active requests in queue - accepted_queue_timeout (int): timeout for putting request into queue + shutdown_timeout (int): the total time, in seconds, to + wait for worker threads to cleanly exit + accepted_queue_size (int): maximum number of active + requests in queue + accepted_queue_timeout (int): timeout for putting request + into queue """ super(Server, self).__init__( bind_addr, gateway=wsgi_gateways[self.wsgi_version], server_name=server_name, + peercreds_enabled=peercreds_enabled, + peercreds_resolve_enabled=peercreds_resolve_enabled, ) self.wsgi_app = wsgi_app self.request_queue_size = request_queue_size @@ -70,14 +83,17 @@ def __init__(self, bind_addr, wsgi_app, numthreads=10, server_name=None, self.requests = threadpool.ThreadPool( self, min=numthreads or 1, max=max, accepted_queue_size=accepted_queue_size, - accepted_queue_timeout=accepted_queue_timeout) + accepted_queue_timeout=accepted_queue_timeout, + ) - def _get_numthreads(self): + @property + def numthreads(self): + """Set minimum number of threads.""" return self.requests.min - def _set_numthreads(self, value): + @numthreads.setter + def numthreads(self, value): self.requests.min = value - numthreads = property(_get_numthreads, _set_numthreads) class Gateway(server.Gateway): @@ -89,7 +105,7 @@ def __init__(self, req): Args: req (HTTPRequest): current HTTP request """ - self.req = req + super(Gateway, self).__init__(req) self.started_response = False self.env = self.get_environ() self.remaining_bytes_out = None @@ -99,21 +115,20 @@ def gateway_map(cls): """Create a mapping of gateways and their versions. Returns: - dict[tuple[int,int],class]: map of gateway version and corresponding class + dict[tuple[int,int],class]: map of gateway version and + corresponding class + """ - return dict( - (gw.version, gw) - for gw in cls.__subclasses__() - ) + return {gw.version: gw for gw in cls.__subclasses__()} def get_environ(self): """Return a new environ dict targeting the given wsgi.version.""" - raise NotImplemented + raise NotImplementedError # pragma: no cover def respond(self): """Process the current request. - From PEP 333: + From :pep:`333`: The start_response callable must not actually transmit the response headers. Instead, it must store them for the @@ -129,6 +144,8 @@ def respond(self): raise ValueError('WSGI Applications must yield bytes') self.write(chunk) finally: + # Send headers if not already sent + self.req.ensure_headers_sent() if hasattr(response, 'close'): response.close() @@ -137,8 +154,10 @@ def start_response(self, status, headers, exc_info=None): # "The application may call start_response more than once, # if and only if the exc_info argument is provided." if self.started_response and not exc_info: - raise AssertionError('WSGI start_response called a second ' - 'time with no exc_info.') + raise AssertionError( + 'WSGI start_response called a second ' + 'time with no exc_info.', + ) self.started_response = True # "if exc_info is provided, and the HTTP headers have already been @@ -155,10 +174,12 @@ def start_response(self, status, headers, exc_info=None): for k, v in headers: if not isinstance(k, str): raise TypeError( - 'WSGI response header key %r is not of type str.' % k) + 'WSGI response header key %r is not of type str.' % k, + ) if not isinstance(v, str): raise TypeError( - 'WSGI response header value %r is not of type str.' % v) + 'WSGI response header value %r is not of type str.' % v, + ) if k.lower() == 'content-length': self.remaining_bytes_out = int(v) out_header = ntob(k), ntob(v) @@ -170,8 +191,8 @@ def start_response(self, status, headers, exc_info=None): def _encode_status(status): """Cast status to bytes representation of current Python version. - According to PEP 3333, when using Python 3, the response status - and headers must be bytes masquerading as unicode; that is, they + According to :pep:`3333`, when using Python 3, the response status + and headers must be bytes masquerading as Unicode; that is, they must be of type "str" but are restricted to code points in the "latin-1" set. """ @@ -198,15 +219,14 @@ def write(self, chunk): self.req.simple_response( '500 Internal Server Error', 'The requested resource returned more bytes than the ' - 'declared Content-Length.') + 'declared Content-Length.', + ) else: # Dang. We have probably already sent data. Truncate the chunk # to fit (so the client doesn't hang) and raise an error later. chunk = chunk[:rbo] - if not self.req.sent_headers: - self.req.sent_headers = True - self.req.send_headers() + self.req.ensure_headers_sent() self.req.write(chunk) @@ -214,7 +234,8 @@ def write(self, chunk): rbo -= chunklen if rbo < 0: raise ValueError( - 'Response body exceeds the declared Content-Length.') + 'Response body exceeds the declared Content-Length.', + ) class Gateway_10(Gateway): @@ -225,6 +246,7 @@ class Gateway_10(Gateway): def get_environ(self): """Return a new environ dict targeting the given wsgi.version.""" req = self.req + req_conn = req.conn env = { # set a non-standard environ entry so the WSGI app can know what # the *real* server protocol is (and what features to support). @@ -232,8 +254,8 @@ def get_environ(self): 'ACTUAL_SERVER_PROTOCOL': req.server.protocol, 'PATH_INFO': bton(req.path), 'QUERY_STRING': bton(req.qs), - 'REMOTE_ADDR': req.conn.remote_addr or '', - 'REMOTE_PORT': str(req.conn.remote_port or ''), + 'REMOTE_ADDR': req_conn.remote_addr or '', + 'REMOTE_PORT': str(req_conn.remote_port or ''), 'REQUEST_METHOD': bton(req.method), 'REQUEST_URI': bton(req.uri), 'SCRIPT_NAME': '', @@ -243,6 +265,7 @@ def get_environ(self): 'SERVER_SOFTWARE': req.server.software, 'wsgi.errors': sys.stderr, 'wsgi.input': req.rfile, + 'wsgi.input_terminated': bool(req.chunked_read), 'wsgi.multiprocess': False, 'wsgi.multithread': True, 'wsgi.run_once': False, @@ -254,12 +277,31 @@ def get_environ(self): # AF_UNIX. This isn't really allowed by WSGI, which doesn't # address unix domain sockets. But it's better than nothing. env['SERVER_PORT'] = '' + try: + env['X_REMOTE_PID'] = str(req_conn.peer_pid) + env['X_REMOTE_UID'] = str(req_conn.peer_uid) + env['X_REMOTE_GID'] = str(req_conn.peer_gid) + + env['X_REMOTE_USER'] = str(req_conn.peer_user) + env['X_REMOTE_GROUP'] = str(req_conn.peer_group) + + env['REMOTE_USER'] = env['X_REMOTE_USER'] + except RuntimeError: + """Unable to retrieve peer creds data. + + Unsupported by current kernel or socket error happened, or + unsupported socket type, or disabled. + """ else: env['SERVER_PORT'] = str(req.server.bind_addr[1]) # Request headers env.update( - ('HTTP_' + bton(k).upper().replace('-', '_'), bton(v)) + ( + 'HTTP_{header_name!s}'. + format(header_name=bton(k).upper().replace('-', '_')), + bton(v), + ) for k, v in req.inheaders.items() ) @@ -280,7 +322,7 @@ def get_environ(self): class Gateway_u0(Gateway_10): """A Gateway class to interface HTTPServer with WSGI u.0. - WSGI u.0 is an experimental protocol, which uses unicode for keys + WSGI u.0 is an experimental protocol, which uses Unicode for keys and values in both Python 2 and Python 3. """ @@ -289,7 +331,7 @@ class Gateway_u0(Gateway_10): def get_environ(self): """Return a new environ dict targeting the given wsgi.version.""" req = self.req - env_10 = super(Gateway_u0, self).get_environ(self) + env_10 = super(Gateway_u0, self).get_environ() env = dict(map(self._decode_key, env_10.items())) # Request-URI @@ -318,7 +360,7 @@ def _decode_key(item): def _decode_value(item): k, v = item skip_keys = 'REQUEST_URI', 'wsgi.input' - if six.PY3 or not isinstance(v, bytes) or k in skip_keys: + if not six.PY2 or not isinstance(v, bytes) or k in skip_keys: return k, v return k, v.decode('ISO-8859-1') @@ -326,14 +368,15 @@ def _decode_value(item): wsgi_gateways = Gateway.gateway_map() -class PathInfoDispatcher(object): +class PathInfoDispatcher: """A WSGI dispatcher for dispatch based on the PATH_INFO.""" def __init__(self, apps): """Initialize path info WSGI app dispatcher. Args: - apps (dict[str,object]|list[tuple[str,object]]): URI prefix and WSGI app pairs + apps (dict[str,object]|list[tuple[str,object]]): URI prefix + and WSGI app pairs """ try: apps = list(apps.items()) @@ -341,7 +384,8 @@ def __init__(self, apps): pass # Sort the apps by len(path), descending - by_path_len = lambda app: len(app[0]) + def by_path_len(app): + return len(app[0]) apps.sort(key=by_path_len, reverse=True) # The path_prefix strings must start, but not end, with a slash. @@ -351,26 +395,33 @@ def __init__(self, apps): def __call__(self, environ, start_response): """Process incoming WSGI request. - Ref: PEP 3333 + Ref: :pep:`3333` Args: environ (Mapping): a dict containing WSGI environment variables - start_response (callable): function, which sets response status and headers + start_response (callable): function, which sets response + status and headers Returns: - list[bytes]: iterable containing bytes to be returned in HTTP response body + list[bytes]: iterable containing bytes to be returned in + HTTP response body + """ path = environ['PATH_INFO'] or '/' for p, app in self.apps: # The apps list should be sorted by length, descending. - if path.startswith(p + '/') or path == p: + if path.startswith('{path!s}/'.format(path=p)) or path == p: environ = environ.copy() - environ['SCRIPT_NAME'] = environ['SCRIPT_NAME'] + p + environ['SCRIPT_NAME'] = environ.get('SCRIPT_NAME', '') + p environ['PATH_INFO'] = path[len(p):] return app(environ, start_response) - start_response('404 Not Found', [('Content-Type', 'text/plain'), - ('Content-Length', '0')]) + start_response( + '404 Not Found', [ + ('Content-Type', 'text/plain'), + ('Content-Length', '0'), + ], + ) return [''] diff --git a/lib/cherrypy/PKG-INFO b/lib/cherrypy/PKG-INFO new file mode 100644 index 0000000..284fe18 --- /dev/null +++ b/lib/cherrypy/PKG-INFO @@ -0,0 +1,141 @@ +Metadata-Version: 2.1 +Name: CherryPy +Version: 17.4.2 +Summary: Object-Oriented HTTP framework +Home-page: https://www.cherrypy.org +Author: CherryPy Team +Author-email: team@cherrypy.org +License: UNKNOWN +Project-URL: CI: AppVeyor, https://ci.appveyor.com/project/cherrypy/cherrypy +Project-URL: CI: Travis, https://travis-ci.org/cherrypy/cherrypy +Project-URL: CI: Circle, https://circleci.com/gh/cherrypy/cherrypy +Project-URL: Docs: RTD, https://docs.cherrypy.org +Project-URL: GitHub: issues, https://github.com/cherrypy/cherrypy/issues +Project-URL: GitHub: repo, https://github.com/cherrypy/cherrypy +Description: .. image:: https://img.shields.io/pypi/v/cherrypy.svg + :target: https://pypi.org/project/cherrypy + + .. image:: https://readthedocs.org/projects/cherrypy/badge/?version=latest + :target: https://docs.cherrypy.org/en/latest/?badge=latest + + .. image:: https://img.shields.io/badge/StackOverflow-CherryPy-blue.svg + :target: https://stackoverflow.com/questions/tagged/cheroot+or+cherrypy + + .. image:: https://img.shields.io/gitter/room/cherrypy/cherrypy.svg + :target: https://gitter.im/cherrypy/cherrypy + + .. image:: https://img.shields.io/travis/cherrypy/cherrypy/master.svg?label=Linux%20build%20%40%20Travis%20CI + :target: https://travis-ci.org/cherrypy/cherrypy + + .. image:: https://circleci.com/gh/cherrypy/cherrypy/tree/master.svg?style=svg + :target: https://circleci.com/gh/cherrypy/cherrypy/tree/master + + .. image:: https://img.shields.io/appveyor/ci/CherryPy/cherrypy/master.svg?label=Windows%20build%20%40%20Appveyor + :target: https://ci.appveyor.com/project/CherryPy/cherrypy/branch/master + + .. image:: https://img.shields.io/badge/license-BSD-blue.svg?maxAge=3600 + :target: https://pypi.org/project/cheroot + + .. image:: https://img.shields.io/pypi/pyversions/cherrypy.svg + :target: https://pypi.org/project/cherrypy + + .. image:: https://badges.github.io/stability-badges/dist/stable.svg + :target: https://github.com/badges/stability-badges + :alt: stable + + .. image:: https://api.codacy.com/project/badge/Grade/48b11060b5d249dc86e52dac2be2c715 + :target: https://www.codacy.com/app/webknjaz/cherrypy-upstream?utm_source=github.com&utm_medium=referral&utm_content=cherrypy/cherrypy&utm_campaign=Badge_Grade + + .. image:: https://codecov.io/gh/cherrypy/cherrypy/branch/master/graph/badge.svg + :target: https://codecov.io/gh/cherrypy/cherrypy + :alt: codecov + + Welcome to the GitHub repository of `CherryPy `_! + + CherryPy is a pythonic, object-oriented HTTP framework. + + 1. It allows building web applications in much the same way one would + build any other object-oriented program. + 2. This design results in less and more readable code being developed faster. + It's all just properties and methods. + 3. It is now more than ten years old and has proven fast and very + stable. + 4. It is being used in production by many sites, from the simplest to + the most demanding. + 5. And perhaps most importantly, it is fun to work with :-) + + Here's how easy it is to write "Hello World" in CherryPy: + + .. code:: python + + import cherrypy + + class HelloWorld(object): + @cherrypy.expose + def index(self): + return "Hello World!" + + cherrypy.quickstart(HelloWorld()) + + And it continues to work that intuitively when systems grow, allowing + for the Python object model to be dynamically presented as a web site + and/or API. + + While CherryPy is one of the easiest and most intuitive frameworks out + there, the prerequisite for understanding the `CherryPy + documentation `_ is that you have + a general understanding of Python and web development. + Additionally: + + - Tutorials are included in the repository: + https://github.com/cherrypy/cherrypy/tree/master/cherrypy/tutorial + - A general wiki at: + https://github.com/cherrypy/cherrypy/wiki + + If the docs are insufficient to address your needs, the CherryPy + community has several `avenues for support + `_. + + Contributing + ------------ + + Please follow the `contribution guidelines + `_. + And by all means, absorb the `Zen of + CherryPy `_. + +Platform: UNKNOWN +Classifier: Development Status :: 5 - Production/Stable +Classifier: Environment :: Web Environment +Classifier: Intended Audience :: Developers +Classifier: License :: Freely Distributable +Classifier: Operating System :: OS Independent +Classifier: Framework :: CherryPy +Classifier: License :: OSI Approved :: BSD License +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 2 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.4 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: Implementation +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python :: Implementation :: Jython +Classifier: Programming Language :: Python :: Implementation :: PyPy +Classifier: Topic :: Internet :: WWW/HTTP +Classifier: Topic :: Internet :: WWW/HTTP :: Dynamic Content +Classifier: Topic :: Internet :: WWW/HTTP :: HTTP Servers +Classifier: Topic :: Internet :: WWW/HTTP :: WSGI +Classifier: Topic :: Internet :: WWW/HTTP :: WSGI :: Application +Classifier: Topic :: Internet :: WWW/HTTP :: WSGI :: Server +Classifier: Topic :: Software Development :: Libraries :: Application Frameworks +Requires-Python: >=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.* +Provides-Extra: routes_dispatcher +Provides-Extra: ssl +Provides-Extra: memcached_session +Provides-Extra: xcgi +Provides-Extra: json +Provides-Extra: docs +Provides-Extra: testing diff --git a/lib/cherrypy/__init__.py b/lib/cherrypy/__init__.py index 099990d..8e27c81 100644 --- a/lib/cherrypy/__init__.py +++ b/lib/cherrypy/__init__.py @@ -1,6 +1,5 @@ """CherryPy is a pythonic, object-oriented HTTP framework. - CherryPy consists of not one, but four separate API layers. The APPLICATION LAYER is the simplest. CherryPy applications are written as @@ -53,7 +52,8 @@ * Server API * WSGI API -These API's are described in the `CherryPy specification `_. +These API's are described in the `CherryPy specification +`_. """ try: @@ -63,33 +63,47 @@ from threading import local as _local -from cherrypy._cperror import HTTPError, HTTPRedirect, InternalRedirect # noqa: F401 -from cherrypy._cperror import NotFound, CherryPyException, TimeoutError # noqa: F401 +from ._cperror import ( + HTTPError, HTTPRedirect, InternalRedirect, + NotFound, CherryPyException, +) -from cherrypy import _cplogging +from . import _cpdispatch as dispatch -from cherrypy import _cpdispatch as dispatch # noqa: F401 +from ._cptools import default_toolbox as tools, Tool +from ._helper import expose, popargs, url -from cherrypy import _cptools # noqa: F401 -from cherrypy._cptools import default_toolbox as tools, Tool # noqa: F401 +from . import _cprequest, _cpserver, _cptree, _cplogging, _cpconfig -from cherrypy import _cprequest -from cherrypy.lib import httputil as _httputil +import cherrypy.lib.httputil as _httputil -from cherrypy import _cptree -from cherrypy._cptree import Application # noqa -from cherrypy import _cpwsgi as wsgi # noqa +from ._cptree import Application +from . import _cpwsgi as wsgi -from cherrypy import _cpserver -from cherrypy import process +from . import process try: - from cherrypy.process import win32 + from .process import win32 engine = win32.Win32Bus() engine.console_control_handler = win32.ConsoleCtrlHandler(engine) del win32 except ImportError: engine = process.bus +from . import _cpchecker + +__all__ = ( + 'HTTPError', 'HTTPRedirect', 'InternalRedirect', + 'NotFound', 'CherryPyException', + 'dispatch', 'tools', 'Tool', 'Application', + 'wsgi', 'process', 'tree', 'engine', + 'quickstart', 'serving', 'request', 'response', 'thread_data', + 'log', 'expose', 'popargs', 'url', 'config', +) + + +__import__('cherrypy._cptools') +__import__('cherrypy._cprequest') + tree = _cptree.Tree() @@ -100,34 +114,10 @@ __version__ = 'unknown' -# Timeout monitor. We add two channels to the engine -# to which cherrypy.Application will publish. engine.listeners['before_request'] = set() engine.listeners['after_request'] = set() -class _TimeoutMonitor(process.plugins.Monitor): - - def __init__(self, bus): - self.servings = [] - process.plugins.Monitor.__init__(self, bus, self.run) - - def before_request(self): - self.servings.append((serving.request, serving.response)) - - def after_request(self): - try: - self.servings.remove((serving.request, serving.response)) - except ValueError: - pass - - def run(self): - """Check timeout on all responses. (Internal)""" - for req, resp in self.servings: - resp.check_timeout() -engine.timeout_monitor = _TimeoutMonitor(engine) # noqa: E305 -engine.timeout_monitor.subscribe() - engine.autoreload = process.plugins.Autoreloader(engine) engine.autoreload.subscribe() @@ -138,21 +128,23 @@ def run(self): class _HandleSignalsPlugin(object): + """Handle signals from other processes. - """Handle signals from other processes based on the configured - platform handlers above.""" + Based on the configured platform handlers above. + """ def __init__(self, bus): self.bus = bus def subscribe(self): - """Add the handlers based on the platform""" + """Add the handlers based on the platform.""" if hasattr(self.bus, 'signal_handler'): self.bus.signal_handler.subscribe() if hasattr(self.bus, 'console_control_handler'): self.bus.console_control_handler.subscribe() -engine.signals = _HandleSignalsPlugin(engine) # noqa: E305 + +engine.signals = _HandleSignalsPlugin(engine) server = _cpserver.Server() @@ -187,7 +179,6 @@ def quickstart(root=None, script_name='', config=None): class _Serving(_local): - """An interface for registering request and response objects. Rather than have a separate "thread local" object for the request and @@ -217,7 +208,8 @@ def clear(self): """Remove all attributes of self.""" self.__dict__.clear() -serving = _Serving() # noqa: E305 + +serving = _Serving() class _ThreadLocalProxy(object): @@ -242,12 +234,12 @@ def __delattr__(self, name): child = getattr(serving, self.__attrname__) delattr(child, name) - def _get_dict(self): + @property + def __dict__(self): child = getattr(serving, self.__attrname__) d = child.__class__.__dict__.copy() d.update(child.__dict__) return d - __dict__ = property(_get_dict) def __getitem__(self, key): child = getattr(serving, self.__attrname__) @@ -275,19 +267,21 @@ def __nonzero__(self): # Python 3 __bool__ = __nonzero__ + # Create request and response object (the same objects will be used # throughout the entire life of the webserver, but will redirect # to the "serving" object) -request = _ThreadLocalProxy('request') # noqa: E305 +request = _ThreadLocalProxy('request') response = _ThreadLocalProxy('response') # Create thread_data object as a thread-specific all-purpose storage class _ThreadData(_local): - """A container for thread-specific data.""" -thread_data = _ThreadData() # noqa: E305 + + +thread_data = _ThreadData() # Monkeypatch pydoc to allow help() to go through the threadlocal proxy. @@ -300,7 +294,8 @@ def _cherrypy_pydoc_resolve(thing, forceload=0): thing = getattr(serving, thing.__attrname__) return _pydoc._builtin_resolve(thing, forceload) -try: # noqa: E305 + +try: import pydoc as _pydoc _pydoc._builtin_resolve = _pydoc.resolve _pydoc.resolve = _cherrypy_pydoc_resolve @@ -309,7 +304,6 @@ def _cherrypy_pydoc_resolve(thing, forceload=0): class _GlobalLogManager(_cplogging.LogManager): - """A site-wide LogManager; routes to app.log or global log as appropriate. This :class:`LogManager` implements @@ -320,7 +314,10 @@ class _GlobalLogManager(_cplogging.LogManager): """ def __call__(self, *args, **kwargs): - """Log the given message to the app.log or global log as appropriate. + """Log the given message to the app.log or global log. + + Log the given message to the app.log or global + log as appropriate. """ # Do NOT use try/except here. See # https://github.com/cherrypy/cherrypy/issues/945 @@ -331,7 +328,10 @@ def __call__(self, *args, **kwargs): return log.error(*args, **kwargs) def access(self): - """Log an access message to the app.log or global log as appropriate. + """Log an access message to the app.log or global log. + + Log the given message to the app.log or global + log as appropriate. """ try: return request.app.log.access() @@ -347,14 +347,11 @@ def access(self): log.access_file = '' +@engine.subscribe('log') def _buslog(msg, level): log.error(msg, 'ENGINE', severity=level) -engine.subscribe('log', _buslog) # noqa: E305 -from cherrypy._helper import expose, popargs, url # noqa: F401 -# import _cpconfig last so it can reference other top-level objects -from cherrypy import _cpconfig # noqa: F401 # Use _global_conf_alias so quickstart can use 'config' as an arg # without shadowing cherrypy.config. config = _global_conf_alias = _cpconfig.Config() @@ -369,6 +366,5 @@ def _buslog(msg, level): # Must reset to get our defaults applied. config.reset() -from cherrypy import _cpchecker # noqa: F401 checker = _cpchecker.Checker() engine.subscribe('start', checker) diff --git a/lib/cherrypy/__main__.py b/lib/cherrypy/__main__.py old mode 100644 new mode 100755 index f8c016d..6674f7c --- a/lib/cherrypy/__main__.py +++ b/lib/cherrypy/__main__.py @@ -1,5 +1,5 @@ +"""CherryPy'd cherryd daemon runner.""" from cherrypy.daemon import run -if __name__ == '__main__': - run() +__name__ == '__main__' and run() diff --git a/lib/cherrypy/_cpchecker.py b/lib/cherrypy/_cpchecker.py index d67f9ad..39b7c97 100644 --- a/lib/cherrypy/_cpchecker.py +++ b/lib/cherrypy/_cpchecker.py @@ -1,12 +1,14 @@ +"""Checker for CherryPy sites and mounted apps.""" import os import warnings +import six +from six.moves import builtins + import cherrypy -from cherrypy._cpcompat import iteritems, copykeys, builtins class Checker(object): - """A checker for CherryPy sites and their mounted applications. When this object is called at engine startup, it executes each @@ -24,6 +26,7 @@ class Checker(object): """If True (the default), run all checks; if False, turn off all checks.""" def __init__(self): + """Initialize Checker instance.""" self._populate_known_types() def __call__(self): @@ -41,15 +44,14 @@ def __call__(self): warnings.formatwarning = oldformatwarning def formatwarning(self, message, category, filename, lineno, line=None): - """Function to format a warning.""" + """Format a warning.""" return 'CherryPy Checker:\n%s\n\n' % message # This value should be set inside _cpconfig. global_config_contained_paths = False def check_app_config_entries_dont_start_with_script_name(self): - """Check for Application config with sections that repeat script_name. - """ + """Check for App config with sections that repeat script_name.""" for sn, app in cherrypy.tree.apps.items(): if not isinstance(app, cherrypy.Application): continue @@ -68,14 +70,14 @@ def check_app_config_entries_dont_start_with_script_name(self): def check_site_config_entries_in_app_config(self): """Check for mounted Applications that have site-scoped config.""" - for sn, app in iteritems(cherrypy.tree.apps): + for sn, app in six.iteritems(cherrypy.tree.apps): if not isinstance(app, cherrypy.Application): continue msg = [] - for section, entries in iteritems(app.config): + for section, entries in six.iteritems(app.config): if section.startswith('/'): - for key, value in iteritems(entries): + for key, value in six.iteritems(entries): for n in ('engine.', 'server.', 'tree.', 'checker.'): if key.startswith(n): msg.append('[%s] %s = %s' % @@ -106,9 +108,7 @@ def check_skipped_app_config(self): return def check_app_config_brackets(self): - """Check for Application config with extraneous brackets in section - names. - """ + """Check for App config with extraneous brackets in section names.""" for sn, app in cherrypy.tree.apps.items(): if not isinstance(app, cherrypy.Application): continue @@ -196,7 +196,7 @@ def _compat(self, config): """Process config and warn on each obsolete or deprecated entry.""" for section, conf in config.items(): if isinstance(conf, dict): - for k, v in conf.items(): + for k in conf: if k in self.obsolete: warnings.warn('%r is obsolete. Use %r instead.\n' 'section: [%s]' % @@ -226,16 +226,16 @@ def check_compatibility(self): def _known_ns(self, app): ns = ['wsgi'] - ns.extend(copykeys(app.toolboxes)) - ns.extend(copykeys(app.namespaces)) - ns.extend(copykeys(app.request_class.namespaces)) - ns.extend(copykeys(cherrypy.config.namespaces)) + ns.extend(app.toolboxes) + ns.extend(app.namespaces) + ns.extend(app.request_class.namespaces) + ns.extend(cherrypy.config.namespaces) ns += self.extra_config_namespaces for section, conf in app.config.items(): is_path_section = section.startswith('/') if is_path_section and isinstance(conf, dict): - for k, v in conf.items(): + for k in conf: atoms = k.split('.') if len(atoms) > 1: if atoms[0] not in ns: @@ -295,16 +295,9 @@ def _known_types(self, config): 'which does not match the expected type %r.') for section, conf in config.items(): - if isinstance(conf, dict): - for k, v in conf.items(): - if v is not None: - expected_type = self.known_config_types.get(k, None) - vtype = type(v) - if expected_type and vtype != expected_type: - warnings.warn(msg % (k, section, vtype.__name__, - expected_type.__name__)) - else: - k, v = section, conf + if not isinstance(conf, dict): + conf = {section: conf} + for k, v in conf.items(): if v is not None: expected_type = self.known_config_types.get(k, None) vtype = type(v) diff --git a/lib/cherrypy/_cpcompat.py b/lib/cherrypy/_cpcompat.py index 06b430a..f454505 100644 --- a/lib/cherrypy/_cpcompat.py +++ b/lib/cherrypy/_cpcompat.py @@ -1,6 +1,6 @@ """Compatibility code for using CherryPy with various versions of Python. -CherryPy 3.2 is compatible with Python versions 2.6+. This module provides a +To retain compatibility with older Python versions, this module provides a useful abstraction over the differences between Python versions, sometimes by preferring a newer idiom, sometimes an older one, and sometimes a custom one. @@ -10,19 +10,21 @@ provides two functions: 'ntob', which translates native strings (of type 'str') into byte strings regardless of Python version, and 'ntou', which translates native -strings to unicode strings. This also provides a 'BytesIO' name for dealing -specifically with bytes, and a 'StringIO' name for dealing with native strings. -It also provides a 'base64_decode' function with native strings as input and -output. +strings to unicode strings. + +Try not to use the compatibility functions 'ntob', 'ntou', 'tonative'. +They were created with Python 2.3-2.5 compatibility in mind. +Instead, use unicode literals (from __future__) and bytes literals +and their .encode/.decode methods as needed. """ -import binascii -import os import re import sys import threading import six +from six.moves import urllib + if six.PY3: def ntob(n, encoding='ISO-8859-1'): @@ -91,203 +93,48 @@ def assert_native(n): raise TypeError('n must be a native str (got %s)' % type(n).__name__) -try: - # Python 3.1+ - from base64 import decodebytes as _base64_decodebytes -except ImportError: - # Python 3.0- - # since CherryPy claims compability with Python 2.3, we must use - # the legacy API of base64 - from base64 import decodestring as _base64_decodebytes +# Some platforms don't expose HTTPSConnection, so handle it separately +HTTPSConnection = getattr(six.moves.http_client, 'HTTPSConnection', None) -def base64_decode(n, encoding='ISO-8859-1'): - """Return the native string base64-decoded (as a native string).""" - if isinstance(n, six.text_type): - b = n.encode(encoding) - else: - b = n - b = _base64_decodebytes(b) - if str is six.text_type: - return b.decode(encoding) - else: - return b +def _unquote_plus_compat(string, encoding='utf-8', errors='replace'): + return urllib.parse.unquote_plus(string).decode(encoding, errors) -try: - sorted = sorted -except NameError: - def sorted(i): - i = i[:] - i.sort() - return i +def _unquote_compat(string, encoding='utf-8', errors='replace'): + return urllib.parse.unquote(string).decode(encoding, errors) -try: - reversed = reversed -except NameError: - def reversed(x): - i = len(x) - while i > 0: - i -= 1 - yield x[i] - -try: - # Python 3 - from urllib.parse import urljoin, urlencode - from urllib.parse import quote, quote_plus - from urllib.request import unquote, urlopen - from urllib.request import parse_http_list, parse_keqv_list -except ImportError: - # Python 2 - from urlparse import urljoin # noqa - from urllib import urlencode, urlopen # noqa - from urllib import quote, quote_plus # noqa - from urllib import unquote # noqa - from urllib2 import parse_http_list, parse_keqv_list # noqa - -try: - dict.iteritems - # Python 2 - iteritems = lambda d: d.iteritems() - copyitems = lambda d: d.items() -except AttributeError: - # Python 3 - iteritems = lambda d: d.items() - copyitems = lambda d: list(d.items()) -try: - dict.iterkeys - # Python 2 - iterkeys = lambda d: d.iterkeys() - copykeys = lambda d: d.keys() -except AttributeError: - # Python 3 - iterkeys = lambda d: d.keys() - copykeys = lambda d: list(d.keys()) +def _quote_compat(string, encoding='utf-8', errors='replace'): + return urllib.parse.quote(string.encode(encoding, errors)) -try: - dict.itervalues - # Python 2 - itervalues = lambda d: d.itervalues() - copyvalues = lambda d: d.values() -except AttributeError: - # Python 3 - itervalues = lambda d: d.values() - copyvalues = lambda d: list(d.values()) -try: - # Python 3 - import builtins -except ImportError: - # Python 2 - import __builtin__ as builtins # noqa +unquote_plus = urllib.parse.unquote_plus if six.PY3 else _unquote_plus_compat +unquote = urllib.parse.unquote if six.PY3 else _unquote_compat +quote = urllib.parse.quote if six.PY3 else _quote_compat try: - # Python 2. We try Python 2 first clients on Python 2 - # don't try to import the 'http' module from cherrypy.lib - from Cookie import SimpleCookie, CookieError - from httplib import BadStatusLine, HTTPConnection, IncompleteRead - from httplib import NotConnected - from BaseHTTPServer import BaseHTTPRequestHandler -except ImportError: - # Python 3 - from http.cookies import SimpleCookie, CookieError # noqa - from http.client import BadStatusLine, HTTPConnection, IncompleteRead # noqa - from http.client import NotConnected # noqa - from http.server import BaseHTTPRequestHandler # noqa - -# Some platforms don't expose HTTPSConnection, so handle it separately -if six.PY3: - try: - from http.client import HTTPSConnection - except ImportError: - # Some platforms which don't have SSL don't expose HTTPSConnection - HTTPSConnection = None -else: - try: - from httplib import HTTPSConnection - except ImportError: - HTTPSConnection = None - -try: - # Python 2 - xrange = xrange -except NameError: - # Python 3 - xrange = range - -try: - # Python 3 - from urllib.parse import unquote as parse_unquote - - def unquote_qs(atom, encoding, errors='strict'): - return parse_unquote( - atom.replace('+', ' '), - encoding=encoding, - errors=errors) -except ImportError: - # Python 2 - from urllib import unquote as parse_unquote - - def unquote_qs(atom, encoding, errors='strict'): - return parse_unquote(atom.replace('+', ' ')).decode(encoding, errors) - -try: - # Prefer simplejson, which is usually more advanced than the builtin - # module. + # Prefer simplejson import simplejson as json - json_decode = json.JSONDecoder().decode - _json_encode = json.JSONEncoder().iterencode except ImportError: - if sys.version_info >= (2, 6): - # Python >=2.6 : json is part of the standard library - import json - json_decode = json.JSONDecoder().decode - _json_encode = json.JSONEncoder().iterencode - else: - json = None - - def json_decode(s): - raise ValueError('No JSON library is available') - - def _json_encode(s): - raise ValueError('No JSON library is available') -finally: - if json and six.PY3: - # The two Python 3 implementations (simplejson/json) - # outputs str. We need bytes. - def json_encode(value): - for chunk in _json_encode(value): - yield chunk.encode('utf8') - else: - json_encode = _json_encode - -text_or_bytes = six.text_type, six.binary_type + import json -try: - import cPickle as pickle -except ImportError: - # In Python 2, pickle is a Python version. - # In Python 3, pickle is the sped-up C version. - import pickle # noqa +json_decode = json.JSONDecoder().decode +_json_encode = json.JSONEncoder().iterencode -def random20(): - return binascii.hexlify(os.urandom(20)).decode('ascii') +if six.PY3: + # Encode to bytes on Python 3 + def json_encode(value): + for chunk in _json_encode(value): + yield chunk.encode('utf-8') +else: + json_encode = _json_encode -try: - from _thread import get_ident as get_thread_ident -except ImportError: - from thread import get_ident as get_thread_ident # noqa -try: - # Python 3 - next = next -except NameError: - # Python 2 - def next(i): - return i.next() +text_or_bytes = six.text_type, bytes + if sys.version_info >= (3, 3): Timer = threading.Timer @@ -297,55 +144,17 @@ def next(i): Timer = threading._Timer Event = threading._Event -try: - # Python 2.7+ - from subprocess import _args_from_interpreter_flags -except ImportError: - def _args_from_interpreter_flags(): - """Tries to reconstruct original interpreter args from sys.flags for Python 2.6 - - Backported from Python 3.5. Aims to return a list of - command-line arguments reproducing the current - settings in sys.flags and sys.warnoptions. - """ - flag_opt_map = { - 'debug': 'd', - # 'inspect': 'i', - # 'interactive': 'i', - 'optimize': 'O', - 'dont_write_bytecode': 'B', - 'no_user_site': 's', - 'no_site': 'S', - 'ignore_environment': 'E', - 'verbose': 'v', - 'bytes_warning': 'b', - 'quiet': 'q', - 'hash_randomization': 'R', - 'py3k_warning': '3', - } - - args = [] - for flag, opt in flag_opt_map.items(): - v = getattr(sys.flags, flag) - if v > 0: - if flag == 'hash_randomization': - v = 1 # Handle specification of an exact seed - args.append('-' + opt * v) - for opt in sys.warnoptions: - args.append('-W' + opt) - - return args - # html module come in 3.2 version try: from html import escape except ImportError: from cgi import escape + # html module needed the argument quote=False because in cgi the default # is False. With quote=True the results differ. -def escape_html(s, escape_quote=False): # noqa: E302 +def escape_html(s, escape_quote=False): """Replace special characters "&", "<" and ">" to HTML-safe sequences. When escape_quote=True, escape (') and (") chars. diff --git a/lib/cherrypy/_cpconfig.py b/lib/cherrypy/_cpconfig.py index 37f0600..8e3fd61 100644 --- a/lib/cherrypy/_cpconfig.py +++ b/lib/cherrypy/_cpconfig.py @@ -122,8 +122,11 @@ def index(self): from cherrypy._cpcompat import text_or_bytes from cherrypy.lib import reprconf -# Deprecated in CherryPy 3.2--remove in 3.3 -NamespaceSet = reprconf.NamespaceSet + +def _if_filename_register_autoreload(ob): + """Register for autoreload if ob is a string (presumed filename).""" + is_filename = isinstance(ob, text_or_bytes) + is_filename and cherrypy.engine.autoreload.files.add(ob) def merge(base, other): @@ -132,11 +135,10 @@ def merge(base, other): If the given config is a filename, it will be appended to the list of files to monitor for "autoreload" changes. """ - if isinstance(other, text_or_bytes): - cherrypy.engine.autoreload.files.add(other) + _if_filename_register_autoreload(other) # Load other into base - for section, value_map in reprconf.as_dict(other).items(): + for section, value_map in reprconf.Parser.load(other).items(): if not isinstance(value_map, dict): raise ValueError( 'Application config must include section headers, but the ' @@ -147,15 +149,12 @@ def merge(base, other): class Config(reprconf.Config): - """The 'global' configuration data for the entire CherryPy process.""" def update(self, config): """Update self from a dict, file or filename.""" - if isinstance(config, text_or_bytes): - # Filename - cherrypy.engine.autoreload.files.add(config) - reprconf.Config.update(self, config) + _if_filename_register_autoreload(config) + super(Config, self).update(config) def _apply(self, config): """Update self from a dict.""" @@ -165,16 +164,11 @@ def _apply(self, config): config = config['global'] if 'tools.staticdir.dir' in config: config['tools.staticdir.section'] = 'global' - reprconf.Config._apply(self, config) + super(Config, self)._apply(config) @staticmethod - def __call__(*args, **kwargs): - """Decorator for page handlers to set _cp_config.""" - if args: - raise TypeError( - 'The cherrypy.config decorator does not accept positional ' - 'arguments; you must use keyword arguments.') - + def __call__(**kwargs): + """Decorate for page handlers to set _cp_config.""" def tool_decorator(f): _Vars(f).setdefault('_cp_config', {}).update(kwargs) return f @@ -182,10 +176,8 @@ def tool_decorator(f): class _Vars(object): - """ - Adapter that allows setting a default attribute on a function - or class. - """ + """Adapter allowing setting a default attribute on a function or class.""" + def __init__(self, target): self.target = target @@ -260,34 +252,33 @@ def _server_namespace_handler(k, v): setattr(cherrypy.servers[servername], k, v) else: setattr(cherrypy.server, k, v) -Config.namespaces['server'] = _server_namespace_handler # noqa: E305 + + +Config.namespaces['server'] = _server_namespace_handler def _engine_namespace_handler(k, v): """Config handler for the "engine" namespace.""" engine = cherrypy.engine - if k == 'SIGHUP': - engine.subscribe('SIGHUP', v) - elif k == 'SIGTERM': - engine.subscribe('SIGTERM', v) - elif '.' in k: + if k in {'SIGHUP', 'SIGTERM'}: + engine.subscribe(k, v) + return + + if '.' in k: plugin, attrname = k.split('.', 1) plugin = getattr(engine, plugin) - if attrname == 'on': - if v and hasattr(getattr(plugin, 'subscribe', None), '__call__'): - plugin.subscribe() - return - elif ( - (not v) and - hasattr(getattr(plugin, 'unsubscribe', None), '__call__') - ): - plugin.unsubscribe() - return + op = 'subscribe' if v else 'unsubscribe' + sub_unsub = getattr(plugin, op, None) + if attrname == 'on' and callable(sub_unsub): + sub_unsub() + return setattr(plugin, attrname, v) else: setattr(engine, k, v) -Config.namespaces['engine'] = _engine_namespace_handler # noqa: E305 + + +Config.namespaces['engine'] = _engine_namespace_handler def _tree_namespace_handler(k, v): @@ -300,4 +291,6 @@ def _tree_namespace_handler(k, v): else: cherrypy.tree.graft(v, v.script_name) cherrypy.engine.log('Mounted: %s on %s' % (v, v.script_name or '/')) -Config.namespaces['tree'] = _tree_namespace_handler # noqa: E305 + + +Config.namespaces['tree'] = _tree_namespace_handler diff --git a/lib/cherrypy/_cpdispatch.py b/lib/cherrypy/_cpdispatch.py index 538adc0..83eb79c 100644 --- a/lib/cherrypy/_cpdispatch.py +++ b/lib/cherrypy/_cpdispatch.py @@ -29,32 +29,26 @@ def __init__(self, callable, *args, **kwargs): self.args = args self.kwargs = kwargs - def get_args(self): + @property + def args(self): + """The ordered args should be accessible from post dispatch hooks.""" return cherrypy.serving.request.args - def set_args(self, args): + @args.setter + def args(self, args): cherrypy.serving.request.args = args return cherrypy.serving.request.args - args = property( - get_args, - set_args, - doc='The ordered args should be accessible from post dispatch hooks' - ) - - def get_kwargs(self): + @property + def kwargs(self): + """The named kwargs should be accessible from post dispatch hooks.""" return cherrypy.serving.request.kwargs - def set_kwargs(self, kwargs): + @kwargs.setter + def kwargs(self, kwargs): cherrypy.serving.request.kwargs = kwargs return cherrypy.serving.request.kwargs - kwargs = property( - get_kwargs, - set_kwargs, - doc='The named kwargs should be accessible from post dispatch hooks' - ) - def __call__(self): try: return self.callable(*self.args, **self.kwargs) @@ -64,7 +58,7 @@ def __call__(self): test_callable_spec(self.callable, self.args, self.kwargs) except cherrypy.HTTPError: raise sys.exc_info()[1] - except: + except Exception: raise x raise @@ -209,10 +203,12 @@ def test_callable_spec(callable, callable_args, callable_kwargs): try: import inspect except ImportError: - test_callable_spec = lambda callable, args, kwargs: None # noqa: F811 + def test_callable_spec(callable, args, kwargs): # noqa: F811 + return None else: getargspec = inspect.getargspec - # Python 3 requires using getfullargspec if keyword-only arguments are present + # Python 3 requires using getfullargspec if + # keyword-only arguments are present if hasattr(inspect, 'getfullargspec'): def getargspec(callable): return inspect.getfullargspec(callable)[:4] @@ -228,20 +224,19 @@ class LateParamPageHandler(PageHandler): (it's more complicated than that, but that's the effect). """ - def _get_kwargs(self): + @property + def kwargs(self): + """Page handler kwargs (with cherrypy.request.params copied in).""" kwargs = cherrypy.serving.request.params.copy() if self._kwargs: kwargs.update(self._kwargs) return kwargs - def _set_kwargs(self, kwargs): + @kwargs.setter + def kwargs(self, kwargs): cherrypy.serving.request.kwargs = kwargs self._kwargs = kwargs - kwargs = property(_get_kwargs, _set_kwargs, - doc='page handler kwargs (with ' - 'cherrypy.request.params copied in)') - if sys.version_info < (3, 0): punctuation_to_underscores = string.maketrans( diff --git a/lib/cherrypy/_cperror.py b/lib/cherrypy/_cperror.py index b597c64..e2a8fad 100644 --- a/lib/cherrypy/_cperror.py +++ b/lib/cherrypy/_cperror.py @@ -29,8 +29,9 @@ 300 Multiple Choices Confirm with the user 301 Moved Permanently Confirm with the user 302 Found (Object moved temporarily) Confirm with the user -303 See Other GET the new URI--no confirmation -304 Not modified (for conditional GET only--POST should not raise this error) +303 See Other GET the new URI; no confirmation +304 Not modified for conditional GET only; + POST should not raise this error 305 Use Proxy Confirm with the user 307 Temporary Redirect Confirm with the user ===== ================================= =========== @@ -58,7 +59,8 @@ expected responses (like 404 Not Found). Supply a filename from which the output will be read. The contents will be interpolated with the values %(status)s, %(message)s, %(traceback)s, and %(version)s using plain old Python -`string formatting `_. +`string formatting +`_. :: @@ -100,7 +102,7 @@ def error_page_402(status, message, traceback, version): def handle_error(): cherrypy.response.status = 500 cherrypy.response.body = [ - "Sorry, an error occured" + "Sorry, an error occurred" ] sendMail('error@domain.com', 'Error in your web app', @@ -115,16 +117,22 @@ class Root: and not simply return an error message as a result. """ +import io import contextlib from sys import exc_info as _exc_info from traceback import format_exception as _format_exception from xml.sax import saxutils import six +from six.moves import urllib +from more_itertools import always_iterable + +import cherrypy from cherrypy._cpcompat import escape_html -from cherrypy._cpcompat import text_or_bytes, iteritems, ntob -from cherrypy._cpcompat import tonative, urljoin as _urljoin +from cherrypy._cpcompat import ntob +from cherrypy._cpcompat import tonative +from cherrypy._helper import classproperty from cherrypy.lib import httputil as _httputil @@ -134,12 +142,6 @@ class CherryPyException(Exception): pass -class TimeoutError(CherryPyException): - - """Exception raised when Response.timed_out is detected.""" - pass - - class InternalRedirect(CherryPyException): """Exception raised to switch to the handler for a different URL. @@ -151,7 +153,6 @@ class InternalRedirect(CherryPyException): """ def __init__(self, path, query_string=''): - import cherrypy self.request = cherrypy.serving.request self.query_string = query_string @@ -163,7 +164,7 @@ def __init__(self, path, query_string=''): # 1. a URL relative to root (e.g. "/dummy") # 2. a URL relative to the current path # Note that any query string will be discarded. - path = _urljoin(self.request.path_info, path) + path = urllib.parse.urljoin(self.request.path_info, path) # Set a 'path' member attribute so that code which traps this # error can have access to it. @@ -198,9 +199,6 @@ class HTTPRedirect(CherryPyException): See :ref:`redirectingpost` for additional caveats. """ - status = None - """The integer HTTP status code to emit.""" - urls = None """The list of URL's to emit.""" @@ -208,41 +206,46 @@ class HTTPRedirect(CherryPyException): """The encoding when passed urls are not native strings""" def __init__(self, urls, status=None, encoding=None): - import cherrypy - request = cherrypy.serving.request - - if isinstance(urls, text_or_bytes): - urls = [urls] - - abs_urls = [] - for url in urls: - url = tonative(url, encoding or self.encoding) - + self.urls = abs_urls = [ # Note that urljoin will "do the right thing" whether url is: # 1. a complete URL with host (e.g. "http://www.example.com/test") # 2. a URL relative to root (e.g. "/dummy") # 3. a URL relative to the current path # Note that any query string in cherrypy.request is discarded. - url = _urljoin(cherrypy.url(), url) - abs_urls.append(url) - self.urls = abs_urls - - # RFC 2616 indicates a 301 response code fits our goal; however, - # browser support for 301 is quite messy. Do 302/303 instead. See - # http://www.alanflavell.org.uk/www/post-redirect.html - if status is None: - if request.protocol >= (1, 1): - status = 303 - else: - status = 302 - else: - status = int(status) - if status < 300 or status > 399: - raise ValueError('status must be between 300 and 399.') + urllib.parse.urljoin( + cherrypy.url(), + tonative(url, encoding or self.encoding), + ) + for url in always_iterable(urls) + ] + + status = ( + int(status) + if status is not None + else self.default_status + ) + if not 300 <= status <= 399: + raise ValueError('status must be between 300 and 399.') - self.status = status CherryPyException.__init__(self, abs_urls, status) + @classproperty + def default_status(cls): + """ + The default redirect status for the request. + + RFC 2616 indicates a 301 response code fits our goal; however, + browser support for 301 is quite messy. Use 302/303 instead. See + http://www.alanflavell.org.uk/www/post-redirect.html + """ + return 303 if cherrypy.serving.request.protocol >= (1, 1) else 302 + + @property + def status(self): + """The integer HTTP status code to emit.""" + _, status = self.args[:2] + return status + def set_response(self): """Modify cherrypy.response status, headers, and body to represent self. @@ -250,7 +253,6 @@ def set_response(self): CherryPy uses this internally, but you can also use it to create an HTTPRedirect object and set its output without *raising* the exception. """ - import cherrypy response = cherrypy.serving.response response.status = status = self.status @@ -271,7 +273,10 @@ def set_response(self): 307: 'This resource has moved temporarily to ', }[status] msg += '%s.' - msgs = [msg % (saxutils.quoteattr(u), escape_html(u)) for u in self.urls] + msgs = [ + msg % (saxutils.quoteattr(u), escape_html(u)) + for u in self.urls + ] response.body = ntob('
\n'.join(msgs), 'utf-8') # Previous code may have set C-L, so we have to reset it # (allow finalize to set it). @@ -311,8 +316,6 @@ def __call__(self): def clean_headers(status): """Remove any headers which should not apply to an error response.""" - import cherrypy - response = cherrypy.serving.response # Remove headers which applied to the original content, @@ -386,8 +389,6 @@ def set_response(self): CherryPy uses this internally, but you can also use it to create an HTTPError object and set its output without *raising* the exception. """ - import cherrypy - response = cherrypy.serving.response clean_headers(self.code) @@ -434,7 +435,6 @@ class NotFound(HTTPError): def __init__(self, path=None): if path is None: - import cherrypy request = cherrypy.serving.request path = request.script_name + request.path_info self.args = (path,) @@ -480,8 +480,6 @@ def get_error_page(status, **kwargs): status should be an int or a str. kwargs will be interpolated into the page template. """ - import cherrypy - try: code, reason, message = _httputil.valid_status(status) except ValueError: @@ -498,7 +496,7 @@ def get_error_page(status, **kwargs): if kwargs.get('version') is None: kwargs['version'] = cherrypy.__version__ - for k, v in iteritems(kwargs): + for k, v in six.iteritems(kwargs): if v is None: kwargs[k] = '' else: @@ -528,14 +526,14 @@ def get_error_page(status, **kwargs): if not isinstance(result, bytes): raise ValueError( 'error page function did not ' - 'return a bytestring, six.text_typeing or an ' + 'return a bytestring, six.text_type or an ' 'iterator - returned object of type %s.' % (type(result).__name__)) return result else: # Load the template from this path. - template = tonative(open(error_page, 'rb').read()) - except: + template = io.open(error_page, newline='').read() + except Exception: e = _format_exception(*_exc_info())[-1] m = kwargs['message'] if m: @@ -557,7 +555,6 @@ def get_error_page(status, **kwargs): def _be_ie_unfriendly(status): - import cherrypy response = cherrypy.serving.response # For some statuses, Internet Explorer 5+ shows "friendly error @@ -571,11 +568,11 @@ def _be_ie_unfriendly(status): # Since we are issuing an HTTP error status, we assume that # the entity is short, and we should just collapse it. content = response.collapse_body() - l = len(content) - if l and l < s: + content_length = len(content) + if content_length and content_length < s: # IN ADDITION: the response must be written to IE # in one chunk or it will still get replaced! Bah. - content = content + (ntob(' ') * (s - l)) + content = content + (b' ' * (s - content_length)) response.body = content response.headers['Content-Length'] = str(len(content)) @@ -610,13 +607,13 @@ def bare_error(extrabody=None): # it cannot be allowed to fail. Therefore, don't add to it! # In particular, don't call any other CP functions. - body = ntob('Unrecoverable error in the server.') + body = b'Unrecoverable error in the server.' if extrabody is not None: if not isinstance(extrabody, bytes): extrabody = extrabody.encode('utf-8') - body += ntob('\n') + extrabody + body += b'\n' + extrabody - return (ntob('500 Internal Server Error'), - [(ntob('Content-Type'), ntob('text/plain')), - (ntob('Content-Length'), ntob(str(len(body)), 'ISO-8859-1'))], + return (b'500 Internal Server Error', + [(b'Content-Type', b'text/plain'), + (b'Content-Length', ntob(str(len(body)), 'ISO-8859-1'))], [body]) diff --git a/lib/cherrypy/_cplogging.py b/lib/cherrypy/_cplogging.py index 79fe5a8..53b9add 100644 --- a/lib/cherrypy/_cplogging.py +++ b/lib/cherrypy/_cplogging.py @@ -59,7 +59,8 @@ If you are logging the access log and error log to the same source, then there is a possibility that a specially crafted error message may replicate an access log message as described in CWE-117. In this case it is the application -developer's responsibility to manually escape data before using CherryPy's log() +developer's responsibility to manually escape data before +using CherryPy's log() functionality, or they may create an application that is vulnerable to CWE-117. This would be achieved by using a custom handler escape any special characters, and attached as described below. @@ -116,7 +117,6 @@ import cherrypy from cherrypy import _cperror -from cherrypy._cpcompat import ntob # Silence the no-handlers "warning" (stderr write!) in stdlib logging @@ -216,7 +216,11 @@ def error(self, msg='', context='', severity=logging.INFO, if traceback: exc_info = _cperror._exc_info() - self.error_log.log(severity, ' '.join((self.time(), context, msg)), exc_info=exc_info) + self.error_log.log( + severity, + ' '.join((self.time(), context, msg)), + exc_info=exc_info, + ) def __call__(self, *args, **kwargs): """An alias for ``error``.""" @@ -226,7 +230,8 @@ def access(self): """Write to the access log (in Apache/NCSA Combined Log format). See the - `apache documentation `_ + `apache documentation + `_ for format details. CherryPy calls this automatically for you. Note there are no arguments; @@ -248,7 +253,7 @@ def access(self): if response.output_status is None: status = '-' else: - status = response.output_status.split(ntob(' '), 1)[0] + status = response.output_status.split(b' ', 1)[0] if six.PY3: status = status.decode('ISO-8859-1') @@ -262,6 +267,8 @@ def access(self): 'f': dict.get(inheaders, 'Referer', ''), 'a': dict.get(inheaders, 'User-Agent', ''), 'o': dict.get(inheaders, 'Host', '-'), + 'i': request.unique_id, + 'z': LazyRfc3339UtcTime(), } if six.PY3: for k, v in atoms.items(): @@ -283,7 +290,7 @@ def access(self): try: self.access_log.log( logging.INFO, self.access_log_format.format(**atoms)) - except: + except Exception: self(traceback=True) else: for k, v in atoms.items(): @@ -300,7 +307,7 @@ def access(self): try: self.access_log.log( logging.INFO, self.access_log_format % atoms) - except: + except Exception: self(traceback=True) def time(self): @@ -331,20 +338,21 @@ def _set_screen_handler(self, log, enable, stream=None): elif h: log.handlers.remove(h) - def _get_screen(self): + @property + def screen(self): + """Turn stderr/stdout logging on or off. + + If you set this to True, it'll add the appropriate StreamHandler for + you. If you set it to False, it will remove the handler. + """ h = self._get_builtin_handler has_h = h(self.error_log, 'screen') or h(self.access_log, 'screen') return bool(has_h) - def _set_screen(self, newvalue): + @screen.setter + def screen(self, newvalue): self._set_screen_handler(self.error_log, newvalue, stream=sys.stderr) self._set_screen_handler(self.access_log, newvalue, stream=sys.stdout) - screen = property(_get_screen, _set_screen, - doc="""Turn stderr/stdout logging on or off. - - If you set this to True, it'll add the appropriate StreamHandler for - you. If you set it to False, it will remove the handler. - """) # -------------------------- File handlers -------------------------- # @@ -369,35 +377,37 @@ def _set_file_handler(self, log, filename): h.close() log.handlers.remove(h) - def _get_error_file(self): + @property + def error_file(self): + """The filename for self.error_log. + + If you set this to a string, it'll add the appropriate FileHandler for + you. If you set it to ``None`` or ``''``, it will remove the handler. + """ h = self._get_builtin_handler(self.error_log, 'file') if h: return h.baseFilename return '' - def _set_error_file(self, newvalue): + @error_file.setter + def error_file(self, newvalue): self._set_file_handler(self.error_log, newvalue) - error_file = property(_get_error_file, _set_error_file, - doc="""The filename for self.error_log. + + @property + def access_file(self): + """The filename for self.access_log. If you set this to a string, it'll add the appropriate FileHandler for you. If you set it to ``None`` or ``''``, it will remove the handler. - """) - - def _get_access_file(self): + """ h = self._get_builtin_handler(self.access_log, 'file') if h: return h.baseFilename return '' - def _set_access_file(self, newvalue): + @access_file.setter + def access_file(self, newvalue): self._set_file_handler(self.access_log, newvalue) - access_file = property(_get_access_file, _set_access_file, - doc="""The filename for self.access_log. - - If you set this to a string, it'll add the appropriate FileHandler for - you. If you set it to ``None`` or ``''``, it will remove the handler. - """) # ------------------------- WSGI handlers ------------------------- # @@ -412,19 +422,20 @@ def _set_wsgi_handler(self, log, enable): elif h: log.handlers.remove(h) - def _get_wsgi(self): - return bool(self._get_builtin_handler(self.error_log, 'wsgi')) - - def _set_wsgi(self, newvalue): - self._set_wsgi_handler(self.error_log, newvalue) - wsgi = property(_get_wsgi, _set_wsgi, - doc="""Write errors to wsgi.errors. + @property + def wsgi(self): + """Write errors to wsgi.errors. If you set this to True, it'll add the appropriate :class:`WSGIErrorHandler` for you (which writes errors to ``wsgi.errors``). If you set it to False, it will remove the handler. - """) + """ + return bool(self._get_builtin_handler(self.error_log, 'wsgi')) + + @wsgi.setter + def wsgi(self, newvalue): + self._set_wsgi_handler(self.error_log, newvalue) class WSGIErrorHandler(logging.Handler): @@ -460,5 +471,12 @@ def emit(self, record): except UnicodeError: stream.write(fs % msg.encode('UTF-8')) self.flush() - except: + except Exception: self.handleError(record) + + +class LazyRfc3339UtcTime(object): + def __str__(self): + """Return now() in RFC3339 UTC Format.""" + now = datetime.datetime.now() + return now.isoformat('T') + 'Z' diff --git a/lib/cherrypy/_cpmodpy.py b/lib/cherrypy/_cpmodpy.py index bbb84ae..ac91e62 100644 --- a/lib/cherrypy/_cpmodpy.py +++ b/lib/cherrypy/_cpmodpy.py @@ -61,8 +61,11 @@ def setup_server(): import re import sys +import six + +from more_itertools import always_iterable + import cherrypy -from cherrypy._cpcompat import copyitems, ntob, text_or_bytes from cherrypy._cperror import format_exc, bare_error from cherrypy.lib import httputil @@ -100,6 +103,7 @@ def setup(req): engine.autoreload.unsubscribe() cherrypy.server.unsubscribe() + @engine.subscribe('log') def _log(msg, level): newlevel = apache.APLOG_ERR if logging.DEBUG >= level: @@ -112,7 +116,6 @@ def _log(msg, level): # http://www.modpython.org/pipermail/mod_python/2003-October/014291.html # Also, "When server is not specified...LogLevel does not apply..." apache.log_error(msg, newlevel, req.server) - engine.subscribe('log', _log) engine.start() @@ -194,7 +197,7 @@ def handler(req): path = req.uri qs = req.args or '' reqproto = req.protocol - headers = copyitems(req.headers_in) + headers = list(six.iteritems(req.headers_in)) rfile = _ReadOnlyRequest(req) prev = None @@ -241,7 +244,7 @@ def handler(req): response.body, response.stream) finally: app.release_serving() - except: + except Exception: tb = format_exc() cherrypy.log(tb, 'MOD_PYTHON', severity=logging.ERROR) s, h, b = bare_error() @@ -266,11 +269,8 @@ def send_response(req, status, headers, body, stream=False): req.flush() # Set response body - if isinstance(body, text_or_bytes): - req.write(body) - else: - for seg in body: - req.write(seg) + for seg in always_iterable(body): + req.write(seg) # --------------- Startup tools for CherryPy + mod_python --------------- # @@ -294,7 +294,7 @@ def read_process(cmd, args=''): try: firstline = pipeout.readline() cmd_not_found = re.search( - ntob('(not recognized|No such file|not found)'), + b'(not recognized|No such file|not found)', firstline, re.IGNORECASE ) diff --git a/lib/cherrypy/_cpnative_server.py b/lib/cherrypy/_cpnative_server.py index e39a29f..e9671d2 100644 --- a/lib/cherrypy/_cpnative_server.py +++ b/lib/cherrypy/_cpnative_server.py @@ -9,31 +9,38 @@ import cherrypy from cherrypy._cperror import format_exc, bare_error from cherrypy.lib import httputil +from ._cpcompat import tonative class NativeGateway(cheroot.server.Gateway): + """Native gateway implementation allowing to bypass WSGI.""" recursive = False def respond(self): + """Obtain response from CherryPy machinery and then send it.""" req = self.req try: # Obtain a Request object from CherryPy - local = req.server.bind_addr + local = req.server.bind_addr # FIXME: handle UNIX sockets + local = tonative(local[0]), local[1] local = httputil.Host(local[0], local[1], '') - remote = req.conn.remote_addr, req.conn.remote_port + remote = tonative(req.conn.remote_addr), req.conn.remote_port remote = httputil.Host(remote[0], remote[1], '') - scheme = req.scheme - sn = cherrypy.tree.script_name(req.uri or '/') + scheme = tonative(req.scheme) + sn = cherrypy.tree.script_name(tonative(req.uri or '/')) if sn is None: self.send_response('404 Not Found', [], ['']) else: app = cherrypy.tree.apps[sn] - method = req.method - path = req.path - qs = req.qs or '' - headers = req.inheaders.items() + method = tonative(req.method) + path = tonative(req.path) + qs = tonative(req.qs or '') + headers = ( + (tonative(h), tonative(v)) + for h, v in req.inheaders.items() + ) rfile = req.rfile prev = None @@ -50,8 +57,11 @@ def respond(self): # Run the CherryPy Request object and obtain the # response try: - request.run(method, path, qs, - req.request_protocol, headers, rfile) + request.run( + method, path, qs, + tonative(req.request_protocol), + headers, rfile, + ) break except cherrypy.InternalRedirect: ir = sys.exc_info()[1] @@ -81,7 +91,7 @@ def respond(self): response.body) finally: app.release_serving() - except: + except Exception: tb = format_exc() # print tb cherrypy.log(tb, 'NATIVE_ADAPTER', severity=logging.ERROR) @@ -89,10 +99,11 @@ def respond(self): self.send_response(s, h, b) def send_response(self, status, headers, body): + """Send response to HTTP request.""" req = self.req # Set response status - req.status = str(status or '500 Server Error') + req.status = status or b'500 Server Error' # Set response headers for header, value in headers: @@ -107,7 +118,6 @@ def send_response(self, status, headers, body): class CPHTTPServer(cheroot.server.HTTPServer): - """Wrapper for cheroot.server.HTTPServer. cheroot has been designed to not reference CherryPy in any way, @@ -117,6 +127,7 @@ class CPHTTPServer(cheroot.server.HTTPServer): """ def __init__(self, server_adapter=cherrypy.server): + """Initialize CPHTTPServer.""" self.server_adapter = server_adapter server_name = (self.server_adapter.socket_host or diff --git a/lib/cherrypy/_cpreqbody.py b/lib/cherrypy/_cpreqbody.py index 0d821a5..893fe5f 100644 --- a/lib/cherrypy/_cpreqbody.py +++ b/lib/cherrypy/_cpreqbody.py @@ -135,7 +135,7 @@ def unquote_plus(bs): import cheroot.server import cherrypy -from cherrypy._cpcompat import text_or_bytes, ntob, ntou +from cherrypy._cpcompat import ntou, unquote from cherrypy.lib import httputil @@ -147,14 +147,14 @@ def process_urlencoded(entity): for charset in entity.attempt_charsets: try: params = {} - for aparam in qs.split(ntob('&')): - for pair in aparam.split(ntob(';')): + for aparam in qs.split(b'&'): + for pair in aparam.split(b';'): if not pair: continue - atoms = pair.split(ntob('='), 1) + atoms = pair.split(b'=', 1) if len(atoms) == 1: - atoms.append(ntob('')) + atoms.append(b'') key = unquote_plus(atoms[0]).decode(charset) value = unquote_plus(atoms[1]).decode(charset) @@ -318,7 +318,8 @@ class Entity(object): :attr:`request.body.parts`. You can enable it with:: - cherrypy.request.body.processors['multipart'] = _cpreqbody.process_multipart + cherrypy.request.body.processors['multipart'] = \ + _cpreqbody.process_multipart in an ``on_start_resource`` tool. """ @@ -328,14 +329,15 @@ class Entity(object): # absence of a charset parameter, is US-ASCII." # However, many browsers send data in utf-8 with no charset. attempt_charsets = ['utf-8'] - """A list of strings, each of which should be a known encoding. + r"""A list of strings, each of which should be a known encoding. When the Content-Type of the request body warrants it, each of the given encodings will be tried in order. The first one to successfully decode the entity without raising an error is stored as :attr:`entity.charset`. This defaults to ``['utf-8']`` (plus 'ISO-8859-1' for "text/\*" types, as required by - `HTTP/1.1 `_), + `HTTP/1.1 + `_), but ``['us-ascii', 'utf-8']`` for multipart parts. """ @@ -468,13 +470,10 @@ def __init__(self, fp, headers, params=None, parts=None): self.filename.endswith('"') ): self.filename = self.filename[1:-1] - - # The 'type' attribute is deprecated in 3.2; remove it in 3.3. - type = property( - lambda self: self.content_type, - doc='A deprecated alias for ' - ':attr:`content_type`.' - ) + if 'filename*' in disp.params: + # @see https://tools.ietf.org/html/rfc5987 + encoding, lang, filename = disp.params['filename*'].split("'") + self.filename = unquote(str(filename), encoding) def read(self, size=None, fp_out=None): return self.fp.read(size, fp_out) @@ -577,14 +576,15 @@ class Part(Entity): # "The default character set, which must be assumed in the absence of a # charset parameter, is US-ASCII." attempt_charsets = ['us-ascii', 'utf-8'] - """A list of strings, each of which should be a known encoding. + r"""A list of strings, each of which should be a known encoding. When the Content-Type of the request body warrants it, each of the given encodings will be tried in order. The first one to successfully decode the entity without raising an error is stored as :attr:`entity.charset`. This defaults to ``['utf-8']`` (plus 'ISO-8859-1' for "text/\*" types, as required by - `HTTP/1.1 `_), + `HTTP/1.1 + `_), but ``['us-ascii', 'utf-8']`` for multipart parts. """ @@ -630,17 +630,17 @@ def read_headers(cls, fp): # No more data--illegal end of headers raise EOFError('Illegal end of headers.') - if line == ntob('\r\n'): + if line == b'\r\n': # Normal end of headers break - if not line.endswith(ntob('\r\n')): + if not line.endswith(b'\r\n'): raise ValueError('MIME requires CRLF terminators: %r' % line) - if line[0] in ntob(' \t'): + if line[0] in b' \t': # It's a continuation line. v = line.strip().decode('ISO-8859-1') else: - k, v = line.split(ntob(':'), 1) + k, v = line.split(b':', 1) k = k.strip().decode('ISO-8859-1') v = v.strip().decode('ISO-8859-1') @@ -661,8 +661,8 @@ def read_lines_to_boundary(self, fp_out=None): object that supports the 'write' method; all bytes read will be written to the fp, and that fp is returned. """ - endmarker = self.boundary + ntob('--') - delim = ntob('') + endmarker = self.boundary + b'--' + delim = b'' prev_lf = True lines = [] seen = 0 @@ -670,7 +670,7 @@ def read_lines_to_boundary(self, fp_out=None): line = self.fp.readline(1 << 16) if not line: raise EOFError('Illegal end of multipart body.') - if line.startswith(ntob('--')) and prev_lf: + if line.startswith(b'--') and prev_lf: strippedline = line.strip() if strippedline == self.boundary: break @@ -680,16 +680,16 @@ def read_lines_to_boundary(self, fp_out=None): line = delim + line - if line.endswith(ntob('\r\n')): - delim = ntob('\r\n') + if line.endswith(b'\r\n'): + delim = b'\r\n' line = line[:-2] prev_lf = True - elif line.endswith(ntob('\n')): - delim = ntob('\n') + elif line.endswith(b'\n'): + delim = b'\n' line = line[:-1] prev_lf = True else: - delim = ntob('') + delim = b'' prev_lf = False if fp_out is None: @@ -703,7 +703,7 @@ def read_lines_to_boundary(self, fp_out=None): fp_out.write(line) if fp_out is None: - result = ntob('').join(lines) + result = b''.join(lines) return result else: fp_out.seek(0) @@ -718,7 +718,7 @@ def default_proc(self): self.file = self.read_into_file() else: result = self.read_lines_to_boundary() - if isinstance(result, text_or_bytes): + if isinstance(result, bytes): self.value = result else: self.file = result @@ -733,7 +733,8 @@ def read_into_file(self, fp_out=None): self.read_lines_to_boundary(fp_out=fp_out) return fp_out -Entity.part_class = Part # noqa: E305 + +Entity.part_class = Part inf = float('inf') @@ -746,7 +747,7 @@ def __init__(self, fp, length, maxbytes, bufsize=DEFAULT_BUFFER_SIZE, self.fp = fp self.length = length self.maxbytes = maxbytes - self.buffer = ntob('') + self.buffer = b'' self.bufsize = bufsize self.bytes_read = 0 self.done = False @@ -782,7 +783,7 @@ def read(self, size=None, fp_out=None): if remaining == 0: self.finish() if fp_out is None: - return ntob('') + return b'' else: return None @@ -792,7 +793,7 @@ def read(self, size=None, fp_out=None): if self.buffer: if remaining is inf: data = self.buffer - self.buffer = ntob('') + self.buffer = b'' else: data = self.buffer[:remaining] self.buffer = self.buffer[remaining:] @@ -841,7 +842,7 @@ def read(self, size=None, fp_out=None): fp_out.write(data) if fp_out is None: - return ntob('').join(chunks) + return b''.join(chunks) def readline(self, size=None): """Read a line from the request body and return it.""" @@ -853,7 +854,7 @@ def readline(self, size=None): data = self.read(chunksize) if not data: break - pos = data.find(ntob('\n')) + 1 + pos = data.find(b'\n') + 1 if pos: chunks.append(data[:pos]) remainder = data[pos:] @@ -862,7 +863,7 @@ def readline(self, size=None): break else: chunks.append(data) - return ntob('').join(chunks) + return b''.join(chunks) def readlines(self, sizehint=None): """Read lines from the request body and return them.""" @@ -891,12 +892,12 @@ def finish(self): try: for line in self.fp.read_trailer_lines(): - if line[0] in ntob(' \t'): + if line[0] in b' \t': # It's a continuation line. v = line.strip() else: try: - k, v = line.split(ntob(':'), 1) + k, v = line.split(b':', 1) except ValueError: raise ValueError('Illegal header line.') k = k.strip().title() @@ -905,7 +906,7 @@ def finish(self): if k in cheroot.server.comma_separated_headers: existing = self.trailers.get(k) if existing: - v = ntob(', ').join((existing, v)) + v = b', '.join((existing, v)) self.trailers[k] = v except Exception: e = sys.exc_info()[1] diff --git a/lib/cherrypy/_cprequest.py b/lib/cherrypy/_cprequest.py index cd1ebfc..3cc0c81 100644 --- a/lib/cherrypy/_cprequest.py +++ b/lib/cherrypy/_cprequest.py @@ -1,15 +1,18 @@ import sys import time -import warnings + +import uuid import six +from six.moves.http_cookies import SimpleCookie, CookieError + +from more_itertools import consume import cherrypy -from cherrypy._cpcompat import text_or_bytes, copykeys, ntob -from cherrypy._cpcompat import SimpleCookie, CookieError -from cherrypy import _cpreqbody, _cpconfig +from cherrypy._cpcompat import ntob +from cherrypy import _cpreqbody from cherrypy._cperror import format_exc, bare_error -from cherrypy.lib import httputil, file_generator +from cherrypy.lib import httputil, reprconf, encoding class Hook(object): @@ -51,13 +54,12 @@ def __init__(self, callback, failsafe=None, priority=None, **kwargs): self.kwargs = kwargs def __lt__(self, other): - # Python 3 + """ + Hooks sort by priority, ascending, such that + hooks of lower priority are run first. + """ return self.priority < other.priority - def __cmp__(self, other): - # Python 2 - return cmp(self.priority, other.priority) # noqa: F821 - def __call__(self): """Run self.callback(**self.kwargs).""" return self.callback(**self.kwargs) @@ -107,7 +109,7 @@ def run(self, point): except (cherrypy.HTTPError, cherrypy.HTTPRedirect, cherrypy.InternalRedirect): exc = sys.exc_info()[1] - except: + except Exception: exc = sys.exc_info()[1] cherrypy.log(traceback=True, severity=40) if exc: @@ -127,7 +129,7 @@ def __repr__(self): return '%s.%s(points=%r)' % ( cls.__module__, cls.__name__, - copykeys(self) + list(self) ) @@ -139,8 +141,8 @@ def hooks_namespace(k, v): # hookpoint per path (e.g. "hooks.before_handler.1"). # Little-known fact you only get from reading source ;) hookpoint = k.split('.', 1)[0] - if isinstance(v, text_or_bytes): - v = cherrypy.lib.attributes(v) + if isinstance(v, six.string_types): + v = cherrypy.lib.reprconf.attributes(v) if not isinstance(v, Hook): v = Hook(v) cherrypy.serving.request.hooks[hookpoint].append(v) @@ -467,7 +469,10 @@ class Request(object): A string containing the stage reached in the request-handling process. This is useful when debugging a live server with hung requests.""" - namespaces = _cpconfig.NamespaceSet( + unique_id = None + """A lazy object generating and memorizing UUID4 on ``str()`` render.""" + + namespaces = reprconf.NamespaceSet( **{'hooks': hooks_namespace, 'request': request_namespace, 'response': response_namespace, @@ -498,6 +503,8 @@ def __init__(self, local_host, remote_host, scheme='http', self.stage = None + self.unique_id = LazyUUID4() + def close(self): """Run cleanup code. (Core)""" if not self.closed: @@ -590,7 +597,7 @@ def run(self, method, path, query_string, req_protocol, headers, rfile): except self.throws: raise - except: + except Exception: if self.throw_errors: raise else: @@ -610,85 +617,82 @@ def run(self, method, path, query_string, req_protocol, headers, rfile): try: cherrypy.log.access() - except: + except Exception: cherrypy.log.error(traceback=True) - if response.timed_out: - raise cherrypy.TimeoutError() - return response - # Uncomment for stage debugging - # stage = property(lambda self: self._stage, lambda self, v: print(v)) - def respond(self, path_info): """Generate a response for the resource at self.path_info. (Core)""" - response = cherrypy.serving.response try: try: try: - if self.app is None: - raise cherrypy.NotFound() - - # Get the 'Host' header, so we can HTTPRedirect properly. - self.stage = 'process_headers' - self.process_headers() - - # Make a copy of the class hooks - self.hooks = self.__class__.hooks.copy() - self.toolmaps = {} - - self.stage = 'get_resource' - self.get_resource(path_info) - - self.body = _cpreqbody.RequestBody( - self.rfile, self.headers, request_params=self.params) - - self.namespaces(self.config) - - self.stage = 'on_start_resource' - self.hooks.run('on_start_resource') - - # Parse the querystring - self.stage = 'process_query_string' - self.process_query_string() - - # Process the body - if self.process_request_body: - if self.method not in self.methods_with_bodies: - self.process_request_body = False - self.stage = 'before_request_body' - self.hooks.run('before_request_body') - if self.process_request_body: - self.body.process() - - # Run the handler - self.stage = 'before_handler' - self.hooks.run('before_handler') - if self.handler: - self.stage = 'handler' - response.body = self.handler() - - # Finalize - self.stage = 'before_finalize' - self.hooks.run('before_finalize') - response.finalize() + self._do_respond(path_info) except (cherrypy.HTTPRedirect, cherrypy.HTTPError): inst = sys.exc_info()[1] inst.set_response() self.stage = 'before_finalize (HTTPError)' self.hooks.run('before_finalize') - response.finalize() + cherrypy.serving.response.finalize() finally: self.stage = 'on_end_resource' self.hooks.run('on_end_resource') except self.throws: raise - except: + except Exception: if self.throw_errors: raise self.handle_error() + def _do_respond(self, path_info): + response = cherrypy.serving.response + + if self.app is None: + raise cherrypy.NotFound() + + self.hooks = self.__class__.hooks.copy() + self.toolmaps = {} + + # Get the 'Host' header, so we can HTTPRedirect properly. + self.stage = 'process_headers' + self.process_headers() + + self.stage = 'get_resource' + self.get_resource(path_info) + + self.body = _cpreqbody.RequestBody( + self.rfile, self.headers, request_params=self.params) + + self.namespaces(self.config) + + self.stage = 'on_start_resource' + self.hooks.run('on_start_resource') + + # Parse the querystring + self.stage = 'process_query_string' + self.process_query_string() + + # Process the body + if self.process_request_body: + if self.method not in self.methods_with_bodies: + self.process_request_body = False + self.stage = 'before_request_body' + self.hooks.run('before_request_body') + if self.process_request_body: + self.body.process() + + # Run the handler + self.stage = 'before_handler' + self.hooks.run('before_handler') + if self.handler: + self.stage = 'handler' + response.body = self.handler() + + # Finalize + self.stage = 'before_finalize' + self.hooks.run('before_finalize') + response.finalize() + def process_query_string(self): """Parse the query string into Python structures. (Core)""" try: @@ -718,23 +722,16 @@ def process_headers(self): name = name.title() value = value.strip() - # Warning: if there is more than one header entry for cookies - # (AFAIK, only Konqueror does that), only the last one will - # remain in headers (but they will be correctly stored in - # request.cookie). - if '=?' in value: - dict.__setitem__(headers, name, httputil.decode_TEXT(value)) - else: - dict.__setitem__(headers, name, value) + headers[name] = httputil.decode_TEXT_maybe(value) - # Handle cookies differently because on Konqueror, multiple - # cookies come on different lines with the same key + # Some clients, notably Konquoror, supply multiple + # cookies on different lines with the same key. To + # handle this case, store all cookies in self.cookie. if name == 'Cookie': try: self.cookie.load(value) - except CookieError: - msg = 'Illegal cookie name %s' % value.split('=')[0] - raise cherrypy.HTTPError(400, msg) + except CookieError as exc: + raise cherrypy.HTTPError(400, str(exc)) if not dict.__contains__(headers, 'Host'): # All Internet-based HTTP/1.1 servers MUST respond with a 400 @@ -772,36 +769,13 @@ def handle_error(self): inst.set_response() cherrypy.serving.response.finalize() - # ------------------------- Properties ------------------------- # - - def _get_body_params(self): - warnings.warn( - 'body_params is deprecated in CherryPy 3.2, will be removed in ' - 'CherryPy 3.3.', - DeprecationWarning - ) - return self.body.params - body_params = property(_get_body_params, - doc=""" - If the request Content-Type is 'application/x-www-form-urlencoded' or - multipart, this will be a dict of the params pulled from the entity - body; that is, it will be the portion of request.params that come - from the message body (sometimes called "POST params", although they - can be sent with various HTTP method verbs). This value is set between - the 'before_request_body' and 'before_handler' hooks (assuming that - process_request_body is True). - - Deprecated in 3.2, will be removed for 3.3 in favor of - :attr:`request.body.params`.""") - class ResponseBody(object): """The body of the HTTP response (the response entity).""" - if six.PY3: - unicode_err = ('Page handlers MUST return bytes. Use tools.encode ' - 'if you wish to return unicode.') + unicode_err = ('Page handlers MUST return bytes. Use tools.encode ' + 'if you wish to return unicode.') def __get__(self, obj, objclass=None): if obj is None: @@ -812,30 +786,14 @@ def __get__(self, obj, objclass=None): def __set__(self, obj, value): # Convert the given value to an iterable object. - if six.PY3 and isinstance(value, str): + if isinstance(value, six.text_type): raise ValueError(self.unicode_err) - - if isinstance(value, text_or_bytes): - # strings get wrapped in a list because iterating over a single - # item list is much faster than iterating over every character - # in a long string. - if value: - value = [value] - else: - # [''] doesn't evaluate to False, so replace it with []. - value = [] - elif six.PY3 and isinstance(value, list): + elif isinstance(value, list): # every item in a list must be bytes... - for i, item in enumerate(value): - if isinstance(item, str): - raise ValueError(self.unicode_err) - # Don't use isinstance here; io.IOBase which has an ABC takes - # 1000 times as long as, say, isinstance(value, str) - elif hasattr(value, 'read'): - value = file_generator(value) - elif value is None: - value = [] - obj._body = value + if any(isinstance(item, six.text_type) for item in value): + raise ValueError(self.unicode_err) + + obj._body = encoding.prepare_iter(value) class Response(object): @@ -872,14 +830,6 @@ class Response(object): time = None """The value of time.time() when created. Use in HTTP dates.""" - timeout = 300 - """Seconds after which the response will be aborted.""" - - timed_out = False - """ - Flag to indicate the response should be aborted, because it has - exceeded its timeout.""" - stream = False """If False, buffer the response body.""" @@ -901,19 +851,17 @@ def __init__(self): def collapse_body(self): """Collapse self.body to a single string; replace it and return it.""" - if isinstance(self.body, text_or_bytes): - return self.body + new_body = b''.join(self.body) + self.body = new_body + return new_body - newbody = [] - for chunk in self.body: - if six.PY3 and not isinstance(chunk, bytes): - raise TypeError("Chunk %s is not of type 'bytes'." % - repr(chunk)) - newbody.append(chunk) - newbody = ntob('').join(newbody) - - self.body = newbody - return newbody + def _flush_body(self): + """ + Discard self.body but consume any generator such that + any finalization can occur, such as is required by + caching.tee_output(). + """ + consume(iter(self.body)) def finalize(self): """Transform headers (and cookies) into self.header_list. (Core)""" @@ -926,7 +874,7 @@ def finalize(self): self.status = '%s %s' % (code, reason) self.output_status = ntob(str(code), 'ascii') + \ - ntob(' ') + headers.encode(reason) + b' ' + headers.encode(reason) if self.stream: # The upshot: wsgiserver will chunk the response if @@ -939,7 +887,8 @@ def finalize(self): # and 304 (not modified) responses MUST NOT # include a message-body." dict.pop(headers, 'Content-Length', None) - self.body = ntob('') + self._flush_body() + self.body = b'' else: # Responses which are not streamed should have a Content-Length, # but allow user code to set Content-Length if desired. @@ -960,11 +909,22 @@ def finalize(self): value = headers.encode(value) h.append((name, value)) - def check_timeout(self): - """If now > self.time + self.timeout, set self.timed_out. - This purposefully sets a flag, rather than raising an error, - so that a monitor thread can interrupt the Response thread. +class LazyUUID4(object): + def __str__(self): + """Return UUID4 and keep it for future calls.""" + return str(self.uuid4) + + @property + def uuid4(self): + """Provide unique id on per-request basis using UUID4. + + It's evaluated lazily on render. """ - if time.time() > self.time + self.timeout: - self.timed_out = True + try: + self._uuid4 + except AttributeError: + # evaluate on first access + self._uuid4 = uuid.uuid4() + + return self._uuid4 diff --git a/lib/cherrypy/_cpserver.py b/lib/cherrypy/_cpserver.py index 9571145..0f60e2c 100644 --- a/lib/cherrypy/_cpserver.py +++ b/lib/cherrypy/_cpserver.py @@ -8,11 +8,10 @@ from cherrypy.process.servers import ServerAdapter -__all__ = ['Server'] +__all__ = ('Server', ) class Server(ServerAdapter): - """An adapter for an HTTP server. You can set attributes (like socket_host and socket_port) @@ -28,26 +27,26 @@ class Server(ServerAdapter): _socket_host = '127.0.0.1' - def _get_socket_host(self): + @property + def socket_host(self): # noqa: D401; irrelevant for properties + """The hostname or IP address on which to listen for connections. + + Host values may be any IPv4 or IPv6 address, or any valid hostname. + The string 'localhost' is a synonym for '127.0.0.1' (or '::1', if + your hosts file prefers IPv6). The string '0.0.0.0' is a special + IPv4 entry meaning "any active interface" (INADDR_ANY), and '::' + is the similar IN6ADDR_ANY for IPv6. The empty string or None are + not allowed. + """ return self._socket_host - def _set_socket_host(self, value): + @socket_host.setter + def socket_host(self, value): if value == '': raise ValueError("The empty string ('') is not an allowed value. " "Use '0.0.0.0' instead to listen on all active " 'interfaces (INADDR_ANY).') self._socket_host = value - socket_host = property( - _get_socket_host, - _set_socket_host, - doc="""The hostname or IP address on which to listen for connections. - - Host values may be any IPv4 or IPv6 address, or any valid hostname. - The string 'localhost' is a synonym for '127.0.0.1' (or '::1', if - your hosts file prefers IPv6). The string '0.0.0.0' is a special - IPv4 entry meaning "any active interface" (INADDR_ANY), and '::' - is the similar IN6ADDR_ANY for IPv6. The empty string or None are - not allowed.""") socket_file = None """If given, the name of the UNIX socket to use instead of TCP/IP. @@ -96,7 +95,8 @@ def _set_socket_host(self, value): instance = None """If not None, this should be an HTTP server instance (such as - cheroot.wsgi.Server) which cherrypy.server will control. Use this when you need + cheroot.wsgi.Server) which cherrypy.server will control. + Use this when you need more control over object instantiation than is available in the various configuration options.""" @@ -144,9 +144,29 @@ def _set_socket_host(self, value): which declares it covers WSGI version 1.0.1 but still mandates the wsgi.version (1, 0)] and ('u', 0), an experimental unicode version. You may create and register your own experimental versions of the WSGI - protocol by adding custom classes to the cheroot.server.wsgi_gateways dict.""" + protocol by adding custom classes to the cheroot.server.wsgi_gateways dict. + """ + + peercreds = False + """If True, peer cred lookup for UNIX domain socket will put to WSGI env. + + This information will then be available through WSGI env vars: + * X_REMOTE_PID + * X_REMOTE_UID + * X_REMOTE_GID + """ + + peercreds_resolve = False + """If True, username/group will be looked up in the OS from peercreds. + + This information will then be available through WSGI env vars: + * REMOTE_USER + * X_REMOTE_USER + * X_REMOTE_GROUP + """ def __init__(self): + """Initialize Server instance.""" self.bus = cherrypy.engine self.httpserver = None self.interrupt = None @@ -171,14 +191,20 @@ def start(self): super(Server, self).start() start.priority = 75 - def _get_bind_addr(self): + @property + def bind_addr(self): + """Return bind address. + + A (host, port) tuple for TCP sockets or a str for Unix domain sockts. + """ if self.socket_file: return self.socket_file if self.socket_host is None and self.socket_port is None: return None return (self.socket_host, self.socket_port) - def _set_bind_addr(self, value): + @bind_addr.setter + def bind_addr(self, value): if value is None: self.socket_file = None self.socket_host = None @@ -195,14 +221,11 @@ def _set_bind_addr(self, value): raise ValueError('bind_addr must be a (host, port) tuple ' '(for TCP sockets) or a string (for Unix ' 'domain sockets), not %r' % value) - bind_addr = property( - _get_bind_addr, - _set_bind_addr, - doc='A (host, port) tuple for TCP sockets or ' - 'a str for Unix domain sockets.') def base(self): - """Return the base (scheme://host[:port] or sock file) for this server. + """Return the base for this server. + + e.i. scheme://host[:port] or sock file """ if self.socket_file: return self.socket_file diff --git a/lib/cherrypy/_cptools.py b/lib/cherrypy/_cptools.py index 65f79ed..5746028 100644 --- a/lib/cherrypy/_cptools.py +++ b/lib/cherrypy/_cptools.py @@ -22,13 +22,12 @@ are generally either modules or instances of the tools.Tool class. """ -import sys -import warnings +import six import cherrypy from cherrypy._helper import expose -from cherrypy.lib import cptools, encoding, auth, static, jsontools +from cherrypy.lib import cptools, encoding, static, jsontools from cherrypy.lib import sessions as _sessions, xmlrpcutil as _xmlrpc from cherrypy.lib import caching as _caching from cherrypy.lib import auth_basic, auth_digest @@ -38,7 +37,7 @@ def _getargs(func): """Return the names of all static arguments to the given function.""" # Use this instead of importing inspect for less mem overhead. import types - if sys.version_info >= (3, 0): + if six.PY3: if isinstance(func, types.MethodType): func = func.__func__ co = func.__code__ @@ -72,12 +71,13 @@ def __init__(self, point, callable, name=None, priority=50): self.__doc__ = self.callable.__doc__ self._setargs() - def _get_on(self): + @property + def on(self): raise AttributeError(_attr_error) - def _set_on(self, value): + @on.setter + def on(self, value): raise AttributeError(_attr_error) - on = property(_get_on, _set_on) def _setargs(self): """Copy func parameter names to obj attributes.""" @@ -322,9 +322,12 @@ def regenerate(self): sess.regenerate() # Grab cookie-relevant tool args - conf = dict([(k, v) for k, v in self._merged_args().items() - if k in ('path', 'path_header', 'name', 'timeout', - 'domain', 'secure')]) + relevant = 'path', 'path_header', 'name', 'timeout', 'domain', 'secure' + conf = dict( + (k, v) + for k, v in self._merged_args().items() + if k in relevant + ) _sessions.set_response_cookie(**conf) @@ -392,11 +395,7 @@ def default(self, *vpath, **params): class SessionAuthTool(HandlerTool): - - def _setargs(self): - for name in dir(cptools.SessionAuth): # noqa: F821 - if not name.startswith('__'): - setattr(self, name, None) + pass class CachingTool(Tool): @@ -411,8 +410,8 @@ def _wrapper(self, **kwargs): if request.cacheable: # Note the devious technique here of adding hooks on the fly request.hooks.attach('before_finalize', _caching.tee_output, - priority=90) - _wrapper.priority = 20 + priority=100) + _wrapper.priority = 90 def _setup(self): """Hook caching into cherrypy.request.""" @@ -462,34 +461,18 @@ def __exit__(self, exc_type, exc_val, exc_tb): tool._setup() def register(self, point, **kwargs): - """Return a decorator which registers the function at the given hook point.""" + """ + Return a decorator which registers the function + at the given hook point. + """ def decorator(func): - setattr(self, kwargs.get('name', func.__name__), Tool(point, func, **kwargs)) + attr_name = kwargs.get('name', func.__name__) + tool = Tool(point, func, **kwargs) + setattr(self, attr_name, tool) return func return decorator -class DeprecatedTool(Tool): - - _name = None - warnmsg = 'This Tool is deprecated.' - - def __init__(self, point, warnmsg=None): - self.point = point - if warnmsg is not None: - self.warnmsg = warnmsg - - def __call__(self, *args, **kwargs): - warnings.warn(self.warnmsg) - - def tool_decorator(f): - return f - return tool_decorator - - def _setup(self): - warnings.warn(self.warnmsg) - - default_toolbox = _d = Toolbox('tools') _d.session_auth = SessionAuthTool(cptools.session_auth) _d.allow = Tool('on_start_resource', cptools.allow) @@ -510,20 +493,8 @@ def _setup(self): _d.xmlrpc = ErrorTool(_xmlrpc.on_error) _d.caching = CachingTool('before_handler', _caching.get, 'caching') _d.expires = Tool('before_finalize', _caching.expires) -_d.tidy = DeprecatedTool( - 'before_finalize', - 'The tidy tool has been removed from the standard distribution of ' - 'CherryPy. The most recent version can be found at ' - 'http://tools.cherrypy.org/browser.') -_d.nsgmls = DeprecatedTool( - 'before_finalize', - 'The nsgmls tool has been removed from the standard distribution of ' - 'CherryPy. The most recent version can be found at ' - 'http://tools.cherrypy.org/browser.') _d.ignore_headers = Tool('before_request_body', cptools.ignore_headers) _d.referer = Tool('before_request_body', cptools.referer) -_d.basic_auth = Tool('on_start_resource', auth.basic_auth) -_d.digest_auth = Tool('on_start_resource', auth.digest_auth) _d.trailing_slash = Tool('before_handler', cptools.trailing_slash, priority=60) _d.flatten = Tool('before_finalize', cptools.flatten) _d.accept = Tool('on_start_resource', cptools.accept) @@ -533,6 +504,6 @@ def _setup(self): _d.json_out = Tool('before_handler', jsontools.json_out, priority=30) _d.auth_basic = Tool('before_handler', auth_basic.basic_auth, priority=1) _d.auth_digest = Tool('before_handler', auth_digest.digest_auth, priority=1) -_d.params = Tool('before_handler', cptools.convert_params) +_d.params = Tool('before_handler', cptools.convert_params, priority=15) -del _d, cptools, encoding, auth, static +del _d, cptools, encoding, static diff --git a/lib/cherrypy/_cptree.py b/lib/cherrypy/_cptree.py index f19cb13..ceb5437 100644 --- a/lib/cherrypy/_cptree.py +++ b/lib/cherrypy/_cptree.py @@ -7,11 +7,10 @@ import cherrypy from cherrypy._cpcompat import ntou from cherrypy import _cpconfig, _cplogging, _cprequest, _cpwsgi, tools -from cherrypy.lib import httputil +from cherrypy.lib import httputil, reprconf class Application(object): - """A CherryPy Application. Servers and gateways should not instantiate Request objects directly. @@ -32,7 +31,7 @@ class Application(object): """A dict of {path: pathconf} pairs, where 'pathconf' is itself a dict of {key: value} pairs.""" - namespaces = _cpconfig.NamespaceSet() + namespaces = reprconf.NamespaceSet() toolboxes = {'tools': cherrypy.tools} log = None @@ -47,6 +46,7 @@ class Application(object): relative_urls = False def __init__(self, root, script_name='', config=None): + """Initialize Application with given root.""" self.log = _cplogging.LogManager(id(self), cherrypy.log.logger_root) self.root = root self.script_name = script_name @@ -61,6 +61,7 @@ def __init__(self, root, script_name='', config=None): self.merge(config) def __repr__(self): + """Generate a representation of the Application instance.""" return '%s.%s(%r, %r)' % (self.__module__, self.__class__.__name__, self.root, self.script_name) @@ -80,7 +81,24 @@ def __repr__(self): provided for each call from request.wsgi_environ['SCRIPT_NAME']. """ - def _get_script_name(self): + @property + def script_name(self): # noqa: D401; irrelevant for properties + """The URI "mount point" for this app. + + A mount point is that portion of the URI which is constant for all URIs + that are serviced by this application; it does not include scheme, + host, or proxy ("virtual host") portions of the URI. + + For example, if script_name is "/my/cool/app", then the URL + "http://www.example.com/my/cool/app/page1" might be handled by a + "page1" method on the root object. + + The value of script_name MUST NOT end in a slash. If the script_name + refers to the root of the URI, it MUST be an empty string (not "/"). + + If script_name is explicitly set to None, then the script_name will be + provided for each call from request.wsgi_environ['SCRIPT_NAME']. + """ if self._script_name is not None: return self._script_name @@ -88,12 +106,11 @@ def _get_script_name(self): # should be pulled from WSGI environ. return cherrypy.serving.request.wsgi_environ['SCRIPT_NAME'].rstrip('/') - def _set_script_name(self, value): + @script_name.setter + def script_name(self, value): if value: value = value.rstrip('/') self._script_name = value - script_name = property(fget=_get_script_name, fset=_set_script_name, - doc=script_name_doc) def merge(self, config): """Merge the given config into self.config.""" @@ -144,17 +161,17 @@ def release_serving(self): try: req.close() - except: + except Exception: cherrypy.log(traceback=True, severity=40) cherrypy.serving.clear() def __call__(self, environ, start_response): + """Call a WSGI-callable.""" return self.wsgiapp(environ, start_response) class Tree(object): - """A registry of CherryPy applications, mounted at diverse points. An instance of this class may also be used as a WSGI callable @@ -170,6 +187,7 @@ class Tree(object): WSGI callable if you happen to be using a WSGI server).""" def __init__(self): + """Initialize registry Tree.""" self.apps = {} def mount(self, root, script_name='', config=None): @@ -216,8 +234,17 @@ def mount(self, root, script_name='', config=None): app = Application(root, script_name) # If mounted at "", add favicon.ico - if script_name == '' and root is not None and not hasattr(root, 'favicon_ico'): - favicon = os.path.join(os.getcwd(), os.path.dirname(__file__), 'favicon.ico') + needs_favicon = ( + script_name == '' + and root is not None + and not hasattr(root, 'favicon_ico') + ) + if needs_favicon: + favicon = os.path.join( + os.getcwd(), + os.path.dirname(__file__), + 'favicon.ico', + ) root.favicon_ico = tools.staticfile.handler(favicon) if config: @@ -234,7 +261,7 @@ def graft(self, wsgi_callable, script_name=''): self.apps[script_name] = wsgi_callable def script_name(self, path=None): - """The script_name of the app at the given path, or None. + """Return the script_name of the app at the given path, or None. If path is None, cherrypy.request is used. """ @@ -257,6 +284,7 @@ def script_name(self, path=None): path = path[:path.rfind('/')] def __call__(self, environ, start_response): + """Pre-initialize WSGI env and call WSGI-callable.""" # If you're calling this, then you're probably setting SCRIPT_NAME # to '' (some WSGI servers always set SCRIPT_NAME to ''). # Try to look up the app using the full path. diff --git a/lib/cherrypy/_cpwsgi.py b/lib/cherrypy/_cpwsgi.py index abb62b2..0b4942f 100644 --- a/lib/cherrypy/_cpwsgi.py +++ b/lib/cherrypy/_cpwsgi.py @@ -13,7 +13,7 @@ import six import cherrypy as _cherrypy -from cherrypy._cpcompat import ntob, ntou +from cherrypy._cpcompat import ntou from cherrypy import _cperror from cherrypy.lib import httputil from cherrypy.lib import is_closable_iterator @@ -192,7 +192,7 @@ def trap(self, func, *args, **kwargs): raise except StopIteration: raise - except: + except Exception: tb = _cperror.format_exc() _cherrypy.log(tb, severity=40) if not _cherrypy.request.show_tracebacks: @@ -213,7 +213,7 @@ def trap(self, func, *args, **kwargs): try: self.start_response(s, h, _sys.exc_info()) - except: + except Exception: # "The application must not trap any exceptions raised by # start_response, if it called start_response with exc_info. # Instead, it should allow such exceptions to propagate @@ -223,7 +223,7 @@ def trap(self, func, *args, **kwargs): raise if self.started_response: - return ntob('').join(b) + return b''.join(b) else: return b @@ -275,7 +275,7 @@ def __init__(self, environ, start_response, cpapp): self.iter_response = iter(r.body) self.write = start_response(outstatus, outheaders) - except: + except BaseException: self.close() raise diff --git a/lib/cherrypy/_cpwsgi_server.py b/lib/cherrypy/_cpwsgi_server.py index 24f57f6..11dd846 100644 --- a/lib/cherrypy/_cpwsgi_server.py +++ b/lib/cherrypy/_cpwsgi_server.py @@ -1,6 +1,7 @@ """ -WSGI server interface (see PEP 333). This adds some CP-specific bits to -the framework-agnostic cheroot package. +WSGI server interface (see PEP 333). + +This adds some CP-specific bits to the framework-agnostic cheroot package. """ import sys @@ -10,8 +11,28 @@ import cherrypy -class CPWSGIServer(cheroot.wsgi.Server): +class CPWSGIHTTPRequest(cheroot.server.HTTPRequest): + """Wrapper for cheroot.server.HTTPRequest. + + This is a layer, which preserves URI parsing mode like it which was + before Cheroot v5.8.0. + """ + + def __init__(self, server, conn): + """Initialize HTTP request container instance. + + Args: + server (cheroot.server.HTTPServer): + web server object receiving this request + conn (cheroot.server.HTTPConnection): + HTTP connection object for this request + """ + super(CPWSGIHTTPRequest, self).__init__( + server, conn, proxy_mode=True + ) + +class CPWSGIServer(cheroot.wsgi.Server): """Wrapper for cheroot.wsgi.Server. cheroot has been designed to not reference CherryPy in any way, @@ -20,9 +41,15 @@ class CPWSGIServer(cheroot.wsgi.Server): and apply some attributes from config -> cherrypy.server -> wsgi.Server. """ - version = 'CherryPy/' + cherrypy.__version__ + ' ' + cheroot.wsgi.Server.version + fmt = 'CherryPy/{cherrypy.__version__} {cheroot.wsgi.Server.version}' + version = fmt.format(**globals()) def __init__(self, server_adapter=cherrypy.server): + """Initialize CPWSGIServer instance. + + Args: + server_adapter (cherrypy._cpserver.Server): ... + """ self.server_adapter = server_adapter self.max_request_header_size = ( self.server_adapter.max_request_header_size or 0 @@ -36,17 +63,22 @@ def __init__(self, server_adapter=cherrypy.server): None) self.wsgi_version = self.server_adapter.wsgi_version - s = cheroot.wsgi.Server - s.__init__(self, server_adapter.bind_addr, cherrypy.tree, - self.server_adapter.thread_pool, - server_name, - max=self.server_adapter.thread_pool_max, - request_queue_size=self.server_adapter.socket_queue_size, - timeout=self.server_adapter.socket_timeout, - shutdown_timeout=self.server_adapter.shutdown_timeout, - accepted_queue_size=self.server_adapter.accepted_queue_size, - accepted_queue_timeout=self.server_adapter.accepted_queue_timeout, - ) + + super(CPWSGIServer, self).__init__( + server_adapter.bind_addr, cherrypy.tree, + self.server_adapter.thread_pool, + server_name, + max=self.server_adapter.thread_pool_max, + request_queue_size=self.server_adapter.socket_queue_size, + timeout=self.server_adapter.socket_timeout, + shutdown_timeout=self.server_adapter.shutdown_timeout, + accepted_queue_size=self.server_adapter.accepted_queue_size, + accepted_queue_timeout=self.server_adapter.accepted_queue_timeout, + peercreds_enabled=self.server_adapter.peercreds, + peercreds_resolve_enabled=self.server_adapter.peercreds_resolve, + ) + self.ConnectionClass.RequestHandlerClass = CPWSGIHTTPRequest + self.protocol = self.server_adapter.protocol_version self.nodelay = self.server_adapter.nodelay @@ -74,4 +106,5 @@ def __init__(self, server_adapter=cherrypy.server): self.server_adapter, 'statistics', False) def error_log(self, msg='', level=20, traceback=False): + """Write given message to the error log.""" cherrypy.engine.log(msg, level, traceback) diff --git a/lib/cherrypy/_helper.py b/lib/cherrypy/_helper.py index 0968e6d..314550c 100644 --- a/lib/cherrypy/_helper.py +++ b/lib/cherrypy/_helper.py @@ -1,19 +1,17 @@ -""" -Helper functions for CP apps -""" +"""Helper functions for CP apps.""" import six +from six.moves import urllib -from cherrypy._cpcompat import urljoin as _urljoin, urlencode as _urlencode from cherrypy._cpcompat import text_or_bytes import cherrypy def expose(func=None, alias=None): - """ - Expose the function or class, optionally providing - an alias or set of aliases. + """Expose the function or class. + + Optionally provide an alias or set of aliases. """ def expose_(func): func.exposed = True @@ -59,8 +57,9 @@ def expose_(func): def popargs(*args, **kwargs): - """A decorator for _cp_dispatch - (cherrypy.dispatch.Dispatcher.dispatch_method_name). + """Decorate _cp_dispatch. + + (cherrypy.dispatch.Dispatcher.dispatch_method_name) Optional keyword argument: handler=(Object or Function) @@ -136,7 +135,6 @@ def index(self): #... """ - # Since keyword arg comes after *args, we have to process it ourselves # for lower versions of python. @@ -219,7 +217,7 @@ def url(path='', qs='', script_name=None, base=None, relative=None): relative to the server root; i.e., it will start with a slash. """ if isinstance(qs, (tuple, list, dict)): - qs = _urlencode(qs) + qs = urllib.parse.urlencode(qs) if qs: qs = '?' + qs @@ -239,7 +237,7 @@ def url(path='', qs='', script_name=None, base=None, relative=None): if path == '': path = pi else: - path = _urljoin(pi, path) + path = urllib.parse.urljoin(pi, path) if script_name is None: script_name = cherrypy.request.script_name @@ -287,6 +285,7 @@ def url(path='', qs='', script_name=None, base=None, relative=None): def normalize_path(path): + """Resolve given path from relative into absolute form.""" if './' not in path: return path @@ -309,3 +308,37 @@ def normalize_path(path): newpath = '/' + newpath return newpath + + +#### +# Inlined from jaraco.classes 1.4.3 +# Ref #1673 +class _ClassPropertyDescriptor(object): + """Descript for read-only class-based property. + + Turns a classmethod-decorated func into a read-only property of that class + type (means the value cannot be set). + """ + + def __init__(self, fget, fset=None): + """Initialize a class property descriptor. + + Instantiated by ``_helper.classproperty``. + """ + self.fget = fget + self.fset = fset + + def __get__(self, obj, klass=None): + """Return property value.""" + if klass is None: + klass = type(obj) + return self.fget.__get__(obj, klass)() + + +def classproperty(func): # noqa: D401; irrelevant for properties + """Decorator like classmethod to implement a static class property.""" + if not isinstance(func, (classmethod, staticmethod)): + func = classmethod(func) + + return _ClassPropertyDescriptor(func) +#### diff --git a/lib/cherrypy/daemon.py b/lib/cherrypy/daemon.py old mode 100644 new mode 100755 index 772b2fc..74488c0 --- a/lib/cherrypy/daemon.py +++ b/lib/cherrypy/daemon.py @@ -65,7 +65,7 @@ def start(configfiles=None, daemonize=False, environment=None, # Always start the engine; this will start all other services try: engine.start() - except: + except Exception: # Assume the error has been logged already via bus.log. sys.exit(1) else: @@ -73,6 +73,7 @@ def start(configfiles=None, daemonize=False, environment=None, def run(): + """Run cherryd CLI.""" from optparse import OptionParser p = OptionParser() diff --git a/lib/cherrypy/lib/__init__.py b/lib/cherrypy/lib/__init__.py index af08028..f815f76 100644 --- a/lib/cherrypy/lib/__init__.py +++ b/lib/cherrypy/lib/__init__.py @@ -1,10 +1,14 @@ -"""CherryPy Library""" +"""CherryPy Library.""" def is_iterator(obj): - '''Returns a boolean indicating if the object provided implements - the iterator protocol (i.e. like a generator). This will return - false for objects which iterable, but not iterators themselves.''' + """Detect if the object provided implements the iterator protocol. + + (i.e. like a generator). + + This will return False for objects which are iterable, + but not iterators themselves. + """ from types import GeneratorType if isinstance(obj, GeneratorType): return True @@ -17,7 +21,7 @@ def is_iterator(obj): def is_closable_iterator(obj): - + """Detect if the given object is both closable and iterator.""" # Not an iterator. if not is_iterator(obj): return False @@ -41,17 +45,22 @@ def is_closable_iterator(obj): class file_generator(object): + """Yield the given input (a file object) in chunks (default 64k). - """Yield the given input (a file object) in chunks (default 64k). (Core)""" + (Core) + """ def __init__(self, input, chunkSize=65536): + """Initialize file_generator with file ``input`` for chunked access.""" self.input = input self.chunkSize = chunkSize def __iter__(self): + """Return iterator.""" return self def __next__(self): + """Return next chunk of file.""" chunk = self.input.read(self.chunkSize) if chunk: return chunk @@ -63,8 +72,10 @@ def __next__(self): def file_generator_limited(fileobj, count, chunk_size=65536): - """Yield the given file object in chunks, stopping after `count` - bytes has been emitted. Default chunk size is 64kB. (Core) + """Yield the given file object in chunks. + + Stopps after `count` bytes has been emitted. + Default chunk size is 64kB. (Core) """ remaining = count while remaining > 0: @@ -77,7 +88,7 @@ def file_generator_limited(fileobj, count, chunk_size=65536): def set_vary_header(response, header_name): - 'Add a Vary header to a response' + """Add a Vary header to a response.""" varies = response.headers.get('Vary', '') varies = [x.strip() for x in varies.split(',') if x.strip()] if header_name not in varies: diff --git a/lib/cherrypy/lib/auth.py b/lib/cherrypy/lib/auth.py deleted file mode 100644 index 34ad688..0000000 --- a/lib/cherrypy/lib/auth.py +++ /dev/null @@ -1,97 +0,0 @@ -import cherrypy -from cherrypy.lib import httpauth - - -def check_auth(users, encrypt=None, realm=None): - """If an authorization header contains credentials, return True or False. - """ - request = cherrypy.serving.request - if 'authorization' in request.headers: - # make sure the provided credentials are correctly set - ah = httpauth.parseAuthorization(request.headers['authorization']) - if ah is None: - raise cherrypy.HTTPError(400, 'Bad Request') - - if not encrypt: - encrypt = httpauth.DIGEST_AUTH_ENCODERS[httpauth.MD5] - - if hasattr(users, '__call__'): - try: - # backward compatibility - users = users() # expect it to return a dictionary - - if not isinstance(users, dict): - raise ValueError( - 'Authentication users must be a dictionary') - - # fetch the user password - password = users.get(ah['username'], None) - except TypeError: - # returns a password (encrypted or clear text) - password = users(ah['username']) - else: - if not isinstance(users, dict): - raise ValueError('Authentication users must be a dictionary') - - # fetch the user password - password = users.get(ah['username'], None) - - # validate the authorization by re-computing it here - # and compare it with what the user-agent provided - if httpauth.checkResponse(ah, password, method=request.method, - encrypt=encrypt, realm=realm): - request.login = ah['username'] - return True - - request.login = False - return False - - -def basic_auth(realm, users, encrypt=None, debug=False): - """If auth fails, raise 401 with a basic authentication header. - - realm - A string containing the authentication realm. - - users - A dict of the form: {username: password} or a callable returning - a dict. - - encrypt - callable used to encrypt the password returned from the user-agent. - if None it defaults to a md5 encryption. - - """ - if check_auth(users, encrypt): - if debug: - cherrypy.log('Auth successful', 'TOOLS.BASIC_AUTH') - return - - # inform the user-agent this path is protected - cherrypy.serving.response.headers[ - 'www-authenticate'] = httpauth.basicAuth(realm) - - raise cherrypy.HTTPError( - 401, 'You are not authorized to access that resource') - - -def digest_auth(realm, users, debug=False): - """If auth fails, raise 401 with a digest authentication header. - - realm - A string containing the authentication realm. - users - A dict of the form: {username: password} or a callable returning - a dict. - """ - if check_auth(users, realm=realm): - if debug: - cherrypy.log('Auth successful', 'TOOLS.DIGEST_AUTH') - return - - # inform the user-agent this path is protected - cherrypy.serving.response.headers[ - 'www-authenticate'] = httpauth.digestAuth(realm) - - raise cherrypy.HTTPError( - 401, 'You are not authorized to access that resource') diff --git a/lib/cherrypy/lib/auth_basic.py b/lib/cherrypy/lib/auth_basic.py index c9c9bd5..ad379a2 100644 --- a/lib/cherrypy/lib/auth_basic.py +++ b/lib/cherrypy/lib/auth_basic.py @@ -1,14 +1,9 @@ # This file is part of CherryPy # -*- coding: utf-8 -*- # vim:ts=4:sw=4:expandtab:fileencoding=utf-8 +"""HTTP Basic Authentication tool. -import binascii - -import cherrypy -from cherrypy._cpcompat import base64_decode - - -__doc__ = """This module provides a CherryPy 3.x tool which implements +This module provides a CherryPy 3.x tool which implements the server-side of HTTP Basic Access Authentication, as described in :rfc:`2617`. @@ -20,11 +15,20 @@ basic_auth = {'tools.auth_basic.on': True, 'tools.auth_basic.realm': 'earth', 'tools.auth_basic.checkpassword': checkpassword, + 'tools.auth_basic.accept_charset': 'UTF-8', } app_config = { '/' : basic_auth } """ +import binascii +import unicodedata +import base64 + +import cherrypy +from cherrypy._cpcompat import ntou, tonative + + __author__ = 'visteya' __date__ = 'April 2009' @@ -44,9 +48,10 @@ def checkpassword(realm, user, password): return checkpassword -def basic_auth(realm, checkpassword, debug=False): +def basic_auth(realm, checkpassword, debug=False, accept_charset='utf-8'): """A CherryPy tool which hooks at before_handler to perform - HTTP Basic Access Authentication, as specified in :rfc:`2617`. + HTTP Basic Access Authentication, as specified in :rfc:`2617` + and :rfc:`7617`. If the request has an 'authorization' header with a 'Basic' scheme, this tool attempts to authenticate the credentials supplied in that header. If @@ -66,6 +71,8 @@ def basic_auth(realm, checkpassword, debug=False): """ + fallback_charset = 'ISO-8859-1' + if '"' in realm: raise ValueError('Realm cannot contain the " (quote) character.') request = cherrypy.serving.request @@ -73,18 +80,41 @@ def basic_auth(realm, checkpassword, debug=False): auth_header = request.headers.get('authorization') if auth_header is not None: # split() error, base64.decodestring() error - with cherrypy.HTTPError.handle((ValueError, binascii.Error), 400, 'Bad Request'): + msg = 'Bad Request' + with cherrypy.HTTPError.handle((ValueError, binascii.Error), 400, msg): scheme, params = auth_header.split(' ', 1) if scheme.lower() == 'basic': - username, password = base64_decode(params).split(':', 1) + charsets = accept_charset, fallback_charset + decoded_params = base64.b64decode(params.encode('ascii')) + decoded_params = _try_decode(decoded_params, charsets) + decoded_params = ntou(decoded_params) + decoded_params = unicodedata.normalize('NFC', decoded_params) + decoded_params = tonative(decoded_params) + username, password = decoded_params.split(':', 1) if checkpassword(realm, username, password): if debug: cherrypy.log('Auth succeeded', 'TOOLS.AUTH_BASIC') request.login = username return # successful authentication + charset = accept_charset.upper() + charset_declaration = ( + (', charset="%s"' % charset) + if charset != fallback_charset + else '' + ) # Respond with 401 status and a WWW-Authenticate header - cherrypy.serving.response.headers[ - 'www-authenticate'] = 'Basic realm="%s"' % realm + cherrypy.serving.response.headers['www-authenticate'] = ( + 'Basic realm="%s"%s' % (realm, charset_declaration) + ) raise cherrypy.HTTPError( 401, 'You are not authorized to access that resource') + + +def _try_decode(subject, charsets): + for charset in charsets[:-1]: + try: + return tonative(subject, charset) + except ValueError: + pass + return tonative(subject, charsets[-1]) diff --git a/lib/cherrypy/lib/auth_digest.py b/lib/cherrypy/lib/auth_digest.py index 0818b24..9b4f55c 100644 --- a/lib/cherrypy/lib/auth_digest.py +++ b/lib/cherrypy/lib/auth_digest.py @@ -1,15 +1,9 @@ # This file is part of CherryPy # -*- coding: utf-8 -*- # vim:ts=4:sw=4:expandtab:fileencoding=utf-8 +"""HTTP Digest Authentication tool. -import time -from hashlib import md5 - -import cherrypy -from cherrypy._cpcompat import ntob, parse_http_list, parse_keqv_list - - -__doc__ = """An implementation of the server-side of HTTP Digest Access +An implementation of the server-side of HTTP Digest Access Authentication, which is described in :rfc:`2617`. Example usage, using the built-in get_ha1_dict_plain function which uses a dict @@ -21,15 +15,28 @@ 'tools.auth_digest.realm': 'wonderland', 'tools.auth_digest.get_ha1': get_ha1, 'tools.auth_digest.key': 'a565c27146791cfb', + 'tools.auth_digest.accept_charset': 'UTF-8', } app_config = { '/' : digest_auth } """ +import time +import functools +from hashlib import md5 + +from six.moves.urllib.request import parse_http_list, parse_keqv_list + +import cherrypy +from cherrypy._cpcompat import ntob, tonative + + __author__ = 'visteya' __date__ = 'April 2009' -md5_hex = lambda s: md5(ntob(s)).hexdigest() +def md5_hex(s): + return md5(ntob(s, 'utf-8')).hexdigest() + qop_auth = 'auth' qop_auth_int = 'auth-int' @@ -37,6 +44,9 @@ valid_algorithms = ('MD5', 'MD5-sess') +FALLBACK_CHARSET = 'ISO-8859-1' +DEFAULT_CHARSET = 'UTF-8' + def TRACE(msg): cherrypy.log(msg, context='TOOLS.AUTH_DIGEST') @@ -131,24 +141,47 @@ def H(s): return md5_hex(s) -class HttpDigestAuthorization (object): +def _try_decode_header(header, charset): + global FALLBACK_CHARSET + + for enc in (charset, FALLBACK_CHARSET): + try: + return tonative(ntob(tonative(header, 'latin1'), 'latin1'), enc) + except ValueError as ve: + last_err = ve + else: + raise last_err + - """Class to parse a Digest Authorization header and perform re-calculation - of the digest. +class HttpDigestAuthorization(object): """ + Parses a Digest Authorization header and performs + re-calculation of the digest. + """ + + scheme = 'digest' def errmsg(self, s): return 'Digest Authorization header: %s' % s - def __init__(self, auth_header, http_method, debug=False): + @classmethod + def matches(cls, header): + scheme, _, _ = header.partition(' ') + return scheme.lower() == cls.scheme + + def __init__( + self, auth_header, http_method, + debug=False, accept_charset=DEFAULT_CHARSET[:], + ): self.http_method = http_method self.debug = debug - scheme, params = auth_header.split(' ', 1) - self.scheme = scheme.lower() - if self.scheme != 'digest': + + if not self.matches(auth_header): raise ValueError('Authorization scheme is not "Digest"') - self.auth_header = auth_header + self.auth_header = _try_decode_header(auth_header, accept_charset) + + scheme, params = self.auth_header.split(' ', 1) # make a dict of the params items = parse_http_list(params) @@ -303,25 +336,44 @@ def request_digest(self, ha1, entity_body=''): return digest -def www_authenticate(realm, key, algorithm='MD5', nonce=None, qop=qop_auth, - stale=False): +def _get_charset_declaration(charset): + global FALLBACK_CHARSET + charset = charset.upper() + return ( + (', charset="%s"' % charset) + if charset != FALLBACK_CHARSET + else '' + ) + + +def www_authenticate( + realm, key, algorithm='MD5', nonce=None, qop=qop_auth, + stale=False, accept_charset=DEFAULT_CHARSET[:], +): """Constructs a WWW-Authenticate header for Digest authentication.""" if qop not in valid_qops: raise ValueError("Unsupported value for qop: '%s'" % qop) if algorithm not in valid_algorithms: raise ValueError("Unsupported value for algorithm: '%s'" % algorithm) + HEADER_PATTERN = ( + 'Digest realm="%s", nonce="%s", algorithm="%s", qop="%s"%s%s' + ) + if nonce is None: nonce = synthesize_nonce(realm, key) - s = 'Digest realm="%s", nonce="%s", algorithm="%s", qop="%s"' % ( - realm, nonce, algorithm, qop) - if stale: - s += ', stale="true"' - return s + + stale_param = ', stale="true"' if stale else '' + + charset_declaration = _get_charset_declaration(accept_charset) + + return HEADER_PATTERN % ( + realm, nonce, algorithm, qop, stale_param, charset_declaration, + ) -def digest_auth(realm, get_ha1, key, debug=False): - """A CherryPy tool which hooks at before_handler to perform +def digest_auth(realm, get_ha1, key, debug=False, accept_charset='utf-8'): + """A CherryPy tool that hooks at before_handler to perform HTTP Digest Access Authentication, as specified in :rfc:`2617`. If the request has an 'authorization' header with a 'Digest' scheme, @@ -334,7 +386,7 @@ def digest_auth(realm, get_ha1, key, debug=False): A string containing the authentication realm. get_ha1 - A callable which looks up a username in a credentials store + A callable that looks up a username in a credentials store and returns the HA1 string, which is defined in the RFC to be MD5(username : realm : password). The function's signature is: ``get_ha1(realm, username)`` @@ -350,39 +402,61 @@ def digest_auth(realm, get_ha1, key, debug=False): request = cherrypy.serving.request auth_header = request.headers.get('authorization') - nonce_is_stale = False - if auth_header is not None: - with cherrypy.HTTPError.handle( - ValueError, 400, 'The Authorization header could not be parsed.'): - auth = HttpDigestAuthorization( - auth_header, request.method, debug=debug) - - if debug: - TRACE(str(auth)) - - if auth.validate_nonce(realm, key): - ha1 = get_ha1(realm, auth.username) - if ha1 is not None: - # note that for request.body to be available we need to - # hook in at before_handler, not on_start_resource like - # 3.1.x digest_auth does. - digest = auth.request_digest(ha1, entity_body=request.body) - if digest == auth.response: # authenticated - if debug: - TRACE('digest matches auth.response') - # Now check if nonce is stale. - # The choice of ten minutes' lifetime for nonce is somewhat - # arbitrary - nonce_is_stale = auth.is_nonce_stale(max_age_seconds=600) - if not nonce_is_stale: - request.login = auth.username - if debug: - TRACE('authentication of %s successful' % - auth.username) - return - - # Respond with 401 status and a WWW-Authenticate header - header = www_authenticate(realm, key, stale=nonce_is_stale) + + respond_401 = functools.partial( + _respond_401, realm, key, accept_charset, debug) + + if not HttpDigestAuthorization.matches(auth_header or ''): + respond_401() + + msg = 'The Authorization header could not be parsed.' + with cherrypy.HTTPError.handle(ValueError, 400, msg): + auth = HttpDigestAuthorization( + auth_header, request.method, + debug=debug, accept_charset=accept_charset, + ) + + if debug: + TRACE(str(auth)) + + if not auth.validate_nonce(realm, key): + respond_401() + + ha1 = get_ha1(realm, auth.username) + + if ha1 is None: + respond_401() + + # note that for request.body to be available we need to + # hook in at before_handler, not on_start_resource like + # 3.1.x digest_auth does. + digest = auth.request_digest(ha1, entity_body=request.body) + if digest != auth.response: + respond_401() + + # authenticated + if debug: + TRACE('digest matches auth.response') + # Now check if nonce is stale. + # The choice of ten minutes' lifetime for nonce is somewhat + # arbitrary + if auth.is_nonce_stale(max_age_seconds=600): + respond_401(stale=True) + + request.login = auth.username + if debug: + TRACE('authentication of %s successful' % auth.username) + + +def _respond_401(realm, key, accept_charset, debug, **kwargs): + """ + Respond with 401 status and a WWW-Authenticate header + """ + header = www_authenticate( + realm, key, + accept_charset=accept_charset, + **kwargs + ) if debug: TRACE(header) cherrypy.serving.response.headers['WWW-Authenticate'] = header diff --git a/lib/cherrypy/lib/caching.py b/lib/cherrypy/lib/caching.py index f363a67..1673b3c 100644 --- a/lib/cherrypy/lib/caching.py +++ b/lib/cherrypy/lib/caching.py @@ -37,9 +37,11 @@ import threading import time +import six + import cherrypy from cherrypy.lib import cptools, httputil -from cherrypy._cpcompat import copyitems, ntob, sorted, Event +from cherrypy._cpcompat import Event class Cache(object): @@ -48,19 +50,19 @@ class Cache(object): def get(self): """Return the current variant if in the cache, else None.""" - raise NotImplemented + raise NotImplementedError def put(self, obj, size): """Store the current variant in the cache.""" - raise NotImplemented + raise NotImplementedError def delete(self): """Remove ALL cached variants of the current resource.""" - raise NotImplemented + raise NotImplementedError def clear(self): """Reset the cache to its initial, empty state.""" - raise NotImplemented + raise NotImplementedError # ------------------------------ Memory Cache ------------------------------- # @@ -197,7 +199,8 @@ def expire_cache(self): now = time.time() # Must make a copy of expirations so it doesn't change size # during iteration - for expiration_time, objects in copyitems(self.expirations): + items = list(six.iteritems(self.expirations)) + for expiration_time, objects in items: if expiration_time <= now: for obj_size, uri, sel_header_values in objects: try: @@ -402,10 +405,19 @@ def tee(body): output.append(chunk) yield chunk - # save the cache data - body = ntob('').join(output) - cherrypy._cache.put((response.status, response.headers or {}, - body, response.time), len(body)) + # Save the cache data, but only if the body isn't empty. + # e.g. a 304 Not Modified on a static file response will + # have an empty body. + # If the body is empty, delete the cache because it + # contains a stale Threading._Event object that will + # stall all consecutive requests until the _Event times + # out + body = b''.join(output) + if not body: + cherrypy._cache.delete() + else: + cherrypy._cache.put((response.status, response.headers or {}, + body, response.time), len(body)) response = cherrypy.serving.response response.body = tee(response.body) diff --git a/lib/cherrypy/lib/covercp.py b/lib/cherrypy/lib/covercp.py index 5b214a2..0bafca1 100644 --- a/lib/cherrypy/lib/covercp.py +++ b/lib/cherrypy/lib/covercp.py @@ -26,8 +26,9 @@ import os import os.path +from six.moves import urllib + import cherrypy -from cherrypy._cpcompat import quote_plus localFile = os.path.join(os.path.dirname(__file__), 'coverage.cache') @@ -212,7 +213,7 @@ def _show_branch(root, base, path, pct=0, showpct=False, exclude='', yield ( "%s\n" % - (newpath, quote_plus(exclude), name) + (newpath, urllib.parse.quote_plus(exclude), name) ) for chunk in _show_branch( @@ -233,7 +234,7 @@ def _show_branch(root, base, path, pct=0, showpct=False, exclude='', if showpct: try: _, statements, _, missing, _ = coverage.analysis2(newpath) - except: + except Exception: # Yes, we really want to pass on all errors. pass else: @@ -290,7 +291,6 @@ def __init__(self, coverage, root=None): if root is None: # Guess initial depth. Files outside this path will not be # reachable from the web interface. - import cherrypy root = os.path.dirname(cherrypy.__file__) self.root = root @@ -316,7 +316,7 @@ def menu(self, base='/', pct='50', showpct='', for atom in atoms: path += atom + os.sep yield ("%s %s" - % (path, quote_plus(exclude), atom, os.sep)) + % (path, urllib.parse.quote_plus(exclude), atom, os.sep)) yield '' yield "
" @@ -380,7 +380,6 @@ def serve(path=localFile, port=8080, root=None): cov = coverage(data_file=path) cov.load() - import cherrypy cherrypy.config.update({'server.socket_port': int(port), 'server.thread_pool': 10, 'environment': 'production', diff --git a/lib/cherrypy/lib/cpstats.py b/lib/cherrypy/lib/cpstats.py index b2debd8..ae9f747 100644 --- a/lib/cherrypy/lib/cpstats.py +++ b/lib/cherrypy/lib/cpstats.py @@ -193,6 +193,8 @@ import threading import time +import six + import cherrypy from cherrypy._cpcompat import json @@ -248,7 +250,9 @@ def extrapolate_statistics(scope): 'Requests': {}, }) -proc_time = lambda s: time.time() - s['Start Time'] + +def proc_time(s): + return time.time() - s['Start Time'] class ByteCountWrapper(object): @@ -294,7 +298,8 @@ def next(self): return data -average_uriset_time = lambda s: s['Count'] and (s['Sum'] / s['Count']) or 0 +def average_uriset_time(s): + return s['Count'] and (s['Sum'] / s['Count']) or 0 def _get_threading_ident(): @@ -402,8 +407,13 @@ def record_stop( missing = object() -locale_date = lambda v: time.strftime('%c', time.gmtime(v)) -iso_format = lambda v: time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(v)) + +def locale_date(v): + return time.strftime('%c', time.gmtime(v)) + + +def iso_format(v): + return time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(v)) def pause_resume(ns): @@ -603,12 +613,7 @@ def get_dict_collection(self, v, formatting): """Return ([headers], [rows]) for the given collection.""" # E.g., the 'Requests' dict. headers = [] - try: - # python2 - vals = v.itervalues() - except AttributeError: - # python3 - vals = v.values() + vals = six.itervalues(v) for record in vals: for k3 in record: format = formatting.get(k3, missing) diff --git a/lib/cherrypy/lib/cptools.py b/lib/cherrypy/lib/cptools.py index 2ad5a44..1c07963 100644 --- a/lib/cherrypy/lib/cptools.py +++ b/lib/cherrypy/lib/cptools.py @@ -5,6 +5,7 @@ from hashlib import md5 import six +from six.moves import urllib import cherrypy from cherrypy._cpcompat import text_or_bytes @@ -195,10 +196,8 @@ def proxy(base=None, local='X-Forwarded-Host', remote='X-Forwarded-For', if lbase is not None: base = lbase.split(',')[0] if not base: - base = request.headers.get('Host', '127.0.0.1') - port = request.local.port - if port != 80: - base += ':%s' % port + default = urllib.parse.urlparse(request.base).netloc + base = request.headers.get('Host', default) if base.find('://') == -1: # add http:// or https:// if needed @@ -212,8 +211,8 @@ def proxy(base=None, local='X-Forwarded-Host', remote='X-Forwarded-For', cherrypy.log('Testing remote %r:%r' % (remote, xff), 'TOOLS.PROXY') if xff: if remote == 'X-Forwarded-For': - # Bug #1268 - xff = xff.split(',')[0].strip() + # Grab the first IP in a comma-separated list. Ref #1268. + xff = next(ip.strip() for ip in xff.split(',')) request.remote.ip = xff @@ -240,7 +239,9 @@ def response_headers(headers=None, debug=False): 'TOOLS.RESPONSE_HEADERS') for name, value in (headers or []): cherrypy.serving.response.headers[name] = value -response_headers.failsafe = True # noqa: E305 + + +response_headers.failsafe = True def referer(pattern, accept=True, accept_missing=False, error=403, @@ -409,7 +410,9 @@ def session_auth(**kwargs): for k, v in kwargs.items(): setattr(sa, k, v) return sa.run() -session_auth.__doc__ = ( # noqa: E305 + + +session_auth.__doc__ = ( """Session authentication hook. Any attribute of the SessionAuth class may be overridden via a keyword arg @@ -586,26 +589,13 @@ def accept(media=None, debug=False): class MonitoredHeaderMap(_httputil.HeaderMap): - def __init__(self): - self.accessed_headers = set() - - def __getitem__(self, key): + def transform_key(self, key): self.accessed_headers.add(key) - return _httputil.HeaderMap.__getitem__(self, key) + return super(MonitoredHeaderMap, self).transform_key(key) - def __contains__(self, key): - self.accessed_headers.add(key) - return _httputil.HeaderMap.__contains__(self, key) - - def get(self, key, default=None): - self.accessed_headers.add(key) - return _httputil.HeaderMap.get(self, key, default=default) - - if hasattr({}, 'has_key'): - # Python 2 - def has_key(self, key): - self.accessed_headers.add(key) - return _httputil.HeaderMap.has_key(self, key) # noqa: W601 + def __init__(self): + self.accessed_headers = set() + super(MonitoredHeaderMap, self).__init__() def autovary(ignore=None, debug=False): diff --git a/lib/cherrypy/lib/encoding.py b/lib/cherrypy/lib/encoding.py index 72e58f9..3d001ca 100644 --- a/lib/cherrypy/lib/encoding.py +++ b/lib/cherrypy/lib/encoding.py @@ -5,7 +5,7 @@ import six import cherrypy -from cherrypy._cpcompat import text_or_bytes, ntob +from cherrypy._cpcompat import text_or_bytes from cherrypy.lib import file_generator from cherrypy.lib import is_closable_iterator from cherrypy.lib import set_vary_header @@ -220,19 +220,7 @@ def __call__(self, *args, **kwargs): response = cherrypy.serving.response self.body = self.oldhandler(*args, **kwargs) - if isinstance(self.body, text_or_bytes): - # strings get wrapped in a list because iterating over a single - # item list is much faster than iterating over every character - # in a long string. - if self.body: - self.body = [self.body] - else: - # [''] doesn't evaluate to False, so replace it with []. - self.body = [] - elif hasattr(self.body, 'read'): - self.body = file_generator(self.body) - elif self.body is None: - self.body = [] + self.body = prepare_iter(self.body) ct = response.headers.elements('Content-Type') if self.debug: @@ -269,6 +257,29 @@ def __call__(self, *args, **kwargs): return self.body + +def prepare_iter(value): + """ + Ensure response body is iterable and resolves to False when empty. + """ + if isinstance(value, text_or_bytes): + # strings get wrapped in a list because iterating over a single + # item list is much faster than iterating over every character + # in a long string. + if value: + value = [value] + else: + # [''] doesn't evaluate to False, so replace it with []. + value = [] + # Don't use isinstance here; io.IOBase which has an ABC takes + # 1000 times as long as, say, isinstance(value, str) + elif hasattr(value, 'read'): + value = file_generator(value) + elif value is None: + value = [] + return value + + # GZIP @@ -277,15 +288,15 @@ def compress(body, compress_level): import zlib # See http://www.gzip.org/zlib/rfc-gzip.html - yield ntob('\x1f\x8b') # ID1 and ID2: gzip marker - yield ntob('\x08') # CM: compression method - yield ntob('\x00') # FLG: none set + yield b'\x1f\x8b' # ID1 and ID2: gzip marker + yield b'\x08' # CM: compression method + yield b'\x00' # FLG: none set # MTIME: 4 bytes yield struct.pack(' -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - * Neither the name of Sylvain Hellegouarch nor the names of his - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -""" - -__all__ = ('digestAuth', 'basicAuth', 'doAuth', 'checkResponse', - 'parseAuthorization', 'SUPPORTED_ALGORITHM', 'md5SessionKey', - 'calculateNonce', 'SUPPORTED_QOP') - -########################################################################## - -MD5 = 'MD5' -MD5_SESS = 'MD5-sess' -AUTH = 'auth' -AUTH_INT = 'auth-int' - -SUPPORTED_ALGORITHM = (MD5, MD5_SESS) -SUPPORTED_QOP = (AUTH, AUTH_INT) - -########################################################################## -# doAuth -# -DIGEST_AUTH_ENCODERS = { - MD5: lambda val: md5(ntob(val)).hexdigest(), - MD5_SESS: lambda val: md5(ntob(val)).hexdigest(), - # SHA: lambda val: sha.new(ntob(val)).hexdigest (), -} - - -def calculateNonce(realm, algorithm=MD5): - """This is an auxaliary function that calculates 'nonce' value. It is used - to handle sessions.""" - - global SUPPORTED_ALGORITHM, DIGEST_AUTH_ENCODERS - assert algorithm in SUPPORTED_ALGORITHM - - try: - encoder = DIGEST_AUTH_ENCODERS[algorithm] - except KeyError: - raise NotImplementedError('The chosen algorithm (%s) does not have ' - 'an implementation yet' % algorithm) - - return encoder('%d:%s' % (time.time(), realm)) - - -def digestAuth(realm, algorithm=MD5, nonce=None, qop=AUTH): - """Challenges the client for a Digest authentication.""" - global SUPPORTED_ALGORITHM, DIGEST_AUTH_ENCODERS, SUPPORTED_QOP - assert algorithm in SUPPORTED_ALGORITHM - assert qop in SUPPORTED_QOP - - if nonce is None: - nonce = calculateNonce(realm, algorithm) - - return 'Digest realm="%s", nonce="%s", algorithm="%s", qop="%s"' % ( - realm, nonce, algorithm, qop - ) - - -def basicAuth(realm): - """Challengenes the client for a Basic authentication.""" - assert '"' not in realm, "Realms cannot contain the \" (quote) character." - - return 'Basic realm="%s"' % realm - - -def doAuth(realm): - """'doAuth' function returns the challenge string b giving priority over - Digest and fallback to Basic authentication when the browser doesn't - support the first one. - - This should be set in the HTTP header under the key 'WWW-Authenticate'.""" - - return digestAuth(realm) + ' ' + basicAuth(realm) - - -########################################################################## -# Parse authorization parameters -# -def _parseDigestAuthorization(auth_params): - # Convert the auth params to a dict - items = parse_http_list(auth_params) - params = parse_keqv_list(items) - - # Now validate the params - - # Check for required parameters - required = ['username', 'realm', 'nonce', 'uri', 'response'] - for k in required: - if k not in params: - return None - - # If qop is sent then cnonce and nc MUST be present - if 'qop' in params and not ('cnonce' in params and 'nc' in params): - return None - - # If qop is not sent, neither cnonce nor nc can be present - if ('cnonce' in params or 'nc' in params) and 'qop' not in params: - return None - - return params - - -def _parseBasicAuthorization(auth_params): - username, password = base64_decode(auth_params).split(':', 1) - return {'username': username, 'password': password} - -AUTH_SCHEMES = { # noqa: E305 - 'basic': _parseBasicAuthorization, - 'digest': _parseDigestAuthorization, -} - - -def parseAuthorization(credentials): - """parseAuthorization will convert the value of the 'Authorization' key in - the HTTP header to a map itself. If the parsing fails 'None' is returned. - """ - - global AUTH_SCHEMES - - auth_scheme, auth_params = credentials.split(' ', 1) - auth_scheme = auth_scheme.lower() - - parser = AUTH_SCHEMES[auth_scheme] - params = parser(auth_params) - - if params is None: - return - - assert 'auth_scheme' not in params - params['auth_scheme'] = auth_scheme - return params - - -########################################################################## -# Check provided response for a valid password -# -def md5SessionKey(params, password): - """ - If the "algorithm" directive's value is "MD5-sess", then A1 - [the session key] is calculated only once - on the first request by the - client following receipt of a WWW-Authenticate challenge from the server. - - This creates a 'session key' for the authentication of subsequent - requests and responses which is different for each "authentication - session", thus limiting the amount of material hashed with any one - key. - - Because the server need only use the hash of the user - credentials in order to create the A1 value, this construction could - be used in conjunction with a third party authentication service so - that the web server would not need the actual password value. The - specification of such a protocol is beyond the scope of this - specification. -""" - - keys = ('username', 'realm', 'nonce', 'cnonce') - params_copy = {} - for key in keys: - params_copy[key] = params[key] - - params_copy['algorithm'] = MD5_SESS - return _A1(params_copy, password) - - -def _A1(params, password): - algorithm = params.get('algorithm', MD5) - H = DIGEST_AUTH_ENCODERS[algorithm] - - if algorithm == MD5: - # If the "algorithm" directive's value is "MD5" or is - # unspecified, then A1 is: - # A1 = unq(username-value) ":" unq(realm-value) ":" passwd - return '%s:%s:%s' % (params['username'], params['realm'], password) - - elif algorithm == MD5_SESS: - - # This is A1 if qop is set - # A1 = H( unq(username-value) ":" unq(realm-value) ":" passwd ) - # ":" unq(nonce-value) ":" unq(cnonce-value) - h_a1 = H('%s:%s:%s' % (params['username'], params['realm'], password)) - return '%s:%s:%s' % (h_a1, params['nonce'], params['cnonce']) - - -def _A2(params, method, kwargs): - # If the "qop" directive's value is "auth" or is unspecified, then A2 is: - # A2 = Method ":" digest-uri-value - - qop = params.get('qop', 'auth') - if qop == 'auth': - return method + ':' + params['uri'] - elif qop == 'auth-int': - # If the "qop" value is "auth-int", then A2 is: - # A2 = Method ":" digest-uri-value ":" H(entity-body) - entity_body = kwargs.get('entity_body', '') - H = kwargs['H'] - - return '%s:%s:%s' % ( - method, - params['uri'], - H(entity_body) - ) - - else: - raise NotImplementedError("The 'qop' method is unknown: %s" % qop) - - -def _computeDigestResponse(auth_map, password, method='GET', A1=None, - **kwargs): - """ - Generates a response respecting the algorithm defined in RFC 2617 - """ - params = auth_map - - algorithm = params.get('algorithm', MD5) - - H = DIGEST_AUTH_ENCODERS[algorithm] - KD = lambda secret, data: H(secret + ':' + data) - - qop = params.get('qop', None) - - H_A2 = H(_A2(params, method, kwargs)) - - if algorithm == MD5_SESS and A1 is not None: - H_A1 = H(A1) - else: - H_A1 = H(_A1(params, password)) - - if qop in ('auth', 'auth-int'): - # If the "qop" value is "auth" or "auth-int": - # request-digest = <"> < KD ( H(A1), unq(nonce-value) - # ":" nc-value - # ":" unq(cnonce-value) - # ":" unq(qop-value) - # ":" H(A2) - # ) <"> - request = '%s:%s:%s:%s:%s' % ( - params['nonce'], - params['nc'], - params['cnonce'], - params['qop'], - H_A2, - ) - elif qop is None: - # If the "qop" directive is not present (this construction is - # for compatibility with RFC 2069): - # request-digest = - # <"> < KD ( H(A1), unq(nonce-value) ":" H(A2) ) > <"> - request = '%s:%s' % (params['nonce'], H_A2) - - return KD(H_A1, request) - - -def _checkDigestResponse(auth_map, password, method='GET', A1=None, **kwargs): - """This function is used to verify the response given by the client when - he tries to authenticate. - Optional arguments: - entity_body - when 'qop' is set to 'auth-int' you MUST provide the - raw data you are going to send to the client (usually the - HTML page. - request_uri - the uri from the request line compared with the 'uri' - directive of the authorization map. They must represent - the same resource (unused at this time). - """ - - if auth_map['realm'] != kwargs.get('realm', None): - return False - - response = _computeDigestResponse( - auth_map, password, method, A1, **kwargs) - - return response == auth_map['response'] - - -def _checkBasicResponse(auth_map, password, method='GET', encrypt=None, - **kwargs): - # Note that the Basic response doesn't provide the realm value so we cannot - # test it - pass_through = lambda password, username=None: password - encrypt = encrypt or pass_through - try: - candidate = encrypt(auth_map['password'], auth_map['username']) - except TypeError: - # if encrypt only takes one parameter, it's the password - candidate = encrypt(auth_map['password']) - return candidate == password - -AUTH_RESPONSES = { # noqa: E305 - 'basic': _checkBasicResponse, - 'digest': _checkDigestResponse, -} - - -def checkResponse(auth_map, password, method='GET', encrypt=None, **kwargs): - """'checkResponse' compares the auth_map with the password and optionally - other arguments that each implementation might need. - - If the response is of type 'Basic' then the function has the following - signature:: - - checkBasicResponse(auth_map, password) -> bool - - If the response is of type 'Digest' then the function has the following - signature:: - - checkDigestResponse(auth_map, password, method='GET', A1=None) -> bool - - The 'A1' argument is only used in MD5_SESS algorithm based responses. - Check md5SessionKey() for more info. - """ - checker = AUTH_RESPONSES[auth_map['auth_scheme']] - return checker(auth_map, password, method=method, encrypt=encrypt, - **kwargs) diff --git a/lib/cherrypy/lib/httputil.py b/lib/cherrypy/lib/httputil.py index 1eb3e64..59bcc74 100644 --- a/lib/cherrypy/lib/httputil.py +++ b/lib/cherrypy/lib/httputil.py @@ -12,17 +12,15 @@ import re from binascii import b2a_base64 from cgi import parse_header -try: - # Python 3 - from email.header import decode_header -except ImportError: - from email.Header import decode_header +from email.header import decode_header import six +from six.moves import range, builtins, map +from six.moves.BaseHTTPServer import BaseHTTPRequestHandler -from cherrypy._cpcompat import BaseHTTPRequestHandler, ntob, ntou -from cherrypy._cpcompat import text_or_bytes, iteritems -from cherrypy._cpcompat import reversed, sorted, unquote_qs +import cherrypy +from cherrypy._cpcompat import ntob, ntou +from cherrypy._cpcompat import unquote_plus response_codes = BaseHTTPRequestHandler.responses.copy() @@ -40,7 +38,7 @@ def urljoin(*atoms): - """Return the given path \*atoms, joined into a single URL. + r"""Return the given path \*atoms, joined into a single URL. This will correctly join a SCRIPT_NAME and PATH_INFO into the original URL, even if either atom is blank. @@ -58,11 +56,11 @@ def urljoin_bytes(*atoms): This will correctly join a SCRIPT_NAME and PATH_INFO into the original URL, even if either atom is blank. """ - url = ntob('/').join([x for x in atoms if x]) - while ntob('//') in url: - url = url.replace(ntob('//'), ntob('/')) + url = b'/'.join([x for x in atoms if x]) + while b'//' in url: + url = url.replace(b'//', b'/') # Special-case the final url of "", and return "/" instead. - return url or ntob('/') + return url or b'/' def protocol_from_http(protocol_str): @@ -139,13 +137,13 @@ def __init__(self, value, params=None): self.params = params def __cmp__(self, other): - return cmp(self.value, other.value) # noqa: F821 + return builtins.cmp(self.value, other.value) def __lt__(self, other): return self.value < other.value def __str__(self): - p = [';%s=%s' % (k, v) for k, v in iteritems(self.params)] + p = [';%s=%s' % (k, v) for k, v in six.iteritems(self.params)] return str('%s%s' % (self.value, ''.join(p))) def __bytes__(self): @@ -204,12 +202,26 @@ def qvalue(self): val = self.params.get('q', '1') if isinstance(val, HeaderElement): val = val.value - return float(val) + try: + return float(val) + except ValueError as val_err: + """Fail client requests with invalid quality value. + + Ref: https://github.com/cherrypy/cherrypy/issues/1370 + """ + six.raise_from( + cherrypy.HTTPError( + 400, + 'Malformed HTTP header: `{}`'. + format(str(self)), + ), + val_err, + ) def __cmp__(self, other): - diff = cmp(self.qvalue, other.qvalue) # noqa: F821 + diff = builtins.cmp(self.qvalue, other.qvalue) if diff == 0: - diff = cmp(str(self), str(other)) # noqa: F821 + diff = builtins.cmp(str(self), str(other)) return diff def __lt__(self, other): @@ -218,8 +230,11 @@ def __lt__(self, other): else: return self.qvalue < other.qvalue -RE_HEADER_SPLIT = re.compile(',(?=(?:[^"]*"[^"]*")*[^"]*$)') # noqa: E305 -def header_elements(fieldname, fieldvalue): # noqa: E302 + +RE_HEADER_SPLIT = re.compile(',(?=(?:[^"]*"[^"]*")*[^"]*$)') + + +def header_elements(fieldname, fieldvalue): """Return a sorted HeaderElement list from a comma-separated header string. """ if not fieldvalue: @@ -237,7 +252,12 @@ def header_elements(fieldname, fieldvalue): # noqa: E302 def decode_TEXT(value): - r"""Decode :rfc:`2047` TEXT (e.g. "=?utf-8?q?f=C3=BCr?=" -> "f\xfcr").""" + r""" + Decode :rfc:`2047` TEXT + + >>> decode_TEXT("=?utf-8?q?f=C3=BCr?=") == b'f\xfcr'.decode('latin-1') + True + """ atoms = decode_header(value) decodedvalue = '' for atom, charset in atoms: @@ -247,31 +267,41 @@ def decode_TEXT(value): return decodedvalue +def decode_TEXT_maybe(value): + """ + Decode the text but only if '=?' appears in it. + """ + return decode_TEXT(value) if '=?' in value else value + + def valid_status(status): """Return legal HTTP status Code, Reason-phrase and Message. - The status arg must be an int, or a str that begins with an int. + The status arg must be an int, a str that begins with an int + or the constant from ``http.client`` stdlib module. + + If status has no reason-phrase is supplied, a default reason- + phrase will be provided. - If status is an int, or a str and no reason-phrase is supplied, - a default reason-phrase will be provided. + >>> from six.moves import http_client + >>> from six.moves.BaseHTTPServer import BaseHTTPRequestHandler + >>> valid_status(http_client.ACCEPTED) == ( + ... int(http_client.ACCEPTED), + ... ) + BaseHTTPRequestHandler.responses[http_client.ACCEPTED] + True """ if not status: status = 200 - status = str(status) - parts = status.split(' ', 1) - if len(parts) == 1: - # No reason supplied. - code, = parts - reason = None - else: - code, reason = parts - reason = reason.strip() + code, reason = status, None + if isinstance(status, six.string_types): + code, _, reason = status.partition(' ') + reason = reason.strip() or None try: code = int(code) - except ValueError: + except (TypeError, ValueError): raise ValueError('Illegal response status from server ' '(%s is non-numeric).' % repr(code)) @@ -329,8 +359,8 @@ def _parse_qs(qs, keep_blank_values=0, strict_parsing=0, encoding='utf-8'): else: continue if len(nv[1]) or keep_blank_values: - name = unquote_qs(nv[0], encoding) - value = unquote_qs(nv[1], encoding) + name = unquote_plus(nv[0], encoding, errors='strict') + value = unquote_plus(nv[1], encoding, errors='strict') if name in d: if not isinstance(d[name], list): d[name] = [d[name]] @@ -360,53 +390,77 @@ def parse_query_string(query_string, keep_blank_values=True, encoding='utf-8'): return pm -class CaseInsensitiveDict(dict): +#### +# Inlined from jaraco.collections 1.5.2 +# Ref #1673 +class KeyTransformingDict(dict): + """ + A dict subclass that transforms the keys before they're used. + Subclasses may override the default transform_key to customize behavior. + """ + @staticmethod + def transform_key(key): + return key - """A case-insensitive dict subclass. + def __init__(self, *args, **kargs): + super(KeyTransformingDict, self).__init__() + # build a dictionary using the default constructs + d = dict(*args, **kargs) + # build this dictionary using transformed keys. + for item in d.items(): + self.__setitem__(*item) - Each key is changed on entry to str(key).title(). - """ + def __setitem__(self, key, val): + key = self.transform_key(key) + super(KeyTransformingDict, self).__setitem__(key, val) def __getitem__(self, key): - return dict.__getitem__(self, str(key).title()) + key = self.transform_key(key) + return super(KeyTransformingDict, self).__getitem__(key) - def __setitem__(self, key, value): - dict.__setitem__(self, str(key).title(), value) + def __contains__(self, key): + key = self.transform_key(key) + return super(KeyTransformingDict, self).__contains__(key) def __delitem__(self, key): - dict.__delitem__(self, str(key).title()) + key = self.transform_key(key) + return super(KeyTransformingDict, self).__delitem__(key) - def __contains__(self, key): - return dict.__contains__(self, str(key).title()) + def get(self, key, *args, **kwargs): + key = self.transform_key(key) + return super(KeyTransformingDict, self).get(key, *args, **kwargs) - def get(self, key, default=None): - return dict.get(self, str(key).title(), default) + def setdefault(self, key, *args, **kwargs): + key = self.transform_key(key) + return super(KeyTransformingDict, self).setdefault( + key, *args, **kwargs) - if hasattr({}, 'has_key'): - def has_key(self, key): - return str(key).title() in self + def pop(self, key, *args, **kwargs): + key = self.transform_key(key) + return super(KeyTransformingDict, self).pop(key, *args, **kwargs) - def update(self, E): - for k in E.keys(): - self[str(k).title()] = E[k] + def matching_key_for(self, key): + """ + Given a key, return the actual key stored in self that matches. + Raise KeyError if the key isn't found. + """ + try: + return next(e_key for e_key in self.keys() if e_key == key) + except StopIteration: + raise KeyError(key) +#### - @classmethod - def fromkeys(cls, seq, value=None): - newdict = cls() - for k in seq: - newdict[str(k).title()] = value - return newdict - def setdefault(self, key, x=None): - key = str(key).title() - try: - return self[key] - except KeyError: - self[key] = x - return x +class CaseInsensitiveDict(KeyTransformingDict): - def pop(self, key, default): - return dict.pop(self, str(key).title(), default) + """A case-insensitive dict subclass. + + Each key is changed on entry to str(key).title(). + """ + + @staticmethod + def transform_key(key): + return str(key).title() # TEXT = @@ -415,9 +469,9 @@ def pop(self, key, default): # field continuation. It is expected that the folding LWS will be # replaced with a single SP before interpretation of the TEXT value." if str == bytes: - header_translate_table = ''.join([chr(i) for i in six.moves.xrange(256)]) + header_translate_table = ''.join([chr(i) for i in range(256)]) header_translate_deletechars = ''.join( - [chr(i) for i in six.moves.xrange(32)]) + chr(127) + [chr(i) for i in range(32)]) + chr(127) else: header_translate_table = None header_translate_deletechars = bytes(range(32)) + bytes([127]) @@ -464,23 +518,21 @@ def encode_header_items(cls, header_items): transmitting on the wire for HTTP. """ for k, v in header_items: - if isinstance(k, six.text_type): - k = cls.encode(k) + if not isinstance(v, six.string_types) and \ + not isinstance(v, six.binary_type): + v = six.text_type(v) - if not isinstance(v, text_or_bytes): - v = str(v) + yield tuple(map(cls.encode_header_item, (k, v))) - if isinstance(v, six.text_type): - v = cls.encode(v) - - # See header_translate_* constants above. - # Replace only if you really know what you're doing. - k = k.translate(header_translate_table, - header_translate_deletechars) - v = v.translate(header_translate_table, - header_translate_deletechars) + @classmethod + def encode_header_item(cls, item): + if isinstance(item, six.text_type): + item = cls.encode(item) - yield (k, v) + # See header_translate_* constants above. + # Replace only if you really know what you're doing. + return item.translate( + header_translate_table, header_translate_deletechars) @classmethod def encode(cls, v): @@ -498,7 +550,7 @@ def encode(cls, v): # because we never want to fold lines--folding has # been deprecated by the HTTP working group. v = b2a_base64(v.encode('utf-8')) - return (ntob('=?utf-8?b?') + v.strip(ntob('\n')) + ntob('?=')) + return (b'=?utf-8?b?' + v.strip(b'\n') + b'?=') raise ValueError('Could not encode header part %r using ' 'any of the encodings %r.' % diff --git a/lib/cherrypy/lib/jsontools.py b/lib/cherrypy/lib/jsontools.py index 91ea74e..4868309 100644 --- a/lib/cherrypy/lib/jsontools.py +++ b/lib/cherrypy/lib/jsontools.py @@ -34,9 +34,6 @@ def json_in(content_type=[ntou('application/json'), ntou('text/javascript')], request header, or it will raise "411 Length Required". If for any other reason the request entity cannot be deserialized from JSON, it will raise "400 Bad Request: Invalid JSON document". - - You must be using Python 2.6 or greater, or have the 'simplejson' - package importable; otherwise, ValueError is raised during processing. """ request = cherrypy.serving.request if isinstance(content_type, text_or_bytes): @@ -72,9 +69,6 @@ def json_out(content_type='application/json', debug=False, Provide your own handler to use a custom encoder. For example cherrypy.config['tools.json_out.handler'] = , or @json_out(handler=function). - - You must be using Python 2.6 or greater, or have the 'simplejson' - package importable; otherwise, ValueError is raised during processing. """ request = cherrypy.serving.request # request.handler may be set to None by e.g. the caching tool diff --git a/lib/cherrypy/lib/lockfile.py b/lib/cherrypy/lib/lockfile.py deleted file mode 100644 index 336b558..0000000 --- a/lib/cherrypy/lib/lockfile.py +++ /dev/null @@ -1,142 +0,0 @@ -""" -Platform-independent file locking. Inspired by and modeled after zc.lockfile. -""" - -import os - -try: - import msvcrt -except ImportError: - pass - -try: - import fcntl -except ImportError: - pass - - -class LockError(Exception): - - 'Could not obtain a lock' - - msg = 'Unable to lock %r' - - def __init__(self, path): - super(LockError, self).__init__(self.msg % path) - - -class UnlockError(LockError): - - 'Could not release a lock' - - msg = 'Unable to unlock %r' - - -# first, a default, naive locking implementation -class LockFile(object): - - """ - A default, naive locking implementation. Always fails if the file - already exists. - """ - - def __init__(self, path): - self.path = path - try: - fd = os.open(path, os.O_CREAT | os.O_WRONLY | os.O_EXCL) - except OSError: - raise LockError(self.path) - os.close(fd) - - def release(self): - os.remove(self.path) - - def remove(self): - pass - - -class SystemLockFile(object): - - """ - An abstract base class for platform-specific locking. - """ - - def __init__(self, path): - self.path = path - - try: - # Open lockfile for writing without truncation: - self.fp = open(path, 'r+') - except IOError: - # If the file doesn't exist, IOError is raised; Use a+ instead. - # Note that there may be a race here. Multiple processes - # could fail on the r+ open and open the file a+, but only - # one will get the the lock and write a pid. - self.fp = open(path, 'a+') - - try: - self._lock_file() - except: - self.fp.seek(1) - self.fp.close() - del self.fp - raise - - self.fp.write(' %s\n' % os.getpid()) - self.fp.truncate() - self.fp.flush() - - def release(self): - if not hasattr(self, 'fp'): - return - self._unlock_file() - self.fp.close() - del self.fp - - def remove(self): - """ - Attempt to remove the file - """ - try: - os.remove(self.path) - except: - pass - - def _unlock_file(self): - """Attempt to obtain the lock on self.fp. Raise UnlockError if not - released.""" - - -class WindowsLockFile(SystemLockFile): - - def _lock_file(self): - # Lock just the first byte - try: - msvcrt.locking(self.fp.fileno(), msvcrt.LK_NBLCK, 1) - except IOError: - raise LockError(self.fp.name) - - def _unlock_file(self): - try: - self.fp.seek(0) - msvcrt.locking(self.fp.fileno(), msvcrt.LK_UNLCK, 1) - except IOError: - raise UnlockError(self.fp.name) - -if 'msvcrt' in globals(): # noqa: E305 - LockFile = WindowsLockFile # noqa: F811 - - -class UnixLockFile(SystemLockFile): - - def _lock_file(self): - flags = fcntl.LOCK_EX | fcntl.LOCK_NB - try: - fcntl.flock(self.fp.fileno(), flags) - except IOError: - raise LockError(self.fp.name) - - # no need to implement _unlock_file, it will be unlocked on close() - -if 'fcntl' in globals(): # noqa: E305 - LockFile = UnixLockFile diff --git a/lib/cherrypy/lib/profiler.py b/lib/cherrypy/lib/profiler.py index 94b8798..fccf2eb 100644 --- a/lib/cherrypy/lib/profiler.py +++ b/lib/cherrypy/lib/profiler.py @@ -51,7 +51,11 @@ def new_func_strip_path(func_name): """ filename, line, name = func_name if filename.endswith('__init__.py'): - return os.path.basename(filename[:-12]) + filename[-12:], line, name + return ( + os.path.basename(filename[:-12]) + filename[-12:], + line, + name, + ) return os.path.basename(filename), line, name pstats.func_strip_path = new_func_strip_path diff --git a/lib/cherrypy/lib/reprconf.py b/lib/cherrypy/lib/reprconf.py index 0c7400a..fc75849 100644 --- a/lib/cherrypy/lib/reprconf.py +++ b/lib/cherrypy/lib/reprconf.py @@ -18,35 +18,12 @@ and the handler must be either a callable or a context manager. """ -try: - # Python 3.0+ - from configparser import ConfigParser -except ImportError: - from ConfigParser import ConfigParser - -try: - text_or_bytes -except NameError: - text_or_bytes = str - -try: - # Python 3 - import builtins -except ImportError: - # Python 2 - import __builtin__ as builtins - -import operator as _operator -import sys - +from cherrypy._cpcompat import text_or_bytes +from six.moves import configparser +from six.moves import builtins -def as_dict(config): - """Return a dict from 'config' whether it is a dict, file, or filename.""" - if isinstance(config, text_or_bytes): - config = Parser().dict_from_file(config) - elif hasattr(config, 'read'): - config = Parser().dict_from_file(config) - return config +import operator +import sys class NamespaceSet(dict): @@ -85,9 +62,9 @@ def __call__(self, config): # I chose __enter__ and __exit__ so someday this could be # rewritten using Python 2.5's 'with' statement: - # for ns, handler in self.iteritems(): + # for ns, handler in six.iteritems(self): # with handler as callable: - # for k, v in ns_confs.get(ns, {}).iteritems(): + # for k, v in six.iteritems(ns_confs.get(ns, {})): # callable(k, v) for ns, handler in self.items(): exit = getattr(handler, '__exit__', None) @@ -98,7 +75,7 @@ def __call__(self, config): try: for k, v in ns_confs.get(ns, {}).items(): callable(k, v) - except: + except Exception: # The exceptional case is handled here no_exc = False if exit is None: @@ -149,16 +126,8 @@ def reset(self): dict.update(self, self.defaults) def update(self, config): - """Update self from a dict, file or filename.""" - if isinstance(config, text_or_bytes): - # Filename - config = Parser().dict_from_file(config) - elif hasattr(config, 'read'): - # Open file object - config = Parser().dict_from_file(config) - else: - config = config.copy() - self._apply(config) + """Update self from a dict, file, or filename.""" + self._apply(Parser.load(config)) def _apply(self, config): """Update self from a dict.""" @@ -177,7 +146,7 @@ def __setitem__(self, k, v): self.namespaces({k: v}) -class Parser(ConfigParser): +class Parser(configparser.ConfigParser): """Sub-class of ConfigParser that keeps the case of options and that raises an exception if the file cannot be read. @@ -227,6 +196,17 @@ def dict_from_file(self, file): self.read(file) return self.as_dict() + @classmethod + def load(self, input): + """Resolve 'input' to dict from a dict, file, or filename.""" + is_file = ( + # Filename + isinstance(input, text_or_bytes) + # Open file object + or hasattr(input, 'read') + ) + return Parser().dict_from_file(input) if is_file else input.copy() + # public domain "unrepr" implementation, found on the web and then improved. @@ -471,6 +451,8 @@ def build_Name(self, o): def build_NameConstant(self, o): return o.value + build_Constant = build_NameConstant # Python 3.8 change + def build_UnaryOp(self, o): op, operand = map(self.build, [o.op, o.operand]) return op(operand) @@ -480,13 +462,13 @@ def build_BinOp(self, o): return op(left, right) def build_Add(self, o): - return _operator.add + return operator.add def build_Mult(self, o): - return _operator.mul + return operator.mul def build_USub(self, o): - return _operator.neg + return operator.neg def build_Attribute(self, o): parent = self.build(o.value) diff --git a/lib/cherrypy/lib/sessions.py b/lib/cherrypy/lib/sessions.py index 9a763dc..5b49ee1 100644 --- a/lib/cherrypy/lib/sessions.py +++ b/lib/cherrypy/lib/sessions.py @@ -57,6 +57,17 @@ data for that id. Therefore, if you never save any session data, **you will get a new session id for every request**. +A side effect of CherryPy overwriting unrecognised session ids is that if you +have multiple, separate CherryPy applications running on a single domain (e.g. +on different ports), each app will overwrite the other's session id because by +default they use the same cookie name (``"session_id"``) but do not recognise +each others sessions. It is therefore a good idea to use a different name for +each, for example:: + + [/] + ... + tools.sessions.name = "my_app_session_id" + ================ Sharing Sessions ================ @@ -94,14 +105,24 @@ import os import time import threading +import binascii + +import six +from six.moves import cPickle as pickle +import contextlib2 + +import zc.lockfile import cherrypy -from cherrypy._cpcompat import copyitems, pickle, random20 from cherrypy.lib import httputil -from cherrypy.lib import lockfile from cherrypy.lib import locking from cherrypy.lib import is_iterator + +if six.PY2: + FileNotFoundError = OSError + + missing = object() @@ -114,14 +135,16 @@ class Session(object): id_observers = None "A list of callbacks to which to pass new id's." - def _get_id(self): + @property + def id(self): + """Return the current session id.""" return self._id - def _set_id(self, value): + @id.setter + def id(self, value): self._id = value for o in self.id_observers: o(value) - id = property(_get_id, _set_id, doc='The current session ID.') timeout = 60 'Number of minutes after which to delete session data.' @@ -235,7 +258,7 @@ def clean_up(self): def generate_id(self): """Return a new session id.""" - return random20() + return binascii.hexlify(os.urandom(20)).decode('ascii') def save(self): """Save session data.""" @@ -334,13 +357,6 @@ def __contains__(self, key): self.load() return key in self._data - if hasattr({}, 'has_key'): - def has_key(self, key): - """D.has_key(k) -> True if D has a key k, else False.""" - if not self.loaded: - self.load() - return key in self._data - def get(self, key, default=None): """D.get(k[,d]) -> D[k] if k in D, else d. d defaults to None.""" if not self.loaded: @@ -394,7 +410,7 @@ def clean_up(self): """Clean up expired sessions.""" now = self.now() - for _id, (data, expiration_time) in copyitems(self.cache): + for _id, (data, expiration_time) in list(six.iteritems(self.cache)): if expiration_time <= now: try: del self.cache[_id] @@ -409,7 +425,11 @@ def clean_up(self): # added to remove obsolete lock objects for _id in list(self.locks): - if _id not in self.cache and self.locks[_id].acquire(blocking=False): + locked = ( + _id not in self.cache + and self.locks[_id].acquire(blocking=False) + ) + if locked: lock = self.locks.pop(_id) lock.release() @@ -470,7 +490,9 @@ def __init__(self, id=None, **kwargs): if isinstance(self.lock_timeout, (int, float)): self.lock_timeout = datetime.timedelta(seconds=self.lock_timeout) if not isinstance(self.lock_timeout, (datetime.timedelta, type(None))): - raise ValueError('Lock timeout must be numeric seconds or a timedelta instance.') + raise ValueError( + 'Lock timeout must be numeric seconds or a timedelta instance.' + ) @classmethod def setup(cls, **kwargs): @@ -538,8 +560,8 @@ def acquire_lock(self, path=None): checker = locking.LockChecker(self.id, self.lock_timeout) while not checker.expired(): try: - self.lock = lockfile.LockFile(path) - except lockfile.LockError: + self.lock = zc.lockfile.LockFile(path) + except zc.lockfile.LockError: time.sleep(0.1) else: break @@ -549,8 +571,9 @@ def acquire_lock(self, path=None): def release_lock(self, path=None): """Release the lock on the currently-loaded session data.""" - self.lock.release() - self.lock.remove() + self.lock.close() + with contextlib2.suppress(FileNotFoundError): + os.remove(self.lock._path) self.locked = False def clean_up(self): @@ -558,7 +581,11 @@ def clean_up(self): now = self.now() # Iterate over all session files in self.storage_path for fname in os.listdir(self.storage_path): - if fname.startswith(self.SESSION_PREFIX) and not fname.endswith(self.LOCK_SUFFIX): + have_session = ( + fname.startswith(self.SESSION_PREFIX) + and not fname.endswith(self.LOCK_SUFFIX) + ) + if have_session: # We have a session file: lock and load it and check # if it's expired. If it fails, nevermind. path = os.path.join(self.storage_path, fname) @@ -594,7 +621,7 @@ class MemcachedSession(Session): # Wrap all .get and .set operations in a single lock. mc_lock = threading.RLock() - # This is a seperate set of locks per session id. + # This is a separate set of locks per session id. locks = {} servers = ['127.0.0.1:11211'] @@ -682,7 +709,9 @@ def save(): if is_iterator(response.body): response.collapse_body() cherrypy.session.save() -save.failsafe = True # noqa: E305 + + +save.failsafe = True def close(): @@ -693,7 +722,9 @@ def close(): sess.release_lock() if sess.debug: cherrypy.log('Lock released on close.', 'TOOLS.SESSIONS') -close.failsafe = True # noqa: E305 + + +close.failsafe = True close.priority = 90 @@ -885,3 +916,4 @@ def expire(): one_year = 60 * 60 * 24 * 365 e = time.time() - one_year cherrypy.serving.response.cookie[name]['expires'] = httputil.HTTPDate(e) + cherrypy.serving.response.cookie[name].pop('max-age', None) diff --git a/lib/cherrypy/lib/static.py b/lib/cherrypy/lib/static.py index acedf51..da9d937 100644 --- a/lib/cherrypy/lib/static.py +++ b/lib/cherrypy/lib/static.py @@ -1,23 +1,32 @@ +"""Module with helpers for serving static files.""" + import os +import platform import re import stat import mimetypes -try: - from io import UnsupportedOperation -except ImportError: - UnsupportedOperation = object() +from email.generator import _make_boundary as make_boundary +from io import UnsupportedOperation + +from six.moves import urllib import cherrypy -from cherrypy._cpcompat import ntob, unquote +from cherrypy._cpcompat import ntob from cherrypy.lib import cptools, httputil, file_generator_limited -mimetypes.init() -mimetypes.types_map['.dwg'] = 'image/x-dwg' -mimetypes.types_map['.ico'] = 'image/x-icon' -mimetypes.types_map['.bz2'] = 'application/x-bzip2' -mimetypes.types_map['.gz'] = 'application/x-gzip' +def _setup_mimetypes(): + """Pre-initialize global mimetype map.""" + if not mimetypes.inited: + mimetypes.init() + mimetypes.types_map['.dwg'] = 'image/x-dwg' + mimetypes.types_map['.ico'] = 'image/x-icon' + mimetypes.types_map['.bz2'] = 'application/x-bzip2' + mimetypes.types_map['.gz'] = 'application/x-gzip' + + +_setup_mimetypes() def serve_file(path, content_type=None, disposition=None, name=None, @@ -33,7 +42,6 @@ def serve_file(path, content_type=None, disposition=None, name=None, to the basename of path. If disposition is None, no Content-Disposition header will be written. """ - response = cherrypy.serving.response # If path is relative, users should fix it by making path absolute. @@ -98,7 +106,7 @@ def serve_file(path, content_type=None, disposition=None, name=None, def serve_fileobj(fileobj, content_type=None, disposition=None, name=None, - debug=False, filesize=None): + debug=False): """Set status, headers, and body in order to serve the given file object. The Content-Type header will be set to the content_type arg, if provided. @@ -115,7 +123,6 @@ def serve_fileobj(fileobj, content_type=None, disposition=None, name=None, serve_fileobj(), expecting that the data would be served starting from that position. """ - response = cherrypy.serving.response try: @@ -123,7 +130,7 @@ def serve_fileobj(fileobj, content_type=None, disposition=None, name=None, except AttributeError: if debug: cherrypy.log('os has no fstat attribute', 'TOOLS.STATIC') - content_length = filesize + content_length = None except UnsupportedOperation: content_length = None else: @@ -144,7 +151,7 @@ def serve_fileobj(fileobj, content_type=None, disposition=None, name=None, cd = disposition else: cd = '%s; filename="%s"' % (disposition, name) - response.headers["Content-Disposition"] = cd + response.headers['Content-Disposition'] = cd if debug: cherrypy.log('Content-Disposition: %r' % cd, 'TOOLS.STATIC') @@ -188,12 +195,6 @@ def _serve_fileobj(fileobj, content_type, content_length, debug=False): else: # Return a multipart/byteranges response. response.status = '206 Partial Content' - try: - # Python 3 - from email.generator import _make_boundary as make_boundary - except ImportError: - # Python 2 - from mimetools import choose_boundary as make_boundary boundary = make_boundary() ct = 'multipart/byteranges; boundary=%s' % boundary response.headers['Content-Type'] = ct @@ -203,7 +204,7 @@ def _serve_fileobj(fileobj, content_type, content_length, debug=False): def file_ranges(): # Apache compatibility: - yield ntob('\r\n') + yield b'\r\n' for start, stop in r: if debug: @@ -222,12 +223,12 @@ def file_ranges(): gen = file_generator_limited(fileobj, stop - start) for chunk in gen: yield chunk - yield ntob('\r\n') + yield b'\r\n' # Final boundary yield ntob('--' + boundary + '--', 'ascii') # Apache compatibility: - yield ntob('\r\n') + yield b'\r\n' response.body = file_ranges() return response.body else: @@ -318,7 +319,15 @@ def staticdir(section, dir, root='', match='', content_types=None, index='', section = '/' section = section.rstrip(r'\/') branch = request.path_info[len(section) + 1:] - branch = unquote(branch.lstrip(r'\/')) + branch = urllib.parse.unquote(branch.lstrip(r'\/')) + + # Requesting a file in sub-dir of the staticdir results + # in mixing of delimiter styles, e.g. C:\static\js/script.js. + # Windows accepts this form except not when the path is + # supplied in extended-path notation, e.g. \\?\C:\static\js/script.js. + # http://bit.ly/1vdioCX + if platform.system() == 'Windows': + branch = branch.replace('/', '\\') # If branch is "", filename will end in a slash filename = os.path.join(dir, branch) diff --git a/lib/cherrypy/lib/xmlrpcutil.py b/lib/cherrypy/lib/xmlrpcutil.py index 9fc9564..ddaac86 100644 --- a/lib/cherrypy/lib/xmlrpcutil.py +++ b/lib/cherrypy/lib/xmlrpcutil.py @@ -1,21 +1,19 @@ +"""XML-RPC tool helpers.""" import sys +from six.moves.xmlrpc_client import ( + loads as xmlrpc_loads, dumps as xmlrpc_dumps, + Fault as XMLRPCFault +) + import cherrypy from cherrypy._cpcompat import ntob -def get_xmlrpclib(): - try: - import xmlrpc.client as x - except ImportError: - import xmlrpclib as x - return x - - def process_body(): """Return (params, method) from request body.""" try: - return get_xmlrpclib().loads(cherrypy.request.body.read()) + return xmlrpc_loads(cherrypy.request.body.read()) except Exception: return ('ERROR PARAMS', ), 'ERRORMETHOD' @@ -31,9 +29,10 @@ def patched_path(path): def _set_response(body): + """Set up HTTP status, headers and body within CherryPy.""" # The XML-RPC spec (http://www.xmlrpc.com/spec) says: # "Unless there's a lower-level error, always return 200 OK." - # Since Python's xmlrpclib interprets a non-200 response + # Since Python's xmlrpc_client interprets a non-200 response # as a "Protocol Error", we'll just return 200 every time. response = cherrypy.response response.status = '200 OK' @@ -43,15 +42,20 @@ def _set_response(body): def respond(body, encoding='utf-8', allow_none=0): - xmlrpclib = get_xmlrpclib() - if not isinstance(body, xmlrpclib.Fault): + """Construct HTTP response body.""" + if not isinstance(body, XMLRPCFault): body = (body,) - _set_response(xmlrpclib.dumps(body, methodresponse=1, - encoding=encoding, - allow_none=allow_none)) + + _set_response( + xmlrpc_dumps( + body, methodresponse=1, + encoding=encoding, + allow_none=allow_none + ) + ) def on_error(*args, **kwargs): + """Construct HTTP response body for an error response.""" body = str(sys.exc_info()[1]) - xmlrpclib = get_xmlrpclib() - _set_response(xmlrpclib.dumps(xmlrpclib.Fault(1, body))) + _set_response(xmlrpc_dumps(XMLRPCFault(1, body))) diff --git a/lib/cherrypy/process/__init__.py b/lib/cherrypy/process/__init__.py index 97f91ce..f242d22 100644 --- a/lib/cherrypy/process/__init__.py +++ b/lib/cherrypy/process/__init__.py @@ -10,5 +10,8 @@ for each class. """ -from cherrypy.process.wspbus import bus # noqa -from cherrypy.process import plugins, servers # noqa +from .wspbus import bus +from . import plugins, servers + + +__all__ = ('bus', 'plugins', 'servers') diff --git a/lib/cherrypy/process/plugins.py b/lib/cherrypy/process/plugins.py index a94c3cd..8c246c8 100644 --- a/lib/cherrypy/process/plugins.py +++ b/lib/cherrypy/process/plugins.py @@ -7,7 +7,9 @@ import time import threading -from cherrypy._cpcompat import text_or_bytes, get_thread_ident +from six.moves import _thread + +from cherrypy._cpcompat import text_or_bytes from cherrypy._cpcompat import ntob, Timer # _module__file__base is used by Autoreload to make @@ -220,7 +222,8 @@ class DropPrivileges(SimplePlugin): """Drop privileges. uid/gid arguments not available on Windows. - Special thanks to `Gavin Baker `_ + Special thanks to `Gavin Baker + `_ """ def __init__(self, bus, umask=None, uid=None, gid=None): @@ -230,10 +233,13 @@ def __init__(self, bus, umask=None, uid=None, gid=None): self.gid = gid self.umask = umask - def _get_uid(self): + @property + def uid(self): + """The uid under which to run. Availability: Unix.""" return self._uid - def _set_uid(self, val): + @uid.setter + def uid(self, val): if val is not None: if pwd is None: self.bus.log('pwd module not available; ignoring uid.', @@ -242,13 +248,14 @@ def _set_uid(self, val): elif isinstance(val, text_or_bytes): val = pwd.getpwnam(val)[2] self._uid = val - uid = property(_get_uid, _set_uid, - doc='The uid under which to run. Availability: Unix.') - def _get_gid(self): + @property + def gid(self): + """The gid under which to run. Availability: Unix.""" return self._gid - def _set_gid(self, val): + @gid.setter + def gid(self, val): if val is not None: if grp is None: self.bus.log('grp module not available; ignoring gid.', @@ -257,13 +264,18 @@ def _set_gid(self, val): elif isinstance(val, text_or_bytes): val = grp.getgrnam(val)[2] self._gid = val - gid = property(_get_gid, _set_gid, - doc='The gid under which to run. Availability: Unix.') - def _get_umask(self): + @property + def umask(self): + """The default permission mode for newly created files and directories. + + Usually expressed in octal format, for example, ``0644``. + Availability: Unix, Windows. + """ return self._umask - def _set_umask(self, val): + @umask.setter + def umask(self, val): if val is not None: try: os.umask @@ -272,15 +284,6 @@ def _set_umask(self, val): level=30) val = None self._umask = val - umask = property( - _get_umask, - _set_umask, - doc="""The default permission mode for newly created files and - directories. - - Usually expressed in octal format, for example, ``0644``. - Availability: Unix, Windows. - """) def start(self): # uid/gid @@ -344,7 +347,7 @@ class Daemonizer(SimplePlugin): process still return proper exit codes. Therefore, if you use this plugin to daemonize, don't use the return code as an accurate indicator of whether the process fully started. In fact, that return code only - indicates if the process succesfully finished the first fork. + indicates if the process successfully finished the first fork. """ def __init__(self, bus, stdin='/dev/null', stdout='/dev/null', @@ -369,6 +372,15 @@ def start(self): 'Daemonizing now may cause strange failures.' % threading.enumerate(), level=30) + self.daemonize(self.stdin, self.stdout, self.stderr, self.bus.log) + + self.finalized = True + start.priority = 65 + + @staticmethod + def daemonize( + stdin='/dev/null', stdout='/dev/null', stderr='/dev/null', + logger=lambda msg: None): # See http://www.erlenstar.demon.co.uk/unix/faq_2.html#SEC16 # (or http://www.faqs.org/faqs/unix-faq/programmer/faq/ section 1.7) # and http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/66012 @@ -377,40 +389,29 @@ def start(self): sys.stdout.flush() sys.stderr.flush() - # Do first fork. - try: - pid = os.fork() - if pid == 0: - # This is the child process. Continue. - pass - else: - # This is the first parent. Exit, now that we've forked. - self.bus.log('Forking once.') - os._exit(0) - except OSError: - # Python raises OSError rather than returning negative numbers. - exc = sys.exc_info()[1] - sys.exit('%s: fork #1 failed: (%d) %s\n' - % (sys.argv[0], exc.errno, exc.strerror)) - - os.setsid() - - # Do second fork - try: - pid = os.fork() - if pid > 0: - self.bus.log('Forking twice.') - os._exit(0) # Exit second parent - except OSError: - exc = sys.exc_info()[1] - sys.exit('%s: fork #2 failed: (%d) %s\n' - % (sys.argv[0], exc.errno, exc.strerror)) + error_tmpl = ( + '{sys.argv[0]}: fork #{n} failed: ({exc.errno}) {exc.strerror}\n' + ) + + for fork in range(2): + msg = ['Forking once.', 'Forking twice.'][fork] + try: + pid = os.fork() + if pid > 0: + # This is the parent; exit. + logger(msg) + os._exit(0) + except OSError as exc: + # Python raises OSError rather than returning negative numbers. + sys.exit(error_tmpl.format(sys=sys, exc=exc, n=fork + 1)) + if fork == 0: + os.setsid() os.umask(0) - si = open(self.stdin, 'r') - so = open(self.stdout, 'a+') - se = open(self.stderr, 'a+') + si = open(stdin, 'r') + so = open(stdout, 'a+') + se = open(stderr, 'a+') # os.dup2(fd, fd2) will close fd2 if necessary, # so we don't explicitly close stdin/out/err. @@ -419,9 +420,7 @@ def start(self): os.dup2(so.fileno(), sys.stdout.fileno()) os.dup2(se.fileno(), sys.stderr.fileno()) - self.bus.log('Daemonized to PID: %s' % os.getpid()) - self.finalized = True - start.priority = 65 + logger('Daemonized to PID: %s' % os.getpid()) class PIDFile(SimplePlugin): @@ -449,7 +448,7 @@ def exit(self): self.bus.log('PID file removed: %r.' % self.pidfile) except (KeyboardInterrupt, SystemExit): raise - except: + except Exception: pass @@ -628,23 +627,40 @@ def start(self): def sysfiles(self): """Return a Set of sys.modules filenames to monitor.""" - files = set() - for k, m in list(sys.modules.items()): - if re.match(self.match, k): - if ( - hasattr(m, '__loader__') and - hasattr(m.__loader__, 'archive') - ): - f = m.__loader__.archive - else: - f = getattr(m, '__file__', None) - if f is not None and not os.path.isabs(f): - # ensure absolute paths so a os.chdir() in the app - # doesn't break me - f = os.path.normpath( - os.path.join(_module__file__base, f)) - files.add(f) - return files + search_mod_names = filter(re.compile(self.match).match, sys.modules) + mods = map(sys.modules.get, search_mod_names) + return set(filter(None, map(self._file_for_module, mods))) + + @classmethod + def _file_for_module(cls, module): + """Return the relevant file for the module.""" + return ( + cls._archive_for_zip_module(module) + or cls._file_for_file_module(module) + ) + + @staticmethod + def _archive_for_zip_module(module): + """Return the archive filename for the module if relevant.""" + try: + return module.__loader__.archive + except AttributeError: + pass + + @classmethod + def _file_for_file_module(cls, module): + """Return the file for the module.""" + try: + return module.__file__ and cls._make_absolute(module.__file__) + except AttributeError: + pass + + @staticmethod + def _make_absolute(filename): + """Ensure filename is absolute to avoid effect of os.chdir.""" + return filename if os.path.isabs(filename) else ( + os.path.normpath(os.path.join(_module__file__base, filename)) + ) def run(self): """Reload the process if registered files have been modified.""" @@ -713,7 +729,7 @@ def acquire_thread(self): If the current thread has already been seen, any 'start_thread' listeners will not be run again. """ - thread_ident = get_thread_ident() + thread_ident = _thread.get_ident() if thread_ident not in self.threads: # We can't just use get_ident as the thread ID # because some platforms reuse thread ID's. @@ -723,7 +739,7 @@ def acquire_thread(self): def release_thread(self): """Release the current thread and run 'stop_thread' listeners.""" - thread_ident = get_thread_ident() + thread_ident = _thread.get_ident() i = self.threads.pop(thread_ident, None) if i is not None: self.bus.publish('stop_thread', i) diff --git a/lib/cherrypy/process/servers.py b/lib/cherrypy/process/servers.py index 1013f24..dcb34de 100644 --- a/lib/cherrypy/process/servers.py +++ b/lib/cherrypy/process/servers.py @@ -1,4 +1,4 @@ -""" +r""" Starting in CherryPy 3.1, cherrypy.server is implemented as an :ref:`Engine Plugin`. It's an instance of :class:`cherrypy._cpserver.Server`, which is a subclass of @@ -232,7 +232,7 @@ def _start_http_thread(self): self.interrupt = sys.exc_info()[1] self.bus.exit() raise - except: + except Exception: self.interrupt = sys.exc_info()[1] self.bus.log('Error in HTTP server: shutting down', traceback=True, level=40) @@ -246,13 +246,18 @@ def wait(self): raise self.interrupt time.sleep(.1) - # Wait for port to be occupied - if not os.environ.get('LISTEN_PID', None): - # Wait for port to be occupied if not running via socket-activation - # (for socket-activation the port will be managed by systemd ) - if isinstance(self.bind_addr, tuple): - with _safe_wait(*self.bound_addr): - portend.occupied(*self.bound_addr, timeout=Timeouts.occupied) + # bypass check when LISTEN_PID is set + if os.environ.get('LISTEN_PID', None): + return + + # bypass check when running via socket-activation + # (for socket-activation the port will be managed by systemd) + if not isinstance(self.bind_addr, tuple): + return + + # wait for port to be occupied + with _safe_wait(*self.bound_addr): + portend.occupied(*self.bound_addr, timeout=Timeouts.occupied) @property def bound_addr(self): diff --git a/lib/cherrypy/process/win32.py b/lib/cherrypy/process/win32.py index 74b5067..096b027 100644 --- a/lib/cherrypy/process/win32.py +++ b/lib/cherrypy/process/win32.py @@ -90,14 +90,15 @@ def _get_state_event(self, state): self.events[state] = event return event - def _get_state(self): + @property + def state(self): return self._state - def _set_state(self, value): + @state.setter + def state(self, value): self._state = value event = self._get_state_event(value) win32event.PulseEvent(event) - state = property(_get_state, _set_state) def wait(self, state, interval=0.1, channel=None): """Wait for the given state(s), KeyboardInterrupt or SystemExit. @@ -137,7 +138,8 @@ def key_for(self, obj): return key raise ValueError('The given object could not be found: %r' % obj) -control_codes = _ControlCodes({'graceful': 138}) # noqa: E305 + +control_codes = _ControlCodes({'graceful': 138}) def signal_child(service, command): diff --git a/lib/cherrypy/process/wspbus.py b/lib/cherrypy/process/wspbus.py index 634859c..d91dba4 100644 --- a/lib/cherrypy/process/wspbus.py +++ b/lib/cherrypy/process/wspbus.py @@ -1,4 +1,4 @@ -"""An implementation of the Web Site Process Bus. +r"""An implementation of the Web Site Process Bus. This module is completely standalone, depending only on the stdlib. @@ -78,11 +78,11 @@ import time import traceback as _traceback import warnings +import subprocess +import functools import six -from cherrypy._cpcompat import _args_from_interpreter_flags - # Here I save the value of os.getcwd(), which, if I am imported early enough, # will be the directory from which the startup script was run. This is needed @@ -94,13 +94,13 @@ class ChannelFailures(Exception): + """Exception raised during errors on Bus.publish().""" - """Exception raised when errors occur in a listener during Bus.publish(). - """ delimiter = '\n' def __init__(self, *args, **kwargs): - super(Exception, self).__init__(*args, **kwargs) + """Initialize ChannelFailures errors wrapper.""" + super(ChannelFailures, self).__init__(*args, **kwargs) self._exceptions = list() def handle_exception(self): @@ -112,12 +112,14 @@ def get_instances(self): return self._exceptions[:] def __str__(self): + """Render the list of errors, which happened in channel.""" exception_strings = map(repr, self.get_instances()) return self.delimiter.join(exception_strings) __repr__ = __str__ def __bool__(self): + """Determine whether any error happened in channel.""" return bool(self._exceptions) __nonzero__ = __bool__ @@ -136,7 +138,9 @@ def __setattr__(self, key, value): if isinstance(value, self.State): value.name = key object.__setattr__(self, key, value) -states = _StateEnum() # noqa: E305 + + +states = _StateEnum() states.STOPPED = states.State() states.STARTING = states.State() states.STARTED = states.State() @@ -156,7 +160,6 @@ def __setattr__(self, key, value): class Bus(object): - """Process state-machine and messenger for HTTP site deployment. All listeners for a given channel are guaranteed to be called even @@ -172,6 +175,7 @@ class Bus(object): max_cloexec_files = max_files def __init__(self): + """Initialize pub/sub bus.""" self.execv = False self.state = states.STOPPED channels = 'start', 'stop', 'exit', 'graceful', 'log', 'main' @@ -181,8 +185,19 @@ def __init__(self): ) self._priorities = {} - def subscribe(self, channel, callback, priority=None): - """Add the given callback at the given channel (if not present).""" + def subscribe(self, channel, callback=None, priority=None): + """Add the given callback at the given channel (if not present). + + If callback is None, return a partial suitable for decorating + the callback. + """ + if callback is None: + return functools.partial( + self.subscribe, + channel, + priority=priority, + ) + ch_listeners = self.listeners.setdefault(channel, set()) ch_listeners.add(callback) @@ -221,7 +236,7 @@ def publish(self, channel, *args, **kwargs): if exc and e.code == 0: e.code = 1 raise - except: + except Exception: exc.handle_exception() if channel == 'log': # Assume any further messages to 'log' will fail. @@ -234,7 +249,7 @@ def publish(self, channel, *args, **kwargs): return output def _clean_exit(self): - """An atexit handler which asserts the Bus is not running.""" + """Assert that the Bus is not running in atexit handler callback.""" if self.state != states.EXITING: warnings.warn( 'The main thread is exiting, but the Bus is in the %r state; ' @@ -255,13 +270,13 @@ def start(self): self.log('Bus STARTED') except (KeyboardInterrupt, SystemExit): raise - except: + except Exception: self.log('Shutting down due to error in start listener:', level=40, traceback=True) e_info = sys.exc_info()[1] try: self.exit() - except: + except Exception: # Any stop/exit errors will be logged inside publish(). pass # Re-raise the original error @@ -280,7 +295,7 @@ def exit(self): # This isn't strictly necessary, but it's better than seeing # "Waiting for child threads to terminate..." and then nothing. self.log('Bus EXITED') - except: + except Exception: # This method is often called asynchronously (whether thread, # signal handler, console handler, or atexit handler), so we # can't just let exceptions propagate out unhandled. @@ -343,7 +358,8 @@ def block(self, interval=0.1): if ( t != threading.currentThread() and not isinstance(t, threading._MainThread) and - # Note that any dummy (external) threads are always daemonic. + # Note that any dummy (external) threads are + # always daemonic. not t.daemon ): self.log('Waiting for thread %s.' % t.getName()) @@ -359,23 +375,9 @@ def wait(self, state, interval=0.1, channel=None): else: states = [state] - def _wait(): - while self.state not in states: - time.sleep(interval) - self.publish(channel) - - # From http://psyco.sourceforge.net/psycoguide/bugs.html: - # "The compiled machine code does not include the regular polling - # done by Python, meaning that a KeyboardInterrupt will not be - # detected before execution comes back to the regular Python - # interpreter. Your program cannot be interrupted if caught - # into an infinite Psyco-compiled loop." - try: - sys.modules['psyco'].cannotcompile(_wait) - except (KeyError, AttributeError): - pass - - _wait() + while self.state not in states: + time.sleep(interval) + self.publish(channel) def _do_execv(self): """Re-execute the current process. @@ -407,7 +409,7 @@ def _do_execv(self): @staticmethod def _get_interpreter_argv(): - """Retrieve current Python interpreter's arguments + """Retrieve current Python interpreter's arguments. Returns empty tuple in case of frozen mode, uses built-in arguments reproduction function otherwise. @@ -421,11 +423,11 @@ def _get_interpreter_argv(): """ return ([] if getattr(sys, 'frozen', False) - else _args_from_interpreter_flags()) + else subprocess._args_from_interpreter_flags()) @staticmethod def _get_true_argv(): - """Retrieves all real arguments of the python interpreter + """Retrieve all real arguments of the python interpreter. ...even those not listed in ``sys.argv`` @@ -433,14 +435,16 @@ def _get_true_argv(): :seealso: http://stackoverflow.com/a/6683222/595220 :seealso: http://stackoverflow.com/a/28414807/595220 """ - try: char_p = ctypes.c_char_p if six.PY2 else ctypes.c_wchar_p argv = ctypes.POINTER(char_p)() argc = ctypes.c_int() - ctypes.pythonapi.Py_GetArgcArgv(ctypes.byref(argc), ctypes.byref(argv)) + ctypes.pythonapi.Py_GetArgcArgv( + ctypes.byref(argc), + ctypes.byref(argv), + ) _argv = argv[:argc.value] @@ -465,7 +469,7 @@ def _get_true_argv(): try: c_ind = _argv.index('-c') - if m_ind < argv_len - 1 and _argv[c_ind + 1] == '-c': + if c_ind < argv_len - 1 and _argv[c_ind + 1] == '-c': is_command = True except (IndexError, ValueError): c_ind = None @@ -500,7 +504,7 @@ def _get_true_argv(): :seealso: https://github.com/cherrypy/cherrypy/issues/1506 :seealso: https://github.com/cherrypy/cherrypy/issues/1512 - :ref: https://chromium.googlesource.com/infra/infra/+/69eb0279c12bcede5937ce9298020dd4581e38dd%5E!/ + :ref: http://bit.ly/2gK6bXK """ raise NotImplementedError else: @@ -508,7 +512,8 @@ def _get_true_argv(): @staticmethod def _extend_pythonpath(env): - """ + """Prepend current working dir to PATH environment variable if needed. + If sys.path[0] is an empty string, the interpreter was likely invoked with -m and the effective path is about to change on re-exec. Add the current directory to $PYTHONPATH to ensure @@ -581,4 +586,5 @@ def log(self, msg='', level=20, traceback=False): msg += '\n' + ''.join(_traceback.format_exception(*sys.exc_info())) self.publish('log', msg, level) -bus = Bus() # noqa: E305 + +bus = Bus() diff --git a/lib/cherrypy/scaffold/__init__.py b/lib/cherrypy/scaffold/__init__.py index 52b40b3..bcddba2 100644 --- a/lib/cherrypy/scaffold/__init__.py +++ b/lib/cherrypy/scaffold/__init__.py @@ -21,9 +21,11 @@ @cherrypy.config(**{'tools.log_tracebacks.on': True}) class Root: + """Declaration of the CherryPy app URI structure.""" @cherrypy.expose def index(self): + """Render HTML-template at the root path of the web-app.""" return """ Try some other path, or a default path.
@@ -34,10 +36,12 @@ def index(self): @cherrypy.expose def default(self, *args, **kwargs): + """Render catch-all args and kwargs.""" return 'args: %s kwargs: %s' % (args, kwargs) @cherrypy.expose def other(self, a=2, b='bananas', c=None): + """Render number of fruits based on third argument.""" cherrypy.response.headers['Content-Type'] = 'text/plain' if c is None: return 'Have %d %s.' % (int(a), b) diff --git a/lib/cherrypy/scaffold/apache-fcgi.conf b/lib/cherrypy/scaffold/apache-fcgi.conf index 922398e..6e4f144 100644 --- a/lib/cherrypy/scaffold/apache-fcgi.conf +++ b/lib/cherrypy/scaffold/apache-fcgi.conf @@ -19,4 +19,4 @@ RewriteRule ^(.*)$ /fastcgi.pyc [L] # If filename does not begin with a slash (/) then it is assumed to be relative to the ServerRoot. # The filename does not have to exist in the local filesystem. URIs that Apache resolves to this # filename will be handled by this external FastCGI application. -FastCgiExternalServer "C:/fastcgi.pyc" -host 127.0.0.1:8088 \ No newline at end of file +FastCgiExternalServer "C:/fastcgi.pyc" -host 127.0.0.1:8088 diff --git a/lib/cherrypy/scaffold/example.conf b/lib/cherrypy/scaffold/example.conf index 93a6e53..63250fe 100644 --- a/lib/cherrypy/scaffold/example.conf +++ b/lib/cherrypy/scaffold/example.conf @@ -1,3 +1,3 @@ [/] log.error_file: "error.log" -log.access_file: "access.log" \ No newline at end of file +log.access_file: "access.log" diff --git a/lib/cherrypy/scaffold/static/made_with_cherrypy_small.png b/lib/cherrypy/scaffold/static/made_with_cherrypy_small.png index c3aafeed952190f5da9982bb359aa75b107ff079..724f9d72d9ca5aede0b788fc1216286aa967e212 100644 GIT binary patch literal 6347 zcmV;+7&PaJP)Z@}2(x zGrQ>KkN5dJKIfkEx#!;Ze&)u^bMcE`{@Q0`WVpG$hJr009vb{C38$aGPQv&7ed&=w zPZh;fF7Hylv=d#JIX(LSCJBj^MMjU#gK`MeukUYTG)<5GzevInYh{Ts{a_ZRM+Z$0 z{l7y(*!gJn`Qb?#hN?#q{WK*G<|VzE9`}ENgu}ywI7-UP7+Lp_%Q>Z9on1{CasJQU z@5QGZ|I>$LT0#&HOHT5OxbGf(F}v8aufH9f&8_&vT*K&0jrbfrApB#8V_L!yT75xK zjI?vmnWS9nuAU~uNvcVU$oENG>tKy(TiXo(tNMEsZ05;@Qk^S&)*s2BUo$cH~!!jbnQ z@9T&AFnw(q(J4wW`Q9eUwbLVqyE73Cb!C)QIe}o~(O4G>gejn}&aJoB`L97)kX-#c zqfgG`)Kl#^-cL@}Cd8^wQpKgO{#jq6Uw?Dx@Ic+4CgE`J$N*)yr=Yd*t$)N#m!LDn zrB~_3W7sVFw zhe#jxx+2P9`rER^;Rp+oXzXd39cg}L!VE=p^o96 zlD5Whudu7`!512<1BW`2S&&!qG&Dt#4KO%&piKciwT|7@mK>36c1icAWe)oU0wi}C z@qKG7_`U6p)nHp~T>;Bu8_twbW#n>fL*T95zGB-ozh3xO`8d?-qCNE!-(j}?#~97i z!4LN<&6`vCS{D3OVR9zxen>ID9IbH*$ka6Tv4)C#?(6X^@1i~hyDaFD!ps>JAu?ct zNAsJ7~mM2XL%>d6l_|$>f_ln8;f+oZpRs`F5)7@C^D0CoyE;J{RBo?zys~?7c{< zTX-r%rMoERPldpcLC9X0L|)4tLTn_V);EiBk@8^sOYHQ%x-IbbZm=oaS;=7ugJoi3aD~Pi_PP*fm)ikLS#4D%P9qbvnpqv`W{{B;m-=U{_&Q zo<5vEi%ZN0+v5k@6IhT{X!0sijy=#=7hp~S1N6Xdsu@S1!=+*JA$BX;;K`S>TQLSm z5hej4RsoE!M51(=xKiYGEp`tL54lh zSZ5$l1+CGr;B-+;*yE#w-yk-lQg2j3Gzuxgf(8=S=BVlOEX%aJR+p##&RxP%gcm{q zH+Ohn5gAKjzh_Npz`dqoZ5Vn=iV~wjuLX^2!y2%(g^iR$t@tX(Su9rP6cjCmc?&#m znQ!85snUmHp34n~eSBcF;|(DXy;KLOm1uCr>yj{Hw_;F;glyi3_)fj6o%T5-9Bd0u zkns+`-a`-j&>A%!M1p%@yfjDPaIQd}8u}W%I>|k&_p&n-(49x&1@FTxyZEYd{$jt2w^U)<~<89o)_4t}^7j0xKU zzcK%JHrrR#WT`+g)$F|Wwv~Q2M3HE=!&Tc|tH5X{7(mmQ2x-J0s{;#w5MtBIv_V~@ z-N``6Q*Uj?eFg~|=w;4^(fU+oH9`OLGhV|vt&^-j8JbGfi3u`@ z-HOr1ML+5w>Ve%s(!<9{F{-q{rlcxl+Fq;(zS~lzg^hfKklP8m#XdK)9WGaAKBlHA zG?l20B`Nlm%&@so;CX{yrZK*`n(QZIMN*<{J6;#D8;exQNpfRJ@;z_lxm|0;>eLsg z&@vPee;ZB*7cB+?N{$a4+BjieI zmO6~!WQfaD^NzXqE20r&0yJ0qjIRs7C-pW4chm;;HH8c`|LVjSyWEtxcJjnAzh|od zKR>j$&Z(1(g4g?JqMfAIP8Jm1Dgr)Is1o4=WDZH~xPs7*XXlXpE3**y6 z^c`q=HP9T)g7|Fp7twdv1ElS2@E@zv*YvV9+b%4~?5U}Wyo}_%+gIhJ#jQ*q_R~kB zby&~uzWe%%FTQx;syx2A{wDvPYP&Aj8i~691d#HT)9h7KEFULZK1no@PcV{8(343r zl1;alD+rfuMr%$vGd@@8_e$9KCNg!^TF`7f(gil>>^x9-=2F?sdzDi1=*N#KwmKa} z9s~5CF{I_`Jd2QI58s&k{Y}5}YpUE{xN3`Czx3meKhB&v6Qn|(6UTN8G=~78ndrGz zfEPS3z0{#NDVqHI8@>A*y#8KL%WN}*wx_HRjNsvW$-O$@r&S@`tK)e2)A%=K@7$j% zbCH2}o@!?6mL@D`342MD^mRfL>X*85ZIL|US1w9t}i8SocH$7Pl>UQ4YzQxRPcX}LO89|7a~G}A0dDR?ZzR|wDN7BQ z%|v%ogU4Tg{WU+|%CbBkn3v#Wf*)dZRKM6Ex*jA9_8G194@VuwIeqtDkgz7-f!W{J z;GcHuGz??p=1o1d-c!g-t8i#zkT^&PQxH~w$s~kTG;iNgIOE`T4ZEu_NR@6h28!UM zy1@8)1Ns`heWUNWhTmX=f+7?!gMBpDt`%Ika6wfeRGbu@)z1&^69Ne%UunUwy{ShHzk!*NqK3*nGlNIr0h$bw`xv1K@0uNN68TE6 zv)U71Xd51A;9{rh?WUHOXbiNR1ml=cec({b?ccsMaJ5tOa#c%>L{;Wm<)@gIW?J;t zdoefHfO9EGH7iUtgMNN2I;R~t5XsF)%d%$aSXHPd91lZz*0d2k2?e&rsb=LF=H0c= znK{sbk~H(uRMQR$>wD44K|(SI32QvPL8{3B2b|nJhfA(Y;xu#_i76u=21f=58A&NP z4ZYIql4X)t`x>4j$jsAZFCh{0v=QrgQ~cbF88ft1Z^1$jwI)Mnwa2kT!XTl&`8{Kl z;=aARmn>N_XU?2?^X74L|8)8M?k2p`7`eCJGd^5j=Fu%~ZtmH$XT#SPq35fQ{tlh?!wiKBPVJyO_@0*kp{)L@5dcJ-c;(sQk_y}!+93sbPd=@ zn7V3R_O?B6mZ_9dAR3v&S_%N8Qd$9=IXm|CRJ--nKLgOo4$fiM9A96V5KduNTWQXz zkvjbR{Ih1xL>D+i6*lOsb~}As6eQfTc_RxA+Rd&_~5Qv5@&bq*lceppAoG!bLI>PJ+r^d+&z8G zn{(#O2906%L$L|(m~vo$G)S0i#!Q2z(>&BOHf{?(eROoaEqt25w=S7iAm31unU>|E zoW6F;%LDuR8vR+ufm~|ISwi#^H8v8$8k@0R7}nC_OJWrdWYCY*0ALhfzLB^8D1Q(d=gb*Lvop)~^4IMqO9{P`D zhEx~M9R=qrA1%bgAUY%-J_r&LlT45f0J#oro%m8a_wKzAo;Agx7NgEu*Ni)(TVw2A z(e4@-U?Ojt^!RQ9>{&KS%macLjb&s^94mV=%6|v5_&p3WC>|J z`5C5{$d;9Tb?#8iiPsSp+6*(SmFmSzqn4U4Uc3-8WJGI1v5joMQ&bR&CVF>z>s*+b zT{Op^Pi%#doYckMahG(wo$o?u&YW4KB2&bZ9jCov!}_0o{I;XU8}Tp*wjd!KJ~J)l zGK2IpeNoAdI;rYU-krYSzk)x>PP4Dx6Jg?`G?G>d#PRTxhlKIY@{{8G}w7RZJJ$0&9A<=w1LAF!8~ex@KJu&4KyeM|$=X+u#f1kzb$= zQ9bobAmQqj-}Z943)4Zh`%S|8EFm{*N!Y8l39W|VhN73V3~ZA^n^%ViXutB-^!HTr zaaVorqT*)vL_~NqWOy6!7-*eT`(uZ;Krz5WdVF&?uIz!3t(oNb5FKy5iuY@Xn(CfM zJT5lUixw@?(*UwF;$aYM5|12&fl3zKlDu(itiYyVUVhKD8$zTcNx5!FKM_DTl1}Xf zX2#nqSDB?8IULW!AFD5wK=Voc0+>Z2 z{6w|fIJGTXHc4DR)>Gp+MyRbYPC0lGmPd4i78KcojxkTzCX~R#k9j4J&DG1ga54DI zQP?IYXvnpK5i_B-!Z7vV0gw>$Mh%LbV416Z z1qLu(>Qs)uQqtC4`F>iHG0{0X?JX2`60$kba75|7Q*hH{G9+inj!Br4t2pMjH@$>> zD>!#@b6vT?WYt^orU&>HKP`1)kkL7MDi8E`ah9+fjfEZV59yb&aZ_u)E}LsC zT@$hlQ6^dM_4aF9`@uB@&fM#bZ z#**1Y=Q3=5yF}N(RSYP0lFc66+zHJ2^SBJu&l2;sJISU9lYuiLywmC+nIiW7J)1@) zbiK!tu$^Rb`TQ;jSsGlP>~)|z|LSiRF5o(_Pt@Mx0o+YujzXW)_wP>t34LUt$Pv(` z3U<$(oo@o+?eg&3R)U#}D( zWtvS$7`^+HqlT}D{Jl5WhTp%3AG?32eq=YeUSRY)(Mm8F?}Y1h0Ui%`?^p*XNkmIH zslvbT$lN~#k~d>f+k{theL2?xb&aL2^qm7-TyuA96M##bW6oxrv99X*@4sD!tkOuT z88f~R+6?QhH4qvqb+>F>1qnUvZ?hz9!5bXhyBwk|h`*~i{D5%c=vL--5s>mV zUTxgsxsor-;nD9L7IdV1n!u)@z1zEKwi6QQX#Lc;Pu>soa*&WkMWUHy+L{e)^h@8o zx%wq?ljk#_i)vX@VVxSK7viHF?5z|NqK(P7>7qFx(H5c&gX9eN(phT3VHNzXWM|gf zP!!`fvTa@x>b0K=WP1~+UX*U$P938`LOZC~L9rR%lh$!s_^LRpML=JvhuYT^^#>1x60XGhD>h`Cd_kl$b50N)g=Ob*MM0>iMD)XP^#>!QsDG=YO zP>cSvfJ-rN{1t7?<p6a`swDCg z6UN?_NE6r`#l!dF=y7ym6yhw)^Q$Nfg`Kazuk$niq4xAMJUr}RZ>OWJ9UB`n^f~_P zxQbUVxTOF9 N002ovPDHLkV1nY?RmA`R literal 7455 zcmV+)9pK`LP)7N6iYPr5@U(R*fj>!*b#f9NC#;mT|kg1D2jq&Lj>tf5D-D6cTjpq zM2*V(&+dUE$T@H@a&NdlKF>Vo`?k!^&c5I5?CvbS4*K`nzw94S`&vnU?rYUm<*)VZ zKlrsb-hAs{CSiv-Eoy)P>)-P4@xvMfTz0~N?aQ%e@i^>WG#2rZLH`!v(s_J^DycTIW{%{Z8$3ex2 zJwN{Ye*4vGhvaBGVAYbvdG-)hRoQS0(8QrrbKv5!75RlXDZQFg?b9mUNpTedsvcPE z_x}BCVY&M9FZ=uK??QBtq&g@;?Xw5}_|tgBz^VnF-bd}Q_mf%$u73`!+D8PcbeaJ}k)Q2^zs@ef5tl{CNA!}w`+bR9j(v1($D9<&6fHUBMQR|~V-NCgX(7O9Ij&o-L`i}^O5*hpVzB~2X3{8((H8QI zbKQnO0*w+O1=c7^$*0W=Wfpxy5a z>?c5&jOoMQ5B|^-Y)UIj(nWEcHmqiRk5jA0JZ?7ur09lAJ&>Q>> z+?D3QdBbGvdhcxl)@Gg`=eh+tgT5v}F2)tb*}QJ1zd|EfZ#t}lTK7-L3LW6-F-{w? z?TyzF>mny;gVL2g|B%49hx0f%suzJulgq#D_1B?FQ=?k9t|+23FRl5|=>0j&Cicfk zl2@zm7rZa`NzRt^Q=F_v{&Hm71MAttR6L^9!Ox5ULvXRt~1X1Y=?`_ zHi;-c(ON7oaczi8ugSH={Y?5QTR~9{YoJbpS(&EG>tzM(#g4b$K>kft{$8H6AA3LQ zgcSGfP56ddN<)9>w>&-OWwrC{I zZT_hdPuAAH&p$@e<*hv3QD!XcmyW`zKgaX;go|5V zU5}@EY0iVqFGF})_MQ{0h#hagiGnD#7WGp>JhJXsw=uBDBtEH%L~xioU;Rbh(kCEz z5?vEX!I3&R%S&7v;?f@#o+70&#Y#@`=QnXHRbQtxUK2ateG8=tn!+?@ z1yZhVa==1&Tg6lCWrZ?{>bp%jeTEyM9#T3Tx6%vG`oP;3AK{u`RtkH z34o<8^NKnifrk-N>p<8?#`5>m z3eWh*-ZXvf;br~w=EI2msS|&U7S;_hUka0Pz4?22iYAzh370kr^RKEft2$ixUKk?) zcSHG_lOu&boYJC}w;o>FV&xUaz>=oG#CQ_|z@=vCMwA{LJ!D%&X#~cW__i+pBC2yt zB?1*wZ1@prX!ib0SU!Um9n29fx~*I{Xc~M#lHBAt?ftkDO)(?juxunb!#$u?SGP1# z4JKoIL!;NH)1J!Loe^?q8RwJYu?1>0{V|`+e(6ZcABBD-o>q~j zM%GG}R)qWbdxqaO1ews$mGc_1YWt9Qd84pyd5S8c99AI2d@-_vcF?MC8wzg8H{u<2 zd?jsHETF0F58B3HgWOa`CR0wx&PI7@B}=PYl@@ivcqWJKAzP8``Qf(O;TcZifo`#KT;ti6;eC!m)T=ovB4x0T%g2v~V8 zuJG?agahhD3K}sMF=~o9t~njip5(|TIG{T9I35+8p#gRlNuD7JIC^EZ#AHvvtITt3 z#H*&@G@?UH;p(J^1G;;#Rc&_>OlcWTepf1e@$HP#!gs^pY%hEbcfiMRC%kmF;+pmrTvL;SxAtZPm~BDC`OPSb zC*STB_ANP-78);OvGOlmhF|aRM1rawWB~)DaDD6)M9y7=m=$YrM|LaXckDohi3%!i z+LBdsYDE5FDmZd!rNngHeH|VaJm--^kr79&m9hNyMfm2MZ}0~B1PnO!G;78f0uWg) zYPTrW4&M#v!ShZZQ)nW~i?Z=|0@=fSdpsluzr9dqj0pmV{|Min197Q)Kb-DH8@J!~ zh70|A^yp89VJJezPKKY{LKG)kzOsI0#u%ZbfTA8+_}9pJe~^}efI-8X1(jc~?+{E@ zE8>@Vh+Qj-BqcRu7@Koq&v#@uquAGkEXXM#Mc>ww7*q^^u0C6ZbyIJdW532u^ytwY zojSb>nLp;k-c}vH?q;}p-T>x?+u-<@hOm5T=246sKClj0dmG8oP^R|&d^g?-0i(y_ zQn$VcKW%`lTPIMM?2N3N4!C(r2X2dI;Ti#qjaQhWG<~QiUx_ZVVl7ZonA{Ss8VCB_ zUL)HqsF=m_?>7XY_k(mZ#lbvl{nPj*t~N&M7zD61Ep) zBqr}*d0ww5)dsqnJMh_OpP_%hUWf^E;FheO@AF>@@2G35BKs)Dj2_H^H~PYKbbd|H9Ns!#S`npV9!(y z)#TYDKI|aQI%+~sdpi{Otc9NDMw~yXgyJ-tMs*63tr6m>L+y~WoNb5XNE4htr3ow3 zy|6zlk1((Oyw3G&`{9089jtujZ8KavtqE&Wc^tErhu>ul+zCC1a9>?S2N~c&E=|WT z^fNEX3K4#Kh$LeDWBGYWmhhi41(&<^Lsqbn$iBp#RpN`Uv)%imEX58f5hjQt9=8MO zIOnc_PeWp|-tyN+w4W|2GDPbq>ox;cy5zvhv9oUxRNP{bQy)5E8gBi*5=n}xDE9KD zAj2o9eewjw-oBKkhG@!*S-BK7d4CBhC`?HrP%-eOCLBP1q6Jn+&%^7qcci>`755Pu z@)P%5nQe<1KaJqPGSgp6)97|gnm8VvJ9oz0Z@-QA-g^(-x_yhq^T*RbA8N$&L9QJ< z&#Oaj<0>lC4R5{m7We1a@8_R?jsX_X z18_Iqyh%S@$nyR??I#SA8jP|``cO%)xd|5VpZXKr2v}*p1|oG6>!FF@;bU;R-#}Dm zI^dSoZroU~6oEg_LQarwlRCF9Ya;5;b?}%r69v%}?AKuY>Yp=Wj9X{1l2kP%0+wIz zK?oT&fjjPZWVazzTc1ydjM(#?P1|_h^)ZuZ)pb}9iAYqKZVpS=Ww1E6q**#dQ4*zD zmh>g7&;azun>4WT5tc$e_58&BR#KaDW{&2-8YVS_2J~CVuKyi}jCVp`YXe4&7{K!w zV2$=KZ^SpmQxnUUF2K9*z6)iA4Y(a1 zf|=n~1YX%kykF;Kj+m`!(pPn*74%WO#X-m{Dr>tS*#f>(CgTbNmXBVOI!s1?TLlCT z83pG#)A=-bFY*Ac^&12qG7#ZfE1T2_Cg25pHw>qzOrlxFQKZh(qYYaJtoWVs5&?@% zmw|&v@aZsW(Q?GDm!lP-JSX-XwU{>JCQs*ODq@Yvv{M8kBZ@?H57^csupF+eLiIgs z*qY1nDe$W=yP`VR{yCpIUiCZ~Uru%HR1hfZfwgqebQGnZqC8t+net3)3>(s&58|8| zQj}-I^Gedpp}6}`4k`m}IsUaaGWhewH-?R}1{E zs}t#588mucNhZ8Qy&JO$-{?jej|k z#;jLlnhSwtXU@WZo5s2;?);TopOM3Qp@zb4S()peDi|?bid&%4OajLX^)jRN@$ttW z@&OLKqQvvd(#Z3*gwZ zKfFzqNPl+1ddY5z@a{DL9^LyR-BVeZo?ym^%XL6va|joGQ7G zm8`K}Vqh`EGvEXa9LDGHkkJ!4v1evVS)O0tL8#6$ZxUM_Kt~cL6c8wm!XsFqzXjA3 z7I9$p>GcH-hU78p6`7_DfVEC!Z1-WKmE6Y}HM~#VKwIFLr5yLehD-G!M_n0b?3LhX zCl3dkz4*&&4-T8|#K;j+eDE)x+`;))W}1;dH;kv9wA~~&j{J`$6L=XDy_I6i6eb&? zcMrkFuCD5QVL2<~P2{hA1T{rgmwhCHxc%Z3V|e{Mm5-SZjf2mSA8>WRP&gCN?7sV+ zLTOeaEws*>H;DCTGJDUSC^${ou|UW#4JN=d}9YVJPSUGmbS_Q27 zl_Ccb`CXjoxjy~Ij0tq;Yu1D+vrJ%rSw<|dvJ-W99euTh99ZwsLGQgd1ChK$@h{IX zZUC&cBJ0_it)@M(PM9!uAWaVzyo{FeGVY`^U#4r$9w z?#}YOPGzPE=FJ(y!Qf~sD^|w!tEO?E&O}$H*?wp$&gHy@522tv(o<2gA!L@%#{2Jg zLVAp4qq@0qdhq>u8a&DJTveCFb$b;=xa>!2=pj_)oZui7r5~0Sb3~T=SyzS)p|j6} zv;s7Nx8i!7BMX<}Z`7=e=~3D!YYniU9lo)A&l&C{Uqm@#2oDC|ntazRBjDuMC+js%6Y&M9MQFiqV z0kCwM??+0620r?*6E7R?C67=Kc?=si6f(chpuuM0)z4%0@5a&$Gj%GT(&8`elUP5D zMarbo%>^IP$!GkiVf1IH4kameZ(sP7dsLv z3Z{s}?1N${l1|wVz5py2Puf_mt5Za=7RQPrsu-IT7gpu|_4y>LO$6k`cE*FTGy*{rz-GCMV0{_X;X1$m?hWJx@ z3NK@(B}9UBL2Dx%1bgN_pw;X);RFZZKOu8olRk`1}&0L z;O0ga>v|}QAAerYOm~rRYhd0lQn*K#0b1f)nyibd6NmD8R)#Vn%OBKU&iAw?jG++T zEK(%38SS?hohXox)zgswg(3u4OnCEsC`;ADte*wII&pZN$nupL29RDdNdT<FKwtAXK9T!ME$zNHmmfG}C`!ZGg@cyk%d0Zp&%b+=}T^N0_%3fr|c=LTe78 zS=m$_gO>)DU!U)A*O~6iWirJ}PNAXOVB@-k?!DPG`nbRktG?jr-7x@%tKCZ5sL~I!5Lp&i-g+vf_-t|Y*tMgR zwi^gAgZQeg9%O9Q$OeFs+`8Eum_>;iw7cGirnPrqpuxg_o653j^%PnazJoEN2Jkz1 z4a*eKSE;&pKEC>*t8f~wZ{zjXX~ibRrcWbYC-06bZMOF2ZP}UqDYC7SWKEiy2lFzz zs`T+lpc_YPX}(cAYx)mOG0Q*Ukex223t9@{8Ea#5P88OV?S+vnvmJo zz_}B9csmzPDd1)R-6za!T&E<}l=G^fZ4Pn$DQ!WX7wesOSc^|S=?WvAm121^xl8G^ zpfufz+Lk<)zhPT3?i5b_<3xDMON-V+-NloVBz1&bkwaE$s6^IEq{V7j^I>it*#1`B z)?Kt2D%AvJ?ARt*LkD-G&mk>bo=(tN+%*gN?Vy*{L3ye=4Cx$??VUvjwyhq04{wIY z&>s-v%5D&eZmT#!71t>yJ|^5*VoB4IL>Yx^R zS@frg=zifnx(L87542Ux_5U*8GSZuy(`L+?ISo=Y#a8lZy5)->tu4c1;)5=1Lx}7K zoE$2JGTJ14kw+E6dP)j;s@#_~Hx5HR}E9U+>i=Q~?Ypf*Q?S19?3v}59&qEX|Cc6pV6szUBO9q)y z3cQ_+$UR^&?Ki#TaLsugLcGmTQI^{(OI2U^l>)1tDck3`ml=-47+1vHIDu%2{Olm{ zI*1H9im6k^afh8POlHlTfw)_j+eBwq|CAwzT?%d#Cx3MMO!}Wc+=T7Kgq=WaBwe2) zU+Q5^I0?`Ps8)FgG)WiZcB'), - esc('
') + ntob('(.*)') + esc('
')) + esc('
') + b'(.*)' + esc('
')) m = re.match(epage, self.body, re.DOTALL) if not m: self._handlewebError( @@ -387,7 +390,9 @@ def _test_method_sorter(_, x, y): if x < y: return -1 return 0 -unittest.TestLoader.sortTestMethodsUsing = _test_method_sorter # noqa: E305 + + +unittest.TestLoader.sortTestMethodsUsing = _test_method_sorter def setup_client(): @@ -452,11 +457,11 @@ def start(self, imports=None): args = [ '-m', - 'cherrypy.__main__', # __main__ is needed for `-m` in Python 2.6 + 'cherrypy', '-c', self.config_file, '-p', self.pid_file, ] - """ + r""" Command for running cherryd server with autoreload enabled Using diff --git a/lib/cherrypy/test/logtest.py b/lib/cherrypy/test/logtest.py index 5361cfb..ed8f154 100644 --- a/lib/cherrypy/test/logtest.py +++ b/lib/cherrypy/test/logtest.py @@ -2,6 +2,7 @@ import sys import time +from uuid import UUID import six @@ -46,7 +47,7 @@ class LogCase(object): logfile = None lastmarker = None - markerPrefix = ntob('test suite marker: ') + markerPrefix = b'test suite marker: ' def _handleLogError(self, msg, data, marker, pattern): print('') @@ -161,6 +162,33 @@ def assertNotInLog(self, line, marker=None): msg = '%r found in log' % line self._handleLogError(msg, data, marker, line) + def assertValidUUIDv4(self, marker=None): + """Fail if the given UUIDv4 is not valid. + + The log will be searched from the given marker to the next marker. + If marker is None, self.lastmarker is used. If the log hasn't + been marked (using self.markLog), the entire log will be searched. + """ + data = self._read_marked_region(marker) + data = [ + chunk.decode('utf-8').rstrip('\n').rstrip('\r') + for chunk in data + ] + for log_chunk in data: + try: + uuid_log = data[-1] + uuid_obj = UUID(uuid_log, version=4) + except (TypeError, ValueError): + pass # it might be in other chunk + else: + if str(uuid_obj) == uuid_log: + return + msg = '%r is not a valid UUIDv4' % uuid_log + self._handleLogError(msg, data, marker, log_chunk) + + msg = 'UUIDv4 not found in log' + self._handleLogError(msg, data, marker, log_chunk) + def assertLog(self, sliceargs, lines, marker=None): """Fail if log.readlines()[sliceargs] is not contained in 'lines'. diff --git a/lib/cherrypy/test/modpy.py b/lib/cherrypy/test/modpy.py index 6da9c53..7c288d2 100644 --- a/lib/cherrypy/test/modpy.py +++ b/lib/cherrypy/test/modpy.py @@ -37,6 +37,7 @@ import os import re +import cherrypy from cherrypy.test import helper curdir = os.path.join(os.getcwd(), os.path.dirname(__file__)) @@ -132,7 +133,6 @@ def wsgisetup(req): loaded = True options = req.get_options() - import cherrypy cherrypy.config.update({ 'log.error_file': os.path.join(curdir, 'test.log'), 'environment': 'test_suite', @@ -155,7 +155,6 @@ def cpmodpysetup(req): loaded = True options = req.get_options() - import cherrypy cherrypy.config.update({ 'log.error_file': os.path.join(curdir, 'test.log'), 'environment': 'test_suite', diff --git a/lib/cherrypy/test/modwsgi.py b/lib/cherrypy/test/modwsgi.py index cd40ad5..f558e22 100644 --- a/lib/cherrypy/test/modwsgi.py +++ b/lib/cherrypy/test/modwsgi.py @@ -39,7 +39,10 @@ import portend -from cherrypy.test import helper, webtest +from cheroot.test import webtest + +import cherrypy +from cherrypy.test import helper curdir = os.path.abspath(os.path.dirname(__file__)) @@ -87,7 +90,7 @@ def read_process(cmd, args=''): WSGIScriptAlias / "%(curdir)s/modwsgi.py" SetEnv testmod %(testmod)s -""" +""" # noqa E501 class ModWSGISupervisor(helper.Supervisor): @@ -134,7 +137,6 @@ def stop(self): def application(environ, start_response): - import cherrypy global loaded if not loaded: loaded = True diff --git a/lib/cherrypy/test/sessiondemo.py b/lib/cherrypy/test/sessiondemo.py old mode 100644 new mode 100755 index 2aac468..8226c1b --- a/lib/cherrypy/test/sessiondemo.py +++ b/lib/cherrypy/test/sessiondemo.py @@ -4,9 +4,11 @@ import calendar from datetime import datetime import sys + +import six + import cherrypy from cherrypy.lib import sessions -from cherrypy._cpcompat import copyitems page = """ @@ -93,7 +95,7 @@ Python Version:%(pyversion)s -""" +""" # noqa E501 class Root(object): @@ -121,7 +123,7 @@ def page(self): 'changemsg': '
'.join(changemsg), 'respcookie': cherrypy.response.cookie.output(), 'reqcookie': cherrypy.request.cookie.output(), - 'sessiondata': copyitems(cherrypy.session), + 'sessiondata': list(six.iteritems(cherrypy.session)), 'servertime': ( datetime.utcnow().strftime('%Y/%m/%d %H:%M') + ' UTC' ), diff --git a/lib/cherrypy/test/static/dirback.jpg b/lib/cherrypy/test/static/dirback.jpg index 530e6d6a386fc097f3a1dbabbde2d80fec1175ac..80403dc227c19b9192158420f5288121e7f2669a 100644 GIT binary patch literal 16585 zcmbt*XHZjJ)b2?k2_^L2jDScNBy`k7T0p=M0wRb+K@7cviX{oX3JM58C3F%%kX|fw z5dlLN5D*kZq^Jnkxp}{D=Ki>U@6JpnIg{CEPS#%Kd7icQ-|W8w1Z^y>EddY+06?4< zus;i!0bE@0|F5;*2?!xTykIW~AOwJf0I(3S-viJ900R2&M*nw#zz_h6;9UD}{oktq z00u#zT!H`y0)}wG!7$D@Fz1&b&OQfqgcY!mqbD6Q2^TJggi;Ide4!Q9b&W1TMHF?- z9(Rs-eshfJ9$OUpe|F?Noe+3`2H=DI&nF?^1n~Vu^T3ZI_>QeL%Ggu5I;IpTZh4}j zErN>A74Ka=3t6Y@@U`te4t|Oc`i2Y_T}~n>kkZEjrY%(L_%GbVO&pJvv{SDqa4YBK z5&2%mC`qp}RUW)|Ubt6?15(cfiRZQa;fbHlOlr-)`MS7++VnzaFzA)aK&`Fen9kDQ z+kcsb)T3c1K7A#*YP<|j4a$Eh>3NhV-oxY(Ee%wkoA=S_|q6=&SAAFi>&SkYW| zrdLsciEwk?#yPza$wULmwArF;OOty)zC5``q!HsK=ImsZETR_7=vK zVTOAc<6>7qpKTW_6uj=Ui3bn8qQ_FrB%!CQVbdPERX=rF^iU0(c-cn;P zcU`#>PCOtGMofiNqAW9Q%R(<9&d5R#zmrxKX~;$cBEJFm)5*= zw1-rLfYxcWDUlB`;TZh{)o*P&QW>O-&`(Q*3TUcAS=rY`3(Y#z@YAnx1jm^m>D$4@ zrP@$Idzo|44~BH2CT`_};t)v0L(f*obO7mYeHOO@3|>%3pRgldM3UdPl{Nf6df814 zG#pYUED(yP6i8U0E*{%muM+Ly>!g{gQ3b$eshMwC4tQ_Ibyn78`K@e?#C@RkZK=gR z5LyRf=~zlg^+2Q41g{vT@%eC_q~P*kFG0{W*;?RqN)fq9Wafu+4$_pZKI3UYJM?J9 zSQVSJEmU<`i+7emFb-^G~Of*#XN-M*um9BSD8Hdrd} znfru>)1xT;WJ3>$1)f6hyC-q=;ce#cV#-YDmxz)wvX(nV!>*`IsoSSmg-!9&wlD5D z7(KO{Ew2va@z<4UKg}~UvFkwKR%?Zj5la2(<`-=QrQc#@vnI6j7F{Lvy2#Tb!jX`( zz!?i$Q&a~(**v(Jo(+SIU)0+=o2DxhU*|;1zrWeJOAs2n5?o5lw4_BGUyB2_~1>ewZCxt5iA>8QNXejy) z3k)TkrguJNTd%~IPhY&F)X5|O#}$~lc#{Mtq@*d0O98|Xop3sp%}A;=AXapzgdTd% zB$ahX$1il^k4)!Dy=9`LL3iFuM|oRti{Q7vGi_2~Q9X8+VrE4T`t^yh2mJ@ra%&e& z-e!;EMdw==QH1V->21=5=Ecaj;@02dv z^9xTjdGgm(jd3GS>#kDzm*yNb&?XT=)%CbSkN|DSi$Wm9S3L#lp|*M1=jiF|6{BZ5 zs!%~&IQV+jgsgQ1>bE+djwHi8N6s4ZtYQRMQ-6y$Aj)=d&uusgXJg2Puhwn)@;$9= z1rbh+4UBTn*h#8eh}BtK@CNC-6YWltdjFK&tEy;=yrt((w{3V z^Qt2diQh1tkJ5}$D3i8TeqVeP;~1W#@GUkBFcPSoEX4zYp*<-!Xih2NvQwvY{=#jqoJY&+4iU1yuLtAv+yQ>PcJyI_O?JWL7f3hlex4Ai55t=!1IStU{vP?;rRor8EzzO2rB8r1Rh!S-efXpjd(ZS zIPdix-lNrN*__yYHJ^; z12qR82GNoIE({s{-G|gK`jnTk{L+V0QY@@boKMs<5{17hI>+Fi3zCqiwxKdQ2d2Ac zzF(Piu*;C_v1nxpH0g~hCJ}H*XpSm@l1y;}C$uX1bmV0$th1CW6mr%Y1hEajMfp<` zk9)Cae=><+d-Z3^Z%q*r-rI^sPl(q9!&@bnFMq>TEGZfl8#X6rJwh7fj}VA{)QS+H z5N@?>ADt)H)J7Z(?z%yW*z>CrRBVLEaGt}QmV;rE$PmMrb_N2qL?h8VCAQl3E@ zO^(q?k}?ZkKSdfU9#5sx&{ zAyKr~3%{P%(tR=)4EvUuGfw>hRTuTVdUd#}NK(SUmMRy*`>~^194P4EaWbOYhMx%m zxD+1xGUj^~72L3nR?-mzJFiPmX5J49(pXjJ4{_`+FEo;~w=~N0lGVAt54?yX7Hrl= z=sgm&<-r1KVt%K!o4J6d&`JnsX#^6@a4$_eUFRUB?bWtXLEsy^^7EglZ@US-;EA7k zK^VNECsjL`c8x)@5Ij*=_%ylm>1`tPca`UT$n<|kWgrzod(mep>zzF+H`pdGVKanE z%S_HG8OyUYXhX`aPVW7{9D89Ab6;eTV@wC{1f7CI=;QrUGb3?l6b*%!WJIOoVqiiz z|DPz7p*cXML^4p|hy!ZW^d)K}1zV^_{s{gkFz}Js!SA7dGWUZZ$qGf4$3Q#eJ{~P8 zu|rx32qgXEQq$C)YhM3x1F`bxRr>6cr@picLoER_10~u)ag)zy1%a{3$cKIXr7>3N zEfICF+wK!^ABBu<!?SD)W>W?9k1Dh4Fx873ce1-@( zHWsS$wc%WBbH#P8Y zBiGEoC>DpH{KsA3PtlKim9k-M=XFG3@G0l^km9ipQNojBSwLb@g!F(~0w6 zD$O8EQRZTpb~7`_b(*E^_*~@XB{(1QN%uo^gML%l3pBsLM z&njvFB-aaEDJ-HJHK66r=mByI>L<}cQRv^Oa&zICO*(Z!GW!rQ4r&_KeQ&&OgrZ>BrUKOS%J2+yU_cqdqfT6Vymj@*=ADB~FdE zRh|{I6y^07hdA=cV`%BDFyjHXKo1}I)FIvSHm}C!5rJx(xO#ZlPKbu@EN?RJEv|DV z`jmUQ-QJ(_9O1mRX(Q)xDt+{OfPF?CFD$*^35Jx*;DB}*wSUL8|87S$GM}|laZ;CV zyei@43p+ez1&v`&y1p=MZqEoo{zHBg5dTIUpEO1Zs=8ju=AlGm@mG$Ozy!$E0-J)0 z>wY89PlI#Mve6o29!=Ha)04iZ_W>`-&Tz=vEWX1hjg{cYYWrnMgV2CUHXOrk43#2N zt}18VT4|-pzSj}|hhTf&KL<)A^0VzoBJ+-n36!q3;B?Bm5IWpm_i}#exRs z(0ztfR+2>^tdGPlBmPPle3UZQ3XP1#ZU?0#5ghmj!jLOpn2r}2gIb%i$KtHR_kr}w zVlDpK3`c&gUWpazVgm~q_h1i;nf%elAic?7xWnFAxrUn9!uC%tsMeQ=_*dv+AGyp; z{>;Ii{@VGXJ^o&A5`NZuWls4)l{(;govY5o{lPetCopBVmT6O+JNM^R3BQh`Y5}2N zh3$vh!#V+w_GQTxA(`sA$Y=7hYeW->*^hSge ze{Qn6c^izzY&2jFAL?*a;1j@|;;A35-%F38Qt+n1s9{;4SU$pDBtYjtH`-3shIUIp8yMy~J z+-^&t8g6DraGv)0HZ}mZh_&F1a!~G#A;@ z;!^5RLLq~4-u!sj#Ps;Om&VD#cxvj*LdsQwz&iD5PJH@S7W_X8{=~NJVBr+&@}X}` z9=H7PxrqYTocLj{pc{>66iYddcs$t(VRZXX^zaC7?Lsb)oU5(P5(jo^j4#nRSs%P} z%%-6F%NUr+Xp+da{6<_)qEy=4FXv?c!Pg5SY{fKOZuy}Y2qQS}1CI9aC2o(Nso|kALE+fW;>7iB%aq z#m{lQIi4J8xMCYM0~@7gzR0i!w(;B{CP$PGd3e9-ibTLuDB>S z814c!JeP=(RSEEt?V%yF;sdHC)K0%uKNTIz-EbID5$MOrx^FV`Z$eermS?x&tA@lI zGy1LXOl$Fs_ek=?%L$)d<*El&j?DgBRnhk!+d(m=YlZy!Ss?2D$-eCHijS0p!pt6i5=SPlxZZ{7XQ)u#`5fg0Pg-1YCJ_

9L zEE=UV_buke9PIN1+@#me#P|o1;^wHS{HW^JSXtAc;NltnC1NMr--+XYT#E6zX_3*x zW5SuEZ`jGOI~VX3$B)FFq@;WQ$vA#+SDc3Ujerax@wUIxW`B=U(aaFIM_0ocuOI@I z8`Ni;*r3Ue8U)#(blwU|%F>YbI55&3WYeGVlu~nokPV2U#o7Bn1Y>G|aiZ7qC*lLk;NevUsPuVFzz0; z&=3nhbbh4y&@jOLVEcQugpU~XD0KKTgU9k!zbZjzdBoS;zy%@H z(ZVl1c@GnpzTp?n@4Dtawx=pK4Q5;EVP*K0=ouH|xU`%6uQ3i*mw99t#<)!%!}}xG z|1d~!MT8C-eatzXq!L;m)wp;$4aWwI&OUi#`1B@P{0$^=q!tPuWpRZV&RKr(%ipOY z!HqFgXj_=uQRTLqk66@S&t_t7*id(m;U!$g?8z@s62%5re-SIJb-dlnqbIXj`@n7C zC~rfD!$~2G`Wxr`NWmuAm=1Z|f&cL}%#COGAkPVvk5>kLrYCSv;8;(^p;5+Zk3domGX^fEG?j2p?^RaCQlYHw?0okt%W;`)-wp&vd}?u z_%U@Zeb)&fgn3uTB^GCUM@e$i0)aZO)og~_#9-ygI^R?b&R3w2WUimT#@RArj7Hzb zWnK~xWWfoKG|PVKIw&_O}UrdCHQU6^Gr{r+r z;adHy;pZP`A<)K{b);`LB??NrPko@4^ErI#vJh$lec8mX6dFgZvwzPu0(Gw&7vBohF!%If^i+|9-YOmJL~_zHeU! zaRMf%yq7k)O_be|N_UU8vD9p(+4~)E9bixM?S}HF@iZBICm7OwZR`FPOk{=?{WiGN z?oIaObqZB`;ac`LOFSX+nFNNtIn@PY9ZJ*@;R0VU9kF%Y^fPNP&ub&y9kR2sP*qVH z!dChV*<8?LtTw|?qkyd>7%lXcLh)1N#B5b_K;!~ydxy!Vhi=wO^G+RVQ5-*wr6@`{eXIcB0nE&bl%qtUxobl3_AVOZy{-cVH0GFrHn*6{@j>IA3!R2_aAB zjdzUp4XyrU@BH~-`_me$G+a-ve%{31z&;>NtgZzNJHT`kVeEa~c9r?SX9ry2lArG_ zSR$SF0QWKh=}Rgh#8+Y`J{z`++qv)p%Z{+We6 zl^NEcK8#4b$=+mb=TI(GIGXJ^O}^F6IKH4_&^!Q{VmX_iQ}d}gaWY#pdE#m10avzu zz@W;e<_+?ZLf0NcVmihxu~U#ww*I$hXJik;z@R%0U+-#uDsHl4A&Hav#UbZE@ceei z=Nd2<2@Iv)*Akxy>h*bghg*q=cV3-h4Y5qXr1z(iqjf39C>#WdGVQ`1#_Z4E5KGHg{0;iwA{^y+95t6{maya=!*L?j`}?k1<+*al9oU ztt&cz*k@zk8Kr;sK9CJH&(E;!vHFe~-1c&{dWU6qC@m!>A=Hr{j2WXw51-HZnjbh` zq1$YSY+asnEmxULotv0HvR2~R-hFc_VKT|%!ci~%8QF8EF*QoZMkMlj7k3yM(e2M_ z1gi1D?_Ja7k8D1+LEkkK60uUShli}S={(z{dR%sln>sF*oheyZtTA`phA6B}*MOm1C3+iUk*BK~|cNe#CJbcGe>7pGc;)zb2 z#N55tSW``&)ycgS<>AV^>iigaEGr!q7@{9H(fwDnbEN)`L4uekPa$b4vY7rxjcvgV zWhLo3BR2&vyG%f>wZmk-5g5v@x-xH}qSG={i+C&9zUPQcpF7?HD!Aeo!LEE!sBw;t z;U)P~tVlY=gPu|B$_;rKG+BFmaI)){5ilcYpK{F$Mo-oUwK$52W(NIXP0GC4abgHY zPdcBzpkO=7llRUg)jlZb?JEA z{fTdmmi%9@t2(%DwV2bRy9za#O`ey@H1i&_hx`FLA*f;f;kD-MJ$CpyYt&kc_BmGA zCs**b3g#V!Jog}YNYJ6Ey{g%C;R@RgF~OpC%`Cg{fx2!aa|bB$Rz$f!a$<%xNYjAg zh_BbiC-^SSdA&X%P|dXqE{my2T5A=W;oBORT@aTSL!srnZ~kY@HKdQ8&)sW3oJ%^Z zMdHd%UsGBjlr^8P&>xE!UyQ7HVaPZ-DR7HJ)zMT>e_!>)y_%IY(5HHL)x#yMSi|SM+#Ap=$NZotOp|*mHch zmPzNdK7J@;rdW^Lum_>G`HcMLG^TwH$z_1h@+6xYD_-a_a}k* zr6r;Y8&@q5;;t7X_laS3m7wH0{p`t)Oe|k@_)2L}UxN@dcMH>gLC0)8Tcv}M#>h%` zMAnN!oBLuepa&f^Koa3ac`u%5$TAjmnh&9t3du_3H^v*ba2Y<(4Ys z1T->6wXnXWZnN43;}L#S6D)FF#u0qlZ8&{x&Obq*h!HUy=YFcMQ zLr@|*&a>lnu@ARzCKG$ct}X<)s@CfnVAAsxFx$LpE(m7wpAts_9fZNy1PB~1=Nu1& z)`W6$!HB#%@Ag;KLO}0O(f``EqD*=l^ZVptC25<-6=Pk}oZOGJQhW(nE~H#_!t2)j zb4XRpT%#PgSz#`#E6ux`iM`6mjjYVKtIs>In+5mfI%2G5V;*Iaq>yYrd7k=(S3;Um zt1y}bksZN!wq6oE0tV zHMZL zJH1FJCI*5xQ;q-FQ&!Qzrynr)>jqh~cBy&(kb_oK>hQqa=2W3gy0ut5*sRz4!a$5?c5%8y(0!hG30V<%g2U`$2cf(V6&imy;BZLTwp5fUG~ zU;DU9P_|02;?$wjG}%KX4jjOVy}?sv^j-cG>cjN?+3rsSV;Pif93C%1T0EXzO9U(9 z)EK^d8aT!o*QHgF)8*fbD0H6P45!39hTUmaEiQm!ZiYL8Q3|3$=wPmQ25%VTQE{4b z!#5TOCXp#V4@R!GWfYdRT4d75!);6xn6n#0+okLQU-JP4Tk;ET7EHYIed z%oFGr=a-H)e`opCc*8{Or?#+zeJu@w`d%{2Shl2Muv-@`&Ch}kW{@HsP|}U8-9+97i~&(ix?5BtlxK&dfN~rkv=$RBT6C!FvVer%LDH0FH&b2O zL_5Ugt1oT%vD28~qWmeih#XY-omKfSRoXj4idjS8JtwX_g-w`APUktZ_9Djn>5Fb0 z-GQ*QDoFU~{4dMVaBsKaJL*P3-J$#Dnbwoeoc|J8fOZr#;=<-bGUR%-3D^;mFlBJn zIWa+M@wwi?5R;6CIf&jcow}@AD9<<8#9b1*rd{o$_p0~OQI#Kd-iQ+~owYk->IAig zFsQ55|5fh5?;V%^RPWkn;rSgHUPgzo=q(E;@AYa zLJ0s8iHHzgR2*gOUKfDFYIrvgB>tMc!S<;IG{`jYa+$d)>g-{UmGPwYuA7i9Bfcxt)JIVV^m-YeLp|RS;kTH!j!u(M}S{#6R1LpHU$`!v4gcAzu zJP~P1)ixC4H$J6KV6)ci#r(6}^m!I-Ly^1e{0+a~H0p@3$zJ86#Ni-Xldz!0TI53N$d=Hc`wb-36c#NH(H&)Nv|G&3lf5R`OR zzvjlb3dzm?_$*gwlswB)v1P{+k_DB3muuL7WA)#4w1i#dIDwJzO78S5jzM)vTWu!&)nByJxQH1h>1+>X#X zX8Q$2(6lT2eCrk!6n?R3EMZqwF4}M(usuH+TBxkk&C!#5F-=RP;1|wmSz^;oT4474 zf=bR*_ztm*!yIUh1+zK+JMV(xrdf-I`QPxhL{XpyS7(;rnYZ0~QO~R~#Tg|Xz4hMc zEtL+X>kpwvra6MBs^vXt@Yq|~Mzz%N0h4Vt==%>xUdKsl@>d^s0={OkgSSk0Iz^x& zd`Us#fhVNtpi~9vSKF^ThG%BV&-{3QZ7Q&(RKb^DD&5)RjG6?ff$@1}9%FV^A%@k7 zk-1Vsui!JtTuU9H_7@fpe0EJ!?v5@_x z#`laF&9y{%6B?(jAczYo2+PUCQixwb-Y+2W9qMeq5l1bl)VjVAYw_SAc z00Ko(54C|MxHKFHmxL2&!;`Np4OhBM+2C`^55G0Qs8JYQA4e?c{`Zvz@%Vlg*US(K z^7Ki4;ZbF7q;B%_QZJatB4*=oB4SfT{`#*eA9q zQNFh>U-6)cFwaI@<6hT;LxL)ypJoV!Vb_m6RN_<4yNL%YUnplW-d&A_vpNU4R{fVz zK&h=j%A%{w&p)V-Gf9d!7g|YqqEa4h3!~J;d$-2LII5S)iBC60Pvt#|(A6zF@P-3j z+6?l#Dzp#hU#k+(GHzN$w@qK-qUdE-t9MnL@@pX*28Nc}m>!P0+V*R_q6cpta>cEq z=i(;L;Iq2R;M=@T474Fq!-PqUkN!9ghESRF%jwS~wVDhn9 zsk+8C;Yt9gPfB=t=<`=580J2^`&1rH*azN&V3actm23jVjNERHDviCJmmFFo7B{tb zxD_O9t-5Fcf|%xO6&&USUPHu1*pztc#qas`l%OOS^^Jrv{|^2PXm^zp@9|zg+4tes z+n0jG!#B9+%MKQ4z6zFzC&_)V(g}-qCES=^_n=`k6}nt z;$_h;^)(U@21?*dgJ`gUps{;C_ZcagMj$UOJlExQD zI<>ta3a(by%RmiEElDsP5A#(DZz=#CPQF-tb+LKH|LeQJNT46i@dmJbjGs=?98 zDgtet1E5Eawyw-VjNIJ`a?jqwWXl4f)tCZb-H_?$OtNFynZSSJd>8HX?=7!l-<(*K zm@{v6#U+|(M^JC|7h2IpG=SEwX(u?@)X-!ydGSV+*Pc6|8Gc8ao6h0#KWfW4L1*~N z9y|!$QXuH-x8FK7UV)9Lm|5 zr~0ihneRsCjv7}>>-9?H`Zb0fqd#;v>UNTSVp2?Y`_SJbUzw7hEVoQjw6;xsS@-@+ z^uXd&SCU=FMUxEo(DfJFMb3Qwn}1ji^pRJ~B+}%!V@@!Do8mW5DRPP4u~5MDw)YO< zTnYy?U$aY#>%=&EM&X$^)fZ(CB=^g4Cza95}I*>MpcNBaNMI zDxyqk_SC^IWOxG;M8qkqwz;2^BB2%}Jo6(1B*rLDA~BJrnx|NN3WB1N*+ni7jNNhP z?4NSUvB;WXNy(tMj^+hV$jAu$MC4QyD_iw$4u?9oK5z*?dVBz~gW@DU7o!5hbUfBL zo5OuWDZoPuNxR$x!1c&XovM3vFKB+k_>?vUm_)0Qbl*C?^V!A-v~@fus3vWytE?P% zuO=o%izN$T)UIisI-U3(i>w`b(Uz(n_x!`0zg4N|Yr)fh?mfAB)O7Q295e_UAX*oA0QOA z?GQPk3Db_v8gu@9FM9HEdg#&n)$Lcx6B^TO3UPu*^*5Kk>h?80dj54ip!=7DYREK) zv;6lNvJ`;;mv-sgF!g0?Tj-J4D1P8NwOA$h)@x-A{X`|luluF#8+y4WMfaG`5@?I_T*;>WtMulg+*Bm zQlPNOc+U^ciImGx$$=#*uvYZPMT{%|!F$WF8T3nSwFpb|ZtD^~o5C0iCGN>4-!e7n z;m_Mk7>E=OC_oF?f0v{#Oh{%T?|vMzNj6gt{HoID=Yl{bZDX6t*zj|zuDK6@rZ|Oc zVAMN;mSjNW!L2y?QhqaehhwXX=}2slpA%Y1{>2KR_;i7VH8}d;e_1(Xp?TXU?+jzXPL5VJ+UASUt7pqS2RRL`I^PHduTdRnKz>6+GFSswOZ2V?oY;7Xp^% zCVj7;b~Il~P<~Kzvai4kI6RR3T}eD5JN$)ds^UxBvQL2TLisNZ3F)Qoo39gosPXg5 zo$b|@>wZG34qnk<@xad?6uxSjVS7)8XOAKLqYm^yR8xs24@Ko^G6gLw!T0qZf8;ba zf`L*QV7ICc21jDVww$=I#<5($;Z}0iGYjbHWrFsRYWP34_r3mC_GAgaWV@n8NsOG0 z!L&_oY>NSt_ zt2TkN)`D48w&rw@{iur9H}MUW&xuPgEp+)n#`ua8Xo9p75~CD|Hw&%UwW6h_^N%C zP%PrxH2mc&s;MjZz=;{AYNry9h=3y~jhP8JurC|L5ygUj-A`~T?9+0MeBK6DhQ>^I z!fTwck(3L^0}2hV< zh$pj)Y^06kD~coBtTZ4BaPi6q+&|#srNx`fm}KShB18bve&=}*PY>0oYV*JwiUc=0 zN<7Q8+>+g(4MrSl$V?UtyC_#Se&#SwCR?`O$fDAi#V1$4`OE!}psekBVtoFUrwD%i zs2l&dn1%nafg54;0f&+@CHgmDp z0(I)QZR9R1F8^Pt=4IOhrDXDq9iXUTJs4v-RVU}RrD>n3`Go z`}~1-CBl7#^0lw1M=z`W>_uu!Z4!(EJ^#%=Ej_qPno{D<>&Y-N?{oBZ{>%k3M)xJh zfTItQj|T8#=B{bVj^szB6jbAIf}vAd&*HMn-`L*4h<;3|%{cqJ{H3GpKJa}X;K>}S z4~E_o#E5t2$3!dSmgB8SwkOC_b-L z=7vBLE+AJDhXPfJiYQ3|;KL0Kk|G!S^DrHNIY0`-V9+`9I~dHDu_3aykthJDK%qr| zckTR2iORQtR*u+Pm=}V*8Hu%Hq7&0D84D}DNxOe72b)U4fH$VPYQ^uO1lcCZh36FO zs2r6H@@B`yO7~^YMXgdigE;cU@#iGk1)9vuq1G<=LRNxVpH$+1SdEw3T&8BZ(RlzL z)N6WSY8sEy@e)U?0;l3E%3+nIrE{xU{I6I3t@gqTy3n3a!ZEkK?&j_Tp8yRJ!`pX2 z|FZ3)&UeDi3ae@~!%!dmdc*h=WsUQ@8{50ap{AHYH4|B!hHRJ7Cv9r!Luu#!Y-th# zjE)ta5r)#BId54-hc+o8Nw8M;>Atdxb00CoY9jH z$qd&Z_q?)16<((0w?emShq7Oo7KA685>(B%)*OCtFNpkwdl6?{UE3j0h@pg;mpq5*1m1AKw zqi6EfC_}VNZvkPA=EK?*WHF&C}-%oUssHX5FocEapkEWWW58` z5BPC`2^fl{R-`Ws>63KZi+^;cv#nnW-!a(3n8S-TgaO`NfByRAI#f`OZWBkm;2_{k zV-iku3_Uln!1sgUCTCIaInN##uK6c8o(-%ngp;D-otbV*T` zu%dhwTG6`>(6x;xn5b=CY)R zvp2KEU`546FrTlyQ<`i%*niRNT!jWaxY0)G2A$@H%~9fMGR$@uj)HkFJWgtV7N>R* zN0F?P-%HIyz}a>Xo(9_@vos_K6VisnJ{T)Jz5zO6F2j? zzSNNmBUZSL!5;gIR~+Kv-wq7!DzoC)-fDsq{3q0OAR5|tG~qq0$78jruvjOsQ0}n$ z>o_urZF{QamKuytZ1D6F_QC~sF~}yRe}&>mA3;C|Mc|H~&#gLB;i`*&A3`0P42SjeV`!)`@}kNWS{%gz3nAjCnuEfd2gM zW~D9FZ1?jK39#txZv4XX&i7V3n(`g<+d&7r>8vM=EX-W{_lYaURf9u{U4(nn`qBKB zIanr#xT473cFIbR3ZnCFJ30Y;eQ8TmNBiOjfyIOM#-gILGWZa`xHHzcF|3*G!kkTN z-!9#lW&O&{ww(1JK%=fC2R_A<)BWHmsr2)ctdp5<|FgNRz|~czh5)CcANcxU+9UG! zKXZOqnAO(Jt&?0xxdi&HBVhgq?eT&$gXlrGEgt^hUqO*quH(k^wjn}ylQ%X3QIRIS zC*n6OjWLlWS5NXmT2n$X@8r$CGtM;jTmr{cD7*Xs-aTEK&2JTi)PQH zkJL+^Eh;qvRXiHxo_JoakP#UO=GFwH1`MMFT?}7BlcS29d_t;tM`c-H2xsD;;c==C zZs2AKoU8=6;&Mj)E2U(6%V1_xUA=1Y!A(|}X|k1Hb}=%%8hG5)a*eUF&(7FEA^$-@&gsA(V=r^?!r=x{ z(J9AERE3<0nWaUHEZ5Q9#_pa5H_xDO5lv_w)cQj`E<2`n5Ux*grv^ zWlkfdOpQfMF5hg~J}`s>@XIETrn)$^eQH<2mNa~3@@PVI^+nPZw{QWiQs`$Q8*7>{ zPL~u~m>_qg!3Q`vwn}z5)sptB;3T~Q550?o>kWI6o0MF4Sn%0T7@yZXhXh9|Evl1j zvbg$CV1>LK+9_r>zNfwZJb^;HDz%Nd#({5OLa=2teYNp$4O(_n>Vefl zK_LKm&fWaR1?l*`z4JpbGT^+lLemGDX0V>pxUJRu4W@U>c@vLp2s};J|Gj1KB5~r) zA7_26^|3hzMUOg~+ewnMTC(|*MNY%o%=%*xzqW3^-P4=UQR7Jap6;`(lJ6^r>bp7V z)n7D%io8674ukL;MaHvS>(BY&(l0p23se-YHn?8R907eAft90W z=HP~6GH*Se;xJIzc6*^)BBG7%iOj;em5MVyZP#-i4>cGGltLu~u`ndR%%#gzT)j1g zd_9>5-Npr<JvMwpPn_y;raK^M?lmVEY69nDR;ombu-?0`0_+_0BY!Ih%daX{JuT+)Yga{P} zg>q;b9#qS0P4y|8k=#HJI?o0rhaJ(yDn*D6XwLjw^G!#hK+8#_;3a?PuWYb#LphzX zK;R&Nk6o9;BM2hg>*>$uyt$y6N|{wB+1@)pumkUOq)$Zu;LHi?#Ki6WIeZit@y|c~ zyrVYaR>q>F*O!&s&jt#Ir|=*3zw85g>7;W($^Yb)-w{hAgA#lGMsrjW6X(fsO?qVJ zp?$#N>704yh18a&;P7qQ!{pXH*8O!C9ALjmx>TDRK=UTPc!!H$5 zTP^}EoVFbPkPrS_^Cc)4|C!*g9F~y7_Iyf^P@=Rte=<; zIjsCDUz9xOXq1_y$=oCf!MNm?}9hohTC-LBaOK47Gtb4G`D z8G|kd{v;UN*?+!ut#DX=bEG}udsP^*b($5pFv_&APEwNk@-ItHy2wthzld=eZ9#(u z3kBeHb7wAze@N88T=;b<~XPWlezamdi719fSy@Wy5VNv2Jtc5dI(C(J#j(B0H}(N~MtD@=LTRP6iQLz{S!_UUb&GS7Aj$!fp>FOeXyhBC z?On$ho2OW>I1`k==$y54wDhz{wXP04XfAQxfqXwo6%T8&1CQTtY-C`hC;J)1GHi0D Sse&CiQ+RTS)_Y=q`u_k!0j`Vy literal 18238 zcmb5VRajfk7cQI-+=Dy8ixwxi1qsEiMSnO1cPq5GyESOh;1mg3+T!jWq{RyqC|W9% z9{%UzeAnOF&)yfapS5P)tasMT`_8|$f7<|ZEp@m$00;yEG#?+pzYTyY00)GPjSa$i z{NUi=;NlVE<2@P~5fK3~n2dq~Oa=y1(lF6dQZZ12!E|hN49v`|tgMu@?40Z@oJ=gN zEdL7ve00Ub#UsVXCuN}mQ?dO2wtsy9Fg}n7%K-#r2VjALATaRX5P$&y06iuP1pGe( zVu7#$IQWnM6vzQU5EcmcF>P!tAT|gH06sc`*eFDl*+mU(eKN5rBg;6%R1EF1TKac< zqvkJEji@-q%P;Mt2NoXv>HlwF(Ep1J_@6%r8|Q!1g8w(?W5oZ@fM7NeWvqYe0OH5t z$7#R-MZj-6JTiY9Z~0Z+2Qq&pR9Op8Meoo~Tq;rKtSEkl_E3tu zCM=B{oR-z5Uw`_JYl%ol9qWg4hL^5pGP6&Xaz}Gvz%!&qN_iet7@9s1g4FTs1rcyp^&2O1O0JGrrK!FN<9$ zFaN_y?{ufh+PM3#bOK!}lwQ%FTrC@6@S*hL$gzbk@vk&_6TzV$=9zSvcPx`)g%Z`E zw9nPZ=kj`vMTrfHVN!y#1WYKUF zj4(lda${I45pXf3jR)GbdcD}^(3KblfRCV(EVb~COmu5+WNPHI7FXXir>o9V{x%&h(ax_sg_y%QzJ%}NlS&ks%Lf4jKsQ;u@iEV@l;JHO;12~U$u zS8INEphU1F>2A?L_7C%!<@oa5kGCwly%>COoL_!2jz`37iim*OwT)OV>Sz4}G;A2-3kO>J>M0ck4hhyxomHl^Z!F`Ql|O5-8UaRe zh~M(bA+I*cI3s87UzM2vUV604T~t5ye3GYzh96Rv2)q+C9$^-?Ngt>+NEQ3h;-~g( z7fU=U+6^~rHJ>A^#y=6X<*J!rZnL3d8o1z3jyR@o^x_u1GhhjsC7e!TJ5cq|v*tsa zn=^TX2flrjqj%F;=ItpfveDqu%>OTOM$%6!)LpFiwB5>Bc3M^_6;g`99t(UXJ*kCT z6#duMU-tVaSgK#-R)}&F=il(`T5ay2(`aR^a~uUcnEemS71L!hnq@~DLF{SyElQY}xo`%f_Ce-h{r1KsFmBDo`OeTCuq)R}Kp$Xyk$p%c zj6Zh+moU?da$7i|)BbMeOAYq7gJUlPtKDMwB-=S4A}iSYSfK%Qt~&$Dg_PeOsFL?s z%q@Z>mAfeWb&FpNotm)5SZYYEDN~Aa?z;imra+p#hk@>;Pe>=>LSLAP6ec9? znF4Xe3{5glCE?9@wwet@^s>6O@7%R#TIv755ey5OJXj+?3b=9%baafY771!*R z4A>7Xw{5y~cPm=3DaIZsME0rEw51{LUlcT8++weVQ2Ra>Ttm$V5zuzuPwbPw2NqpYX%ogL^;4(E1nQAGG5_@~6MmV_F zRp2+(gViWH57=?pVG=V-EKB1iD%y<8FJBD4_LGdOT?m#O!aU^a{~90h+Zq#8j$#Hd zIeh*BqriVtf~in?Gd`qwNUbE7SGmLUVpxhkEy+&|z2j4D5`^v|XJ*=Yn)Ce+{y2@R z@x7G3 zwHUZvl7P~uOMVv}p-rz3kJC*oOi0mz{N|>c(j*vmF+@Zs0K+MFfV!**Q1QHrzX#D3 zFCIYGRl!1(^v(q9cS9esMUh7$)!;AfD3QWz9o-TT+fYqQ^I*#>Y}7>s34CB1A2w2S z*IT%tkjQ6;>)mSssoe!U-SX*(6_kHN@+R3e8H^+lOI+ElnFk4nR+32%zURZ!EXt1R z8r`wfL92(vDDp+Zh3%+BvTvh32}h&__BVnVnZY;ny@N4l1nS9~K2|6w=XlIy=zte7 zC^-gw1M974G|CA@*#7a8>Y2{1Sag^kTNFR&NYOQ_V4IJ_ zqcE8@sTnt41M}2BxHJ7Lryj*oWm#tSM6L z3(>Db0IKlagp|@CB_eIf<7hqH(!9zLPYBS%*DOt!QcsAPrXV6ow@s~$58Qf&a?J5l zQzL(&Hz@jgqB9!&T=Jm0Jc9!mt)0lKchR_mAE2V_e&Jdkl%xn=4MtnsTJpQk;3=h& zUqX9#chdSCmt_%hx>(3%?)US~`cnPPKN_#mmK7IsgymP;W zHV!bFmDjFQ%$Ikc16$1pNnf`jOHU20Pru&&1Mm$m$};J)Tj@tEawb32-is9M?HaTD za@6@K_pv58aoiYJ!Q$g--`@QLptqNL)wq+|I-KBX$7+&sNRMX;t)0nEGL4=~Ok+y= z&R>jrglk6ax=Cr5`1GU#4X^S6OzMF46wlD!*RPn@DzB7Q) zo~<*h)q+K)qSp3OMW58!lar(>evLs691*Yg)e?H7WF-W|-@^?>szDv}&{yY0%W}dj z!6bvW!<22POOJw7;RWej84qp(s{YJOlS|lcqC4!2)2*r$x~P+F78kGX|2;o9usw<> zTD5gr6YNa{d2ywS%R5AWk7B4n4SLJZUu~S`b}4~rw+*nt2zkOenr3kTK}7ViomfIN z>-UqlJ+WpZftDWCb$oFFwDy_$L;HrU*~3cOP?DDr%dkm?Old^9xrC+lmn2POl-M{E zO}s;!AtTEp9K6jmLmZr#oTQ{z>&b-WQPDMD+B^je`LSg2!W zUAVnN#^&fx9VPP1X0;5LWgqHF;cTYogz3`F$rWaix1Jm7zVX@G;TVo5QpjH^TtUch zUV)kjf{d>*Gbmz@E_2FUcc#~;Lf6=%r`?T%x0K7`X>4ZtvVTlNG6K;uf;cNv0%rjg z^kYZ$#6pl4@L+|+8n{qA$juC}t^6ioVZ6qxFd>bIvbEUMct}LLx(-_zmtNk;hKmkJdctlDyvCA!{0dt~6?;CmK zTNj9u@=$Ux zyU*i2&Jv_1E*cx>h5Ks;wN7lla;I6EdidtV@5M67?|HUf(&JQ+s10pjU=hR(r~GO9 zX~wPW0X0rHANz&m&!@etcd>3^J5-j2ihx5?XyHXa;yyZ~og@f*kzIC*p)*@mg@s}(bvE3XM|4c?(#A!q> zNMDH^*Q3hwySpw}TfM6Y;5*ByJN6SPp>sTCL2fy(fgSPLF(@h8kh?@>W*RrFEi~NL zZH1aWXPl%vSIJCTGPD+;>r}J>yM&PQ&Yq%mg*78&Q6wbfjG(#<02r$z%mj8KF^A#_ zmDXx=D1O5j)M7fB9>6yM$8cI$YyLh~0blmVr@n)sQ&v3ZrC+2ajw$#LUhigQe-y*z z@#iQXt*ZScuykkoX1{Im*Q@H==Veifx1@^rmYZ)6T&s1qvAmO&0Rn?xv%*@?UQ*_& z9S*&g>l2Zt!qL!({Tc?FFvCg$lBmCaB-*3qB0XQB7DyA9Xz1bi2bByN``3;J%TL-n zNoLDaGFG3xMRu7s*v7m3=B0;hVUq9Ebc<8(0&Yyode6#3f7epBVD~sH(CF=qXL080 zu4FZq*o+$R7W~-Zb`rBP*=hRFU6$g2|CLSi%+EpXK(RbcGX6jORW7BkP$o_@!-`Pq zb%&P~uZUV$2y_+dp>DB*&~MAuA=D5zyQq2m78P7_=8N5R4z)@X8t;?hclEgDJW8oN zOG4`|AhjYkBZc+sO`(S&E5 z;#pQJl=`A)S9?q}&W_6rIx6&WgZf>D-eBUAgttXI?y=cSR|G@)>!}GO+BiB&R-h?KaKHwmM5~I@)>=iXaP|-Djy$12wGnv;PfWuUcKkCALBF2rC(0^E2LL(4hedCIyQ}K)-HE$0HwP zkSKbzh-VVrvglS&;Uq+St0}V1oAtWZAmR(9-V@#k+z^&;9609_cB3Z;%WeR&$~&{d zc+x%GarD)~NgYT8H=pVkyx63d`)4z9abBm?MtB9p(d1e}v5Zg0E`9?$ooiZ^X&H>@ zqff3lSi0(?1C|if{VH)6!!<_@8olA_u+mA(vS!fDXrP;O@1u&SA-R|c4tpVCKqu1D z1?~iGc~9*AeeNtOxXe2TWjQf1Cy?b_bFDvmS+j!Oy+I*Uajeo6W_S9Ob{6)L?hI{T z8_q;2{fTOliTIG#pxr6ztH0R|TG7AZZ0nX^^-gPMdhV?%<*SfiHgCW~hwY--7kz>> zAG#azjeT%4*hYRgiWhwaT>?8&58JTL5P$lPip0MesKx9QIs6w@VU_HlaM|zvmf$VY zXewM8;%RtgE8i7UZjyq%ds^)>jY&(&eP!0`Z-n%`Zo{u>YqR@qowOum<@@@NA!pq8 zE_#{&xXc-6;%Ng6{YsZ2Lf9gvi)6X#Y|l5?F)D)VH2(SBv7xC^niHFuHh#exoFF~w z{&z(i7%MKGEbL+|(|D4jn1Jyo&RG$4yGJ{3pE5OoB{d+-kF!8#-;eRc;IhY-0iV-Mt<--dW4M#(o2pKxrpodD*#4_^XCk@Bk+5lTaj=auGBh(g4$6UYJS z6;S&pZr#UhTRCF+Z3W?4N_@arr{Idv3trI*OB>jYo~NK{eL>H?IeOf!&vDX%)1_2* z<`+F8H@BYhAcBceXo}a4y@(;^OFXB6+Qc)8n*V>mohRM&%Pj?$LjMtO^%Hw@85YSWi~OweN1o`__yq%@=gHmQPv*oI)yH zOPV;!r&b1o>=**NT2~l7_Q=b5=UMd=c10uVsnk-T)^i)0tNF$ksqB?I(M+K{RL2Ax zl%k&XI~(vOwlwV9TwCW$?SfANpPtsy8dYlpz6ztG@3?Dq-d&OnVF36a7A13q0VKjj z42P{nO1Na=XVuy+8isGiQD5PRlDfD;d;dD69RL6W?kksJ`4$byVmOwYkM6rOPZ`zV zzcp*M#n@mZx&4XYW5{z;+b$bX;o=Zd7Zj9&!F|rbAtd)doNH2*)pRuoJ}L-j7sl|47t|`%S8M4ENhMdXg2#Ji3EE`t@ADyObdKy zyqmF+W|4vI>vDkNM)n80Y)w|nP(F#iaM<{EhcP`#sR1q~rfcIN#(W!=D$Rvss7g$C zSh))+(w%asH$G*qIYRk{Cqfb;UucG8yefuy`G|2ge{Ba$>_!^8yxeS+y z_ZYC-qy^o?Czjx6;b_1!=~V*zkuL4I35^hOyTsi>N?rXl#`74P@UkaBFV1lJRi!`r zlJlB+YBb*+Hhj3v&g&Ks*CRiw|7hWXUArq5`u5)BTgQLUq74VCCb5@IQ2uXdTb-e?;M3aJ zMPD{0TDxB(Ut+EEX-T2!bjG@D_ZArZ;HeJE(^M>MQuvTYYP3k!l%}9;y+w0>Q4YL} z21vs$7R248CqCUr-fhpILBOFq7DtaHR6a=U_Ip?GTAYafG&ABy_Zfgm*vox^;b{W( zQ?VyMGem;n9sifp=NcSh#c^=8yv%EdS2itr4 zu?Cr}%W_BS0i|m(?wc}-nKdbHVmlu=v9k=}an#7?Y*nn~wbjYOCAih5>u1EpSm_pS zHBh2fuaF9riG#BU3ve%rXj^^}Fb5Hj=&l@D0?9}Vi}}beB^oZV?;5Q|aD!sop! z>DX&88+8Y`stjn#1e>lH76R3S{_4A(*tZ$wrL`XJbM9088$rlGY#2(P{{6fik zdQ!LMW)Zd=5U1Ky+xHPJc=&NpZ_Ds$))E%5C_D0J_=4J%?qf+W6#xBrG`f>+@@4&H zQ~JQmTA9Vn$C_p(KfPFXNz>?=HK?AF1u1q@13fpY;;sCq0{}Sc5vyr6-EVjZxPP(K z1p82TT1yw@g$t5y@k1Gb|10m2-DP5>BCqvI$@B%ZAzh(>H+!(asjRM;o?_>1>j&C< z=n#f`2%m`Jis8j#MGSY}t6K&WZh%kzACqX;@P|p?V=)*}kHKji?eN>v0%8?4E+Z0d zJ1FnW_%Y1JYC69CPx8Hoheqte(CtmC#%6d9gtzBJdT3bU}t<1N$E-_-Ynh`Q@#%s&)U$p8k7%T{HqIo2 zZD@AUwf*g1d-hJU3C5bQdu&C#w~g_x1@?4p^IG> zQriEL1e=9d9o`Q{xsJQVTV^t_lG~xYCSj@mogu$YsJ~nXj8jhi;dF&2ax)o_ig5in zs&BIO_hZgMM_JkYg>nNTvC&QUHG%5n-Wb_3C-xERwKO6)BagqpB8jZM5Pt16mQ*3e zS3q#^VkKLw!fVfpARgTFzpTjJ;Lul=?U!mDSS@|{BW>1Ayqly>c^>d-x zh_mZ=0_o?b?tOVZp*0;9@3`8$>VFyAa`q=Awq-Stli>yFydpjCBGo;AKX+#V_&aK$ zn%_FLlEFem|GxGRZL%5gF*|?1f>DSLkqq>kcB&zb8e5w+8hZqx?8KosI0L;w25%(Q zc!^x@JtxJkux^)$tdNhJt1LN|o%A76K6VbD^IGc%tx1ZY=uSxpwX#lW0t(yAh0Zz4 zXL0KZ?be<4|LQ)5#QRUv!{U}VQ+mMTl5WWCE5BsT_0`#0p8DmTp})N({pC?ondMgN zGak8g<0qngpxEvI#FhiWsT)j>ENTRM4Q^q4L#qDsz+-*Jr1LF4khi}MacY2~aK9(H z&FSpvJ6$4mTB1TG;WqKjBKH=l!Fxd}$LWa6#=cl2tvx=LwU`dx!?@;(!WRl?T{AhP zi=va7{-Z79CDFSfp<65`uT=Z>ti0wKEZu`QRi_{}$8q`rS; z=u=?l%5)7h8q92OS4fC?=hi~U{!3H2BGEI9R_?9v8EFjrVB(1P`xu^iEGrLGv_CwI z8Fk?BQ(3l*2@=jXAQ%O0iV;xA{jzQx|0&=}McT;qsEO)aELj5`{iTUIdyW!^(356E z1o?TCabae22K39V9&B(ITQ^O;+H_QEkP=};R|<@n?&fJyEq3P?!@*@Pb5fFyv0fq{ zRT}CqNlF=xcTwIZq`;(aNtK=Fvj-ijDAxMmzN-3?ZzA*b_6hh>Xz)X=IWYS63C+FE zVo8+~GFxq?pJulpw+@p=ng2vwI|YwDl{-w|JCmWdiofXTF!qL0j$B5drA+K+j+ZBs zu$+ygpCuegjR&ms`F}GdaDx;Q-uV5{k=ww)6-7QBX%i>5icPV}t(O5u9roKKu1M!j z(eHzVThW=aK}T)R-rc5HV^2H z#)DK^0h@nEe1R{!)N6L(*V3@DmF|HJ`iQ?;RB_&!eWJeeBEB{c6PHQke4{KK^RCzB+FX3{Nrts`a4vVKGdT%XQ8AmWCMm{|e+Z*h80oJA5qhUBy%-OK@G>CZs1rJE zdOb{Z;EpUy%jF!4^kql3jf>r5B3ACSHTov+<>n*ifpOAQ48+7bVq>WEf%6m1m1vzG z91e%>ZOwGv-Y(6nZ%xXZ3v^q?4!=26t{@>c*gpQ@TIDqCIol`#85S#(dyc>&+tLfX ze0{Nl{y|FQYp&N=kdM>M)Ur=jC?9vY_(hlM2}W*-n9AAd$-0y2g70f6hoLvG`IZ<~ zq*nkXI7aAF)ToT*0DR^?Te6S=ZjaAXK+GY{vus4^)c4?sdcx>({;el^%bx|WN7$ZY z3xVUd_-b~+k6+e7JZi%Je+i<0d19s5JGu#A_e9>5=Nol_)~7bi&CUnA{JD)l4P*n)3|%qc~^T&o7g&9MhJ;xO!t zZV7X|33!6iZ1b3g5~t2@$yG!tE);(9M=CRRU+Wybtd-SyStA#w+A7s`V@{@0dU;7Y zCbe{CP zKg97l&{C#W|9Ktj1SWw#oW>AmwM+$Qy2T#iQagDI{fNy673vewWc3$%mzXa8l&$yZer-kr_?2dZM*kYzE9hbQ|7>B;J z^2Q7J&?oA^n-I{Hs7g4=7w>QLhKX+&v&yypZ)tPcLQ^;LRK=Hh=;;b%VyCg`**fnu z%8B4912)@Y>>*UqDIK(W+!h7?^$EV`5gdPc0h!q^Q6%(nn-_*6G)Va+g@yviNF-eN5AugZl`0TE5O%ikribhz;W}zc__utQ}?( zKy(35v#RWV6m~ZU$3%3kn>9V-gv9TX*zWjGhpRy1QXU$gTe4PO=(Rxl}Bw>Tdij=A6%GhqV z4~yPyMs{zO$;?y==DU|1*nLmPPUPR%aTsB=^QuI5N!dA~{)1$&R}AUahAI2_xnJAo znbj)eWUNPFKwkO*$>iq6*Zpi~{0tYCS#f?yyMB8DuoVvHu_Tn)nQcVP+Ct*c%%8#clj&cv*58Uh# zm}!UM{#>LH!Vm5tySC+8#g*qWSV@g?*@!%k{mF9l4`2(U`>p5QL0Qk89f9CXNi?$O z!_v#z9MO1d5*kOR37~7?Q>juR!Eo@GAC`xhfRwHZw2yYhnU?PvEfGp2gedHjv6>?{=q3t@7o`05=~Y(E&p3vDnK#@vh>RZ{`}lYgTCthZ zM#Vx6pi*~ne_rzm-6#h9?5uZX=SVHk6$v>W)rJbTMcp2{DxQA)K)udlzQE@qKJA240Eem${R3##U~6X> z*FMu->`&9`+F%H@h}p2r{KPdl@9%5G6<&y>h2MM~ywTHlKg3xeA}ZE6ssN5KNLEOg zt2^o(ZfyE7Q_hVI@ft3P=x=>21HcY3*aM+yo}!Y@X7E;mi1CwHkW=3dIkbGfNTMXx z2c+IE##Vg{%G9M*p2UaQ^(O4#C_wyGX}NfU&P(Z3@36I}8nI_$Va9<^FtbT@+D6_P zFGOROFqfRIki+*=HU0?n8VYr{7Y~ruf-3Ja4IJfdBp4#;MTyt2Z_T%lxiWMrx$FtL z`kOT0qYN?}GCq$d7luJ)o36IFZK5BH2zBU%wS1$78|ED4)Z=^Dnj!ei&owxld9WvS_* zM;Q|-{dg!2h>jUMe76hIr2Jiji&h?dv#WNh3}CGqm@jzE_15oCr8i?$d`~^xO$m$5 zklfyUl-7d+#x2A|E3uy~D5!2Rh&`X!-I4w>jrf-rqf658 zI;S$@F07)%PkSU{*ESEZq5Aqji;wPE#U!{gSG}W@oDQ&XqV2b7Ta_jU%GD^udLex# zs3b&&HT}u>bc=$x=se?BCx(9gH+$coU3Y(egEywp@IEP#+SVDRzEf?di~JzV^LNH! zJnm6~y+;QAG+o1Tq6+ojrm1Dg27uF5Vt2kwD{pb^>MX#KAFJSW>Vt)`|?Gx5&m6fKT3{vTo|i3qOqL0*ncB0`Vx0* z{HN82|5Ut8moWS;ausL+mac%G2dFU3vHS9=o?vM4PNof03R*45i3N^KQO9PZ6RZpG zjQnr(y(XC{UkW_16c)|>LSSoRJ{_S6y|9MJU?p^Rm5a_}Qzs2LohqDIbdOR%GT2VX z7zCSSo;PBjCfDB$NDk=5nIB`vgOqB$?~V)*fqG|4BPPqMlv_n<>1+SWP8T9_bL>;d zA<=%-7qzfxVP!*3CAMk$<9WlPoj|%`fdsfw_^NZASFALtbm_$@tVgEFMrI&$gy8G4 zzwo;JQXkuP#Ks|9hnk&;A>62DTioGko*R1XPLtse69-4rv-HBGKHHN@zonW8pM9L4 zD4`l2r;rJ%b+Iszv{oI5J4r;#YL22}oe0Qlp89fQawdpu?liWK)(c=VQMWB#TojVsDfQ)B@|k<&&SZN; zrSDXf=k9cLz4y;gNYjNe-6sb@&e`u9&b@|;K=M}ePBltGf$H6|ztp2&J%KVvPvBI8 zp^WdAmji&8461-s)hd$uT`wBEOg`mQauSi5-+VvIa$7b|#Y(;A>BqSm-eRKT55LIRoN*A&62#Apj zTeMti8<9q2Bzz7aRt*c~m+lS=(eScxMK0DH`>BcID~X&73A!$7+4YvU0!wpJ2sIQ~LC5VUh-0E-Bc`ovy;|Fi+{MO1iKUvJ- z_OJ{^@gEBm%pA!&$|KBxH&l1bIczo!i#7`fKcEB~_G{xV1lq%2JG zt741&y_K;|NOlY;`TVc};&WbZP6F@ap`|iZ;-DErN>eQMetMH|7s@DoG;Cn}sS+Pg36B#w{W|duI(#$5ns#MN;W|GobrKc8`mjB_1Q zma=?3AR%bnOQ{Ewu+gE6jbtmxfYWs6u8APZ4r3t!(?5yPv{9&Q87tisbSKWl z_0=J@mDlI~r%-`DqBR44A-yi<6RHU@hUd}*>ef1=JS>UlPNbze;t;!MqVzVA#p4g#l zmiAuGk6%0k$e4ECE%>m|&ERd(ItNN*rq$`3b@1_%&)~UanZOS&yA-WCE2ijG#^U!( zNK5Q}rdr2})JB|Sol%}OoGa~LX!wyvA}u!04<*1nLgd>FCZ(Cit0ekgm$xaAry zrI0M#2W(bt9|yyDyECE3nK^L?j8)UccU&A%Ay38FV)RxVXw8!3hsSSjH%o@&aNfex zD)Gf|aO)JH7^ljJjOI^64x$H$sAN)0@VaPUwC`xi@N&_n^8OOEB}u^e@n{DRCXD^V zu!+40H)X_WQo(;_r)%oUysj1L^x05MQ}J@rZoVl@aX>)xqpCRof$Jgn$OEuvCjvX? zW+FLKiL@_!|0fN2UBGr;lxyx*&DM|sd>!WwOJj|?U}h8%E>QCWZLb*)Rr9AjO7{s z8dv2a$mOr)$|t<5)UH=aH%oT>x-=wZzTX~dFPAV7$^C@k4HLf9pYc1bU!OalH^QUA zxJUq_atU8bcX};{!z;(~LD82rN$v^2`DG|-N+f1riKi2*u*cfs*}Lf{U-FemKFrpq z%Pn_&qDa7-+LWH26-~+>fTRH2H(Y}s?|59Nj4j^t?f~NuYOTYMauZ+^1Rv~nV|LSM z4F-0JqoBHjq)GwRk3kk^O!Sw}UCB7Caq>@4}=Kh>sX0kCf16AJ|>sizGzw(BR2% z2=Q07WZMOo?~ZddtJmrhVX9OA0X}PLxFLY-Pwm4NBL*A@ zg+l|78mhB;3X1(`+-K~on|b+UkhY!b!W*H@NZ0BoB(%jS8M zy&ERz?hjki#^d_|>beo13y8w1vWlPhFmmQdYfaXd* zeJp^-KLEi$0RFEWrC%NictYV6dA4ptXe@B#k9L@8rMUa+-t6}2noVsN$TwaGIT&Gx zI05anlH4z|MQm@6)}uzVnC3W%HV(P05EC_MlCN&`8#N=~PzKl3*cU;QUY4g^i;vAK zeCdRy)3!3@YWLmFU?D7C9K@GYfiwFktMC=sC1oR_5q7%IV4DtRn*u{C*^+7`!=;Fd zu8fJ=W|Pr+#(PtL{QrE)m3b8s-ReLejS+u7YD45f;fYS$e4*^8Dbo-82M7v)UP9CX z-@YcAB7JP0wsm3EjPpkz$YGl=kjqX3a^>D+(?3mn(Ca^T0u9CZUg03n66!}^`tmYgwrly>1nrS>g@js{q*Gid4b?=m84 z?us|UIW44CI@@4$tnkpXBw73Yc3*yx)-B(v%!X)aop_O`YCIPWv64GeW7smN2pd-h z7oJo5D|7Kglj&&FSM)eV$b88?eW?`p^Ukc%%E?~sX`9AS3f{|fV`H2OP-J!W*1S}# z3z=}Aq~9=KusUs+?1pvXJ?EZ}Al9aH)NEDg`&egeLP$~cYsb*{X-93@{{X0+)&lyY z4^<<(Izyd5pI7>*7Mbse+1?A>SkLWRA9j~I#{G77tTZkcBnCfRXiT)I7rfA9d$U^` zTj_}(E7_CXI-phRr$_qDhq;2F8Ou-oo&p^u_#1Wezn;VrFi6Sw%Kl>9@AX^>DsR=W z@@zBz?bf(D(}($9E_oh1+YxgZFMR~atVHQrM~O_ZFD|ly3c=qTf4cLP%NqcWk?MI) z{;U9``L>%$a2O)#%WWKwKMZ{VUkOmhTf@xCKbI(lswzhg(?qkIl>`|YQMVDb_-L+t zCXo%$Ao3;QRk7-a8P}$f(v|Q%-qXM*1XfSKxA;(0UP0JSeEx#jQ9>sDDPTKI-Pj@% zW82AkAi*!+y7G0Di%SkR)|`@(%Wf@hb36Tfh!R39(ojn)%g(ED%l9J(f$IZiP&UTN ze%mv142WPvK5l>}Tg1uxq^Y&?xZVc!K-JpL-+%(_yi~U|@tKxNp6qeWi@mHH<=Aqn z(<`tibwLl>ma|Hh)U==-Y!eqwlsxE&SDcp_lxVRSHgEW4ydU^}!az#{iHwXXOwqZ$ z`bCm%JxBxM!Vr(>l7)A!th*F;NRAYghvZOqTxi*)suINTeZ)2*2eg*v#Y-uOT3Z@g~GvY`KJSz&J<%6`4!5@fwipT2K;Fpj(R>JrXIOu9=$gouXlor$QmJZH%% zE2VW(8#(iui-pqjetw|QeWS;3ip>EuGQI?ZWp-28{l1p2{4^E<2H4 z`4IgRN@YbNNSUcuD`ixWrl1^fEi_GI1WoT_AGKEV@fw1*I+;% zL>+cftBe$89dIvrJg_NlHf)F!o8i=0u86B&rtwx6zGqWRq-XUts?I-cv)NE2!92>9 z$~CC*aTZC11LO^EA{)H1e>n%G!#;zX_XGy)d2>?SyqV*H?ti7ZFFHcjygQudofl2( z0=1^T1^d0OmMCx9G28MY)gGB}lb|uIQ|Hbj-x0O&MT?ei&@48#sEQU6nP#ZJ4>OHzWz?o=O}m$_PsQei=x<*$&4W>d9n~`Qh`tV zr;6WP>AfGGHM}MvI1ayL%=}#fEn;9ng95c;1b7pt$6h#;$l+LBYXBLtt!7=mub->+ z;1I==MY$o`7Z`whPYB&{muCHk%Q@EE99zVK`6QC#{-k(48IfV>0fZ4D z)q&3GYBqSFE$7aAv`C;yuS5q*vR^yPI8lD|_nK)?Z z^Xe~o4R^&4SDE`z2!q_Q_=%TO<#r!&hzT>t{gru`l@f0pu~MHH{p;pd+^-jPM4$hL z2$rz7z1ZYrYK!HzH8pb{G3zXFG8r@_g3B&9Ry!P`7zM&Tv{`n#%X) zgSL_r$dBlmh5mOaxCqdeRGMroP3eRnys7&YWo`$p!uQY>oQw*L2L?t3@4p8Z%a%Xu z4$@yRvEN*?H^+z67Dzl}SPcpJO7EDP{oF4aCQHDhW32O>(9V@Ui~Alcack#{(O0}(DICX8GSDsmf z(C4z5)>bEjoP0~Px@`F*m9VnMm?U}@x8oQ%Z3I86!ItX)>=~lsC-0PpZ>k0fw(|?x zvHA})$>a{lqs=Wnj8cE$NWLdA+@5IaY2%}m<9*{T03va+vKWur z?#r5FToA27dY;QLZ^Xrc+5Z5QqmBKRVK~Oav&{PPMNR@5ejA{)=6NQf&?nFtfS+lLt1nL0(Oy! zs@qcnbSpa2zArJd9c@e~hBh{s(QL6Rc%TcLveVSUh7d?4jmrC9%BYHh@&*=yBHKul zQr{2VAaphw2TV!ZP@I6D>Qtq~zBezzQE7BZX$Pv|h?6b1RNjv6w1D(I)?hZs!qPge zK926Q`EJKPSnjh@P=3~TT$=-hunzwK%H^I;m98bSNgBi>Iqe?Hu{v{H9O%tjv6by> zsuPBx+Mm%DhMK|HDg!($Zn)m7fy*EN0MJ5>>lx7h0RE6%!H^WHkN2Y0<7zZB1i7Zi z5FhUal9`MrQmNyf_S7M+AYmHyR6xp@T4uALh!!8^2Ry)cm~W;XL@73#oWj$@^=UW3@&JE0@m+#o>BNs+`Bwp`gF+d1dSYVf9f zF}qF9LK9S{PTLL3f_MRj1ST_jV!}hiaj+h3zonB?ksEZgy`o2MYIN#6y6&8_=(gZ& z?u@u26FVDF?1M;-pTc%x!~ybPn}DrNjDxZ=8Yc3o$v6gTkZ780)N47W0C7FnVH_7V zqn~oL0qtRE#7=XXiKXGoV-=_FYY58Db5sUUxGpNya5s27E^3ma% z)>4L<3nfN0wX!GuD@TM8Kt~l$+GQkTKRQda)oEog+4zS1PD>RRxz+CCZgpB-OG}`U zm+Gj>!Loi~?yHOO1(u87_Z+`)os}PnyGKQ-Q${bu5ykj|%jP~t(bx4>@PxUfxXF~2 zCi}r>7tzMYKUJk-1j|gEj=Y?Tgf+p3Ve@Mjb5G!;6SEfLp+sT^Z(;i@crEvpnRzRC zkCg6Wo9_fSKv9xY) zb!u3fX0ou&NWkp)^;(u7Nit5q^Ev(TMGt}juB)UKP59kr_-#x$Y z6`xAJDsixew2^z5ONyne-^`!HwXQPZCkmNkU1XpThF9<}R~L zO_r))H(#pgQu(bIS)OxsLe}|)%5A2>QKU6wt>Oa+0L1lB^4I`4T1os$^*|&@l`_`a z%BmDKx7c9UJySJ-q&P8>KN10qld=<"' chars.""" - self.assertEqual("""xx&<>"aa'""", compat.escape_html("""xx&<>"aa'""")) + self.assertEqual( + """xx&<>"aa'""", + compat.escape_html("""xx&<>"aa'"""), + ) diff --git a/lib/cherrypy/test/test_config.py b/lib/cherrypy/test/test_config.py index 059edc2..be17df9 100644 --- a/lib/cherrypy/test/test_config.py +++ b/lib/cherrypy/test/test_config.py @@ -8,7 +8,6 @@ import six import cherrypy -import cherrypy._cpcompat as compat from cherrypy.test import helper @@ -16,7 +15,8 @@ localDir = os.path.join(os.getcwd(), os.path.dirname(__file__)) -StringIOFromNative = lambda x: io.StringIO(six.text_type(x)) +def StringIOFromNative(x): + return io.StringIO(six.text_type(x)) def setup_server(): @@ -147,13 +147,13 @@ class ConfigTests(helper.CPWebCase): def testConfig(self): tests = [ - ('/', 'nex', 'None'), # noqa: E241 - ('/', 'foo', 'this'), # noqa: E241 - ('/', 'bar', 'that'), # noqa: E241 - ('/xyz', 'foo', 'this'), # noqa: E241 - ('/foo/', 'foo', 'this2'), # noqa: E241 - ('/foo/', 'bar', 'that'), # noqa: E241 - ('/foo/', 'bax', 'None'), # noqa: E241 + ('/', 'nex', 'None'), + ('/', 'foo', 'this'), + ('/', 'bar', 'that'), + ('/xyz', 'foo', 'this'), + ('/foo/', 'foo', 'this2'), + ('/foo/', 'bar', 'that'), + ('/foo/', 'bax', 'None'), ('/foo/bar', 'baz', "'that2'"), ('/foo/nex', 'baz', 'that2'), # If 'foo' == 'this', then the mount point '/another' leaks into @@ -240,7 +240,7 @@ def test_request_body_namespace(self): self.getPage('/plain', method='POST', headers=[ ('Content-Type', 'application/x-www-form-urlencoded'), ('Content-Length', '13')], - body=compat.ntob('\xff\xfex\x00=\xff\xfea\x00b\x00c\x00')) + body=b'\xff\xfex\x00=\xff\xfea\x00b\x00c\x00') self.assertBody('abc') diff --git a/lib/cherrypy/test/test_conn.py b/lib/cherrypy/test/test_conn.py index 3287245..7d60c6f 100644 --- a/lib/cherrypy/test/test_conn.py +++ b/lib/cherrypy/test/test_conn.py @@ -6,16 +6,16 @@ import time import six +from six.moves import urllib +from six.moves.http_client import BadStatusLine, HTTPConnection, NotConnected + import pytest +from cheroot.test import webtest + import cherrypy -from cherrypy._cpcompat import ( - HTTPConnection, HTTPSConnection, - NotConnected, BadStatusLine, - ntob, tonative, - urlopen, -) -from cherrypy.test import helper, webtest +from cherrypy._cpcompat import HTTPSConnection, ntob, tonative +from cherrypy.test import helper timeout = 1 @@ -304,7 +304,7 @@ def test_HTTP11_Timeout(self): conn = self.HTTP_CONN conn.auto_open = False conn.connect() - conn.send(ntob('GET /hello HTTP/1.1')) + conn.send(b'GET /hello HTTP/1.1') conn.send(('Host: %s' % self.HOST).encode('ascii')) # Wait for our socket timeout @@ -337,7 +337,7 @@ def test_HTTP11_Timeout_after_request(self): self.assertBody(str(timeout)) # Make a second request on the same socket - conn._output(ntob('GET /hello HTTP/1.1')) + conn._output(b'GET /hello HTTP/1.1') conn._output(ntob('Host: %s' % self.HOST, 'ascii')) conn._send_output() response = conn.response_class(conn.sock, method='GET') @@ -350,13 +350,13 @@ def test_HTTP11_Timeout_after_request(self): time.sleep(timeout * 2) # Make another request on the same socket, which should error - conn._output(ntob('GET /hello HTTP/1.1')) + conn._output(b'GET /hello HTTP/1.1') conn._output(ntob('Host: %s' % self.HOST, 'ascii')) conn._send_output() response = conn.response_class(conn.sock, method='GET') try: response.begin() - except: + except Exception: if not isinstance(sys.exc_info()[1], (socket.error, BadStatusLine)): self.fail("Writing to timed out socket didn't fail" @@ -383,13 +383,13 @@ def test_HTTP11_Timeout_after_request(self): # Make another request on the same socket, # but timeout on the headers - conn.send(ntob('GET /hello HTTP/1.1')) + conn.send(b'GET /hello HTTP/1.1') # Wait for our socket timeout time.sleep(timeout * 2) response = conn.response_class(conn.sock, method='GET') try: response.begin() - except: + except Exception: if not isinstance(sys.exc_info()[1], (socket.error, BadStatusLine)): self.fail("Writing to timed out socket didn't fail" @@ -431,7 +431,7 @@ def test_HTTP11_pipelining(self): for trial in range(5): # Put next request - conn._output(ntob('GET /hello HTTP/1.1')) + conn._output(b'GET /hello HTTP/1.1') conn._output(ntob('Host: %s' % self.HOST, 'ascii')) conn._send_output() @@ -439,21 +439,21 @@ def test_HTTP11_pipelining(self): response = conn.response_class(conn.sock, method='GET') # there is a bug in python3 regarding the buffering of # ``conn.sock``. Until that bug get's fixed we will - # monkey patch the ``reponse`` instance. + # monkey patch the ``response`` instance. # https://bugs.python.org/issue23377 if six.PY3: response.fp = conn.sock.makefile('rb', 0) response.begin() body = response.read(13) self.assertEqual(response.status, 200) - self.assertEqual(body, ntob('Hello, world!')) + self.assertEqual(body, b'Hello, world!') # Retrieve final response response = conn.response_class(conn.sock, method='GET') response.begin() body = response.read() self.assertEqual(response.status, 200) - self.assertEqual(body, ntob('Hello, world!')) + self.assertEqual(body, b'Hello, world!') conn.close() @@ -506,7 +506,7 @@ def test_100_Continue(self): break # ...send the body - body = ntob('I am a small file') + body = b'I am a small file' conn.send(body) # ...get the final response @@ -566,11 +566,11 @@ def test_readall_or_close(self): self.assertStatus(500) # Now try a working page with an Expect header... - conn._output(ntob('POST /upload HTTP/1.1')) + conn._output(b'POST /upload HTTP/1.1') conn._output(ntob('Host: %s' % self.HOST, 'ascii')) - conn._output(ntob('Content-Type: text/plain')) - conn._output(ntob('Content-Length: 17')) - conn._output(ntob('Expect: 100-continue')) + conn._output(b'Content-Type: text/plain') + conn._output(b'Content-Length: 17') + conn._output(b'Expect: 100-continue') conn._send_output() response = conn.response_class(conn.sock, method='POST') @@ -583,7 +583,7 @@ def test_readall_or_close(self): break # ...send the body - body = ntob('I am a small file') + body = b'I am a small file' conn.send(body) # ...get the final response @@ -654,7 +654,7 @@ def test_Chunked_Encoding(self): response = conn.getresponse() self.status, self.headers, self.body = webtest.shb(response) self.assertStatus('200 OK') - self.assertBody("thanks for '%s'" % ntob('xx\r\nxxxxyyyyy')) + self.assertBody("thanks for '%s'" % b'xx\r\nxxxxyyyyy') # Try a chunked request that exceeds server.max_request_body_size. # Note that the delimiters and trailer are included. @@ -723,8 +723,13 @@ def test_Content_Length_out_postheaders(self): conn.close() def test_598(self): - remote_data_conn = urlopen('%s://%s:%s/one_megabyte_of_a/' % - (self.scheme, self.HOST, self.PORT,)) + tmpl = '{scheme}://{host}:{port}/one_megabyte_of_a/' + url = tmpl.format( + scheme=self.scheme, + host=self.HOST, + port=self.PORT, + ) + remote_data_conn = urllib.request.urlopen(url) buf = remote_data_conn.read(512) time.sleep(timeout * 0.6) remaining = (1024 * 1024) - 512 @@ -796,7 +801,8 @@ def test_queue_full(self): conn.endheaders() conns.append(conn) - # Now try a 16th conn, which should be closed by the server immediately. + # Now try a 16th conn, which should be closed by the + # server immediately. overflow_conn = self.HTTP_CONN(self.HOST, self.PORT) # Manually connect since httplib won't let us set a timeout for res in socket.getaddrinfo(self.HOST, self.PORT, 0, @@ -810,7 +816,10 @@ def test_queue_full(self): overflow_conn.putrequest('GET', '/', skip_host=True) overflow_conn.putheader('Host', self.HOST) overflow_conn.endheaders() - response = overflow_conn.response_class(overflow_conn.sock, method='GET') + response = overflow_conn.response_class( + overflow_conn.sock, + method='GET', + ) try: response.begin() except socket.error as exc: @@ -830,7 +839,7 @@ def test_queue_full(self): raise AssertionError('Overflow conn did not get RST ') finally: for conn in conns: - conn.send(ntob('done')) + conn.send(b'done') response = conn.response_class(conn.sock, method='POST') response.begin() self.body = response.read() @@ -848,7 +857,7 @@ def test_No_CRLF(self): self.persistent = True conn = self.HTTP_CONN - conn.send(ntob('GET /hello HTTP/1.1\n\n')) + conn.send(b'GET /hello HTTP/1.1\n\n') response = conn.response_class(conn.sock, method='GET') response.begin() self.body = response.read() @@ -856,7 +865,7 @@ def test_No_CRLF(self): conn.close() conn.connect() - conn.send(ntob('GET /hello HTTP/1.1\r\n\n')) + conn.send(b'GET /hello HTTP/1.1\r\n\n') response = conn.response_class(conn.sock, method='GET') response.begin() self.body = response.read() diff --git a/lib/cherrypy/test/test_core.py b/lib/cherrypy/test/test_core.py index 252c1ac..9834c1f 100644 --- a/lib/cherrypy/test/test_core.py +++ b/lib/cherrypy/test/test_core.py @@ -6,8 +6,10 @@ import sys import types +import six + import cherrypy -from cherrypy._cpcompat import itervalues, ntob, ntou +from cherrypy._cpcompat import ntou from cherrypy import _cptools, tools from cherrypy.lib import httputil, static @@ -55,7 +57,7 @@ class TestType(type): """ def __init__(cls, name, bases, dct): type.__init__(cls, name, bases, dct) - for value in itervalues(dct): + for value in six.itervalues(dct): if isinstance(value, types.FunctionType): value.exposed = True setattr(root, name.lower(), cls()) @@ -154,7 +156,8 @@ def url_with_quote(self): raise cherrypy.HTTPRedirect("/some\"url/that'we/want") def url_with_xss(self): - raise cherrypy.HTTPRedirect("/someurl/that'we/want") + raise cherrypy.HTTPRedirect( + "/someurl/that'we/want") def url_with_unicode(self): raise cherrypy.HTTPRedirect(ntou('тест', 'utf-8')) @@ -232,7 +235,7 @@ def as_list(self): return ['con', 'tent'] def as_yield(self): - yield ntob('content') + yield b'content' @cherrypy.config(**{'tools.flatten.on': True}) def as_dblyield(self): @@ -276,8 +279,8 @@ class MultiHeader(Test): def header_list(self): pass header_list = cherrypy.tools.append_headers(header_list=[ - (ntob('WWW-Authenticate'), ntob('Negotiate')), - (ntob('WWW-Authenticate'), ntob('Basic realm="foo"')), + (b'WWW-Authenticate', b'Negotiate'), + (b'WWW-Authenticate', b'Basic realm="foo"'), ])(header_list) def commas(self): @@ -322,8 +325,10 @@ def testSlashes(self): # Make sure GET params are preserved. self.getPage('/redirect?id=3') self.assertStatus(301) - self.assertMatchesBody('' - '%s/redirect/[?]id=3' % (self.base(), self.base())) + self.assertMatchesBody( + '' + '%s/redirect/[?]id=3' % (self.base(), self.base()) + ) if self.prefix(): # Corner case: the "trailing slash" redirect could be tricky if @@ -338,9 +343,11 @@ def testSlashes(self): # Make sure GET params are preserved. self.getPage('/redirect/by_code/?code=307') self.assertStatus(301) - self.assertMatchesBody("" - '%s/redirect/by_code[?]code=307' - % (self.base(), self.base())) + self.assertMatchesBody( + "" + '%s/redirect/by_code[?]code=307' + % (self.base(), self.base()) + ) # If the trailing_slash tool is off, CP should just continue # as if the slashes were correct. But it needs some help @@ -426,9 +433,14 @@ def testRedirect(self): def assertValidXHTML(): from xml.etree import ElementTree try: - ElementTree.fromstring('%s' % self.body) - except ElementTree.ParseError as e: # noqa: F841 - self._handlewebError('automatically generated redirect did not generate well-formed html') + ElementTree.fromstring( + '%s' % self.body, + ) + except ElementTree.ParseError: + self._handlewebError( + 'automatically generated redirect did not ' + 'generate well-formed html', + ) # check redirects to URLs generated valid HTML - we check this # by seeing if it appears as valid XHTML. @@ -442,7 +454,8 @@ def assertValidXHTML(): assertValidXHTML() def test_redirect_with_xss(self): - """A redirect to a URL with HTML injected should result in page contents escaped.""" + """A redirect to a URL with HTML injected should result + in page contents escaped.""" self.getPage('/redirect/url_with_xss') self.assertStatus(303) assert b'