ndb_json performance tweaks.

.travis.yml

@@ -11,7 +11,7 @@ install:
 - pip install .
 - pip install -r requirements.txt
 script:
-- python setup.py test
+- nosetests tests
 deploy:
   provider: pypi
   user: erichiggins

gaek/ndb_json.py

@@ -34,9 +34,11 @@
 import base64
 import datetime
 import json
+import re
 import time
 import types
 
+import arrow
 import dateutil.parser
 from google.appengine.ext import ndb
 
@@ -50,6 +52,9 @@
 )
 
 
+DATE_RE = re.compile(r'^\d{4}[-/]\d{2}[-/]\d{2}')
+
+
 def encode_model(obj):
   """Encode objects like ndb.Model which have a `.to_dict()` method."""
   obj_dict = obj.to_dict()
@@ -131,38 +136,51 @@ def encode_basevalue(obj):
 }
 
 # Sort the types so any iteration is in a deterministic order
-NDB_TYPES = sorted(NDB_TYPE_ENCODING.keys(), key=lambda t: t.__name__)
+NDB_TYPES = (
+  ndb.Future,
+  ndb.Key,
+  ndb.MetaModel,
+  ndb.Query,
+  ndb.QueryIterator,
+  ndb.model._BaseValue,
+  types.ComplexType,
+  datetime.date,
+  datetime.datetime,
+  time.struct_time,
+)
+
+
+def _object_hook_handler(val):
+  """Handles decoding of nested date strings."""
+  return {k: _decode_date(v) for k, v in val.iteritems()}
+
+
+def _decode_date(val):
+  """Tries to decode strings that look like dates into datetime objects."""
+  if isinstance(val, basestring) and DATE_RE.match(val):
+    try:
+      dt = arrow.parser.DateTimeParser().parse_iso(val)
+      # Check for UTC.
+      if val.endswith(('+00:00', '-00:00', 'Z')):
+        # Then remove tzinfo for gae, which is offset-naive.
+        dt = dt.replace(tzinfo=None)
+      return dt
+    except (TypeError, ValueError):
+      pass
+  return val
 
 
 class NdbDecoder(json.JSONDecoder):
   """Extend the JSON decoder to add support for datetime objects."""
 
   def __init__(self, **kwargs):
     """Override the default __init__ in order to specify our own parameters."""
-    json.JSONDecoder.__init__(self, object_hook=self.object_hook_handler, **kwargs)
-
-  def object_hook_handler(self, val):
-    """Handles decoding of nested date strings."""
-    return {k: self.decode_date(v) for k, v in val.iteritems()}
-
-  def decode_date(self, val):
-    """Tries to decode strings that look like dates into datetime objects."""
-    if isinstance(val, basestring) and val.count('-') == 2 and len(val) > 9:
-      try:
-        dt = dateutil.parser.parse(val)
-        # Check for UTC.
-        if val.endswith(('+00:00', '-00:00', 'Z')):
-          # Then remove tzinfo for gae, which is offset-naive.
-          dt = dt.replace(tzinfo=None)
-        return dt
-      except (TypeError, ValueError):
-        pass
-    return val
+    json.JSONDecoder.__init__(self, object_hook=_object_hook_handler, **kwargs)
 
   def decode(self, val):
     """Override of the default decode method that also uses decode_date."""
     # First try the date decoder.
-    new_val = self.decode_date(val)
+    new_val = _decode_date(val)
     if val != new_val:
       return new_val
     # Fall back to the default decoder.
@@ -172,47 +190,46 @@ def decode(self, val):
 class NdbEncoder(json.JSONEncoder):
   """Extend the JSON encoder to add support for NDB Models."""
 
-
   def __init__(self, **kwargs):
     self._ndb_type_encoding = NDB_TYPE_ENCODING.copy()
+    self._cached_type_encoding = {}
 
     keys_as_entities = kwargs.pop('ndb_keys_as_entities', False)
     keys_as_pairs = kwargs.pop('ndb_keys_as_pairs', False)
     keys_as_urlsafe = kwargs.pop('ndb_keys_as_urlsafe', False)
 
-    # Validate that only one of three flags is True
-    if ((keys_as_entities and keys_as_pairs)
-        or (keys_as_entities and keys_as_urlsafe)
-        or (keys_as_pairs and keys_as_urlsafe)):
-      raise ValueError('Only one of arguments ndb_keys_as_entities, ndb_keys_as_pairs, ndb_keys_as_urlsafe can be True')
-
-    if keys_as_pairs:
-      self._ndb_type_encoding[ndb.Key] = encode_key_as_pair
-    elif keys_as_urlsafe:
-      self._ndb_type_encoding[ndb.Key] = encode_key_as_urlsafe
-    else:
-      self._ndb_type_encoding[ndb.Key] = encode_key_as_entity
-
+    if any((keys_as_entities, keys_as_pairs, keys_as_urlsafe)):
+      # Validate that only one of three flags is True
+      if ((keys_as_entities and keys_as_pairs)
+          or (keys_as_entities and keys_as_urlsafe)
+          or (keys_as_pairs and keys_as_urlsafe)):
+        raise ValueError('Only one of arguments ndb_keys_as_entities, ndb_keys_as_pairs, ndb_keys_as_urlsafe can be True')
+
+      if keys_as_pairs:
+        self._ndb_type_encoding[ndb.Key] = encode_key_as_pair
+      elif keys_as_urlsafe:
+        self._ndb_type_encoding[ndb.Key] = encode_key_as_urlsafe
+      else:
+        self._ndb_type_encoding[ndb.Key] = encode_key_as_entity
 
     json.JSONEncoder.__init__(self, **kwargs)
 
   def default(self, obj):
     """Overriding the default JSONEncoder.default for NDB support."""
     obj_type = type(obj)
-    # NDB Models return a repr to calls from type().
-    if obj_type not in self._ndb_type_encoding:
-      if hasattr(obj, '__metaclass__'):
-        obj_type = obj.__metaclass__
-      else:
-        # Try to encode subclasses of types
-        for ndb_type in NDB_TYPES:
-          if isinstance(obj, ndb_type):
-            obj_type = ndb_type
-            break
+    fn = self._ndb_type_encoding.get(obj_type) or self._cached_type_encoding.get(obj_type) or self._ndb_type_encoding.get(getattr(obj, '__metaclass__', None))
+    if fn:
+      return fn(obj)
 
-    fn = self._ndb_type_encoding.get(obj_type)
+    # Try to encode subclasses of types
+    for ndb_type in NDB_TYPES:
+      if isinstance(obj, ndb_type):
+        obj_type = ndb_type
+        break
 
+    fn = self._ndb_type_encoding.get(obj_type)
     if fn:
+      self._cached_type_encoding[getattr(obj, '__metaclass__', type(obj))] = fn
       return fn(obj)
 
     return json.JSONEncoder.default(self, obj)

requirements.txt

@@ -1 +1,2 @@
+arrow>=0.10.0
 python-dateutil>=2.4.2

setup.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
-GAE_SDK_SHA1='abe54d95c4ce6ffc35452e027ca701f5d21dd56a'
-GAE_SDK_FILE='google_appengine_1.9.35.zip'
+GAE_SDK_SHA1='02ca467d77f5681c52741c7223fb8e97dff999da'
+GAE_SDK_FILE='google_appengine_1.9.50.zip'
 
 # Create virtual environment.
 echo 'Creating virtual environment...'
@@ -10,6 +10,9 @@ source .dev_env/bin/activate
 pip install --upgrade ndg-httpsclient
 pip install --upgrade pip
 
+pip install -r requirements.txt
+pip install -r requirements_test.txt
+
 # Download the App Engine SDK.
 echo "Downloading $GAE_SDK_FILE..."
 curl -O https://storage.googleapis.com/appengine-sdks/featured/$GAE_SDK_FILE

tox.ini

@@ -4,6 +4,6 @@ envlist = py26, py27, py33, py34
 [testenv]
 setenv =
     PYTHONPATH = {toxinidir}:{toxinidir}/gaek
-commands = python setup.py test
+commands = nosetests tests
 deps =
     -r{toxinidir}/requirements.txt