TF 2.12 (and Python 3.11) support

[jonas-eschle]

Hi there, I was wondering what's the status of supporting TensorFlow version 2.12 and Python 3.11?

This currently fails with TF 2.12 installed (but setting PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python as suggested works afai see). It seems like kernels just have to be recompiled if I get that right? I.e. a new version with the new protobuf library, and that's it?

Details

In [1]: import tf_quant_finance
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
Cell In[1], line 1
----> 1 import tf_quant_finance

File ~/mambaforge/envs/tmp310/lib/python3.10/site-packages/tf_quant_finance/__init__.py:73
     71 from tf_quant_finance import black_scholes
     72 from tf_quant_finance import datetime
---> 73 from tf_quant_finance import experimental
     74 from tf_quant_finance import math
     75 from tf_quant_finance import models

File ~/mambaforge/envs/tmp310/lib/python3.10/site-packages/tf_quant_finance/experimental/__init__.py:17
     14 """Experimental modules."""
     16 from tf_quant_finance.experimental import instruments
---> 17 from tf_quant_finance.experimental import local_stochastic_volatility
     18 from tf_quant_finance.experimental import local_volatility
     19 from tf_quant_finance.experimental import lsm_algorithm

File ~/mambaforge/envs/tmp310/lib/python3.10/site-packages/tf_quant_finance/experimental/local_stochastic_volatility/__init__.py:16
      1 # Copyright 2020 Google LLC
      2 #
      3 # Licensed under the Apache License, Version 2.0 (the "License");
   (...)
     12 # See the License for the specific language governing permissions and
     13 # limitations under the License.
     14 """Local volatility model."""
---> 16 from tf_quant_finance.experimental.local_stochastic_volatility.local_stochastic_volatility_model import LocalStochasticVolatilityModel
     17 from tf_quant_finance.experimental.local_stochastic_volatility.lsv_variance_model import LSVVarianceModel
     19 from tensorflow.python.util.all_util import remove_undocumented  # pylint: disable=g-direct-tensorflow-import

File ~/mambaforge/envs/tmp310/lib/python3.10/site-packages/tf_quant_finance/experimental/local_stochastic_volatility/local_stochastic_volatility_model.py:25
     23 from tf_quant_finance import math
     24 from tf_quant_finance import utils as tff_utils
---> 25 from tf_quant_finance.experimental import local_volatility as lvm
     26 from tf_quant_finance.experimental.pricing_platform.framework.market_data import utils
     27 from tf_quant_finance.math import pde

File ~/mambaforge/envs/tmp310/lib/python3.10/site-packages/tf_quant_finance/experimental/local_volatility/__init__.py:17
      1 # Copyright 2020 Google LLC
      2 #
      3 # Licensed under the Apache License, Version 2.0 (the "License");
   (...)
     12 # See the License for the specific language governing permissions and
     13 # limitations under the License.
     15 """Local volatility model."""
---> 17 from tf_quant_finance.experimental.local_volatility.local_volatility_model import LocalVolatilityModel
     19 from tensorflow.python.util.all_util import remove_undocumented  # pylint: disable=g-direct-tensorflow-import
     21 _allowed_symbols = [
     22     "LocalVolatilityModel",
     23 ]

File ~/mambaforge/envs/tmp310/lib/python3.10/site-packages/tf_quant_finance/experimental/local_volatility/local_volatility_model.py:26
     24 from tf_quant_finance import math
     25 from tf_quant_finance import types
---> 26 from tf_quant_finance.experimental.pricing_platform.framework.market_data import volatility_surface
     27 from tf_quant_finance.math import interpolation
     28 from tf_quant_finance.math import piecewise

File ~/mambaforge/envs/tmp310/lib/python3.10/site-packages/tf_quant_finance/experimental/pricing_platform/__init__.py:16
      1 # Copyright 2020 Google LLC
      2 #
      3 # Licensed under the Apache License, Version 2.0 (the "License");
   (...)
     12 # See the License for the specific language governing permissions and
     13 # limitations under the License.
     14 """Pricing platform module."""
---> 16 from tf_quant_finance.experimental.pricing_platform import framework
     17 from tf_quant_finance.experimental.pricing_platform import instrument_protos
     19 from tensorflow.python.util.all_util import remove_undocumented  # pylint: disable=g-direct-tensorflow-import

File ~/mambaforge/envs/tmp310/lib/python3.10/site-packages/tf_quant_finance/experimental/pricing_platform/framework/__init__.py:16
      1 # Copyright 2020 Google LLC
      2 #
      3 # Licensed under the Apache License, Version 2.0 (the "License");
   (...)
     12 # See the License for the specific language governing permissions and
     13 # limitations under the License.
     14 """Framework module."""
---> 16 from tf_quant_finance.experimental.pricing_platform.framework import core
     17 from tf_quant_finance.experimental.pricing_platform.framework import equity_instruments
     18 from tf_quant_finance.experimental.pricing_platform.framework import market_data

File ~/mambaforge/envs/tmp310/lib/python3.10/site-packages/tf_quant_finance/experimental/pricing_platform/framework/core/__init__.py:16
      1 # Copyright 2020 Google LLC
      2 #
      3 # Licensed under the Apache License, Version 2.0 (the "License");
   (...)
     12 # See the License for the specific language governing permissions and
     13 # limitations under the License.
     14 """Pricing platform module."""
---> 16 from tf_quant_finance.experimental.pricing_platform.framework.core import business_days
     17 from tf_quant_finance.experimental.pricing_platform.framework.core import currencies
     18 from tf_quant_finance.experimental.pricing_platform.framework.core import curve_types

File ~/mambaforge/envs/tmp310/lib/python3.10/site-packages/tf_quant_finance/experimental/pricing_platform/framework/core/business_days.py:17
     14 """Supported bank holidays and business day conventions."""
     16 import enum
---> 17 from tf_quant_finance.experimental.pricing_platform.instrument_protos import business_days_pb2
     19 __all__ = ["BusinessDayConvention",
     20            "convention_from_proto_value",
     21            "BankHolidays",
     22            "holiday_from_proto_value"]
     25 BusinessDayConvention = enum.Enum(
     26     "BusinessDayConvention",
     27     zip(business_days_pb2.BusinessDayConvention.keys(),
     28         business_days_pb2.BusinessDayConvention.keys()))

File ~/mambaforge/envs/tmp310/lib/python3.10/site-packages/tf_quant_finance/experimental/pricing_platform/instrument_protos/__init__.py:16
      1 # Copyright 2020 Google LLC
      2 #
      3 # Licensed under the Apache License, Version 2.0 (the "License");
   (...)
     12 # See the License for the specific language governing permissions and
     13 # limitations under the License.
     14 """Instrument protos module."""
---> 16 from tf_quant_finance.experimental.pricing_platform.instrument_protos import all_instruments_pb2 as instruments
     17 from tf_quant_finance.experimental.pricing_platform.instrument_protos import american_equity_option_pb2 as american_equity_option
     18 from tf_quant_finance.experimental.pricing_platform.instrument_protos import business_days_pb2 as business_days

File ~/mambaforge/envs/tmp310/lib/python3.10/site-packages/tf_quant_finance/experimental/pricing_platform/instrument_protos/all_instruments_pb2.py:14
      9 # @@protoc_insertion_point(imports)
     11 _sym_db = _symbol_database.Default()
---> 14 from tf_quant_finance.experimental.pricing_platform.instrument_protos import american_equity_option_pb2 as tf__quant__finance_dot_experimental_dot_pricing__platform_dot_instrument__protos_dot_american__equity__option__pb2
     15 from tf_quant_finance.experimental.pricing_platform.instrument_protos import interest_rate_swap_pb2 as tf__quant__finance_dot_experimental_dot_pricing__platform_dot_instrument__protos_dot_interest__rate__swap__pb2
     16 from tf_quant_finance.experimental.pricing_platform.instrument_protos import forward_rate_agreement_pb2 as tf__quant__finance_dot_experimental_dot_pricing__platform_dot_instrument__protos_dot_forward__rate__agreement__pb2

File ~/mambaforge/envs/tmp310/lib/python3.10/site-packages/tf_quant_finance/experimental/pricing_platform/instrument_protos/american_equity_option_pb2.py:14
      9 # @@protoc_insertion_point(imports)
     11 _sym_db = _symbol_database.Default()
---> 14 from tf_quant_finance.experimental.pricing_platform.instrument_protos import business_days_pb2 as tf__quant__finance_dot_experimental_dot_pricing__platform_dot_instrument__protos_dot_business__days__pb2
     15 from tf_quant_finance.experimental.pricing_platform.instrument_protos import currencies_pb2 as tf__quant__finance_dot_experimental_dot_pricing__platform_dot_instrument__protos_dot_currencies__pb2
     16 from tf_quant_finance.experimental.pricing_platform.instrument_protos import date_pb2 as tf__quant__finance_dot_experimental_dot_pricing__platform_dot_instrument__protos_dot_date__pb2

File ~/mambaforge/envs/tmp310/lib/python3.10/site-packages/tf_quant_finance/experimental/pricing_platform/instrument_protos/business_days_pb2.py:33
     12 _sym_db = _symbol_database.Default()
     17 DESCRIPTOR = _descriptor.FileDescriptor(
     18   name='tf_quant_finance/experimental/pricing_platform/instrument_protos/business_days.proto',
     19   package='tf_quant_finance.experimental.pricing_platform.instrument_protos',
   (...)
     23   serialized_pb=b'\nTtf_quant_finance/experimental/pricing_platform/instrument_protos/business_days.proto\x12@tf_quant_finance.experimental.pricing_platform.instrument_protos*\xd7\x01\n\x15\x42usinessDayConvention\x12#\n\x1f\x42USINESS_DAY_CONVENTION_UNKNOWN\x10\x00\x12\x11\n\rNO_ADJUSTMENT\x10\x01\x12\x0c\n\x08PREVIOUS\x10\x02\x12\r\n\tFOLLOWING\x10\x03\x12\x15\n\x11MODIFIED_PREVIOUS\x10\x04\x12\x16\n\x12MODIFIED_FOLLOWING\x10\x05\x12\x15\n\x11\x45OM_NO_ADJUSTMENT\x10\x06\x12\x10\n\x0c\x45OM_PREVIOUS\x10\x07\x12\x11\n\rEOM_FOLLOWING\x10\x08*9\n\x0c\x42\x61nkHolidays\x12\x19\n\x15\x42\x41NK_HOLIDAYS_UNKNOWN\x10\x00\x12\x06\n\x02US\x10\x01\x12\x06\n\x02UK\x10\x02\x42\x02P\x01\x62\x06proto3'
     24 )
     26 _BUSINESSDAYCONVENTION = _descriptor.EnumDescriptor(
     27   name='BusinessDayConvention',
     28   full_name='tf_quant_finance.experimental.pricing_platform.instrument_protos.BusinessDayConvention',
     29   filename=None,
     30   file=DESCRIPTOR,
     31   create_key=_descriptor._internal_create_key,
     32   values=[
---> 33     _descriptor.EnumValueDescriptor(
     34       name='BUSINESS_DAY_CONVENTION_UNKNOWN', index=0, number=0,
     35       serialized_options=None,
     36       type=None,
     37       create_key=_descriptor._internal_create_key),
     38     _descriptor.EnumValueDescriptor(
     39       name='NO_ADJUSTMENT', index=1, number=1,
     40       serialized_options=None,
     41       type=None,
     42       create_key=_descriptor._internal_create_key),
     43     _descriptor.EnumValueDescriptor(
     44       name='PREVIOUS', index=2, number=2,
     45       serialized_options=None,
     46       type=None,
     47       create_key=_descriptor._internal_create_key),
     48     _descriptor.EnumValueDescriptor(
     49       name='FOLLOWING', index=3, number=3,
     50       serialized_options=None,
     51       type=None,
     52       create_key=_descriptor._internal_create_key),
     53     _descriptor.EnumValueDescriptor(
     54       name='MODIFIED_PREVIOUS', index=4, number=4,
     55       serialized_options=None,
     56       type=None,
     57       create_key=_descriptor._internal_create_key),
     58     _descriptor.EnumValueDescriptor(
     59       name='MODIFIED_FOLLOWING', index=5, number=5,
     60       serialized_options=None,
     61       type=None,
     62       create_key=_descriptor._internal_create_key),
     63     _descriptor.EnumValueDescriptor(
     64       name='EOM_NO_ADJUSTMENT', index=6, number=6,
     65       serialized_options=None,
     66       type=None,
     67       create_key=_descriptor._internal_create_key),
     68     _descriptor.EnumValueDescriptor(
     69       name='EOM_PREVIOUS', index=7, number=7,
     70       serialized_options=None,
     71       type=None,
     72       create_key=_descriptor._internal_create_key),
     73     _descriptor.EnumValueDescriptor(
     74       name='EOM_FOLLOWING', index=8, number=8,
     75       serialized_options=None,
     76       type=None,
     77       create_key=_descriptor._internal_create_key),
     78   ],
     79   containing_type=None,
     80   serialized_options=None,
     81   serialized_start=155,
     82   serialized_end=370,
     83 )
     84 _sym_db.RegisterEnumDescriptor(_BUSINESSDAYCONVENTION)
     86 BusinessDayConvention = enum_type_wrapper.EnumTypeWrapper(_BUSINESSDAYCONVENTION)

File ~/mambaforge/envs/tmp310/lib/python3.10/site-packages/google/protobuf/descriptor.py:796, in EnumValueDescriptor.__new__(cls, name, index, number, type, options, serialized_options, create_key)
    793 def __new__(cls, name, index, number,
    794             type=None,  # pylint: disable=redefined-builtin
    795             options=None, serialized_options=None, create_key=None):
--> 796   _message.Message._CheckCalledFromGeneratedFile()
    797   # There is no way we can build a complete EnumValueDescriptor with the
    798   # given parameters (the name of the Enum is not known, for example).
    799   # Fortunately generated files just pass it to the EnumDescriptor()
    800   # constructor, which will ignore it, so returning None is good enough.
    801   return None

TypeError: Descriptors cannot not be created directly.
If this call came from a _pb2.py file, your generated code is out of date and must be regenerated with protoc >= 3.19.0.
If you cannot immediately regenerate your protos, some other possible workarounds are:
 1. Downgrade the protobuf package to 3.20.x or lower.
 2. Set PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python (but this will use pure-Python parsing and will be much slower).

More information: https://developers.google.com/protocol-buffers/docs/news/2022-05-06#python-updates

[jonas-eschle]

@perezju I was wondering what are the plans for this? Is this on the timeline? Or anything we can do about it?

[YuMan-Tam]

Encountered the same error with newer python version. @jonas-eschle's method works.