diff --git a/tfx/components/distribution_validator/executor_test.py b/tfx/components/distribution_validator/executor_test.py index b46f9dcf41..1bb30aa707 100644 --- a/tfx/components/distribution_validator/executor_test.py +++ b/tfx/components/distribution_validator/executor_test.py @@ -294,7 +294,7 @@ def testSplitPairs(self, split_pairs, expected_split_pair_names, step: 'company' } validations { - sql_expression: 'feature_test.string_stats.unique > feature_base.string_stats.unique' + sql_expression: 'feature_test.string_stats.unique > feature_base.string_stats.unique * 2' severity: ERROR description: 'Test feature has too few unique values.' } @@ -308,7 +308,7 @@ def testSplitPairs(self, split_pairs, expected_split_pair_names, reason { type: CUSTOM_VALIDATION short_description: "Test feature has too few unique values." - description: "Custom validation triggered anomaly. Query: feature_test.string_stats.unique > feature_base.string_stats.unique Test dataset: default slice Base dataset: Base path: company" } + description: "Custom validation triggered anomaly. Query: feature_test.string_stats.unique > feature_base.string_stats.unique * 2 Test dataset: default slice Base dataset: Base path: company" } path { step: "company" } diff --git a/tfx/components/evaluator/executor.py b/tfx/components/evaluator/executor.py index 39a2a141dd..938a031671 100644 --- a/tfx/components/evaluator/executor.py +++ b/tfx/components/evaluator/executor.py @@ -21,7 +21,6 @@ import tensorflow_model_analysis as tfma # Need to import the following module so that the fairness indicator post-export # metric is registered. -import tensorflow_model_analysis.addons.fairness.post_export_metrics.fairness_indicators # pylint: disable=unused-import from tfx import types from tfx.components.evaluator import constants from tfx.components.util import udf_utils @@ -102,16 +101,6 @@ def Do(self, input_dict: Dict[str, List[types.Artifact]], self._log_startup(input_dict, output_dict, exec_properties) - # Add fairness indicator metric callback if necessary. - fairness_indicator_thresholds = json_utils.loads( - exec_properties.get( - standard_component_specs.FAIRNESS_INDICATOR_THRESHOLDS_KEY, 'null')) - add_metrics_callbacks = None - if fairness_indicator_thresholds: - add_metrics_callbacks = [ - tfma.post_export_metrics.fairness_indicators( # pytype: disable=module-attr - thresholds=fairness_indicator_thresholds), - ] output_uri = artifact_utils.get_single_uri( output_dict[constants.EVALUATION_KEY]) @@ -196,7 +185,7 @@ def Do(self, input_dict: Dict[str, List[types.Artifact]], eval_saved_model_path=model_path, model_name=model_spec.name, eval_config=eval_config, - add_metrics_callbacks=add_metrics_callbacks)) + add_metrics_callbacks=None)) else: eval_config = None assert (standard_component_specs.FEATURE_SLICING_SPEC_KEY @@ -219,7 +208,7 @@ def Do(self, input_dict: Dict[str, List[types.Artifact]], eval_saved_model_path=model_path, model_name='', eval_config=None, - add_metrics_callbacks=add_metrics_callbacks)) + add_metrics_callbacks=None)) eval_shared_model = models[0] if len(models) == 1 else models schema = None diff --git a/tfx/components/evaluator/executor_test.py b/tfx/components/evaluator/executor_test.py index de64be3619..93bdf201e7 100644 --- a/tfx/components/evaluator/executor_test.py +++ b/tfx/components/evaluator/executor_test.py @@ -16,6 +16,7 @@ import glob import os +import pytest from absl import logging from absl.testing import parameterized @@ -147,6 +148,7 @@ def testEvalution(self, exec_properties, model_agnostic=False): column_for_slicing=['trip_start_day', 'trip_miles']), ])), })) + @pytest.mark.xfail(run=False, reason="EvalSavedModel is deprecated.") def testDoLegacySingleEvalSavedModelWFairness(self, exec_properties): source_data_dir = os.path.join( os.path.dirname(os.path.dirname(__file__)), 'testdata') @@ -180,7 +182,8 @@ def testDoLegacySingleEvalSavedModelWFairness(self, exec_properties): # post-export metric is registered. This may raise an ImportError if the # currently-installed version of TFMA does not support fairness # indicators. - import tensorflow_model_analysis.addons.fairness.post_export_metrics.fairness_indicators # noqa: F401 + # Note: tensorflow_model_analysis.addons is deprecated from 0.47.0. + # import tensorflow_model_analysis.addons.fairness.post_export_metrics.fairness_indicators # noqa: F401 exec_properties[ standard_component_specs .FAIRNESS_INDICATOR_THRESHOLDS_KEY] = '[0.1, 0.3, 0.5, 0.7, 0.9]' diff --git a/tfx/components/model_validator/executor_test.py b/tfx/components/model_validator/executor_test.py index cdcc2bfb6f..4495f573a3 100644 --- a/tfx/components/model_validator/executor_test.py +++ b/tfx/components/model_validator/executor_test.py @@ -14,6 +14,7 @@ """Tests for tfx.components.model_validator.executor.""" import os +import pytest import tensorflow as tf from tfx.components.model_validator import constants @@ -23,6 +24,8 @@ from tfx.types import standard_artifacts +@pytest.mark.xfail(run=False, + reason="Model validator is deprecated and this doesn't work with TFMA 0.47.0") class ExecutorTest(tf.test.TestCase): def setUp(self): diff --git a/tfx/components/testdata/module_file/trainer_module.py b/tfx/components/testdata/module_file/trainer_module.py index 4fdc7550e6..30f24cecc4 100644 --- a/tfx/components/testdata/module_file/trainer_module.py +++ b/tfx/components/testdata/module_file/trainer_module.py @@ -240,7 +240,7 @@ def _build_keras_model( output = tf.keras.layers.Dense(1, activation='sigmoid')( tf.keras.layers.concatenate([deep, wide]) ) - output = tf.squeeze(output, -1) + output = tf.keras.layers.Reshape((1,))(output) model = tf.keras.Model(input_layers, output) model.compile( @@ -365,4 +365,4 @@ def run_fn(fn_args: fn_args_utils.FnArgs): model, tf_transform_output ), } - model.save(fn_args.serving_model_dir, save_format='tf', signatures=signatures) + tf.saved_model.save(model, fn_args.serving_model_dir, signatures=signatures) diff --git a/tfx/examples/penguin/experimental/sklearn_predict_extractor_test.py b/tfx/examples/penguin/experimental/sklearn_predict_extractor_test.py index 7fda470dc5..8f0200c471 100644 --- a/tfx/examples/penguin/experimental/sklearn_predict_extractor_test.py +++ b/tfx/examples/penguin/experimental/sklearn_predict_extractor_test.py @@ -13,172 +13,173 @@ # limitations under the License. """Tests for the custom scikit-learn Evaluator module.""" -import os -import pickle -import pytest +# Note: tfma.test has been deprecated from TFMA 0.47.0") -import apache_beam as beam -from apache_beam.testing import util -from sklearn import neural_network as nn -import tensorflow_model_analysis as tfma -from tfx.examples.penguin.experimental import sklearn_predict_extractor -from tfx_bsl.tfxio import tensor_adapter -from tfx_bsl.tfxio import test_util - -from google.protobuf import text_format -from tensorflow_metadata.proto.v0 import schema_pb2 - - -class SklearnPredictExtractorTest(tfma.test.TestCase): - - def setUp(self): - super().setUp() - self._eval_export_dir = os.path.join(self._getTempDir(), 'eval_export') - self._create_sklearn_model(self._eval_export_dir) - self._eval_config = tfma.EvalConfig(model_specs=[tfma.ModelSpec()]) - self._eval_shared_model = ( - sklearn_predict_extractor.custom_eval_shared_model( - eval_saved_model_path=self._eval_export_dir, - model_name=None, - eval_config=self._eval_config)) - self._schema = text_format.Parse( - """ - feature { - name: "age" - type: FLOAT - } - feature { - name: "language" - type: FLOAT - } - feature { - name: "label" - type: INT - } - """, schema_pb2.Schema()) - self._tfx_io = test_util.InMemoryTFExampleRecord( - schema=self._schema, - raw_record_column_name=tfma.ARROW_INPUT_COLUMN) - self._tensor_adapter_config = tensor_adapter.TensorAdapterConfig( - arrow_schema=self._tfx_io.ArrowSchema(), - tensor_representations=self._tfx_io.TensorRepresentations()) - self._examples = [ - self._makeExample(age=3.0, language=1.0, label=1), - self._makeExample(age=3.0, language=0.0, label=0), - self._makeExample(age=4.0, language=1.0, label=1), - self._makeExample(age=5.0, language=0.0, label=0), - ] - - @pytest.mark.xfail(run=False, reason="This is based on experimental implementation," -"and the test fails.", strict=True) - def testMakeSklearnPredictExtractor(self): - """Tests that predictions are made from extracts for a single model.""" - feature_extractor = tfma.extractors.FeaturesExtractor(self._eval_config) - prediction_extractor = ( - sklearn_predict_extractor._make_sklearn_predict_extractor( - self._eval_shared_model)) - with beam.Pipeline() as pipeline: - predict_extracts = ( - pipeline - | 'Create' >> beam.Create( - [e.SerializeToString() for e in self._examples]) - | 'BatchExamples' >> self._tfx_io.BeamSource() - | 'InputsToExtracts' >> tfma.BatchedInputsToExtracts() # pylint: disable=no-value-for-parameter - | feature_extractor.stage_name >> feature_extractor.ptransform - | prediction_extractor.stage_name >> prediction_extractor.ptransform - ) - - def check_result(actual): - try: - for item in actual: - self.assertEqual(item['labels'].shape, item['predictions'].shape) - - except AssertionError as err: - raise util.BeamAssertException(err) - - util.assert_that(predict_extracts, check_result) - - @pytest.mark.xfail(run=False, reason="This is based on experimental implementation," -"and the test fails.", strict=True) - def testMakeSklearnPredictExtractorWithMultiModels(self): - """Tests that predictions are made from extracts for multiple models.""" - eval_config = tfma.EvalConfig(model_specs=[ - tfma.ModelSpec(name='model1'), - tfma.ModelSpec(name='model2'), - ]) - eval_export_dir_1 = os.path.join(self._eval_export_dir, '1') - self._create_sklearn_model(eval_export_dir_1) - eval_shared_model_1 = sklearn_predict_extractor.custom_eval_shared_model( - eval_saved_model_path=eval_export_dir_1, - model_name='model1', - eval_config=eval_config) - eval_export_dir_2 = os.path.join(self._eval_export_dir, '2') - self._create_sklearn_model(eval_export_dir_2) - eval_shared_model_2 = sklearn_predict_extractor.custom_eval_shared_model( - eval_saved_model_path=eval_export_dir_2, - model_name='model2', - eval_config=eval_config) - - feature_extractor = tfma.extractors.FeaturesExtractor(self._eval_config) - prediction_extractor = ( - sklearn_predict_extractor._make_sklearn_predict_extractor( - eval_shared_model={ - 'model1': eval_shared_model_1, - 'model2': eval_shared_model_2, - })) - with beam.Pipeline() as pipeline: - predict_extracts = ( - pipeline - | 'Create' >> beam.Create( - [e.SerializeToString() for e in self._examples]) - | 'BatchExamples' >> self._tfx_io.BeamSource() - | 'InputsToExtracts' >> tfma.BatchedInputsToExtracts() # pylint: disable=no-value-for-parameter - | feature_extractor.stage_name >> feature_extractor.ptransform - | prediction_extractor.stage_name >> prediction_extractor.ptransform - ) - - def check_result(actual): - try: - for item in actual: - self.assertEqual(item['labels'].shape, item['predictions'].shape) - self.assertIn('model1', item['predictions'][0]) - self.assertIn('model2', item['predictions'][0]) - - except AssertionError as err: - raise util.BeamAssertException(err) - - util.assert_that(predict_extracts, check_result) - - def test_custom_eval_shared_model(self): - """Tests that an EvalSharedModel is created with a custom sklearn loader.""" - model_file = os.path.basename(self._eval_shared_model.model_path) - self.assertEqual(model_file, 'model.pkl') - model = self._eval_shared_model.model_loader.construct_fn() - self.assertIsInstance(model, nn.MLPClassifier) - - def test_custom_extractors(self): - """Tests that the sklearn extractor is used when creating extracts.""" - extractors = sklearn_predict_extractor.custom_extractors( - self._eval_shared_model, self._eval_config, self._tensor_adapter_config) - self.assertLen(extractors, 6) - self.assertIn( - 'SklearnPredict', [extractor.stage_name for extractor in extractors]) - - def _create_sklearn_model(self, eval_export_dir): - """Creates and pickles a toy scikit-learn model. - - Args: - eval_export_dir: Directory to store a pickled scikit-learn model. This - directory is created if it does not exist. - """ - x_train = [[3, 0], [4, 1]] - y_train = [0, 1] - model = nn.MLPClassifier(max_iter=1) - model.feature_keys = ['age', 'language'] - model.label_key = 'label' - model.fit(x_train, y_train) - - os.makedirs(eval_export_dir) - model_path = os.path.join(eval_export_dir, 'model.pkl') - with open(model_path, 'wb+') as f: - pickle.dump(model, f) +#import os +#import pickle +#import pytest +# +#import apache_beam as beam +#from apache_beam.testing import util +#from sklearn import neural_network as nn +#import tensorflow_model_analysis as tfma +#from tfx.examples.penguin.experimental import sklearn_predict_extractor +#from tfx_bsl.tfxio import tensor_adapter +#from tfx_bsl.tfxio import test_util +# +#from google.protobuf import text_format +#from tensorflow_metadata.proto.v0 import schema_pb2 +# +#class SklearnPredictExtractorTest(tfma.test.TestCase): +# +# def setUp(self): +# super().setUp() +# self._eval_export_dir = os.path.join(self._getTempDir(), 'eval_export') +# self._create_sklearn_model(self._eval_export_dir) +# self._eval_config = tfma.EvalConfig(model_specs=[tfma.ModelSpec()]) +# self._eval_shared_model = ( +# sklearn_predict_extractor.custom_eval_shared_model( +# eval_saved_model_path=self._eval_export_dir, +# model_name=None, +# eval_config=self._eval_config)) +# self._schema = text_format.Parse( +# """ +# feature { +# name: "age" +# type: FLOAT +# } +# feature { +# name: "language" +# type: FLOAT +# } +# feature { +# name: "label" +# type: INT +# } +# """, schema_pb2.Schema()) +# self._tfx_io = test_util.InMemoryTFExampleRecord( +# schema=self._schema, +# raw_record_column_name=tfma.ARROW_INPUT_COLUMN) +# self._tensor_adapter_config = tensor_adapter.TensorAdapterConfig( +# arrow_schema=self._tfx_io.ArrowSchema(), +# tensor_representations=self._tfx_io.TensorRepresentations()) +# self._examples = [ +# self._makeExample(age=3.0, language=1.0, label=1), +# self._makeExample(age=3.0, language=0.0, label=0), +# self._makeExample(age=4.0, language=1.0, label=1), +# self._makeExample(age=5.0, language=0.0, label=0), +# ] +# +# @pytest.mark.xfail(run=False, reason="This is based on experimental implementation," +#"and the test fails.", strict=True) +# def testMakeSklearnPredictExtractor(self): +# """Tests that predictions are made from extracts for a single model.""" +# feature_extractor = tfma.extractors.FeaturesExtractor(self._eval_config) +# prediction_extractor = ( +# sklearn_predict_extractor._make_sklearn_predict_extractor( +# self._eval_shared_model)) +# with beam.Pipeline() as pipeline: +# predict_extracts = ( +# pipeline +# | 'Create' >> beam.Create( +# [e.SerializeToString() for e in self._examples]) +# | 'BatchExamples' >> self._tfx_io.BeamSource() +# | 'InputsToExtracts' >> tfma.BatchedInputsToExtracts() # pylint: disable=no-value-for-parameter +# | feature_extractor.stage_name >> feature_extractor.ptransform +# | prediction_extractor.stage_name >> prediction_extractor.ptransform +# ) +# +# def check_result(actual): +# try: +# for item in actual: +# self.assertEqual(item['labels'].shape, item['predictions'].shape) +# +# except AssertionError as err: +# raise util.BeamAssertException(err) +# +# util.assert_that(predict_extracts, check_result) +# +# @pytest.mark.xfail(run=False, reason="This is based on experimental implementation," +#"and the test fails.", strict=True) +# def testMakeSklearnPredictExtractorWithMultiModels(self): +# """Tests that predictions are made from extracts for multiple models.""" +# eval_config = tfma.EvalConfig(model_specs=[ +# tfma.ModelSpec(name='model1'), +# tfma.ModelSpec(name='model2'), +# ]) +# eval_export_dir_1 = os.path.join(self._eval_export_dir, '1') +# self._create_sklearn_model(eval_export_dir_1) +# eval_shared_model_1 = sklearn_predict_extractor.custom_eval_shared_model( +# eval_saved_model_path=eval_export_dir_1, +# model_name='model1', +# eval_config=eval_config) +# eval_export_dir_2 = os.path.join(self._eval_export_dir, '2') +# self._create_sklearn_model(eval_export_dir_2) +# eval_shared_model_2 = sklearn_predict_extractor.custom_eval_shared_model( +# eval_saved_model_path=eval_export_dir_2, +# model_name='model2', +# eval_config=eval_config) +# +# feature_extractor = tfma.extractors.FeaturesExtractor(self._eval_config) +# prediction_extractor = ( +# sklearn_predict_extractor._make_sklearn_predict_extractor( +# eval_shared_model={ +# 'model1': eval_shared_model_1, +# 'model2': eval_shared_model_2, +# })) +# with beam.Pipeline() as pipeline: +# predict_extracts = ( +# pipeline +# | 'Create' >> beam.Create( +# [e.SerializeToString() for e in self._examples]) +# | 'BatchExamples' >> self._tfx_io.BeamSource() +# | 'InputsToExtracts' >> tfma.BatchedInputsToExtracts() # pylint: disable=no-value-for-parameter +# | feature_extractor.stage_name >> feature_extractor.ptransform +# | prediction_extractor.stage_name >> prediction_extractor.ptransform +# ) +# +# def check_result(actual): +# try: +# for item in actual: +# self.assertEqual(item['labels'].shape, item['predictions'].shape) +# self.assertIn('model1', item['predictions'][0]) +# self.assertIn('model2', item['predictions'][0]) +# +# except AssertionError as err: +# raise util.BeamAssertException(err) +# +# util.assert_that(predict_extracts, check_result) +# +# def test_custom_eval_shared_model(self): +# """Tests that an EvalSharedModel is created with a custom sklearn loader.""" +# model_file = os.path.basename(self._eval_shared_model.model_path) +# self.assertEqual(model_file, 'model.pkl') +# model = self._eval_shared_model.model_loader.construct_fn() +# self.assertIsInstance(model, nn.MLPClassifier) +# +# def test_custom_extractors(self): +# """Tests that the sklearn extractor is used when creating extracts.""" +# extractors = sklearn_predict_extractor.custom_extractors( +# self._eval_shared_model, self._eval_config, self._tensor_adapter_config) +# self.assertLen(extractors, 6) +# self.assertIn( +# 'SklearnPredict', [extractor.stage_name for extractor in extractors]) +# +# def _create_sklearn_model(self, eval_export_dir): +# """Creates and pickles a toy scikit-learn model. +# +# Args: +# eval_export_dir: Directory to store a pickled scikit-learn model. This +# directory is created if it does not exist. +# """ +# x_train = [[3, 0], [4, 1]] +# y_train = [0, 1] +# model = nn.MLPClassifier(max_iter=1) +# model.feature_keys = ['age', 'language'] +# model.label_key = 'label' +# model.fit(x_train, y_train) +# +# os.makedirs(eval_export_dir) +# model_path = os.path.join(eval_export_dir, 'model.pkl') +# with open(model_path, 'wb+') as f: +# pickle.dump(model, f) diff --git a/tfx/experimental/templates/taxi/models/keras_model/model_test.py b/tfx/experimental/templates/taxi/models/keras_model/model_test.py index 7dd6110a6b..e2b97c5e9a 100644 --- a/tfx/experimental/templates/taxi/models/keras_model/model_test.py +++ b/tfx/experimental/templates/taxi/models/keras_model/model_test.py @@ -13,10 +13,12 @@ # limitations under the License. import tensorflow as tf +import pytest from tfx.experimental.templates.taxi.models.keras_model import model +@pytest.mark.xfail(run=False, reason="_build_keras_model is not compatible with Keras3.") class ModelTest(tf.test.TestCase): def testBuildKerasModel(self):