From 7a7ed031f8adce4627c8fa73f772a6f0eba4a402 Mon Sep 17 00:00:00 2001 From: tmartins Date: Fri, 4 Nov 2022 09:05:39 -0300 Subject: [PATCH 1/3] remove serialization code --- vespa/package.py | 297 +++--------------------------------------- vespa/test_package.py | 121 ----------------- 2 files changed, 15 insertions(+), 403 deletions(-) diff --git a/vespa/package.py b/vespa/package.py index 7d2c6102..0a8568a3 100644 --- a/vespa/package.py +++ b/vespa/package.py @@ -8,10 +8,8 @@ from jinja2 import Environment, PackageLoader, select_autoescape from io import BytesIO -from vespa.json_serialization import ToJson, FromJson - -class HNSW(ToJson, FromJson["HNSW"]): +class HNSW(object): def __init__( self, distance_metric="euclidean", @@ -31,22 +29,6 @@ def __init__( self.max_links_per_node = max_links_per_node self.neighbors_to_explore_at_insert = neighbors_to_explore_at_insert - @staticmethod - def from_dict(mapping: Mapping) -> "HNSW": - return HNSW( - distance_metric=mapping["distance_metric"], - max_links_per_node=mapping["max_links_per_node"], - neighbors_to_explore_at_insert=mapping["neighbors_to_explore_at_insert"], - ) - - @property - def to_dict(self) -> Mapping: - return { - "distance_metric": self.distance_metric, - "max_links_per_node": self.max_links_per_node, - "neighbors_to_explore_at_insert": self.neighbors_to_explore_at_insert, - } - def __eq__(self, other): if not isinstance(other, self.__class__): return False @@ -66,7 +48,7 @@ def __repr__(self): ) -class Field(ToJson, FromJson["Field"]): +class Field(object): def __init__( self, name: str, @@ -129,31 +111,6 @@ def indexing_to_text(self) -> Optional[str]: if self.indexing is not None: return " | ".join(self.indexing) - @staticmethod - def from_dict(mapping: Mapping) -> "Field": - ann = mapping.get("ann", None) - return Field( - name=mapping["name"], - type=mapping["type"], - indexing=mapping.get("indexing", None), - index=mapping.get("index", None), - attribute=mapping.get("attribute", None), - ann=FromJson.map(ann) if ann is not None else None, - ) - - @property - def to_dict(self) -> Mapping: - m = {"name": self.name, "type": self.type} - if self.indexing is not None: - m.update(indexing=self.indexing) - if self.index is not None: - m.update(index=self.index) - if self.attribute is not None: - m.update(attribute=self.attribute) - if self.ann is not None: - m.update(ann=self.ann.to_envelope) - return m - def __eq__(self, other): if not isinstance(other, self.__class__): return False @@ -178,7 +135,7 @@ def __repr__(self): ) -class ImportedField(ToJson, FromJson["ImportedField"]): +class ImportedField(object): def __init__( self, name: str, @@ -207,22 +164,6 @@ def __init__( self.reference_field = reference_field self.field_to_import = field_to_import - @staticmethod - def from_dict(mapping: Mapping) -> "ImportedField": - return ImportedField( - name=mapping["name"], - reference_field=mapping["reference_field"], - field_to_import=mapping["field_to_import"], - ) - - @property - def to_dict(self) -> Mapping: - return { - "name": self.name, - "reference_field": self.reference_field, - "field_to_import": self.field_to_import, - } - def __eq__(self, other): if not isinstance(other, self.__class__): return False @@ -241,7 +182,7 @@ def __repr__(self): ) -class Document(ToJson, FromJson["Document"]): +class Document(object): def __init__( self, fields: Optional[List[Field]] = None, inherits: Optional[str] = None ) -> None: @@ -285,20 +226,6 @@ def add_fields(self, *fields: Field) -> None: for field in fields: self._fields.update({field.name: field}) - @staticmethod - def from_dict(mapping: Mapping) -> "Document": - return Document( - fields=[FromJson.map(field) for field in mapping.get("fields")], - inherits=mapping.get("inherits", None), - ) - - @property - def to_dict(self) -> Mapping: - return { - "fields": [field.to_envelope for field in self.fields], - "inherits": self.inherits, - } - def __eq__(self, other): if not isinstance(other, self.__class__): return False @@ -312,7 +239,7 @@ def __repr__(self): ) -class FieldSet(ToJson, FromJson["FieldSet"]): +class FieldSet(object): def __init__(self, name: str, fields: List[str]) -> None: """ Create a Vespa field set. @@ -335,14 +262,6 @@ def fields_to_text(self): if self.fields is not None: return ", ".join(self.fields) - @staticmethod - def from_dict(mapping: Mapping) -> "FieldSet": - return FieldSet(name=mapping["name"], fields=mapping["fields"]) - - @property - def to_dict(self) -> Mapping: - return {"name": self.name, "fields": self.fields} - def __eq__(self, other): if not isinstance(other, self.__class__): return False @@ -354,7 +273,7 @@ def __repr__(self): ) -class Function(ToJson, FromJson["Function"]): +class Function(object): def __init__( self, name: str, expression: str, args: Optional[List[str]] = None ) -> None: @@ -402,16 +321,6 @@ def args_to_text(self) -> str: else: return "" - @staticmethod - def from_dict(mapping: Mapping) -> "Function": - return Function( - name=mapping["name"], expression=mapping["expression"], args=mapping["args"] - ) - - @property - def to_dict(self) -> Mapping: - return {"name": self.name, "expression": self.expression, "args": self.args} - def __eq__(self, other): if not isinstance(other, self.__class__): return False @@ -430,7 +339,7 @@ def __repr__(self): ) -class SecondPhaseRanking(ToJson, FromJson["SecondPhaseRanking"]): +class SecondPhaseRanking(object): def __init__(self, expression: str, rerank_count: int = 100) -> None: r""" Create a Vespa second phase ranking configuration. @@ -450,16 +359,6 @@ def __init__(self, expression: str, rerank_count: int = 100) -> None: self.expression = expression self.rerank_count = rerank_count - @staticmethod - def from_dict(mapping: Mapping) -> "SecondPhaseRanking": - return SecondPhaseRanking( - expression=mapping["expression"], rerank_count=mapping["rerank_count"] - ) - - @property - def to_dict(self) -> Mapping: - return {"expression": self.expression, "rerank_count": self.rerank_count} - def __eq__(self, other): if not isinstance(other, self.__class__): return False @@ -476,7 +375,7 @@ def __repr__(self): ) -class RankProfile(ToJson, FromJson["RankProfile"]): +class RankProfile(object): def __init__( self, name: str, @@ -556,44 +455,6 @@ def __init__( self.summary_features = summary_features self.second_phase = second_phase - @staticmethod - def from_dict(mapping: Mapping) -> "RankProfile": - functions = mapping.get("functions", None) - if functions is not None: - functions = [FromJson.map(f) for f in functions] - second_phase = mapping.get("second_phase", None) - if second_phase is not None: - second_phase = FromJson.map(second_phase) - - return RankProfile( - name=mapping["name"], - first_phase=mapping["first_phase"], - inherits=mapping.get("inherits", None), - constants=mapping.get("constants", None), - functions=functions, - summary_features=mapping.get("summary_features", None), - second_phase=second_phase, - ) - - @property - def to_dict(self) -> Mapping: - m = { - "name": self.name, - "first_phase": self.first_phase, - } - if self.inherits is not None: - m.update({"inherits": self.inherits}) - if self.constants is not None: - m.update({"constants": self.constants}) - if self.functions is not None: - m.update({"functions": [f.to_envelope for f in self.functions]}) - if self.summary_features is not None: - m.update({"summary_features": self.summary_features}) - if self.second_phase is not None: - m.update({"second_phase": self.second_phase.to_envelope}) - - return m - def __eq__(self, other): if not isinstance(other, self.__class__): return False @@ -620,7 +481,7 @@ def __repr__(self): ) -class OnnxModel(ToJson, FromJson["OnnxModel"]): +class OnnxModel(object): def __init__( self, model_name: str, @@ -664,24 +525,6 @@ def __init__( self.model_file_name = self.model_name + ".onnx" self.file_path = os.path.join("files", self.model_file_name) - @staticmethod - def from_dict(mapping: Mapping) -> "OnnxModel": - return OnnxModel( - model_name=mapping["model_name"], - model_file_path=mapping["model_file_path"], - inputs=mapping["inputs"], - outputs=mapping["outputs"], - ) - - @property - def to_dict(self) -> Mapping: - return { - "model_name": self.model_name, - "model_file_path": self.model_file_path, - "inputs": self.inputs, - "outputs": self.outputs, - } - def __eq__(self, other): if not isinstance(other, self.__class__): return False @@ -702,7 +545,7 @@ def __repr__(self): ) -class Schema(ToJson, FromJson["Schema"]): +class Schema(object): def __init__( self, name: str, @@ -819,60 +662,6 @@ def schema_to_text(self): imported_fields=self.imported_fields, ) - @staticmethod - def from_dict(mapping: Mapping) -> "Schema": - fieldsets = mapping.get("fieldsets", None) - if fieldsets: - fieldsets = [FromJson.map(fieldset) for fieldset in mapping["fieldsets"]] - rank_profiles = mapping.get("rank_profiles", None) - if rank_profiles: - rank_profiles = [ - FromJson.map(rank_profile) for rank_profile in mapping["rank_profiles"] - ] - models = mapping.get("models", None) - if models: - models = [FromJson.map(model) for model in mapping["models"]] - imported_fields = mapping.get("imported_fields", None) - if imported_fields: - imported_fields = [ - FromJson.map(imported_field) for imported_field in imported_fields - ] - - return Schema( - name=mapping["name"], - document=FromJson.map(mapping["document"]), - fieldsets=fieldsets, - rank_profiles=rank_profiles, - models=models, - global_document=mapping["global_document"], - imported_fields=imported_fields, - ) - - @property - def to_dict(self) -> Mapping: - m = { - "name": self.name, - "document": self.document.to_envelope, - "global_document": self.global_document, - } - if self.fieldsets: - m["fieldsets"] = [ - self.fieldsets[name].to_envelope for name in self.fieldsets.keys() - ] - if self.rank_profiles: - m["rank_profiles"] = [ - self.rank_profiles[name].to_envelope - for name in self.rank_profiles.keys() - ] - if self.models: - m["models"] = [model.to_envelope for model in self.models] - if self.imported_fields: - m["imported_fields"] = [ - self.imported_fields[name].to_envelope - for name in self.imported_fields.keys() - ] - return m - def __eq__(self, other): if not isinstance(other, self.__class__): return False @@ -909,7 +698,7 @@ def __repr__(self): ) -class QueryTypeField(ToJson, FromJson["QueryTypeField"]): +class QueryTypeField(object): def __init__( self, name: str, @@ -930,17 +719,6 @@ def __init__( self.name = name self.type = type - @staticmethod - def from_dict(mapping: Mapping) -> "QueryTypeField": - return QueryTypeField( - name=mapping["name"], - type=mapping["type"], - ) - - @property - def to_dict(self) -> Mapping: - return {"name": self.name, "type": self.type} - def __eq__(self, other): if not isinstance(other, self.__class__): return False @@ -954,7 +732,7 @@ def __repr__(self): ) -class QueryProfileType(ToJson, FromJson["QueryProfileType"]): +class QueryProfileType(object): def __init__(self, fields: Optional[List[QueryTypeField]] = None) -> None: """ Create a Vespa Query Profile Type. @@ -1001,16 +779,6 @@ def add_fields(self, *fields: QueryTypeField) -> None: """ self.fields.extend(fields) - @staticmethod - def from_dict(mapping: Mapping) -> "QueryProfileType": - return QueryProfileType( - fields=[FromJson.map(field) for field in mapping.get("fields")] - ) - - @property - def to_dict(self) -> Mapping: - return {"fields": [field.to_envelope for field in self.fields]} - def __eq__(self, other): if not isinstance(other, self.__class__): return False @@ -1022,7 +790,7 @@ def __repr__(self): ) -class QueryField(ToJson, FromJson["QueryField"]): +class QueryField(object): def __init__( self, name: str, @@ -1040,17 +808,6 @@ def __init__( self.name = name self.value = value - @staticmethod - def from_dict(mapping: Mapping) -> "QueryField": - return QueryField( - name=mapping["name"], - value=mapping["value"], - ) - - @property - def to_dict(self) -> Mapping: - return {"name": self.name, "value": self.value} - def __eq__(self, other): if not isinstance(other, self.__class__): return False @@ -1064,7 +821,7 @@ def __repr__(self): ) -class QueryProfile(ToJson, FromJson["QueryProfile"]): +class QueryProfile(object): def __init__(self, fields: Optional[List[QueryField]] = None) -> None: """ Create a Vespa Query Profile. @@ -1097,16 +854,6 @@ def add_fields(self, *fields: QueryField) -> None: """ self.fields.extend(fields) - @staticmethod - def from_dict(mapping: Mapping) -> "QueryProfile": - return QueryProfile( - fields=[FromJson.map(field) for field in mapping.get("fields")] - ) - - @property - def to_dict(self) -> Mapping: - return {"fields": [field.to_envelope for field in self.fields]} - def __eq__(self, other): if not isinstance(other, self.__class__): return False @@ -1118,7 +865,7 @@ def __repr__(self): ) -class ApplicationPackage(ToJson, FromJson["ApplicationPackage"]): +class ApplicationPackage(object): def __init__( self, name: str, @@ -1271,20 +1018,6 @@ def services_to_text(self): stateless_model_evaluation=self.stateless_model_evaluation, ) - @staticmethod - def from_dict(mapping: Mapping) -> "ApplicationPackage": - schema = mapping.get("schema", None) - if schema is not None: - schema = [FromJson.map(x) for x in schema] - return ApplicationPackage(name=mapping["name"], schema=schema) - - @property - def to_dict(self) -> Mapping: - m = {"name": self.name} - if self._schema is not None: - m.update({"schema": [x.to_envelope for x in self.schemas]}) - return m - @staticmethod def _application_package_file_name(disk_folder): return os.path.join(disk_folder, "application_package.json") diff --git a/vespa/test_package.py b/vespa/test_package.py index 38134343..e903df97 100644 --- a/vespa/test_package.py +++ b/vespa/test_package.py @@ -29,9 +29,7 @@ def test_field_name_type(self): field = Field(name="test_name", type="string") self.assertEqual(field.name, "test_name") self.assertEqual(field.type, "string") - self.assertEqual(field.to_dict, {"name": "test_name", "type": "string"}) self.assertEqual(field, Field(name="test_name", type="string")) - self.assertEqual(field, Field.from_dict(field.to_dict)) self.assertIsNone(field.indexing_to_text) def test_field_name_type_indexing_index(self): @@ -45,15 +43,6 @@ def test_field_name_type_indexing_index(self): self.assertEqual(field.type, "string") self.assertEqual(field.indexing, ["index", "summary"]) self.assertEqual(field.index, "enable-bm25") - self.assertEqual( - field.to_dict, - { - "name": "body", - "type": "string", - "indexing": ["index", "summary"], - "index": "enable-bm25", - }, - ) self.assertEqual( field, Field( @@ -63,23 +52,8 @@ def test_field_name_type_indexing_index(self): index="enable-bm25", ), ) - self.assertEqual(field, Field.from_dict(field.to_dict)) self.assertEqual(field.indexing_to_text, "index | summary") - def test_tensor_with_hnsw(self): - field = Field( - name="tensor_field", - type="tensor(x[128])", - indexing=["attribute"], - attribute=["fast-search", "fast-access"], - ann=HNSW( - distance_metric="euclidean", - max_links_per_node=16, - neighbors_to_explore_at_insert=200, - ), - ) - self.assertEqual(field, Field.from_dict(field.to_dict)) - class TestImportField(unittest.TestCase): def test_import_field(self): @@ -91,24 +65,18 @@ def test_import_field(self): self.assertEqual(imported_field.name, "global_category_ctrs") self.assertEqual(imported_field.reference_field, "category_ctr_ref") self.assertEqual(imported_field.field_to_import, "ctrs") - self.assertEqual( - imported_field, ImportedField.from_dict(imported_field.to_dict) - ) class TestDocument(unittest.TestCase): def test_empty_document(self): document = Document() self.assertEqual(document.fields, []) - self.assertEqual(document.to_dict, {"fields": [], "inherits": None}) - self.assertEqual(document, Document.from_dict(document.to_dict)) def test_document_one_field(self): document = Document(inherits="context") field = Field(name="test_name", type="string") document.add_fields(field) self.assertEqual(document.fields, [field]) - self.assertEqual(document, Document.from_dict(document.to_dict)) self.assertEqual(document, Document([field], "context")) def test_document_two_fields(self): @@ -122,7 +90,6 @@ def test_document_two_fields(self): ) document.add_fields(field_1, field_2) self.assertEqual(document.fields, [field_1, field_2]) - self.assertEqual(document, Document.from_dict(document.to_dict)) self.assertEqual(document, Document([field_1, field_2])) def test_update_field(self): @@ -142,21 +109,9 @@ def test_fieldset(self): field_set = FieldSet(name="default", fields=["title", "body"]) self.assertEqual(field_set.name, "default") self.assertEqual(field_set.fields, ["title", "body"]) - self.assertEqual(field_set, FieldSet.from_dict(field_set.to_dict)) self.assertEqual(field_set.fields_to_text, "title, body") -class TestSecondPhaseRanking(unittest.TestCase): - def test_second_phase_ranking(self): - second_phase_ranking = SecondPhaseRanking( - expression="sum(eval)", rerank_count=10 - ) - self.assertEqual( - second_phase_ranking, - SecondPhaseRanking.from_dict(second_phase_ranking.to_dict), - ) - - class TestFunction(unittest.TestCase): def test_function_no_argument(self): function = Function( @@ -166,7 +121,6 @@ def test_function_no_argument(self): self.assertEqual( function.expression, "fieldMatch(title) + freshness(timestamp)" ) - self.assertEqual(function, Function.from_dict(function.to_dict)) self.assertEqual(function.args_to_text, "") def test_function_one_argument(self): @@ -178,7 +132,6 @@ def test_function_one_argument(self): self.assertEqual(function.name, "myfeature") self.assertEqual(function.expression, "fieldMatch(title) + freshness(foo)") self.assertEqual(function.args, ["foo"]) - self.assertEqual(function, Function.from_dict(function.to_dict)) self.assertEqual(function.args_to_text, "foo") def test_function_multiple_argument(self): @@ -190,7 +143,6 @@ def test_function_multiple_argument(self): self.assertEqual(function.name, "myfeature") self.assertEqual(function.expression, "fieldMatch(bar) + freshness(foo)") self.assertEqual(function.args, ["foo", "bar"]) - self.assertEqual(function, Function.from_dict(function.to_dict)) self.assertEqual(function.args_to_text, "foo, bar") def test_function_multiple_lines(self): @@ -220,7 +172,6 @@ def test_function_multiple_lines(self): """, ) self.assertEqual(function.args, None) - self.assertEqual(function, Function.from_dict(function.to_dict)) self.assertEqual(function.args_to_text, "") @@ -229,7 +180,6 @@ def test_rank_profile(self): rank_profile = RankProfile(name="bm25", first_phase="bm25(title) + bm25(body)") self.assertEqual(rank_profile.name, "bm25") self.assertEqual(rank_profile.first_phase, "bm25(title) + bm25(body)") - self.assertEqual(rank_profile, RankProfile.from_dict(rank_profile.to_dict)) def test_rank_profile_inherits(self): rank_profile = RankProfile( @@ -237,7 +187,6 @@ def test_rank_profile_inherits(self): ) self.assertEqual(rank_profile.name, "bm25") self.assertEqual(rank_profile.first_phase, "bm25(title) + bm25(body)") - self.assertEqual(rank_profile, RankProfile.from_dict(rank_profile.to_dict)) def test_rank_profile_bert_second_phase(self): rank_profile = RankProfile( @@ -305,25 +254,6 @@ def test_rank_profile_bert_second_phase(self): rank_profile.summary_features, ["onnx(bert).logits", "input_ids", "attention_mask", "token_type_ids"], ) - self.assertEqual(rank_profile, RankProfile.from_dict(rank_profile.to_dict)) - - -class TestOnnxModel(unittest.TestCase): - def test_onnx_model(self): - onnx_model = OnnxModel( - model_name="bert", - model_file_path="bert.onnx", - inputs={ - "input_ids": "input_ids", - "token_type_ids": "token_type_ids", - "attention_mask": "attention_mask", - }, - outputs={"logits": "logits"}, - ) - self.assertEqual( - onnx_model, - OnnxModel.from_dict(onnx_model.to_dict), - ) class TestSchema(unittest.TestCase): @@ -357,9 +287,6 @@ def setUp(self) -> None: ], ) - def test_serialization(self): - self.assertEqual(self.schema, Schema.from_dict(self.schema.to_dict)) - def test_rank_profile(self): self.assertDictEqual( self.schema.rank_profiles, @@ -418,28 +345,19 @@ def test_field_name_type(self): field = QueryTypeField(name="test_name", type="string") self.assertEqual(field.name, "test_name") self.assertEqual(field.type, "string") - self.assertEqual(field.to_dict, {"name": "test_name", "type": "string"}) self.assertEqual(field, QueryTypeField(name="test_name", type="string")) - self.assertEqual(field, QueryTypeField.from_dict(field.to_dict)) class TestQueryProfileType(unittest.TestCase): def test_empty(self): query_profile_type = QueryProfileType() self.assertEqual(query_profile_type.fields, []) - self.assertEqual(query_profile_type.to_dict, {"fields": []}) - self.assertEqual( - query_profile_type, QueryProfileType.from_dict(query_profile_type.to_dict) - ) def test_one_field(self): query_profile_type = QueryProfileType() field = QueryTypeField(name="test_name", type="string") query_profile_type.add_fields(field) self.assertEqual(query_profile_type.fields, [field]) - self.assertEqual( - query_profile_type, QueryProfileType.from_dict(query_profile_type.to_dict) - ) self.assertEqual(query_profile_type, QueryProfileType([field])) def test_two_fields(self): @@ -451,9 +369,6 @@ def test_two_fields(self): ) query_profile_type.add_fields(field_1, field_2) self.assertEqual(query_profile_type.fields, [field_1, field_2]) - self.assertEqual( - query_profile_type, QueryProfileType.from_dict(query_profile_type.to_dict) - ) self.assertEqual(query_profile_type, QueryProfileType([field_1, field_2])) @@ -462,24 +377,19 @@ def test_field_name_type(self): field = QueryField(name="test_name", value=1) self.assertEqual(field.name, "test_name") self.assertEqual(field.value, 1) - self.assertEqual(field.to_dict, {"name": "test_name", "value": 1}) self.assertEqual(field, QueryField(name="test_name", value=1)) - self.assertEqual(field, QueryField.from_dict(field.to_dict)) class TestQueryProfile(unittest.TestCase): def test_empty(self): query_profile = QueryProfile() self.assertEqual(query_profile.fields, []) - self.assertEqual(query_profile.to_dict, {"fields": []}) - self.assertEqual(query_profile, QueryProfile.from_dict(query_profile.to_dict)) def test_one_field(self): query_profile = QueryProfile() field = QueryField(name="test_name", value=2.0) query_profile.add_fields(field) self.assertEqual(query_profile.fields, [field]) - self.assertEqual(query_profile, QueryProfile.from_dict(query_profile.to_dict)) self.assertEqual(query_profile, QueryProfile([field])) def test_two_fields(self): @@ -491,7 +401,6 @@ def test_two_fields(self): ) query_profile.add_fields(field_1, field_2) self.assertEqual(query_profile.fields, [field_1, field_2]) - self.assertEqual(query_profile, QueryProfile.from_dict(query_profile.to_dict)) self.assertEqual(query_profile, QueryProfile([field_1, field_2])) @@ -621,11 +530,6 @@ def setUp(self) -> None: query_profile_type=test_query_profile_type, ) - def test_application_package(self): - self.assertEqual( - self.app_package, ApplicationPackage.from_dict(self.app_package.to_dict) - ) - def test_get_schema(self): self.assertEqual(self.app_package.schema, self.test_schema) self.assertEqual(self.app_package.schema, self.app_package.get_schema()) @@ -834,11 +738,6 @@ def setUp(self) -> None: schema=[self.news_schema, self.user_schema, self.category_ctr_schema], ) - def test_application_package(self): - self.assertEqual( - self.app_package, ApplicationPackage.from_dict(self.app_package.to_dict) - ) - def test_get_schema(self): self.assertEqual(self.app_package.get_schema(name="news"), self.news_schema) self.assertEqual(self.app_package.get_schema(name="user"), self.user_schema) @@ -963,11 +862,6 @@ def setUp(self) -> None: self.disk_folder = "saved_app" - def test_application_package(self): - self.assertEqual( - self.app_package, ApplicationPackage.from_dict(self.app_package.to_dict) - ) - def test_news_schema_to_text(self): expected_result = ( "schema news {\n" @@ -1164,11 +1058,6 @@ def setUp(self) -> None: QueryField(name="anotherField", value="string_value"), ) - def test_application_package(self): - self.assertEqual( - self.app_package, ApplicationPackage.from_dict(self.app_package.to_dict) - ) - def test_schema_to_text(self): expected_result = ( "schema testapp {\n" @@ -1279,11 +1168,6 @@ def setUp(self) -> None: ) ) - def test_application_package(self): - self.assertEqual( - self.app_package, ApplicationPackage.from_dict(self.app_package.to_dict) - ) - def test_schema_to_text(self): expected_news_result = ( "schema news {\n" @@ -1392,11 +1276,6 @@ def setUp(self) -> None: ) self.disk_folder = "saved_app" - def test_application_package(self): - self.assertEqual( - self.app_package, ApplicationPackage.from_dict(self.app_package.to_dict) - ) - def test_schema_to_text(self): expected_result = ( "schema testapp {\n" From 59dbb7ff87553f0ffac7ec00aeec77e9c440cf2b Mon Sep 17 00:00:00 2001 From: tmartins Date: Fri, 4 Nov 2022 09:24:32 -0300 Subject: [PATCH 2/3] remove more serialization code --- vespa/deployment.py | 44 ++------------------------------ vespa/package.py | 14 +--------- vespa/test_integration_docker.py | 7 ----- vespa/test_package.py | 12 --------- 4 files changed, 3 insertions(+), 74 deletions(-) diff --git a/vespa/deployment.py b/vespa/deployment.py index bb5f52dc..4062fce4 100644 --- a/vespa/deployment.py +++ b/vespa/deployment.py @@ -9,7 +9,7 @@ from io import BytesIO from pathlib import Path from time import sleep, strftime, gmtime -from typing import Union, IO, Optional, Mapping +from typing import Union, IO, Optional import docker import requests @@ -19,7 +19,6 @@ from cryptography.hazmat.primitives.asymmetric import ec from vespa.application import Vespa -from vespa.json_serialization import ToJson, FromJson from vespa.package import ApplicationPackage CFG_SERVER_START_TIMEOUT = 300 @@ -27,7 +26,7 @@ DOCKER_TIMEOUT = 600 -class VespaDocker(ToJson, FromJson["VespaDocker"]): +class VespaDocker(object): def __init__( self, port: int = 8080, @@ -310,45 +309,6 @@ def restart_services(self): self.stop_services() self.start_services() - @staticmethod - def from_dict(mapping: Mapping) -> "VespaDocker": - try: - if mapping["container_id"] is not None: - vespa_docker = VespaDocker.from_container_name_or_id( - name_or_id=mapping["container_id"] - ) - return vespa_docker - elif mapping["container_name"] is not None: - vespa_docker = VespaDocker.from_container_name_or_id( - name_or_id=mapping["container_name"] - ) - return vespa_docker - else: - print( - "Unable to instantiate VespaDocker from a running container. Starting new container." - ) - except ValueError: - print( - "Unable to instantiate VespaDocker from a running container. Starting new container." - ) - vespa_docker = VespaDocker( - port=mapping["port"], - container_memory=mapping["container_memory"], - container_image=mapping["container_image"], - ) - return vespa_docker - - @property - def to_dict(self) -> Mapping: - return { - "container_id": self.container_id, - "container_name": self.container_name, - "url": self.url, - "port": self.local_port, - "container_memory": self.container_memory, - "container_image": self.container_image, - } - def __eq__(self, other): if not isinstance(other, self.__class__): return False diff --git a/vespa/package.py b/vespa/package.py index 0a8568a3..aadecc68 100644 --- a/vespa/package.py +++ b/vespa/package.py @@ -3,7 +3,7 @@ from pathlib import Path from shutil import copyfile -from typing import List, Mapping, Optional, Union, Dict +from typing import List, Optional, Union, Dict from collections import OrderedDict from jinja2 import Environment, PackageLoader, select_autoescape from io import BytesIO @@ -1022,18 +1022,6 @@ def services_to_text(self): def _application_package_file_name(disk_folder): return os.path.join(disk_folder, "application_package.json") - def save(self, disk_folder: str) -> None: - Path(disk_folder).mkdir(parents=True, exist_ok=True) - file_path = ApplicationPackage._application_package_file_name(disk_folder) - with open(file_path, "w") as f: - f.write(self.to_json) - - @staticmethod - def load(disk_folder: str) -> "ApplicationPackage": - file_path = ApplicationPackage._application_package_file_name(disk_folder) - with open(file_path, "r") as f: - return ApplicationPackage.from_json(f.read()) - def to_zip(self) -> BytesIO: """ Return the application package as zipped bytes, diff --git a/vespa/test_integration_docker.py b/vespa/test_integration_docker.py index 6aa19331..c10e304e 100644 --- a/vespa/test_integration_docker.py +++ b/vespa/test_integration_docker.py @@ -237,13 +237,6 @@ def deploy(self, application_package, container_image=None): self.vespa_docker.deploy(application_package=application_package) except RuntimeError as e: assert False, "Deployment error: {}".format(e) - # - # Test VespaDocker serialization - # - self.assertEqual( - repr(self.vespa_docker), - repr(VespaDocker.from_dict(self.vespa_docker.to_dict)), - ) def create_vespa_docker_from_container_name_or_id(self, application_package): # diff --git a/vespa/test_package.py b/vespa/test_package.py index e903df97..bc50c146 100644 --- a/vespa/test_package.py +++ b/vespa/test_package.py @@ -994,12 +994,6 @@ def test_query_profile_type_to_text(self): ) self.assertEqual(self.app_package.query_profile_type_to_text, expected_result) - def test_save_load(self): - self.app_package.save(disk_folder=self.disk_folder) - self.assertEqual( - self.app_package, ApplicationPackage.load(disk_folder=self.disk_folder) - ) - def tearDown(self) -> None: rmtree(self.disk_folder, ignore_errors=True) @@ -1416,12 +1410,6 @@ def test_query_profile_type_to_text(self): ) self.assertEqual(self.app_package.query_profile_type_to_text, expected_result) - def test_save_load(self): - self.app_package.save(disk_folder=self.disk_folder) - self.assertEqual( - self.app_package, ApplicationPackage.load(disk_folder=self.disk_folder) - ) - def tearDown(self) -> None: rmtree(self.disk_folder, ignore_errors=True) From 53cbacadb4d39886896fc89cb58d42a8d71faf01 Mon Sep 17 00:00:00 2001 From: tmartins Date: Fri, 4 Nov 2022 09:25:22 -0300 Subject: [PATCH 3/3] remove serialization code --- vespa/json_serialization.py | 78 ------------------------------------- 1 file changed, 78 deletions(-) delete mode 100644 vespa/json_serialization.py diff --git a/vespa/json_serialization.py b/vespa/json_serialization.py deleted file mode 100644 index a9bf43a5..00000000 --- a/vespa/json_serialization.py +++ /dev/null @@ -1,78 +0,0 @@ -import datetime -import json -import typing -from pkg_resources import get_distribution - -T = typing.TypeVar("T") - - -class ToJson(object): - """ - Utility mix-in class for serializing an object to JSON. It does not really - do any conversion on its own, but forces serialization into a standardized - API. - - The serialized class is put into an envelope with some data to make it easier - to understand what has happened. - - { - "version": 0.1.0, - "class": "Field", - "serialized_at": "2018-10-24T12:55:32+00:00", - "data": { ... } - } - - * version: pyvespa package version. - * class: The name of the class we serialized. For debugging purposes. - * serialized_at: The time we serialized the instance of the class. For debugging purposes. - * data: The actual data of the serialized class. - - All serialization is based on converting objects to a `dict` which is then converted - to JSON using the standard Python json library. - """ - - @property - def to_dict(self) -> typing.Mapping: - raise NotImplementedError - - @property - def to_envelope(self) -> typing.Mapping: - return { - "version": get_distribution("pyvespa").version, - "class": self.__class__.__name__, - "serialized_at": datetime.datetime.utcnow().isoformat(), - "data": self.to_dict, - } - - @property - def to_json(self) -> str: - mapping = self.to_envelope - return json.dumps(mapping) - - -class FromJson(typing.Generic[T]): - """ - A mix-in class for deserializing from JSON to an object that implements this class. - All JSON must have the same envelope as ToJson to be able to properly deserialize the - contents of the mapping. - """ - - deserializers: typing.MutableMapping[str, "FromJson"] = {} - - def __init_subclass__(cls, **kwargs): - super().__init_subclass__(**kwargs) # type: ignore - FromJson.deserializers[cls.__name__] = cls - - @staticmethod - def from_json(json_string: str) -> T: - mapping = json.loads(json_string) - return FromJson.map(mapping) - - @staticmethod - def map(mapping: typing.Mapping) -> T: - mapping_class = FromJson.deserializers[mapping["class"]] - return mapping_class.from_dict(mapping["data"]) - - @staticmethod - def from_dict(mapping: typing.Mapping) -> T: - raise NotImplementedError