From 994196c5354c02f40c223cf4fe3388aba09ea020 Mon Sep 17 00:00:00 2001 From: Arne H Juul Date: Fri, 11 Mar 2022 09:41:24 +0000 Subject: [PATCH 1/9] ensure unique ReferenceDataType instances --- .../com/yahoo/documentmodel/NewDocumentType.java | 16 ++++++++++++++++ .../searchdefinition/DocumentModelBuilder.java | 13 ++----------- .../documentmanager_refs_to_same_type.cfg | 8 +++----- 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java index d98869e9dd30..4ff54d7ff1cb 100644 --- a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java +++ b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java @@ -4,8 +4,10 @@ import com.yahoo.document.DataType; import com.yahoo.document.Document; import com.yahoo.document.Field; +import com.yahoo.document.ReferenceDataType; import com.yahoo.document.StructDataType; import com.yahoo.document.StructuredDataType; +import com.yahoo.document.TemporaryStructuredDataType; import com.yahoo.document.annotation.AnnotationType; import com.yahoo.document.annotation.AnnotationTypeRegistry; import com.yahoo.document.datatypes.FieldValue; @@ -383,4 +385,18 @@ public boolean equals(Object other) { } + private ReferenceDataType refToThis = null; + + @SuppressWarnings("deprecation") + public ReferenceDataType getReferenceDataType() { + if (refToThis == null) { + // super ugly, the APIs for this are horribly inconsistent + var tmptmp = TemporaryStructuredDataType.create(getName()); + var tmp = ReferenceDataType.createWithInferredId(tmptmp); + tmp.setTargetType((StructuredDataType) this); + refToThis = tmp; + } + return refToThis; + } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java index 4a449dc898f3..2d9c81085fe1 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java @@ -292,17 +292,8 @@ else if (type instanceof WeightedSetDataType) { else if (type instanceof ReferenceDataType) { ReferenceDataType t = (ReferenceDataType) type; var tt = t.getTargetType(); - if (tt instanceof TemporaryStructuredDataType) { - DataType targetType = resolveTemporariesRecurse(tt, repo, docs, replacements); - t.setTargetType((StructuredDataType) targetType); - } else if (tt instanceof DocumentType) { - DataType targetType = resolveTemporariesRecurse(tt, repo, docs, replacements); - // super ugly, the APIs for this are horribly inconsistent - var tmptmp = TemporaryStructuredDataType.create(tt.getName()); - var tmp = new ReferenceDataType(tmptmp, t.getId()); - tmp.setTargetType((StructuredDataType) targetType); - type = tmp; - } + var doc = getDocumentType(docs, tt.getId()); + type = doc.getReferenceDataType(); } if (type != original) { replacements.add(new TypeReplacement(original, type)); diff --git a/config-model/src/test/configmodel/types/references/documentmanager_refs_to_same_type.cfg b/config-model/src/test/configmodel/types/references/documentmanager_refs_to_same_type.cfg index 876ed00d0c4c..ef1bb4c5ad4a 100644 --- a/config-model/src/test/configmodel/types/references/documentmanager_refs_to_same_type.cfg +++ b/config-model/src/test/configmodel/types/references/documentmanager_refs_to_same_type.cfg @@ -47,8 +47,6 @@ doctype[1].fieldsets{[document]}.fields[0] "campaign_ref" doctype[1].fieldsets{[document]}.fields[1] "other_campaign_ref" doctype[1].documentref[0].idx 10017 doctype[1].documentref[0].targettype 10018 -doctype[1].documentref[1].idx 10019 -doctype[1].documentref[1].targettype 10018 doctype[1].structtype[0].idx 10016 doctype[1].structtype[0].name "ad.header" doctype[1].structtype[0].field[0].name "campaign_ref" @@ -56,10 +54,10 @@ doctype[1].structtype[0].field[0].internalid 23963250 doctype[1].structtype[0].field[0].type 10017 doctype[1].structtype[0].field[1].name "other_campaign_ref" doctype[1].structtype[0].field[1].internalid 874751172 -doctype[1].structtype[0].field[1].type 10019 +doctype[1].structtype[0].field[1].type 10017 doctype[2].name "campaign" doctype[2].idx 10018 doctype[2].inherits[0].idx 10000 -doctype[2].contentstruct 10020 -doctype[2].structtype[0].idx 10020 +doctype[2].contentstruct 10019 +doctype[2].structtype[0].idx 10019 doctype[2].structtype[0].name "campaign.header" From 9976089dd3ea2516425cf24677ff693f29ba3e8c Mon Sep 17 00:00:00 2001 From: Arne H Juul Date: Fri, 11 Mar 2022 11:43:17 +0000 Subject: [PATCH 2/9] inherit directly * instead of indirect (temporary) inherit from name, inherit the already-converted document directly. * keep schema order. --- .../searchdefinition/parser/ConvertSchemaCollection.java | 9 ++++++++- .../searchdefinition/parser/IntermediateCollection.java | 4 ++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertSchemaCollection.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertSchemaCollection.java index d32fae99f5ab..67e6c88d0437 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertSchemaCollection.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertSchemaCollection.java @@ -33,7 +33,9 @@ import com.yahoo.vespa.documentmodel.SummaryField; import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.Optional; /** @@ -122,6 +124,8 @@ public void convertTypes() { typeConverter.convert(true); } + private Map convertedDocuments = new LinkedHashMap(); + public List convertToSchemas() { typeConverter = new ConvertParsedTypes(orderedInput, docMan); typeConverter.convert(false); @@ -150,7 +154,9 @@ private void convertDocument(Schema schema, ParsedDocument parsed, { SDDocumentType document = new SDDocumentType(parsed.name()); for (String inherit : parsed.getInherited()) { - document.inherit(new DataTypeName(inherit)); + var parent = convertedDocuments.get(inherit); + assert(parent != null); + document.inherit(parent); } for (var struct : parsed.getStructs()) { var structProxy = fieldConverter.convertStructDeclaration(schema, struct); @@ -165,6 +171,7 @@ private void convertDocument(Schema schema, ParsedDocument parsed, document.setFieldId(sdf, field.idOverride()); } } + convertedDocuments.put(parsed.name(), document); schema.addDocument(document); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/IntermediateCollection.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/IntermediateCollection.java index 536caf551110..23b5195486b0 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/IntermediateCollection.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/IntermediateCollection.java @@ -12,7 +12,7 @@ import java.io.File; import java.util.ArrayList; -import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -27,7 +27,7 @@ public class IntermediateCollection { private final DeployLogger deployLogger; private final ModelContext.Properties modelProperties; - private Map parsedSchemas = new HashMap<>(); + private Map parsedSchemas = new LinkedHashMap<>(); IntermediateCollection() { this.deployLogger = new BaseDeployLogger(); From ba2758ab815ebe7ab2fa9b702b3ace7d5c0fd0d7 Mon Sep 17 00:00:00 2001 From: Arne H Juul Date: Fri, 11 Mar 2022 13:28:14 +0000 Subject: [PATCH 3/9] connect SDField to "repo" * when creating SDField inside a struct, it needs connection to the "repo" (SDDocument) for lookup of other types. The StructProxy would be the "owner" here, and is not necessary. --- .../searchdefinition/parser/ConvertParsedFields.java | 8 ++++---- .../searchdefinition/parser/ConvertSchemaCollection.java | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedFields.java index 36c11b33b23b..caeebd65f4fe 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedFields.java @@ -291,13 +291,12 @@ void convertExtraIndex(Schema schema, ParsedIndex parsed) { schema.addIndex(index); } - SDDocumentType convertStructDeclaration(Schema schema, ParsedStruct parsed) { + SDDocumentType convertStructDeclaration(Schema schema, SDDocumentType document, ParsedStruct parsed) { // TODO - can we cleanup this mess var structProxy = new SDDocumentType(parsed.name(), schema); - structProxy.setStruct(context.resolveStruct(parsed)); for (var parsedField : parsed.getFields()) { var fieldType = context.resolveType(parsedField.getType()); - var field = new SDField(structProxy, parsedField.name(), fieldType); + var field = new SDField(document, parsedField.name(), fieldType); convertCommonFieldSettings(field, parsedField); structProxy.addField(field); if (parsedField.hasIdOverride()) { @@ -307,6 +306,7 @@ SDDocumentType convertStructDeclaration(Schema schema, ParsedStruct parsed) { for (String inherit : parsed.getInherited()) { structProxy.inherit(new DataTypeName(inherit)); } + structProxy.setStruct(context.resolveStruct(parsed)); return structProxy; } @@ -314,7 +314,7 @@ void convertAnnotation(Schema schema, SDDocumentType document, ParsedAnnotation var annType = context.resolveAnnotation(parsed.name()); var payload = parsed.getStruct(); if (payload.isPresent()) { - var structProxy = convertStructDeclaration(schema, payload.get()); + var structProxy = convertStructDeclaration(schema, document, payload.get()); document.addType(structProxy); } document.addAnnotation(annType); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertSchemaCollection.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertSchemaCollection.java index 67e6c88d0437..99614227714a 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertSchemaCollection.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertSchemaCollection.java @@ -159,7 +159,7 @@ private void convertDocument(Schema schema, ParsedDocument parsed, document.inherit(parent); } for (var struct : parsed.getStructs()) { - var structProxy = fieldConverter.convertStructDeclaration(schema, struct); + var structProxy = fieldConverter.convertStructDeclaration(schema, document, struct); document.addType(structProxy); } for (var annotation : parsed.getAnnotations()) { @@ -221,7 +221,7 @@ private void convertFieldSet(Schema schema, ParsedFieldSet parsed) { } if (parsed.getMatchSettings().isPresent()) { // same ugliness as SDParser.jj used to have: - var tmp = new SDField(setName, DataType.STRING); + var tmp = new SDField(null, setName, DataType.STRING); ConvertParsedFields.convertMatchSettings(tmp, parsed.matchSettings()); schema.fieldSets().userFieldSets().get(setName).setMatching(tmp.getMatching()); } From ee4fb1f44497518575d0cd38afee55b4a20706cf Mon Sep 17 00:00:00 2001 From: Arne H Juul Date: Tue, 8 Mar 2022 11:27:42 +0000 Subject: [PATCH 4/9] even TemporarySDField should be connected to a "repo" --- .../document/TemporarySDField.java | 8 ++++---- .../parser/ConvertSchemaCollection.java | 2 +- config-model/src/main/javacc/SDParser.jj | 20 ++++++++++--------- .../DocumentGraphValidatorTest.java | 2 +- .../SDDocumentTypeOrdererTestCase.java | 12 +++++------ .../derived/SchemaOrdererTestCase.java | 2 +- .../derived/VsmFieldsTestCase.java | 5 +++-- .../AdjustPositionSummaryFieldsTestCase.java | 5 +++-- .../ImportedFieldsResolverTestCase.java | 15 +++++++------- .../processing/ParentChildSearchModel.java | 10 +++++----- .../java/com/yahoo/document/DataType.java | 1 + 11 files changed, 44 insertions(+), 38 deletions(-) diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java index 4ced104fa55d..6a47d08e3ac1 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java @@ -8,12 +8,12 @@ */ public class TemporarySDField extends SDField { - public TemporarySDField(String name, DataType dataType, SDDocumentType owner) { - super(owner, name, dataType, owner, false); + public TemporarySDField(SDDocumentType repo, String name, DataType dataType, SDDocumentType owner) { + super(repo, name, dataType, owner, false); } - public TemporarySDField(String name, DataType dataType) { - super(null, name, dataType, false); + public TemporarySDField(SDDocumentType repo, String name, DataType dataType) { + super(repo, name, dataType, false); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertSchemaCollection.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertSchemaCollection.java index 99614227714a..2d9a788cfef5 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertSchemaCollection.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertSchemaCollection.java @@ -221,7 +221,7 @@ private void convertFieldSet(Schema schema, ParsedFieldSet parsed) { } if (parsed.getMatchSettings().isPresent()) { // same ugliness as SDParser.jj used to have: - var tmp = new SDField(null, setName, DataType.STRING); + var tmp = new SDField(setName, DataType.STRING); ConvertParsedFields.convertMatchSettings(tmp, parsed.matchSettings()); schema.fieldSets().userFieldSets().get(setName).setMatching(tmp.getMatching()); } diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj index 1a0b518bbb2a..1fec33c4f128 100644 --- a/config-model/src/main/javacc/SDParser.jj +++ b/config-model/src/main/javacc/SDParser.jj @@ -639,7 +639,7 @@ void field(SDDocumentType document, Schema schema) : if (name != null && Schema.isReservedName(name.toLowerCase())) { throw new IllegalArgumentException("Reserved name '" + name + "' can not be used as a field name."); } - field = new TemporarySDField(name, type, document); + field = new TemporarySDField(document, name, type, document); } lbrace() (fieldBody(field, schema, document) ()*)* { @@ -698,11 +698,13 @@ void annotationOutside(Schema schema) : name = identifier() { type = new SDAnnotationType(name.trim()); + if (schema.getDocument() == null) { + throw new IllegalArgumentException("Can't add annotation '"+name+"' to a document type, define a document type first or declare the annotation inside of one."); + } } [ inheritsAnnotation(type) ()* ] - lbrace() (type = annotationBody(schema, type)) + lbrace() (type = annotationBody(schema, schema.getDocument(), type)) { - if (schema.getDocument()==null) throw new IllegalArgumentException("Can't add annotation '"+name+"' to a document type, define a document type first or declare the annotation inside of one."); schema.addAnnotation(type); } } @@ -723,7 +725,7 @@ void annotation(Schema schema, SDDocumentType document) : type = new SDAnnotationType(name.trim()); } [ inheritsAnnotation(type) ()* ] - lbrace() (type = annotationBody(schema, type)) + lbrace() (type = annotationBody(schema, document, type)) { document.addAnnotation(type); } @@ -737,12 +739,12 @@ void annotation(Schema schema, SDDocumentType document) : * @param type the type being built * @return a modified or new AnnotationType instance */ -SDAnnotationType annotationBody(Schema schema, SDAnnotationType type) : +SDAnnotationType annotationBody(Schema schema, SDDocumentType repo, SDAnnotationType type) : { SDDocumentType struct = new SDDocumentType("annotation." + type.getName(), schema); } { - (structFieldDefinition(struct) ()*)* + (structFieldDefinition(repo, struct) ()*)* { if (struct.getFieldCount() > 0) { // Must account for the temporary TemporarySDField. type = new SDAnnotationType(type.getName(), struct, type.getInherits()); @@ -811,7 +813,7 @@ SDDocumentType structDefinition(Schema schema, SDDocumentType repo) : { ( name = identifier() ()* { struct = new SDDocumentType(name, schema); } [ inheritsDocument(struct) ()* ] - lbrace() (structFieldDefinition(struct) ()*)* ) + lbrace() (structFieldDefinition(repo, struct) ()*)* ) { try { docMan.getDataType(name); @@ -921,7 +923,7 @@ DataType wildCardType() : * * @param struct The struct to modify. */ -void structFieldDefinition(SDDocumentType struct) : +void structFieldDefinition(SDDocumentType document, SDDocumentType struct) : { String name; SDField field; @@ -932,7 +934,7 @@ void structFieldDefinition(SDDocumentType struct) : if (name != null && Schema.isReservedName(name.toLowerCase())) { throw new IllegalArgumentException("Reserved name '" + name + "' can not be used as a field name."); } - field = new TemporarySDField(name, type, struct); + field = new TemporarySDField(document, name, type, struct); struct.addField(field); } lbrace() (id(field,struct) ()*)? (match(field) ()*)* { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentGraphValidatorTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/DocumentGraphValidatorTest.java index 81a44261daf1..8edbd789a27c 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentGraphValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/DocumentGraphValidatorTest.java @@ -156,8 +156,8 @@ private static Schema createSearchWithName(String name, Schema... parents) { @SuppressWarnings("deprecation") private static void createDocumentReference(Schema from, Schema to, String refFieldName) { - SDField refField = new TemporarySDField(refFieldName, ReferenceDataType.createWithInferredId(TemporaryStructuredDataType.create(to.getName()))); SDDocumentType fromDocument = from.getDocument(); + SDField refField = new TemporarySDField(fromDocument, refFieldName, ReferenceDataType.createWithInferredId(TemporaryStructuredDataType.create(to.getName()))); fromDocument.addField(refField); Map originalMap = fromDocument.getDocumentReferences().get().referenceMap(); HashMap modifiedMap = new HashMap<>(originalMap); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/SDDocumentTypeOrdererTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/SDDocumentTypeOrdererTestCase.java index c7ec2d9e9d14..652a06a60252 100755 --- a/config-model/src/test/java/com/yahoo/searchdefinition/SDDocumentTypeOrdererTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/SDDocumentTypeOrdererTestCase.java @@ -37,22 +37,22 @@ public void testOrder() { g.inherit(new TemporarySDDocumentType(new DataTypeName("e"))); g.inherit(new TemporarySDDocumentType(new DataTypeName("c"))); - SDField aFieldTypeB = new TemporarySDField("atypeb", DataType.STRING); + SDField aFieldTypeB = new TemporarySDField(a, "atypeb", DataType.STRING); a.addField(aFieldTypeB); - SDField bFieldTypeC = new TemporarySDField("btypec", DataType.STRING); + SDField bFieldTypeC = new TemporarySDField(b, "btypec", DataType.STRING); b.addField(bFieldTypeC); - SDField cFieldTypeG = new TemporarySDField("ctypeg", DataType.STRING); + SDField cFieldTypeG = new TemporarySDField(c, "ctypeg", DataType.STRING); c.addField(cFieldTypeG); - SDField gFieldTypeF = new TemporarySDField("gtypef", DataType.STRING); + SDField gFieldTypeF = new TemporarySDField(g, "gtypef", DataType.STRING); g.addField(gFieldTypeF); - SDField fFieldTypeC = new TemporarySDField("ftypec", DataType.STRING); + SDField fFieldTypeC = new TemporarySDField(f, "ftypec", DataType.STRING); f.addField(fFieldTypeC); - SDField dFieldTypeE = new TemporarySDField("dtypee", DataType.STRING); + SDField dFieldTypeE = new TemporarySDField(d, "dtypee", DataType.STRING); d.addField(dFieldTypeE); types.add(a); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SchemaOrdererTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SchemaOrdererTestCase.java index 34d33a00d9eb..2522f8f56e27 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SchemaOrdererTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SchemaOrdererTestCase.java @@ -91,8 +91,8 @@ private static void assertOrder(List expectedSearchOrder, List i @SuppressWarnings("deprecation") private static void createDocumentReference(Schema from, Schema to, String refFieldName) { - SDField refField = new TemporarySDField(refFieldName, ReferenceDataType.createWithInferredId(TemporaryStructuredDataType.create(to.getName()))); SDDocumentType fromDocument = from.getDocument(); + SDField refField = new TemporarySDField(fromDocument, refFieldName, ReferenceDataType.createWithInferredId(TemporaryStructuredDataType.create(to.getName()))); fromDocument.addField(refField); Map originalMap = fromDocument.getDocumentReferences().get().referenceMap(); HashMap modifiedMap = new HashMap<>(originalMap); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/VsmFieldsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/VsmFieldsTestCase.java index 5ab5a8057e84..9c9742256056 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/VsmFieldsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/VsmFieldsTestCase.java @@ -25,8 +25,9 @@ public class VsmFieldsTestCase { @Test public void reference_type_field_is_unsearchable() { Schema schema = new Schema("test", MockApplicationPackage.createEmpty(), new MockFileRegistry(), new TestableDeployLogger(), new TestProperties()); - schema.addDocument(new SDDocumentType("test")); - SDField refField = new TemporarySDField("ref_field", ReferenceDataType.createWithInferredId(TemporaryStructuredDataType.create("parent_type"))); + var sdoc = new SDDocumentType("test"); + schema.addDocument(sdoc); + SDField refField = new TemporarySDField(sdoc, "ref_field", ReferenceDataType.createWithInferredId(TemporaryStructuredDataType.create("parent_type"))); refField.parseIndexingScript("{ summary }"); schema.getDocument().addField(refField); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AdjustPositionSummaryFieldsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AdjustPositionSummaryFieldsTestCase.java index 5d2590a420d1..926e26451d77 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AdjustPositionSummaryFieldsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AdjustPositionSummaryFieldsTestCase.java @@ -186,9 +186,10 @@ static class SearchModel extends ParentChildSearchModel { private void createPositionField(Schema schema, boolean setupPosAttr, boolean setupBadAttr) { String ilScript = setupPosAttr ? "{ summary | attribute }" : "{ summary }"; - schema.getDocument().addField(createField("pos", PositionDataType.INSTANCE, ilScript)); + var doc = schema.getDocument(); + doc.addField(createField(doc, "pos", PositionDataType.INSTANCE, ilScript)); if (setupBadAttr) { - schema.getDocument().addField(createField("pos_zcurve", DataType.LONG, "{ attribute }")); + doc.addField(createField(doc, "pos_zcurve", DataType.LONG, "{ attribute }")); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolverTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolverTestCase.java index 324010f9e83d..d1d3f4489cea 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolverTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolverTestCase.java @@ -110,13 +110,14 @@ static class SearchModel extends ParentChildSearchModel { public SearchModel() { super(); grandParentSchema = createSearch("grandparent"); - grandParentSchema.getDocument().addField(createField("ancient_field", DataType.INT, "{ attribute }")); - - parentSchema.getDocument().addField(createField("attribute_field", DataType.INT, "{ attribute }")); - parentSchema.getDocument().addField(createField("attribute_and_index", DataType.INT, "{ attribute | index }")); - parentSchema.getDocument().addField(new TemporarySDField("not_attribute", DataType.INT)); - parentSchema.getDocument().addField(createField("tensor_field", new TensorDataType(TensorType.fromSpec("tensor(x[5])")), "{ attribute }")); - parentSchema.getDocument().addField(createField("predicate_field", DataType.PREDICATE, "{ attribute }")); + var grandParentDoc = grandParentSchema.getDocument(); + grandParentDoc.addField(createField(grandParentDoc, "ancient_field", DataType.INT, "{ attribute }")); + var parentDoc = parentSchema.getDocument(); + parentDoc.addField(createField(parentDoc, "attribute_field", DataType.INT, "{ attribute }")); + parentDoc.addField(createField(parentDoc, "attribute_and_index", DataType.INT, "{ attribute | index }")); + parentDoc.addField(new TemporarySDField(parentDoc, "not_attribute", DataType.INT)); + parentDoc.addField(createField(parentDoc, "tensor_field", new TensorDataType(TensorType.fromSpec("tensor(x[5])")), "{ attribute }")); + parentDoc.addField(createField(parentDoc, "predicate_field", DataType.PREDICATE, "{ attribute }")); addRefField(parentSchema, grandParentSchema, "ref"); addImportedField(parentSchema, "ancient_field", "ref", "ancient_field"); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ParentChildSearchModel.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ParentChildSearchModel.java index b14c7287537f..3b4612ee87ad 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ParentChildSearchModel.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ParentChildSearchModel.java @@ -37,19 +37,19 @@ protected Schema createSearch(String name) { return result; } - protected static TemporarySDField createField(String name, DataType dataType, String indexingScript) { - TemporarySDField result = new TemporarySDField(name, dataType); + protected static TemporarySDField createField(SDDocumentType repo, String name, DataType dataType, String indexingScript) { + TemporarySDField result = new TemporarySDField(repo, name, dataType); result.parseIndexingScript(indexingScript); return result; } @SuppressWarnings("deprecation") - protected static SDField createRefField(String parentType, String fieldName) { - return new TemporarySDField(fieldName, ReferenceDataType.createWithInferredId(TemporaryStructuredDataType.create(parentType))); + protected static SDField createRefField(SDDocumentType repo, String parentType, String fieldName) { + return new TemporarySDField(repo, fieldName, ReferenceDataType.createWithInferredId(TemporaryStructuredDataType.create(parentType))); } protected static void addRefField(Schema child, Schema parent, String fieldName) { - SDField refField = createRefField(parent.getName(), fieldName); + SDField refField = createRefField(child.getDocument(), parent.getName(), fieldName); child.getDocument().addField(refField); child.getDocument().setDocumentReferences(new DocumentReferences(ImmutableMap.of(refField.getName(), new DocumentReference(refField, parent)))); diff --git a/document/src/main/java/com/yahoo/document/DataType.java b/document/src/main/java/com/yahoo/document/DataType.java index e47c60863ff3..2fcbe7333b48 100644 --- a/document/src/main/java/com/yahoo/document/DataType.java +++ b/document/src/main/java/com/yahoo/document/DataType.java @@ -62,6 +62,7 @@ public abstract class DataType extends Identifiable implements Serializable, Com static { TAG.setTag(true); } + public static int lastPredefinedDataTypeId() { return 21; } From 136ed7365ccf60898a84bb568d83da6a5d012ec3 Mon Sep 17 00:00:00 2001 From: Arne H Juul Date: Tue, 8 Mar 2022 13:50:36 +0000 Subject: [PATCH 5/9] populate struct fields on demand * most SDField instances should know their "repo" * populate with struct fields in a lazy manner * just remember what we were told in the old API * note: old API will be renamed so it makes a bit more sense --- .../searchdefinition/document/SDField.java | 95 +++++++++++-------- .../fieldoperation/IndexingOperation.java | 2 + .../processing/CreatePositionZCurve.java | 6 +- .../searchdefinition/processing/UriHack.java | 2 +- .../src/main/javacc/IntermediateParser.jj | 5 +- config-model/src/main/javacc/SDParser.jj | 13 ++- .../DocumentReferenceResolverTest.java | 17 ++-- .../ImportedFieldsEnumeratorTest.java | 6 +- .../derived/EmptyRankProfileTestCase.java | 6 +- .../searchdefinition/derived/IdTestCase.java | 2 +- .../derived/TypeConversionTestCase.java | 2 +- ...ransformToSummaryOfImportedFieldsTest.java | 5 +- .../processing/ValidateFieldTypesTest.java | 4 +- .../model/search/test/SchemaClusterTest.java | 4 +- 14 files changed, 99 insertions(+), 70 deletions(-) diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java index bd6625d4bde6..e72d72171d62 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java @@ -111,8 +111,8 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, /** Struct fields defined in this field */ private final Map structFields = new java.util.LinkedHashMap<>(0); - /** The document that this field was declared in, or null*/ - private SDDocumentType ownerDocType = null; + /** The document that this field was declared in, or null */ + private SDDocumentType repoDocType = null; /** The aliases declared for this field. May pertain to indexes or attributes */ private final Map aliasToName = new HashMap<>(); @@ -132,6 +132,7 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, */ protected SDField(SDDocumentType repo, String name, int id, DataType dataType, boolean populate) { super(name, id, dataType); + this.repoDocType = repo; populate(populate, repo, name, dataType); } @@ -142,12 +143,14 @@ public SDField(SDDocumentType repo, String name, int id, DataType dataType) { /** Creates a new field */ public SDField(SDDocumentType repo, String name, DataType dataType, boolean populate) { super(name, dataType); + this.repoDocType = repo; populate(populate, repo, name, dataType); } /** Creates a new field */ protected SDField(SDDocumentType repo, String name, DataType dataType, SDDocumentType owner, boolean populate) { super(name, dataType, owner == null ? null : owner.getDocumentType()); + this.repoDocType = repo; populate(populate, repo, name, dataType); } @@ -162,6 +165,8 @@ protected SDField(SDDocumentType repo, String name, DataType dataType, SDDocumen protected SDField(SDDocumentType repo, String name, DataType dataType, SDDocumentType owner, Matching fieldMatching, boolean populate, int recursion) { super(name, dataType, owner == null ? null : owner.getDocumentType()); + this.repoDocType = repo; + this.structFieldDepth = recursion; if (fieldMatching != null) this.setMatching(fieldMatching); populate(populate, repo, name, dataType, fieldMatching, recursion); @@ -171,15 +176,22 @@ public SDField(SDDocumentType repo, String name, DataType dataType) { this(repo, name, dataType, true); } + /* public SDField(String name, DataType dataType) { this(null, name, dataType); } + */ private void populate(boolean populate, SDDocumentType repo, String name, DataType dataType) { populate(populate, repo, name, dataType, null, 0); } - private void populate(boolean populate, SDDocumentType repo, String name, DataType dataType, Matching fieldMatching, int recursion) { + private int structFieldDepth = 0; + + private void populate(boolean populate, SDDocumentType repo, String name, DataType dataType, Matching fieldMatching, int recursion) { + if (recursion > structFieldDepth) { + structFieldDepth = recursion; + } if (dataType instanceof TensorDataType) { TensorType type = ((TensorDataType)dataType).getTensorType(); if (type.dimensions().stream().anyMatch(d -> d.isIndexed() && d.size().isEmpty())) @@ -273,34 +285,46 @@ public void addSummaryFieldSources(SummaryField summaryField) { } } - @SuppressWarnings("deprecation") public void populateWithStructFields(SDDocumentType sdoc, String name, DataType dataType, int recursion) { + if (repoDocType == null) { + repoDocType = sdoc; + } DataType dt = getFirstStructOrMapRecursive(); - if (dt == null) return; + if (dt == null) { + // nothing to do + doneStructFields = true; + } + } + + private boolean doneStructFields = false; + @SuppressWarnings("deprecation") + private void actuallyMakeStructFields() { + if (doneStructFields) return; + var sdoc = repoDocType; + var dataType = getDataType(); java.util.function.BiConsumer supplyStructField = (fieldName, fieldType) -> { if (structFields.containsKey(fieldName)) return; - String subName = name.concat(".").concat(fieldName); - var subField = new SDField(sdoc, subName, fieldType, - ownerDocType, new Matching(), - true, recursion + 1); + String subName = getName().concat(".").concat(fieldName); + var subField = new SDField(sdoc, subName, fieldType, sdoc, + null, true, structFieldDepth + 1); structFields.put(fieldName, subField); }; - if (dataType instanceof MapDataType) { MapDataType mdt = (MapDataType) dataType; supplyStructField.accept("key", mdt.getKeyType()); supplyStructField.accept("value", mdt.getValueType()); } else { - if (recursion >= 10) return; + if (structFieldDepth >= 10) { + // too risky + doneStructFields = true; + return; + } if (dataType instanceof CollectionDataType) { dataType = ((CollectionDataType)dataType).getNestedType(); } - if (dataType instanceof TemporaryStructuredDataType) { - SDDocumentType subType = sdoc != null ? sdoc.getType(dataType.getName()) : null; - if (subType == null) { - throw new IllegalArgumentException("Could not find struct '" + dataType.getName() + "'."); - } + SDDocumentType subType = sdoc != null ? sdoc.getType(dataType.getName()) : null; + if (dataType instanceof TemporaryStructuredDataType && subType != null) { for (Field field : subType.fieldSet()) { supplyStructField.accept(field.getName(), field.getDataType()); } @@ -310,32 +334,12 @@ ownerDocType, new Matching(), supplyStructField.accept(field.getName(), field.getDataType()); } } - } - } - - public void populateWithStructMatching(SDDocumentType sdoc, DataType dataType, Matching superFieldMatching) { - if (sdoc == null) return; - if (superFieldMatching == null) return; - DataType dt = getFirstStructOrMapRecursive(); - if (dt == null) return; - - if (dataType instanceof MapDataType) { - // old code here would never do anything useful, should we do something here? - return; - } else { - if (dataType instanceof CollectionDataType) { - dataType = ((CollectionDataType)dataType).getNestedType(); - } - if (dataType instanceof StructDataType) { - SDDocumentType subType = sdoc.getType(dataType.getName()); - if (subType == null) { - throw new IllegalArgumentException("Could not find struct " + dataType.getName()); - } + if (subType != null) { for (Field f : subType.fieldSet()) { if (f instanceof SDField) { SDField field = (SDField) f; Matching subFieldMatching = new Matching(); - subFieldMatching.merge(superFieldMatching); + subFieldMatching.merge(matchingForStructFields); subFieldMatching.merge(field.getMatching()); SDField subField = structFields.get(field.getName()); if (subField != null) { @@ -349,6 +353,15 @@ public void populateWithStructMatching(SDDocumentType sdoc, DataType dataType, M } } } + doneStructFields = true; + } + + private Matching matchingForStructFields = null; + + public void populateWithStructMatching(SDDocumentType sdoc, DataType dataType, Matching superFieldMatching) { + if (superFieldMatching == null) return; + if (getFirstStructOrMapRecursive() == null) return; + matchingForStructFields = superFieldMatching; } public void addOperation(FieldOperation op) { @@ -723,7 +736,10 @@ public SummaryField getSummaryField(String name,boolean create) { /** Returns list of static struct fields */ @Override - public Collection getStructFields() { return structFields.values(); } + public Collection getStructFields() { + actuallyMakeStructFields(); + return structFields.values(); + } /** * Returns a struct field defined in this field, @@ -732,6 +748,7 @@ public SummaryField getSummaryField(String name,boolean create) { */ @Override public SDField getStructField(String name) { + actuallyMakeStructFields(); if (name.contains(".")) { String superFieldName = name.substring(0,name.indexOf(".")); String subFieldName = name.substring(name.indexOf(".")+1); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingOperation.java index fe3ac11af278..a5f5f961ab50 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingOperation.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingOperation.java @@ -24,6 +24,8 @@ public IndexingOperation(ScriptExpression script) { this.script = script; } + public ScriptExpression getScript() { return script; } + public void apply(SDField field) { field.setIndexingScript(script); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java index 0bb1b7da769d..d7882c7f8fbc 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java @@ -10,6 +10,7 @@ import com.yahoo.searchdefinition.Schema; import com.yahoo.searchdefinition.document.Attribute; import com.yahoo.searchdefinition.document.GeoPos; +import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.documentmodel.SummaryTransform; @@ -35,8 +36,11 @@ */ public class CreatePositionZCurve extends Processor { + private final SDDocumentType repo; + public CreatePositionZCurve(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { super(schema, deployLogger, rankProfileRegistry, queryProfiles); + this.repo = schema.getDocument(); } private boolean useV8GeoPositions = false; @@ -105,7 +109,7 @@ private SDField createZCurveField(SDField inputField, String fieldName, boolean "' already created."); } boolean isArray = inputField.getDataType() instanceof ArrayDataType; - SDField field = new SDField(fieldName, isArray ? DataType.getArray(DataType.LONG) : DataType.LONG); + SDField field = new SDField(repo, fieldName, isArray ? DataType.getArray(DataType.LONG) : DataType.LONG); Attribute attribute = new Attribute(fieldName, Attribute.Type.LONG, isArray ? Attribute.CollectionType.ARRAY : Attribute.CollectionType.SINGLE); attribute.setPosition(true); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java index 84dc6d369fc4..7397f9a289ca 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java @@ -61,7 +61,7 @@ else if (uriField.getDataType() instanceof WeightedSetDataType) { String partName = uriName + "." + suffix; // I wonder if this is explicit in qrs or implicit in backend? // search.addFieldSetItem(uriName, partName); - SDField partField = new SDField(partName, generatedType); + SDField partField = new SDField(schema.getDocument(), partName, generatedType); partField.setIndexStructureField(uriField.doesIndexing()); partField.setRankType(uriField.getRankType()); partField.setStemming(Stemming.NONE); diff --git a/config-model/src/main/javacc/IntermediateParser.jj b/config-model/src/main/javacc/IntermediateParser.jj index 7447eee5cec3..ba955f071b22 100644 --- a/config-model/src/main/javacc/IntermediateParser.jj +++ b/config-model/src/main/javacc/IntermediateParser.jj @@ -958,10 +958,7 @@ void indexingOperation(ParsedField field, boolean multiLine) : { } { { IndexingOperation oldOp = newIndexingOperation(multiLine); - // TODO conversion via SDField is very ugly - SDField tmpField = new SDField("temp", com.yahoo.document.DataType.STRING); - oldOp.apply(tmpField); - ParsedIndexingOp newOp = new ParsedIndexingOp(tmpField.getIndexingScript()); + ParsedIndexingOp newOp = new ParsedIndexingOp(oldOp.getScript()); field.setIndexingOperation(newOp); } } diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj index 1fec33c4f128..e90df2776e04 100644 --- a/config-model/src/main/javacc/SDParser.jj +++ b/config-model/src/main/javacc/SDParser.jj @@ -639,7 +639,7 @@ void field(SDDocumentType document, Schema schema) : if (name != null && Schema.isReservedName(name.toLowerCase())) { throw new IllegalArgumentException("Reserved name '" + name + "' can not be used as a field name."); } - field = new TemporarySDField(document, name, type, document); + field = new TemporarySDField(document == null ? schema.getDocument() : document, name, type, document); } lbrace() (fieldBody(field, schema, document) ()*)* { @@ -668,7 +668,7 @@ void fieldSet(Schema schema) : | ( (queryCommand = identifierWithDash() | queryCommand = quotedString())) { queryCommands.add(queryCommand); } | - ( matchSetting = match(new SDField(setName, DataType.STRING)) ) { matchSettings.add(matchSetting); } + ( matchSetting = match(new SDField(null, setName, DataType.STRING)) ) { matchSettings.add(matchSetting); } )()*)+ { @@ -811,7 +811,11 @@ SDDocumentType structDefinition(Schema schema, SDDocumentType repo) : SDDocumentType struct; } { - ( name = identifier() ()* { struct = new SDDocumentType(name, schema); } + ( name = identifier() ()* { + if (repo == null) + throw new IllegalArgumentException("Can't add struct '"+name+"' to a document type, define a document type first or declare the struct inside of one."); + struct = new SDDocumentType(name, schema); + } [ inheritsDocument(struct) ()* ] lbrace() (structFieldDefinition(repo, struct) ()*)* ) { @@ -821,7 +825,6 @@ SDDocumentType structDefinition(Schema schema, SDDocumentType repo) : } catch (IllegalArgumentException e) { // empty } - if (repo==null) throw new IllegalArgumentException("Can't add struct '"+name+"' to a document type, define a document type first or declare the struct inside of one."); SDDocumentType sdtype = repo.getOwnedType(struct.getDocumentName()); DataType stype = sdtype != null ? sdtype.getStruct() @@ -830,7 +833,7 @@ SDDocumentType structDefinition(Schema schema, SDDocumentType repo) : return struct; } } - + /** * This rule consumes a data type block from within a field element. * diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java index 66f1850bd10e..4fa145afae97 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java @@ -38,12 +38,12 @@ public void reference_from_one_document_to_another_is_resolved() { barSchema.addDocument(barDocument); // Create foo document with document reference to bar and add another field - SDField fooRefToBarField = new SDField - ("bar_ref", ReferenceDataType.createWithInferredId(barDocument.getDocumentType())); - AttributeUtils.addAttributeAspect(fooRefToBarField); - SDField irrelevantField = new SDField("irrelevant_stuff", DataType.INT); Schema fooSchema = new Schema(FOO, MockApplicationPackage.createEmpty()); SDDocumentType fooDocument = new SDDocumentType("foo", fooSchema); + SDField fooRefToBarField = new SDField + (fooDocument, "bar_ref", ReferenceDataType.createWithInferredId(barDocument.getDocumentType())); + AttributeUtils.addAttributeAspect(fooRefToBarField); + SDField irrelevantField = new SDField(fooDocument, "irrelevant_stuff", DataType.INT); fooDocument.addField(fooRefToBarField); fooDocument.addField(irrelevantField); fooSchema.addDocument(fooDocument); @@ -62,11 +62,12 @@ public void reference_from_one_document_to_another_is_resolved() { @Test public void throws_user_friendly_exception_if_referenced_document_does_not_exist() { // Create foo document with document reference to non-existing document bar + Schema fooSchema = new Schema(FOO, MockApplicationPackage.createEmpty()); + SDDocumentType fooDocument = new SDDocumentType("foo", fooSchema); SDField fooRefToBarField = new SDField( + fooDocument, "bar_ref", ReferenceDataType.createWithInferredId(TemporaryStructuredDataType.create("bar"))); AttributeUtils.addAttributeAspect(fooRefToBarField); - Schema fooSchema = new Schema(FOO, MockApplicationPackage.createEmpty()); - SDDocumentType fooDocument = new SDDocumentType("foo", fooSchema); fooDocument.addField(fooRefToBarField); fooSchema.addDocument(fooDocument); @@ -86,10 +87,10 @@ public void throws_exception_if_reference_is_not_an_attribute() { barSchema.addDocument(barDocument); // Create foo document with document reference to bar - SDField fooRefToBarField = new SDField - ("bar_ref", ReferenceDataType.createWithInferredId(barDocument.getDocumentType())); Schema fooSchema = new Schema(FOO, MockApplicationPackage.createEmpty()); SDDocumentType fooDocument = new SDDocumentType("foo", fooSchema); + SDField fooRefToBarField = new SDField + (fooDocument, "bar_ref", ReferenceDataType.createWithInferredId(barDocument.getDocumentType())); fooDocument.addField(fooRefToBarField); fooSchema.addDocument(fooDocument); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java index 7d2386030da1..7e708f93a96e 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java @@ -22,16 +22,18 @@ public void imported_fields_are_enumerated_and_copied_from_correct_search_instan String PARENT = "parent"; Schema parentSchema = new Schema(PARENT, MockApplicationPackage.createEmpty()); SDDocumentType parentDocument = new SDDocumentType(PARENT, parentSchema); - var parentField = new SDField("their_field", DataType.INT); + var parentField = new SDField(parentDocument, "their_field", DataType.INT); AttributeUtils.addAttributeAspect(parentField); parentDocument.addField(parentField); parentSchema.addDocument(parentDocument); String FOO = "foo"; Schema fooSchema = new Schema(FOO, MockApplicationPackage.createEmpty()); + /* SDField fooRefToParent = new SDField( "foo_ref", ReferenceDataType.createWithInferredId(parentDocument.getDocumentType())); AttributeUtils.addAttributeAspect(fooRefToParent); + */ var fooImports = fooSchema.temporaryImportedFields().get(); fooImports.add(new TemporaryImportedField("my_first_import", "foo_ref", "their_field")); fooImports.add(new TemporaryImportedField("my_second_import", "foo_ref", "their_field")); @@ -40,9 +42,11 @@ public void imported_fields_are_enumerated_and_copied_from_correct_search_instan String BAR = "bar"; Schema barSchema = new Schema(BAR, MockApplicationPackage.createEmpty()); + /* SDField barRefToParent = new SDField( "bar_ref", ReferenceDataType.createWithInferredId(parentDocument.getDocumentType())); AttributeUtils.addAttributeAspect(barRefToParent); + */ var barImports = barSchema.temporaryImportedFields().get(); barImports.add(new TemporaryImportedField("my_cool_import", "my_ref", "their_field")); SDDocumentType barDocument = new SDDocumentType(BAR, barSchema); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java index 237e6c8c9927..a3123550efa8 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java @@ -25,11 +25,11 @@ public void testDeriving() { RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(schema); SDDocumentType doc = new SDDocumentType("test"); schema.addDocument(doc); - doc.addField(new SDField("a", DataType.STRING)); - SDField field = new SDField("b", DataType.STRING); + doc.addField(new SDField(doc, "a", DataType.STRING)); + SDField field = new SDField(doc, "b", DataType.STRING); field.setLiteralBoost(500); doc.addField(field); - doc.addField(new SDField("c", DataType.STRING)); + doc.addField(new SDField(doc, "c", DataType.STRING)); schema = ApplicationBuilder.buildFromRawSchema(schema, rankProfileRegistry, new QueryProfileRegistry()); new DerivedConfiguration(schema, rankProfileRegistry); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java index 440f067dd003..ba485b7b96b8 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java @@ -29,7 +29,7 @@ public void testExplicitUpperCaseIdField() { Schema schema = new Schema("test", MockApplicationPackage.createEmpty()); SDDocumentType document = new SDDocumentType("test"); schema.addDocument(document); - SDField uri = new SDField("URI", DataType.URI); + SDField uri = new SDField(document, "URI", DataType.URI); uri.parseIndexingScript("{ summary | index }"); document.addField(uri); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java index dbb32e611444..62a79e491462 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java @@ -30,7 +30,7 @@ public void testExactStringToStringTypeConversion() { RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(schema); SDDocumentType document = new SDDocumentType("test"); schema.addDocument(document); - SDField a = new SDField("a", DataType.STRING); + SDField a = new SDField(document, "a", DataType.STRING); a.parseIndexingScript("{ index }"); document.addField(a); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java index 9ca97f4dbc75..0adaba4cf680 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java @@ -59,8 +59,9 @@ private static Schema createSearchWithDocument(String documentName) { } private static ImportedFields createSingleImportedField(String fieldName) { - Schema targetSchema = createSearch("target_doc"); - SDField targetField = new SDField("target_field", DataType.INT); + Schema targetSchema = createSearchWithDocument("target_doc"); + var doc = targetSchema.getDocument(); + SDField targetField = new SDField(doc, "target_field", DataType.INT); DocumentReference documentReference = new DocumentReference(new Field("reference_field"), targetSchema); ImportedField importedField = new ImportedSimpleField(fieldName, documentReference, targetField); return new ImportedFields(Collections.singletonMap(fieldName, importedField)); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java index 22fd4e45c4a7..8c801d9deaf6 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java @@ -65,8 +65,8 @@ private static Schema createSearchWithDocument(String documentName) { } private static ImportedFields createSingleImportedField(String fieldName, DataType dataType) { - Schema targetSchema = createSearch("target_doc"); - SDField targetField = new SDField("target_field", dataType); + Schema targetSchema = createSearchWithDocument("target_doc"); + SDField targetField = new SDField(targetSchema.getDocument(), "target_field", dataType); DocumentReference documentReference = new DocumentReference(new Field("reference_field"), targetSchema); ImportedField importedField = new ImportedSimpleField(fieldName, documentReference, targetField); return new ImportedFields(Collections.singletonMap(fieldName, importedField)); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaClusterTest.java index e58c29cc4fd8..405c1127842e 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaClusterTest.java @@ -42,7 +42,7 @@ public void testSdConfigLogical() { // sd1 SDDocumentType sdt1 = new SDDocumentType("s1"); Schema schema1 = new Schema("s1", MockApplicationPackage.createEmpty()); - SDField f1 = new SDField("f1", DataType.STRING); + SDField f1 = new SDField(sdt1, "f1", DataType.STRING); f1.addAttribute(new Attribute("f1", DataType.STRING)); f1.setIndexingScript(new ScriptExpression(new StatementExpression(new AttributeExpression("f1")))); sdt1.addField(f1); @@ -51,7 +51,7 @@ public void testSdConfigLogical() { // sd2 SDDocumentType sdt2 = new SDDocumentType("s2"); Schema schema2 = new Schema("s2", MockApplicationPackage.createEmpty()); - SDField f2=new SDField("f2", DataType.STRING); + SDField f2=new SDField(sdt2, "f2", DataType.STRING); f2.addAttribute(new Attribute("f2", DataType.STRING)); f2.setIndexingScript(new ScriptExpression(new StatementExpression(new AttributeExpression("f2")))); sdt2.addField(f2); From cb7c08cfff57cc712e0a6b0b34b6102bde5d14e9 Mon Sep 17 00:00:00 2001 From: Arne H Juul Date: Fri, 11 Mar 2022 07:54:54 +0000 Subject: [PATCH 6/9] removing some useless stuff --- .../FieldOperationApplierForStructs.java | 3 +- .../searchdefinition/document/SDField.java | 46 ++++++++----------- .../twostreamingstructs/streamingstruct.sd | 1 + 3 files changed, 20 insertions(+), 30 deletions(-) diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java b/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java index 5e5623e2319c..7557b7ba11f2 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java @@ -47,8 +47,7 @@ private void maybePopulateField(SDDocumentType sdoc, SDField field, SDDocumentTy } if (structUsedByField.getName().equals(structType.getName())) { //this field is using this type!! - field.populateWithStructFields(sdoc, field.getName(), field.getDataType(), 0); - field.populateWithStructMatching(sdoc, field.getDataType(), field.getMatching()); + // field.populateWithStructFields(sdoc, field.getName(), field.getDataType(), 0); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java index e72d72171d62..a8ddc24d4fd2 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java @@ -133,7 +133,7 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, protected SDField(SDDocumentType repo, String name, int id, DataType dataType, boolean populate) { super(name, id, dataType); this.repoDocType = repo; - populate(populate, repo, name, dataType); + populate(populate, repo, name, dataType, null, 0); } public SDField(SDDocumentType repo, String name, int id, DataType dataType) { @@ -144,14 +144,14 @@ public SDField(SDDocumentType repo, String name, int id, DataType dataType) { public SDField(SDDocumentType repo, String name, DataType dataType, boolean populate) { super(name, dataType); this.repoDocType = repo; - populate(populate, repo, name, dataType); + populate(populate, repo, name, dataType, null, 0); } /** Creates a new field */ protected SDField(SDDocumentType repo, String name, DataType dataType, SDDocumentType owner, boolean populate) { super(name, dataType, owner == null ? null : owner.getDocumentType()); this.repoDocType = repo; - populate(populate, repo, name, dataType); + populate(populate, repo, name, dataType, null, 0); } /** @@ -180,11 +180,11 @@ public SDField(SDDocumentType repo, String name, DataType dataType) { public SDField(String name, DataType dataType) { this(null, name, dataType); } - */ private void populate(boolean populate, SDDocumentType repo, String name, DataType dataType) { populate(populate, repo, name, dataType, null, 0); } + */ private int structFieldDepth = 0; @@ -206,10 +206,6 @@ else if (dataType instanceof WeightedSetDataType) { else { addQueryCommand("type " + dataType.getName()); } - if (populate || (dataType instanceof MapDataType)) { - populateWithStructFields(repo, name, dataType, recursion); - populateWithStructMatching(repo, dataType, fieldMatching); - } } public void setIsExtraField(boolean isExtra) { @@ -285,22 +281,15 @@ public void addSummaryFieldSources(SummaryField summaryField) { } } - public void populateWithStructFields(SDDocumentType sdoc, String name, DataType dataType, int recursion) { - if (repoDocType == null) { - repoDocType = sdoc; - } - DataType dt = getFirstStructOrMapRecursive(); - if (dt == null) { - // nothing to do - doneStructFields = true; - } - } - private boolean doneStructFields = false; @SuppressWarnings("deprecation") private void actuallyMakeStructFields() { if (doneStructFields) return; + if (getFirstStructOrMapRecursive() == null) { + doneStructFields = true; + return; + } var sdoc = repoDocType; var dataType = getDataType(); java.util.function.BiConsumer supplyStructField = (fieldName, fieldType) -> { @@ -334,17 +323,24 @@ private void actuallyMakeStructFields() { supplyStructField.accept(field.getName(), field.getDataType()); } } + // if ((subType == null) && (structFields.size() > 0)) { + if ((subType == null) && (structFields.size() > 0)) { + throw new IllegalArgumentException("Cannot find matching (repo=" + sdoc + ") for subfields in " + + this + " [" + getDataType() + getDataType().getClass() + + "] with " + structFields.size() + " struct fields"); + } + // populate struct fields with matching if (subType != null) { for (Field f : subType.fieldSet()) { if (f instanceof SDField) { SDField field = (SDField) f; Matching subFieldMatching = new Matching(); - subFieldMatching.merge(matchingForStructFields); + subFieldMatching.merge(this.matching); subFieldMatching.merge(field.getMatching()); SDField subField = structFields.get(field.getName()); if (subField != null) { - subFieldMatching.merge(subField.getMatching()); - subField.populateWithStructMatching(sdoc, field.getDataType(), subFieldMatching); + // we just made this with no matching, so nop: + // subFieldMatching.merge(subField.getMatching()); subField.setMatching(subFieldMatching); } } else { @@ -358,12 +354,6 @@ private void actuallyMakeStructFields() { private Matching matchingForStructFields = null; - public void populateWithStructMatching(SDDocumentType sdoc, DataType dataType, Matching superFieldMatching) { - if (superFieldMatching == null) return; - if (getFirstStructOrMapRecursive() == null) return; - matchingForStructFields = superFieldMatching; - } - public void addOperation(FieldOperation op) { pendingOperations.add(op); } diff --git a/config-model/src/test/derived/twostreamingstructs/streamingstruct.sd b/config-model/src/test/derived/twostreamingstructs/streamingstruct.sd index 32823e46592a..0636e7a537e2 100644 --- a/config-model/src/test/derived/twostreamingstructs/streamingstruct.sd +++ b/config-model/src/test/derived/twostreamingstructs/streamingstruct.sd @@ -21,6 +21,7 @@ search streamingstruct { # Allow default matchtypes in struct. Can be overridden. # No index/attribute related stuff. It is only a datatype definition. } + struct ns1 { field nf1 type s1 { } field nf1s type s1 { match: substring } From c7d95723f10dca553d2892e60e178cf13a467258 Mon Sep 17 00:00:00 2001 From: Arne H Juul Date: Fri, 11 Mar 2022 12:02:15 +0000 Subject: [PATCH 7/9] remove old fixups no longer needed --- .../yahoo/searchdefinition/Application.java | 1 - .../FieldOperationApplierForStructs.java | 38 ------------------- 2 files changed, 39 deletions(-) diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Application.java b/config-model/src/main/java/com/yahoo/searchdefinition/Application.java index 16eef798acde..64688a7e70d1 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/Application.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/Application.java @@ -84,7 +84,6 @@ public Application(ApplicationPackage applicationPackage, List schemasSomewhatOrdered = new ArrayList<>(schemas); for (Schema schema : new SearchOrderer().order(schemasSomewhatOrdered)) { - new FieldOperationApplierForStructs().processSchemaFields(schema); new FieldOperationApplierForSearch().process(schema); // TODO: Why is this not in the regular list? new Processing(properties).process(schema, logger, diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java b/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java index 7557b7ba11f2..4a5a858f8282 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java @@ -20,46 +20,8 @@ public void process(SDDocumentType sdoc) { for (SDDocumentType type : sdoc.getAllTypes()) { if (type.isStruct()) { apply(type); - copyFields(type, sdoc); } } } - @SuppressWarnings("deprecation") - private void copyFields(SDDocumentType structType, SDDocumentType sdoc) { - //find all fields in OTHER types that have this type: - List list = new ArrayList<>(); - list.add(sdoc); - list.addAll(sdoc.getTypes()); - for (SDDocumentType anyType : list) { - Iterator fields = anyType.fieldIterator(); - while (fields.hasNext()) { - SDField field = (SDField) fields.next(); - maybePopulateField(sdoc, field, structType); - } - } - } - - private void maybePopulateField(SDDocumentType sdoc, SDField field, SDDocumentType structType) { - DataType structUsedByField = field.getFirstStructRecursive(); - if (structUsedByField == null) { - return; - } - if (structUsedByField.getName().equals(structType.getName())) { - //this field is using this type!! - // field.populateWithStructFields(sdoc, field.getName(), field.getDataType(), 0); - } - } - - public void processSchemaFields(Schema schema) { - var sdoc = schema.getDocument(); - if (sdoc == null) return; - for (SDDocumentType type : sdoc.getAllTypes()) { - if (type.isStruct()) { - for (SDField field : schema.allExtraFields()) { - maybePopulateField(sdoc, field, type); - } - } - } - } } From 5fe393bf28fd9e0fc96a3a1dd0f56510b30ed629 Mon Sep 17 00:00:00 2001 From: Arne H Juul Date: Fri, 11 Mar 2022 13:10:19 +0000 Subject: [PATCH 8/9] cleanup constructors --- .../document/SDDocumentType.java | 4 +- .../searchdefinition/document/SDField.java | 59 +++++++------------ .../document/TemporarySDField.java | 4 +- .../processing/AddExtraFieldsToDocument.java | 2 +- 4 files changed, 27 insertions(+), 42 deletions(-) diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java index a037d5046a73..b87bdd8907e2 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java @@ -268,8 +268,8 @@ public SDField addField(String string, DataType dataType) { return field; } - public Field addField(String string, DataType dataType, boolean header, int code) { - SDField field = new SDField(this, string, code, dataType, header); + public Field addField(String fName, DataType dataType, boolean header, int code) { + SDField field = new SDField(this, fName, code, dataType); addField(field); return field; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java index a8ddc24d4fd2..8263352e87f5 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java @@ -130,28 +130,24 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, * @param name the name of the field * @param dataType the datatype of the field */ - protected SDField(SDDocumentType repo, String name, int id, DataType dataType, boolean populate) { + public SDField(SDDocumentType repo, String name, int id, DataType dataType) { super(name, id, dataType); this.repoDocType = repo; - populate(populate, repo, name, dataType, null, 0); + populate(name, dataType); } - public SDField(SDDocumentType repo, String name, int id, DataType dataType) { - this(repo, name, id, dataType, true); + public SDField(String name, DataType dataType) { + this(null, name, dataType); } /** Creates a new field */ - public SDField(SDDocumentType repo, String name, DataType dataType, boolean populate) { - super(name, dataType); - this.repoDocType = repo; - populate(populate, repo, name, dataType, null, 0); + public SDField(SDDocumentType repo, String name, DataType dataType) { + this(repo, name, dataType, null); } /** Creates a new field */ - protected SDField(SDDocumentType repo, String name, DataType dataType, SDDocumentType owner, boolean populate) { - super(name, dataType, owner == null ? null : owner.getDocumentType()); - this.repoDocType = repo; - populate(populate, repo, name, dataType, null, 0); + protected SDField(SDDocumentType repo, String name, DataType dataType, SDDocumentType owner) { + this(repo, name, dataType, owner, null, 0); } /** @@ -162,36 +158,24 @@ protected SDField(SDDocumentType repo, String name, DataType dataType, SDDocumen * @param owner the owning document (used to check for id collisions) * @param fieldMatching the matching object to set for the field */ - protected SDField(SDDocumentType repo, String name, DataType dataType, SDDocumentType owner, - Matching fieldMatching, boolean populate, int recursion) { + protected SDField(SDDocumentType repo, + String name, + DataType dataType, + SDDocumentType owner, + Matching fieldMatching, + int recursion) + { super(name, dataType, owner == null ? null : owner.getDocumentType()); this.repoDocType = repo; this.structFieldDepth = recursion; if (fieldMatching != null) this.setMatching(fieldMatching); - populate(populate, repo, name, dataType, fieldMatching, recursion); - } - - public SDField(SDDocumentType repo, String name, DataType dataType) { - this(repo, name, dataType, true); + populate(name, dataType); } - /* - public SDField(String name, DataType dataType) { - this(null, name, dataType); - } - - private void populate(boolean populate, SDDocumentType repo, String name, DataType dataType) { - populate(populate, repo, name, dataType, null, 0); - } - */ - private int structFieldDepth = 0; - private void populate(boolean populate, SDDocumentType repo, String name, DataType dataType, Matching fieldMatching, int recursion) { - if (recursion > structFieldDepth) { - structFieldDepth = recursion; - } + private void populate(String name, DataType dataType) { if (dataType instanceof TensorDataType) { TensorType type = ((TensorDataType)dataType).getTensorType(); if (type.dimensions().stream().anyMatch(d -> d.isIndexed() && d.size().isEmpty())) @@ -292,20 +276,22 @@ private void actuallyMakeStructFields() { } var sdoc = repoDocType; var dataType = getDataType(); + java.util.function.BiConsumer supplyStructField = (fieldName, fieldType) -> { if (structFields.containsKey(fieldName)) return; String subName = getName().concat(".").concat(fieldName); - var subField = new SDField(sdoc, subName, fieldType, sdoc, - null, true, structFieldDepth + 1); + var subField = new SDField(sdoc, subName, fieldType, null, + null, structFieldDepth + 1); structFields.put(fieldName, subField); }; + if (dataType instanceof MapDataType) { MapDataType mdt = (MapDataType) dataType; supplyStructField.accept("key", mdt.getKeyType()); supplyStructField.accept("value", mdt.getValueType()); } else { if (structFieldDepth >= 10) { - // too risky + // too risky, infinite recursion doneStructFields = true; return; } @@ -323,7 +309,6 @@ private void actuallyMakeStructFields() { supplyStructField.accept(field.getName(), field.getDataType()); } } - // if ((subType == null) && (structFields.size() > 0)) { if ((subType == null) && (structFields.size() > 0)) { throw new IllegalArgumentException("Cannot find matching (repo=" + sdoc + ") for subfields in " + this + " [" + getDataType() + getDataType().getClass() + diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java index 6a47d08e3ac1..8c17b607f941 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java @@ -9,11 +9,11 @@ public class TemporarySDField extends SDField { public TemporarySDField(SDDocumentType repo, String name, DataType dataType, SDDocumentType owner) { - super(repo, name, dataType, owner, false); + super(repo, name, dataType, owner); } public TemporarySDField(SDDocumentType repo, String name, DataType dataType) { - super(repo, name, dataType, false); + super(repo, name, dataType); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java index 51defffa00b0..0be48d1fd25d 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java @@ -70,7 +70,7 @@ private void addSummaryField(Schema schema, SDDocumentType document, SummaryFiel if (docField == null) { ImmutableSDField existingField = schema.getField(field.getName()); if (existingField == null) { - SDField newField = new SDField(document, field.getName(), field.getDataType(), true); + SDField newField = new SDField(document, field.getName(), field.getDataType()); newField.setIsExtraField(true); document.addField(newField); } else if (!existingField.isImportedField()) { From 0f066685e6472da01060ee06b2395af33e37d6d1 Mon Sep 17 00:00:00 2001 From: Arne H Juul Date: Fri, 11 Mar 2022 13:23:21 +0000 Subject: [PATCH 9/9] on-demand struct fields solves two problems seen here --- .../derived/MatchSettingsResolvingTestCase.java | 7 ------- 1 file changed, 7 deletions(-) diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/MatchSettingsResolvingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/MatchSettingsResolvingTestCase.java index 63e3df2205f6..275c94465aeb 100755 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/MatchSettingsResolvingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/MatchSettingsResolvingTestCase.java @@ -54,12 +54,8 @@ public void testMapDefaults() throws IOException, ParseException { @Test public void testMapWithStructSettings() throws IOException, ParseException { - /* - * does not work - * does not pick up settings from struct declaration assertCorrectDeriving("matchsettings_map_wss", new TestProperties().setExperimentalSdParsing(false)); - */ assertCorrectDeriving("matchsettings_map_wss", new TestProperties().setExperimentalSdParsing(true)); } @@ -74,11 +70,8 @@ public void testMapWithFieldSettings() throws IOException, ParseException { @Test public void testMapAfter() throws IOException, ParseException { - /* fails with: - java.lang.IllegalArgumentException: Could not find struct 'elem'. assertCorrectDeriving("matchsettings_map_after", new TestProperties().setExperimentalSdParsing(false)); - */ assertCorrectDeriving("matchsettings_map_after", new TestProperties().setExperimentalSdParsing(true)); }