Skip to content

Commit

Permalink
Merge pull request #21659 from vespa-engine/arnej/on-demand-struct-fi…
Browse files Browse the repository at this point in the history 
…elds

Arnej/on demand struct fields
  • Loading branch information
@bratseth
bratseth authored Mar 12, 2022
2 parents ff15841 + 0f06668 commit 64f4ae9
Show file tree
Hide file tree
Showing 35 changed files with 185 additions and 209 deletions.
16 changes: 16 additions & 0 deletions config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,10 @@
import com.yahoo.document.DataType;
import com.yahoo.document.Document;
import com.yahoo.document.Field;
import com.yahoo.document.ReferenceDataType;
import com.yahoo.document.StructDataType;
import com.yahoo.document.StructuredDataType;
import com.yahoo.document.TemporaryStructuredDataType;
import com.yahoo.document.annotation.AnnotationType;
import com.yahoo.document.annotation.AnnotationTypeRegistry;
import com.yahoo.document.datatypes.FieldValue;
Expand Down Expand Up @@ -383,4 +385,18 @@ public boolean equals(Object other) {

}

private ReferenceDataType refToThis = null;

@SuppressWarnings("deprecation")
public ReferenceDataType getReferenceDataType() {
if (refToThis == null) {
// super ugly, the APIs for this are horribly inconsistent
var tmptmp = TemporaryStructuredDataType.create(getName());
var tmp = ReferenceDataType.createWithInferredId(tmptmp);
tmp.setTargetType((StructuredDataType) this);
refToThis = tmp;
}
return refToThis;
}

}
1 change: 0 additions & 1 deletion config-model/src/main/java/com/yahoo/searchdefinition/Application.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,6 @@ public Application(ApplicationPackage applicationPackage,

List<Schema> schemasSomewhatOrdered = new ArrayList<>(schemas);
for (Schema schema : new SearchOrderer().order(schemasSomewhatOrdered)) {
new FieldOperationApplierForStructs().processSchemaFields(schema);
new FieldOperationApplierForSearch().process(schema); // TODO: Why is this not in the regular list?
new Processing(properties).process(schema,
logger,
Expand Down
13 changes: 2 additions & 11 deletions config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -292,17 +292,8 @@ else if (type instanceof WeightedSetDataType) {
else if (type instanceof ReferenceDataType) {
ReferenceDataType t = (ReferenceDataType) type;
var tt = t.getTargetType();
if (tt instanceof TemporaryStructuredDataType) {
DataType targetType = resolveTemporariesRecurse(tt, repo, docs, replacements);
t.setTargetType((StructuredDataType) targetType);
} else if (tt instanceof DocumentType) {
DataType targetType = resolveTemporariesRecurse(tt, repo, docs, replacements);
// super ugly, the APIs for this are horribly inconsistent
var tmptmp = TemporaryStructuredDataType.create(tt.getName());
var tmp = new ReferenceDataType(tmptmp, t.getId());
tmp.setTargetType((StructuredDataType) targetType);
type = tmp;
}
var doc = getDocumentType(docs, tt.getId());
type = doc.getReferenceDataType();
}
if (type != original) {
replacements.add(new TypeReplacement(original, type));
Expand Down
39 changes: 0 additions & 39 deletions config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,47 +20,8 @@ public void process(SDDocumentType sdoc) {
for (SDDocumentType type : sdoc.getAllTypes()) {
if (type.isStruct()) {
apply(type);
copyFields(type, sdoc);
}
}
}

@SuppressWarnings("deprecation")
private void copyFields(SDDocumentType structType, SDDocumentType sdoc) {
//find all fields in OTHER types that have this type:
List<SDDocumentType> list = new ArrayList<>();
list.add(sdoc);
list.addAll(sdoc.getTypes());
for (SDDocumentType anyType : list) {
Iterator<Field> fields = anyType.fieldIterator();
while (fields.hasNext()) {
SDField field = (SDField) fields.next();
maybePopulateField(sdoc, field, structType);
}
}
}

private void maybePopulateField(SDDocumentType sdoc, SDField field, SDDocumentType structType) {
DataType structUsedByField = field.getFirstStructRecursive();
if (structUsedByField == null) {
return;
}
if (structUsedByField.getName().equals(structType.getName())) {
//this field is using this type!!
field.populateWithStructFields(sdoc, field.getName(), field.getDataType(), 0);
field.populateWithStructMatching(sdoc, field.getDataType(), field.getMatching());
}
}

public void processSchemaFields(Schema schema) {
var sdoc = schema.getDocument();
if (sdoc == null) return;
for (SDDocumentType type : sdoc.getAllTypes()) {
if (type.isStruct()) {
for (SDField field : schema.allExtraFields()) {
maybePopulateField(sdoc, field, type);
}
}
}
}
}
4 changes: 2 additions & 2 deletions config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -268,8 +268,8 @@ public SDField addField(String string, DataType dataType) {
return field;
}

public Field addField(String string, DataType dataType, boolean header, int code) {
SDField field = new SDField(this, string, code, dataType, header);
public Field addField(String fName, DataType dataType, boolean header, int code) {
SDField field = new SDField(this, fName, code, dataType);
addField(field);
return field;
}
Expand Down
128 changes: 60 additions & 68 deletions config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,8 +111,8 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer,
/** Struct fields defined in this field */
private final Map<String,SDField> structFields = new java.util.LinkedHashMap<>(0);

/** The document that this field was declared in, or null*/
private SDDocumentType ownerDocType = null;
/** The document that this field was declared in, or null */
private SDDocumentType repoDocType = null;

/** The aliases declared for this field. May pertain to indexes or attributes */
private final Map<String, String> aliasToName = new HashMap<>();
Expand All @@ -130,25 +130,24 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer,
* @param name the name of the field
* @param dataType the datatype of the field
*/
protected SDField(SDDocumentType repo, String name, int id, DataType dataType, boolean populate) {
public SDField(SDDocumentType repo, String name, int id, DataType dataType) {
super(name, id, dataType);
populate(populate, repo, name, dataType);
this.repoDocType = repo;
populate(name, dataType);
}

public SDField(SDDocumentType repo, String name, int id, DataType dataType) {
this(repo, name, id, dataType, true);
public SDField(String name, DataType dataType) {
this(null, name, dataType);
}

/** Creates a new field */
public SDField(SDDocumentType repo, String name, DataType dataType, boolean populate) {
super(name, dataType);
populate(populate, repo, name, dataType);
public SDField(SDDocumentType repo, String name, DataType dataType) {
this(repo, name, dataType, null);
}

/** Creates a new field */
protected SDField(SDDocumentType repo, String name, DataType dataType, SDDocumentType owner, boolean populate) {
super(name, dataType, owner == null ? null : owner.getDocumentType());
populate(populate, repo, name, dataType);
protected SDField(SDDocumentType repo, String name, DataType dataType, SDDocumentType owner) {
this(repo, name, dataType, owner, null, 0);
}

/**
Expand All @@ -159,27 +158,24 @@ protected SDField(SDDocumentType repo, String name, DataType dataType, SDDocumen
* @param owner the owning document (used to check for id collisions)
* @param fieldMatching the matching object to set for the field
*/
protected SDField(SDDocumentType repo, String name, DataType dataType, SDDocumentType owner,
Matching fieldMatching, boolean populate, int recursion) {
protected SDField(SDDocumentType repo,
String name,
DataType dataType,
SDDocumentType owner,
Matching fieldMatching,
int recursion)
{
super(name, dataType, owner == null ? null : owner.getDocumentType());
this.repoDocType = repo;
this.structFieldDepth = recursion;
if (fieldMatching != null)
this.setMatching(fieldMatching);
populate(populate, repo, name, dataType, fieldMatching, recursion);
populate(name, dataType);
}

public SDField(SDDocumentType repo, String name, DataType dataType) {
this(repo, name, dataType, true);
}
private int structFieldDepth = 0;

public SDField(String name, DataType dataType) {
this(null, name, dataType);
}

private void populate(boolean populate, SDDocumentType repo, String name, DataType dataType) {
populate(populate, repo, name, dataType, null, 0);
}

private void populate(boolean populate, SDDocumentType repo, String name, DataType dataType, Matching fieldMatching, int recursion) {
private void populate(String name, DataType dataType) {
if (dataType instanceof TensorDataType) {
TensorType type = ((TensorDataType)dataType).getTensorType();
if (type.dimensions().stream().anyMatch(d -> d.isIndexed() && d.size().isEmpty()))
Expand All @@ -194,10 +190,6 @@ else if (dataType instanceof WeightedSetDataType) {
else {
addQueryCommand("type " + dataType.getName());
}
if (populate || (dataType instanceof MapDataType)) {
populateWithStructFields(repo, name, dataType, recursion);
populateWithStructMatching(repo, dataType, fieldMatching);
}
}

public void setIsExtraField(boolean isExtra) {
Expand Down Expand Up @@ -273,17 +265,23 @@ public void addSummaryFieldSources(SummaryField summaryField) {
}
}

private boolean doneStructFields = false;

@SuppressWarnings("deprecation")
public void populateWithStructFields(SDDocumentType sdoc, String name, DataType dataType, int recursion) {
DataType dt = getFirstStructOrMapRecursive();
if (dt == null) return;
private void actuallyMakeStructFields() {
if (doneStructFields) return;
if (getFirstStructOrMapRecursive() == null) {
doneStructFields = true;
return;
}
var sdoc = repoDocType;
var dataType = getDataType();

java.util.function.BiConsumer<String, DataType> supplyStructField = (fieldName, fieldType) -> {
if (structFields.containsKey(fieldName)) return;
String subName = name.concat(".").concat(fieldName);
var subField = new SDField(sdoc, subName, fieldType,
ownerDocType, new Matching(),
true, recursion + 1);
String subName = getName().concat(".").concat(fieldName);
var subField = new SDField(sdoc, subName, fieldType, null,
null, structFieldDepth + 1);
structFields.put(fieldName, subField);
};

Expand All @@ -292,15 +290,16 @@ ownerDocType, new Matching(),
supplyStructField.accept("key", mdt.getKeyType());
supplyStructField.accept("value", mdt.getValueType());
} else {
if (recursion >= 10) return;
if (structFieldDepth >= 10) {
// too risky, infinite recursion
doneStructFields = true;
return;
}
if (dataType instanceof CollectionDataType) {
dataType = ((CollectionDataType)dataType).getNestedType();
}
if (dataType instanceof TemporaryStructuredDataType) {
SDDocumentType subType = sdoc != null ? sdoc.getType(dataType.getName()) : null;
if (subType == null) {
throw new IllegalArgumentException("Could not find struct '" + dataType.getName() + "'.");
}
SDDocumentType subType = sdoc != null ? sdoc.getType(dataType.getName()) : null;
if (dataType instanceof TemporaryStructuredDataType && subType != null) {
for (Field field : subType.fieldSet()) {
supplyStructField.accept(field.getName(), field.getDataType());
}
Expand All @@ -310,37 +309,23 @@ ownerDocType, new Matching(),
supplyStructField.accept(field.getName(), field.getDataType());
}
}
}
}

public void populateWithStructMatching(SDDocumentType sdoc, DataType dataType, Matching superFieldMatching) {
if (sdoc == null) return;
if (superFieldMatching == null) return;
DataType dt = getFirstStructOrMapRecursive();
if (dt == null) return;

if (dataType instanceof MapDataType) {
// old code here would never do anything useful, should we do something here?
return;
} else {
if (dataType instanceof CollectionDataType) {
dataType = ((CollectionDataType)dataType).getNestedType();
if ((subType == null) && (structFields.size() > 0)) {
throw new IllegalArgumentException("Cannot find matching (repo=" + sdoc + ") for subfields in "
+ this + " [" + getDataType() + getDataType().getClass() +
"] with " + structFields.size() + " struct fields");
}
if (dataType instanceof StructDataType) {
SDDocumentType subType = sdoc.getType(dataType.getName());
if (subType == null) {
throw new IllegalArgumentException("Could not find struct " + dataType.getName());
}
// populate struct fields with matching
if (subType != null) {
for (Field f : subType.fieldSet()) {
if (f instanceof SDField) {
SDField field = (SDField) f;
Matching subFieldMatching = new Matching();
subFieldMatching.merge(superFieldMatching);
subFieldMatching.merge(this.matching);
subFieldMatching.merge(field.getMatching());
SDField subField = structFields.get(field.getName());
if (subField != null) {
subFieldMatching.merge(subField.getMatching());
subField.populateWithStructMatching(sdoc, field.getDataType(), subFieldMatching);
// we just made this with no matching, so nop:
// subFieldMatching.merge(subField.getMatching());
subField.setMatching(subFieldMatching);
}
} else {
Expand All @@ -349,8 +334,11 @@ public void populateWithStructMatching(SDDocumentType sdoc, DataType dataType, M
}
}
}
doneStructFields = true;
}

private Matching matchingForStructFields = null;

public void addOperation(FieldOperation op) {
pendingOperations.add(op);
}
Expand Down Expand Up @@ -723,7 +711,10 @@ public SummaryField getSummaryField(String name,boolean create) {

/** Returns list of static struct fields */
@Override
public Collection<SDField> getStructFields() { return structFields.values(); }
public Collection<SDField> getStructFields() {
actuallyMakeStructFields();
return structFields.values();
}

/**
* Returns a struct field defined in this field,
Expand All @@ -732,6 +723,7 @@ public SummaryField getSummaryField(String name,boolean create) {
*/
@Override
public SDField getStructField(String name) {
actuallyMakeStructFields();
if (name.contains(".")) {
String superFieldName = name.substring(0,name.indexOf("."));
String subFieldName = name.substring(name.indexOf(".")+1);
Expand Down
8 changes: 4 additions & 4 deletions config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,12 @@
*/
public class TemporarySDField extends SDField {

public TemporarySDField(String name, DataType dataType, SDDocumentType owner) {
super(owner, name, dataType, owner, false);
public TemporarySDField(SDDocumentType repo, String name, DataType dataType, SDDocumentType owner) {
super(repo, name, dataType, owner);
}

public TemporarySDField(String name, DataType dataType) {
super(null, name, dataType, false);
public TemporarySDField(SDDocumentType repo, String name, DataType dataType) {
super(repo, name, dataType);
}

}
2 changes: 2 additions & 0 deletions config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingOperation.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ public IndexingOperation(ScriptExpression script) {
this.script = script;
}

public ScriptExpression getScript() { return script; }

public void apply(SDField field) {
field.setIndexingScript(script);
}
Expand Down
Loading

0 comments on commit 64f4ae9

Please sign in to comment.