Skip to content

Commit

Permalink
Merge pull request #4249 from vespa-engine/havardpe/extend-perf-test
Browse files Browse the repository at this point in the history 
extend perf test with drop strategy
  • Loading branch information
@arnej27959
arnej27959 authored Oct 30, 2024
2 parents e6ce623 + dcaa60f commit 1c326de
Show file tree
Hide file tree
Showing 2 changed files with 40 additions and 0 deletions.
22 changes: 22 additions & 0 deletions tests/performance/wand_stopwords/MicroBmSearcher.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,10 +102,26 @@ public Result search(Query weakAndQuery, Execution execution) {
execution.fill(weakAndResult05);
var weakAndSet05 = getHitIds(weakAndResult05);

Query weakAndQueryD20 = changeRoot(weakAndQuery, new WeakAndItem());
weakAndQueryD20.properties().set("rankproperty.vespa.matching.weakand.stop_word_limit", "0.20");
weakAndQueryD20.properties().set("rankproperty.vespa.matching.weakand.stop_word_strategy", "drop");
Result weakAndResultD20 = execution.search(weakAndQueryD20);
execution.fill(weakAndResultD20);
var weakAndSetD20 = getHitIds(weakAndResultD20);

Query weakAndQueryD05 = changeRoot(weakAndQuery, new WeakAndItem());
weakAndQueryD05.properties().set("rankproperty.vespa.matching.weakand.stop_word_limit", "0.05");
weakAndQueryD05.properties().set("rankproperty.vespa.matching.weakand.stop_word_strategy", "drop");
Result weakAndResultD05 = execution.search(weakAndQueryD05);
execution.fill(weakAndResultD05);
var weakAndSetD05 = getHitIds(weakAndResultD05);

// double orTime = timeQuery(orQuery, execution, 1);
double weakAndTime = timeQuery(weakAndQuery, execution);
double weakAndTime20 = timeQuery(weakAndQuery20, execution);
double weakAndTime05 = timeQuery(weakAndQuery05, execution);
double weakAndTimeD20 = timeQuery(weakAndQueryD20, execution);
double weakAndTimeD05 = timeQuery(weakAndQueryD05, execution);
double andTime = timeQuery(andQuery, execution);

Hit meta = new Hit("meta");
Expand All @@ -114,16 +130,22 @@ public Result search(Query weakAndQuery, Execution execution) {
meta.setField("weakAndQuality", quality(orSet, weakAndSet));
meta.setField("weakAndQuality20", quality(orSet, weakAndSet20));
meta.setField("weakAndQuality05", quality(orSet, weakAndSet05));
meta.setField("weakAndQualityD20", quality(orSet, weakAndSetD20));
meta.setField("weakAndQualityD05", quality(orSet, weakAndSetD05));
meta.setField("orHits", orResult.getTotalHitCount());
meta.setField("andHits", andResult.getTotalHitCount());
meta.setField("weakAndHits", weakAndResult.getTotalHitCount());
meta.setField("weakAndHits20", weakAndResult20.getTotalHitCount());
meta.setField("weakAndHits05", weakAndResult05.getTotalHitCount());
meta.setField("weakAndHitsD20", weakAndResultD20.getTotalHitCount());
meta.setField("weakAndHitsD05", weakAndResultD05.getTotalHitCount());
meta.setField("orTime", orTime);
meta.setField("andTime", andTime);
meta.setField("weakAndTime", weakAndTime);
meta.setField("weakAndTime20", weakAndTime20);
meta.setField("weakAndTime05", weakAndTime05);
meta.setField("weakAndTimeD20", weakAndTimeD20);
meta.setField("weakAndTimeD05", weakAndTimeD05);
Result result = new Result(weakAndQuery);
result.setTotalHitCount(weakAndResult.getTotalHitCount());
result.hits().add(meta);
Expand Down
18 changes: 18 additions & 0 deletions tests/performance/wand_stopwords/wand_stopwords.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,16 +71,22 @@ def measure_wand_quality
waQ = []
wa20Q = []
wa05Q = []
waD20Q = []
waD05Q = []
andH = []
orH = []
waH = []
wa20H = []
wa05H = []
waD20H = []
waD05H = []
andT = []
orT = []
waT = []
wa20T = []
wa05T = []
waD20T = []
waD05T = []
counter = 0
q_file = download_file('squad2-questions.raw.141k.txt.zst', vespa.adminserver)
vespa.adminserver.execute("zstdcat #{q_file} | head -n 1000 > #{q_file}.raw")
Expand All @@ -95,16 +101,22 @@ def measure_wand_quality
waQ.append(h.field['weakAndQuality'])
wa20Q.append(h.field['weakAndQuality20'])
wa05Q.append(h.field['weakAndQuality05'])
waD20Q.append(h.field['weakAndQualityD20'])
waD05Q.append(h.field['weakAndQualityD05'])
andH.append(h.field['andHits'])
orH.append(h.field['orHits'])
waH.append(h.field['weakAndHits'])
wa20H.append(h.field['weakAndHits20'])
wa05H.append(h.field['weakAndHits05'])
waD20H.append(h.field['weakAndHitsD20'])
waD05H.append(h.field['weakAndHitsD05'])
andT.append(h.field['andTime'])
orT.append(h.field['orTime'])
waT.append(h.field['weakAndTime'])
wa20T.append(h.field['weakAndTime20'])
wa05T.append(h.field['weakAndTime05'])
waD20T.append(h.field['weakAndTimeD20'])
waD05T.append(h.field['weakAndTimeD05'])
quality = h.field['weakAndQuality']
wantedHits = max(h.field['andHits'], min(100, h.field['orHits']))
hitsFactor = (1000 * h.field['weakAndHits']) / wantedHits
Expand All @@ -121,17 +133,23 @@ def measure_wand_quality
process("WeakAnd-100-recall", "recall@100", waQ)
process("WeakAnd-20-recall", "recall@100", wa20Q)
process("WeakAnd-5-recall", "recall@100", wa05Q)
process("WeakAnd-D20-recall", "recall@100", waD20Q)
process("WeakAnd-D5-recall", "recall@100", waD05Q)

process("AND-hits", "hits", andH)
process("WeakAnd-100-hits", "hits", waH)
process("WeakAnd-20-hits", "hits", wa20H)
process("WeakAnd-5-hits", "hits", wa05H)
process("WeakAnd-D20-hits", "hits", waD20H)
process("WeakAnd-D5-hits", "hits", waD05H)
process("OR-hits", "hits", orH)

process("AND-ms", "latency", andT)
process("WeakAnd-100-ms", "latency", waT)
process("WeakAnd-20-ms", "latency", wa20T)
process("WeakAnd-5-ms", "latency", wa05T)
process("WeakAnd-D20-ms", "latency", waD20T)
process("WeakAnd-D5-ms", "latency", waD05T)
process("OR-ms", "latency", orT)
end

Expand Down

0 comments on commit 1c326de

Please sign in to comment.