Skip to content

Commit

Permalink
Add query execution metrics to JDBC QueryStats
Browse files Browse the repository at this point in the history 
Adds planningTimeMillis, analysisTimeMillis, finishingTimeMillis,
physicalInputBytes, physicalWrittenBytes and internalNetworkInputBytes
to allow JDBC clients to get some important metrics about query execution
  • Loading branch information
@raunaqmorarka
raunaqmorarka committed Dec 24, 2024
1 parent 3257342 commit c6c77cf
Show file tree
Hide file tree
Showing 12 changed files with 204 additions and 2 deletions.
72 changes: 72 additions & 0 deletions client/trino-client/src/main/java/io/trino/client/StatementStats.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,14 +36,18 @@ public class StatementStats
private final int queuedSplits;
private final int runningSplits;
private final int completedSplits;
private final long planningTimeMillis;
private final long analysisTimeMillis;
private final long cpuTimeMillis;
private final long wallTimeMillis;
private final long queuedTimeMillis;
private final long elapsedTimeMillis;
private final long finishingTimeMillis;
private final long processedRows;
private final long processedBytes;
private final long physicalInputBytes;
private final long physicalWrittenBytes;
private final long internalNetworkInputBytes;
private final long peakMemoryBytes;
private final long spilledBytes;
private final StageStats rootStage;
Expand All @@ -60,14 +64,18 @@ public StatementStats(
@JsonProperty("queuedSplits") int queuedSplits,
@JsonProperty("runningSplits") int runningSplits,
@JsonProperty("completedSplits") int completedSplits,
@JsonProperty("planningTimeMillis") long planningTimeMillis,
@JsonProperty("analysisTimeMillis") long analysisTimeMillis,
@JsonProperty("cpuTimeMillis") long cpuTimeMillis,
@JsonProperty("wallTimeMillis") long wallTimeMillis,
@JsonProperty("queuedTimeMillis") long queuedTimeMillis,
@JsonProperty("elapsedTimeMillis") long elapsedTimeMillis,
@JsonProperty("finishingTimeMillis") long finishingTimeMillis,
@JsonProperty("processedRows") long processedRows,
@JsonProperty("processedBytes") long processedBytes,
@JsonProperty("physicalInputBytes") long physicalInputBytes,
@JsonProperty("physicalWrittenBytes") long physicalWrittenBytes,
@JsonProperty("internalNetworkInputBytes") long internalNetworkInputBytes,
@JsonProperty("peakMemoryBytes") long peakMemoryBytes,
@JsonProperty("spilledBytes") long spilledBytes,
@JsonProperty("rootStage") StageStats rootStage)
Expand All @@ -82,14 +90,18 @@ public StatementStats(
this.queuedSplits = queuedSplits;
this.runningSplits = runningSplits;
this.completedSplits = completedSplits;
this.planningTimeMillis = planningTimeMillis;
this.analysisTimeMillis = analysisTimeMillis;
this.cpuTimeMillis = cpuTimeMillis;
this.wallTimeMillis = wallTimeMillis;
this.queuedTimeMillis = queuedTimeMillis;
this.elapsedTimeMillis = elapsedTimeMillis;
this.finishingTimeMillis = finishingTimeMillis;
this.processedRows = processedRows;
this.processedBytes = processedBytes;
this.physicalInputBytes = physicalInputBytes;
this.physicalWrittenBytes = physicalWrittenBytes;
this.internalNetworkInputBytes = internalNetworkInputBytes;
this.peakMemoryBytes = peakMemoryBytes;
this.spilledBytes = spilledBytes;
this.rootStage = rootStage;
Expand Down Expand Up @@ -155,6 +167,18 @@ public int getCompletedSplits()
return completedSplits;
}

@JsonProperty
public long getPlanningTimeMillis()
{
return planningTimeMillis;
}

@JsonProperty
public long getAnalysisTimeMillis()
{
return analysisTimeMillis;
}

@JsonProperty
public long getCpuTimeMillis()
{
Expand All @@ -179,6 +203,12 @@ public long getElapsedTimeMillis()
return elapsedTimeMillis;
}

@JsonProperty
public long getFinishingTimeMillis()
{
return finishingTimeMillis;
}

@JsonProperty
public long getProcessedRows()
{
Expand All @@ -203,6 +233,12 @@ public long getPhysicalWrittenBytes()
return physicalWrittenBytes;
}

@JsonProperty
public long getInternalNetworkInputBytes()
{
return internalNetworkInputBytes;
}

@JsonProperty
public long getPeakMemoryBytes()
{
Expand Down Expand Up @@ -236,14 +272,18 @@ public String toString()
.add("queuedSplits", queuedSplits)
.add("runningSplits", runningSplits)
.add("completedSplits", completedSplits)
.add("planningTimeMillis", planningTimeMillis)
.add("analysisTimeMillis", analysisTimeMillis)
.add("cpuTimeMillis", cpuTimeMillis)
.add("wallTimeMillis", wallTimeMillis)
.add("queuedTimeMillis", queuedTimeMillis)
.add("elapsedTimeMillis", elapsedTimeMillis)
.add("finishingTimeMillis", finishingTimeMillis)
.add("processedRows", processedRows)
.add("processedBytes", processedBytes)
.add("physicalInputBytes", physicalInputBytes)
.add("physicalWrittenBytes", physicalWrittenBytes)
.add("internalNetworkInputBytes", internalNetworkInputBytes)
.add("peakMemoryBytes", peakMemoryBytes)
.add("spilledBytes", spilledBytes)
.add("rootStage", rootStage)
Expand All @@ -267,14 +307,18 @@ public static class Builder
private int queuedSplits;
private int runningSplits;
private int completedSplits;
private long planningTimeMillis;
private long analysisTimeMillis;
private long cpuTimeMillis;
private long wallTimeMillis;
private long queuedTimeMillis;
private long elapsedTimeMillis;
private long finishingTimeMillis;
private long processedRows;
private long processedBytes;
private long physicalInputBytes;
private long physicalWrittenBytes;
private long internalNetworkInputBytes;
private long peakMemoryBytes;
private long spilledBytes;
private StageStats rootStage;
Expand Down Expand Up @@ -341,6 +385,18 @@ public Builder setCompletedSplits(int completedSplits)
return this;
}

public Builder setPlanningTimeMillis(long planningTimeMillis)
{
this.planningTimeMillis = planningTimeMillis;
return this;
}

public Builder setAnalysisTimeMillis(long analysisTimeMillis)
{
this.analysisTimeMillis = analysisTimeMillis;
return this;
}

public Builder setCpuTimeMillis(long cpuTimeMillis)
{
this.cpuTimeMillis = cpuTimeMillis;
Expand All @@ -365,6 +421,12 @@ public Builder setElapsedTimeMillis(long elapsedTimeMillis)
return this;
}

public Builder setFinishingTimeMillis(long finishingTimeMillis)
{
this.finishingTimeMillis = finishingTimeMillis;
return this;
}

public Builder setProcessedRows(long processedRows)
{
this.processedRows = processedRows;
Expand All @@ -389,6 +451,12 @@ public Builder setPhysicalWrittenBytes(long physicalWrittenBytes)
return this;
}

public Builder setInternalNetworkInputBytes(long internalNetworkInputBytes)
{
this.internalNetworkInputBytes = internalNetworkInputBytes;
return this;
}

public Builder setPeakMemoryBytes(long peakMemoryBytes)
{
this.peakMemoryBytes = peakMemoryBytes;
Expand Down Expand Up @@ -420,14 +488,18 @@ public StatementStats build()
queuedSplits,
runningSplits,
completedSplits,
planningTimeMillis,
analysisTimeMillis,
cpuTimeMillis,
wallTimeMillis,
queuedTimeMillis,
elapsedTimeMillis,
finishingTimeMillis,
processedRows,
processedBytes,
physicalInputBytes,
physicalWrittenBytes,
internalNetworkInputBytes,
peakMemoryBytes,
spilledBytes,
rootStage);
Expand Down
2 changes: 1 addition & 1 deletion client/trino-client/src/test/java/io/trino/client/TestRetry.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ private String newQueryResults(String state)
TypedQueryData.of(IntStream.range(0, numRecords)
.mapToObj(index -> Stream.of((Object) index, "a").collect(toList()))
.collect(toList())),
new StatementStats(state, state.equals("QUEUED"), true, OptionalDouble.of(0), OptionalDouble.of(0), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, null),
new StatementStats(state, state.equals("QUEUED"), true, OptionalDouble.of(0), OptionalDouble.of(0), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, null),
null,
ImmutableList.of(),
null,
Expand Down
54 changes: 54 additions & 0 deletions client/trino-jdbc/src/main/java/io/trino/jdbc/QueryStats.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,13 +32,19 @@ public final class QueryStats
private final int queuedSplits;
private final int runningSplits;
private final int completedSplits;
private final long planningTimeMillis;
private final long analysisTimeMillis;
private final long cpuTimeMillis;
private final long wallTimeMillis;
private final long queuedTimeMillis;
private final long elapsedTimeMillis;
private final long finishingTimeMillis;
private final long processedRows;
private final long processedBytes;
private final long peakMemoryBytes;
private final long physicalInputBytes;
private final long physicalWrittenBytes;
private final long internalNetworkInputBytes;
private final Optional<StageStats> rootStage;

public QueryStats(
Expand All @@ -52,13 +58,19 @@ public QueryStats(
int queuedSplits,
int runningSplits,
int completedSplits,
long planningTimeMillis,
long analysisTimeMillis,
long cpuTimeMillis,
long wallTimeMillis,
long queuedTimeMillis,
long elapsedTimeMillis,
long finishingTimeMillis,
long processedRows,
long processedBytes,
long peakMemoryBytes,
long physicalInputBytes,
long physicalWrittenBytes,
long internalNetworkInputBytes,
Optional<StageStats> rootStage)
{
this.queryId = requireNonNull(queryId, "queryId is null");
Expand All @@ -71,13 +83,19 @@ public QueryStats(
this.queuedSplits = queuedSplits;
this.runningSplits = runningSplits;
this.completedSplits = completedSplits;
this.planningTimeMillis = planningTimeMillis;
this.analysisTimeMillis = analysisTimeMillis;
this.cpuTimeMillis = cpuTimeMillis;
this.wallTimeMillis = wallTimeMillis;
this.queuedTimeMillis = queuedTimeMillis;
this.elapsedTimeMillis = elapsedTimeMillis;
this.finishingTimeMillis = finishingTimeMillis;
this.processedRows = processedRows;
this.processedBytes = processedBytes;
this.peakMemoryBytes = peakMemoryBytes;
this.physicalInputBytes = physicalInputBytes;
this.physicalWrittenBytes = physicalWrittenBytes;
this.internalNetworkInputBytes = internalNetworkInputBytes;
this.rootStage = requireNonNull(rootStage, "rootStage is null");
}

Expand All @@ -94,13 +112,19 @@ static QueryStats create(String queryId, StatementStats stats)
stats.getQueuedSplits(),
stats.getRunningSplits(),
stats.getCompletedSplits(),
stats.getPlanningTimeMillis(),
stats.getAnalysisTimeMillis(),
stats.getCpuTimeMillis(),
stats.getWallTimeMillis(),
stats.getQueuedTimeMillis(),
stats.getElapsedTimeMillis(),
stats.getFinishingTimeMillis(),
stats.getProcessedRows(),
stats.getProcessedBytes(),
stats.getPeakMemoryBytes(),
stats.getPhysicalInputBytes(),
stats.getPhysicalWrittenBytes(),
stats.getInternalNetworkInputBytes(),
Optional.ofNullable(stats.getRootStage()).map(StageStats::create));
}

Expand Down Expand Up @@ -154,6 +178,16 @@ public int getCompletedSplits()
return completedSplits;
}

public long getPlanningTimeMillis()
{
return planningTimeMillis;
}

public long getAnalysisTimeMillis()
{
return analysisTimeMillis;
}

public long getCpuTimeMillis()
{
return cpuTimeMillis;
Expand All @@ -174,6 +208,11 @@ public long getElapsedTimeMillis()
return elapsedTimeMillis;
}

public long getFinishingTimeMillis()
{
return finishingTimeMillis;
}

public long getProcessedRows()
{
return processedRows;
Expand All @@ -189,6 +228,21 @@ public long getPeakMemoryBytes()
return peakMemoryBytes;
}

public long getPhysicalInputBytes()
{
return physicalInputBytes;
}

public long getPhysicalWrittenBytes()
{
return physicalWrittenBytes;
}

public long getInternalNetworkInputBytes()
{
return internalNetworkInputBytes;
}

public Optional<StageStats> getRootStage()
{
return rootStage;
Expand Down
4 changes: 4 additions & 0 deletions client/trino-jdbc/src/test/java/io/trino/jdbc/TestAsyncResultIterator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -333,6 +333,10 @@ public StatementStats getStats()
100,
100,
100,
100,
100,
100,
100,
StageStats.builder()
.setStageId("id")
.setDone(false)
Expand Down
2 changes: 1 addition & 1 deletion client/trino-jdbc/src/test/java/io/trino/jdbc/TestProgressMonitor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ private String newQueryResults(Integer partialCancelId, Integer nextUriId, List<
nextUriId == null ? null : server.url(format("/v1/statement/%s/%s", queryId, nextUriId)).uri(),
responseColumns,
TypedQueryData.of(data),
new StatementStats(state, state.equals("QUEUED"), true, OptionalDouble.of(0), OptionalDouble.of(0), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, null),
new StatementStats(state, state.equals("QUEUED"), true, OptionalDouble.of(0), OptionalDouble.of(0), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, null),
null,
ImmutableList.of(),
null,
Expand Down
4 changes: 4 additions & 0 deletions core/trino-main/src/main/java/io/trino/execution/QueryStateMachine.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -547,6 +547,7 @@ private BasicQueryStats createBasicQueryStats(BasicStageStats stageStats)
stageStats.getSpilledDataSize(),
stageStats.getPhysicalInputDataSize(),
stageStats.getPhysicalWrittenDataSize(),
stageStats.getInternalNetworkInputDataSize(),

stageStats.getCumulativeUserMemory(),
stageStats.getFailedCumulativeUserMemory(),
Expand All @@ -555,10 +556,13 @@ private BasicQueryStats createBasicQueryStats(BasicStageStats stageStats)
succinctBytes(getPeakUserMemoryInBytes()),
succinctBytes(getPeakTotalMemoryInBytes()),

queryStateTimer.getPlanningTime(),
queryStateTimer.getAnalysisTime(),
stageStats.getTotalCpuTime(),
stageStats.getFailedCpuTime(),
stageStats.getTotalScheduledTime(),
stageStats.getFailedScheduledTime(),
queryStateTimer.getFinishingTime(),

stageStats.isFullyBlocked(),
stageStats.getBlockedReasons(),
Expand Down
Loading

0 comments on commit c6c77cf

Please sign in to comment.