Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove cpuTimeAcc from DataSource#createSegmentMapFunction's signature #17623

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
Open

Remove cpuTimeAcc from DataSource#createSegmentMapFunction's signature #17623

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
f41b171
remove cpuTimeAcc from createSegmentMapF
kgyrtkirk Jan 14, 2025
faac1c6
undo
kgyrtkirk Jan 14, 2025
9aaff07
cleanup
kgyrtkirk Jan 14, 2025
0b483a7
cleanup
kgyrtkirk Jan 14, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions ...tage-query/src/main/java/org/apache/druid/msq/querykit/BaseLeafFrameProcessorFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,6 @@
import java.util.ArrayList;
import java.util.List;
import java.util.Queue;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Consumer;
import java.util.function.Function;

Expand Down Expand Up @@ -163,7 +162,7 @@ public ProcessorsAndChannels<Object, Long> makeProcessors(

if (segmentMapFnProcessor == null) {
final Function<SegmentReference, SegmentReference> segmentMapFn =
query.getDataSource().createSegmentMapFunction(query, new AtomicLong());
query.getDataSource().createSegmentMapFunction(query);
processorManager = processorManagerFn.apply(ImmutableList.of(segmentMapFn));
} else {
processorManager = new ChainedProcessorManager<>(ProcessorManagers.of(() -> segmentMapFnProcessor), processorManagerFn);
Expand Down
2 changes: 1 addition & 1 deletion ...query/src/main/java/org/apache/druid/msq/querykit/BroadcastJoinSegmentMapFnProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -193,7 +193,7 @@ private void addFrame(final int channelNumber, final Frame frame)

private Function<SegmentReference, SegmentReference> createSegmentMapFunction()
{
return inlineChannelData(query.getDataSource()).createSegmentMapFunction(query, new AtomicLong());
return inlineChannelData(query.getDataSource()).createSegmentMapFunction(query);
}

DataSource inlineChannelData(final DataSource originalDataSource)
Expand Down
3 changes: 1 addition & 2 deletions .../multi-stage-query/src/main/java/org/apache/druid/msq/querykit/InputNumberDataSource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Function;

/**
Expand Down Expand Up @@ -97,7 +96,7 @@ public boolean isConcrete()
}

@Override
public Function<SegmentReference, SegmentReference> createSegmentMapFunction(Query query, AtomicLong cpuTimeAcc)
public Function<SegmentReference, SegmentReference> createSegmentMapFunction(Query query)
{
return Function.identity();
}
Expand Down
2 changes: 1 addition & 1 deletion ...-stage-query/src/main/java/org/apache/druid/msq/querykit/SimpleSegmentMapFnProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ public List<WritableFrameChannel> outputChannels()
@Override
public ReturnOrAwait<Function<SegmentReference, SegmentReference>> runIncrementally(final IntSet readableInputs)
{
return ReturnOrAwait.returnObject(query.getDataSource().createSegmentMapFunction(query, new AtomicLong()));
return ReturnOrAwait.returnObject(query.getDataSource().createSegmentMapFunction(query));
}

@Override
Expand Down
3 changes: 1 addition & 2 deletions processing/src/main/java/org/apache/druid/query/DataSource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@

import java.util.List;
import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Function;

/**
Expand Down Expand Up @@ -108,7 +107,7 @@ public interface DataSource
* @param cpuTimeAcc the cpu time accumulator
* @return the segment function
*/
Function<SegmentReference, SegmentReference> createSegmentMapFunction(Query query, AtomicLong cpuTimeAcc);
Function<SegmentReference, SegmentReference> createSegmentMapFunction(Query query);

/**
* Returns an updated datasource based on the specified new source.
Expand Down
18 changes: 4 additions & 14 deletions processing/src/main/java/org/apache/druid/query/FilteredDataSource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,13 +27,12 @@
import org.apache.druid.query.planning.DataSourceAnalysis;
import org.apache.druid.segment.FilteredSegment;
import org.apache.druid.segment.SegmentReference;
import org.apache.druid.utils.JvmUtils;

import javax.annotation.Nullable;

import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Function;

/**
Expand Down Expand Up @@ -122,19 +121,10 @@ public boolean isConcrete()
}

@Override
public Function<SegmentReference, SegmentReference> createSegmentMapFunction(
Query query,
AtomicLong cpuTimeAccumulator
)
public Function<SegmentReference, SegmentReference> createSegmentMapFunction(Query query)
{
final Function<SegmentReference, SegmentReference> segmentMapFn = base.createSegmentMapFunction(
query,
cpuTimeAccumulator
);
return JvmUtils.safeAccumulateThreadCpuTime(
cpuTimeAccumulator,
() -> baseSegment -> new FilteredSegment(segmentMapFn.apply(baseSegment), filter)
);
final Function<SegmentReference, SegmentReference> segmentMapFn = base.createSegmentMapFunction(query);
return baseSegment -> new FilteredSegment(segmentMapFn.apply(baseSegment), filter);
}

@Override
Expand Down
3 changes: 1 addition & 2 deletions processing/src/main/java/org/apache/druid/query/FrameBasedInlineDataSource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,6 @@
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Function;
import java.util.stream.Collectors;

Expand Down Expand Up @@ -171,7 +170,7 @@ public boolean isConcrete()
}

@Override
public Function<SegmentReference, SegmentReference> createSegmentMapFunction(Query query, AtomicLong cpuTimeAcc)
public Function<SegmentReference, SegmentReference> createSegmentMapFunction(Query query)
{
return Function.identity();
}
Expand Down
6 changes: 1 addition & 5 deletions processing/src/main/java/org/apache/druid/query/InlineDataSource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,6 @@
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
Expand Down Expand Up @@ -243,10 +242,7 @@ public boolean isConcrete()
}

@Override
public Function<SegmentReference, SegmentReference> createSegmentMapFunction(
Query query,
AtomicLong cpuTimeAcc
)
public Function<SegmentReference, SegmentReference> createSegmentMapFunction(Query query)
{
return Function.identity();
}
Expand Down
16 changes: 2 additions & 14 deletions processing/src/main/java/org/apache/druid/query/JoinDataSource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,6 @@
import org.apache.druid.segment.join.filter.JoinFilterPreAnalysisKey;
import org.apache.druid.segment.join.filter.JoinableClauses;
import org.apache.druid.segment.join.filter.rewrite.JoinFilterRewriteConfig;
import org.apache.druid.utils.JvmUtils;

import javax.annotation.Nullable;
import java.util.ArrayList;
import java.util.Arrays;
Expand All @@ -65,7 +63,6 @@
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Function;
import java.util.stream.Collectors;

Expand Down Expand Up @@ -301,14 +298,12 @@ public Set<String> getVirtualColumnCandidates()

@Override
public Function<SegmentReference, SegmentReference> createSegmentMapFunction(
Query query,
AtomicLong cpuTimeAccumulator
Query query
)
{
return createSegmentMapFunctionInternal(
analysis.getJoinBaseTableFilter().map(Filters::toFilter).orElse(null),
analysis.getPreJoinableClauses(),
cpuTimeAccumulator,
analysis.getBaseQuery().orElse(query)
);
}
Expand Down Expand Up @@ -444,14 +439,10 @@ private DataSourceAnalysis getAnalysisForDataSource()
private Function<SegmentReference, SegmentReference> createSegmentMapFunctionInternal(
@Nullable final Filter baseFilter,
final List<PreJoinableClause> clauses,
final AtomicLong cpuTimeAccumulator,
final Query<?> query
)
{
// compute column correlations here and RHS correlated values
return JvmUtils.safeAccumulateThreadCpuTime(
cpuTimeAccumulator,
() -> {
if (clauses.isEmpty()) {
return Function.identity();
} else {
Expand Down Expand Up @@ -510,8 +501,7 @@ private Function<SegmentReference, SegmentReference> createSegmentMapFunctionInt
baseMapFn = Function.identity();
} else {
baseMapFn = left.createSegmentMapFunction(
query,
cpuTimeAccumulator
query
);
}
return baseSegment ->
Expand All @@ -522,8 +512,6 @@ private Function<SegmentReference, SegmentReference> createSegmentMapFunctionInt
joinFilterPreAnalysis
);
}
}
);
}

/**
Expand Down
6 changes: 1 addition & 5 deletions processing/src/main/java/org/apache/druid/query/LookupDataSource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Function;

/**
Expand Down Expand Up @@ -101,10 +100,7 @@ public boolean isConcrete()
}

@Override
public Function<SegmentReference, SegmentReference> createSegmentMapFunction(
Query query,
AtomicLong cpuTime
)
public Function<SegmentReference, SegmentReference> createSegmentMapFunction(Query query)
{
return Function.identity();
}
Expand Down
6 changes: 2 additions & 4 deletions processing/src/main/java/org/apache/druid/query/QueryDataSource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Function;

@JsonTypeName("query")
Expand Down Expand Up @@ -110,12 +109,11 @@ public boolean isConcrete()

@Override
public Function<SegmentReference, SegmentReference> createSegmentMapFunction(
Query query,
AtomicLong cpuTime
Query query
)
{
final Query<?> subQuery = this.getQuery();
return subQuery.getDataSource().createSegmentMapFunction(subQuery, cpuTime);
return subQuery.getDataSource().createSegmentMapFunction(subQuery);
}

@Override
Expand Down
6 changes: 1 addition & 5 deletions processing/src/main/java/org/apache/druid/query/TableDataSource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Function;

@JsonTypeName("table")
Expand Down Expand Up @@ -98,10 +97,7 @@ public boolean isConcrete()
}

@Override
public Function<SegmentReference, SegmentReference> createSegmentMapFunction(
Query query,
AtomicLong cpuTime
)
public Function<SegmentReference, SegmentReference> createSegmentMapFunction(Query query)
{
return Function.identity();
}
Expand Down
6 changes: 1 addition & 5 deletions processing/src/main/java/org/apache/druid/query/UnionDataSource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Function;
import java.util.stream.Collectors;

Expand Down Expand Up @@ -144,10 +143,7 @@ public boolean isConcrete()
}

@Override
public Function<SegmentReference, SegmentReference> createSegmentMapFunction(
Query query,
AtomicLong cpuTime
)
public Function<SegmentReference, SegmentReference> createSegmentMapFunction(Query query)
{
return Function.identity();
}
Expand Down
14 changes: 3 additions & 11 deletions processing/src/main/java/org/apache/druid/query/UnnestDataSource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,13 +28,10 @@
import org.apache.druid.segment.SegmentReference;
import org.apache.druid.segment.UnnestSegment;
import org.apache.druid.segment.VirtualColumn;
import org.apache.druid.utils.JvmUtils;

import javax.annotation.Nullable;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Function;

/**
Expand Down Expand Up @@ -135,18 +132,13 @@ public boolean isConcrete()

@Override
public Function<SegmentReference, SegmentReference> createSegmentMapFunction(
Query query,
AtomicLong cpuTimeAccumulator
Query query
)
{
final Function<SegmentReference, SegmentReference> segmentMapFn = base.createSegmentMapFunction(
query,
cpuTimeAccumulator
);
return JvmUtils.safeAccumulateThreadCpuTime(
cpuTimeAccumulator,
() -> baseSegment -> new UnnestSegment(segmentMapFn.apply(baseSegment), virtualColumn, unnestFilter)
query
);
return baseSegment -> new UnnestSegment(segmentMapFn.apply(baseSegment), virtualColumn, unnestFilter);
}

@Override
Expand Down
3 changes: 1 addition & 2 deletions processing/src/main/java/org/apache/druid/query/union/UnionQuery.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,6 @@
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Function;

public class UnionQuery implements Query<Object>
Expand Down Expand Up @@ -271,7 +270,7 @@ public boolean isConcrete()
}

@Override
public Function<SegmentReference, SegmentReference> createSegmentMapFunction(Query query, AtomicLong cpuTimeAcc)
public Function<SegmentReference, SegmentReference> createSegmentMapFunction(Query query)
{
throw methodNotSupported();
}
Expand Down
9 changes: 3 additions & 6 deletions processing/src/test/java/org/apache/druid/query/QueryDataSourceTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@
import org.junit.rules.ExpectedException;

import java.util.Collections;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Function;

public class QueryDataSourceTest
Expand Down Expand Up @@ -174,17 +173,15 @@ public void test_serde() throws Exception
public void test_withSegmentMapFunction()
{
Function<SegmentReference, SegmentReference> parentsegmentMapFunction = queryDataSource.createSegmentMapFunction(
groupByQuery,
new AtomicLong()
groupByQuery
);

Function<SegmentReference, SegmentReference> childsegmentMapFunction = queryOnTableDataSource.createSegmentMapFunction(
groupByQuery,
new AtomicLong()
groupByQuery
);
// The segment functions should both be identity functions and equal
Assert.assertEquals(parentsegmentMapFunction, childsegmentMapFunction);
}


}
2 changes: 1 addition & 1 deletion processing/src/test/java/org/apache/druid/query/QueryRunnerTestHelper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -559,7 +559,7 @@ public static <T, QueryType extends Query<T>> QueryRunner<T> makeQueryRunnerWith
{
final DataSource base = query.getDataSource();

final SegmentReference segmentReference = base.createSegmentMapFunction(query, new AtomicLong())
final SegmentReference segmentReference = base.createSegmentMapFunction(query)
.apply(ReferenceCountingSegment.wrapRootGenerationSegment(adapter));
return makeQueryRunner(factory, segmentReference, runnerName);
}
Expand Down
Loading
Loading