Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/candidate-9.6.x' into candidate-…
Browse files Browse the repository at this point in the history 
…9.8.x

Signed-off-by: Gavin Halliday <[email protected]>

# Conflicts:
#	helm/hpcc/Chart.yaml
#	helm/hpcc/templates/_helpers.tpl
#	version.cmake
  • Loading branch information
@ghalliday
ghalliday committed Nov 1, 2024
2 parents 2ca6a2b + 144df3b commit 7d0a1a3
Show file tree
Hide file tree
Showing 26 changed files with 260 additions and 157 deletions.
15 changes: 14 additions & 1 deletion dali/base/dadfs.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2912,7 +2912,7 @@ class CDistributedFileBase : implements INTERFACE, public CInterface
if (history)
queryAttributes().removeTree(history);
}
void lockFileAttrLock(CFileAttrLock & attrLock)
virtual void lockFileAttrLock(CFileAttrLock & attrLock)
{
if (!attrLock.init(logicalName, DXB_File, RTM_LOCK_WRITE, conn, defaultTimeout, "CDistributedFile::lockFileAttrLock"))
{
Expand Down Expand Up @@ -6482,6 +6482,19 @@ class CDistributedSuperFile: public CDistributedFileBase<IDistributedSuperFile>
return new cSubFileIterator(subfiles,supersub);
}

virtual void lockFileAttrLock(CFileAttrLock & attrLock) override
{
if (!attrLock.init(logicalName, DXB_SuperFile, RTM_LOCK_WRITE, conn, defaultTimeout, "CDistributedFile::lockFileAttrLock"))
{
// In unlikely event File/Attr doesn't exist, must ensure created, commited and root connection is reloaded.
verifyex(attrLock.init(logicalName, DXB_SuperFile, RTM_LOCK_WRITE|RTM_CREATE_QUERY, conn, defaultTimeout, "CDistributedFile::lockFileAttrLock"));
attrLock.commit();
conn->commit();
conn->reload();
root.setown(conn->getRoot());
}
}

void updateFileAttrs()
{
if (subfiles.ordinality()==0) {
Expand Down
15 changes: 10 additions & 5 deletions dali/dfu/dfuutil.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -530,13 +530,18 @@ class CFileCloner
if (iskey&&!cluster2.isEmpty())
dstfdesc->addCluster(cluster2,grp2,spec2);

for (unsigned pn=0; pn<numParts; pn++) {
offset_t sz = srcfdesc->queryPart(pn)->queryProperties().getPropInt64("@size",-1);
for (unsigned pn=0; pn<numParts; pn++)
{
IPropertyTree &srcProps = srcfdesc->queryPart(pn)->queryProperties();
IPropertyTree &dstProps = dstfdesc->queryPart(pn)->queryProperties();
offset_t sz = srcProps.getPropInt64("@size",-1);
if (sz!=(offset_t)-1)
dstfdesc->queryPart(pn)->queryProperties().setPropInt64("@size",sz);
dstProps.setPropInt64("@size",sz);
StringBuffer dates;
if (srcfdesc->queryPart(pn)->queryProperties().getProp("@modified",dates))
dstfdesc->queryPart(pn)->queryProperties().setProp("@modified",dates.str());
if (srcProps.getProp("@modified",dates))
dstProps.setProp("@modified",dates.str());
if (srcProps.hasProp("@kind"))
dstProps.setProp("@kind", srcProps.queryProp("@kind"));
}

if (!copyphysical) //cloneFrom tells roxie where to copy from.. it's unnecessary if we already did the copy
Expand Down
3 changes: 3 additions & 0 deletions ecl/eclcc/eclcc.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1535,8 +1535,11 @@ void EclCC::processSingleQuery(const EclRepositoryManager & localRepositoryManag

updateWorkunitStat(instance.wu, SSToperation, ">compile:>parse", StTimeElapsed, NULL, parseTimeNs);
stat_type sourceDownloadTime = localRepositoryManager.getStatistic(StTimeElapsed);
stat_type sourceDownloadBlockedTime = localRepositoryManager.getStatistic(StTimeBlocked);
if (sourceDownloadTime)
updateWorkunitStat(instance.wu, SSToperation, ">compile:>parse:>download", StTimeElapsed, NULL, sourceDownloadTime);
if (sourceDownloadBlockedTime)
updateWorkunitStat(instance.wu, SSToperation, ">compile:>parse:>download", StTimeBlocked, NULL, sourceDownloadBlockedTime);

if (optExtraStats)
{
Expand Down
11 changes: 9 additions & 2 deletions ecl/hql/hqlrepository.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -676,6 +676,8 @@ unsigned __int64 EclRepositoryManager::getStatistic(StatisticKind kind) const
{
case StTimeElapsed:
return cycle_to_nanosec(gitDownloadCycles);
case StTimeBlocked:
return cycle_to_nanosec(gitDownloadBlockedCycles);
}
return 0;
}
Expand Down Expand Up @@ -823,7 +825,12 @@ IEclSourceCollection * EclRepositoryManager::resolveGitCollection(const char * r
throw makeStringExceptionV(99, "Unsupported repository link format '%s'", defaultUrl);

bool alreadyExists = false;

CCycleTimer gitDownloadTimer;
Owned<IInterface> gitUpdateLock(getGitUpdateLock(repoPath));
cycle_t blockedCycles = gitDownloadTimer.elapsedCycles();
gitDownloadBlockedCycles += blockedCycles;

if (checkDirExists(repoPath))
{
if (options.cleanRepos)
Expand Down Expand Up @@ -853,7 +860,6 @@ IEclSourceCollection * EclRepositoryManager::resolveGitCollection(const char * r

bool ok = false;
Owned<IError> error;
CCycleTimer gitDownloadTimer;
if (alreadyExists)
{
if (options.updateRepos)
Expand Down Expand Up @@ -890,7 +896,8 @@ IEclSourceCollection * EclRepositoryManager::resolveGitCollection(const char * r
//this could become a read/write lock if that proved to be an issue.
gitUpdateLock.clear();

gitDownloadCycles += gitDownloadTimer.elapsedCycles();
gitDownloadCycles += (gitDownloadTimer.elapsedCycles() - blockedCycles);

if (error)
{
if (errorReceiver)
Expand Down
1 change: 1 addition & 0 deletions ecl/hql/hqlrepository.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,7 @@ class HQL_API EclRepositoryManager
IArrayOf<IEclRepository> overrideSources; // -D options
IArrayOf<IEclRepository> allSources; // also includes -D options
cycle_t gitDownloadCycles = 0;
cycle_t gitDownloadBlockedCycles = 0;

//Include all options in a nested struct to make it easy to ensure they are cloned
struct {
Expand Down
3 changes: 3 additions & 0 deletions esp/src/src-react/components/Metrics.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,9 @@ export const Metrics: React.FunctionComponent<MetricsProps> = ({
selection,
fullscreen = false
}) => {
if (querySet && queryId) {
wuid = "";
}
const [_uiState, _setUIState] = React.useState({ ...defaultUIState });
const [selectedMetricsSource, setSelectedMetricsSource] = React.useState<SelectedMetricsSource>("");
const [selectedMetrics, setSelectedMetrics] = React.useState<IScope[]>([]);
Expand Down
177 changes: 95 additions & 82 deletions esp/src/src-react/hooks/metrics.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import { scopedLogger } from "@hpcc-js/util";
import { singletonHook } from "react-singleton-hook";
import { userKeyValStore } from "src/KeyValStore";
import { DockPanelLayout } from "../layouts/DockPanel";
import { singletonDebounce } from "../util/throttle";
import { useWorkunit } from "./workunit";
import { useQuery } from "./query";
import { useCounter } from "./util";
Expand Down Expand Up @@ -214,14 +215,19 @@ function useMetricsViewsImpl(): useMetricsViewsResult {

export const useMetricsViews = singletonHook(defaultState, useMetricsViewsImpl);

let wuDetailsMetaResponse: Promise<WsWorkunits.WUDetailsMetaResponse>;

export function useMetricMeta(): [string[], string[]] {

const service = useConst(() => new WorkunitsService({ baseUrl: "" }));
const [scopeTypes, setScopeTypes] = React.useState<string[]>([]);
const [properties, setProperties] = React.useState<string[]>([]);

React.useEffect(() => {
service?.WUDetailsMeta({}).then(response => {
if (!wuDetailsMetaResponse && service) {
wuDetailsMetaResponse = service.WUDetailsMeta({});
}
wuDetailsMetaResponse?.then(response => {
setScopeTypes(response?.ScopeTypes?.ScopeType || []);
setProperties((response?.Properties?.Property.map(p => p.Name) || []).sort());
});
Expand Down Expand Up @@ -274,45 +280,48 @@ export function useWorkunitMetrics(
const [count, increment] = useCounter();

React.useEffect(() => {
setStatus(FetchStatus.STARTED);
workunit?.fetchDetailsNormalized({
ScopeFilter: scopeFilter,
NestedFilter: nestedFilter,
PropertiesToReturn: {
AllScopes: true,
AllAttributes: true,
AllProperties: true,
AllNotes: true,
AllStatistics: true,
AllHints: true
},
ScopeOptions: {
IncludeId: true,
IncludeScope: true,
IncludeScopeType: true,
IncludeMatchedScopesInResults: true
},
PropertyOptions: {
IncludeName: true,
IncludeRawValue: true,
IncludeFormatted: true,
IncludeMeasure: true,
IncludeCreator: false,
IncludeCreatorType: false
}
}).then(response => {
setData(response?.data);
setColumns(response?.columns);
setActivities(response?.meta?.Activities?.Activity || []);
setProperties(response?.meta?.Properties?.Property || []);
setMeasures(response?.meta?.Measures?.Measure || []);
setScopeTypes(response?.meta?.ScopeTypes?.ScopeType || []);
}).catch(e => {
logger.error(e);
}).finally(() => {
setStatus(FetchStatus.COMPLETE);
});
}, [workunit, state, count, scopeFilter, nestedFilter]);
if (wuid && workunit) {
const fetchDetailsNormalized = singletonDebounce(workunit, "fetchDetailsNormalized");
setStatus(FetchStatus.STARTED);
fetchDetailsNormalized({
ScopeFilter: scopeFilter,
NestedFilter: nestedFilter,
PropertiesToReturn: {
AllScopes: true,
AllAttributes: true,
AllProperties: true,
AllNotes: true,
AllStatistics: true,
AllHints: true
},
ScopeOptions: {
IncludeId: true,
IncludeScope: true,
IncludeScopeType: true,
IncludeMatchedScopesInResults: true
},
PropertyOptions: {
IncludeName: true,
IncludeRawValue: true,
IncludeFormatted: true,
IncludeMeasure: true,
IncludeCreator: false,
IncludeCreatorType: false
}
}).then(response => {
setData(response?.data);
setColumns(response?.columns);
setActivities(response?.meta?.Activities?.Activity || []);
setProperties(response?.meta?.Properties?.Property || []);
setMeasures(response?.meta?.Measures?.Measure || []);
setScopeTypes(response?.meta?.ScopeTypes?.ScopeType || []);
}).catch(e => {
logger.error(e);
}).finally(() => {
setStatus(FetchStatus.COMPLETE);
});
}
}, [workunit, state, count, scopeFilter, nestedFilter, wuid]);

return { metrics: data, columns, activities, properties, measures, scopeTypes, status, refresh: increment };
}
Expand All @@ -335,45 +344,48 @@ export function useQueryMetrics(
const [count, increment] = useCounter();

React.useEffect(() => {
setStatus(FetchStatus.STARTED);
query?.fetchDetailsNormalized({
ScopeFilter: scopeFilter,
NestedFilter: nestedFilter,
PropertiesToReturn: {
AllScopes: true,
AllAttributes: true,
AllProperties: true,
AllNotes: true,
AllStatistics: true,
AllHints: true
},
ScopeOptions: {
IncludeId: true,
IncludeScope: true,
IncludeScopeType: true,
IncludeMatchedScopesInResults: true
},
PropertyOptions: {
IncludeName: true,
IncludeRawValue: true,
IncludeFormatted: true,
IncludeMeasure: true,
IncludeCreator: false,
IncludeCreatorType: false
}
}).then(response => {
setData(response?.data);
setColumns(response?.columns);
setActivities(response?.meta?.Activities?.Activity || []);
setProperties(response?.meta?.Properties?.Property || []);
setMeasures(response?.meta?.Measures?.Measure || []);
setScopeTypes(response?.meta?.ScopeTypes?.ScopeType || []);
}).catch(e => {
logger.error(e);
}).finally(() => {
setStatus(FetchStatus.COMPLETE);
});
}, [query, state, count, scopeFilter, nestedFilter]);
if (querySet && queryId && query) {
const fetchDetailsNormalized = singletonDebounce(query, "fetchDetailsNormalized");
setStatus(FetchStatus.STARTED);
fetchDetailsNormalized({
ScopeFilter: scopeFilter,
NestedFilter: nestedFilter,
PropertiesToReturn: {
AllScopes: true,
AllAttributes: true,
AllProperties: true,
AllNotes: true,
AllStatistics: true,
AllHints: true
},
ScopeOptions: {
IncludeId: true,
IncludeScope: true,
IncludeScopeType: true,
IncludeMatchedScopesInResults: true
},
PropertyOptions: {
IncludeName: true,
IncludeRawValue: true,
IncludeFormatted: true,
IncludeMeasure: true,
IncludeCreator: false,
IncludeCreatorType: false
}
}).then(response => {
setData(response?.data);
setColumns(response?.columns);
setActivities(response?.meta?.Activities?.Activity || []);
setProperties(response?.meta?.Properties?.Property || []);
setMeasures(response?.meta?.Measures?.Measure || []);
setScopeTypes(response?.meta?.ScopeTypes?.ScopeType || []);
}).catch(e => {
logger.error(e);
}).finally(() => {
setStatus(FetchStatus.COMPLETE);
});
}
}, [query, state, count, scopeFilter, nestedFilter, querySet, queryId]);

return { metrics: data, columns, activities, properties, measures, scopeTypes, status, refresh: increment };
}
Expand All @@ -385,7 +397,8 @@ export function useWUQueryMetrics(
scopeFilter: Partial<WsWorkunits.ScopeFilter> = scopeFilterDefault,
nestedFilter: WsWorkunits.NestedFilter = nestedFilterDefault
): useMetricsResult {
const wuMetrics = useWorkunitMetrics(wuid, scopeFilter, nestedFilter);
const queryMetrics = useQueryMetrics(querySet, queryId, scopeFilter, nestedFilter);
return querySet && queryId ? { ...queryMetrics } : { ...wuMetrics };
const isQuery = querySet && queryId;
const wuMetrics = useWorkunitMetrics(isQuery ? "" : wuid, scopeFilter, nestedFilter);
const queryMetrics = useQueryMetrics(isQuery ? querySet : "", isQuery ? queryId : "", scopeFilter, nestedFilter);
return isQuery ? { ...queryMetrics } : { ...wuMetrics };
}
2 changes: 1 addition & 1 deletion esp/src/src-react/hooks/workunit.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ export function useWorkunit(wuid: string, full: boolean = false): [Workunit, WUS
const [retVal, setRetVal] = React.useState<{ workunit: Workunit, state: number, lastUpdate: number, isComplete: boolean, refresh: RefreshFunc }>();

React.useEffect(() => {
if (wuid === undefined || wuid === null) {
if (!wuid) {
setRetVal({ workunit: undefined, state: WUStateID.NotFound, lastUpdate: Date.now(), isComplete: undefined, refresh: (full?: boolean) => Promise.resolve(undefined) });
return;
}
Expand Down
14 changes: 14 additions & 0 deletions fs/dafsserver/dafsserver.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2278,6 +2278,7 @@ class CRemoteIndexReadActivity : public CRemoteIndexBaseActivity

Owned<const IDynamicTransform> translator;
unsigned __int64 chooseN = 0;
bool cleanupBlobs = false;
public:
CRemoteIndexReadActivity(IPropertyTree &config, IFileDescriptor *fileDesc) : PARENT(config, fileDesc)
{
Expand Down Expand Up @@ -2316,6 +2317,12 @@ class CRemoteIndexReadActivity : public CRemoteIndexBaseActivity
}
dbgassertex(retSz);

if (cleanupBlobs)
{
keyManager->releaseBlobs();
cleanupBlobs = false;
}

const void *ret = outBuilder.getSelf();
outBuilder.finishRow(retSz);
++processed;
Expand Down Expand Up @@ -2350,6 +2357,13 @@ class CRemoteIndexReadActivity : public CRemoteIndexBaseActivity
{
return out.appendf("indexread[%s]", fileName.get());
}

virtual const byte * lookupBlob(unsigned __int64 id) override
{
size32_t dummy;
cleanupBlobs = true;
return (byte *) keyManager->loadBlob(id, dummy, nullptr);
}
};


Expand Down
2 changes: 1 addition & 1 deletion roxie/ccd/ccdlistener.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -967,7 +967,7 @@ StringBuffer & ContextLogger::getStats(StringBuffer &s) const
ids.append(slowestActivityIds[i]);
formatStatistic(times, cycle_to_nanosec(slowestActivityTimes[i]), SMeasureTimeNs);
}
s.appendf(", slowestActivities={ ids=[%s] times=[%s] }", ids.str(), times.str());
s.appendf(" slowestActivities={ ids=[%s] times=[%s] }", ids.str(), times.str());
}
return s;
}
Expand Down
Loading

0 comments on commit 7d0a1a3

Please sign in to comment.