From 93cc5bc94a3e9b8ca771d541b8dbe871374a43c1 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Fri, 13 Dec 2024 10:26:47 +0000 Subject: [PATCH 1/5] HPCC-33126 Add FNV-1a hashing function and use in jptree Resolves some pathologically slow cases of hash collisions in jptree maps. Signed-off-by: Jake Smith --- dali/daliadmin/daadmin.cpp | 5 +- dali/daliadmin/daadmin.hpp | 2 +- dali/daliadmin/daliadmin.cpp | 4 +- system/jlib/jhash.cpp | 145 +++++++++++++++++++++++++++++--- system/jlib/jhash.hpp | 26 ++++-- system/jlib/jlzw.cpp | 2 +- system/jlib/jptree.cpp | 2 +- system/jlib/jptree.ipp | 6 +- system/jlib/jsuperhash.hpp | 13 ++- testing/unittests/jlibtests.cpp | 122 ++++++++++++++++++++++++++- 10 files changed, 292 insertions(+), 35 deletions(-) diff --git a/dali/daliadmin/daadmin.cpp b/dali/daliadmin/daadmin.cpp index cee095e3321..837fe3882a4 100644 --- a/dali/daliadmin/daadmin.cpp +++ b/dali/daliadmin/daadmin.cpp @@ -2472,7 +2472,7 @@ void xmlSize(const char *filename, double pc) } } -void loadXMLTest(const char *filename, bool parseOnly, bool useLowMemPTree, bool saveFormattedXML) +void loadXMLTest(const char *filename, bool parseOnly, bool useLowMemPTree, bool saveFormattedXML, bool freePTree) { OwnedIFile iFile = createIFile(filename); OwnedIFileIO iFileIO = iFile->open(IFOread); @@ -2544,7 +2544,8 @@ void loadXMLTest(const char *filename, bool parseOnly, bool useLowMemPTree, bool PROGLOG("Save took: %.2f", (float)timer.elapsedMs()/1000); } - ::LINK(iMaker->queryRoot()); // intentionally leak (avoid time clearing up) + if (!freePTree) + ::LINK(iMaker->queryRoot()); // intentionally leak (avoid time clearing up) } void translateToXpath(const char *logicalfile, DfsXmlBranchKind tailType) diff --git a/dali/daliadmin/daadmin.hpp b/dali/daliadmin/daadmin.hpp index 9230e245d41..b6f27473e45 100644 --- a/dali/daliadmin/daadmin.hpp +++ b/dali/daliadmin/daadmin.hpp @@ -28,7 +28,7 @@ namespace daadmin extern DALIADMIN_API void setDaliConnectTimeoutMs(unsigned timeoutMs); extern DALIADMIN_API void xmlSize(const char *filename, double pc); -extern DALIADMIN_API void loadXMLTest(const char *filename, bool parseOnly, bool useLowMemPTree, bool saveFormattedXML); +extern DALIADMIN_API void loadXMLTest(const char *filename, bool parseOnly, bool useLowMemPTree, bool saveFormattedXML, bool freePTree); extern DALIADMIN_API void translateToXpath(const char *logicalfile, DfsXmlBranchKind tailType = DXB_File); extern DALIADMIN_API void exportToFile(const char *path, const char *filename, bool safe = false); diff --git a/dali/daliadmin/daliadmin.cpp b/dali/daliadmin/daliadmin.cpp index c402fa6f62c..f3612d39606 100644 --- a/dali/daliadmin/daliadmin.cpp +++ b/dali/daliadmin/daliadmin.cpp @@ -233,13 +233,15 @@ int main(int argc, const char* argv[]) { bool useLowMemPTree = false; bool saveFormatedTree = false; + bool freePTree = false; bool parseOnly = getComponentConfigSP()->getPropBool("@parseonly"); if (!parseOnly) { useLowMemPTree = getComponentConfigSP()->getPropBool("@lowmem"); saveFormatedTree = getComponentConfigSP()->getPropBool("@savexml"); + freePTree = getComponentConfigSP()->getPropBool("@free"); } - loadXMLTest(params.item(1), parseOnly, useLowMemPTree, saveFormatedTree); + loadXMLTest(params.item(1), parseOnly, useLowMemPTree, saveFormatedTree, freePTree); } else { diff --git a/system/jlib/jhash.cpp b/system/jlib/jhash.cpp index da42a193ef2..5ea87918a9d 100644 --- a/system/jlib/jhash.cpp +++ b/system/jlib/jhash.cpp @@ -64,12 +64,14 @@ MODULE_EXIT() // do not want these in your leak checking call releaseAtoms() } -#define HASHONE(hash, c) { hash *= 0x01000193; hash ^= c; } // Fowler/Noll/Vo Hash... seems to work pretty well, and fast +#define HASHONE(hash, c) { hash *= fnvPrime32; hash ^= c; } // Fowler/Noll/Vo Hash... seems to work pretty well, and fast +#define HASHONE_FNV1a(hash, c) { hash ^= c; hash *= fnvPrime32; } //Following is potentially quicker, but not tested //#define HASHONE(hash, c) { hash += (hash<<1) + (hash<<4) + (hash<<7) + (hash<<8) + (hash<<24); hash ^= c; } -unsigned hashc( const unsigned char *k, unsigned length, unsigned initval) +// deprecatedHash* functions (non template versions) kept for unittest only +unsigned deprecatedHashc(const unsigned char *k, unsigned length, unsigned initval) { unsigned hash = initval; unsigned char c; @@ -94,7 +96,7 @@ unsigned hashc( const unsigned char *k, unsigned length, unsigned initval) return hash; } -unsigned hashcz( const unsigned char *k, unsigned initval) +unsigned deprecatedHashcz(const unsigned char *k, unsigned initval) { unsigned hash = initval; for (;;) @@ -107,8 +109,87 @@ unsigned hashcz( const unsigned char *k, unsigned initval) return hash; } -template -inline unsigned doHashValue( T value, unsigned initval) +// FNV Hash Operations +struct FNV1 +{ + static inline void apply(unsigned &hash, unsigned char c) + { + hash *= fnvPrime32; + hash ^= c; + } +}; + +struct FNV1a +{ + static inline void apply(unsigned &hash, unsigned char c) + { + hash ^= c; + hash *= fnvPrime32; + } +}; + +template +unsigned policyHashc(const unsigned char *k, unsigned length, unsigned initval) +{ + unsigned hash = initval; + unsigned char c; + while (length >= 8) + { + c = (*k++); HashPolicy::apply(hash, c); + c = (*k++); HashPolicy::apply(hash, c); + c = (*k++); HashPolicy::apply(hash, c); + c = (*k++); HashPolicy::apply(hash, c); + length-=4; + } + switch (length) + { + case 7: c = (*k++); HashPolicy::apply(hash, c); // fallthrough + case 6: c = (*k++); HashPolicy::apply(hash, c); // fallthrough + case 5: c = (*k++); HashPolicy::apply(hash, c); // fallthrough + case 4: c = (*k++); HashPolicy::apply(hash, c); // fallthrough + case 3: c = (*k++); HashPolicy::apply(hash, c); // fallthrough + case 2: c = (*k++); HashPolicy::apply(hash, c); // fallthrough + case 1: c = (*k++); HashPolicy::apply(hash, c); + } + return hash; +} + +template +unsigned policyHashcz(const unsigned char *k, unsigned initval) +{ + unsigned hash = initval; + for (;;) + { + unsigned char c = (*k++); + if (c == 0) + break; + HashPolicy::apply(hash, c); + } + return hash; +} + +unsigned hashc(const unsigned char *k, unsigned length, unsigned initval) +{ + return policyHashc(k, length, initval); +} + +unsigned hashcz(const unsigned char *k, unsigned initval) +{ + return policyHashcz(k, initval); +} + +unsigned hashc_fnv1a(const unsigned char *k, unsigned length, unsigned initval) +{ + return policyHashc(k, length, initval); +} + +unsigned hashcz_fnv1a(const unsigned char *k, unsigned initval) +{ + return policyHashcz(k, initval); +} + +template +inline unsigned doHashValue(T value, unsigned initval) { //The values returned from this function are only consistent with those from hashn() if running on little endian architecture unsigned hash = initval; @@ -117,30 +198,47 @@ inline unsigned doHashValue( T value, unsigned initval) { c = (byte)value; value >>= 8; - HASHONE(hash, c); + HashPolicy::apply(hash, c); } return hash; } -unsigned hashvalue( unsigned value, unsigned initval) +unsigned hashvalue(unsigned value, unsigned initval) { - return doHashValue(value, initval); + return doHashValue(value, initval); } unsigned hashvalue( unsigned __int64 value, unsigned initval) { - return doHashValue(value, initval); + return doHashValue(value, initval); } unsigned hashvalue( const void * value, unsigned initval) { - return doHashValue((memsize_t)value, initval); + return doHashValue((memsize_t)value, initval); +} + +unsigned hashvalue_fnv1a(unsigned value, unsigned initval) +{ + return doHashValue(value, initval); +} + +unsigned hashvalue_fnv1a(unsigned __int64 value, unsigned initval) +{ + return doHashValue(value, initval); +} + +unsigned hashvalue_fnv1a(const void * value, unsigned initval) +{ + return doHashValue((memsize_t)value, initval); } + #define GETWORDNC(k,n) ((GETBYTE0(n)+GETBYTE1(n)+GETBYTE2(n)+GETBYTE3(n))&0xdfdfdfdf) -unsigned hashnc( const unsigned char *k, unsigned length, unsigned initval) +template +unsigned policyHashnc(const unsigned char *k, unsigned length, unsigned initval) { unsigned hash = initval; unsigned char c; @@ -165,8 +263,8 @@ unsigned hashnc( const unsigned char *k, unsigned length, unsigned initval) return hash; } - -unsigned hashncz( const unsigned char *k, unsigned initval) +template +unsigned policyHashncz(const unsigned char *k, unsigned initval) { unsigned hash = initval; for (;;) @@ -180,6 +278,26 @@ unsigned hashncz( const unsigned char *k, unsigned initval) return hash; } +unsigned hashnc(const unsigned char *k, unsigned length, unsigned initval) +{ + return policyHashnc(k, length, initval); +} + +unsigned hashncz(const unsigned char *k, unsigned initval) +{ + return policyHashncz(k, initval); +} + +unsigned hashnc_fnv1a(const unsigned char *k, unsigned length, unsigned initval) +{ + return policyHashnc(k, length, initval); +} + +unsigned hashncz_fnv1a(const unsigned char *k, unsigned initval) +{ + return policyHashncz(k, initval); +} + MappingKey::MappingKey(const void * inKey, int keysize) { int ksm = keysize; @@ -201,6 +319,7 @@ MappingKey::MappingKey(const void * inKey, int keysize) key = temp; } + //-- Mapping --------------------------------------------------- unsigned MappingBase::getHash() const { return hash; } diff --git a/system/jlib/jhash.hpp b/system/jlib/jhash.hpp index c8790fe2e4c..48e06f1e34e 100644 --- a/system/jlib/jhash.hpp +++ b/system/jlib/jhash.hpp @@ -425,13 +425,25 @@ extern jlib_decl IIdAtom * createIdAtom(const char *value); extern jlib_decl IIdAtom * createIdAtom(const char *value, size32_t len); extern jlib_decl void releaseAtoms(); -extern jlib_decl unsigned hashc( const unsigned char *k, unsigned length, unsigned initval); -extern jlib_decl unsigned hashnc( const unsigned char *k, unsigned length, unsigned initval); -extern jlib_decl unsigned hashcz( const unsigned char *k, unsigned initval); -extern jlib_decl unsigned hashncz( const unsigned char *k, unsigned initval); -extern jlib_decl unsigned hashvalue( unsigned value, unsigned initval); -extern jlib_decl unsigned hashvalue( unsigned __int64 value, unsigned initval); -extern jlib_decl unsigned hashvalue( const void * value, unsigned initval); + +extern jlib_decl unsigned deprecatedHashc(const unsigned char *k, unsigned length, unsigned initval); +extern jlib_decl unsigned deprecatedHashcz(const unsigned char *k, unsigned initval); + +extern jlib_decl unsigned hashc(const unsigned char *k, unsigned length, unsigned initval); +extern jlib_decl unsigned hashnc(const unsigned char *k, unsigned length, unsigned initval); +extern jlib_decl unsigned hashcz(const unsigned char *k, unsigned initval); +extern jlib_decl unsigned hashncz(const unsigned char *k, unsigned initval); +extern jlib_decl unsigned hashvalue(unsigned value, unsigned initval); +extern jlib_decl unsigned hashvalue(unsigned __int64 value, unsigned initval); +extern jlib_decl unsigned hashvalue(const void * value, unsigned initval); + +extern jlib_decl unsigned hashc_fnv1a(const unsigned char *k, unsigned length, unsigned initval); +extern jlib_decl unsigned hashnc_fnv1a(const unsigned char *k, unsigned length, unsigned initval); +extern jlib_decl unsigned hashcz_fnv1a(const unsigned char *k, unsigned initval); +extern jlib_decl unsigned hashncz_fnv1a(const unsigned char *k, unsigned initval); +extern jlib_decl unsigned hashvalue_fnv1a(unsigned value, unsigned initval); +extern jlib_decl unsigned hashvalue_fnv1a(unsigned __int64 value, unsigned initval); +extern jlib_decl unsigned hashvalue_fnv1a(const void * value, unsigned initval); //================================================ // Minimal Hash table template - slightly less overhead that HashTable/SuperHashTable diff --git a/system/jlib/jlzw.cpp b/system/jlib/jlzw.cpp index 3c18d04dc93..e66dda15d49 100644 --- a/system/jlib/jlzw.cpp +++ b/system/jlib/jlzw.cpp @@ -310,7 +310,7 @@ void CLZWCompressor::open(void *buf,size32_t max) -#define HASHC(code,ch) (((0x01000193*(unsigned)code)^(unsigned char)ch)%LZW_HASH_TABLE_SIZE) +#define HASHC(code,ch) (((fnvPrime32*(unsigned)code)^(unsigned char)ch)%LZW_HASH_TABLE_SIZE) #define BE_MEMCPY4(dst,src) { if (supportbigendian) _WINCPYREV4(dst,src); else memcpy(dst,src,4); } diff --git a/system/jlib/jptree.cpp b/system/jlib/jptree.cpp index 5b4151b2461..49de17a2900 100644 --- a/system/jlib/jptree.cpp +++ b/system/jlib/jptree.cpp @@ -3847,7 +3847,7 @@ unsigned CAtomPTree::queryHash() const { const char *_name = name.get(); size32_t nl = strlen(_name); - return isnocase() ? hashnc((const byte *) _name, nl, fnvInitialHash32): hashc((const byte *) _name, nl, fnvInitialHash32); + return isnocase() ? hashnc_fnv1a((const byte *) _name, nl, fnvInitialHash32): hashc_fnv1a((const byte *) _name, nl, fnvInitialHash32); } } diff --git a/system/jlib/jptree.ipp b/system/jlib/jptree.ipp index fd5cb63fd89..4c5c0d194a7 100644 --- a/system/jlib/jptree.ipp +++ b/system/jlib/jptree.ipp @@ -58,7 +58,7 @@ protected: virtual unsigned getHashFromElement(const void *e) const override; virtual unsigned getHashFromFindParam(const void *fp) const override { - return hashcz((const unsigned char *)fp, fnvInitialHash32); + return hashcz_fnv1a((const unsigned char *)fp, fnvInitialHash32); } virtual bool matchesFindParam(const void *e, const void *fp, unsigned fphash) const override { @@ -108,7 +108,7 @@ public: // SuperHashTable definitions virtual unsigned getHashFromFindParam(const void *fp) const override { - return hashncz((const unsigned char *)fp, fnvInitialHash32); + return hashncz_fnv1a((const unsigned char *)fp, fnvInitialHash32); } virtual bool matchesFindParam(const void *e, const void *fp, unsigned fphash) const override { @@ -844,7 +844,7 @@ public: const char *myname = queryName(); assert(myname); size32_t nl = strlen(myname); - return isnocase() ? hashnc((const byte *)myname, nl, fnvInitialHash32): hashc((const byte *)myname, nl, fnvInitialHash32); + return isnocase() ? hashnc_fnv1a((const byte *)myname, nl, fnvInitialHash32): hashc_fnv1a((const byte *)myname, nl, fnvInitialHash32); } virtual void setName(const char *_name) override; virtual void setAttribute(const char *attr, const char *val, bool encoded) override; diff --git a/system/jlib/jsuperhash.hpp b/system/jlib/jsuperhash.hpp index 2c82a8fb5e8..51f4d62078c 100644 --- a/system/jlib/jsuperhash.hpp +++ b/system/jlib/jsuperhash.hpp @@ -29,11 +29,16 @@ #include "jmutex.hpp" constexpr unsigned fnvInitialHash32 = 0x811C9DC5; +constexpr unsigned fnvPrime32 = 0x01000193; extern jlib_decl unsigned hashc( const unsigned char *k, unsigned length, unsigned initval); extern jlib_decl unsigned hashnc( const unsigned char *k, unsigned length, unsigned initval); extern jlib_decl unsigned hashcz( const unsigned char *k, unsigned initval); extern jlib_decl unsigned hashncz( const unsigned char *k, unsigned initval); +extern jlib_decl unsigned hashc_fnv1a(const unsigned char *k, unsigned length, unsigned initval); +extern jlib_decl unsigned hashnc_fnv1a(const unsigned char *k, unsigned length, unsigned initval); +extern jlib_decl unsigned hashcz_fnv1a(const unsigned char *k, unsigned initval); +extern jlib_decl unsigned hashncz_fnv1a(const unsigned char *k, unsigned initval); class jlib_decl SuperHashTable : public CInterface { @@ -516,9 +521,9 @@ class jlib_decl AtomRefTable : public SuperHashTableOfkeyPtr()), key, l+1); if (nocase) - hke->hashValue = hashnc((const unsigned char *)key, l, fnvInitialHash32); + hke->hashValue = hashnc_fnv1a((const unsigned char *)key, l, fnvInitialHash32); else - hke->hashValue = hashc((const unsigned char *)key, l, fnvInitialHash32); + hke->hashValue = hashc_fnv1a((const unsigned char *)key, l, fnvInitialHash32); hke->linkCount = 0; return hke; } @@ -611,9 +616,9 @@ class jlib_decl AtomRefTable : public SuperHashTableOf -#include #include +#include +#include +#include +#include + #include "jsem.hpp" #include "jfile.hpp" #include "jdebug.hpp" @@ -3055,5 +3058,120 @@ class JLibUnicodeTest : public CppUnit::TestFixture CPPUNIT_TEST_SUITE_REGISTRATION( JLibUnicodeTest ); CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( JLibUnicodeTest, "JLibUnicodeTest" ); +class HashFuncTests : public CppUnit::TestFixture +{ +public: + virtual void setUp() override + { + generateFixedRandomNullTermiantedStrings(); + } + + CPPUNIT_TEST_SUITE(HashFuncTests); + CPPUNIT_TEST(fnvTests); + CPPUNIT_TEST_SUITE_END(); + +protected: + static constexpr unsigned maxCalls = 10'000'000; + static constexpr unsigned minLen = 5, maxLen = 100; + static constexpr unsigned lenRange = maxLen - minLen + 1; + static constexpr unsigned randomSeed = 42; + static constexpr size_t testBufferSize = 1'000'000; + CCycleTimer timer; + std::vector buffer; + + unsigned getOffsetLenHash(unsigned offset, unsigned hash) + { + hash ^= (offset * 0x27D4EB2D); // use MurMurHash3 multiplier to introduce more randomness + hash *= fnvPrime32; + return hash; + } + void generateFixedRandomNullTermiantedStrings() + { + buffer.resize(testBufferSize); + std::mt19937 rng(randomSeed); + std::uniform_int_distribution dist(1, 255); + + unsigned offset = 0; + unsigned lenHash = fnvInitialHash32; + while (offset < testBufferSize) + { + // create str lengths between min and max based on offset, + // so that we can predictably read them back + lenHash = getOffsetLenHash(offset, lenHash); + unsigned len = (lenHash % lenRange) + minLen; + + if (offset + len + 1 >= testBufferSize) + break; + + for (unsigned i=0; i + void testHashc() + { + unsigned hashResult = fnvInitialHash32; + + unsigned offset = 0; + for (unsigned i=0; i buffer.size()) + offset = 0; + + hashResult ^= HASHCFUNC(&buffer[offset], len, hashResult); + offset += len; + } + CPPUNIT_ASSERT(hashResult != 0); + } + template + void testHashcz() + { + unsigned hashResult = fnvInitialHash32; + + unsigned lenHash = fnvInitialHash32; + unsigned offset = 0; + for (unsigned i=0; i buffer.size()) + { + offset = 0; + lenHash = getOffsetLenHash(offset, fnvInitialHash32); + len = (lenHash % lenRange) + minLen; + } + dbgassertex(len == strlen((const char *)&buffer[offset])); + hashResult ^= HASHCZFUNC(&buffer[offset], hashResult); + offset += len + 1; + } + CPPUNIT_ASSERT(hashResult != 0); + } + void measure(const char *funcName, const std::function &func) + { + timer.reset(); + func(); + unsigned elapsed = timer.elapsedMs(); + double throughput = static_cast(maxCalls) / elapsed * 1000; + PROGLOG("%s: %u calls took %u ms (%.2f hashes/sec)", funcName, maxCalls, elapsed, throughput); + } + void fnvTests() + { + measure("deprecatedHashc (fnv1)", [this]() { testHashc(); }); + measure("deprecatedHashcz (fnv1)", [this]() { testHashcz(); }); + measure("hashc (fnv1)", [this]() { testHashc(); }); + measure("hashcz (fnv1)", [this]() { testHashcz(); }); + measure("hashc_fnv1a", [this]() { testHashc(); }); + measure("hashcz_fnv1a", [this]() { testHashcz(); }); + } +}; + +CPPUNIT_TEST_SUITE_REGISTRATION( HashFuncTests ); +CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( HashFuncTests, "HashFuncTests" ); #endif // _USE_CPPUNIT From fb4db8fb971c55366e46160588e53870683f3881 Mon Sep 17 00:00:00 2001 From: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> Date: Fri, 3 Jan 2025 12:46:25 -0500 Subject: [PATCH 2/5] HPCC-33140 ECL Watch v9 fix WU details flicker fixes an issue on the WU details page, where the data shown would flicker between different values as you scroll the window up & down (most noticeable when viewing archived WUs) Signed-off-by: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> --- esp/src/src-react/components/WorkunitSummary.tsx | 8 +++----- esp/src/src-react/components/controls/StateIcon.tsx | 2 +- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/esp/src/src-react/components/WorkunitSummary.tsx b/esp/src/src-react/components/WorkunitSummary.tsx index c5e0f5c4b6c..c73e6cf244f 100644 --- a/esp/src/src-react/components/WorkunitSummary.tsx +++ b/esp/src/src-react/components/WorkunitSummary.tsx @@ -251,12 +251,10 @@ export const WorkunitSummary: React.FunctionComponent = ({ } - +
-
- -
- + +
= ({ } }, [workunit, workunit?.StateID, theme.semanticColors.errorIcon, theme.semanticColors.successIcon, theme.semanticColors.warningIcon]); - return + return {showProtected && From 7cb34d90d1b0987ca48fe4e9c3cef5ad754b577c Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Tue, 7 Jan 2025 15:19:15 +0000 Subject: [PATCH 3/5] HPCC-33164 Remove experimental decorators (Not supported in esbuild / vite) Signed-off-by: Gordon Smith --- esp/src/src/DataPatternsWidget.ts | 9 +++++---- esp/src/src/DeclareDecorator.ts | 5 +++++ esp/src/src/ECLArchiveWidget.ts | 9 +++++---- esp/src/src/Graph7Widget.ts | 8 ++++---- esp/src/src/GraphTree7Widget.ts | 11 ++++++----- esp/src/src/GraphTreeWidget.ts | 8 ++++---- esp/src/tsconfig.json | 1 - 7 files changed, 29 insertions(+), 22 deletions(-) diff --git a/esp/src/src/DataPatternsWidget.ts b/esp/src/src/DataPatternsWidget.ts index 75c479c049f..f030a85e231 100644 --- a/esp/src/src/DataPatternsWidget.ts +++ b/esp/src/src/DataPatternsWidget.ts @@ -34,7 +34,7 @@ import "dijit/TooltipDialog"; import "hpcc/TableContainer"; import "hpcc/TargetSelectWidget"; -import { declareDecorator } from "./DeclareDecorator"; +import { declareMixin } from "./DeclareDecorator"; import { WUStatus } from "./WUStatus"; type _TabContainerWidget = { @@ -50,11 +50,10 @@ type _TabContainerWidget = { export const supportedFileType = (contentType: string): boolean => ["flat", "csv", "thor"].indexOf((contentType || "").toLowerCase()) >= 0; -export interface DataPatternsWidget extends _TabContainerWidget { +interface _DataPatternsWidget extends _TabContainerWidget { } -@declareDecorator("DataPatternsWidget", _TabContainerWidget) -export class DataPatternsWidget { +class _DataPatternsWidget { templateString = template; static baseClass = "DataPatternsWidget"; i18n = nlsHPCC; @@ -296,3 +295,5 @@ export class DataPatternsWidget { ; } } + +export const DataPatternsWidget = declareMixin(_DataPatternsWidget, "DataPatternsWidget", _TabContainerWidget); diff --git a/esp/src/src/DeclareDecorator.ts b/esp/src/src/DeclareDecorator.ts index cb5e5d8debd..603f1ed7e5c 100644 --- a/esp/src/src/DeclareDecorator.ts +++ b/esp/src/src/DeclareDecorator.ts @@ -10,3 +10,8 @@ export function declareDecorator(classID: string, ...mixins: object[]) { return declare(classID, mixins, target.prototype); }; } + +type Constructor = new (...args: any[]) => T; +export function declareMixin(target: T, classID: string, ...mixins: object[]): T { + return declare(classID, mixins, target.prototype); +} diff --git a/esp/src/src/ECLArchiveWidget.ts b/esp/src/src/ECLArchiveWidget.ts index 8076a41433e..14d58ff37fa 100644 --- a/esp/src/src/ECLArchiveWidget.ts +++ b/esp/src/src/ECLArchiveWidget.ts @@ -18,7 +18,7 @@ import { themeIsDark } from "./Utility"; import * as template from "dojo/text!hpcc/templates/ECLArchiveWidget.html"; // @ts-ignore import * as _Widget from "hpcc/_Widget"; -import { declareDecorator } from "./DeclareDecorator"; +import { declareMixin } from "./DeclareDecorator"; const TIME_NAMES = ["TimeMaxLocalExecute", "TimeAvgLocalExecute", "TimeLocalExecute"]; @@ -36,11 +36,10 @@ type _Widget = { setDisabled(id: string, disabled: boolean, icon?: string, disabledIcon?: string); }; -export interface ECLArchiveWidget extends _Widget { +interface _ECLArchiveWidget extends _Widget { } -@declareDecorator("ECLArchiveWidget", _Widget) -export class ECLArchiveWidget { +class _ECLArchiveWidget { protected templateString = template; static baseClass = "ECLArchiveWidget"; protected i18n = nlsHPCC; @@ -714,3 +713,5 @@ export class ECLArchiveWidget { } } } +export const ECLArchiveWidget = declareMixin(_ECLArchiveWidget, "ECLArchiveWidget", _Widget); + diff --git a/esp/src/src/Graph7Widget.ts b/esp/src/src/Graph7Widget.ts index 1c72eba37e4..5bb3ee1f6d2 100644 --- a/esp/src/src/Graph7Widget.ts +++ b/esp/src/src/Graph7Widget.ts @@ -21,7 +21,7 @@ import "dijit/layout/ContentPane"; import "dijit/Toolbar"; import "dijit/ToolbarSeparator"; -import { declareDecorator } from "./DeclareDecorator"; +import { declareMixin } from "./DeclareDecorator"; import nlsHPCC from "./nlsHPCC"; import { WUScopeController } from "./WUScopeController"; @@ -33,11 +33,10 @@ type _Widget = { setDisabled(id: string, disabled: boolean, icon?: string, disabledIcon?: string); }; -export interface Graph7Widget extends _Widget { +interface _Graph7Widget extends _Widget { } -@declareDecorator("Graph7Widget", _Widget) -export class Graph7Widget { +class _Graph7Widget { templateString = template; static baseClass = "Graph7Widget"; i18n = nlsHPCC; @@ -252,3 +251,4 @@ export class Graph7Widget { ; } } +export const Graph7Widget = declareMixin(_Graph7Widget, "Graph7Widget", _Widget); diff --git a/esp/src/src/GraphTree7Widget.ts b/esp/src/src/GraphTree7Widget.ts index 5238395e430..0f792c18095 100644 --- a/esp/src/src/GraphTree7Widget.ts +++ b/esp/src/src/GraphTree7Widget.ts @@ -15,7 +15,7 @@ import { hashSum } from "@hpcc-js/util"; // @ts-ignore import * as _Widget from "hpcc/_Widget"; -import { declareDecorator } from "./DeclareDecorator"; +import { declareMixin } from "./DeclareDecorator"; import { Grid, maximizeWidget } from "./ESPUtil"; import { GraphStore, GraphTreeStore } from "./GraphStore"; import nlsHPCC from "./nlsHPCC"; @@ -88,11 +88,10 @@ type _Widget = { setDisabled(id: string, disabled: boolean, icon?: string, disabledIcon?: string); }; -export interface GraphTree7Widget extends _Widget { +interface _GraphTree7Widget extends _Widget { } -@declareDecorator("GraphTree7Widget", _Widget) -export class GraphTree7Widget { +class _GraphTree7Widget { templateString = template; static baseClass = "GraphTree7Widget"; i18n = nlsHPCC; @@ -761,7 +760,7 @@ export class GraphTree7Widget { } } -GraphTree7Widget.prototype._syncSelectionFrom = debounce(function (this: GraphTree7Widget, sourceControlOrGlobalIDs) { +_GraphTree7Widget.prototype._syncSelectionFrom = debounce(function (this: _GraphTree7Widget, sourceControlOrGlobalIDs) { this.inSyncSelectionFrom = true; const sourceControl = sourceControlOrGlobalIDs instanceof Array ? null : sourceControlOrGlobalIDs; let selectedGlobalIDs = sourceControlOrGlobalIDs instanceof Array ? sourceControlOrGlobalIDs : []; @@ -846,3 +845,5 @@ GraphTree7Widget.prototype._syncSelectionFrom = debounce(function (this: GraphTr } this.inSyncSelectionFrom = false; }, 500, false); + +export const GraphTree7Widget = declareMixin(_GraphTree7Widget, "GraphTree7Widget", _Widget); \ No newline at end of file diff --git a/esp/src/src/GraphTreeWidget.ts b/esp/src/src/GraphTreeWidget.ts index 0a2e74d80fc..10fec1ba60b 100644 --- a/esp/src/src/GraphTreeWidget.ts +++ b/esp/src/src/GraphTreeWidget.ts @@ -48,13 +48,12 @@ import "dijit/ToolbarSeparator"; import "hpcc/JSGraphWidget"; import "hpcc/TimingTreeMapWidget"; -import { declareDecorator } from "./DeclareDecorator"; +import { declareMixin } from "./DeclareDecorator"; type _Widget = any; -export interface GraphTreeWidget extends _Widget { } +interface _GraphTreeWidget extends _Widget { } -@declareDecorator("GraphTreeWidget", _Widget) -export class GraphTreeWidget { +class _GraphTreeWidget { templateString = template; baseClass = "GraphTreeWidget"; i18n = nlsHPCC; @@ -796,3 +795,4 @@ export class GraphTreeWidget { this.setDisabled(this.id + "ActivityMetric", tab.id !== this.id + "ActivitiesTreeMap"); } } +export const GraphTreeWidget = declareMixin(_GraphTreeWidget, "GraphTreeWidget", _Widget); diff --git a/esp/src/tsconfig.json b/esp/src/tsconfig.json index 55fd90ef44b..a2340603d81 100644 --- a/esp/src/tsconfig.json +++ b/esp/src/tsconfig.json @@ -14,7 +14,6 @@ "noUnusedLocals": true, "strictNullChecks": false, "importHelpers": true, - "experimentalDecorators": true, "allowJs": true, "skipLibCheck": true, "noImplicitUseStrict": true, From 611dce32dfaae52a4674162cbd206fa0f0604b6e Mon Sep 17 00:00:00 2001 From: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> Date: Tue, 7 Jan 2025 11:49:52 -0500 Subject: [PATCH 4/5] HPCC-33169 ECL Watch v9 fix spinner on playground initial load fix an issue where the graph and results/errors components on the ECL playground page in the v9 UI would display spinners on intial load Signed-off-by: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> --- esp/src/src-react/components/ECLPlayground.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/esp/src/src-react/components/ECLPlayground.tsx b/esp/src/src-react/components/ECLPlayground.tsx index ceca0b9eed9..c6d9eca7320 100644 --- a/esp/src/src-react/components/ECLPlayground.tsx +++ b/esp/src/src-react/components/ECLPlayground.tsx @@ -471,10 +471,11 @@ export const ECLPlayground: React.FunctionComponent = (props useOnEvent(document, "eclwatch-theme-toggle", handleThemeToggle); const submissionComplete = React.useMemo(() => { + if (!workunit?.Wuid) return true; return workunit?.StateID === WUStateID.Completed || workunit?.StateID === WUStateID.Failed || (workunit?.ActionEx === "compile" && workunit?.StateID === WUStateID.Compiled); - }, [workunit?.StateID, workunit?.ActionEx]); + }, [workunit?.StateID, workunit?.ActionEx, workunit?.Wuid]); const handleEclChange = React.useMemo(() => debounce((evt) => { if (editor.hasFocus()) { From ec9ddf051633ba8ce08c95f1ad70f616a96f384f Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Wed, 8 Jan 2025 10:00:07 +0000 Subject: [PATCH 5/5] HPCC-33179 Ensure relative pathing for "src" folder Signed-off-by: Gordon Smith --- esp/src/src/ESPSearch.ts | 8 ++++---- esp/src/src/Session.ts | 2 +- esp/src/src/Utility.ts | 12 ++++++------ esp/src/src/WsTopology.ts | 4 ++-- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/esp/src/src/ESPSearch.ts b/esp/src/src/ESPSearch.ts index 357ba2432c5..5991bd39265 100644 --- a/esp/src/src/ESPSearch.ts +++ b/esp/src/src/ESPSearch.ts @@ -1,8 +1,8 @@ import * as Observable from "dojo/store/Observable"; -import * as ESPWorkunit from "src/ESPWorkunit"; -import * as ESPDFUWorkunit from "src/ESPDFUWorkunit"; -import * as ESPLogicalFile from "src/ESPLogicalFile"; -import * as ESPQuery from "src/ESPQuery"; +import * as ESPWorkunit from "./ESPWorkunit"; +import * as ESPDFUWorkunit from "./ESPDFUWorkunit"; +import * as ESPLogicalFile from "./ESPLogicalFile"; +import * as ESPQuery from "./ESPQuery"; import { Memory } from "./store/Memory"; import * as WsWorkunits from "./WsWorkunits"; import * as FileSpray from "./FileSpray"; diff --git a/esp/src/src/Session.ts b/esp/src/src/Session.ts index 10c6fe5e186..eb0afbd70a9 100644 --- a/esp/src/src/Session.ts +++ b/esp/src/src/Session.ts @@ -4,7 +4,7 @@ import * as topic from "dojo/topic"; import { format as d3Format } from "@hpcc-js/common"; import { SMCService } from "@hpcc-js/comms"; import { scopedLogger } from "@hpcc-js/util"; -import { cookieKeyValStore, sessionKeyValStore, userKeyValStore } from "src/KeyValStore"; +import { cookieKeyValStore, sessionKeyValStore, userKeyValStore } from "./KeyValStore"; import { singletonDebounce } from "../src-react/util/throttle"; import { parseSearch, replaceUrl } from "../src-react/util/history"; import { ModernMode } from "./BuildInfo"; diff --git a/esp/src/src/Utility.ts b/esp/src/src/Utility.ts index d17b93257cb..be9e751b388 100644 --- a/esp/src/src/Utility.ts +++ b/esp/src/src/Utility.ts @@ -5,7 +5,7 @@ import * as arrayUtil from "dojo/_base/array"; import * as domConstruct from "dojo/dom-construct"; import * as entities from "dojox/html/entities"; import { darkTheme } from "../src-react/themes"; -import nlsHPCC from "src/nlsHPCC"; +import nlsHPCC from "./nlsHPCC"; declare const dojoConfig; declare const ActiveXObject; @@ -631,7 +631,7 @@ export function resolve(hpccWidget, callback) { require(["hpcc/viz/DojoD3NDChart"], doLoad); break; case "DataPatternsWidget": - require(["src/DataPatternsWidget"], doLoad); + require(["./DataPatternsWidget"], doLoad); break; case "DynamicESDLDefinitionDetailsWidget": require(["hpcc/DynamicESDLDefinitionDetailsWidget"], doLoad); @@ -655,7 +655,7 @@ export function resolve(hpccWidget, callback) { require(["hpcc/ECLPlaygroundWidget"], doLoad); break; case "ECLArchiveWidget": - require(["src/ECLArchiveWidget"], doLoad); + require(["./ECLArchiveWidget"], doLoad); break; case "ECLSourceWidget": require(["hpcc/ECLSourceWidget"], doLoad); @@ -710,13 +710,13 @@ export function resolve(hpccWidget, callback) { require(["hpcc/GraphsLFWidget"], doLoad); break; case "GraphTreeWidget": - require(["src/GraphTreeWidget"], doLoad); + require(["./GraphTreeWidget"], doLoad); break; case "GraphTree7Widget": - require(["src/GraphTree7Widget"], doLoad); + require(["./GraphTree7Widget"], doLoad); break; case "Graph7Widget": - require(["src/Graph7Widget"], doLoad); + require(["./Graph7Widget"], doLoad); break; case "GridDetailsWidget": require(["hpcc/GridDetailsWidget"], doLoad); diff --git a/esp/src/src/WsTopology.ts b/esp/src/src/WsTopology.ts index 6c88185fe2f..b8bc7ff363a 100644 --- a/esp/src/src/WsTopology.ts +++ b/esp/src/src/WsTopology.ts @@ -1,7 +1,7 @@ import { Connection, ResourcesService, Topology } from "@hpcc-js/comms"; import { scopedLogger } from "@hpcc-js/util"; -import { containerized } from "src/BuildInfo"; -import { Memory } from "src/store/Memory"; +import { containerized } from "./BuildInfo"; +import { Memory } from "./store/Memory"; import * as arrayUtil from "dojo/_base/array"; import * as Deferred from "dojo/_base/Deferred"; import * as lang from "dojo/_base/lang";