set fastest choice

erigon-lib/commitment/commitment.go

@@ -796,13 +796,12 @@ func ParseCommitmentMode(s string) Mode {
 }
 
 type UpdateTree struct {
-	keccak    cryptozerocopy.KeccakState
-	hasher    keyHasher
-	tree      *btree.BTreeG[*KeyUpdate]
-	mode      Mode
-	updates   uint64
-	collector *etl.Collector
-	tmpdir    string
+	keccak cryptozerocopy.KeccakState
+	hasher keyHasher
+	keys   map[string]struct{}
+	tree   *btree.BTreeG[*KeyUpdate]
+	mode   Mode
+	tmpdir string
 }
 
 type keyHasher func(key []byte) []byte
@@ -813,24 +812,17 @@ func NewUpdateTree(m Mode, tmpdir string, hasher keyHasher) *UpdateTree {
 	t := &UpdateTree{
 		keccak: sha3.NewLegacyKeccak256().(cryptozerocopy.KeccakState),
 		hasher: hasher,
-		mode:   m,
 		tmpdir: tmpdir,
+		mode:   m,
 	}
 	if t.mode == ModeDirect {
-		t.initCollector()
+		t.keys = make(map[string]struct{})
 	} else if t.mode == ModeUpdate {
 		t.tree = btree.NewG[*KeyUpdate](64, keyUpdateLessFn)
 	}
 	return t
 }
 
-func (t *UpdateTree) initCollector() {
-	t.collector = etl.NewCollector("commitment", t.tmpdir, etl.NewSortableBuffer(etl.BufferOptimalSize/4), log.Root().New("update-tree"))
-	t.collector.LogLvl(log.LvlDebug)
-	t.collector.SortAndFlushInBackground(true)
-	t.updates = 0
-}
-
 // TouchPlainKey marks plainKey as updated and applies different fn for different key types
 // (different behaviour for Code, Account and Storage key modifications).
 func (t *UpdateTree) TouchPlainKey(key, val []byte, fn func(c *KeyUpdate, val []byte)) {
@@ -851,26 +843,20 @@ func (t *UpdateTree) TouchPlainKey(key, val []byte, fn func(c *KeyUpdate, val []
 			fn(pivot, val)
 			t.tree.ReplaceOrInsert(pivot)
 		}
-		t.updates++
 	case ModeDirect:
-		err := t.collector.Collect(t.hasher(key), key)
-		if err != nil {
-			log.Error("[commitment] failed to collect key", "err", err)
-		}
-		t.updates++
+		t.keys[string(key)] = struct{}{}
 	default:
 	}
 }
 
-func (t *UpdateTree) Size() (updates uint64, unique bool) {
+func (t *UpdateTree) Size() (updates uint64) {
 	switch t.mode {
 	case ModeDirect:
-		//return uint64(len(t.keys)), true
-		return t.updates, false
+		return uint64(len(t.keys))
 	case ModeUpdate:
-		return uint64(t.tree.Len()), true
+		return uint64(t.tree.Len())
 	default:
-		return 0, false
+		return 0
 	}
 }
 
@@ -929,9 +915,8 @@ func (t *UpdateTree) TouchCode(c *KeyUpdate, val []byte) {
 }
 
 func (t *UpdateTree) Close() {
-	if t.collector != nil {
-		t.collector.Close()
-		t.collector = nil
+	if t.keys != nil {
+		clear(t.keys)
 	}
 	if t.tree != nil {
 		t.tree.Clear(true)
@@ -942,25 +927,30 @@ func (t *UpdateTree) Close() {
 func (t *UpdateTree) HashSort(ctx context.Context, fn func(hk, pk []byte) error) error {
 	switch t.mode {
 	case ModeDirect:
-		var phk []byte
-		var initialised bool
+		collector := etl.NewCollector("commitment", t.tmpdir, etl.NewSortableBuffer(etl.BufferOptimalSize/4), log.Root().New("update-tree"))
+		defer collector.Close()
+		collector.LogLvl(log.LvlDebug)
+		collector.SortAndFlushInBackground(true)
 
-		if err := t.collector.Load(nil, "", func(k, v []byte, table etl.CurrentTableReader, next etl.LoadNextFunc) error {
-			if initialised && bytes.Equal(phk, k) {
+		for k := range t.keys {
+			select {
+			case <-ctx.Done():
 				return nil
+			default:
 			}
-			if err := fn(k, v); err != nil {
+			if err := collector.Collect(t.hasher([]byte(k)), []byte(k)); err != nil {
 				return err
 			}
-			phk = append(phk[:0], k...)
-			initialised = true
-			return nil
-		}, etl.TransformArgs{Quit: ctx.Done()}); err != nil {
-			t.collector.Close()
+		}
+
+		err := collector.Load(nil, "", func(k, v []byte, table etl.CurrentTableReader, next etl.LoadNextFunc) error {
+			return fn(k, v)
+		}, etl.TransformArgs{Quit: ctx.Done()})
+		if err != nil {
 			return err
 		}
-		t.collector.Close()
-		t.initCollector()
+
+		clear(t.keys)
 	case ModeUpdate:
 		t.tree.Ascend(func(item *KeyUpdate) bool {
 			select {
@@ -986,15 +976,14 @@ func (t *UpdateTree) HashSort(ctx context.Context, fn func(hk, pk []byte) error)
 func (t *UpdateTree) List(clear bool) ([][]byte, []Update) {
 	switch t.mode {
 	case ModeDirect:
-		plainKeys := make([][]byte, 0, t.updates)
+		plainKeys := make([][]byte, 0, len(t.keys))
 		err := t.HashSort(context.Background(), func(hk, pk []byte) error {
 			plainKeys = append(plainKeys, common.Copy(pk))
 			return nil
 		})
 		if err != nil {
 			return nil, nil
 		}
-		t.updates = 0
 		return plainKeys, nil
 	case ModeUpdate:
 		plainKeys := make([][]byte, t.tree.Len())

erigon-lib/commitment/commitment_test.go

@@ -295,15 +295,15 @@ func TestNewUpdateTree(t *testing.T) {
 
 		require.NotNil(t, ut.tree)
 		require.NotNil(t, ut.keccak)
-		require.Nil(t, ut.collector)
+		require.Nil(t, ut.keys)
 		require.Equal(t, ModeUpdate, ut.mode)
 	})
 
 	t.Run("ModeDirect", func(t *testing.T) {
 		ut := NewUpdateTree(ModeDirect, t.TempDir(), keyHasherNoop)
 
 		require.NotNil(t, ut.keccak)
-		require.NotNil(t, ut.collector)
+		require.NotNil(t, ut.keys)
 		require.Equal(t, ModeDirect, ut.mode)
 	})
 
@@ -345,13 +345,11 @@ func TestUpdateTree_TouchPlainKey(t *testing.T) {
 		return bytes.Compare(sortedUniqUpds[i].key, sortedUniqUpds[j].key) < 0
 	})
 
-	sz, uniq := utUpdate.Size()
+	sz := utUpdate.Size()
 	require.EqualValues(t, 3, sz)
-	require.True(t, uniq)
 
-	sz, uniq = utDirect.Size()
-	require.EqualValues(t, len(upds), sz)
-	require.False(t, uniq)
+	sz = utDirect.Size()
+	require.EqualValues(t, 3, sz)
 
 	pk, upd := utUpdate.List(true)
 	require.Len(t, pk, 3)

erigon-lib/commitment/hex_patricia_hashed.go

@@ -1286,15 +1286,14 @@ func (hph *HexPatriciaHashed) ProcessTree(ctx context.Context, tree *UpdateTree,
 		ki uint64
 	)
 	defer logEvery.Stop()
-	updatesCount, unique := tree.Size()
+	updatesCount := tree.Size()
 
 	err = tree.HashSort(ctx, func(hashedKey, plainKey []byte) error {
 		select {
 		case <-logEvery.C:
 			dbg.ReadMemStats(&m)
 			log.Info(fmt.Sprintf("[%s][agg] computing trie", logPrefix),
-				"progress", fmt.Sprintf("%dk/%dk", ki/1000, updatesCount/1000),
-				"countUnique", unique, "alloc", common.ByteCount(m.Alloc), "sys", common.ByteCount(m.Sys))
+				"progress", fmt.Sprintf("%dk/%dk", ki/1000, updatesCount/1000), "alloc", common.ByteCount(m.Alloc), "sys", common.ByteCount(m.Sys))
 		default:
 		}
 

erigon-lib/state/domain_shared.go

@@ -1089,7 +1089,7 @@ func (sdc *SharedDomainsCommitmentContext) TempDir() string {
 }
 
 func (sdc *SharedDomainsCommitmentContext) KeysCount() uint64 {
-	s, _ := sdc.updates.Size()
+	s := sdc.updates.Size()
 	return s
 }
 
@@ -1124,9 +1124,9 @@ func (sdc *SharedDomainsCommitmentContext) ComputeCommitment(ctx context.Context
 	defer mxCommitmentRunning.Dec()
 	defer func(s time.Time) { mxCommitmentTook.ObserveDuration(s) }(time.Now())
 
-	updateCount, areUnique := sdc.updates.Size()
+	updateCount := sdc.updates.Size()
 	if sdc.sd.trace {
-		defer sdc.sd.logger.Trace("ComputeCommitment", "block", blockNum, "keys", updateCount, "unique", areUnique, "mode", sdc.mode)
+		defer sdc.sd.logger.Trace("ComputeCommitment", "block", blockNum, "keys", updateCount, "mode", sdc.mode)
 	}
 	if updateCount == 0 {
 		rootHash, err = sdc.patriciaTrie.RootHash()