From d5abac7ceb68aaefdc3f0ae48622afd93ae51e5d Mon Sep 17 00:00:00 2001 From: vladenysiuk Date: Tue, 14 May 2024 17:34:39 +0200 Subject: [PATCH 01/18] basic batchaggregator --- .../chain-deps/facets/Executor.sol | 7 +++++++ .../chain-interfaces/IExecutor.sol | 3 ++- system-contracts/contracts/BatchAggregator.sol | 18 ++++++++++++++++++ system-contracts/contracts/Constants.sol | 6 +++++- system-contracts/contracts/L1Messenger.sol | 11 ++++++++++- .../contracts/interfaces/IBatchAggregator.sol | 11 +++++++++++ 6 files changed, 53 insertions(+), 3 deletions(-) create mode 100644 system-contracts/contracts/BatchAggregator.sol create mode 100644 system-contracts/contracts/interfaces/IBatchAggregator.sol diff --git a/l1-contracts/contracts/state-transition/chain-deps/facets/Executor.sol b/l1-contracts/contracts/state-transition/chain-deps/facets/Executor.sol index 506379eb4..0a8ce8ce3 100644 --- a/l1-contracts/contracts/state-transition/chain-deps/facets/Executor.sol +++ b/l1-contracts/contracts/state-transition/chain-deps/facets/Executor.sol @@ -11,6 +11,10 @@ import {UnsafeBytes} from "../../../common/libraries/UnsafeBytes.sol"; import {L2_BOOTLOADER_ADDRESS, L2_TO_L1_MESSENGER_SYSTEM_CONTRACT_ADDR, L2_SYSTEM_CONTEXT_SYSTEM_CONTRACT_ADDR, L2_PUBDATA_CHUNK_PUBLISHER_ADDR} from "../../../common/L2ContractAddresses.sol"; import {PubdataPricingMode} from "../ZkSyncHyperchainStorage.sol"; import {IStateTransitionManager} from "../../IStateTransitionManager.sol"; +import {IBatchAggregator} from "../../../../../system-contracts/contracts/interfaces/IBatchAggregator.sol"; + +// TODO: replace with actual system address +constant address BATCH_AGGREGATOR_ADDRESS = 0x0000000000000000000000000000000000009012; // While formally the following import is not used, it is needed to inherit documentation from it import {IZkSyncHyperchainBase} from "../../chain-interfaces/IZkSyncHyperchainBase.sol"; @@ -69,6 +73,9 @@ contract ExecutorFacet is ZkSyncHyperchainBase, IExecutor { .pubdataCommitments .length] ); + } else if (pubdataSource == uint8(PubdataSource.ForwardedMessage)) { + // Similar to Calldata, forwards packed pubdataCommitments of hyperchains to the lower layer + } require(_previousBatch.batchHash == logOutput.previousBatchHash, "l"); diff --git a/l1-contracts/contracts/state-transition/chain-interfaces/IExecutor.sol b/l1-contracts/contracts/state-transition/chain-interfaces/IExecutor.sol index 43a7485c7..fc1ea1e6c 100644 --- a/l1-contracts/contracts/state-transition/chain-interfaces/IExecutor.sol +++ b/l1-contracts/contracts/state-transition/chain-interfaces/IExecutor.sol @@ -25,7 +25,8 @@ enum SystemLogKey { /// @dev Enum used to determine the source of pubdata. At first we will support calldata and blobs but this can be extended. enum PubdataSource { Calldata, - Blob + Blob, + ForwardedMessage } struct LogProcessingOutput { diff --git a/system-contracts/contracts/BatchAggregator.sol b/system-contracts/contracts/BatchAggregator.sol new file mode 100644 index 000000000..22b814732 --- /dev/null +++ b/system-contracts/contracts/BatchAggregator.sol @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: MIT + +pragma solidity 0.8.20; + +import {IBatchAggregator} from "./interfaces/IBatchAggregator.sol"; +import {ISystemContract} from "./interfaces/ISystemContract.sol"; +import {CommitBatchInfo} from "../../l1-contracts/contracts/state-transition/chain-interfaces/IExecutor.sol"; + +contract BatchAggregator is IBatchAggregator, ISystemContract { + CommitBatchInfo[] batchStorage; + function commitBatch(CommitBatchInfo batch) external{ + batchStorage.push(batch); + } + function returnBatchesAndClearState() external returns(bytes32 batchInfo) { + batchInfo = abi.encode(batchStorage); + batchStorage = []; + } +} \ No newline at end of file diff --git a/system-contracts/contracts/Constants.sol b/system-contracts/contracts/Constants.sol index 0f8e2307f..d0a8960f0 100644 --- a/system-contracts/contracts/Constants.sol +++ b/system-contracts/contracts/Constants.sol @@ -14,6 +14,7 @@ import {ICompressor} from "./interfaces/ICompressor.sol"; import {IComplexUpgrader} from "./interfaces/IComplexUpgrader.sol"; import {IBootloaderUtilities} from "./interfaces/IBootloaderUtilities.sol"; import {IPubdataChunkPublisher} from "./interfaces/IPubdataChunkPublisher.sol"; +import {IBatchAggregator} from "./interfaces/IBatchAggregator.sol"; /// @dev All the system contracts introduced by zkSync have their addresses /// started from 2^15 in order to avoid collision with Ethereum precompiles. @@ -88,6 +89,8 @@ IPubdataChunkPublisher constant PUBDATA_CHUNK_PUBLISHER = IPubdataChunkPublisher address(SYSTEM_CONTRACTS_OFFSET + 0x11) ); +IBatchAggregator constant BATCH_AGGREGATOR = IBatchAggregator(address(SYSTEM_CONTRACTS_OFFSET + 0x12)) + /// @dev If the bitwise AND of the extraAbi[2] param when calling the MSG_VALUE_SIMULATOR /// is non-zero, the call will be assumed to be a system one. uint256 constant MSG_VALUE_SIMULATOR_IS_SYSTEM_BIT = 1; @@ -119,7 +122,8 @@ enum SystemLogKey { BLOB_FOUR_HASH_KEY, BLOB_FIVE_HASH_KEY, BLOB_SIX_HASH_KEY, - EXPECTED_SYSTEM_CONTRACT_UPGRADE_TX_HASH_KEY + EXPECTED_SYSTEM_CONTRACT_UPGRADE_TX_HASH_KEY, + AGGREGATED_HYPERCHAIN_PUBDATA } /// @dev The number of leaves in the L2->L1 log Merkle tree. diff --git a/system-contracts/contracts/L1Messenger.sol b/system-contracts/contracts/L1Messenger.sol index 2b584d110..b1c5dce57 100644 --- a/system-contracts/contracts/L1Messenger.sol +++ b/system-contracts/contracts/L1Messenger.sol @@ -7,7 +7,7 @@ import {ISystemContract} from "./interfaces/ISystemContract.sol"; import {SystemContractHelper} from "./libraries/SystemContractHelper.sol"; import {EfficientCall} from "./libraries/EfficientCall.sol"; import {Utils} from "./libraries/Utils.sol"; -import {SystemLogKey, SYSTEM_CONTEXT_CONTRACT, KNOWN_CODE_STORAGE_CONTRACT, COMPRESSOR_CONTRACT, STATE_DIFF_ENTRY_SIZE, L2_TO_L1_LOGS_MERKLE_TREE_LEAVES, PUBDATA_CHUNK_PUBLISHER, COMPUTATIONAL_PRICE_FOR_PUBDATA} from "./Constants.sol"; +import {SystemLogKey, SYSTEM_CONTEXT_CONTRACT, KNOWN_CODE_STORAGE_CONTRACT, COMPRESSOR_CONTRACT, BATCH_AGGREGATOR, STATE_DIFF_ENTRY_SIZE, L2_TO_L1_LOGS_MERKLE_TREE_LEAVES, PUBDATA_CHUNK_PUBLISHER, COMPUTATIONAL_PRICE_FOR_PUBDATA} from "./Constants.sol"; /** * @author Matter Labs @@ -194,6 +194,14 @@ contract L1Messenger is IL1Messenger, ISystemContract { function publishPubdataAndClearState( bytes calldata _totalL2ToL1PubdataAndStateDiffs ) external onlyCallFromBootloader { + // Send hyperchain batches + SystemContractHelper.toL1( + true, + bytes32(uint256(SystemLogKey.AGGREGATED_HYPERCHAIN_PUBDATA_KEY)), + EfficientCall.keccak(BATCH_AGGREGATOR.returnBatchesAndClearState()) + ); + + uint256 calldataPtr = 0; /// Check logs @@ -308,6 +316,7 @@ contract L1Messenger is IL1Messenger, ISystemContract { stateDiffs, compressedStateDiffs ); + // /// Check for calldata strict format require(calldataPtr == _totalL2ToL1PubdataAndStateDiffs.length, "Extra data in the totalL2ToL1Pubdata array"); diff --git a/system-contracts/contracts/interfaces/IBatchAggregator.sol b/system-contracts/contracts/interfaces/IBatchAggregator.sol new file mode 100644 index 000000000..7d825152f --- /dev/null +++ b/system-contracts/contracts/interfaces/IBatchAggregator.sol @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: MIT + +pragma solidity 0.8.20; + +import {CommitBatchInfo} from "../../../l1-contracts/contracts/state-transition/chain-interfaces/IExecutor.sol"; + +interface IBatchAggregator{ + bytes32 batchStorage; + function commitBatch(CommitBatchInfo batch) external; + function returnBatchesAndClearState() external; +} \ No newline at end of file From 120387d5f8a28b057b4f624d2da24fa87e5a9dae Mon Sep 17 00:00:00 2001 From: vladenysiuk Date: Tue, 14 May 2024 19:41:34 +0200 Subject: [PATCH 02/18] fix build --- .../contracts/common/interfaces/IBatchAggregator.sol | 8 ++++++++ .../state-transition/chain-deps/facets/Executor.sol | 4 ++-- system-contracts/contracts/BatchAggregator.sol | 9 ++++----- system-contracts/contracts/Constants.sol | 4 ++-- system-contracts/contracts/L1Messenger.sol | 4 ++-- .../contracts/interfaces/IBatchAggregator.sol | 7 ++----- 6 files changed, 20 insertions(+), 16 deletions(-) create mode 100644 l1-contracts/contracts/common/interfaces/IBatchAggregator.sol diff --git a/l1-contracts/contracts/common/interfaces/IBatchAggregator.sol b/l1-contracts/contracts/common/interfaces/IBatchAggregator.sol new file mode 100644 index 000000000..5ff0857db --- /dev/null +++ b/l1-contracts/contracts/common/interfaces/IBatchAggregator.sol @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: MIT + +pragma solidity 0.8.24; + +interface IBatchAggregator{ + function commitBatch(bytes memory batch) external; + function returnBatchesAndClearState() external returns (bytes memory); +} \ No newline at end of file diff --git a/l1-contracts/contracts/state-transition/chain-deps/facets/Executor.sol b/l1-contracts/contracts/state-transition/chain-deps/facets/Executor.sol index 0a8ce8ce3..eb5da7529 100644 --- a/l1-contracts/contracts/state-transition/chain-deps/facets/Executor.sol +++ b/l1-contracts/contracts/state-transition/chain-deps/facets/Executor.sol @@ -11,10 +11,10 @@ import {UnsafeBytes} from "../../../common/libraries/UnsafeBytes.sol"; import {L2_BOOTLOADER_ADDRESS, L2_TO_L1_MESSENGER_SYSTEM_CONTRACT_ADDR, L2_SYSTEM_CONTEXT_SYSTEM_CONTRACT_ADDR, L2_PUBDATA_CHUNK_PUBLISHER_ADDR} from "../../../common/L2ContractAddresses.sol"; import {PubdataPricingMode} from "../ZkSyncHyperchainStorage.sol"; import {IStateTransitionManager} from "../../IStateTransitionManager.sol"; -import {IBatchAggregator} from "../../../../../system-contracts/contracts/interfaces/IBatchAggregator.sol"; +import {IBatchAggregator} from "../../../common/interfaces/IBatchAggregator.sol"; // TODO: replace with actual system address -constant address BATCH_AGGREGATOR_ADDRESS = 0x0000000000000000000000000000000000009012; +address constant BATCH_AGGREGATOR_ADDRESS = 0x0000000000000000000000000000000000009012; // While formally the following import is not used, it is needed to inherit documentation from it import {IZkSyncHyperchainBase} from "../../chain-interfaces/IZkSyncHyperchainBase.sol"; diff --git a/system-contracts/contracts/BatchAggregator.sol b/system-contracts/contracts/BatchAggregator.sol index 22b814732..715c32682 100644 --- a/system-contracts/contracts/BatchAggregator.sol +++ b/system-contracts/contracts/BatchAggregator.sol @@ -4,15 +4,14 @@ pragma solidity 0.8.20; import {IBatchAggregator} from "./interfaces/IBatchAggregator.sol"; import {ISystemContract} from "./interfaces/ISystemContract.sol"; -import {CommitBatchInfo} from "../../l1-contracts/contracts/state-transition/chain-interfaces/IExecutor.sol"; contract BatchAggregator is IBatchAggregator, ISystemContract { - CommitBatchInfo[] batchStorage; - function commitBatch(CommitBatchInfo batch) external{ + bytes[] batchStorage; + function commitBatch(bytes memory batch) external{ batchStorage.push(batch); } - function returnBatchesAndClearState() external returns(bytes32 batchInfo) { + function returnBatchesAndClearState() external returns(bytes memory batchInfo) { batchInfo = abi.encode(batchStorage); - batchStorage = []; + delete batchStorage; } } \ No newline at end of file diff --git a/system-contracts/contracts/Constants.sol b/system-contracts/contracts/Constants.sol index d0a8960f0..7cfed1660 100644 --- a/system-contracts/contracts/Constants.sol +++ b/system-contracts/contracts/Constants.sol @@ -89,7 +89,7 @@ IPubdataChunkPublisher constant PUBDATA_CHUNK_PUBLISHER = IPubdataChunkPublisher address(SYSTEM_CONTRACTS_OFFSET + 0x11) ); -IBatchAggregator constant BATCH_AGGREGATOR = IBatchAggregator(address(SYSTEM_CONTRACTS_OFFSET + 0x12)) +IBatchAggregator constant BATCH_AGGREGATOR = IBatchAggregator(address(SYSTEM_CONTRACTS_OFFSET + 0x12)); /// @dev If the bitwise AND of the extraAbi[2] param when calling the MSG_VALUE_SIMULATOR /// is non-zero, the call will be assumed to be a system one. @@ -123,7 +123,7 @@ enum SystemLogKey { BLOB_FIVE_HASH_KEY, BLOB_SIX_HASH_KEY, EXPECTED_SYSTEM_CONTRACT_UPGRADE_TX_HASH_KEY, - AGGREGATED_HYPERCHAIN_PUBDATA + HYPERCHAIN_PUBDATA_KEY } /// @dev The number of leaves in the L2->L1 log Merkle tree. diff --git a/system-contracts/contracts/L1Messenger.sol b/system-contracts/contracts/L1Messenger.sol index b1c5dce57..9c0dcc0bb 100644 --- a/system-contracts/contracts/L1Messenger.sol +++ b/system-contracts/contracts/L1Messenger.sol @@ -197,8 +197,8 @@ contract L1Messenger is IL1Messenger, ISystemContract { // Send hyperchain batches SystemContractHelper.toL1( true, - bytes32(uint256(SystemLogKey.AGGREGATED_HYPERCHAIN_PUBDATA_KEY)), - EfficientCall.keccak(BATCH_AGGREGATOR.returnBatchesAndClearState()) + bytes32(uint256(SystemLogKey.HYPERCHAIN_PUBDATA_KEY)), + keccak256(BATCH_AGGREGATOR.returnBatchesAndClearState()) ); diff --git a/system-contracts/contracts/interfaces/IBatchAggregator.sol b/system-contracts/contracts/interfaces/IBatchAggregator.sol index 7d825152f..f1cb15b6e 100644 --- a/system-contracts/contracts/interfaces/IBatchAggregator.sol +++ b/system-contracts/contracts/interfaces/IBatchAggregator.sol @@ -2,10 +2,7 @@ pragma solidity 0.8.20; -import {CommitBatchInfo} from "../../../l1-contracts/contracts/state-transition/chain-interfaces/IExecutor.sol"; - interface IBatchAggregator{ - bytes32 batchStorage; - function commitBatch(CommitBatchInfo batch) external; - function returnBatchesAndClearState() external; + function commitBatch(bytes memory batch) external; + function returnBatchesAndClearState() external returns(bytes memory); } \ No newline at end of file From 1255a766f53decc2addcb62ce517af47c06e09bf Mon Sep 17 00:00:00 2001 From: vladenysiuk Date: Wed, 15 May 2024 15:52:00 +0200 Subject: [PATCH 03/18] bug fixed + pubdata parsing --- .../common/interfaces/IBatchAggregator.sol | 2 +- .../chain-deps/facets/Executor.sol | 5 +- .../contracts/BatchAggregator.sol | 101 +++++++++++++++++- .../contracts/interfaces/IBatchAggregator.sol | 4 +- 4 files changed, 103 insertions(+), 9 deletions(-) diff --git a/l1-contracts/contracts/common/interfaces/IBatchAggregator.sol b/l1-contracts/contracts/common/interfaces/IBatchAggregator.sol index 5ff0857db..1cff35a8e 100644 --- a/l1-contracts/contracts/common/interfaces/IBatchAggregator.sol +++ b/l1-contracts/contracts/common/interfaces/IBatchAggregator.sol @@ -3,6 +3,6 @@ pragma solidity 0.8.24; interface IBatchAggregator{ - function commitBatch(bytes memory batch) external; + function commitBatch(bytes calldata _totalL2ToL1PubdataAndStateDiffs, uint256 chainId, uint256 batchNumber) external; function returnBatchesAndClearState() external returns (bytes memory); } \ No newline at end of file diff --git a/l1-contracts/contracts/state-transition/chain-deps/facets/Executor.sol b/l1-contracts/contracts/state-transition/chain-deps/facets/Executor.sol index eb5da7529..c957bc9a0 100644 --- a/l1-contracts/contracts/state-transition/chain-deps/facets/Executor.sol +++ b/l1-contracts/contracts/state-transition/chain-deps/facets/Executor.sol @@ -36,7 +36,7 @@ contract ExecutorFacet is ZkSyncHyperchainBase, IExecutor { StoredBatchInfo memory _previousBatch, CommitBatchInfo calldata _newBatch, bytes32 _expectedSystemContractUpgradeTxHash - ) internal view returns (StoredBatchInfo memory) { + ) internal returns (StoredBatchInfo memory) { require(_newBatch.batchNumber == _previousBatch.batchNumber + 1, "f"); // only commit next batch uint8 pubdataSource = uint8(bytes1(_newBatch.pubdataCommitments[0])); @@ -75,7 +75,8 @@ contract ExecutorFacet is ZkSyncHyperchainBase, IExecutor { ); } else if (pubdataSource == uint8(PubdataSource.ForwardedMessage)) { // Similar to Calldata, forwards packed pubdataCommitments of hyperchains to the lower layer - + // Add batch to BatchAggregator + IBatchAggregator(BATCH_AGGREGATOR_ADDRESS).commitBatch(_newBatch.pubdataCommitments,0,_newBatch.batchNumber); } require(_previousBatch.batchHash == logOutput.previousBatchHash, "l"); diff --git a/system-contracts/contracts/BatchAggregator.sol b/system-contracts/contracts/BatchAggregator.sol index 715c32682..76cd3a28f 100644 --- a/system-contracts/contracts/BatchAggregator.sol +++ b/system-contracts/contracts/BatchAggregator.sol @@ -4,14 +4,107 @@ pragma solidity 0.8.20; import {IBatchAggregator} from "./interfaces/IBatchAggregator.sol"; import {ISystemContract} from "./interfaces/ISystemContract.sol"; +import {L2_L1_LOGS_TREE_DEFAULT_LEAF_HASH, L2_TO_L1_LOG_SERIALIZE_SIZE, STATE_DIFF_COMPRESSION_VERSION_NUMBER} from "./interfaces/IL1Messenger.sol"; +import {SystemLogKey, SYSTEM_CONTEXT_CONTRACT, KNOWN_CODE_STORAGE_CONTRACT, COMPRESSOR_CONTRACT, BATCH_AGGREGATOR, STATE_DIFF_ENTRY_SIZE, L2_TO_L1_LOGS_MERKLE_TREE_LEAVES, PUBDATA_CHUNK_PUBLISHER, COMPUTATIONAL_PRICE_FOR_PUBDATA} from "./Constants.sol"; contract BatchAggregator is IBatchAggregator, ISystemContract { bytes[] batchStorage; - function commitBatch(bytes memory batch) external{ - batchStorage.push(batch); + bytes[] chainData; + mapping(uint256 => bytes[]) messageStorage; + mapping(uint256 => bytes[]) logStorage; + mapping(uint256 => bytes[]) stateDiffStorage; + mapping(uint256 => bytes[]) bytecodeStorage; + mapping(uint256 => bool) chainSet; + uint256[] chainList; + function addChain(uint256 chainId) internal { + if (chainSet[chainId]==false){ + chainList.push(chainId); + chainSet[chainId] = true; + } + } + function commitBatch(bytes calldata _totalL2ToL1PubdataAndStateDiffs, uint256 chainId, uint256 batchNumber) external{ + addChain(chainId); + + uint256 calldataPtr = 0; + + /// Check logs + uint32 numberOfL2ToL1Logs = uint32(bytes4(_totalL2ToL1PubdataAndStateDiffs[calldataPtr:calldataPtr + 4])); + require(numberOfL2ToL1Logs <= L2_TO_L1_LOGS_MERKLE_TREE_LEAVES, "Too many L2->L1 logs"); + logStorage[chainId].push(_totalL2ToL1PubdataAndStateDiffs[calldataPtr:calldataPtr + 4 + numberOfL2ToL1Logs*L2_TO_L1_LOG_SERIALIZE_SIZE]); + calldataPtr += 4 + L2_TO_L1_LOG_SERIALIZE_SIZE*numberOfL2ToL1Logs; + + /// Check messages + uint32 numberOfMessages = uint32(bytes4(_totalL2ToL1PubdataAndStateDiffs[calldataPtr:calldataPtr + 4])); + messageStorage[chainId].push(_totalL2ToL1PubdataAndStateDiffs[calldataPtr:calldataPtr + 4+numberOfMessages*4]); + calldataPtr += 4+numberOfMessages*4; + + /// Check bytecodes + uint32 numberOfBytecodes = uint32(bytes4(_totalL2ToL1PubdataAndStateDiffs[calldataPtr:calldataPtr + 4])); + uint256 bytecodeSliceStart = calldataPtr; + calldataPtr += 4; + bytes32 reconstructedChainedL1BytecodesRevealDataHash; + for (uint256 i = 0; i < numberOfBytecodes; ++i) { + uint32 currentBytecodeLength = uint32( + bytes4(_totalL2ToL1PubdataAndStateDiffs[calldataPtr:calldataPtr + 4]) + ); + calldataPtr += 4 + currentBytecodeLength; + } + + bytecodeStorage[chainId].push(_totalL2ToL1PubdataAndStateDiffs[bytecodeSliceStart:calldataPtr]); + /// Check State Diffs + /// encoding is as follows: + /// header (1 byte version, 3 bytes total len of compressed, 1 byte enumeration index size) + /// body (`compressedStateDiffSize` bytes, 4 bytes number of state diffs, `numberOfStateDiffs` * `STATE_DIFF_ENTRY_SIZE` bytes for the uncompressed state diffs) + /// encoded state diffs: [20bytes address][32bytes key][32bytes derived key][8bytes enum index][32bytes initial value][32bytes final value] + require( + uint256(uint8(bytes1(_totalL2ToL1PubdataAndStateDiffs[calldataPtr]))) == + STATE_DIFF_COMPRESSION_VERSION_NUMBER, + "state diff compression version mismatch" + ); + uint256 stateDiffSliceStart = calldataPtr; + calldataPtr++; + + uint24 compressedStateDiffSize = uint24(bytes3(_totalL2ToL1PubdataAndStateDiffs[calldataPtr:calldataPtr + 3])); + calldataPtr += 3; + + uint8 enumerationIndexSize = uint8(bytes1(_totalL2ToL1PubdataAndStateDiffs[calldataPtr])); + calldataPtr++; + + bytes calldata compressedStateDiffs = _totalL2ToL1PubdataAndStateDiffs[calldataPtr:calldataPtr + + compressedStateDiffSize]; + calldataPtr += compressedStateDiffSize; + + bytes calldata totalL2ToL1Pubdata = _totalL2ToL1PubdataAndStateDiffs[:calldataPtr]; + + uint32 numberOfStateDiffs = uint32(bytes4(_totalL2ToL1PubdataAndStateDiffs[calldataPtr:calldataPtr + 4])); + calldataPtr += 4; + + bytes calldata stateDiffs = _totalL2ToL1PubdataAndStateDiffs[calldataPtr:calldataPtr + + (numberOfStateDiffs * STATE_DIFF_ENTRY_SIZE)]; + calldataPtr += numberOfStateDiffs * STATE_DIFF_ENTRY_SIZE; + + stateDiffStorage[chainId].push(_totalL2ToL1PubdataAndStateDiffs[stateDiffSliceStart:calldataPtr]); + } function returnBatchesAndClearState() external returns(bytes memory batchInfo) { - batchInfo = abi.encode(batchStorage); - delete batchStorage; + for (uint256 i = 0;i Date: Wed, 15 May 2024 15:52:29 +0200 Subject: [PATCH 04/18] fix lint --- .../common/interfaces/IBatchAggregator.sol | 10 ++-- .../chain-deps/facets/Executor.sol | 6 ++- .../contracts/BatchAggregator.sol | 47 +++++++++++++------ system-contracts/contracts/L1Messenger.sol | 3 +- .../contracts/interfaces/IBatchAggregator.sol | 10 ++-- 5 files changed, 52 insertions(+), 24 deletions(-) diff --git a/l1-contracts/contracts/common/interfaces/IBatchAggregator.sol b/l1-contracts/contracts/common/interfaces/IBatchAggregator.sol index 1cff35a8e..06eef0ae2 100644 --- a/l1-contracts/contracts/common/interfaces/IBatchAggregator.sol +++ b/l1-contracts/contracts/common/interfaces/IBatchAggregator.sol @@ -2,7 +2,11 @@ pragma solidity 0.8.24; -interface IBatchAggregator{ - function commitBatch(bytes calldata _totalL2ToL1PubdataAndStateDiffs, uint256 chainId, uint256 batchNumber) external; +interface IBatchAggregator { + function commitBatch( + bytes calldata _totalL2ToL1PubdataAndStateDiffs, + uint256 chainId, + uint256 batchNumber + ) external; function returnBatchesAndClearState() external returns (bytes memory); -} \ No newline at end of file +} diff --git a/l1-contracts/contracts/state-transition/chain-deps/facets/Executor.sol b/l1-contracts/contracts/state-transition/chain-deps/facets/Executor.sol index c957bc9a0..6639a7af8 100644 --- a/l1-contracts/contracts/state-transition/chain-deps/facets/Executor.sol +++ b/l1-contracts/contracts/state-transition/chain-deps/facets/Executor.sol @@ -76,7 +76,11 @@ contract ExecutorFacet is ZkSyncHyperchainBase, IExecutor { } else if (pubdataSource == uint8(PubdataSource.ForwardedMessage)) { // Similar to Calldata, forwards packed pubdataCommitments of hyperchains to the lower layer // Add batch to BatchAggregator - IBatchAggregator(BATCH_AGGREGATOR_ADDRESS).commitBatch(_newBatch.pubdataCommitments,0,_newBatch.batchNumber); + IBatchAggregator(BATCH_AGGREGATOR_ADDRESS).commitBatch( + _newBatch.pubdataCommitments, + 0, + _newBatch.batchNumber + ); } require(_previousBatch.batchHash == logOutput.previousBatchHash, "l"); diff --git a/system-contracts/contracts/BatchAggregator.sol b/system-contracts/contracts/BatchAggregator.sol index 76cd3a28f..eb3c03934 100644 --- a/system-contracts/contracts/BatchAggregator.sol +++ b/system-contracts/contracts/BatchAggregator.sol @@ -17,12 +17,16 @@ contract BatchAggregator is IBatchAggregator, ISystemContract { mapping(uint256 => bool) chainSet; uint256[] chainList; function addChain(uint256 chainId) internal { - if (chainSet[chainId]==false){ + if (chainSet[chainId] == false) { chainList.push(chainId); chainSet[chainId] = true; } } - function commitBatch(bytes calldata _totalL2ToL1PubdataAndStateDiffs, uint256 chainId, uint256 batchNumber) external{ + function commitBatch( + bytes calldata _totalL2ToL1PubdataAndStateDiffs, + uint256 chainId, + uint256 batchNumber + ) external { addChain(chainId); uint256 calldataPtr = 0; @@ -30,14 +34,21 @@ contract BatchAggregator is IBatchAggregator, ISystemContract { /// Check logs uint32 numberOfL2ToL1Logs = uint32(bytes4(_totalL2ToL1PubdataAndStateDiffs[calldataPtr:calldataPtr + 4])); require(numberOfL2ToL1Logs <= L2_TO_L1_LOGS_MERKLE_TREE_LEAVES, "Too many L2->L1 logs"); - logStorage[chainId].push(_totalL2ToL1PubdataAndStateDiffs[calldataPtr:calldataPtr + 4 + numberOfL2ToL1Logs*L2_TO_L1_LOG_SERIALIZE_SIZE]); - calldataPtr += 4 + L2_TO_L1_LOG_SERIALIZE_SIZE*numberOfL2ToL1Logs; - + logStorage[chainId].push( + _totalL2ToL1PubdataAndStateDiffs[calldataPtr:calldataPtr + + 4 + + numberOfL2ToL1Logs * + L2_TO_L1_LOG_SERIALIZE_SIZE] + ); + calldataPtr += 4 + L2_TO_L1_LOG_SERIALIZE_SIZE * numberOfL2ToL1Logs; + /// Check messages uint32 numberOfMessages = uint32(bytes4(_totalL2ToL1PubdataAndStateDiffs[calldataPtr:calldataPtr + 4])); - messageStorage[chainId].push(_totalL2ToL1PubdataAndStateDiffs[calldataPtr:calldataPtr + 4+numberOfMessages*4]); - calldataPtr += 4+numberOfMessages*4; - + messageStorage[chainId].push( + _totalL2ToL1PubdataAndStateDiffs[calldataPtr:calldataPtr + 4 + numberOfMessages * 4] + ); + calldataPtr += 4 + numberOfMessages * 4; + /// Check bytecodes uint32 numberOfBytecodes = uint32(bytes4(_totalL2ToL1PubdataAndStateDiffs[calldataPtr:calldataPtr + 4])); uint256 bytecodeSliceStart = calldataPtr; @@ -84,14 +95,20 @@ contract BatchAggregator is IBatchAggregator, ISystemContract { calldataPtr += numberOfStateDiffs * STATE_DIFF_ENTRY_SIZE; stateDiffStorage[chainId].push(_totalL2ToL1PubdataAndStateDiffs[stateDiffSliceStart:calldataPtr]); - } - function returnBatchesAndClearState() external returns(bytes memory batchInfo) { - for (uint256 i = 0;i Date: Wed, 15 May 2024 21:49:16 +0200 Subject: [PATCH 05/18] implement stateDiff compression --- .../contracts/BatchAggregator.sol | 97 ++++++++++++++++--- .../contracts/interfaces/IBatchAggregator.sol | 2 + 2 files changed, 88 insertions(+), 11 deletions(-) diff --git a/system-contracts/contracts/BatchAggregator.sol b/system-contracts/contracts/BatchAggregator.sol index eb3c03934..d467c2a18 100644 --- a/system-contracts/contracts/BatchAggregator.sol +++ b/system-contracts/contracts/BatchAggregator.sol @@ -2,17 +2,23 @@ pragma solidity 0.8.20; -import {IBatchAggregator} from "./interfaces/IBatchAggregator.sol"; +import {UnsafeBytesCalldata} from "./libraries/UnsafeBytesCalldata.sol"; +import {IBatchAggregator, COMPRESSED_STATE_DIFF_SIZE} from "./interfaces/IBatchAggregator.sol"; +import {OPERATION_BITMASK, LENGTH_BITS_OFFSET, MAX_ENUMERATION_INDEX_SIZE} from "./interfaces/ICompressor.sol"; import {ISystemContract} from "./interfaces/ISystemContract.sol"; import {L2_L1_LOGS_TREE_DEFAULT_LEAF_HASH, L2_TO_L1_LOG_SERIALIZE_SIZE, STATE_DIFF_COMPRESSION_VERSION_NUMBER} from "./interfaces/IL1Messenger.sol"; import {SystemLogKey, SYSTEM_CONTEXT_CONTRACT, KNOWN_CODE_STORAGE_CONTRACT, COMPRESSOR_CONTRACT, BATCH_AGGREGATOR, STATE_DIFF_ENTRY_SIZE, L2_TO_L1_LOGS_MERKLE_TREE_LEAVES, PUBDATA_CHUNK_PUBLISHER, COMPUTATIONAL_PRICE_FOR_PUBDATA} from "./Constants.sol"; contract BatchAggregator is IBatchAggregator, ISystemContract { + using UnsafeBytesCalldata for bytes; + bytes[] batchStorage; bytes[] chainData; mapping(uint256 => bytes[]) messageStorage; mapping(uint256 => bytes[]) logStorage; - mapping(uint256 => bytes[]) stateDiffStorage; + mapping(uint256 => mapping(bytes32 => uint256)) stateDiffStorage; + mapping(uint256 => mapping(bytes32 => bool)) keyStatus; + mapping(uint256 => bytes32[]) touchedKeys; mapping(uint256 => bytes[]) bytecodeStorage; mapping(uint256 => bool) chainSet; uint256[] chainList; @@ -72,7 +78,6 @@ contract BatchAggregator is IBatchAggregator, ISystemContract { STATE_DIFF_COMPRESSION_VERSION_NUMBER, "state diff compression version mismatch" ); - uint256 stateDiffSliceStart = calldataPtr; calldataPtr++; uint24 compressedStateDiffSize = uint24(bytes3(_totalL2ToL1PubdataAndStateDiffs[calldataPtr:calldataPtr + 3])); @@ -94,19 +99,93 @@ contract BatchAggregator is IBatchAggregator, ISystemContract { (numberOfStateDiffs * STATE_DIFF_ENTRY_SIZE)]; calldataPtr += numberOfStateDiffs * STATE_DIFF_ENTRY_SIZE; - stateDiffStorage[chainId].push(_totalL2ToL1PubdataAndStateDiffs[stateDiffSliceStart:calldataPtr]); + require(enumerationIndexSize <= MAX_ENUMERATION_INDEX_SIZE, "enumeration index size is too large"); + uint256 numberOfInitialWrites = uint256(compressedStateDiffs.readUint16(0)); + bytes32[] memory derivedKeyTable = new bytes32[](numberOfInitialWrites); + + uint256 stateDiffPtr = 2; + uint256 numInitialWritesProcessed = 0; + // Process initial writes + for (uint256 i = 0; i < numberOfStateDiffs * STATE_DIFF_ENTRY_SIZE; i += STATE_DIFF_ENTRY_SIZE) { + bytes calldata stateDiff = stateDiffs[i:i + STATE_DIFF_ENTRY_SIZE]; + uint64 enumIndex = stateDiff.readUint64(84); + if (enumIndex != 0) { + // It is a repeated write, so we skip it. + continue; + } + + bytes32 derivedKey = stateDiff.readBytes32(52); + derivedKeyTable[numInitialWritesProcessed] = derivedKey; + uint256 initValue = stateDiff.readUint256(92); + uint256 finalValue = stateDiff.readUint256(124); + + numInitialWritesProcessed++; + require(derivedKey == compressedStateDiffs.readBytes32(stateDiffPtr), "iw: initial key mismatch"); + if (keyStatus[chainId][derivedKey] == false) { + keyStatus[chainId][derivedKey] = true; + touchedKeys[chainId].push(derivedKey); + } + stateDiffStorage[chainId][derivedKey] = finalValue; + + stateDiffPtr += 32; + + uint8 metadata = uint8(bytes1(compressedStateDiffs[stateDiffPtr])); + stateDiffPtr++; + uint8 operation = metadata & OPERATION_BITMASK; + uint8 len = operation == 0 ? 32 : metadata >> LENGTH_BITS_OFFSET; + + stateDiffPtr += len; + } + + require(numInitialWritesProcessed == numberOfInitialWrites, "Incorrect number of initial storage diffs"); + + // Process repeated writes + for (uint256 i = 0; i < numberOfStateDiffs * STATE_DIFF_ENTRY_SIZE; i += STATE_DIFF_ENTRY_SIZE) { + bytes calldata stateDiff = stateDiffs[i:i + STATE_DIFF_ENTRY_SIZE]; + uint64 enumIndex = stateDiff.readUint64(84); + if (enumIndex == 0) { + continue; + } + bytes32 derivedKey = derivedKeyTable[enumIndex]; + uint256 initValue = stateDiff.readUint256(92); + uint256 finalValue = stateDiff.readUint256(124); + + stateDiffStorage[chainId][derivedKey] = finalValue; + stateDiffPtr += enumerationIndexSize; + + uint8 metadata = uint8(bytes1(compressedStateDiffs[stateDiffPtr])); + stateDiffPtr += 1; + uint8 operation = metadata & OPERATION_BITMASK; + uint8 len = operation == 0 ? 32 : metadata >> LENGTH_BITS_OFFSET; + + stateDiffPtr += len; + } + + require(stateDiffPtr == compressedStateDiffs.length, "Extra data in _compressedStateDiffs"); } function returnBatchesAndClearState() external returns (bytes memory batchInfo) { for (uint256 i = 0; i < chainList.length; i += 1) { uint256 chainId = chainList[i]; - + bytes memory compressedStateDiff = new bytes(touchedKeys[chainId].length * COMPRESSED_STATE_DIFF_SIZE); + uint256 stateDiffPtr = 0; + for (uint256 keyIndex = 0; keyIndex < touchedKeys[chainId].length; keyIndex += 1) { + bytes32 derivedKey = touchedKeys[chainId][keyIndex]; + uint256 finalValue = stateDiffStorage[chainId][derivedKey]; + assembly { + mstore(add(compressedStateDiff, stateDiffPtr), derivedKey) + mstore(add(compressedStateDiff, add(stateDiffPtr, 0x20)), finalValue) + } + delete stateDiffStorage[chainId][derivedKey]; + delete keyStatus[chainId][derivedKey]; + stateDiffPtr += COMPRESSED_STATE_DIFF_SIZE; + } chainData.push( abi.encode( chainId, logStorage[chainId], messageStorage[chainId], bytecodeStorage[chainId], - stateDiffStorage[chainId] + compressedStateDiff ) ); @@ -114,14 +193,10 @@ contract BatchAggregator is IBatchAggregator, ISystemContract { delete logStorage[chainId]; delete messageStorage[chainId]; delete bytecodeStorage[chainId]; - delete stateDiffStorage[chainId]; + delete touchedKeys[chainId]; } delete chainList; batchInfo = abi.encode(chainData); delete chainData; - /*delete messageStorage; - delete logStorage; - delete stateDiffStorage; - delete bytecodeStorage;*/ } } diff --git a/system-contracts/contracts/interfaces/IBatchAggregator.sol b/system-contracts/contracts/interfaces/IBatchAggregator.sol index ed90f5ba8..0e8571527 100644 --- a/system-contracts/contracts/interfaces/IBatchAggregator.sol +++ b/system-contracts/contracts/interfaces/IBatchAggregator.sol @@ -2,6 +2,8 @@ pragma solidity 0.8.20; +uint256 constant COMPRESSED_STATE_DIFF_SIZE = 64; + interface IBatchAggregator { function commitBatch( bytes calldata _totalL2ToL1PubdataAndStateDiffs, From 84ab8eb90e9689e4a3abc3678fd66b3f382aff8a Mon Sep 17 00:00:00 2001 From: vladenysiuk Date: Tue, 21 May 2024 14:19:44 +0200 Subject: [PATCH 06/18] Revert "implement stateDiff compression" This reverts commit 0d760f7b74a29dd043a9a36c05e755a5c0d9b1fd. --- .../contracts/BatchAggregator.sol | 97 +++---------------- .../contracts/interfaces/IBatchAggregator.sol | 2 - 2 files changed, 11 insertions(+), 88 deletions(-) diff --git a/system-contracts/contracts/BatchAggregator.sol b/system-contracts/contracts/BatchAggregator.sol index d467c2a18..eb3c03934 100644 --- a/system-contracts/contracts/BatchAggregator.sol +++ b/system-contracts/contracts/BatchAggregator.sol @@ -2,23 +2,17 @@ pragma solidity 0.8.20; -import {UnsafeBytesCalldata} from "./libraries/UnsafeBytesCalldata.sol"; -import {IBatchAggregator, COMPRESSED_STATE_DIFF_SIZE} from "./interfaces/IBatchAggregator.sol"; -import {OPERATION_BITMASK, LENGTH_BITS_OFFSET, MAX_ENUMERATION_INDEX_SIZE} from "./interfaces/ICompressor.sol"; +import {IBatchAggregator} from "./interfaces/IBatchAggregator.sol"; import {ISystemContract} from "./interfaces/ISystemContract.sol"; import {L2_L1_LOGS_TREE_DEFAULT_LEAF_HASH, L2_TO_L1_LOG_SERIALIZE_SIZE, STATE_DIFF_COMPRESSION_VERSION_NUMBER} from "./interfaces/IL1Messenger.sol"; import {SystemLogKey, SYSTEM_CONTEXT_CONTRACT, KNOWN_CODE_STORAGE_CONTRACT, COMPRESSOR_CONTRACT, BATCH_AGGREGATOR, STATE_DIFF_ENTRY_SIZE, L2_TO_L1_LOGS_MERKLE_TREE_LEAVES, PUBDATA_CHUNK_PUBLISHER, COMPUTATIONAL_PRICE_FOR_PUBDATA} from "./Constants.sol"; contract BatchAggregator is IBatchAggregator, ISystemContract { - using UnsafeBytesCalldata for bytes; - bytes[] batchStorage; bytes[] chainData; mapping(uint256 => bytes[]) messageStorage; mapping(uint256 => bytes[]) logStorage; - mapping(uint256 => mapping(bytes32 => uint256)) stateDiffStorage; - mapping(uint256 => mapping(bytes32 => bool)) keyStatus; - mapping(uint256 => bytes32[]) touchedKeys; + mapping(uint256 => bytes[]) stateDiffStorage; mapping(uint256 => bytes[]) bytecodeStorage; mapping(uint256 => bool) chainSet; uint256[] chainList; @@ -78,6 +72,7 @@ contract BatchAggregator is IBatchAggregator, ISystemContract { STATE_DIFF_COMPRESSION_VERSION_NUMBER, "state diff compression version mismatch" ); + uint256 stateDiffSliceStart = calldataPtr; calldataPtr++; uint24 compressedStateDiffSize = uint24(bytes3(_totalL2ToL1PubdataAndStateDiffs[calldataPtr:calldataPtr + 3])); @@ -99,93 +94,19 @@ contract BatchAggregator is IBatchAggregator, ISystemContract { (numberOfStateDiffs * STATE_DIFF_ENTRY_SIZE)]; calldataPtr += numberOfStateDiffs * STATE_DIFF_ENTRY_SIZE; - require(enumerationIndexSize <= MAX_ENUMERATION_INDEX_SIZE, "enumeration index size is too large"); - uint256 numberOfInitialWrites = uint256(compressedStateDiffs.readUint16(0)); - bytes32[] memory derivedKeyTable = new bytes32[](numberOfInitialWrites); - - uint256 stateDiffPtr = 2; - uint256 numInitialWritesProcessed = 0; - // Process initial writes - for (uint256 i = 0; i < numberOfStateDiffs * STATE_DIFF_ENTRY_SIZE; i += STATE_DIFF_ENTRY_SIZE) { - bytes calldata stateDiff = stateDiffs[i:i + STATE_DIFF_ENTRY_SIZE]; - uint64 enumIndex = stateDiff.readUint64(84); - if (enumIndex != 0) { - // It is a repeated write, so we skip it. - continue; - } - - bytes32 derivedKey = stateDiff.readBytes32(52); - derivedKeyTable[numInitialWritesProcessed] = derivedKey; - uint256 initValue = stateDiff.readUint256(92); - uint256 finalValue = stateDiff.readUint256(124); - - numInitialWritesProcessed++; - require(derivedKey == compressedStateDiffs.readBytes32(stateDiffPtr), "iw: initial key mismatch"); - if (keyStatus[chainId][derivedKey] == false) { - keyStatus[chainId][derivedKey] = true; - touchedKeys[chainId].push(derivedKey); - } - stateDiffStorage[chainId][derivedKey] = finalValue; - - stateDiffPtr += 32; - - uint8 metadata = uint8(bytes1(compressedStateDiffs[stateDiffPtr])); - stateDiffPtr++; - uint8 operation = metadata & OPERATION_BITMASK; - uint8 len = operation == 0 ? 32 : metadata >> LENGTH_BITS_OFFSET; - - stateDiffPtr += len; - } - - require(numInitialWritesProcessed == numberOfInitialWrites, "Incorrect number of initial storage diffs"); - - // Process repeated writes - for (uint256 i = 0; i < numberOfStateDiffs * STATE_DIFF_ENTRY_SIZE; i += STATE_DIFF_ENTRY_SIZE) { - bytes calldata stateDiff = stateDiffs[i:i + STATE_DIFF_ENTRY_SIZE]; - uint64 enumIndex = stateDiff.readUint64(84); - if (enumIndex == 0) { - continue; - } - bytes32 derivedKey = derivedKeyTable[enumIndex]; - uint256 initValue = stateDiff.readUint256(92); - uint256 finalValue = stateDiff.readUint256(124); - - stateDiffStorage[chainId][derivedKey] = finalValue; - stateDiffPtr += enumerationIndexSize; - - uint8 metadata = uint8(bytes1(compressedStateDiffs[stateDiffPtr])); - stateDiffPtr += 1; - uint8 operation = metadata & OPERATION_BITMASK; - uint8 len = operation == 0 ? 32 : metadata >> LENGTH_BITS_OFFSET; - - stateDiffPtr += len; - } - - require(stateDiffPtr == compressedStateDiffs.length, "Extra data in _compressedStateDiffs"); + stateDiffStorage[chainId].push(_totalL2ToL1PubdataAndStateDiffs[stateDiffSliceStart:calldataPtr]); } function returnBatchesAndClearState() external returns (bytes memory batchInfo) { for (uint256 i = 0; i < chainList.length; i += 1) { uint256 chainId = chainList[i]; - bytes memory compressedStateDiff = new bytes(touchedKeys[chainId].length * COMPRESSED_STATE_DIFF_SIZE); - uint256 stateDiffPtr = 0; - for (uint256 keyIndex = 0; keyIndex < touchedKeys[chainId].length; keyIndex += 1) { - bytes32 derivedKey = touchedKeys[chainId][keyIndex]; - uint256 finalValue = stateDiffStorage[chainId][derivedKey]; - assembly { - mstore(add(compressedStateDiff, stateDiffPtr), derivedKey) - mstore(add(compressedStateDiff, add(stateDiffPtr, 0x20)), finalValue) - } - delete stateDiffStorage[chainId][derivedKey]; - delete keyStatus[chainId][derivedKey]; - stateDiffPtr += COMPRESSED_STATE_DIFF_SIZE; - } + chainData.push( abi.encode( chainId, logStorage[chainId], messageStorage[chainId], bytecodeStorage[chainId], - compressedStateDiff + stateDiffStorage[chainId] ) ); @@ -193,10 +114,14 @@ contract BatchAggregator is IBatchAggregator, ISystemContract { delete logStorage[chainId]; delete messageStorage[chainId]; delete bytecodeStorage[chainId]; - delete touchedKeys[chainId]; + delete stateDiffStorage[chainId]; } delete chainList; batchInfo = abi.encode(chainData); delete chainData; + /*delete messageStorage; + delete logStorage; + delete stateDiffStorage; + delete bytecodeStorage;*/ } } diff --git a/system-contracts/contracts/interfaces/IBatchAggregator.sol b/system-contracts/contracts/interfaces/IBatchAggregator.sol index 0e8571527..ed90f5ba8 100644 --- a/system-contracts/contracts/interfaces/IBatchAggregator.sol +++ b/system-contracts/contracts/interfaces/IBatchAggregator.sol @@ -2,8 +2,6 @@ pragma solidity 0.8.20; -uint256 constant COMPRESSED_STATE_DIFF_SIZE = 64; - interface IBatchAggregator { function commitBatch( bytes calldata _totalL2ToL1PubdataAndStateDiffs, From 38e82b44aaa642c09265940ce9ec4f135a638b4f Mon Sep 17 00:00:00 2001 From: vladenysiuk Date: Wed, 22 May 2024 13:37:19 +0200 Subject: [PATCH 07/18] compress uncompressed state diffs --- .../contracts/BatchAggregator.sol | 168 ++++++++++++++++-- 1 file changed, 158 insertions(+), 10 deletions(-) diff --git a/system-contracts/contracts/BatchAggregator.sol b/system-contracts/contracts/BatchAggregator.sol index eb3c03934..7cf72e9ba 100644 --- a/system-contracts/contracts/BatchAggregator.sol +++ b/system-contracts/contracts/BatchAggregator.sol @@ -6,14 +6,21 @@ import {IBatchAggregator} from "./interfaces/IBatchAggregator.sol"; import {ISystemContract} from "./interfaces/ISystemContract.sol"; import {L2_L1_LOGS_TREE_DEFAULT_LEAF_HASH, L2_TO_L1_LOG_SERIALIZE_SIZE, STATE_DIFF_COMPRESSION_VERSION_NUMBER} from "./interfaces/IL1Messenger.sol"; import {SystemLogKey, SYSTEM_CONTEXT_CONTRACT, KNOWN_CODE_STORAGE_CONTRACT, COMPRESSOR_CONTRACT, BATCH_AGGREGATOR, STATE_DIFF_ENTRY_SIZE, L2_TO_L1_LOGS_MERKLE_TREE_LEAVES, PUBDATA_CHUNK_PUBLISHER, COMPUTATIONAL_PRICE_FOR_PUBDATA} from "./Constants.sol"; - +import {UnsafeBytesCalldata} from "./libraries/UnsafeBytesCalldata.sol"; +import {ICompressor, OPERATION_BITMASK, LENGTH_BITS_OFFSET, MAX_ENUMERATION_INDEX_SIZE} from "./interfaces/ICompressor.sol"; contract BatchAggregator is IBatchAggregator, ISystemContract { + using UnsafeBytesCalldata for bytes; bytes[] batchStorage; bytes[] chainData; + // log data mapping(uint256 => bytes[]) messageStorage; mapping(uint256 => bytes[]) logStorage; - mapping(uint256 => bytes[]) stateDiffStorage; mapping(uint256 => bytes[]) bytecodeStorage; + // state diff data + mapping(uint256 => mapping(bytes32 => bytes)) uncompressedWrites; + mapping(uint256 => mapping(bytes32 => bool)) slotStatus; + mapping(uint256 => bytes32[]) accesedSlots; + // chain data mapping(uint256 => bool) chainSet; uint256[] chainList; function addChain(uint256 chainId) internal { @@ -22,6 +29,149 @@ contract BatchAggregator is IBatchAggregator, ISystemContract { chainSet[chainId] = true; } } + // TODO: import or delete + function _sliceToUint256(bytes calldata _calldataSlice) internal pure returns (uint256 number) { + number = uint256(bytes32(_calldataSlice)); + number >>= (256 - (_calldataSlice.length * 8)); + } + function _verifyValueCompression( + uint256 _initialValue, + uint256 _finalValue, + uint256 _operation, + bytes calldata _compressedValue + ) internal pure { + uint256 convertedValue = _sliceToUint256(_compressedValue); + + unchecked { + if (_operation == 0 || _operation == 3) { + require(convertedValue == _finalValue, "transform or no compression: compressed and final mismatch"); + } else if (_operation == 1) { + require( + _initialValue + convertedValue == _finalValue, + "add: initial plus converted not equal to final" + ); + } else if (_operation == 2) { + require( + _initialValue - convertedValue == _finalValue, + "sub: initial minus converted not equal to final" + ); + } else { + revert("unsupported operation"); + } + } + } + + + function addInitialWrite(uint256 chainId, bytes calldata stateDiff) internal{ + bytes32 derivedKey = stateDiff.readBytes32(52); + uncompressedWrites[chainId][derivedKey] = stateDiff; + slotStatus[chainId][derivedKey] = true; + accesedSlots[chainId].push(derivedKey); + } + function addRepeatedWrite(uint256 chainId, bytes calldata stateDiff) internal{ + bytes32 derivedKey = stateDiff.readBytes32(52); + if (slotStatus[chainId][derivedKey]==false){ + uncompressedWrites[chainId][derivedKey] = stateDiff; + slotStatus[chainId][derivedKey] = true; + accesedSlots[chainId].push(derivedKey); + } + else{ + bytes memory slotData = uncompressedWrites[chainId][derivedKey]; + uint64 enumIndex; + bytes32 finalValue; + assembly{ + let offset := slotData + enumIndex := mload(add(sub(offset,24),84)) + finalValue := mload(add(offset,124)) + } + assembly { + let start := add(slotData, 0x20) + mstore(add(start,84), enumIndex) + mstore(add(start,124),finalValue) + } + } + } + function repackStateDiffs(uint256 chainId, + uint256 _numberOfStateDiffs, + uint256 _enumerationIndexSize, + bytes calldata _stateDiffs, + bytes calldata _compressedStateDiffs + ) internal{ + // We do not enforce the operator to use the optimal, i.e. the minimally possible _enumerationIndexSize. + // We do enforce however, that the _enumerationIndexSize is not larger than 8 bytes long, which is the + // maximal ever possible size for enumeration index. + require(_enumerationIndexSize <= MAX_ENUMERATION_INDEX_SIZE, "enumeration index size is too large"); + + uint256 numberOfInitialWrites = uint256(_compressedStateDiffs.readUint16(0)); + + uint256 stateDiffPtr = 2; + uint256 numInitialWritesProcessed = 0; + + // Process initial writes + for (uint256 i = 0; i < _numberOfStateDiffs * STATE_DIFF_ENTRY_SIZE; i += STATE_DIFF_ENTRY_SIZE) { + bytes calldata stateDiff = _stateDiffs[i:i + STATE_DIFF_ENTRY_SIZE]; + uint64 enumIndex = stateDiff.readUint64(84); + if (enumIndex != 0) { + // It is a repeated write, so we skip it. + continue; + } + addInitialWrite(chainId, stateDiff); + + numInitialWritesProcessed++; + + bytes32 derivedKey = stateDiff.readBytes32(52); + uint256 initValue = stateDiff.readUint256(92); + uint256 finalValue = stateDiff.readUint256(124); + require(derivedKey == _compressedStateDiffs.readBytes32(stateDiffPtr), "iw: initial key mismatch"); + stateDiffPtr += 32; + + uint8 metadata = uint8(bytes1(_compressedStateDiffs[stateDiffPtr])); + stateDiffPtr++; + uint8 operation = metadata & OPERATION_BITMASK; + uint8 len = operation == 0 ? 32 : metadata >> LENGTH_BITS_OFFSET; + _verifyValueCompression( + initValue, + finalValue, + operation, + _compressedStateDiffs[stateDiffPtr:stateDiffPtr + len] + ); + stateDiffPtr += len; + } + + require(numInitialWritesProcessed == numberOfInitialWrites, "Incorrect number of initial storage diffs"); + + // Process repeated writes + for (uint256 i = 0; i < _numberOfStateDiffs * STATE_DIFF_ENTRY_SIZE; i += STATE_DIFF_ENTRY_SIZE) { + bytes calldata stateDiff = _stateDiffs[i:i + STATE_DIFF_ENTRY_SIZE]; + uint64 enumIndex = stateDiff.readUint64(84); + if (enumIndex == 0) { + continue; + } + addRepeatedWrite(chainId, stateDiff); + uint256 initValue = stateDiff.readUint256(92); + uint256 finalValue = stateDiff.readUint256(124); + uint256 compressedEnumIndex = _sliceToUint256( + _compressedStateDiffs[stateDiffPtr:stateDiffPtr + _enumerationIndexSize] + ); + require(enumIndex == compressedEnumIndex, "rw: enum key mismatch"); + stateDiffPtr += _enumerationIndexSize; + + uint8 metadata = uint8(bytes1(_compressedStateDiffs[stateDiffPtr])); + stateDiffPtr += 1; + uint8 operation = metadata & OPERATION_BITMASK; + uint8 len = operation == 0 ? 32 : metadata >> LENGTH_BITS_OFFSET; + _verifyValueCompression( + initValue, + finalValue, + operation, + _compressedStateDiffs[stateDiffPtr:stateDiffPtr + len] + ); + stateDiffPtr += len; + } + + require(stateDiffPtr == _compressedStateDiffs.length, "Extra data in _compressedStateDiffs"); + + } function commitBatch( bytes calldata _totalL2ToL1PubdataAndStateDiffs, uint256 chainId, @@ -92,9 +242,13 @@ contract BatchAggregator is IBatchAggregator, ISystemContract { bytes calldata stateDiffs = _totalL2ToL1PubdataAndStateDiffs[calldataPtr:calldataPtr + (numberOfStateDiffs * STATE_DIFF_ENTRY_SIZE)]; + calldataPtr += numberOfStateDiffs * STATE_DIFF_ENTRY_SIZE; - stateDiffStorage[chainId].push(_totalL2ToL1PubdataAndStateDiffs[stateDiffSliceStart:calldataPtr]); + repackStateDiffs(chainId, numberOfStateDiffs, enumerationIndexSize, stateDiffs, compressedStateDiffs); + + + } function returnBatchesAndClearState() external returns (bytes memory batchInfo) { for (uint256 i = 0; i < chainList.length; i += 1) { @@ -105,8 +259,7 @@ contract BatchAggregator is IBatchAggregator, ISystemContract { chainId, logStorage[chainId], messageStorage[chainId], - bytecodeStorage[chainId], - stateDiffStorage[chainId] + bytecodeStorage[chainId] ) ); @@ -114,14 +267,9 @@ contract BatchAggregator is IBatchAggregator, ISystemContract { delete logStorage[chainId]; delete messageStorage[chainId]; delete bytecodeStorage[chainId]; - delete stateDiffStorage[chainId]; } delete chainList; batchInfo = abi.encode(chainData); delete chainData; - /*delete messageStorage; - delete logStorage; - delete stateDiffStorage; - delete bytecodeStorage;*/ } } From 1fc16b0100890c04bec82f783f325fac34e4ef91 Mon Sep 17 00:00:00 2001 From: vladenysiuk Date: Wed, 22 May 2024 15:46:53 +0200 Subject: [PATCH 08/18] add compressed state diffs --- .../contracts/BatchAggregator.sol | 55 ++++++++++++------- 1 file changed, 34 insertions(+), 21 deletions(-) diff --git a/system-contracts/contracts/BatchAggregator.sol b/system-contracts/contracts/BatchAggregator.sol index 7cf72e9ba..a141ff938 100644 --- a/system-contracts/contracts/BatchAggregator.sol +++ b/system-contracts/contracts/BatchAggregator.sol @@ -17,9 +17,13 @@ contract BatchAggregator is IBatchAggregator, ISystemContract { mapping(uint256 => bytes[]) logStorage; mapping(uint256 => bytes[]) bytecodeStorage; // state diff data - mapping(uint256 => mapping(bytes32 => bytes)) uncompressedWrites; - mapping(uint256 => mapping(bytes32 => bool)) slotStatus; - mapping(uint256 => bytes32[]) accesedSlots; + mapping(uint256 => bytes[]) initialWrites; + mapping(uint256 => bytes[]) compressedInitialWrites; + mapping(uint256 => mapping(uint64 => bytes)) uncompressedWrites; + mapping(uint256 => mapping(uint64 => bytes)) compressedWrites; + mapping(uint256 => mapping(uint64 => bool)) slotStatus; + mapping(uint256 => uint64[]) accesedSlots; + // chain data mapping(uint256 => bool) chainSet; uint256[] chainList; @@ -62,33 +66,37 @@ contract BatchAggregator is IBatchAggregator, ISystemContract { } - function addInitialWrite(uint256 chainId, bytes calldata stateDiff) internal{ - bytes32 derivedKey = stateDiff.readBytes32(52); - uncompressedWrites[chainId][derivedKey] = stateDiff; - slotStatus[chainId][derivedKey] = true; - accesedSlots[chainId].push(derivedKey); + function addInitialWrite(uint256 chainId, bytes calldata stateDiff, bytes calldata compressedStateDiff) internal{ + initialWrites[chainId].push(stateDiff); + compressedInitialWrites[chainId].push(compressedStateDiff); } - function addRepeatedWrite(uint256 chainId, bytes calldata stateDiff) internal{ - bytes32 derivedKey = stateDiff.readBytes32(52); - if (slotStatus[chainId][derivedKey]==false){ - uncompressedWrites[chainId][derivedKey] = stateDiff; - slotStatus[chainId][derivedKey] = true; - accesedSlots[chainId].push(derivedKey); + function addRepeatedWrite(uint256 chainId, bytes calldata stateDiff, bytes calldata compressedStateDiff) internal{ + uint64 enumIndex = stateDiff.readUint64(84); + if (slotStatus[chainId][enumIndex]==false){ + uncompressedWrites[chainId][enumIndex] = stateDiff; + compressedWrites[chainId][enumIndex] = compressedStateDiff; + slotStatus[chainId][enumIndex] = true; + accesedSlots[chainId].push(enumIndex); } else{ - bytes memory slotData = uncompressedWrites[chainId][derivedKey]; - uint64 enumIndex; + bytes memory slotData = uncompressedWrites[chainId][enumIndex]; bytes32 finalValue; assembly{ let offset := slotData - enumIndex := mload(add(sub(offset,24),84)) finalValue := mload(add(offset,124)) } assembly { let start := add(slotData, 0x20) - mstore(add(start,84), enumIndex) mstore(add(start,124),finalValue) } + uncompressedWrites[chainId][enumIndex] = slotData; + bytes memory compressedWrite = new bytes(33); + compressedWrite[0] = bytes1(uint8(0 + LENGTH_BITS_OFFSET*32)); + assembly{ + let start := add(compressedWrite, 0x20) + mstore(add(start,1),finalValue) + } + compressedWrites[chainId][enumIndex] = compressedWrite; } } function repackStateDiffs(uint256 chainId, @@ -115,16 +123,17 @@ contract BatchAggregator is IBatchAggregator, ISystemContract { // It is a repeated write, so we skip it. continue; } - addInitialWrite(chainId, stateDiff); + numInitialWritesProcessed++; bytes32 derivedKey = stateDiff.readBytes32(52); uint256 initValue = stateDiff.readUint256(92); uint256 finalValue = stateDiff.readUint256(124); + + uint256 sliceStart = stateDiffPtr; require(derivedKey == _compressedStateDiffs.readBytes32(stateDiffPtr), "iw: initial key mismatch"); stateDiffPtr += 32; - uint8 metadata = uint8(bytes1(_compressedStateDiffs[stateDiffPtr])); stateDiffPtr++; uint8 operation = metadata & OPERATION_BITMASK; @@ -136,6 +145,7 @@ contract BatchAggregator is IBatchAggregator, ISystemContract { _compressedStateDiffs[stateDiffPtr:stateDiffPtr + len] ); stateDiffPtr += len; + addInitialWrite(chainId, stateDiff, _compressedStateDiffs[sliceStart:stateDiffPtr]); } require(numInitialWritesProcessed == numberOfInitialWrites, "Incorrect number of initial storage diffs"); @@ -147,9 +157,10 @@ contract BatchAggregator is IBatchAggregator, ISystemContract { if (enumIndex == 0) { continue; } - addRepeatedWrite(chainId, stateDiff); + uint256 initValue = stateDiff.readUint256(92); uint256 finalValue = stateDiff.readUint256(124); + uint256 sliceStart = stateDiffPtr; uint256 compressedEnumIndex = _sliceToUint256( _compressedStateDiffs[stateDiffPtr:stateDiffPtr + _enumerationIndexSize] ); @@ -167,6 +178,8 @@ contract BatchAggregator is IBatchAggregator, ISystemContract { _compressedStateDiffs[stateDiffPtr:stateDiffPtr + len] ); stateDiffPtr += len; + addRepeatedWrite(chainId, stateDiff, _compressedStateDiffs[sliceStart:stateDiffPtr]); + } require(stateDiffPtr == _compressedStateDiffs.length, "Extra data in _compressedStateDiffs"); From 150eb61fb83777d4b8858a8f5c35554922dc1b75 Mon Sep 17 00:00:00 2001 From: vladenysiuk Date: Wed, 22 May 2024 16:40:57 +0200 Subject: [PATCH 09/18] aggregate uncompressed statediffs --- .../contracts/BatchAggregator.sol | 71 ++++++++----------- 1 file changed, 30 insertions(+), 41 deletions(-) diff --git a/system-contracts/contracts/BatchAggregator.sol b/system-contracts/contracts/BatchAggregator.sol index a141ff938..fc5a35461 100644 --- a/system-contracts/contracts/BatchAggregator.sol +++ b/system-contracts/contracts/BatchAggregator.sol @@ -19,6 +19,7 @@ contract BatchAggregator is IBatchAggregator, ISystemContract { // state diff data mapping(uint256 => bytes[]) initialWrites; mapping(uint256 => bytes[]) compressedInitialWrites; + mapping(uint256 => bytes32[]) initialWritesSlots; mapping(uint256 => mapping(uint64 => bytes)) uncompressedWrites; mapping(uint256 => mapping(uint64 => bytes)) compressedWrites; mapping(uint256 => mapping(uint64 => bool)) slotStatus; @@ -33,42 +34,17 @@ contract BatchAggregator is IBatchAggregator, ISystemContract { chainSet[chainId] = true; } } - // TODO: import or delete + function _sliceToUint256(bytes calldata _calldataSlice) internal pure returns (uint256 number) { number = uint256(bytes32(_calldataSlice)); number >>= (256 - (_calldataSlice.length * 8)); } - function _verifyValueCompression( - uint256 _initialValue, - uint256 _finalValue, - uint256 _operation, - bytes calldata _compressedValue - ) internal pure { - uint256 convertedValue = _sliceToUint256(_compressedValue); - - unchecked { - if (_operation == 0 || _operation == 3) { - require(convertedValue == _finalValue, "transform or no compression: compressed and final mismatch"); - } else if (_operation == 1) { - require( - _initialValue + convertedValue == _finalValue, - "add: initial plus converted not equal to final" - ); - } else if (_operation == 2) { - require( - _initialValue - convertedValue == _finalValue, - "sub: initial minus converted not equal to final" - ); - } else { - revert("unsupported operation"); - } - } - } - function addInitialWrite(uint256 chainId, bytes calldata stateDiff, bytes calldata compressedStateDiff) internal{ + bytes32 derivedKey = stateDiff.readBytes32(52); initialWrites[chainId].push(stateDiff); compressedInitialWrites[chainId].push(compressedStateDiff); + initialWritesSlots[chainId].push(derivedKey); } function addRepeatedWrite(uint256 chainId, bytes calldata stateDiff, bytes calldata compressedStateDiff) internal{ uint64 enumIndex = stateDiff.readUint64(84); @@ -138,12 +114,7 @@ contract BatchAggregator is IBatchAggregator, ISystemContract { stateDiffPtr++; uint8 operation = metadata & OPERATION_BITMASK; uint8 len = operation == 0 ? 32 : metadata >> LENGTH_BITS_OFFSET; - _verifyValueCompression( - initValue, - finalValue, - operation, - _compressedStateDiffs[stateDiffPtr:stateDiffPtr + len] - ); + stateDiffPtr += len; addInitialWrite(chainId, stateDiff, _compressedStateDiffs[sliceStart:stateDiffPtr]); } @@ -171,12 +142,7 @@ contract BatchAggregator is IBatchAggregator, ISystemContract { stateDiffPtr += 1; uint8 operation = metadata & OPERATION_BITMASK; uint8 len = operation == 0 ? 32 : metadata >> LENGTH_BITS_OFFSET; - _verifyValueCompression( - initValue, - finalValue, - operation, - _compressedStateDiffs[stateDiffPtr:stateDiffPtr + len] - ); + stateDiffPtr += len; addRepeatedWrite(chainId, stateDiff, _compressedStateDiffs[sliceStart:stateDiffPtr]); @@ -266,13 +232,36 @@ contract BatchAggregator is IBatchAggregator, ISystemContract { function returnBatchesAndClearState() external returns (bytes memory batchInfo) { for (uint256 i = 0; i < chainList.length; i += 1) { uint256 chainId = chainList[i]; + uint256 numberOfInitialWrites = initialWritesSlots[chainId].length; + uint256 numberOfRepeatedWrites = accesedSlots[chainId].length; + bytes memory stateDiffs = new bytes((numberOfRepeatedWrites+numberOfInitialWrites)*STATE_DIFF_ENTRY_SIZE); + // append initial writes + uint256 initialWritesPtr = 0; + for(uint256 i = 0;i Date: Thu, 23 May 2024 10:29:09 +0200 Subject: [PATCH 10/18] optimal statediff compression !!!! mload, mstore need to be handled in the next commit --- .../contracts/BatchAggregator.sol | 119 ++++++++++++++---- 1 file changed, 98 insertions(+), 21 deletions(-) diff --git a/system-contracts/contracts/BatchAggregator.sol b/system-contracts/contracts/BatchAggregator.sol index fc5a35461..b68264e7d 100644 --- a/system-contracts/contracts/BatchAggregator.sol +++ b/system-contracts/contracts/BatchAggregator.sol @@ -8,6 +8,10 @@ import {L2_L1_LOGS_TREE_DEFAULT_LEAF_HASH, L2_TO_L1_LOG_SERIALIZE_SIZE, STATE_DI import {SystemLogKey, SYSTEM_CONTEXT_CONTRACT, KNOWN_CODE_STORAGE_CONTRACT, COMPRESSOR_CONTRACT, BATCH_AGGREGATOR, STATE_DIFF_ENTRY_SIZE, L2_TO_L1_LOGS_MERKLE_TREE_LEAVES, PUBDATA_CHUNK_PUBLISHER, COMPUTATIONAL_PRICE_FOR_PUBDATA} from "./Constants.sol"; import {UnsafeBytesCalldata} from "./libraries/UnsafeBytesCalldata.sol"; import {ICompressor, OPERATION_BITMASK, LENGTH_BITS_OFFSET, MAX_ENUMERATION_INDEX_SIZE} from "./interfaces/ICompressor.sol"; + +uint256 constant BYTE_BITMASK = (1<<8)-1; +uint256 constant DERIVED_KEY_SIZE = 32; +uint256 constant LOOSE_COMPRESION = 33; contract BatchAggregator is IBatchAggregator, ISystemContract { using UnsafeBytesCalldata for bytes; bytes[] batchStorage; @@ -18,10 +22,8 @@ contract BatchAggregator is IBatchAggregator, ISystemContract { mapping(uint256 => bytes[]) bytecodeStorage; // state diff data mapping(uint256 => bytes[]) initialWrites; - mapping(uint256 => bytes[]) compressedInitialWrites; mapping(uint256 => bytes32[]) initialWritesSlots; mapping(uint256 => mapping(uint64 => bytes)) uncompressedWrites; - mapping(uint256 => mapping(uint64 => bytes)) compressedWrites; mapping(uint256 => mapping(uint64 => bool)) slotStatus; mapping(uint256 => uint64[]) accesedSlots; @@ -43,36 +45,23 @@ contract BatchAggregator is IBatchAggregator, ISystemContract { function addInitialWrite(uint256 chainId, bytes calldata stateDiff, bytes calldata compressedStateDiff) internal{ bytes32 derivedKey = stateDiff.readBytes32(52); initialWrites[chainId].push(stateDiff); - compressedInitialWrites[chainId].push(compressedStateDiff); initialWritesSlots[chainId].push(derivedKey); } function addRepeatedWrite(uint256 chainId, bytes calldata stateDiff, bytes calldata compressedStateDiff) internal{ uint64 enumIndex = stateDiff.readUint64(84); if (slotStatus[chainId][enumIndex]==false){ uncompressedWrites[chainId][enumIndex] = stateDiff; - compressedWrites[chainId][enumIndex] = compressedStateDiff; slotStatus[chainId][enumIndex] = true; accesedSlots[chainId].push(enumIndex); } else{ bytes memory slotData = uncompressedWrites[chainId][enumIndex]; - bytes32 finalValue; - assembly{ - let offset := slotData - finalValue := mload(add(offset,124)) - } + bytes32 finalValue = stateDiff.readBytes32(124); assembly { let start := add(slotData, 0x20) mstore(add(start,124),finalValue) } uncompressedWrites[chainId][enumIndex] = slotData; - bytes memory compressedWrite = new bytes(33); - compressedWrite[0] = bytes1(uint8(0 + LENGTH_BITS_OFFSET*32)); - assembly{ - let start := add(compressedWrite, 0x20) - mstore(add(start,1),finalValue) - } - compressedWrites[chainId][enumIndex] = compressedWrite; } } function repackStateDiffs(uint256 chainId, @@ -225,9 +214,43 @@ contract BatchAggregator is IBatchAggregator, ISystemContract { calldataPtr += numberOfStateDiffs * STATE_DIFF_ENTRY_SIZE; repackStateDiffs(chainId, numberOfStateDiffs, enumerationIndexSize, stateDiffs, compressedStateDiffs); - - - + } + function bytesLength(uint256 value) internal returns (uint8 length){ + while(value>0){ + length += 1; + value = value>>8; + } + } + + function compressValue(uint256 initialValue, uint256 finalValue) internal returns (bytes memory compressedDiff){ + uint8 transform = bytesLength(finalValue); + uint8 add = bytesLength(finalValue-initialValue); + uint8 sub = bytesLength(initialValue-finalValue); + uint8 opt = (transform enumIndex?maxEnumIndex:enumIndex); stateDiffPtr += STATE_DIFF_ENTRY_SIZE; } - + uint256 enumIndexSize = bytesLength(maxEnumIndex); + bytes memory compressedStateDiffs = new bytes((numberOfRepeatedWrites+numberOfInitialWrites)*LOOSE_COMPRESION+numberOfRepeatedWrites*enumIndexSize+numberOfInitialWrites*DERIVED_KEY_SIZE); + uint256 compressionPtr = 0; + // compress initial writes + uint256 compressedStateDiffSize = 0; + initialWritesPtr = 0; + for(uint256 i = 0;i Date: Tue, 28 May 2024 13:10:32 +0200 Subject: [PATCH 11/18] change uncompressed data + fix minor issues --- .../chain-deps/facets/Executor.sol | 2 +- .../contracts/BatchAggregator.sol | 103 ++++++++++-------- system-contracts/contracts/Constants.sol | 1 + 3 files changed, 61 insertions(+), 45 deletions(-) diff --git a/l1-contracts/contracts/state-transition/chain-deps/facets/Executor.sol b/l1-contracts/contracts/state-transition/chain-deps/facets/Executor.sol index 6639a7af8..c04c78fc8 100644 --- a/l1-contracts/contracts/state-transition/chain-deps/facets/Executor.sol +++ b/l1-contracts/contracts/state-transition/chain-deps/facets/Executor.sol @@ -78,7 +78,7 @@ contract ExecutorFacet is ZkSyncHyperchainBase, IExecutor { // Add batch to BatchAggregator IBatchAggregator(BATCH_AGGREGATOR_ADDRESS).commitBatch( _newBatch.pubdataCommitments, - 0, + s.chainId, _newBatch.batchNumber ); } diff --git a/system-contracts/contracts/BatchAggregator.sol b/system-contracts/contracts/BatchAggregator.sol index b68264e7d..73b33e9a9 100644 --- a/system-contracts/contracts/BatchAggregator.sol +++ b/system-contracts/contracts/BatchAggregator.sol @@ -5,7 +5,7 @@ pragma solidity 0.8.20; import {IBatchAggregator} from "./interfaces/IBatchAggregator.sol"; import {ISystemContract} from "./interfaces/ISystemContract.sol"; import {L2_L1_LOGS_TREE_DEFAULT_LEAF_HASH, L2_TO_L1_LOG_SERIALIZE_SIZE, STATE_DIFF_COMPRESSION_VERSION_NUMBER} from "./interfaces/IL1Messenger.sol"; -import {SystemLogKey, SYSTEM_CONTEXT_CONTRACT, KNOWN_CODE_STORAGE_CONTRACT, COMPRESSOR_CONTRACT, BATCH_AGGREGATOR, STATE_DIFF_ENTRY_SIZE, L2_TO_L1_LOGS_MERKLE_TREE_LEAVES, PUBDATA_CHUNK_PUBLISHER, COMPUTATIONAL_PRICE_FOR_PUBDATA} from "./Constants.sol"; +import {SystemLogKey, SYSTEM_CONTEXT_CONTRACT, KNOWN_CODE_STORAGE_CONTRACT, COMPRESSOR_CONTRACT, BATCH_AGGREGATOR, STATE_DIFF_ENTRY_SIZE, STATE_DIFF_AGGREGATION_INFO_SIZE, L2_TO_L1_LOGS_MERKLE_TREE_LEAVES, PUBDATA_CHUNK_PUBLISHER, COMPUTATIONAL_PRICE_FOR_PUBDATA} from "./Constants.sol"; import {UnsafeBytesCalldata} from "./libraries/UnsafeBytesCalldata.sol"; import {ICompressor, OPERATION_BITMASK, LENGTH_BITS_OFFSET, MAX_ENUMERATION_INDEX_SIZE} from "./interfaces/ICompressor.sol"; @@ -23,9 +23,10 @@ contract BatchAggregator is IBatchAggregator, ISystemContract { // state diff data mapping(uint256 => bytes[]) initialWrites; mapping(uint256 => bytes32[]) initialWritesSlots; + /// @dev state diff: [32bytes derived key][8bytes enum index][32bytes initial value][32bytes final value] mapping(uint256 => mapping(uint64 => bytes)) uncompressedWrites; - mapping(uint256 => mapping(uint64 => bool)) slotStatus; - mapping(uint256 => uint64[]) accesedSlots; + mapping(uint256 => mapping(uint64 => bool)) isKeyTouched; + mapping(uint256 => uint64[]) touchedSlots; // chain data mapping(uint256 => bool) chainSet; @@ -40,26 +41,37 @@ contract BatchAggregator is IBatchAggregator, ISystemContract { function _sliceToUint256(bytes calldata _calldataSlice) internal pure returns (uint256 number) { number = uint256(bytes32(_calldataSlice)); number >>= (256 - (_calldataSlice.length * 8)); + } - function addInitialWrite(uint256 chainId, bytes calldata stateDiff, bytes calldata compressedStateDiff) internal{ - bytes32 derivedKey = stateDiff.readBytes32(52); - initialWrites[chainId].push(stateDiff); + function addInitialWrite(uint256 chainId, bytes32 derivedKey, uint64 enumIndex, uint256 initialValue, uint256 finalValue) internal{ + bytes memory slotData = new bytes(STATE_DIFF_AGGREGATION_INFO_SIZE); + assembly{ + mstore(add(slotData,0),derivedKey) + mstore(add(slotData,32),enumIndex) + mstore(add(slotData,40),initialValue) + mstore(add(slotData,72),finalValue) + } + initialWrites[chainId].push(slotData); initialWritesSlots[chainId].push(derivedKey); } - function addRepeatedWrite(uint256 chainId, bytes calldata stateDiff, bytes calldata compressedStateDiff) internal{ - uint64 enumIndex = stateDiff.readUint64(84); - if (slotStatus[chainId][enumIndex]==false){ - uncompressedWrites[chainId][enumIndex] = stateDiff; - slotStatus[chainId][enumIndex] = true; - accesedSlots[chainId].push(enumIndex); + function addRepeatedWrite(uint256 chainId, bytes32 derivedKey, uint64 enumIndex, uint256 initialValue, uint256 finalValue) internal{ + if (isKeyTouched[chainId][enumIndex]==false){ + bytes memory slotData = new bytes(STATE_DIFF_AGGREGATION_INFO_SIZE); + assembly{ + mstore(add(slotData,0),derivedKey) + mstore(add(slotData,32),enumIndex) + mstore(add(slotData,40),initialValue) + mstore(add(slotData,72),finalValue) + } + uncompressedWrites[chainId][enumIndex] = slotData; + isKeyTouched[chainId][enumIndex] = true; + touchedSlots[chainId].push(enumIndex); } else{ bytes memory slotData = uncompressedWrites[chainId][enumIndex]; - bytes32 finalValue = stateDiff.readBytes32(124); assembly { - let start := add(slotData, 0x20) - mstore(add(start,124),finalValue) + mstore(add(slotData,72),finalValue) } uncompressedWrites[chainId][enumIndex] = slotData; } @@ -93,7 +105,7 @@ contract BatchAggregator is IBatchAggregator, ISystemContract { numInitialWritesProcessed++; bytes32 derivedKey = stateDiff.readBytes32(52); - uint256 initValue = stateDiff.readUint256(92); + uint256 initialValue = stateDiff.readUint256(92); uint256 finalValue = stateDiff.readUint256(124); uint256 sliceStart = stateDiffPtr; @@ -105,7 +117,7 @@ contract BatchAggregator is IBatchAggregator, ISystemContract { uint8 len = operation == 0 ? 32 : metadata >> LENGTH_BITS_OFFSET; stateDiffPtr += len; - addInitialWrite(chainId, stateDiff, _compressedStateDiffs[sliceStart:stateDiffPtr]); + addInitialWrite(chainId, derivedKey, enumIndex, initialValue, finalValue); } require(numInitialWritesProcessed == numberOfInitialWrites, "Incorrect number of initial storage diffs"); @@ -117,8 +129,9 @@ contract BatchAggregator is IBatchAggregator, ISystemContract { if (enumIndex == 0) { continue; } - - uint256 initValue = stateDiff.readUint256(92); + + bytes32 derivedKey = stateDiff.readBytes32(52); + uint256 initialValue = stateDiff.readUint256(92); uint256 finalValue = stateDiff.readUint256(124); uint256 sliceStart = stateDiffPtr; uint256 compressedEnumIndex = _sliceToUint256( @@ -131,9 +144,9 @@ contract BatchAggregator is IBatchAggregator, ISystemContract { stateDiffPtr += 1; uint8 operation = metadata & OPERATION_BITMASK; uint8 len = operation == 0 ? 32 : metadata >> LENGTH_BITS_OFFSET; - stateDiffPtr += len; - addRepeatedWrite(chainId, stateDiff, _compressedStateDiffs[sliceStart:stateDiffPtr]); + + addRepeatedWrite(chainId, derivedKey, enumIndex, initialValue, finalValue); } @@ -226,28 +239,28 @@ contract BatchAggregator is IBatchAggregator, ISystemContract { uint8 transform = bytesLength(finalValue); uint8 add = bytesLength(finalValue-initialValue); uint8 sub = bytesLength(initialValue-finalValue); - uint8 opt = (transform enumIndex?maxEnumIndex:enumIndex); - stateDiffPtr += STATE_DIFF_ENTRY_SIZE; + stateDiffPtr += STATE_DIFF_AGGREGATION_INFO_SIZE; } uint256 enumIndexSize = bytesLength(maxEnumIndex); - bytes memory compressedStateDiffs = new bytes((numberOfRepeatedWrites+numberOfInitialWrites)*LOOSE_COMPRESION+numberOfRepeatedWrites*enumIndexSize+numberOfInitialWrites*DERIVED_KEY_SIZE); + + bytes memory compressedStateDiffs = new bytes( + (numberOfRepeatedWrites+numberOfInitialWrites)*LOOSE_COMPRESION // maximal size of metadata + compressed value + +numberOfRepeatedWrites*enumIndexSize // enumIndexSize for repeated writes + +numberOfInitialWrites*DERIVED_KEY_SIZE); // derived key for initial writes uint256 compressionPtr = 0; // compress initial writes uint256 compressedStateDiffSize = 0; - initialWritesPtr = 0; - for(uint256 i = 0;i Date: Tue, 28 May 2024 13:16:34 +0200 Subject: [PATCH 12/18] unchecked math + type().max --- system-contracts/contracts/BatchAggregator.sol | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/system-contracts/contracts/BatchAggregator.sol b/system-contracts/contracts/BatchAggregator.sol index 73b33e9a9..1a4dd64d1 100644 --- a/system-contracts/contracts/BatchAggregator.sol +++ b/system-contracts/contracts/BatchAggregator.sol @@ -9,7 +9,6 @@ import {SystemLogKey, SYSTEM_CONTEXT_CONTRACT, KNOWN_CODE_STORAGE_CONTRACT, COMP import {UnsafeBytesCalldata} from "./libraries/UnsafeBytesCalldata.sol"; import {ICompressor, OPERATION_BITMASK, LENGTH_BITS_OFFSET, MAX_ENUMERATION_INDEX_SIZE} from "./interfaces/ICompressor.sol"; -uint256 constant BYTE_BITMASK = (1<<8)-1; uint256 constant DERIVED_KEY_SIZE = 32; uint256 constant LOOSE_COMPRESION = 33; contract BatchAggregator is IBatchAggregator, ISystemContract { @@ -234,11 +233,13 @@ contract BatchAggregator is IBatchAggregator, ISystemContract { value = value>>8; } } - + function subUnchecked(uint256 a, uint256 b) view public returns(uint256) { + unchecked { return a - b ;} + } function compressValue(uint256 initialValue, uint256 finalValue) internal returns (bytes memory compressedDiff){ uint8 transform = bytesLength(finalValue); - uint8 add = bytesLength(finalValue-initialValue); - uint8 sub = bytesLength(initialValue-finalValue); + uint8 add = bytesLength(subUnchecked(finalValue,initialValue)); + uint8 sub = bytesLength(subUnchecked(initialValue,finalValue)); uint8 optimal = (transform Date: Tue, 28 May 2024 13:28:13 +0200 Subject: [PATCH 13/18] fix messages compression --- system-contracts/contracts/BatchAggregator.sol | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/system-contracts/contracts/BatchAggregator.sol b/system-contracts/contracts/BatchAggregator.sol index 1a4dd64d1..6696b7fcb 100644 --- a/system-contracts/contracts/BatchAggregator.sol +++ b/system-contracts/contracts/BatchAggregator.sol @@ -173,11 +173,15 @@ contract BatchAggregator is IBatchAggregator, ISystemContract { calldataPtr += 4 + L2_TO_L1_LOG_SERIALIZE_SIZE * numberOfL2ToL1Logs; /// Check messages + uint256 messageSliceStart = calldataPtr; uint32 numberOfMessages = uint32(bytes4(_totalL2ToL1PubdataAndStateDiffs[calldataPtr:calldataPtr + 4])); - messageStorage[chainId].push( - _totalL2ToL1PubdataAndStateDiffs[calldataPtr:calldataPtr + 4 + numberOfMessages * 4] - ); - calldataPtr += 4 + numberOfMessages * 4; + calldataPtr += 4; + bytes32 reconstructedChainedMessagesHash; + for (uint256 i = 0; i < numberOfMessages; ++i) { + uint32 currentMessageLength = uint32(bytes4(_totalL2ToL1PubdataAndStateDiffs[calldataPtr:calldataPtr + 4])); + calldataPtr += 4 + currentMessageLength; + } + messageStorage[chainId].push(_totalL2ToL1PubdataAndStateDiffs[messageSliceStart:calldataPtr]); /// Check bytecodes uint32 numberOfBytecodes = uint32(bytes4(_totalL2ToL1PubdataAndStateDiffs[calldataPtr:calldataPtr + 4])); From 3604c6b40d835ea3d06d1565c9ba35c6da7e9fde Mon Sep 17 00:00:00 2001 From: vladenysiuk Date: Tue, 28 May 2024 13:31:17 +0200 Subject: [PATCH 14/18] handle compressedData size = 32 --- system-contracts/contracts/BatchAggregator.sol | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/system-contracts/contracts/BatchAggregator.sol b/system-contracts/contracts/BatchAggregator.sol index 6696b7fcb..d1838eca6 100644 --- a/system-contracts/contracts/BatchAggregator.sol +++ b/system-contracts/contracts/BatchAggregator.sol @@ -249,7 +249,11 @@ contract BatchAggregator is IBatchAggregator, ISystemContract { compressedDiff = new bytes(optimal+1); uint8 mask = (optimal< Date: Wed, 29 May 2024 11:26:48 +0200 Subject: [PATCH 15/18] aggregate initial and repeated writes --- .../contracts/BatchAggregator.sol | 29 ++++++++++++------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/system-contracts/contracts/BatchAggregator.sol b/system-contracts/contracts/BatchAggregator.sol index d1838eca6..ade43091c 100644 --- a/system-contracts/contracts/BatchAggregator.sol +++ b/system-contracts/contracts/BatchAggregator.sol @@ -20,7 +20,8 @@ contract BatchAggregator is IBatchAggregator, ISystemContract { mapping(uint256 => bytes[]) logStorage; mapping(uint256 => bytes[]) bytecodeStorage; // state diff data - mapping(uint256 => bytes[]) initialWrites; + mapping(uint256 => mapping(bytes32 => bytes)) initialWrites; + mapping(uint256 => mapping(bytes32 => bool)) isInitialWrite; mapping(uint256 => bytes32[]) initialWritesSlots; /// @dev state diff: [32bytes derived key][8bytes enum index][32bytes initial value][32bytes final value] mapping(uint256 => mapping(uint64 => bytes)) uncompressedWrites; @@ -51,11 +52,19 @@ contract BatchAggregator is IBatchAggregator, ISystemContract { mstore(add(slotData,40),initialValue) mstore(add(slotData,72),finalValue) } - initialWrites[chainId].push(slotData); + initialWrites[chainId][derivedKey] = slotData; + isInitialWrite[chainId][derivedKey] = true; initialWritesSlots[chainId].push(derivedKey); } function addRepeatedWrite(uint256 chainId, bytes32 derivedKey, uint64 enumIndex, uint256 initialValue, uint256 finalValue) internal{ - if (isKeyTouched[chainId][enumIndex]==false){ + if (isInitialWrite[chainId][derivedKey]==true){ + bytes memory slotData = initialWrites[chainId][derivedKey]; + assembly { + mstore(add(slotData,72),finalValue) + } + initialWrites[chainId][derivedKey] = slotData; + } + else if (isKeyTouched[chainId][enumIndex]==false){ bytes memory slotData = new bytes(STATE_DIFF_AGGREGATION_INFO_SIZE); assembly{ mstore(add(slotData,0),derivedKey) @@ -282,17 +291,16 @@ contract BatchAggregator is IBatchAggregator, ISystemContract { bytes memory stateDiffs = new bytes((numberOfRepeatedWrites+numberOfInitialWrites)*STATE_DIFF_AGGREGATION_INFO_SIZE); uint256 maxEnumIndex = 0; + uint256 stateDiffPtr = 0; // append initial writes - uint256 initialWritesPtr = 0; - for(uint256 i = 0;i Date: Wed, 29 May 2024 11:32:34 +0200 Subject: [PATCH 16/18] fix warnings --- system-contracts/contracts/BatchAggregator.sol | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/system-contracts/contracts/BatchAggregator.sol b/system-contracts/contracts/BatchAggregator.sol index ade43091c..f7eed1afd 100644 --- a/system-contracts/contracts/BatchAggregator.sol +++ b/system-contracts/contracts/BatchAggregator.sol @@ -116,7 +116,6 @@ contract BatchAggregator is IBatchAggregator, ISystemContract { uint256 initialValue = stateDiff.readUint256(92); uint256 finalValue = stateDiff.readUint256(124); - uint256 sliceStart = stateDiffPtr; require(derivedKey == _compressedStateDiffs.readBytes32(stateDiffPtr), "iw: initial key mismatch"); stateDiffPtr += 32; uint8 metadata = uint8(bytes1(_compressedStateDiffs[stateDiffPtr])); @@ -141,7 +140,6 @@ contract BatchAggregator is IBatchAggregator, ISystemContract { bytes32 derivedKey = stateDiff.readBytes32(52); uint256 initialValue = stateDiff.readUint256(92); uint256 finalValue = stateDiff.readUint256(124); - uint256 sliceStart = stateDiffPtr; uint256 compressedEnumIndex = _sliceToUint256( _compressedStateDiffs[stateDiffPtr:stateDiffPtr + _enumerationIndexSize] ); @@ -185,7 +183,6 @@ contract BatchAggregator is IBatchAggregator, ISystemContract { uint256 messageSliceStart = calldataPtr; uint32 numberOfMessages = uint32(bytes4(_totalL2ToL1PubdataAndStateDiffs[calldataPtr:calldataPtr + 4])); calldataPtr += 4; - bytes32 reconstructedChainedMessagesHash; for (uint256 i = 0; i < numberOfMessages; ++i) { uint32 currentMessageLength = uint32(bytes4(_totalL2ToL1PubdataAndStateDiffs[calldataPtr:calldataPtr + 4])); calldataPtr += 4 + currentMessageLength; @@ -196,7 +193,6 @@ contract BatchAggregator is IBatchAggregator, ISystemContract { uint32 numberOfBytecodes = uint32(bytes4(_totalL2ToL1PubdataAndStateDiffs[calldataPtr:calldataPtr + 4])); uint256 bytecodeSliceStart = calldataPtr; calldataPtr += 4; - bytes32 reconstructedChainedL1BytecodesRevealDataHash; for (uint256 i = 0; i < numberOfBytecodes; ++i) { uint32 currentBytecodeLength = uint32( bytes4(_totalL2ToL1PubdataAndStateDiffs[calldataPtr:calldataPtr + 4]) @@ -215,7 +211,6 @@ contract BatchAggregator is IBatchAggregator, ISystemContract { STATE_DIFF_COMPRESSION_VERSION_NUMBER, "state diff compression version mismatch" ); - uint256 stateDiffSliceStart = calldataPtr; calldataPtr++; uint24 compressedStateDiffSize = uint24(bytes3(_totalL2ToL1PubdataAndStateDiffs[calldataPtr:calldataPtr + 3])); @@ -228,8 +223,6 @@ contract BatchAggregator is IBatchAggregator, ISystemContract { compressedStateDiffSize]; calldataPtr += compressedStateDiffSize; - bytes calldata totalL2ToL1Pubdata = _totalL2ToL1PubdataAndStateDiffs[:calldataPtr]; - uint32 numberOfStateDiffs = uint32(bytes4(_totalL2ToL1PubdataAndStateDiffs[calldataPtr:calldataPtr + 4])); calldataPtr += 4; @@ -240,16 +233,16 @@ contract BatchAggregator is IBatchAggregator, ISystemContract { repackStateDiffs(chainId, numberOfStateDiffs, enumerationIndexSize, stateDiffs, compressedStateDiffs); } - function bytesLength(uint256 value) internal returns (uint8 length){ + function bytesLength(uint256 value) pure internal returns (uint8 length){ while(value>0){ length += 1; value = value>>8; } } - function subUnchecked(uint256 a, uint256 b) view public returns(uint256) { + function subUnchecked(uint256 a, uint256 b) pure internal returns(uint256) { unchecked { return a - b ;} } - function compressValue(uint256 initialValue, uint256 finalValue) internal returns (bytes memory compressedDiff){ + function compressValue(uint256 initialValue, uint256 finalValue) pure internal returns (bytes memory compressedDiff){ uint8 transform = bytesLength(finalValue); uint8 add = bytesLength(subUnchecked(finalValue,initialValue)); uint8 sub = bytesLength(subUnchecked(initialValue,finalValue)); @@ -316,7 +309,6 @@ contract BatchAggregator is IBatchAggregator, ISystemContract { (numberOfRepeatedWrites+numberOfInitialWrites)*LOOSE_COMPRESION // maximal size of metadata + compressed value +numberOfRepeatedWrites*enumIndexSize // enumIndexSize for repeated writes +numberOfInitialWrites*DERIVED_KEY_SIZE); // derived key for initial writes - uint256 compressionPtr = 0; // compress initial writes uint256 compressedStateDiffSize = 0; for(uint256 i = 0;i Date: Wed, 29 May 2024 11:53:50 +0200 Subject: [PATCH 17/18] increase stateDiff readability --- .../contracts/BatchAggregator.sol | 86 ++++++++----------- system-contracts/contracts/Constants.sol | 3 +- 2 files changed, 36 insertions(+), 53 deletions(-) diff --git a/system-contracts/contracts/BatchAggregator.sol b/system-contracts/contracts/BatchAggregator.sol index f7eed1afd..b078abb49 100644 --- a/system-contracts/contracts/BatchAggregator.sol +++ b/system-contracts/contracts/BatchAggregator.sol @@ -5,13 +5,18 @@ pragma solidity 0.8.20; import {IBatchAggregator} from "./interfaces/IBatchAggregator.sol"; import {ISystemContract} from "./interfaces/ISystemContract.sol"; import {L2_L1_LOGS_TREE_DEFAULT_LEAF_HASH, L2_TO_L1_LOG_SERIALIZE_SIZE, STATE_DIFF_COMPRESSION_VERSION_NUMBER} from "./interfaces/IL1Messenger.sol"; -import {SystemLogKey, SYSTEM_CONTEXT_CONTRACT, KNOWN_CODE_STORAGE_CONTRACT, COMPRESSOR_CONTRACT, BATCH_AGGREGATOR, STATE_DIFF_ENTRY_SIZE, STATE_DIFF_AGGREGATION_INFO_SIZE, L2_TO_L1_LOGS_MERKLE_TREE_LEAVES, PUBDATA_CHUNK_PUBLISHER, COMPUTATIONAL_PRICE_FOR_PUBDATA} from "./Constants.sol"; +import {SystemLogKey, SYSTEM_CONTEXT_CONTRACT, KNOWN_CODE_STORAGE_CONTRACT, COMPRESSOR_CONTRACT, BATCH_AGGREGATOR, STATE_DIFF_ENTRY_SIZE, L2_TO_L1_LOGS_MERKLE_TREE_LEAVES, PUBDATA_CHUNK_PUBLISHER, COMPUTATIONAL_PRICE_FOR_PUBDATA, DERIVED_KEY_SIZE, LOOSE_COMPRESION} from "./Constants.sol"; import {UnsafeBytesCalldata} from "./libraries/UnsafeBytesCalldata.sol"; import {ICompressor, OPERATION_BITMASK, LENGTH_BITS_OFFSET, MAX_ENUMERATION_INDEX_SIZE} from "./interfaces/ICompressor.sol"; -uint256 constant DERIVED_KEY_SIZE = 32; -uint256 constant LOOSE_COMPRESION = 33; + contract BatchAggregator is IBatchAggregator, ISystemContract { + struct AggregationInfo{ + bytes32 derivedKey; + uint64 enumIndex; + uint256 initialValue; + uint256 finalValue; + } using UnsafeBytesCalldata for bytes; bytes[] batchStorage; bytes[] chainData; @@ -20,11 +25,10 @@ contract BatchAggregator is IBatchAggregator, ISystemContract { mapping(uint256 => bytes[]) logStorage; mapping(uint256 => bytes[]) bytecodeStorage; // state diff data - mapping(uint256 => mapping(bytes32 => bytes)) initialWrites; + mapping(uint256 => mapping(bytes32 => AggregationInfo)) initialWrites; mapping(uint256 => mapping(bytes32 => bool)) isInitialWrite; mapping(uint256 => bytes32[]) initialWritesSlots; - /// @dev state diff: [32bytes derived key][8bytes enum index][32bytes initial value][32bytes final value] - mapping(uint256 => mapping(uint64 => bytes)) uncompressedWrites; + mapping(uint256 => mapping(uint64 => AggregationInfo)) uncompressedWrites; mapping(uint256 => mapping(uint64 => bool)) isKeyTouched; mapping(uint256 => uint64[]) touchedSlots; @@ -45,42 +49,36 @@ contract BatchAggregator is IBatchAggregator, ISystemContract { } function addInitialWrite(uint256 chainId, bytes32 derivedKey, uint64 enumIndex, uint256 initialValue, uint256 finalValue) internal{ - bytes memory slotData = new bytes(STATE_DIFF_AGGREGATION_INFO_SIZE); - assembly{ - mstore(add(slotData,0),derivedKey) - mstore(add(slotData,32),enumIndex) - mstore(add(slotData,40),initialValue) - mstore(add(slotData,72),finalValue) - } + AggregationInfo memory slotData = AggregationInfo({ + derivedKey:derivedKey, + enumIndex:enumIndex, + initialValue:initialValue, + finalValue:finalValue + }); initialWrites[chainId][derivedKey] = slotData; isInitialWrite[chainId][derivedKey] = true; initialWritesSlots[chainId].push(derivedKey); } function addRepeatedWrite(uint256 chainId, bytes32 derivedKey, uint64 enumIndex, uint256 initialValue, uint256 finalValue) internal{ if (isInitialWrite[chainId][derivedKey]==true){ - bytes memory slotData = initialWrites[chainId][derivedKey]; - assembly { - mstore(add(slotData,72),finalValue) - } + AggregationInfo memory slotData = initialWrites[chainId][derivedKey]; + slotData.finalValue = finalValue; initialWrites[chainId][derivedKey] = slotData; } else if (isKeyTouched[chainId][enumIndex]==false){ - bytes memory slotData = new bytes(STATE_DIFF_AGGREGATION_INFO_SIZE); - assembly{ - mstore(add(slotData,0),derivedKey) - mstore(add(slotData,32),enumIndex) - mstore(add(slotData,40),initialValue) - mstore(add(slotData,72),finalValue) - } + AggregationInfo memory slotData = AggregationInfo({ + derivedKey:derivedKey, + enumIndex:enumIndex, + initialValue:initialValue, + finalValue:finalValue + }); uncompressedWrites[chainId][enumIndex] = slotData; isKeyTouched[chainId][enumIndex] = true; touchedSlots[chainId].push(enumIndex); } else{ - bytes memory slotData = uncompressedWrites[chainId][enumIndex]; - assembly { - mstore(add(slotData,72),finalValue) - } + AggregationInfo memory slotData = uncompressedWrites[chainId][enumIndex]; + slotData.finalValue = finalValue; uncompressedWrites[chainId][enumIndex] = slotData; } } @@ -281,27 +279,12 @@ contract BatchAggregator is IBatchAggregator, ISystemContract { uint256 chainId = chainList[i]; uint256 numberOfInitialWrites = initialWritesSlots[chainId].length; uint256 numberOfRepeatedWrites = touchedSlots[chainId].length; - bytes memory stateDiffs = new bytes((numberOfRepeatedWrites+numberOfInitialWrites)*STATE_DIFF_AGGREGATION_INFO_SIZE); uint256 maxEnumIndex = 0; - uint256 stateDiffPtr = 0; - // append initial writes - for(uint256 i = 0;i enumIndex?maxEnumIndex:enumIndex); - stateDiffPtr += STATE_DIFF_AGGREGATION_INFO_SIZE; } uint256 enumIndexSize = bytesLength(maxEnumIndex); @@ -312,10 +295,10 @@ contract BatchAggregator is IBatchAggregator, ISystemContract { // compress initial writes uint256 compressedStateDiffSize = 0; for(uint256 i = 0;i Date: Wed, 29 May 2024 12:31:50 +0200 Subject: [PATCH 18/18] merge hyperchain and statediff hash --- system-contracts/contracts/Compressor.sol | 5 ++++- system-contracts/contracts/L1Messenger.sol | 10 +++------- system-contracts/contracts/interfaces/ICompressor.sol | 3 ++- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/system-contracts/contracts/Compressor.sol b/system-contracts/contracts/Compressor.sol index f52c18ed4..32ace63d0 100644 --- a/system-contracts/contracts/Compressor.sol +++ b/system-contracts/contracts/Compressor.sol @@ -111,7 +111,8 @@ contract Compressor is ICompressor, ISystemContract { uint256 _numberOfStateDiffs, uint256 _enumerationIndexSize, bytes calldata _stateDiffs, - bytes calldata _compressedStateDiffs + bytes calldata _compressedStateDiffs, + bytes memory _hyperchainData ) external onlyCallFrom(address(L1_MESSENGER_CONTRACT)) returns (bytes32 stateDiffHash) { // We do not enforce the operator to use the optimal, i.e. the minimally possible _enumerationIndexSize. // We do enforce however, that the _enumerationIndexSize is not larger than 8 bytes long, which is the @@ -187,6 +188,8 @@ contract Compressor is ICompressor, ISystemContract { require(stateDiffPtr == _compressedStateDiffs.length, "Extra data in _compressedStateDiffs"); stateDiffHash = EfficientCall.keccak(_stateDiffs); + bytes32 hyperchainHash = keccak256(_hyperchainData); + stateDiffHash = keccak256(abi.encode(stateDiffHash,hyperchainHash)); } /// @notice Decode the raw compressed data into the dictionary and the encoded data. diff --git a/system-contracts/contracts/L1Messenger.sol b/system-contracts/contracts/L1Messenger.sol index bd123b633..b9e045f12 100644 --- a/system-contracts/contracts/L1Messenger.sol +++ b/system-contracts/contracts/L1Messenger.sol @@ -194,12 +194,7 @@ contract L1Messenger is IL1Messenger, ISystemContract { function publishPubdataAndClearState( bytes calldata _totalL2ToL1PubdataAndStateDiffs ) external onlyCallFromBootloader { - // Send hyperchain batches - SystemContractHelper.toL1( - true, - bytes32(uint256(SystemLogKey.HYPERCHAIN_PUBDATA_KEY)), - keccak256(BATCH_AGGREGATOR.returnBatchesAndClearState()) - ); + uint256 calldataPtr = 0; @@ -313,7 +308,8 @@ contract L1Messenger is IL1Messenger, ISystemContract { numberOfStateDiffs, enumerationIndexSize, stateDiffs, - compressedStateDiffs + compressedStateDiffs, + BATCH_AGGREGATOR.returnBatchesAndClearState() ); // diff --git a/system-contracts/contracts/interfaces/ICompressor.sol b/system-contracts/contracts/interfaces/ICompressor.sol index 3062ea4f7..4533aa08c 100644 --- a/system-contracts/contracts/interfaces/ICompressor.sol +++ b/system-contracts/contracts/interfaces/ICompressor.sol @@ -25,6 +25,7 @@ interface ICompressor { uint256 _numberOfStateDiffs, uint256 _enumerationIndexSize, bytes calldata _stateDiffs, - bytes calldata _compressedStateDiffs + bytes calldata _compressedStateDiffs, + bytes memory _hyperchainData ) external returns (bytes32 stateDiffHash); }