Skip to content

Commit

Permalink
Merge branch 'master' into pull-improvements
Browse files Browse the repository at this point in the history
  • Loading branch information
vishalchangrani authored Aug 28, 2024
2 parents 8e52fd5 + fc56066 commit b89f92d
Show file tree
Hide file tree
Showing 34 changed files with 2,889 additions and 2,176 deletions.
8 changes: 7 additions & 1 deletion .github/workflows/tools.yml
Original file line number Diff line number Diff line change
Expand Up @@ -46,12 +46,18 @@ jobs:
make tool-bootstrap tool-transit
mkdir boot-tools
mv bootstrap transit boot-tools/
sha256sum boot-tools/bootstrap > boot-tools/bootstrap.sha256sum
cat boot-tools/bootstrap.sha256sum
sha256sum boot-tools/transit > boot-tools/transit.sha256sum
cat boot-tools/transit.sha256sum
tar -czf boot-tools.tar ./boot-tools/
gsutil cp boot-tools.tar gs://flow-genesis-bootstrap/tools/${{ inputs.tag }}/boot-tools.tar
- name: Build and upload util
run: |
make tool-util
tar -czf util.tar util
sha256sum util > util.sha256sum
cat util.sha256sum
tar -czf util.tar util util.sha256sum
gsutil cp util.tar gs://flow-genesis-bootstrap/tools/${{ inputs.tag }}/util.tar
- name: Promote boot-tools
run: |
Expand Down
8 changes: 8 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -540,6 +540,14 @@ docker-native-build-access-corrupt:
-t "$(CONTAINER_REGISTRY)/access-corrupted:$(IMAGE_TAG)" .
./insecure/cmd/mods_restore.sh

# build a binary to run on bare metal without using docker.
# binary is written to file ./bin/app
.PHONY: docker-native-build-access-binary
docker-native-build-access-binary: docker-native-build-access
docker create --name extract "$(CONTAINER_REGISTRY)/access:latest"
docker cp extract:/bin/app ./flow_access_node
docker rm extract

.PHONY: docker-native-build-observer
docker-native-build-observer:
docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/observer --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG) --build-arg GOARCH=$(GOARCH) --build-arg CGO_FLAG=$(CRYPTO_FLAG) --target production \
Expand Down
11 changes: 11 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,17 @@ Build a Docker image for a particular node role (replace `$ROLE` with `collectio
make docker-native-build-$ROLE
```

#### Building a binary for the access node

Build the binary for an access node that can be run directly on the machine without using Docker.

```bash
make docker-native-build-access-binary
```
_this builds a binary for Linux/x86_64 machine_.

The make command will generate a binary called `flow_access_node`

### Importing the module

When importing the `github.com/onflow/flow-go` module in your Go project, testing or building your project may require setting extra Go flags because the module requires [cgo](https://pkg.go.dev/cmd/cgo). In particular, `CGO_ENABLED` must be set to `1` if `cgo` isn't enabled by default. This constraint comes from the underlying cryptography library. Refer to the [crypto repository build](https://github.com/onflow/crypto?tab=readme-ov-file#build) for more details.
Expand Down
41 changes: 36 additions & 5 deletions cmd/util/cmd/check-storage/cmd.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package check_storage

import (
"context"
"slices"

"github.com/rs/zerolog/log"
"github.com/spf13/cobra"
Expand All @@ -10,11 +11,12 @@ import (
"github.com/onflow/cadence/runtime"
"github.com/onflow/cadence/runtime/common"

"github.com/onflow/flow-go/cmd/util/ledger/migrations"
"github.com/onflow/flow-go/cmd/util/ledger/reporters"
"github.com/onflow/flow-go/cmd/util/ledger/util"
"github.com/onflow/flow-go/cmd/util/ledger/util/registers"
"github.com/onflow/flow-go/fvm/systemcontracts"
"github.com/onflow/flow-go/ledger"
"github.com/onflow/flow-go/model/flow"
moduleUtil "github.com/onflow/flow-go/module/util"
)

Expand All @@ -23,6 +25,7 @@ var (
flagState string
flagStateCommitment string
flagOutputDirectory string
flagChain string
flagNWorker int
)

Expand Down Expand Up @@ -74,10 +77,22 @@ func init() {
10,
"number of workers to use",
)

Cmd.Flags().StringVar(
&flagChain,
"chain",
"",
"Chain name",
)
_ = Cmd.MarkFlagRequired("chain")
}

func run(*cobra.Command, []string) {

chainID := flow.ChainID(flagChain)
// Validate chain ID
_ = chainID.Chain()

if flagPayloads == "" && flagState == "" {
log.Fatal().Msg("Either --payloads or --state must be provided")
} else if flagPayloads != "" && flagState != "" {
Expand All @@ -87,6 +102,14 @@ func run(*cobra.Command, []string) {
log.Fatal().Msg("--state-commitment must be provided when --state is provided")
}

// For now, skip EVM storage account since a different decoder is needed for decoding EVM registers.

systemContracts := systemcontracts.SystemContractsForChain(chainID)

acctsToSkip := []string{
flow.AddressToRegisterOwner(systemContracts.EVMStorage.Address),
}

// Create report in JSONL format
rw := reporters.NewReportFileWriterFactoryWithFormat(flagOutputDirectory, log.Logger, reporters.ReportFormatJSONL).
ReportWriter(ReporterName)
Expand Down Expand Up @@ -138,7 +161,7 @@ func run(*cobra.Command, []string) {
len(payloads),
)

failedAccountAddresses, err := checkStorageHealth(registersByAccount, flagNWorker, rw)
failedAccountAddresses, err := checkStorageHealth(registersByAccount, flagNWorker, rw, acctsToSkip)
if err != nil {
log.Fatal().Err(err).Msg("failed to check storage health")
}
Expand All @@ -165,6 +188,7 @@ func checkStorageHealth(
registersByAccount *registers.ByAccount,
nWorkers int,
rw reporters.ReportWriter,
acctsToSkip []string,
) (failedAccountAddresses []string, err error) {

accountCount := registersByAccount.AccountCount()
Expand All @@ -185,6 +209,12 @@ func checkStorageHealth(
// Skip goroutine to avoid overhead
err = registersByAccount.ForEachAccount(
func(accountRegisters *registers.AccountRegisters) error {
defer logAccount(1)

if slices.Contains(acctsToSkip, accountRegisters.Owner()) {
return nil
}

accountStorageIssues := checkAccountStorageHealth(accountRegisters, nWorkers)

if len(accountStorageIssues) > 0 {
Expand All @@ -195,8 +225,6 @@ func checkStorageHealth(
}
}

logAccount(1)

return nil
})

Expand Down Expand Up @@ -253,6 +281,9 @@ func checkStorageHealth(

err = registersByAccount.ForEachAccount(
func(accountRegisters *registers.AccountRegisters) error {
if slices.Contains(acctsToSkip, accountRegisters.Owner()) {
return nil
}
jobs <- job{accountRegisters: accountRegisters}
return nil
})
Expand Down Expand Up @@ -294,7 +325,7 @@ func checkAccountStorageHealth(accountRegisters *registers.AccountRegisters, nWo
ledger := &registers.ReadOnlyLedger{Registers: accountRegisters}
storage := runtime.NewStorage(ledger, nil)

err = util.CheckStorageHealth(address, storage, accountRegisters, migrations.AllStorageMapDomains, nWorkers)
err = util.CheckStorageHealth(address, storage, accountRegisters, util.StorageMapDomains, nWorkers)
if err != nil {
issues = append(
issues,
Expand Down
147 changes: 147 additions & 0 deletions cmd/util/cmd/debug-tx/cmd.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,147 @@
package debug_tx

import (
"context"

"github.com/rs/zerolog/log"
"github.com/spf13/cobra"

sdk "github.com/onflow/flow-go-sdk"

"github.com/onflow/flow-go/model/flow"
"github.com/onflow/flow-go/module/grpcclient"
"github.com/onflow/flow-go/utils/debug"
)

// use the following command to forward port 9000 from the EN to localhost:9001
// `gcloud compute ssh '--ssh-flag=-A' --no-user-output-enabled --tunnel-through-iap migrationmainnet1-execution-001 --project flow-multi-region -- -NL 9001:localhost:9000`

var (
flagAccessAddress string
flagExecutionAddress string
flagChain string
flagTx string
flagComputeLimit uint64
flagAtLatestBlock bool
flagProposalKeySeq uint64
)

var Cmd = &cobra.Command{
Use: "debug-tx",
Short: "debug a transaction",
Run: run,
}

func init() {

Cmd.Flags().StringVar(
&flagChain,
"chain",
"",
"Chain name",
)
_ = Cmd.MarkFlagRequired("chain")

Cmd.Flags().StringVar(&flagAccessAddress, "access-address", "", "address of the access node")
_ = Cmd.MarkFlagRequired("access-address")

Cmd.Flags().StringVar(&flagExecutionAddress, "execution-address", "", "address of the execution node")
_ = Cmd.MarkFlagRequired("execution-address")

Cmd.Flags().StringVar(&flagTx, "tx", "", "transaction ID")
_ = Cmd.MarkFlagRequired("tx")

Cmd.Flags().Uint64Var(&flagComputeLimit, "compute-limit", 9999, "transaction compute limit")

Cmd.Flags().BoolVar(&flagAtLatestBlock, "at-latest-block", false, "run at latest block")

Cmd.Flags().Uint64Var(&flagProposalKeySeq, "proposal-key-seq", 0, "proposal key sequence number")
}

func run(*cobra.Command, []string) {

chainID := flow.ChainID(flagChain)
chain := chainID.Chain()

txID, err := flow.HexStringToIdentifier(flagTx)
if err != nil {
log.Fatal().Err(err).Msg("failed to parse transaction ID")
}

config, err := grpcclient.NewFlowClientConfig(flagAccessAddress, "", flow.ZeroID, true)
if err != nil {
log.Fatal().Err(err).Msg("failed to create flow client config")
}

flowClient, err := grpcclient.FlowClient(config)
if err != nil {
log.Fatal().Err(err).Msg("failed to create flow client")
}

log.Info().Msg("Fetching transaction ...")

tx, err := flowClient.GetTransaction(context.Background(), sdk.Identifier(txID))
if err != nil {
log.Fatal().Err(err).Msg("failed to fetch transaction")
}

log.Info().Msgf("Fetched transaction: %s", tx.ID())

log.Info().Msg("Fetching transaction result ...")

txResult, err := flowClient.GetTransactionResult(context.Background(), sdk.Identifier(txID))
if err != nil {
log.Fatal().Err(err).Msg("failed to fetch transaction result")
}

log.Info().Msgf("Fetched transaction result: %s at block %s", txResult.Status, txResult.BlockID)

log.Info().Msg("Debugging transaction ...")

debugger := debug.NewRemoteDebugger(
flagExecutionAddress,
chain,
log.Logger,
)

txBody := flow.NewTransactionBody().
SetScript(tx.Script).
SetComputeLimit(flagComputeLimit).
SetPayer(flow.Address(tx.Payer))

for _, argument := range tx.Arguments {
txBody.AddArgument(argument)
}

for _, authorizer := range tx.Authorizers {
txBody.AddAuthorizer(flow.Address(authorizer))
}

proposalKeySequenceNumber := tx.ProposalKey.SequenceNumber
if flagProposalKeySeq != 0 {
proposalKeySequenceNumber = flagProposalKeySeq
}

txBody.SetProposalKey(
flow.Address(tx.ProposalKey.Address),
tx.ProposalKey.KeyIndex,
proposalKeySequenceNumber,
)

var txErr, processErr error
if flagAtLatestBlock {
txErr, processErr = debugger.RunTransaction(txBody)
} else {
txErr, processErr = debugger.RunTransactionAtBlockID(
txBody,
flow.Identifier(txResult.BlockID),
"",
)
}
if txErr != nil {
log.Fatal().Err(txErr).Msg("transaction error")
}
if processErr != nil {
log.Fatal().Err(processErr).Msg("process error")
}
}
Loading

0 comments on commit b89f92d

Please sign in to comment.