Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor: persist forkchoice state #1125

Merged
merged 4 commits into from
Jun 3, 2024
Merged

refactor: persist forkchoice state #1125

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
5e03810
refactor: persist forkchoice state
avilagaston9 May 30, 2024
0c5e2de
Add grafana metrics
avilagaston9 May 30, 2024
acc0812
Address comments
avilagaston9 Jun 3, 2024
df85b16
Merge branch 'main' into persist-forkchoice-store
avilagaston9 Jun 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 28 additions & 4 deletions lib/lambda_ethereum_consensus/fork_choice/fork_choice.ex
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,11 +65,13 @@ defmodule LambdaEthereumConsensus.ForkChoice do
:telemetry.execute([:sync, :store], %{slot: Store.get_current_slot(store)})
:telemetry.execute([:sync, :on_block], %{slot: head_slot})

persist_fork_choice_store(store)
{:ok, store}
end

@impl GenServer
def handle_cast({:on_block, block_root, %SignedBeaconBlock{} = signed_block, from}, store) do
def handle_cast({:on_block, block_root, %SignedBeaconBlock{} = signed_block, from}, _store) do
store = fetch_fork_choice_store!()
slot = signed_block.message.slot

Logger.info("[Fork choice] Adding new block", root: block_root, slot: slot)
Expand All @@ -93,6 +95,7 @@ defmodule LambdaEthereumConsensus.ForkChoice do
prune_old_states(last_finalized_checkpoint.epoch, new_finalized_checkpoint.epoch)

GenServer.cast(from, {:block_processed, block_root, true})
persist_fork_choice_store(new_store)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would persist before calling the cast. As a general rule it's useful that the state is stored and consistent before others are made aware of it. As a matter of fact, if we persist it before launching the recompute_head async task, we can remove the persist call in recompute-head.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That's true. Done!

{:noreply, new_store}

{:error, reason} ->
Expand All @@ -103,7 +106,8 @@ defmodule LambdaEthereumConsensus.ForkChoice do
end

@impl GenServer
def handle_cast({:on_attestation, %Attestation{} = attestation}, %Store{} = state) do
def handle_cast({:on_attestation, %Attestation{} = attestation}, %Store{} = _state) do
state = fetch_fork_choice_store!()
id = attestation.signature |> Base.encode16() |> String.slice(0, 8)
Logger.debug("[Fork choice] Adding attestation #{id} to the store")

Expand All @@ -113,12 +117,14 @@ defmodule LambdaEthereumConsensus.ForkChoice do
_ -> state
end

persist_fork_choice_store(state)
{:noreply, state}
end

@impl GenServer
def handle_cast({:attester_slashing, attester_slashing}, state) do
def handle_cast({:attester_slashing, attester_slashing}, _state) do
Logger.info("[Fork choice] Adding attester slashing to the store")
state = fetch_fork_choice_store!()

state =
case Handlers.on_attester_slashing(state, attester_slashing) do
Expand All @@ -130,16 +136,19 @@ defmodule LambdaEthereumConsensus.ForkChoice do
state
end

persist_fork_choice_store(state)
{:noreply, state}
end

@impl GenServer
def handle_cast({:on_tick, time}, store) do
def handle_cast({:on_tick, time}, _store) do
store = fetch_fork_choice_store!()
%Store{finalized_checkpoint: last_finalized_checkpoint} = store

new_store = Handlers.on_tick(store, time)
%Store{finalized_checkpoint: new_finalized_checkpoint} = new_store
prune_old_states(last_finalized_checkpoint.epoch, new_finalized_checkpoint.epoch)
persist_fork_choice_store(new_store)
{:noreply, new_store}
end

Expand Down Expand Up @@ -219,4 +228,19 @@ defmodule LambdaEthereumConsensus.ForkChoice do
Logger.debug("[Fork choice] Store persisted")
end)
end

defp persist_fork_choice_store(store) do
:telemetry.span([:fork_choice, :persist], %{}, fn ->
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let's have the telemetry call on the db methods directly, so they are called no matter who is storing/fetching the store.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done!

{StoreDb.persist_fork_choice_store(store), %{}}
end)
end

defp fetch_fork_choice_store!() do
{:ok, store} =
:telemetry.span([:fork_choice, :fetch], %{}, fn ->
{StoreDb.fetch_fork_choice_store(), %{}}
end)

store
end
end
9 changes: 9 additions & 0 deletions lib/lambda_ethereum_consensus/store/store_db.ex
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,16 +5,25 @@ defmodule LambdaEthereumConsensus.Store.StoreDb do
alias LambdaEthereumConsensus.Store.Db

@store_prefix "store"
@fork_choice_store_prefix "fork_choice_store"
@snapshot_prefix "snapshot"

@spec fetch_store() :: {:ok, Types.Store.t()} | :not_found
def fetch_store(), do: get(@store_prefix)

@spec fetch_fork_choice_store() :: {:ok, Types.Store.t()} | :not_found
def fetch_fork_choice_store(), do: get(@fork_choice_store_prefix)

@spec persist_store(Types.Store.t()) :: :ok
def persist_store(%Types.Store{} = store) do
put(@store_prefix, store)
end

@spec persist_fork_choice_store(Types.Store.t()) :: :ok
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why do we need a separate table? Is it not the same store we're saving?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Merged into a single table!

def persist_fork_choice_store(%Types.Store{} = store) do
put(@fork_choice_store_prefix, store)
end

@spec fetch_deposits_snapshot() :: {:ok, Types.DepositTreeSnapshot.t()} | :not_found
def fetch_deposits_snapshot(), do: get(@snapshot_prefix)

Expand Down
16 changes: 15 additions & 1 deletion lib/lambda_ethereum_consensus/telemetry.ex
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,21 @@ defmodule LambdaEthereumConsensus.Telemetry do
last_value("vm.uptime.total", unit: :millisecond),

# Db Metrics
last_value("db.size.total", unit: :byte)
last_value("db.size.total", unit: :byte),

# ForkChoice Metrics
last_value("fork_choice.persist.stop.duration",
unit: {:native, :millisecond}
),
last_value("fork_choice.persist.exception.duration",
unit: {:native, :millisecond}
),
last_value("fork_choice.fetch.stop.duration",
unit: {:native, :millisecond}
),
last_value("fork_choice.fetch.exception.duration",
unit: {:native, :millisecond}
)
]
end

Expand Down
206 changes: 205 additions & 1 deletion metrics/grafana/provisioning/dashboards/home.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,210 @@
"title": "Database size",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"targets": [
{
"refId": "A",
"expr": "fork_choice_persist_stop_duration",
"range": true,
"instant": true,
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"editorMode": "builder",
"legendFormat": "__auto",
"useBackend": false,
"disableTextWrap": false,
"fullMetaSearch": false,
"includeNullMetadata": true,
"hide": false
}
],
"type": "timeseries",
"title": "ForkChoice Persisting Store Duration",
"gridPos": {
"x": 0,
"y": 0,
"w": 12,
"h": 8
},
"options": {
"tooltip": {
"mode": "single",
"sort": "none",
"maxHeight": 600
},
"legend": {
"showLegend": false,
"displayMode": "list",
"placement": "bottom",
"calcs": []
}
},
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"lineInterpolation": "linear",
"barAlignment": 0,
"lineWidth": 2,
"fillOpacity": 0,
"gradientMode": "none",
"spanNulls": false,
"insertNulls": false,
"showPoints": "auto",
"pointSize": 1,
"stacking": {
"mode": "none",
"group": "A"
},
"axisPlacement": "auto",
"axisLabel": "",
"axisColorMode": "text",
"axisBorderShow": false,
"scaleDistribution": {
"type": "linear"
},
"axisCenteredZero": false,
"hideFrom": {
"tooltip": false,
"viz": false,
"legend": false
},
"thresholdsStyle": {
"mode": "off"
}
},
"color": {
"mode": "palette-classic"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"value": null,
"color": "green"
},
{
"value": 80,
"color": "red"
}
]
},
"unit": "ms"
},
"overrides": []
},
"id": 25
},
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"targets": [
{
"refId": "A",
"expr": "fork_choice_fetch_stop_duration",
"range": true,
"instant": true,
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"editorMode": "builder",
"legendFormat": "__auto",
"useBackend": false,
"disableTextWrap": false,
"fullMetaSearch": false,
"includeNullMetadata": true,
"hide": false
}
],
"type": "timeseries",
"title": "ForkChoice Fetching Store Duration",
"gridPos": {
"x": 0,
"y": 0,
"w": 12,
"h": 8
},
"options": {
"tooltip": {
"mode": "single",
"sort": "none",
"maxHeight": 600
},
"legend": {
"showLegend": false,
"displayMode": "list",
"placement": "bottom",
"calcs": []
}
},
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"lineInterpolation": "linear",
"barAlignment": 0,
"lineWidth": 2,
"fillOpacity": 0,
"gradientMode": "none",
"spanNulls": false,
"insertNulls": false,
"showPoints": "auto",
"pointSize": 1,
"stacking": {
"mode": "none",
"group": "A"
},
"axisPlacement": "auto",
"axisLabel": "",
"axisColorMode": "text",
"axisBorderShow": false,
"scaleDistribution": {
"type": "linear"
},
"axisCenteredZero": false,
"hideFrom": {
"tooltip": false,
"viz": false,
"legend": false
},
"thresholdsStyle": {
"mode": "off"
}
},
"color": {
"mode": "palette-classic"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"value": null,
"color": "green"
},
{
"value": 80,
"color": "red"
}
]
},
"unit": "ms"
},
"overrides": []
},
"id": 26
},
{
"datasource": {
"type": "prometheus",
Expand Down Expand Up @@ -2161,4 +2365,4 @@
"uid": "90EXFQnIk",
"version": 1,
"weekStart": ""
}
}
Loading