-
Notifications
You must be signed in to change notification settings - Fork 68
142 lines (137 loc) · 6 KB
/
devnet_scenarios.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
name: Devnet Scenarios
on:
schedule:
- cron: '0 */2 * * *'
workflow_dispatch:
jobs:
Run:
strategy:
fail-fast: false
matrix:
test:
- FourValidatorsReconnect
- MultipleValidatorsDown
- FourValidatorsReconnectRmDatabase
- FourValidatorsReconnectSpammer
- MacroBlockProduction
- Validators90sDown
- LowBlockProducerTimeout
include:
- test: FourValidatorsReconnect
devnet_args: -t .github/devnet_topologies/four_validators.toml -dt 30
- test: MultipleValidatorsDown
devnet_args: -t .github/devnet_topologies/four_validators.toml -k 2 -ut 100 -dt 30 -as
- test: FourValidatorsReconnectRmDatabase
devnet_args: -t .github/devnet_topologies/four_validators.toml -d -ut 100 -dt 30
- test: FourValidatorsReconnectSpammer
devnet_args: -t .github/devnet_topologies/four_validators_spammer_1.toml -ut 250 -dt 30
- test: MacroBlockProduction
# The number of blocks per epoch needs to stay the same (because of the ZKP keys)
pre: "grep 'blocks_per_batch: 60,' primitives/src/policy.rs &&
grep 'batches_per_epoch: 720,' primitives/src/policy.rs &&
sed -i 's/blocks_per_batch: 60,/blocks_per_batch: 60 \\/ 30,/g' primitives/src/policy.rs &&
sed -i 's/batches_per_epoch: 720,/batches_per_epoch: 720 \\* 30,/g' primitives/src/policy.rs"
devnet_args: -t .github/devnet_topologies/four_validators_spammer_1.toml -k 0 -ut 100
- test: Validators90sDown
devnet_args: -t .github/devnet_topologies/four_validators_spammer_1.toml -dt 90 -ut 100
- test: LowBlockProducerTimeout
pre: "grep 'MIN_PRODUCER_TIMEOUT: u64 = 4 \\* 1000;' primitives/src/policy.rs &&
sed -i 's/MIN_PRODUCER_TIMEOUT: u64 = 4 \\* 1000;/MIN_PRODUCER_TIMEOUT: u64 = 1000;/g' primitives/src/policy.rs"
devnet_args: -t .github/devnet_topologies/four_validators_spammer_1.toml -k 0 -ut 100
runs-on: ubuntu-22.04
steps:
- name: Free Disk Space (Ubuntu)
uses: jlumbroso/free-disk-space@main
with:
tool-cache: true
large-packages: false
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: '3.8'
- name: Install python dependencies
run: pip install scripts/devnet
- name: Set up Rust toolchain
uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2
- name: Optionally patch the source
run: ${{ matrix.pre }}
- name: Build the code
run: cargo build
- name: Retrieve initial timestamp
id: initial_ts
run: |
echo "INITIAL_TS=$(date +%s%N | cut -b1-13)" >> $GITHUB_OUTPUT
- name: Execute the test
run: |
python3 scripts/devnet/devnet.py --run-environment ci ${{ matrix.devnet_args }}
env:
NIMIQ_LOKI_URL: ${{ secrets.LOKI_URL }}
NIMIQ_LOKI_EXTRA_FIELDS: nimiq_ci_name=${{ github.workflow }}:nimiq_ci_run=${{ github.run_number }}:nimiq_test=${{ matrix.test }}
- name: Redact Loki URL
if: always()
run: sed -i 's/url = "[^"]*"/url = "<redacted>"/g' temp-logs/*/conf/*/*.toml
- name: Archive test results
if: always()
uses: actions/upload-artifact@v4
with:
name: ${{ matrix.test }}-logs
path: |
temp-logs/
- name: Retrieve failure reason
if: always()
run: |
if [ -f temp-state/*/RESULT.TXT ]; then
echo "FAIL_REASON=$(cat temp-state/*/RESULT.TXT)" >> $GITHUB_OUTPUT
else
echo "FAIL_REASON=other" >> $GITHUB_OUTPUT
fi
id: reason
- name: Retrieve final timestamp and Run ID
if: always()
id: final_ts_run_id
run: |
echo "FINAL_TS=$(date +%s%N | cut -b1-13)" >> $GITHUB_OUTPUT
echo "RUN_ID=$(cat temp-logs/*/conf/run_id.info)" >> $GITHUB_OUTPUT
- name: Build Grafana dashboard link
if: always()
id: grafana_url
run: |
echo "GRAFANA_URL=http://localhost:3001/d/YbjdvlU7z/nimiq-test?orgId=1&var-env=ci&var-run_id=${{steps.final_ts_run_id.outputs.RUN_ID}}&from=${{steps.initial_ts.outputs.INITIAL_TS}}&to=${{steps.final_ts_run_id.outputs.FINAL_TS}}" >> $GITHUB_OUTPUT
- name: Report potential deadlocks to slack
if: always() && contains(steps.reason.outputs.FAIL_REASON, 'DEADLOCK')
uses: ravsamhq/notify-slack-action@v2
with:
status: ${{ job.status }}
notification_title: 'Potential deadlock detected in ${{ matrix.test }}'
footer: '<{run_url}|View Run>'
env:
SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}
# - name: Report long lock hold times
# if: always() && contains(steps.reason.outputs.FAIL_REASON, 'LONG_LOCK_HOLD_TIME')
# uses: ravsamhq/notify-slack-action@v2
# with:
# status: ${{ job.status }}
# notification_title: 'Long lock hold time in ${{ matrix.test }}'
# footer: '<{run_url}|View Run>'
# env:
# SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}
# - name: Report slow lock acquisitions
# if: always() && contains(steps.reason.outputs.FAIL_REASON, 'SLOW_LOCK_ACQUISITION')
# uses: ravsamhq/notify-slack-action@v2
# with:
# status: ${{ job.status }}
# notification_title: 'Slow lock acquisition detected in ${{ matrix.test }}'
# footer: '<{run_url}|View Run>'
# env:
# SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}
- name: Report Status to Slack
if: always() && github.ref == 'refs/heads/albatross'
uses: ravsamhq/notify-slack-action@v2
with:
status: ${{ job.status }}
notify_when: 'failure'
notification_title: '${{ matrix.test }} failed because of ${{ steps.reason.outputs.FAIL_REASON }}'
footer: '<{run_url}|View Run> | <${{steps.grafana_url.outputs.GRAFANA_URL}}|Grafana dashboard>'
env:
SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}