Skip to content

Commit

Permalink
m4 timeseries benchmark (#1277)
Browse files Browse the repository at this point in the history
* feat: add new  m4 benchmark results

* chore: update FEDOT metrics on m4

* feat: add friedman conover ttest for m4

* style: add tabulation to stat analysis table

* feat: update amlb classification metrics
  • Loading branch information
Lopa10ko authored Aug 26, 2024
1 parent 17b9593 commit b8740e3
Show file tree
Hide file tree
Showing 4 changed files with 173 additions and 762 deletions.
72 changes: 39 additions & 33 deletions docs/source/benchmarks/amlb_res.csv
Original file line number Diff line number Diff line change
@@ -1,33 +1,39 @@
Dataset name,Metric name,AutoGluon,FEDOT,H2O,LAMA
APSFailure,auc,0.99,0.991,0.992,0.992
Amazon_employee_access,auc,0.857,0.865,0.873,0.879
Australian,auc,0.94,0.939,0.939,0.945
Covertype,neg_logloss,-0.071,-0.117,-0.265,
Fashion-MNIST,neg_logloss,-0.329,-0.373,-0.38,-0.248
Jannis,neg_logloss,-0.728,-0.737,-0.691,-0.664
KDDCup09_appetency,auc,0.804,0.822,0.829,0.85
MiniBooNE,auc,0.982,0.981,,0.988
Shuttle,neg_logloss,-0.001,-0.001,-0.0,-0.001
Volkert,neg_logloss,-0.917,-1.097,-0.976,-0.806
adult,auc,0.91,0.925,0.931,0.932
bank-marketing,auc,0.931,0.935,0.939,0.94
blood-transfusion,auc,0.69,0.759,0.765,0.75
car,neg_logloss,-0.117,-0.011,-0.004,-0.002
christine,auc,0.804,0.812,0.823,0.83
cnae-9,neg_logloss,-0.332,-0.211,-0.175,-0.156
connect-4,neg_logloss,-0.502,-0.456,-0.338,-0.337
credit-g,auc,0.795,0.778,0.789,0.796
dilbert,neg_logloss,-0.148,-0.159,-0.05,-0.033
fabert,neg_logloss,-0.788,-0.895,-0.752,-0.766
guillermo,auc,0.9,0.891,,0.926
jasmine,auc,0.883,0.888,0.887,0.88
jungle chess,neg_logloss,-0.431,-0.193,-0.24,-0.149
kc1,auc,0.822,0.843,,0.831
kr-vs-kp,auc,0.999,1.0,,1.0
mfeat-factors,neg_logloss,-0.161,-0.094,,-0.082
nomao,auc,0.995,0.994,0.996,0.997
numerai28_6,auc,0.517,0.529,0.531,0.531
phoneme,auc,0.965,0.965,,0.965
segment,neg_logloss,-0.094,-0.062,,-0.061
sylvine,auc,0.985,0.988,,0.988
vehicle,neg_logloss,-0.515,-0.354,,-0.404
Dataset,Metric,AutoGluon,FEDOT,H2O,TPOT
adult,auc,0.9100126,0.91529255,0.9307700000000001,0.9272897999999999
airlines,auc,0.7249085714285715,0.6537803999999999,0.7303896,0.693676
albert,auc,0.739028,0.7276503,,
amazon_employee_access,auc,0.8571479999999999,0.8591113,0.8728077000000001,0.8662471
apsfailure,auc,0.9906209,0.9899874210526317,0.9925172,0.990437
australian,auc,0.9395274,0.9378541,0.93857085,0.9360440999999999
bank-marketing,auc,0.9312558,0.93245125,0.9385977000000001,0.9346086
blood-transfusion,auc,0.6895855,0.72444385,0.75949435,0.7401904
christine,auc,0.8042872000000001,0.8044556500000001,0.8193608421052632,0.8066902
credit-g,auc,0.7952859,0.7845833,0.79357155,0.7938096
guillermo,auc,0.8996748,0.89125215,,0.7833095714285714
jasmine,auc,0.8831222000000001,0.88548405,0.8873440499999999,0.8903762000000001
kc1,auc,0.8222621,0.8385662,,0.8448118000000001
kddcup09_appetency,auc,0.8044676000000001,0.7877767,0.8291237,0.825562
kr-vs-kp,auc,0.9988583999999999,0.9992477,0.9997232,0.9997627
miniboone,auc,0.9821717,0.98101815,,0.9834643333333334
nomao,auc,0.9948282,0.99419515,0.9959996,0.9953825
numerai28_6,auc,0.5165548,0.5216116000000001,0.5305179,
phoneme,auc,0.9654223,0.9644835,0.9675107000000001,0.970699
riccardo,auc,0.9997026,0.9979384,,
sylvine,auc,0.9847037999999999,0.9849627999999999,0.9893596,0.9933923
car,neg_logloss,-0.11658660000000001,-0.088851992,-0.003471899925,-0.64257486468
cnae-9,neg_logloss,-0.332075,-0.270096135,-0.21849159,-0.15368975
connect-4,neg_logloss,-0.5015701,-0.47033240000000004,-0.33770059999999996,-0.3734921
covertype,neg_logloss,-0.07139724444444445,-0.1409624,-0.2642175,
dilbert,neg_logloss,-0.14967388235294118,-0.24454559000000003,-0.07642755500000001,-0.168390625
dionis,neg_logloss,-2.157603,,,
fabert,neg_logloss,-0.7878137,-0.9015242000000001,-0.77193945,-0.8915912
fashion-mnist,neg_logloss,-0.3325671,-0.38379342857142856,-0.3832832,-0.535493
helena,neg_logloss,-2.784965,-6.348634,-2.9801966666666666,-2.98157375
jannis,neg_logloss,-0.7283778,-0.7619161,-0.691228,-0.703102
jungle_chess_2pcs_raw_endgame_complete,neg_logloss,-0.43063529999999994,-0.270741845,-0.23951890000000003,-0.21872090000000002
mfeat-factors,neg_logloss,-0.1611791,-0.17412199,-0.09295753,-0.10726150999999999
robert,neg_logloss,-1.6843139999999999,-1.745091,,
segment,neg_logloss,-0.09418663,-0.096434561,-0.05962082,-0.07710542000000001
shuttle,neg_logloss,-0.0008124975,-0.0010121353499999998,-0.00035519797666666667,
vehicle,neg_logloss,-0.5154588,-0.42775929999999995,-0.3313683,-0.3915049
volkert,neg_logloss,-0.9200727000000001,-1.0448454545454544,-0.9779738888888888,
Loading

0 comments on commit b8740e3

Please sign in to comment.