From f5f102186d19d77cfb97ae7f4ff60ae64e777a68 Mon Sep 17 00:00:00 2001 From: Samuel Burbulla Date: Wed, 17 Apr 2024 18:46:01 +0200 Subject: [PATCH] Refactor benchmark table. --- benchmarks/README.md | 19 +- benchmarks/html/img/.gitkeep | 0 benchmarks/html/img/SineRegular_BelNet.svg | 1074 ----------------- .../img/SineRegular_DeepNeuralOperator.svg | 1074 ----------------- benchmarks/html/img/SineRegular_DeepONet.svg | 1074 ----------------- .../img/SineRegular_FourierNeuralOperator.svg | 1074 ----------------- benchmarks/html/img/SineUniform_BelNet.svg | 1074 ----------------- .../img/SineUniform_DeepNeuralOperator.svg | 1074 ----------------- benchmarks/html/img/SineUniform_DeepONet.svg | 1074 ----------------- .../img/SineUniform_FourierNeuralOperator.svg | 1074 ----------------- benchmarks/html/style.css | 50 - benchmarks/html/table.html | 25 - benchmarks/process.py | 6 - benchmarks/results/data.json | 98 ++ benchmarks/results/process.py | 59 + benchmarks/{ => sine}/run_all.py | 0 build_scripts/copy_benchmarks.py | 12 +- docs/benchmarks/.gitignore | 6 +- docs/benchmarks/img/.gitattributes | 5 + .../benchmarks/img}/SineRegular.svg | 0 .../benchmarks/img}/SineUniform.svg | 0 .../benchmarks/img}/navierstokes.png | 0 .../benchmarks}/img/ns_test_144.png | 0 .../benchmarks}/img/ns_test_179.png | 0 .../benchmarks}/img/ns_train_237.png | 0 .../benchmarks}/img/ns_train_420.png | 0 docs/benchmarks/index.md | 2 +- docs/benchmarks/tables.html | 83 ++ src/continuiti/benchmarks/navierstokes.py | 2 +- src/continuiti/benchmarks/run/table.py | 145 --- src/continuiti/benchmarks/sine.py | 4 +- tests/benchmarks/test_navierstokes.py | 2 +- 32 files changed, 265 insertions(+), 8845 deletions(-) delete mode 100644 benchmarks/html/img/.gitkeep delete mode 100644 benchmarks/html/img/SineRegular_BelNet.svg delete mode 100644 benchmarks/html/img/SineRegular_DeepNeuralOperator.svg delete mode 100644 benchmarks/html/img/SineRegular_DeepONet.svg delete mode 100644 benchmarks/html/img/SineRegular_FourierNeuralOperator.svg delete mode 100644 benchmarks/html/img/SineUniform_BelNet.svg delete mode 100644 benchmarks/html/img/SineUniform_DeepNeuralOperator.svg delete mode 100644 benchmarks/html/img/SineUniform_DeepONet.svg delete mode 100644 benchmarks/html/img/SineUniform_FourierNeuralOperator.svg delete mode 100644 benchmarks/html/style.css delete mode 100644 benchmarks/html/table.html delete mode 100644 benchmarks/process.py create mode 100644 benchmarks/results/data.json create mode 100644 benchmarks/results/process.py rename benchmarks/{ => sine}/run_all.py (100%) create mode 100644 docs/benchmarks/img/.gitattributes rename {benchmarks/html => docs/benchmarks/img}/SineRegular.svg (100%) rename {benchmarks/html => docs/benchmarks/img}/SineUniform.svg (100%) rename {benchmarks/html => docs/benchmarks/img}/navierstokes.png (100%) rename {benchmarks/html => docs/benchmarks}/img/ns_test_144.png (100%) rename {benchmarks/html => docs/benchmarks}/img/ns_test_179.png (100%) rename {benchmarks/html => docs/benchmarks}/img/ns_train_237.png (100%) rename {benchmarks/html => docs/benchmarks}/img/ns_train_420.png (100%) create mode 100644 docs/benchmarks/tables.html delete mode 100644 src/continuiti/benchmarks/run/table.py diff --git a/benchmarks/README.md b/benchmarks/README.md index c44df600..a725daad 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -2,20 +2,20 @@ ## Run -Run the `run_all.py` script to run all benchmarks. +If you want to evaluate a single benchmark, adopt the `run_single.py` script +and (within the `benchmarks` directory) run ```bash -python run_all.py +python run_single.py ``` -If you only want to evaluate a single benchmark, adopt the `run_single.py` script -and run +To run a specific benchmark, e.g., the sine benchmarks, run ```bash -python run_single.py +python sine/run_all.py ``` -## Visualize +## MLFlow We use MLFlow to log the benchmark runs and you can run @@ -25,8 +25,11 @@ mlflow ui to visualize the logged experiments in your browser. -In order to visualize the best runs for every benchmark in an HTML table, run +## Documentation + +In order to process the data of the best runs for the benchmark page in +the documentation, run ```bash -python process.py +python results/process.py ``` diff --git a/benchmarks/html/img/.gitkeep b/benchmarks/html/img/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/benchmarks/html/img/SineRegular_BelNet.svg b/benchmarks/html/img/SineRegular_BelNet.svg deleted file mode 100644 index 2b679da1..00000000 --- a/benchmarks/html/img/SineRegular_BelNet.svg +++ /dev/null @@ -1,1074 +0,0 @@ - - - - - - - - 2024-04-02T17:23:23.507780 - image/svg+xml - - - Matplotlib v3.8.3, https://matplotlib.org/ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/benchmarks/html/img/SineRegular_DeepNeuralOperator.svg b/benchmarks/html/img/SineRegular_DeepNeuralOperator.svg deleted file mode 100644 index d28df9ba..00000000 --- a/benchmarks/html/img/SineRegular_DeepNeuralOperator.svg +++ /dev/null @@ -1,1074 +0,0 @@ - - - - - - - - 2024-04-02T17:23:23.494453 - image/svg+xml - - - Matplotlib v3.8.3, https://matplotlib.org/ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/benchmarks/html/img/SineRegular_DeepONet.svg b/benchmarks/html/img/SineRegular_DeepONet.svg deleted file mode 100644 index 60c67bfb..00000000 --- a/benchmarks/html/img/SineRegular_DeepONet.svg +++ /dev/null @@ -1,1074 +0,0 @@ - - - - - - - - 2024-04-02T17:23:23.560487 - image/svg+xml - - - Matplotlib v3.8.3, https://matplotlib.org/ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/benchmarks/html/img/SineRegular_FourierNeuralOperator.svg b/benchmarks/html/img/SineRegular_FourierNeuralOperator.svg deleted file mode 100644 index 8ce9d3f3..00000000 --- a/benchmarks/html/img/SineRegular_FourierNeuralOperator.svg +++ /dev/null @@ -1,1074 +0,0 @@ - - - - - - - - 2024-04-02T17:23:23.480146 - image/svg+xml - - - Matplotlib v3.8.3, https://matplotlib.org/ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/benchmarks/html/img/SineUniform_BelNet.svg b/benchmarks/html/img/SineUniform_BelNet.svg deleted file mode 100644 index c4ac2107..00000000 --- a/benchmarks/html/img/SineUniform_BelNet.svg +++ /dev/null @@ -1,1074 +0,0 @@ - - - - - - - - 2024-04-02T17:23:23.587877 - image/svg+xml - - - Matplotlib v3.8.3, https://matplotlib.org/ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/benchmarks/html/img/SineUniform_DeepNeuralOperator.svg b/benchmarks/html/img/SineUniform_DeepNeuralOperator.svg deleted file mode 100644 index 4317a64c..00000000 --- a/benchmarks/html/img/SineUniform_DeepNeuralOperator.svg +++ /dev/null @@ -1,1074 +0,0 @@ - - - - - - - - 2024-04-02T17:23:23.574525 - image/svg+xml - - - Matplotlib v3.8.3, https://matplotlib.org/ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/benchmarks/html/img/SineUniform_DeepONet.svg b/benchmarks/html/img/SineUniform_DeepONet.svg deleted file mode 100644 index 8ed74200..00000000 --- a/benchmarks/html/img/SineUniform_DeepONet.svg +++ /dev/null @@ -1,1074 +0,0 @@ - - - - - - - - 2024-04-02T17:23:23.601280 - image/svg+xml - - - Matplotlib v3.8.3, https://matplotlib.org/ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/benchmarks/html/img/SineUniform_FourierNeuralOperator.svg b/benchmarks/html/img/SineUniform_FourierNeuralOperator.svg deleted file mode 100644 index 66fb11b2..00000000 --- a/benchmarks/html/img/SineUniform_FourierNeuralOperator.svg +++ /dev/null @@ -1,1074 +0,0 @@ - - - - - - - - 2024-04-02T17:23:23.614466 - image/svg+xml - - - Matplotlib v3.8.3, https://matplotlib.org/ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/benchmarks/html/style.css b/benchmarks/html/style.css deleted file mode 100644 index 95142db6..00000000 --- a/benchmarks/html/style.css +++ /dev/null @@ -1,50 +0,0 @@ -.benchmark-table { - border-collapse: collapse; - background: #fff; - color: #000; - overflow: hidden; - width: 100%; - margin: 0 auto; - position: relative; - box-shadow: 0px 5px 15px #000; - font: 1em sans-serif; -} - -.benchmark-table td, table th { - padding-left: 12px; - border: 0; -} - -.benchmark-table thead tr { - height: 40px; - background: #232323; - color: #fff; -} - -.benchmark-table tbody tr { - height: 50px; - font-size: 15px; - color: 000; - line-height: 1.2; -} - -.benchmark-table td { - text-align: left; - vertical-align: middle; - min-width: 90px; -} - -.benchmark-table th { - text-align: left; - vertical-align: middle; -} - -.benchmark-table td:nth-of-type(2n-1) { - background-color: #dbdbdb; -} - -.div-params { - font-size: 6pt; - font-weight: lighter; - color: #232323; -} diff --git a/benchmarks/html/table.html b/benchmarks/html/table.html deleted file mode 100644 index 80857fad..00000000 --- a/benchmarks/html/table.html +++ /dev/null @@ -1,25 +0,0 @@ - -

SineRegular

- - - - - - - - - - -
OperatorParamsLearning Curveloss/trainloss/test
FourierNeuralOperator
(depth=1, width=4)
3053.43e-153.61e-15
DeepNeuralOperator
(depth=32, width=64)
1332496.95e-077.36e-07
BelNet
(D_1=4, D_2=8, K=8, N_1=16, N_2=16)
77681.77e-051.81e-05
DeepONet
(basis_functions=32, branch_depth=8, branch_width=32, trunk_depth=8, trunk_width=32)
180162.5e-052.57e-05
-

SineUniform

- - - - - - - - - - -
OperatorParamsLearning Curveloss/trainloss/test
DeepNeuralOperator
(depth=8, width=64)
334090.0002390.000397
BelNet
(D_1=8, D_2=4, K=16, N_1=16, N_2=8)
115120.0003170.000572
DeepONet
(basis_functions=32, branch_depth=8, branch_width=32, trunk_depth=8, trunk_width=32)
180160.003130.00533
FourierNeuralOperator
(depth=3, width=4)
8890.1990.207
diff --git a/benchmarks/process.py b/benchmarks/process.py deleted file mode 100644 index 413cdb52..00000000 --- a/benchmarks/process.py +++ /dev/null @@ -1,6 +0,0 @@ -from continuiti.benchmarks.run.table import BenchmarkTable - - -if __name__ == "__main__": - table = BenchmarkTable() - table.write_html() diff --git a/benchmarks/results/data.json b/benchmarks/results/data.json new file mode 100644 index 00000000..a124bba9 --- /dev/null +++ b/benchmarks/results/data.json @@ -0,0 +1,98 @@ +{ + "SineRegular": [ + { + "operator": "FourierNeuralOperator", + "num_params": "305", + "loss/train": "3.43e-15", + "loss/test": "3.61e-15", + "params": { + "depth": "1", + "width": "4" + } + }, + { + "operator": "DeepNeuralOperator", + "num_params": "133249", + "loss/train": "6.95e-07", + "loss/test": "7.36e-07", + "params": { + "depth": "32", + "width": "64" + } + }, + { + "operator": "BelNet", + "num_params": "7768", + "loss/train": "1.77e-05", + "loss/test": "1.81e-05", + "params": { + "D_1": "4", + "D_2": "8", + "K": "8", + "N_1": "16", + "N_2": "16" + } + }, + { + "operator": "DeepONet", + "num_params": "18016", + "loss/train": "2.50e-05", + "loss/test": "2.57e-05", + "params": { + "basis_functions": "32", + "branch_depth": "8", + "branch_width": "32", + "trunk_depth": "8", + "trunk_width": "32" + } + } + ], + "SineUniform": [ + { + "operator": "DeepNeuralOperator", + "num_params": "33409", + "loss/train": "0.000239", + "loss/test": "0.000397", + "params": { + "depth": "8", + "width": "64" + } + }, + { + "operator": "BelNet", + "num_params": "11512", + "loss/train": "0.000317", + "loss/test": "0.000572", + "params": { + "D_1": "8", + "D_2": "4", + "K": "16", + "N_1": "16", + "N_2": "8" + } + }, + { + "operator": "DeepONet", + "num_params": "18016", + "loss/train": "0.00313", + "loss/test": "0.00533", + "params": { + "basis_functions": "32", + "branch_depth": "8", + "branch_width": "32", + "trunk_depth": "8", + "trunk_width": "32" + } + }, + { + "operator": "FourierNeuralOperator", + "num_params": "889", + "loss/train": "0.199", + "loss/test": "0.207", + "params": { + "depth": "3", + "width": "4" + } + } + ] +} diff --git a/benchmarks/results/process.py b/benchmarks/results/process.py new file mode 100644 index 00000000..80e2e991 --- /dev/null +++ b/benchmarks/results/process.py @@ -0,0 +1,59 @@ +import mlflow +import json +import pandas as pd + + +def process(): + client = mlflow.MlflowClient() + experiments = client.search_experiments() + experiment_ids = [e.experiment_id for e in experiments] + runs = client.search_runs(experiment_ids) + + all_runs_dicts = [] + for run in runs: + all_runs_dicts.append( + { + "benchmark": run.data.tags["benchmark"], + "operator": run.data.tags["operator"], + "num_params": run.data.metrics["num_params"], + "loss/train": run.data.metrics["loss/train"], + "loss/test": run.data.metrics["loss/test"], + "params": run.data.params, + } + ) + + df = pd.concat([pd.DataFrame([r]) for r in all_runs_dicts]) + + filter_params = ["tol", "seed", "max_epochs", "lr", "batch_size"] + + data = {} + benchmarks = sorted(df["benchmark"].unique()) + for bm in benchmarks: + data[bm] = [] + + df_benchmark = df[df["benchmark"] == bm] + operators = sorted(df_benchmark["operator"].unique()) + + for operator in operators: + operator_data = df_benchmark[df_benchmark["operator"] == operator] + + # Get the best operator run for this benchmark + best = operator_data.sort_values("loss/test").iloc[0].to_dict() + + # Filter out parameters + best["params"] = { + k: v for k, v in best["params"].items() if k not in filter_params + } + + best["num_params"] = f'{int(best["num_params"])}' + best["loss/train"] = f'{best["loss/train"]:.2e}' + best["loss/test"] = f'{best["loss/test"]:.2e}' + + data[bm].append(best) + + with open("results/data.json", "w") as f: + json.dump(data, f, indent=2) + + +if __name__ == "__main__": + process() diff --git a/benchmarks/run_all.py b/benchmarks/sine/run_all.py similarity index 100% rename from benchmarks/run_all.py rename to benchmarks/sine/run_all.py diff --git a/build_scripts/copy_benchmarks.py b/build_scripts/copy_benchmarks.py index c776881d..0c32d153 100644 --- a/build_scripts/copy_benchmarks.py +++ b/build_scripts/copy_benchmarks.py @@ -7,15 +7,15 @@ logger = logging.getLogger(__name__) root_dir = Path(__file__).parent.parent +benchmarks_dir = root_dir / "benchmarks" / "results" docs_benchmarks_dir = root_dir / "docs" / "benchmarks" -benchmarks_dir = root_dir / "benchmarks" / "html" @mkdocs.plugins.event_priority(100) def on_pre_build(config): logger.info("Temporarily copying benchmark results to docs directory") docs_benchmarks_dir.mkdir(parents=True, exist_ok=True) - filepaths = list(benchmarks_dir.glob("*")) + filepaths = list(benchmarks_dir.glob("*.json")) for file in filepaths: target_filepath = docs_benchmarks_dir / file.name @@ -38,11 +38,5 @@ def on_pre_build(config): @mkdocs.plugins.event_priority(-100) def on_shutdown(): logger.info("Removing temporary examples directory") - for file in docs_benchmarks_dir.glob("*.html"): - file.unlink() - for file in docs_benchmarks_dir.glob("*.css"): - file.unlink() - for file in docs_benchmarks_dir.glob("*.png"): - file.unlink() - for file in docs_benchmarks_dir.glob("*.svg"): + for file in docs_benchmarks_dir.glob("*.json"): file.unlink() diff --git a/docs/benchmarks/.gitignore b/docs/benchmarks/.gitignore index 6771891a..2d46485a 100644 --- a/docs/benchmarks/.gitignore +++ b/docs/benchmarks/.gitignore @@ -1,5 +1 @@ -table.html -style.css -img -*.png -*.svg +data.json diff --git a/docs/benchmarks/img/.gitattributes b/docs/benchmarks/img/.gitattributes new file mode 100644 index 00000000..9e2b4ad7 --- /dev/null +++ b/docs/benchmarks/img/.gitattributes @@ -0,0 +1,5 @@ +navierstokes.png filter=lfs diff=lfs merge=lfs -text +ns_test_144.png filter=lfs diff=lfs merge=lfs -text +ns_test_179.png filter=lfs diff=lfs merge=lfs -text +ns_train_237.png filter=lfs diff=lfs merge=lfs -text +ns_train_420.png filter=lfs diff=lfs merge=lfs -text diff --git a/benchmarks/html/SineRegular.svg b/docs/benchmarks/img/SineRegular.svg similarity index 100% rename from benchmarks/html/SineRegular.svg rename to docs/benchmarks/img/SineRegular.svg diff --git a/benchmarks/html/SineUniform.svg b/docs/benchmarks/img/SineUniform.svg similarity index 100% rename from benchmarks/html/SineUniform.svg rename to docs/benchmarks/img/SineUniform.svg diff --git a/benchmarks/html/navierstokes.png b/docs/benchmarks/img/navierstokes.png similarity index 100% rename from benchmarks/html/navierstokes.png rename to docs/benchmarks/img/navierstokes.png diff --git a/benchmarks/html/img/ns_test_144.png b/docs/benchmarks/img/ns_test_144.png similarity index 100% rename from benchmarks/html/img/ns_test_144.png rename to docs/benchmarks/img/ns_test_144.png diff --git a/benchmarks/html/img/ns_test_179.png b/docs/benchmarks/img/ns_test_179.png similarity index 100% rename from benchmarks/html/img/ns_test_179.png rename to docs/benchmarks/img/ns_test_179.png diff --git a/benchmarks/html/img/ns_train_237.png b/docs/benchmarks/img/ns_train_237.png similarity index 100% rename from benchmarks/html/img/ns_train_237.png rename to docs/benchmarks/img/ns_train_237.png diff --git a/benchmarks/html/img/ns_train_420.png b/docs/benchmarks/img/ns_train_420.png similarity index 100% rename from benchmarks/html/img/ns_train_420.png rename to docs/benchmarks/img/ns_train_420.png diff --git a/docs/benchmarks/index.md b/docs/benchmarks/index.md index bc3413e3..9a7f2d27 100644 --- a/docs/benchmarks/index.md +++ b/docs/benchmarks/index.md @@ -39,4 +39,4 @@ rel. error = 4.4655e-01 -{% include 'benchmarks/table.html' %} +{% include 'benchmarks/tables.html' %} diff --git a/docs/benchmarks/tables.html b/docs/benchmarks/tables.html new file mode 100644 index 00000000..20f12c54 --- /dev/null +++ b/docs/benchmarks/tables.html @@ -0,0 +1,83 @@ + + +
diff --git a/src/continuiti/benchmarks/navierstokes.py b/src/continuiti/benchmarks/navierstokes.py index 66c4e304..e220985f 100644 --- a/src/continuiti/benchmarks/navierstokes.py +++ b/src/continuiti/benchmarks/navierstokes.py @@ -38,7 +38,7 @@ class NavierStokes(Benchmark): steps $(-0.9, -0.8, ..., 0.0)$ and the output function by the vorticity field at the following ten time steps $(0.1, 0.2, ..., 1.0)$. - ![Visualization of first training sample.](/continuiti/benchmarks/navierstokes.png) + ![Visualization of first training sample.](/continuiti/benchmarks/img/navierstokes.png) The datasets have the following shapes: diff --git a/src/continuiti/benchmarks/run/table.py b/src/continuiti/benchmarks/run/table.py deleted file mode 100644 index 68d8aa42..00000000 --- a/src/continuiti/benchmarks/run/table.py +++ /dev/null @@ -1,145 +0,0 @@ -import mlflow -import pandas as pd -import matplotlib.pyplot as plt - - -class BenchmarkTable: - def __init__(self): - self.keys = [ - "loss/train", - "loss/test", - ] - - self.client = mlflow.MlflowClient() - experiments = self.client.search_experiments() - experiment_ids = [e.experiment_id for e in experiments] - self.runs = self.client.search_runs(experiment_ids) - - def as_data_frame(self) -> pd.DataFrame: - all_runs_dicts = [] - - for run in self.runs: - history = self.client.get_metric_history(run.info.run_id, "loss/train") - loss_history = [m.value for m in history] - - all_runs_dicts.append( - { - "Benchmark": run.data.tags["benchmark"], - "Operator": run.data.tags["operator"], - "num_params": run.data.metrics["num_params"], - "loss/train": run.data.metrics["loss/train"], - "loss/test": run.data.metrics["loss/test"], - "train_history": loss_history, - "max_epochs": run.data.params["max_epochs"], - "params": run.data.params, - } - ) - - return pd.concat([pd.DataFrame([r]) for r in all_runs_dicts]) - - def by_benchmark_and_operator(self) -> dict: - processed = {} - df = self.as_data_frame() - - benchmarks = df["Benchmark"].unique() - - for bm in benchmarks: - mask = df["Benchmark"] == bm - processed[bm] = df.loc[mask] - - return processed - - def generate_loss_plot(self, operator_data) -> str: - bm = str(operator_data["Benchmark"].values[0]) - op = str(operator_data["Operator"].values[0]) - - fig, ax = plt.subplots(figsize=(5, 2)) - filename = f"img/{bm}_{op}.svg" - - max_epochs = int(operator_data["max_epochs"].iloc[0]) - ax.hlines(1e-5, 0, max_epochs, "black", "--") - - # Plot all runs - train_history = operator_data["train_history"].values - for i in range(1, len(train_history)): - ax.plot(range(len(train_history[i])), train_history[i], "k-", alpha=0.1) - - # Plot the best run - ax.plot(range(len(train_history[0])), train_history[0], "k-") - - ax.axis("off") - plt.xlim(0, max_epochs) - plt.yscale("log") - plt.tight_layout() - fig.savefig("html/" + filename) - return filename - - def write_html(self): - filename = "html/table.html" - by_benchmark_and_operator = self.by_benchmark_and_operator() - - benchmarks = sorted(list(by_benchmark_and_operator.keys())) - - # Path to the API documentation - path = "../api/continuiti/" - - table = '\n' - for bm in benchmarks: - benchmark_data = by_benchmark_and_operator[bm] - - table += f'

{bm}

\n' - - table += '\n\n' - table += "" - - for key in self.keys: - table += f"" - table += "\n\n\n" - - # Sort by test loss - benchmark_data = benchmark_data.sort_values("loss/test") - - visited = set() - - for i, op in enumerate(benchmark_data["Operator"]): - if op in visited: - continue - else: - visited.add(op) - - # Show only the best run for each operator - sorted_data = benchmark_data[benchmark_data["Operator"] == op] - operator_data = sorted_data.iloc[0] - - params_dict = operator_data["params"] - exclude_params = ["max_epochs", "seed", "lr", "tol", "batch_size"] - params_dict = { - k: v for k, v in params_dict.items() if k not in exclude_params - } - sorted_keys = sorted(params_dict.keys()) - param_str = ", ".join([f"{k}={params_dict[k]}" for k in sorted_keys]) - table += ( - f'' - - num_weights = operator_data["num_params"] - table += f"" - - loss_plot = self.generate_loss_plot(sorted_data) - table += f'' - - for key in self.keys: - v = operator_data[key] - if key in ["epoch"]: - table += f"" - elif key in ["loss/test"] and i == 0: - table += f"" - else: - table += f"" - - table += "\n" - table += "\n
OperatorParamsLearning Curve{key}
{op}
({param_str})
{int(num_weights)}{int(v)}{v:.3g}{v:.3g}
\n" - - with open(filename, "w") as f: - f.write(table) diff --git a/src/continuiti/benchmarks/sine.py b/src/continuiti/benchmarks/sine.py index a157d377..73469c25 100644 --- a/src/continuiti/benchmarks/sine.py +++ b/src/continuiti/benchmarks/sine.py @@ -83,7 +83,7 @@ class SineRegular(SineBenchmark): - `n_test` is 1024. - `uniform` is `False`. - ![Visualizations of a few samples.](/continuiti/benchmarks/SineRegular.svg) + ![Visualizations of a few samples.](/continuiti/benchmarks/img/SineRegular.svg) """ @@ -109,7 +109,7 @@ class SineUniform(SineBenchmark): - `n_test` is 4096. - `uniform` is `True`. - ![Visualizations of a few samples.](/continuiti/benchmarks/SineUniform.svg) + ![Visualizations of a few samples.](/continuiti/benchmarks/img/SineUniform.svg) """ diff --git a/tests/benchmarks/test_navierstokes.py b/tests/benchmarks/test_navierstokes.py index 3e70ce4a..b7cf5a56 100644 --- a/tests/benchmarks/test_navierstokes.py +++ b/tests/benchmarks/test_navierstokes.py @@ -47,6 +47,6 @@ def test_navierstokes_shapes_and_plot(): axs[1].set_title("Output") try: - fig.savefig("docs/benchmarks/navierstokes.png", dpi=500) + fig.savefig("docs/benchmarks/img/navierstokes.png", dpi=500) except FileNotFoundError: pass