From e3535bbef58443efbc86c7e7a3d6925be1b33ed6 Mon Sep 17 00:00:00 2001 From: Gabrian Giovano Tanasal Date: Sun, 23 Jun 2024 01:13:25 +0200 Subject: [PATCH 01/14] initial draft for LongerCaller. --- .../perf_tests/feature_perf_cs_collection.py | 62 ++++++++++++++++++- 1 file changed, 60 insertions(+), 2 deletions(-) diff --git a/varats/varats/projects/perf_tests/feature_perf_cs_collection.py b/varats/varats/projects/perf_tests/feature_perf_cs_collection.py index 07e1fadb0..3e484026d 100644 --- a/varats/varats/projects/perf_tests/feature_perf_cs_collection.py +++ b/varats/varats/projects/perf_tests/feature_perf_cs_collection.py @@ -231,6 +231,64 @@ def recompile(self) -> None: _do_feature_perf_cs_collection_recompile(self) +# Create Longer Caller Project +class LongerCaller(VProject): + """Main Class longer than function called.""" + + NAME = 'LongerCaller' + GROUP = 'perf_tests' + DOMAIN = ProjectDomains.TEST + + SOURCE = [ + bb.source.Git( + remote="https://github.com/se-sic/FeaturePerfCSCollection.git", + local=NAME, + refspec="origin/HEAD", + limit=None, + shallow=False, + version_filter=project_filter_generator(NAME) + ), + FeatureSource() + ] + + WORKLOADS = { + WorkloadSet(WorkloadCategory.EXAMPLE): [ + VCommand( + SourceRoot(NAME) / RSBinary("CTCRTP"), label="CompileTime-CRTP" + ) + ] + } + + @staticmethod + def binaries_for_revision( + revision: ShortCommitHash # pylint: disable=W0613 + ) -> tp.List[ProjectBinaryWrapper]: + binary_map = RevisionBinaryMap( + get_local_project_git_path(SynthCTCRTP.NAME) + ) + + binary_map.specify_binary( + "build/bin/CTCRTP", + BinaryType.EXECUTABLE, + only_valid_in=RevisionRange("6d50a6efd5", "master") + ) + + return binary_map[revision] + + def run_tests(self) -> None: + pass + + def compile(self) -> None: + """Compile the project.""" + _do_feature_perf_cs_collection_compile( + self, "FPCSC_ENABLE_PROJECT_LONGERCALLER" + ) + + def recompile(self) -> None: + """Recompile the project.""" + _do_feature_perf_cs_collection_recompile(self) + + class SynthSAContextSensitivity(VProject): """Synthetic case-study project for testing flow sensitivity.""" @@ -1227,11 +1285,11 @@ def binaries_for_revision( revision: ShortCommitHash # pylint: disable=W0613 ) -> tp.List[ProjectBinaryWrapper]: binary_map = RevisionBinaryMap( - get_local_project_git_path(SynthCTCRTP.NAME) + get_local_project_git_path(LongerCaller.NAME) ) binary_map.specify_binary( - "build/bin/CTCRTP", + "build/bin/LongerCaller", BinaryType.EXECUTABLE, only_valid_in=RevisionRange("6d50a6efd5", "master") ) From 0ec18bfd165e54bf4401575507ced4c67fe7e74e Mon Sep 17 00:00:00 2001 From: Gabrian Giovano Tanasal Date: Sun, 23 Jun 2024 01:19:35 +0200 Subject: [PATCH 02/14] changes on SYNTHCTCRTP --- .../projects/perf_tests/feature_perf_cs_collection.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/varats/varats/projects/perf_tests/feature_perf_cs_collection.py b/varats/varats/projects/perf_tests/feature_perf_cs_collection.py index 3e484026d..cdd216d48 100644 --- a/varats/varats/projects/perf_tests/feature_perf_cs_collection.py +++ b/varats/varats/projects/perf_tests/feature_perf_cs_collection.py @@ -254,7 +254,7 @@ class LongerCaller(VProject): WORKLOADS = { WorkloadSet(WorkloadCategory.EXAMPLE): [ VCommand( - SourceRoot(NAME) / RSBinary("CTCRTP"), label="CompileTime-CRTP" + SourceRoot(NAME) / RSBinary("LongerCaller"), label="CompileTime-CRTP" ) ] } @@ -264,11 +264,11 @@ def binaries_for_revision( revision: ShortCommitHash # pylint: disable=W0613 ) -> tp.List[ProjectBinaryWrapper]: binary_map = RevisionBinaryMap( - get_local_project_git_path(SynthCTCRTP.NAME) + get_local_project_git_path(LongerCaller.NAME) ) binary_map.specify_binary( - "build/bin/CTCRTP", + "build/bin/LongerCaller", BinaryType.EXECUTABLE, only_valid_in=RevisionRange("6d50a6efd5", "master") ) @@ -1285,11 +1285,11 @@ def binaries_for_revision( revision: ShortCommitHash # pylint: disable=W0613 ) -> tp.List[ProjectBinaryWrapper]: binary_map = RevisionBinaryMap( - get_local_project_git_path(LongerCaller.NAME) + get_local_project_git_path(SynthCTCRTP.NAME) ) binary_map.specify_binary( - "build/bin/LongerCaller", + "build/bin/CTCRTP", BinaryType.EXECUTABLE, only_valid_in=RevisionRange("6d50a6efd5", "master") ) From 943f1ef22012c717ffcf937c11b2e4f977a801d6 Mon Sep 17 00:00:00 2001 From: Gabrian Giovano Tanasal Date: Fri, 5 Jul 2024 09:36:53 +0200 Subject: [PATCH 03/14] adding LongerCaller projects --- .../projects/perf_tests/feature_perf_cs_collection.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/varats/varats/projects/perf_tests/feature_perf_cs_collection.py b/varats/varats/projects/perf_tests/feature_perf_cs_collection.py index cdd216d48..707fa31fd 100644 --- a/varats/varats/projects/perf_tests/feature_perf_cs_collection.py +++ b/varats/varats/projects/perf_tests/feature_perf_cs_collection.py @@ -243,7 +243,7 @@ class LongerCaller(VProject): bb.source.Git( remote="https://github.com/se-sic/FeaturePerfCSCollection.git", local=NAME, - refspec="origin/HEAD", + refspec="origin/f-calculatedynamicweight", limit=None, shallow=False, version_filter=project_filter_generator(NAME) @@ -254,7 +254,7 @@ class LongerCaller(VProject): WORKLOADS = { WorkloadSet(WorkloadCategory.EXAMPLE): [ VCommand( - SourceRoot(NAME) / RSBinary("LongerCaller"), label="CompileTime-CRTP" + SourceRoot(NAME) / RSBinary("LongerCaller"), label="CompileTime-LongerCaller" ) ] } @@ -266,11 +266,10 @@ def binaries_for_revision( binary_map = RevisionBinaryMap( get_local_project_git_path(LongerCaller.NAME) ) - binary_map.specify_binary( "build/bin/LongerCaller", BinaryType.EXECUTABLE, - only_valid_in=RevisionRange("6d50a6efd5", "master") + only_valid_in=RevisionRange("05687047", "master") #check the hash commit number on f-CalcualteDynamicWeight branch ) return binary_map[revision] From a6a5913a60990e04ee30a61b23bebfb335906ef9 Mon Sep 17 00:00:00 2001 From: Gabrian Giovano Tanasal Date: Fri, 5 Jul 2024 10:36:09 +0200 Subject: [PATCH 04/14] adding LongerCaller projects --- varats/varats/projects/perf_tests/feature_perf_cs_collection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/varats/varats/projects/perf_tests/feature_perf_cs_collection.py b/varats/varats/projects/perf_tests/feature_perf_cs_collection.py index 707fa31fd..814802950 100644 --- a/varats/varats/projects/perf_tests/feature_perf_cs_collection.py +++ b/varats/varats/projects/perf_tests/feature_perf_cs_collection.py @@ -269,7 +269,7 @@ def binaries_for_revision( binary_map.specify_binary( "build/bin/LongerCaller", BinaryType.EXECUTABLE, - only_valid_in=RevisionRange("05687047", "master") #check the hash commit number on f-CalcualteDynamicWeight branch + only_valid_in=RevisionRange("master", "056870472452101e68536dd5140f99aeb6879fc9") #check the hash commit number on f-CalcualteDynamicWeight branch ) return binary_map[revision] From 8a55da970ed38e3d67423ba00551862b6c10499a Mon Sep 17 00:00:00 2001 From: Gabrian Giovano Tanasal Date: Fri, 5 Jul 2024 12:57:04 +0200 Subject: [PATCH 05/14] adding LongerCaller projects --- .../perf_tests/feature_perf_cs_collection.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/varats/varats/projects/perf_tests/feature_perf_cs_collection.py b/varats/varats/projects/perf_tests/feature_perf_cs_collection.py index 814802950..f09022f4a 100644 --- a/varats/varats/projects/perf_tests/feature_perf_cs_collection.py +++ b/varats/varats/projects/perf_tests/feature_perf_cs_collection.py @@ -232,10 +232,10 @@ def recompile(self) -> None: # Create Longer Caller Project -class LongerCaller(VProject): +class LongerCallee(VProject): """Main Class longer than function called.""" - NAME = 'LongerCaller' + NAME = 'LongerCallee' GROUP = 'perf_tests' DOMAIN = ProjectDomains.TEST @@ -254,7 +254,7 @@ class LongerCaller(VProject): WORKLOADS = { WorkloadSet(WorkloadCategory.EXAMPLE): [ VCommand( - SourceRoot(NAME) / RSBinary("LongerCaller"), label="CompileTime-LongerCaller" + SourceRoot(NAME) / RSBinary("LongerCallee"), label="CompileTime-LongerCallee" ) ] } @@ -264,10 +264,10 @@ def binaries_for_revision( revision: ShortCommitHash # pylint: disable=W0613 ) -> tp.List[ProjectBinaryWrapper]: binary_map = RevisionBinaryMap( - get_local_project_git_path(LongerCaller.NAME) + get_local_project_git_path(LongerCallee.NAME) ) binary_map.specify_binary( - "build/bin/LongerCaller", + "build/bin/LongerCallee", BinaryType.EXECUTABLE, only_valid_in=RevisionRange("master", "056870472452101e68536dd5140f99aeb6879fc9") #check the hash commit number on f-CalcualteDynamicWeight branch ) @@ -280,7 +280,7 @@ def run_tests(self) -> None: def compile(self) -> None: """Compile the project.""" _do_feature_perf_cs_collection_compile( - self, "FPCSC_ENABLE_PROJECT_LONGERCALLER" + self, "FPCSC_ENABLE_PROJECT_LONGERCALLEE" ) def recompile(self) -> None: From 84e6bd5747e029dcb221184ecf879d8f92b5c2e4 Mon Sep 17 00:00:00 2001 From: Gabrian Giovano Tanasal Date: Fri, 5 Jul 2024 13:36:26 +0200 Subject: [PATCH 06/14] adding SimpleCall projects --- .../perf_tests/feature_perf_cs_collection.py | 40 ++++++++++++++++--- 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/varats/varats/projects/perf_tests/feature_perf_cs_collection.py b/varats/varats/projects/perf_tests/feature_perf_cs_collection.py index f09022f4a..816f7fab3 100644 --- a/varats/varats/projects/perf_tests/feature_perf_cs_collection.py +++ b/varats/varats/projects/perf_tests/feature_perf_cs_collection.py @@ -232,10 +232,10 @@ def recompile(self) -> None: # Create Longer Caller Project -class LongerCallee(VProject): +class SimpleCall(VProject): """Main Class longer than function called.""" - NAME = 'LongerCallee' + NAME = 'SimpleCall' GROUP = 'perf_tests' DOMAIN = ProjectDomains.TEST @@ -254,7 +254,7 @@ class LongerCallee(VProject): WORKLOADS = { WorkloadSet(WorkloadCategory.EXAMPLE): [ VCommand( - SourceRoot(NAME) / RSBinary("LongerCallee"), label="CompileTime-LongerCallee" + SourceRoot(NAME) / RSBinary("SimpleCall"), label="CompileTime-SimpleCall" ) ] } @@ -264,10 +264,10 @@ def binaries_for_revision( revision: ShortCommitHash # pylint: disable=W0613 ) -> tp.List[ProjectBinaryWrapper]: binary_map = RevisionBinaryMap( - get_local_project_git_path(LongerCallee.NAME) + get_local_project_git_path(SimpleCall.NAME) ) binary_map.specify_binary( - "build/bin/LongerCallee", + "build/bin/SimpleCall", BinaryType.EXECUTABLE, only_valid_in=RevisionRange("master", "056870472452101e68536dd5140f99aeb6879fc9") #check the hash commit number on f-CalcualteDynamicWeight branch ) @@ -280,7 +280,35 @@ def run_tests(self) -> None: def compile(self) -> None: """Compile the project.""" _do_feature_perf_cs_collection_compile( - self, "FPCSC_ENABLE_PROJECT_LONGERCALLEE" + self, "FPCSC_ENABLE_PROJECT_SIMPLECALL" + ) + + def recompile(self) -> None: + """Recompile the project.""" + _do_feature_perf_cs_collection_recompile(self) + + @staticmethod + def binaries_for_revision( + revision: ShortCommitHash # pylint: disable=W0613 + ) -> tp.List[ProjectBinaryWrapper]: + binary_map = RevisionBinaryMap( + get_local_project_git_path(SimpleCall.NAME) + ) + binary_map.specify_binary( + "build/bin/SimpleCall", + BinaryType.EXECUTABLE, + only_valid_in=RevisionRange("master", "056870472452101e68536dd5140f99aeb6879fc9") #check the hash commit number on f-CalcualteDynamicWeight branch + ) + + return binary_map[revision] + + def run_tests(self) -> None: + pass + + def compile(self) -> None: + """Compile the project.""" + _do_feature_perf_cs_collection_compile( + self, "FPCSC_ENABLE_PROJECT_SIMPLECALL" ) def recompile(self) -> None: From a5b1ebb2181461ef8f99481d9c25774b6326e484 Mon Sep 17 00:00:00 2001 From: Gabrian Giovano Tanasal Date: Fri, 5 Jul 2024 14:06:44 +0200 Subject: [PATCH 07/14] fixing revision range --- varats/varats/projects/perf_tests/feature_perf_cs_collection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/varats/varats/projects/perf_tests/feature_perf_cs_collection.py b/varats/varats/projects/perf_tests/feature_perf_cs_collection.py index 816f7fab3..f114ca6cb 100644 --- a/varats/varats/projects/perf_tests/feature_perf_cs_collection.py +++ b/varats/varats/projects/perf_tests/feature_perf_cs_collection.py @@ -297,7 +297,7 @@ def binaries_for_revision( binary_map.specify_binary( "build/bin/SimpleCall", BinaryType.EXECUTABLE, - only_valid_in=RevisionRange("master", "056870472452101e68536dd5140f99aeb6879fc9") #check the hash commit number on f-CalcualteDynamicWeight branch + only_valid_in=RevisionRange("4b440e27e1edd034e19639fd3e3774f8a57fc2db", "4b440e27e1edd034e19639fd3e3774f8a57fc2db") #check the hash commit number on f-CalcualteDynamicWeight branch ) return binary_map[revision] From 976c45651d164b0bc3d618120ff0f46b319049c1 Mon Sep 17 00:00:00 2001 From: Gabrian Giovano Tanasal Date: Sun, 7 Jul 2024 18:52:46 +0200 Subject: [PATCH 08/14] adding the rest of the projects --- .../perf_tests/feature_perf_cs_collection.py | 154 +++++++++++++++++- 1 file changed, 149 insertions(+), 5 deletions(-) diff --git a/varats/varats/projects/perf_tests/feature_perf_cs_collection.py b/varats/varats/projects/perf_tests/feature_perf_cs_collection.py index f114ca6cb..d41215218 100644 --- a/varats/varats/projects/perf_tests/feature_perf_cs_collection.py +++ b/varats/varats/projects/perf_tests/feature_perf_cs_collection.py @@ -231,7 +231,121 @@ def recompile(self) -> None: _do_feature_perf_cs_collection_recompile(self) -# Create Longer Caller Project +# Create SimpleCall Project +class LongerCallee(VProject): + """Main Class longer than function called.""" + + NAME = 'LongerCallee' + GROUP = 'perf_tests' + DOMAIN = ProjectDomains.TEST + + SOURCE = [ + bb.source.Git( + remote="https://github.com/se-sic/FeaturePerfCSCollection.git", + local=NAME, + refspec="origin/f-calculatedynamicweight", + limit=None, + shallow=False, + version_filter=project_filter_generator(NAME) + ), + FeatureSource() + ] + + WORKLOADS = { + WorkloadSet(WorkloadCategory.EXAMPLE): [ + VCommand( + SourceRoot(NAME) / RSBinary("LongerCallee"), label="CompileTime-LongerCallee" + ) + ] + } + + @staticmethod + def binaries_for_revision( + revision: ShortCommitHash # pylint: disable=W0613 + ) -> tp.List[ProjectBinaryWrapper]: + binary_map = RevisionBinaryMap( + get_local_project_git_path(SimpleCall.NAME) + ) + binary_map.specify_binary( + "build/bin/LongerCallee", + BinaryType.EXECUTABLE, + only_valid_in=RevisionRange("0214ccd4cebac656449ce232a69e3b385d5e63e9", "0214ccd4cebac656449ce232a69e3b385d5e63e9") #check the hash commit number on f-CalcualteDynamicWeight branch + ) + + return binary_map[revision] + + def run_tests(self) -> None: + pass + + def compile(self) -> None: + """Compile the project.""" + _do_feature_perf_cs_collection_compile( + self, "FPCSC_ENABLE_PROJECT_LONGERCALLEE" + ) + + def recompile(self) -> None: + """Recompile the project.""" + _do_feature_perf_cs_collection_recompile(self) + + + +class LongerCaller(VProject): + """Main Class longer than function called.""" + + NAME = 'LongerCaller' + GROUP = 'perf_tests' + DOMAIN = ProjectDomains.TEST + + SOURCE = [ + bb.source.Git( + remote="https://github.com/se-sic/FeaturePerfCSCollection.git", + local=NAME, + refspec="origin/f-calculatedynamicweight", + limit=None, + shallow=False, + version_filter=project_filter_generator(NAME) + ), + FeatureSource() + ] + + WORKLOADS = { + WorkloadSet(WorkloadCategory.EXAMPLE): [ + VCommand( + SourceRoot(NAME) / RSBinary("LongerCaller"), label="CompileTime-LongerCaller" + ) + ] + } + + @staticmethod + def binaries_for_revision( + revision: ShortCommitHash # pylint: disable=W0613 + ) -> tp.List[ProjectBinaryWrapper]: + binary_map = RevisionBinaryMap( + get_local_project_git_path(SimpleCall.NAME) + ) + binary_map.specify_binary( + "build/bin/LongerCaller", + BinaryType.EXECUTABLE, + only_valid_in=RevisionRange("0214ccd4cebac656449ce232a69e3b385d5e63e9", "0214ccd4cebac656449ce232a69e3b385d5e63e9") #check the hash commit number on f-CalcualteDynamicWeight branch + ) + + return binary_map[revision] + + def run_tests(self) -> None: + pass + + def compile(self) -> None: + """Compile the project.""" + _do_feature_perf_cs_collection_compile( + self, "FPCSC_ENABLE_PROJECT_LONGERCALLER" + ) + + def recompile(self) -> None: + """Recompile the project.""" + _do_feature_perf_cs_collection_recompile(self) + + + class SimpleCall(VProject): """Main Class longer than function called.""" @@ -269,7 +383,7 @@ def binaries_for_revision( binary_map.specify_binary( "build/bin/SimpleCall", BinaryType.EXECUTABLE, - only_valid_in=RevisionRange("master", "056870472452101e68536dd5140f99aeb6879fc9") #check the hash commit number on f-CalcualteDynamicWeight branch + only_valid_in=RevisionRange("0214ccd4cebac656449ce232a69e3b385d5e63e9", "0214ccd4cebac656449ce232a69e3b385d5e63e9") #check the hash commit number on f-CalcualteDynamicWeight branch ) return binary_map[revision] @@ -287,6 +401,35 @@ def recompile(self) -> None: """Recompile the project.""" _do_feature_perf_cs_collection_recompile(self) + + +class RecursiveCalls(VProject): + """Main Class longer than function called.""" + + NAME = 'RecursiveCalls' + GROUP = 'perf_tests' + DOMAIN = ProjectDomains.TEST + + SOURCE = [ + bb.source.Git( + remote="https://github.com/se-sic/FeaturePerfCSCollection.git", + local=NAME, + refspec="origin/f-calculatedynamicweight", + limit=None, + shallow=False, + version_filter=project_filter_generator(NAME) + ), + FeatureSource() + ] + + WORKLOADS = { + WorkloadSet(WorkloadCategory.EXAMPLE): [ + VCommand( + SourceRoot(NAME) / RSBinary("RecursiveCalls"), label="CompileTime-RecursiveCalls" + ) + ] + } + @staticmethod def binaries_for_revision( revision: ShortCommitHash # pylint: disable=W0613 @@ -295,9 +438,9 @@ def binaries_for_revision( get_local_project_git_path(SimpleCall.NAME) ) binary_map.specify_binary( - "build/bin/SimpleCall", + "build/bin/RecursiveCalls", BinaryType.EXECUTABLE, - only_valid_in=RevisionRange("4b440e27e1edd034e19639fd3e3774f8a57fc2db", "4b440e27e1edd034e19639fd3e3774f8a57fc2db") #check the hash commit number on f-CalcualteDynamicWeight branch + only_valid_in=RevisionRange("0214ccd4cebac656449ce232a69e3b385d5e63e9", "0214ccd4cebac656449ce232a69e3b385d5e63e9") #check the hash commit number on f-CalcualteDynamicWeight branch ) return binary_map[revision] @@ -308,7 +451,7 @@ def run_tests(self) -> None: def compile(self) -> None: """Compile the project.""" _do_feature_perf_cs_collection_compile( - self, "FPCSC_ENABLE_PROJECT_SIMPLECALL" + self, "FPCSC_ENABLE_PROJECT_RECURSIVECALLS" ) def recompile(self) -> None: @@ -316,6 +459,7 @@ def recompile(self) -> None: _do_feature_perf_cs_collection_recompile(self) + class SynthSAContextSensitivity(VProject): """Synthetic case-study project for testing flow sensitivity.""" From 2efd65ba72c8d1b23f38c5e704aabdbb4e24d21e Mon Sep 17 00:00:00 2001 From: Gabrian Giovano Tanasal Date: Sun, 7 Jul 2024 23:12:18 +0200 Subject: [PATCH 09/14] Initial FeatureWeight class --- .../varats/experiments/vara/feature_weight.py | 117 ++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 varats/varats/experiments/vara/feature_weight.py diff --git a/varats/varats/experiments/vara/feature_weight.py b/varats/varats/experiments/vara/feature_weight.py new file mode 100644 index 000000000..bdfd2522a --- /dev/null +++ b/varats/varats/experiments/vara/feature_weight.py @@ -0,0 +1,117 @@ +"""Implements an experiment that times the execution of all project binaries.""" + +import typing as tp +from pathlib import Path + +from benchbuild import Project +from benchbuild.command import cleanup +from benchbuild.extensions import compiler, run +from benchbuild.utils import actions +from benchbuild.utils.cmd import time +from plumbum import local + +from varats.experiment.experiment_util import ( + VersionExperiment, + get_default_compile_error_wrapped, + create_new_success_result_filepath, + ZippedExperimentSteps, + OutputFolderStep, +) +from varats.experiment.workload_util import ( + workload_commands, + WorkloadCategory, + create_workload_specific_filename, +) +from varats.project.project_util import ProjectBinaryWrapper +from varats.project.varats_project import VProject +from varats.report.gnu_time_report import WLTimeReportAggregate +from varats.report.report import ReportSpecification + + +class FeatureWeight(OutputFolderStep): + """Times the execution of all project example workloads.""" + + NAME = "FeatureWeight" + DESCRIPTION = "Analyse Output with -fvara-weight" + + project: VProject + + def __init__( + self, project: Project, num: int, binary: ProjectBinaryWrapper + ): + super().__init__(project=project) + self.__num = num + self.__binary = binary + + def call_with_output_folder(self, tmp_dir: Path) -> actions.StepResult: + return self.analyze(tmp_dir) + + def analyze(self, tmp_dir: Path) -> actions.StepResult: + """Only create a report file.""" + + with local.cwd(self.project.builddir): # changing working directory + for prj_command in workload_commands( + self.project, self.__binary, [WorkloadCategory.EXAMPLE] + ): + pb_cmd = prj_command.command.as_plumbum(project=self.project) # in workload_commands each line of command is now represented by this + + run_report_name = tmp_dir / create_workload_specific_filename( + "weight_report", prj_command.command, self.__num, ".txt" + ) + + # trying to pipe the output of running the binary + run_cmd = local[pb_cmd, f"> {run_report_name}"] + + with cleanup(prj_command): + run_cmd() + + return actions.StepResult.OK + + +class WeightWorkloads(VersionExperiment, shorthand="WWL"): + """Generates Weight report files.""" + + NAME = "TimeWorkloads" + + REPORT_SPEC = ReportSpecification(WLTimeReportAggregate) + + def actions_for_project( + self, project: VProject + ) -> tp.MutableSequence[actions.Step]: + """Returns the specified steps to run the project(s) specified in the + call in a fixed order.""" + + # Add the required runtime extensions to the project(s). + project.runtime_extension = run.RuntimeExtension(project, self) + + # Add the required compiler extensions to the project(s). + project.compiler_extension = compiler.RunCompiler(project, self) \ + << run.WithTimeout() + + project.compile = get_default_compile_error_wrapped( + self.get_handle(), project, self.REPORT_SPEC.main_report + ) + + # Only consider the main/first binary + binary = project.binaries[0] + + measurement_repetitions = 2 + result_filepath = create_new_success_result_filepath( + self.get_handle(), + self.get_handle().report_spec().main_report, project, binary + ) + + analysis_actions = [] + analysis_actions.append(actions.Compile(project)) + + analysis_actions.append( + ZippedExperimentSteps( + result_filepath, [ + FeatureWeight(project, rep_num, binary) + for rep_num in range(0, measurement_repetitions) + ] + ) + ) + analysis_actions.append(actions.Clean(project)) + + return analysis_actions From d446c74a141585c72f0c972fd2e586403fa523ac Mon Sep 17 00:00:00 2001 From: Gabrian Giovano Tanasal Date: Mon, 15 Jul 2024 10:28:29 +0200 Subject: [PATCH 10/14] fixing weight, adding experiment for default weight function --- .../varats/experiments/vara/feature_weight.py | 81 ++++++----------- .../vara/feature_weight_default.py | 86 +++++++++++++++++++ 2 files changed, 111 insertions(+), 56 deletions(-) create mode 100644 varats/varats/experiments/vara/feature_weight_default.py diff --git a/varats/varats/experiments/vara/feature_weight.py b/varats/varats/experiments/vara/feature_weight.py index bdfd2522a..52dbb7895 100644 --- a/varats/varats/experiments/vara/feature_weight.py +++ b/varats/varats/experiments/vara/feature_weight.py @@ -28,52 +28,14 @@ from varats.report.report import ReportSpecification -class FeatureWeight(OutputFolderStep): - """Times the execution of all project example workloads.""" - NAME = "FeatureWeight" - DESCRIPTION = "Analyse Output with -fvara-weight" - project: VProject - - def __init__( - self, project: Project, num: int, binary: ProjectBinaryWrapper - ): - super().__init__(project=project) - self.__num = num - self.__binary = binary - - def call_with_output_folder(self, tmp_dir: Path) -> actions.StepResult: - return self.analyze(tmp_dir) - - def analyze(self, tmp_dir: Path) -> actions.StepResult: - """Only create a report file.""" - - with local.cwd(self.project.builddir): # changing working directory - for prj_command in workload_commands( - self.project, self.__binary, [WorkloadCategory.EXAMPLE] - ): - pb_cmd = prj_command.command.as_plumbum(project=self.project) # in workload_commands each line of command is now represented by this - - run_report_name = tmp_dir / create_workload_specific_filename( - "weight_report", prj_command.command, self.__num, ".txt" - ) - - # trying to pipe the output of running the binary - run_cmd = local[pb_cmd, f"> {run_report_name}"] - - with cleanup(prj_command): - run_cmd() - - return actions.StepResult.OK - - -class WeightWorkloads(VersionExperiment, shorthand="WWL"): +class WeightRegionsCount(VersionExperiment, shorthand="WRP"): """Generates Weight report files.""" - NAME = "TimeWorkloads" + NAME = "WeightAnalysis" - REPORT_SPEC = ReportSpecification(WLTimeReportAggregate) + REPORT_SPEC = ReportSpecification(InstrVerifierReport) def actions_for_project( self, project: VProject @@ -81,35 +43,42 @@ def actions_for_project( """Returns the specified steps to run the project(s) specified in the call in a fixed order.""" + project.cflags += self.get_vara_feature_cflags(project) + + # change the featureInstrType to verify + project.cflags += self.get_vara_tracing_cflags( + FeatureInstrType.TEF, instruction_threshold=1 + ) + + project.cflags += [ + "-fvara-weight-opt=recursive", + "-01", + "-g0", + "-mllvm", + "--vara-use-phasar" + ] + + project.ldflags += self.get_vara_tracing_ldflags() + # Add the required runtime extensions to the project(s). project.runtime_extension = run.RuntimeExtension(project, self) # Add the required compiler extensions to the project(s). project.compiler_extension = compiler.RunCompiler(project, self) \ - << run.WithTimeout() + << WithUnlimitedStackSize() + project.compile = get_default_compile_error_wrapped( self.get_handle(), project, self.REPORT_SPEC.main_report ) - # Only consider the main/first binary - binary = project.binaries[0] - - measurement_repetitions = 2 - result_filepath = create_new_success_result_filepath( - self.get_handle(), - self.get_handle().report_spec().main_report, project, binary - ) - analysis_actions = [] analysis_actions.append(actions.Compile(project)) + analysis_actions.append(actions.Compile(project)) analysis_actions.append( - ZippedExperimentSteps( - result_filepath, [ - FeatureWeight(project, rep_num, binary) - for rep_num in range(0, measurement_repetitions) - ] + RunVaRATracedWorkloads( + project, self.get_handle(), report_file_ending="ivr" ) ) analysis_actions.append(actions.Clean(project)) diff --git a/varats/varats/experiments/vara/feature_weight_default.py b/varats/varats/experiments/vara/feature_weight_default.py new file mode 100644 index 000000000..c0d394363 --- /dev/null +++ b/varats/varats/experiments/vara/feature_weight_default.py @@ -0,0 +1,86 @@ +"""Implements an experiment that times the execution of all project binaries.""" + +import typing as tp +from pathlib import Path + +from benchbuild import Project +from benchbuild.command import cleanup +from benchbuild.extensions import compiler, run +from benchbuild.utils import actions +from benchbuild.utils.cmd import time +from plumbum import local + +from varats.experiment.experiment_util import ( + VersionExperiment, + get_default_compile_error_wrapped, + create_new_success_result_filepath, + ZippedExperimentSteps, + OutputFolderStep, +) +from varats.experiment.workload_util import ( + workload_commands, + WorkloadCategory, + create_workload_specific_filename, +) +from varats.project.project_util import ProjectBinaryWrapper +from varats.project.varats_project import VProject +from varats.report.gnu_time_report import WLTimeReportAggregate +from varats.report.report import ReportSpecification + + + + +class WeightRegionsCount(VersionExperiment, shorthand="WRP"): + """Generates Weight report files.""" + + NAME = "WeightAnalysis" + + REPORT_SPEC = ReportSpecification(InstrVerifierReport) + + def actions_for_project( + self, project: VProject + ) -> tp.MutableSequence[actions.Step]: + """Returns the specified steps to run the project(s) specified in the + call in a fixed order.""" + + project.cflags += self.get_vara_feature_cflags(project) + + # change the featureInstrType to verify + project.cflags += self.get_vara_tracing_cflags( + FeatureInstrType.TEF, instruction_threshold=1 + ) + + project.cflags += [ + "-fvara-weight-opt=default", + "-01", + "-g0", + "-mllvm", + "--vara-use-phasar" + ] + + project.ldflags += self.get_vara_tracing_ldflags() + + # Add the required runtime extensions to the project(s). + project.runtime_extension = run.RuntimeExtension(project, self) + + # Add the required compiler extensions to the project(s). + project.compiler_extension = compiler.RunCompiler(project, self) \ + << WithUnlimitedStackSize() + + + project.compile = get_default_compile_error_wrapped( + self.get_handle(), project, self.REPORT_SPEC.main_report + ) + + analysis_actions = [] + analysis_actions.append(actions.Compile(project)) + + analysis_actions.append(actions.Compile(project)) + analysis_actions.append( + RunVaRATracedWorkloads( + project, self.get_handle(), report_file_ending="ivr" + ) + ) + analysis_actions.append(actions.Clean(project)) + + return analysis_actions From bfeefe1141e689f82368ae9bf05db722e44d390b Mon Sep 17 00:00:00 2001 From: Gabrian Giovano Tanasal Date: Mon, 15 Jul 2024 13:53:41 +0200 Subject: [PATCH 11/14] changing the shorthand and the name of the experiment --- varats/varats/experiments/vara/feature_weight.py | 4 ++-- varats/varats/experiments/vara/feature_weight_default.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/varats/varats/experiments/vara/feature_weight.py b/varats/varats/experiments/vara/feature_weight.py index 52dbb7895..695d52daa 100644 --- a/varats/varats/experiments/vara/feature_weight.py +++ b/varats/varats/experiments/vara/feature_weight.py @@ -30,10 +30,10 @@ -class WeightRegionsCount(VersionExperiment, shorthand="WRP"): +class WeightRegionsCount(VersionExperiment, shorthand="WAR"): """Generates Weight report files.""" - NAME = "WeightAnalysis" + NAME = "WeightAnalysisRecursive" REPORT_SPEC = ReportSpecification(InstrVerifierReport) diff --git a/varats/varats/experiments/vara/feature_weight_default.py b/varats/varats/experiments/vara/feature_weight_default.py index c0d394363..bf49e6283 100644 --- a/varats/varats/experiments/vara/feature_weight_default.py +++ b/varats/varats/experiments/vara/feature_weight_default.py @@ -30,10 +30,10 @@ -class WeightRegionsCount(VersionExperiment, shorthand="WRP"): +class WeightRegionsCount(VersionExperiment, shorthand="WAD"): """Generates Weight report files.""" - NAME = "WeightAnalysis" + NAME = "WeightAnalysisDefault" REPORT_SPEC = ReportSpecification(InstrVerifierReport) From 738483076786398ab5d5ec04c435dd5d5466be52 Mon Sep 17 00:00:00 2001 From: Gabrian Giovano Tanasal Date: Mon, 15 Jul 2024 13:56:38 +0200 Subject: [PATCH 12/14] fixing import issue --- .../varats/experiments/vara/feature_weight.py | 26 ++++++------------- .../vara/feature_weight_default.py | 26 ++++++------------- 2 files changed, 16 insertions(+), 36 deletions(-) diff --git a/varats/varats/experiments/vara/feature_weight.py b/varats/varats/experiments/vara/feature_weight.py index 695d52daa..b8b2da6f3 100644 --- a/varats/varats/experiments/vara/feature_weight.py +++ b/varats/varats/experiments/vara/feature_weight.py @@ -1,35 +1,25 @@ """Implements an experiment that times the execution of all project binaries.""" - import typing as tp -from pathlib import Path -from benchbuild import Project -from benchbuild.command import cleanup from benchbuild.extensions import compiler, run from benchbuild.utils import actions -from benchbuild.utils.cmd import time -from plumbum import local +from varats.data.reports.instrumentation_verifier_report import ( + InstrVerifierReport, +) from varats.experiment.experiment_util import ( - VersionExperiment, get_default_compile_error_wrapped, - create_new_success_result_filepath, - ZippedExperimentSteps, - OutputFolderStep, + WithUnlimitedStackSize, ) -from varats.experiment.workload_util import ( - workload_commands, - WorkloadCategory, - create_workload_specific_filename, +from varats.experiments.vara.feature_experiment import ( + FeatureExperiment, + RunVaRATracedWorkloads, + FeatureInstrType, ) -from varats.project.project_util import ProjectBinaryWrapper from varats.project.varats_project import VProject -from varats.report.gnu_time_report import WLTimeReportAggregate from varats.report.report import ReportSpecification - - class WeightRegionsCount(VersionExperiment, shorthand="WAR"): """Generates Weight report files.""" diff --git a/varats/varats/experiments/vara/feature_weight_default.py b/varats/varats/experiments/vara/feature_weight_default.py index bf49e6283..e8bf8c3fb 100644 --- a/varats/varats/experiments/vara/feature_weight_default.py +++ b/varats/varats/experiments/vara/feature_weight_default.py @@ -1,35 +1,25 @@ """Implements an experiment that times the execution of all project binaries.""" - import typing as tp -from pathlib import Path -from benchbuild import Project -from benchbuild.command import cleanup from benchbuild.extensions import compiler, run from benchbuild.utils import actions -from benchbuild.utils.cmd import time -from plumbum import local +from varats.data.reports.instrumentation_verifier_report import ( + InstrVerifierReport, +) from varats.experiment.experiment_util import ( - VersionExperiment, get_default_compile_error_wrapped, - create_new_success_result_filepath, - ZippedExperimentSteps, - OutputFolderStep, + WithUnlimitedStackSize, ) -from varats.experiment.workload_util import ( - workload_commands, - WorkloadCategory, - create_workload_specific_filename, +from varats.experiments.vara.feature_experiment import ( + FeatureExperiment, + RunVaRATracedWorkloads, + FeatureInstrType, ) -from varats.project.project_util import ProjectBinaryWrapper from varats.project.varats_project import VProject -from varats.report.gnu_time_report import WLTimeReportAggregate from varats.report.report import ReportSpecification - - class WeightRegionsCount(VersionExperiment, shorthand="WAD"): """Generates Weight report files.""" From 70877372d4db04fc9e52b81bffa63da9e373acbe Mon Sep 17 00:00:00 2001 From: Gabrian Giovano Tanasal Date: Mon, 15 Jul 2024 13:57:36 +0200 Subject: [PATCH 13/14] fixing argument issue --- varats/varats/experiments/vara/feature_weight.py | 2 +- varats/varats/experiments/vara/feature_weight_default.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/varats/varats/experiments/vara/feature_weight.py b/varats/varats/experiments/vara/feature_weight.py index b8b2da6f3..1cca78da4 100644 --- a/varats/varats/experiments/vara/feature_weight.py +++ b/varats/varats/experiments/vara/feature_weight.py @@ -20,7 +20,7 @@ from varats.report.report import ReportSpecification -class WeightRegionsCount(VersionExperiment, shorthand="WAR"): +class WeightRegionsCount(FeatureExperiment, shorthand="WAR"): """Generates Weight report files.""" NAME = "WeightAnalysisRecursive" diff --git a/varats/varats/experiments/vara/feature_weight_default.py b/varats/varats/experiments/vara/feature_weight_default.py index e8bf8c3fb..e404abde1 100644 --- a/varats/varats/experiments/vara/feature_weight_default.py +++ b/varats/varats/experiments/vara/feature_weight_default.py @@ -20,7 +20,7 @@ from varats.report.report import ReportSpecification -class WeightRegionsCount(VersionExperiment, shorthand="WAD"): +class WeightRegionsCount(FeatureExperiment, shorthand="WAD"): """Generates Weight report files.""" NAME = "WeightAnalysisDefault" From 8ab634a1ad7a39337ed5a82ce275751482784ac2 Mon Sep 17 00:00:00 2001 From: Gabrian Giovano Tanasal Date: Mon, 15 Jul 2024 14:06:09 +0200 Subject: [PATCH 14/14] chaning class name --- varats/varats/experiments/vara/feature_weight.py | 4 ++-- varats/varats/experiments/vara/feature_weight_default.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/varats/varats/experiments/vara/feature_weight.py b/varats/varats/experiments/vara/feature_weight.py index 1cca78da4..0529eb0e5 100644 --- a/varats/varats/experiments/vara/feature_weight.py +++ b/varats/varats/experiments/vara/feature_weight.py @@ -20,8 +20,8 @@ from varats.report.report import ReportSpecification -class WeightRegionsCount(FeatureExperiment, shorthand="WAR"): - """Generates Weight report files.""" +class WeightRegionsCountRec(FeatureExperiment, shorthand="WAR"): + """Generates Weight report files for Recursive weight function.""" NAME = "WeightAnalysisRecursive" diff --git a/varats/varats/experiments/vara/feature_weight_default.py b/varats/varats/experiments/vara/feature_weight_default.py index e404abde1..b23da26f0 100644 --- a/varats/varats/experiments/vara/feature_weight_default.py +++ b/varats/varats/experiments/vara/feature_weight_default.py @@ -20,8 +20,8 @@ from varats.report.report import ReportSpecification -class WeightRegionsCount(FeatureExperiment, shorthand="WAD"): - """Generates Weight report files.""" +class WeightRegionsCountDef(FeatureExperiment, shorthand="WAD"): + """Generates Weight report files for Default recursive function.""" NAME = "WeightAnalysisDefault"