Feature/slp vectorization

CMakeLists.txt

@@ -35,6 +35,10 @@ endif (${CUDA_GPU_SUPPORT})
 
 set(SPNC_CXX_WARNING_LEVEL "-Wall")
 
+option(SPNC_SLP_DEBUG
+        "Enable additional debug output for the SLP vectorizer"
+        OFF)
+
 #
 # clang-tidy setup
 #

README_SLP.md

@@ -0,0 +1,43 @@
+# SLP-Vectorization #
+
+The SLP directory contains 11 files, each dealing with a different SLP topic.
+
+Please note that a _superword_ in this project describes an SLP vector containing the elements. This term was chosen because a _vector_ is slightly overloaded with meanings in C++. 
+
+* CostModel.h
+  * Contains the cost model, which assigns cost to scalar operations, superwords and entire patterns using a visitor pattern.
+
+* GraphConversion.h
+  * A very important file that contains the ConversionManager class. The conversion manager keeps track of created vector operations, extractions and maintains a ConversionState. The conversion state is responsible for remembering which scalar/superword values have been computed already. The conversion manager is also responsible for gracefully resetting the function state in case an SLP graph is not deemed profitable. 
+
+* PatternVisitors.h
+  * Contains the visitor template and the LeafPatternVisitor, which can determine the scalar values that need to be computed for every leaf pattern (e.g. a BroadcastInsertPattern needs a scalar broadcast value and scalar insert values).
+
+* ScoreModel.h
+  * Contains the Look-Ahead-Score score model from the original Look-Ahead SLP publication [[1]](https://dl.acm.org/doi/10.1145/3168807). Also contains the XOR chain model.
+
+* Seeding.h
+  * Contains the classes used for top-down and bottom-up seeding.
+
+* SLPGraph.h
+  * Contains the superword logic and the logic for actual SLP graphs (nodes and multinodes). Note: there is no explicit SLPGraphEdge class or something similar.
+
+* SLPGraphBuilder.h
+  * Contains a graph builder that constructs SLP graphs as described in Porpodas et al. [[1]](https://dl.acm.org/doi/10.1145/3168807).
+
+* SLPPatternMatch.h
+  * Responsible for selecting the best patterns based on the cost model and the current conversion state.
+
+* SLPVectorizationPatterns.h
+  * The individual patterns that can be applied to superwords and their match and rewrite logic. They were designed in a somewhat similar fashion compared to MLIR's pattern rewrite framework.
+
+* Util.h
+  * Some utility functions, such as _vectorizable(...)_ or _commutative(...)_.
+
+### Known Issues ###
+* ShufflePattern: With shuffle patterns enabled, the output of the kernels sometimes does not match the expected output. This might be due to the reordering changing semantics and the shuffle pattern accessing elements with changed semantics by accident.
+* The SPN compiler options are replicated inside the util class. This is a little bit annoying.
+
+References
+-----
+[[1] Vasileios Porpodas, Rodrigo C. O. Rocha, and Luís F. W. Góes. 2018. Look-ahead SLP: auto-vectorization in the presence of commutative operations. In Proceedings of the 2018 International Symposium on Code Generation and Optimization (CGO 2018). Association for Computing Machinery, New York, NY, USA, 163–174.](https://dl.acm.org/doi/10.1145/3168807)

compiler/src/option/GlobalOptions.cpp

@@ -42,6 +42,26 @@ EnumOpt spnc::option::vectorLibrary{"vector-library", NONE,
 
 Option<bool> spnc::option::replaceGatherWithShuffle{"use-shuffle", false};
 
+Option<unsigned> spnc::option::slpMaxAttempts{"slp-max-attempts", 1, {depends(spnc::option::cpuVectorize, true)}};
+
+Option<unsigned>spnc::option::slpMaxSuccessfulIterations
+    {"slp-max-successful-iterations", 1, {depends(spnc::option::cpuVectorize, true)}};
+
+Option<unsigned> spnc::option::slpMaxNodeSize{"slp-max-node-size", 10, {depends(spnc::option::cpuVectorize, true)}};
+
+Option<unsigned> spnc::option::slpMaxLookAhead{"slp-max-look-ahead", 3, {depends(spnc::option::cpuVectorize, true)}};
+
+Option<bool>
+    spnc::option::slpReorderInstructionsDFS{"slp-reorder-dfs", true, {depends(spnc::option::cpuVectorize, true)}};
+
+Option<bool>spnc::option::slpAllowDuplicateElements
+    {"slp-allow-duplicate-elements", false, {depends(spnc::option::cpuVectorize, true)}};
+
+Option<bool>spnc::option::slpAllowTopologicalMixing
+    {"slp-allow-topological-mixing", false, {depends(spnc::option::cpuVectorize, true)}};
+
+Option<bool>spnc::option::slpUseXorChains{"slp-use-xor-chains", false, {depends(spnc::option::cpuVectorize, true)}};
+
 Option<bool> spnc::option::logSpace{"use-log-space", false};
 
 Option<bool> spnc::option::gpuSharedMem{"gpu-shared-mem", true};

compiler/src/option/GlobalOptions.h

@@ -84,6 +84,44 @@ namespace spnc {
     /// with a combination of regular vector loads and shuffles.
     extern Option<bool> replaceGatherWithShuffle;
 
+    // SLP vectorization options.
+
+    ///
+    /// Maximum number of SLP vectorization attempts.
+    extern Option<unsigned> slpMaxAttempts;
+
+    ///
+    /// Maximum number of successful SLP vectorization runs to be applied to a function.
+    extern Option<unsigned> slpMaxSuccessfulIterations;
+
+    ///
+    /// Maximum multinode size during SLP vectorization in terms of the number of vectors they may contain.
+    extern Option<unsigned> slpMaxNodeSize;
+
+    ///
+    /// Maximum look-ahead depth when reordering multinode operands during SLP vectorization.
+    extern Option<unsigned> slpMaxLookAhead;
+
+    ///
+    /// Flag to indicate the order in which SLP-vectorized instructions should be arranged.
+    /// True to reorder them based on a depth-first search, false to reorder them based on a breadth-first search.
+    extern Option<bool> slpReorderInstructionsDFS;
+
+    ///
+    /// Flag to indicate whether duplicate elements are allowed in vectors during SLP graph building.
+    /// True to keep growing the graph when duplicates are encountered in a vector, false to stop growing.
+    extern Option<bool> slpAllowDuplicateElements;
+
+    ///
+    /// Flag to indicate if elements with different topological depths are allowed in vectors during SLP graph building.
+    /// True to allow mixing, false to stop growing the graph at that vector if mixed topological depths occur.
+    extern Option<bool> slpAllowTopologicalMixing;
+
+    ///
+    /// Flag to indicate if XOR chains should be used to compute look-ahead scores instead of Porpodas's algorithm.
+    /// True to use XOR chains, false to use Porpodas's algorithm.
+    extern Option<bool> slpUseXorChains;
+
     ///
     /// Flag to indicate whether log-space computation should be used.
     extern Option<bool> logSpace;

compiler/src/option/Options.cpp

@@ -16,12 +16,12 @@ std::unordered_map<std::string, Opt*>& Options::options() {
   return *_options;
 };
 
-std::vector<std::unique_ptr<OptModifier>>& Options::allModifiers(){
+std::vector<std::unique_ptr<OptModifier>>& Options::allModifiers() {
   static auto* _modifiers = new std::vector<std::unique_ptr<OptModifier>>();
   return *_modifiers;
 };
 
-std::vector<OptModifier*>& Options::activeModifiers(){
+std::vector<OptModifier*>& Options::activeModifiers() {
   static auto* _modifiers = new std::vector<OptModifier*>();
   return *_modifiers;
 };
@@ -45,6 +45,15 @@ int detail::OptionParsers::parse(const std::string& value) {
   return std::stoi(value);
 }
 
+/// Specialization to parse unsigned integer options,
+/// using standard library facilities to parse the unsigned int from the string.
+/// \param value String.
+/// \return Unsigned integer value.
+template<>
+unsigned detail::OptionParsers::parse(const std::string& value) {
+  return std::stol(value);
+}
+
 /// Specialization to parse floating-point options,
 /// using standard library facilities to parse the double from the string.
 /// \param value String.

compiler/src/pipeline/steps/mlir/conversion/LoSPNtoCPUConversion.cpp

@@ -18,7 +18,17 @@
 void spnc::LoSPNtoCPUConversion::initializePassPipeline(mlir::PassManager* pm, mlir::MLIRContext* ctx) {
   auto* config = getContext()->get<Configuration>();
   bool vectorize = spnc::option::cpuVectorize.get(*config);
-  pm->addPass(mlir::spn::createLoSPNtoCPUStructureConversionPass(vectorize));
+  pm->addPass(std::make_unique<mlir::spn::LoSPNtoCPUStructureConversionPass>(
+      vectorize,
+      spnc::option::slpMaxAttempts.get(*config),
+      spnc::option::slpMaxSuccessfulIterations.get(*config),
+      spnc::option::slpMaxNodeSize.get(*config),
+      spnc::option::slpMaxLookAhead.get(*config),
+      spnc::option::slpReorderInstructionsDFS.get(*config),
+      spnc::option::slpAllowDuplicateElements.get(*config),
+      spnc::option::slpAllowTopologicalMixing.get(*config),
+      spnc::option::slpUseXorChains.get(*config)
+  ));
   if (vectorize) {
     auto useShuffle = spnc::option::replaceGatherWithShuffle.get(*config);
     if (useShuffle) {
@@ -50,4 +60,4 @@ void spnc::LoSPNtoCPUConversion::initializePassPipeline(mlir::PassManager* pm, m
   pm->nest<mlir::FuncOp>().addPass(mlir::createTensorBufferizePass());
   pm->nest<mlir::FuncOp>().addPass(mlir::createFinalizingBufferizePass());
   pm->nest<mlir::FuncOp>().addPass(mlir::createBufferDeallocationPass());
-}
+}

mlir/include/Conversion/HiSPNtoLoSPN/ArithmeticPrecisionAnalysis.h

@@ -136,7 +136,7 @@ namespace mlir {
       /// \param isSum Boolean If set to true will treat operands as addends, otherwise as multiplicands.
       /// \return SmallVector<ErrorEstimationValue> with a single element, representing the whole Sum or Product.
       llvm::SmallVector<spn::detail::ErrorEstimationValue>
-        estimateErrorBinaryOperation(SmallVector<spn::detail::ErrorEstimationValue> operands, bool isSum);
+      estimateErrorBinaryOperation(SmallVector<spn::detail::ErrorEstimationValue> operands, bool isSum);
 
       /// Estimate the error introduced by the given (weighted) addition operation w.r.t. the current format.
       /// \param op Pointer to the defining operation, representing a SPN node.

mlir/include/Conversion/HiSPNtoLoSPN/NodePatterns.h

@@ -119,7 +119,7 @@ namespace mlir {
     };
 
     static inline void populateHiSPNtoLoSPNNodePatterns(OwningRewritePatternList& patterns, MLIRContext* context,
-                                                 TypeConverter& typeConverter) {
+                                                        TypeConverter& typeConverter) {
       patterns.insert<ProductNodeLowering, SumNodeLowering>(typeConverter, context);
       patterns.insert<HistogramNodeLowering, CategoricalNodeLowering, GaussianNodeLowering>(typeConverter, context);
       patterns.insert<RootNodeLowering>(typeConverter, context);

mlir/include/Conversion/HiSPNtoLoSPN/QueryPatterns.h

@@ -28,7 +28,7 @@ namespace mlir {
     };
 
     static inline void populateHiSPNtoLoSPNQueryPatterns(OwningRewritePatternList& patterns, MLIRContext* context,
-                                                  TypeConverter& typeConverter) {
+                                                         TypeConverter& typeConverter) {
       patterns.insert<JointQueryLowering>(typeConverter, context);
     }
 

mlir/include/Conversion/LoSPNtoCPU/LoSPNtoCPUConversionPasses.h

@@ -10,51 +10,92 @@
 #define SPNC_MLIR_INCLUDE_CONVERSION_LOSPNTOCPU_LOSPNTOCPUCONVERSIONPASSES_H
 
 #include "mlir/Pass/Pass.h"
+#include "mlir/Pass/PassOptions.h"
+#include "LoSPNtoCPU/Vectorization/SLP/Util.h"
 
 namespace mlir {
   namespace spn {
 
-    struct LoSPNtoCPUStructureConversionPass :
-        public PassWrapper<LoSPNtoCPUStructureConversionPass, OperationPass<ModuleOp>> {
-
+    struct LoSPNtoCPUStructureConversionPass : public PassWrapper<LoSPNtoCPUStructureConversionPass,
+                                                                  OperationPass<ModuleOp>> {
     public:
 
-      explicit LoSPNtoCPUStructureConversionPass(bool enableVectorization) : vectorize{enableVectorization} {}
+      LoSPNtoCPUStructureConversionPass() = default;
+      /// Constructor for accepting arguments from the driver instead of spnc-opt.
+      LoSPNtoCPUStructureConversionPass(bool vectorize,
+                                        unsigned maxAttempts,
+                                        unsigned maxSuccessfulIterations,
+                                        unsigned maxNodeSize,
+                                        unsigned maxLookAhead,
+                                        bool reorderInstructionsDFS,
+                                        bool allowDuplicateElements,
+                                        bool allowTopologicalMixing,
+                                        bool useXorChains) {
+        this->vectorize.setValue(vectorize);
+        this->maxAttempts.setValue(maxAttempts);
+        this->maxSuccessfulIterations.setValue(maxSuccessfulIterations);
+        this->maxNodeSize.setValue(maxNodeSize);
+        this->maxLookAhead.setValue(maxLookAhead);
+        this->reorderInstructionsDFS.setValue(reorderInstructionsDFS);
+        this->allowDuplicateElements.setValue(allowDuplicateElements);
+        this->allowTopologicalMixing.setValue(allowTopologicalMixing);
+        this->useXorChains.setValue(useXorChains);
+      }
+      LoSPNtoCPUStructureConversionPass(LoSPNtoCPUStructureConversionPass const& pass) : PassWrapper<
+          LoSPNtoCPUStructureConversionPass,
+          OperationPass<ModuleOp>>(pass) {}
 
-    protected:
-      void runOnOperation() override;
-
-    public:
       void getDependentDialects(DialectRegistry& registry) const override;
 
-    private:
+      Option<bool> vectorize
+          {*this, "cpu-vectorize", llvm::cl::desc("Vectorize code generated for CPU targets"), llvm::cl::init(false)};
+      Option<unsigned> maxAttempts
+          {*this, "slp-max-attempts", llvm::cl::desc("Maximum number of SLP vectorization attempts"),
+           llvm::cl::init(1)};
+      Option<unsigned> maxSuccessfulIterations{*this, "slp-max-successful-iterations", llvm::cl::desc(
+          "Maximum number of successful SLP vectorization runs to be applied to a function"), llvm::cl::init(1)};
+      Option<unsigned> maxNodeSize{*this, "slp-max-node-size", llvm::cl::desc(
+          "Maximum multinode size during SLP vectorization in terms of the number of vectors they may contain"),
+                                   llvm::cl::init(10)};
+      Option<unsigned> maxLookAhead{*this, "slp-max-look-ahead", llvm::cl::desc(
+          "Maximum look-ahead depth when reordering multinode operands during SLP vectorization"), llvm::cl::init(3)};
+      Option<bool> reorderInstructionsDFS{*this, "slp-reorder-instructions-dfs", llvm::cl::desc(
+          "Flag to indicate if SLP-vectorized instructions should be arranged in DFS order (true) or in BFS order (false)"),
+                                          llvm::cl::init(true)};
+      Option<bool> allowDuplicateElements{*this, "slp-allow-duplicate-elements", llvm::cl::desc(
+          "Flag to indicate whether duplicate elements are allowed in vectors during SLP graph building"),
+                                          llvm::cl::init(false)};
+      Option<bool> allowTopologicalMixing{*this, "slp-allow-topological-mixing", llvm::cl::desc(
+          "Flag to indicate if elements with different topological depths are allowed in vectors during SLP graph building"),
+                                          llvm::cl::init(false)};
+      Option<bool> useXorChains{*this, "slp-use-xor-chains", llvm::cl::desc(
+          "Flag to indicate if XOR chains should be used to compute look-ahead scores instead of Porpodas's algorithm"),
+                                llvm::cl::init(true)};
 
-      bool vectorize;
+    protected:
+      void runOnOperation() override;
 
     };
 
-    std::unique_ptr<Pass> createLoSPNtoCPUStructureConversionPass(bool enableVectorization);
+    struct LoSPNtoCPUNodeConversionPass : public PassWrapper<LoSPNtoCPUNodeConversionPass, OperationPass<ModuleOp>> {
 
-    struct LoSPNtoCPUNodeConversionPass :
-        public PassWrapper<LoSPNtoCPUNodeConversionPass, OperationPass<ModuleOp>> {
+    public:
+      void getDependentDialects(DialectRegistry& registry) const override;
 
     protected:
       void runOnOperation() override;
 
-    public:
-      void getDependentDialects(DialectRegistry& registry) const override;
-
     };
 
     std::unique_ptr<Pass> createLoSPNtoCPUNodeConversionPass();
 
     struct LoSPNNodeVectorizationPass : public PassWrapper<LoSPNNodeVectorizationPass, OperationPass<ModuleOp>> {
 
-    protected:
-      void runOnOperation() override;
-
     public:
       void getDependentDialects(DialectRegistry& registry) const override;
+
+    protected:
+      void runOnOperation() override;
     };
 
     std::unique_ptr<Pass> createLoSPNNodeVectorizationPass();

mlir/include/Conversion/LoSPNtoCPU/StructurePatterns.h

@@ -54,10 +54,16 @@ namespace mlir {
     };
 
     static inline void populateLoSPNtoCPUStructurePatterns(OwningRewritePatternList& patterns, MLIRContext* context,
-                                                    TypeConverter& typeConverter) {
-      patterns.insert<KernelLowering, BatchTaskLowering, SingleTaskLowering>(typeConverter, context);
+                                                           TypeConverter& typeConverter) {
+      patterns.insert<KernelLowering>(typeConverter, context);
       patterns.insert<BodyLowering>(typeConverter, context);
     }
+
+    static inline void populateLoSPNtoCPUTaskPatterns(OwningRewritePatternList& patterns,
+                                                      MLIRContext* context,
+                                                      TypeConverter& typeConverter) {
+      patterns.insert<BatchTaskLowering, SingleTaskLowering>(typeConverter, context, 1);
+    }
   }
 }