Merge branch 'main' into fix_nightly_test

docs/sphinx/api/default_ops.rst

@@ -631,8 +631,9 @@ operations, each operating on 2 qubits.
 
 .. note:: 
 
-  Custom operations are currently supported only on :doc:`../using/backends/simulators`.
-  Attempt to use with a hardware backend will result in runtime error.
+  When a custom operation is used on hardware backends, it is synthesized to a
+  set of native quantum operations. Currently, only 1-qubit and 2-qubit custom 
+  operations are supported on hardware backends.
 
 
 Photonic Operations on Qudits

include/cudaq/Optimizer/Dialect/CC/CCOps.td

@@ -10,6 +10,7 @@
 #define CUDAQ_OPTIMIZER_DIALECT_CC_OPS
 
 include "mlir/Interfaces/InferTypeOpInterface.td"
+include "mlir/IR/SymbolInterfaces.td"
 include "cudaq/Optimizer/Dialect/Common/Traits.td"
 include "cudaq/Optimizer/Dialect/CC/CCDialect.td"
 include "cudaq/Optimizer/Dialect/CC/CCInterfaces.td"
@@ -911,19 +912,35 @@ def cc_GlobalOp : CCOp<"global", [IsolatedFromAbove, Symbol]> {
 
   let arguments = (ins
     TypeAttr:$global_type,
-    StrAttr:$sym_name,
+    SymbolNameAttr:$sym_name,
     OptionalAttr<AnyAttr>:$value,
+    OptionalAttr<StrAttr>:$sym_visibility,
     UnitAttr:$constant,
     UnitAttr:$external
   );
 
   let hasCustomAssemblyFormat = 1;
 
+  let builders = [
+    OpBuilder<(ins
+      "mlir::Type":$type, "mlir::StringRef":$name,
+      "mlir::Attribute":$value, "bool":$constant, "bool":$external), [{
+        return build($_builder, $_state, type, name, value,
+                     mlir::StringAttr{}, constant, external);
+      }]
+    >];
+
   let extraClassDeclaration = [{
     cudaq::cc::PointerType getType() {
       auto globalTy = getGlobalType();
       return cudaq::cc::PointerType::get(globalTy);
     }
+
+    //===------------------------------------------------------------------===//
+    // SymbolOpInterface Methods
+    //===------------------------------------------------------------------===//
+
+    bool isDeclaration() { return getExternal(); }
   }];
 }
 

include/cudaq/Optimizer/Transforms/Passes.td

@@ -407,7 +407,7 @@ def GetConcreteMatrix : Pass<"get-concrete-matrix", "mlir::func::FuncOp"> {
           return
         }
 
-        cc.global constant @__nvqpp__mlirgen__function_foo_generator_1.bar.rodata_0 ((dense<[(0.000000e+00,0.000000e+00), (1.000000e+00,0.000000e+00), (1.000000e+00,0.000000e+00), (0.000000e+00,0.000000e+00)]> : tensor<4xcomplex<f64>>) : !cc.array<complex<f64> x 4>
+        cc.global constant private @__nvqpp__mlirgen__function_foo_generator_1.bar.rodata_0 ((dense<[(0.000000e+00,0.000000e+00), (1.000000e+00,0.000000e+00), (1.000000e+00,0.000000e+00), (0.000000e+00,0.000000e+00)]> : tensor<4xcomplex<f64>>) : !cc.array<complex<f64> x 4>
       }
     ```
 
@@ -816,7 +816,7 @@ def StatePreparation : Pass<"state-prep", "mlir::ModuleOp"> {
         %2 = quake.init_state %1, %0 : (!quake.veq<2>, !cc.ptr<!cc.array<complex<f32> x 4>>) -> !quake.veq<2>
         return
       }
-      cc.global constant @foo.rodata_0 (dense<[(0.707106769,0.000000e+00), (0.707106769,0.000000e+00), (0.000000e+00,0.000000e+00), (0.000000e+00,0.000000e+00)]> : tensor<4xcomplex<f32>>) : !cc.array<complex<f32> x 4>
+      cc.global constant private @foo.rodata_0 (dense<[(0.707106769,0.000000e+00), (0.707106769,0.000000e+00), (0.000000e+00,0.000000e+00), (0.000000e+00,0.000000e+00)]> : tensor<4xcomplex<f32>>) : !cc.array<complex<f32> x 4>
     }
     ```
     Will be rewritten to:

lib/Frontend/nvqpp/ConvertExpr.cpp

@@ -2498,8 +2498,10 @@ bool QuakeBridgeVisitor::VisitInitListExpr(clang::InitListExpr *x) {
     {
       OpBuilder::InsertionGuard guard(builder);
       builder.setInsertionPointToEnd(module.getBody());
-      builder.create<cc::GlobalOp>(loc, globalTy, name, f64Attr,
-                                   /*constant=*/true);
+      builder
+          .create<cc::GlobalOp>(loc, globalTy, name, f64Attr,
+                                /*constant=*/true, /*external=*/false)
+          .setPrivate();
     }
     auto ptrTy = cc::PointerType::get(globalTy);
     auto globalInit = builder.create<cc::AddressOfOp>(loc, ptrTy, name);

lib/Optimizer/Builder/Intrinsics.cpp

@@ -481,9 +481,12 @@ static cc::GlobalOp buildVectorOfConstantElements(Location loc, ModuleOp module,
   OpBuilder::InsertionGuard guard(builder);
   builder.setInsertionPointToEnd(module.getBody());
   auto globalTy = cc::ArrayType::get(ctx, eleTy, arrayAttr.size());
-  return builder.create<cudaq::cc::GlobalOp>(loc, globalTy, name, arrayAttr,
-                                             /*constant=*/true,
-                                             /*external=*/false);
+  auto global =
+      builder.create<cudaq::cc::GlobalOp>(loc, globalTy, name, arrayAttr,
+                                          /*constant=*/true,
+                                          /*external=*/false);
+  global.setPrivate();
+  return global;
 }
 
 template <typename A>

lib/Optimizer/CodeGen/Pipelines.cpp

@@ -50,6 +50,15 @@ void cudaq::opt::addPipelineTranslateToOpenQASM(PassManager &pm) {
   pm.addNestedPass<func::FuncOp>(createLiftArrayAlloc());
   pm.addPass(createGlobalizeArrayValues());
   pm.addPass(createStatePreparation());
+  pm.addNestedPass<func::FuncOp>(createGetConcreteMatrix());
+  pm.addPass(createUnitarySynthesis());
+  pm.addPass(createSymbolDCEPass());
+  pm.addPass(createCanonicalizerPass());
+  pm.addPass(createCSEPass());
+  pm.addNestedPass<func::FuncOp>(createMultiControlDecompositionPass());
+  pm.addPass(createDecompositionPass(
+      {.enabledPatterns = {"CCZToCX", "RxAdjToRx", "RyAdjToRy", "RzAdjToRz"}}));
+  pm.addPass(createCanonicalizerPass());
 }
 
 void cudaq::opt::addPipelineTranslateToIQMJson(PassManager &pm) {

lib/Optimizer/Dialect/CC/CCOps.cpp

@@ -1003,6 +1003,14 @@ ParseResult cudaq::cc::GlobalOp::parse(OpAsmParser &parser,
     result.addAttribute(getConstantAttrName(result.name),
                         parser.getBuilder().getUnitAttr());
 
+  // Check for the visibility optional keyword third.
+  StringRef visibility;
+  if (parser.parseOptionalKeyword(&visibility, {"public", "private", "nested"}))
+    return failure();
+
+  StringAttr visibilityAttr = parser.getBuilder().getStringAttr(visibility);
+  result.addAttribute(SymbolTable::getVisibilityAttrName(), visibilityAttr);
+
   // Parse the rest of the global.
   //   @<symbol> ( <initializer-attr> ) : <result-type>
   StringAttr name;
@@ -1033,17 +1041,23 @@ void cudaq::cc::GlobalOp::print(OpAsmPrinter &p) {
     p << "extern ";
   if (getConstant())
     p << "constant ";
+
+  if (auto visibility = getSymVisibility())
+    if (visibility.has_value())
+      p << visibility.value().str() << ' ';
+
   p.printSymbolName(getSymName());
   if (auto value = getValue()) {
     p << " (";
     p.printAttribute(*value);
     p << ")";
   }
   p << " : " << getGlobalType();
-  p.printOptionalAttrDictWithKeyword(
-      (*this)->getAttrs(),
-      {getSymNameAttrName(), getValueAttrName(), getGlobalTypeAttrName(),
-       getConstantAttrName(), getExternalAttrName()});
+
+  p.printOptionalAttrDict((*this)->getAttrs(),
+                          {getSymNameAttrName(), getValueAttrName(),
+                           getGlobalTypeAttrName(), getConstantAttrName(),
+                           getExternalAttrName(), getSymVisibilityAttrName()});
 }
 
 //===----------------------------------------------------------------------===//

lib/Optimizer/Transforms/DecompositionPatterns.cpp

@@ -1181,6 +1181,41 @@ struct RxToPhasedRx : public OpRewritePattern<quake::RxOp> {
   }
 };
 
+// quake.rx<adj> (θ) target
+// ─────────────────────────────────
+// quake.rx(-θ) target
+struct RxAdjToRx : public OpRewritePattern<quake::RxOp> {
+  using OpRewritePattern<quake::RxOp>::OpRewritePattern;
+
+  void initialize() { setDebugName("RxAdjToRx"); }
+
+  LogicalResult matchAndRewrite(quake::RxOp op,
+                                PatternRewriter &rewriter) const override {
+    if (!op.getControls().empty())
+      return failure();
+
+    if (!op.isAdj())
+      return failure();
+
+    // Op info
+    Location loc = op->getLoc();
+    Value target = op.getTarget();
+    Value angle = op.getParameter();
+    angle = rewriter.create<arith::NegFOp>(loc, angle);
+
+    // Necessary/Helpful constants
+    SmallVector<Value> noControls;
+    SmallVector<Value> parameters = {angle};
+
+    QuakeOperatorCreator qRewriter(rewriter);
+    qRewriter.create<quake::RxOp>(loc, parameters, noControls, target);
+
+    qRewriter.selectWiresAndReplaceUses(op, target);
+    rewriter.eraseOp(op);
+    return success();
+  }
+};
+
 //===----------------------------------------------------------------------===//
 // RyOp decompositions
 //===----------------------------------------------------------------------===//
@@ -1267,6 +1302,41 @@ struct RyToPhasedRx : public OpRewritePattern<quake::RyOp> {
   }
 };
 
+// quake.ry<adj> (θ) target
+// ─────────────────────────────────
+// quake.ry(-θ) target
+struct RyAdjToRy : public OpRewritePattern<quake::RyOp> {
+  using OpRewritePattern<quake::RyOp>::OpRewritePattern;
+
+  void initialize() { setDebugName("RyAdjToRy"); }
+
+  LogicalResult matchAndRewrite(quake::RyOp op,
+                                PatternRewriter &rewriter) const override {
+    if (!op.getControls().empty())
+      return failure();
+
+    if (!op.isAdj())
+      return failure();
+
+    // Op info
+    Location loc = op->getLoc();
+    Value target = op.getTarget();
+    Value angle = op.getParameter();
+    angle = rewriter.create<arith::NegFOp>(loc, angle);
+
+    // Necessary/Helpful constants
+    SmallVector<Value> noControls;
+    SmallVector<Value> parameters = {angle};
+
+    QuakeOperatorCreator qRewriter(rewriter);
+    qRewriter.create<quake::RyOp>(loc, parameters, noControls, target);
+
+    qRewriter.selectWiresAndReplaceUses(op, target);
+    rewriter.eraseOp(op);
+    return success();
+  }
+};
+
 //===----------------------------------------------------------------------===//
 // RzOp decompositions
 //===----------------------------------------------------------------------===//
@@ -1364,6 +1434,41 @@ struct RzToPhasedRx : public OpRewritePattern<quake::RzOp> {
   }
 };
 
+// quake.rz<adj> (θ) target
+// ─────────────────────────────────
+// quake.rz(-θ) target
+struct RzAdjToRz : public OpRewritePattern<quake::RzOp> {
+  using OpRewritePattern<quake::RzOp>::OpRewritePattern;
+
+  void initialize() { setDebugName("RzAdjToRz"); }
+
+  LogicalResult matchAndRewrite(quake::RzOp op,
+                                PatternRewriter &rewriter) const override {
+    if (!op.getControls().empty())
+      return failure();
+
+    if (!op.isAdj())
+      return failure();
+
+    // Op info
+    Location loc = op->getLoc();
+    Value target = op.getTarget();
+    Value angle = op.getParameter();
+    angle = rewriter.create<arith::NegFOp>(loc, angle);
+
+    // Necessary/Helpful constants
+    SmallVector<Value> noControls;
+    SmallVector<Value> parameters = {angle};
+
+    QuakeOperatorCreator qRewriter(rewriter);
+    qRewriter.create<quake::RzOp>(loc, parameters, noControls, target);
+
+    qRewriter.selectWiresAndReplaceUses(op, target);
+    rewriter.eraseOp(op);
+    return success();
+  }
+};
+
 //===----------------------------------------------------------------------===//
 // U3Op decompositions
 //===----------------------------------------------------------------------===//
@@ -1449,12 +1554,15 @@ void cudaq::populateWithAllDecompositionPatterns(RewritePatternSet &patterns) {
     // RxOp patterns
     CRxToCX,
     RxToPhasedRx,
+    RxAdjToRx,
     // RyOp patterns
     CRyToCX,
     RyToPhasedRx,
+    RyAdjToRy,
     // RzOp patterns
     CRzToCX,
     RzToPhasedRx,
+    RzAdjToRz,
     // Swap
     SwapToCX,
     // U3Op

lib/Optimizer/Transforms/GetConcreteMatrix.cpp

@@ -47,6 +47,8 @@ class CustomUnitaryPattern
     if (!funcOp)
       return failure();
 
+    funcOp.setPrivate();
+
     // The generator function returns a concrete matrix. If prior passes have
     // run to constant fold and lift array values, the generator function will
     // have address of the global variable which holds the concrete matrix.

lib/Optimizer/Transforms/StatePreparation.cpp

@@ -296,7 +296,8 @@ class StateDecomposer {
 ///       !cc.ptr<!cc.array<complex<f32> x 4>>) -> !quake.veq<2>
 ///     return
 ///  }
-///  cc.global constant @foo.rodata_0 (dense<[(0.707106769,0.000000e+00),
+///  cc.global constant private @foo.rodata_0
+///  (dense<[(0.707106769,0.000000e+00),
 ///      (0.707106769,0.000000e+00), (0.000000e+00,0.000000e+00),
 ///      (0.000000e+00,0.000000e+00)]> : tensor<4xcomplex<f32>>) :
 ///    !cc.array<complex<f32> x 4>

python/cudaq/kernel/captured_data.py

@@ -82,7 +82,10 @@ def storeCudaqState(self, value: cudaq_runtime.State):
         globalName = f'nvqpp.cudaq.state.{stateID}'
         setStateName = f'nvqpp.set.cudaq.state.{stateID}'
         with InsertionPoint.at_block_begin(self.module.body):
-            cc.GlobalOp(TypeAttr.get(globalTy), globalName, external=True)
+            cc.GlobalOp(TypeAttr.get(globalTy),
+                        globalName,
+                        sym_visibility=StringAttr.get("private"),
+                        external=True)
             setStateFunc = func.FuncOp(setStateName,
                                        FunctionType.get(inputs=[statePtrTy],
                                                         results=[]),
@@ -133,7 +136,10 @@ def storeArray(self, array: np.ndarray):
         globalName = f'nvqpp.state.{arrayId}'
         setStateName = f'nvqpp.set.state.{arrayId}'
         with InsertionPoint.at_block_begin(self.module.body):
-            cc.GlobalOp(TypeAttr.get(globalTy), globalName, external=True)
+            cc.GlobalOp(TypeAttr.get(globalTy),
+                        globalName,
+                        sym_visibility=StringAttr.get("private"),
+                        external=True)
             setStateFunc = func.FuncOp(setStateName,
                                        FunctionType.get(inputs=[ptrComplex],
                                                         results=[]),

python/tests/mlir/custom_op_builder.py

@@ -80,6 +80,6 @@ def register_custom_operations(matrix):
 # CHECK:           quake.custom_op @__nvqpp__mlirgen__foo_2_generator_1.rodata {{\[}}%[[VAL_4]]] %[[VAL_3]] : (!quake.veq<2>, !quake.veq<1>) -> ()
 # CHECK:           return
 # CHECK:         }
-# CHECK-DAG:         cc.global constant @__nvqpp__mlirgen__foo_0_generator_1.rodata (dense<[{{.*}}]> : tensor<4xcomplex<f64>>) : !cc.array<complex<f64> x 4>
-# CHECK-DAG:         cc.global constant @__nvqpp__mlirgen__foo_1_generator_1.rodata (dense<[{{.*}}]> : tensor<4xcomplex<f64>>) : !cc.array<complex<f64> x 4>
-# CHECK-DAG:         cc.global constant @__nvqpp__mlirgen__foo_2_generator_1.rodata (dense<[{{.*}}]> : tensor<4xcomplex<f64>>) : !cc.array<complex<f64> x 4>
+# CHECK-DAG:     cc.global constant private @__nvqpp__mlirgen__foo_0_generator_1.rodata (dense<[{{.*}}]> : tensor<4xcomplex<f64>>) : !cc.array<complex<f64> x 4>
+# CHECK-DAG:     cc.global constant private @__nvqpp__mlirgen__foo_1_generator_1.rodata (dense<[{{.*}}]> : tensor<4xcomplex<f64>>) : !cc.array<complex<f64> x 4>
+# CHECK-DAG:     cc.global constant private @__nvqpp__mlirgen__foo_2_generator_1.rodata (dense<[{{.*}}]> : tensor<4xcomplex<f64>>) : !cc.array<complex<f64> x 4>

python/tests/mlir/custom_operation.py

@@ -35,8 +35,8 @@ def bell():
 # CHECK:           quake.custom_op @__nvqpp__mlirgen__custom_x_generator_1.rodata {{\[}}%[[VAL_1]]] %[[VAL_2]] : (!quake.ref, !quake.ref) -> ()
 # CHECK:           return
 # CHECK:         }
-# CHECK:         cc.global constant @__nvqpp__mlirgen__custom_h_generator_1.rodata (dense<[(0.70710678118654746,0.000000e+00), (0.70710678118654746,0.000000e+00), (0.70710678118654746,0.000000e+00), (-0.70710678118654746,0.000000e+00)]> : tensor<4xcomplex<f64>>) : !cc.array<complex<f64> x 4>
-# CHECK:         cc.global constant @__nvqpp__mlirgen__custom_x_generator_1.rodata (dense<[(0.000000e+00,0.000000e+00), (1.000000e+00,0.000000e+00), (1.000000e+00,0.000000e+00), (0.000000e+00,0.000000e+00)]> : tensor<4xcomplex<f64>>) : !cc.array<complex<f64> x 4>
+# CHECK:         cc.global constant private @__nvqpp__mlirgen__custom_h_generator_1.rodata (dense<[(0.70710678118654746,0.000000e+00), (0.70710678118654746,0.000000e+00), (0.70710678118654746,0.000000e+00), (-0.70710678118654746,0.000000e+00)]> : tensor<4xcomplex<f64>>) : !cc.array<complex<f64> x 4>
+# CHECK:         cc.global constant private @__nvqpp__mlirgen__custom_x_generator_1.rodata (dense<[(0.000000e+00,0.000000e+00), (1.000000e+00,0.000000e+00), (1.000000e+00,0.000000e+00), (0.000000e+00,0.000000e+00)]> : tensor<4xcomplex<f64>>) : !cc.array<complex<f64> x 4>
 
 
 def test_custom_adjoint():
@@ -64,5 +64,5 @@ def kernel():
 # CHECK:           quake.h %[[VAL_0]] : (!quake.ref) -> ()
 # CHECK:           return
 # CHECK:         }
-# CHECK:         cc.global constant @__nvqpp__mlirgen__custom_s_generator_1.rodata (dense<[(1.000000e+00,0.000000e+00), (0.000000e+00,0.000000e+00), (0.000000e+00,0.000000e+00), (0.000000e+00,1.000000e+00)]> : tensor<4xcomplex<f64>>) : !cc.array<complex<f64> x 4>
-# CHECK:         cc.global constant @__nvqpp__mlirgen__custom_s_adj_generator_1.rodata (dense<[(1.000000e+00,0.000000e+00), (0.000000e+00,0.000000e+00), (0.000000e+00,0.000000e+00), (-0.000000e+00,-1.000000e+00)]> : tensor<4xcomplex<f64>>) : !cc.array<complex<f64> x 4>
+# CHECK:         cc.global constant private @__nvqpp__mlirgen__custom_s_generator_1.rodata (dense<[(1.000000e+00,0.000000e+00), (0.000000e+00,0.000000e+00), (0.000000e+00,0.000000e+00), (0.000000e+00,1.000000e+00)]> : tensor<4xcomplex<f64>>) : !cc.array<complex<f64> x 4>
+# CHECK:          cc.global constant private @__nvqpp__mlirgen__custom_s_adj_generator_1.rodata (dense<[(1.000000e+00,0.000000e+00), (0.000000e+00,0.000000e+00), (0.000000e+00,0.000000e+00), (-0.000000e+00,-1.000000e+00)]> : tensor<4xcomplex<f64>>) : !cc.array<complex<f64> x 4

runtime/cudaq/platform/default/rest/helpers/braket/braket.yml

@@ -17,7 +17,7 @@ config:
   # Tell NVQ++ to generate glue code to set the target backend name
   gen-target-backend: true
   # Define the lowering pipeline
-  platform-lowering-config: "func.func(const-prop-complex,canonicalize,cse,lift-array-alloc),globalize-array-values,state-prep,unitary-synthesis,canonicalize,apply-op-specialization,aggressive-early-inlining,unrolling-pipeline,func.func(lower-to-cfg),canonicalize,func.func(multicontrol-decomposition),func.func(memtoreg{quantum=0}),symbol-dce"
+  platform-lowering-config: "func.func(const-prop-complex,canonicalize,cse,lift-array-alloc),globalize-array-values,state-prep,unitary-synthesis,canonicalize,apply-op-specialization,aggressive-early-inlining,unrolling-pipeline,func.func(lower-to-cfg),canonicalize,func.func(multicontrol-decomposition),decomposition{enable-patterns=CCZToCX,RxAdjToRx,RyAdjToRy,RzAdjToRz},func.func(memtoreg{quantum=0}),symbol-dce"
   # Tell the rest-qpu that we are generating OpenQASM 2.0.
   codegen-emission: qasm2
   # Library mode is only for simulators, physical backends must turn this off