feat: native guest macro for kernel functions

Cargo.toml

@@ -35,6 +35,9 @@ members = [
     "extensions/native/compiler",
     "extensions/native/compiler/derive",
     "extensions/native/recursion",
+    "extensions/native/transpiler",
+    "extensions/native/guest-macro",
+    "extensions/native/tests",
     "extensions/algebra/circuit",
     "extensions/algebra/transpiler",
     "extensions/algebra/guest",
@@ -135,6 +138,9 @@ openvm-native-circuit = { path = "extensions/native/circuit", default-features =
 openvm-native-compiler = { path = "extensions/native/compiler", default-features = false }
 openvm-native-compiler-derive = { path = "extensions/native/compiler/derive", default-features = false }
 openvm-native-recursion = { path = "extensions/native/recursion", default-features = false }
+openvm-native-guest-macro = { path = "extensions/native/guest-macro", default-features = false }
+openvm-native-integration-test = { path = "extensions/native/guest-macro/test", default-features = false }
+openvm-native-transpiler = { path = "extensions/native/transpiler", default-features = false }
 openvm-keccak256-circuit = { path = "extensions/keccak256/circuit", default-features = false }
 openvm-keccak256-transpiler = { path = "extensions/keccak256/transpiler", default-features = false }
 openvm-keccak256-guest = { path = "extensions/keccak256/guest", default-features = false }

benchmarks/Cargo.toml

@@ -15,6 +15,7 @@ openvm-sdk.workspace = true
 openvm-stark-backend.workspace = true
 openvm-stark-sdk.workspace = true
 openvm-transpiler.workspace = true
+openvm-native-transpiler.workspace = true
 
 openvm-algebra-circuit.workspace = true
 openvm-algebra-transpiler.workspace = true

crates/toolchain/build/src/lib.rs

@@ -34,7 +34,12 @@ pub fn get_package(manifest_dir: impl AsRef<Path>) -> Package {
         .manifest_path(&manifest_path)
         .no_deps()
         .exec()
-        .expect("cargo metadata command failed");
+        .unwrap_or_else(|e| {
+            panic!(
+                "cargo metadata command failed for manifest path: {}: {e:?}",
+                manifest_path.display()
+            )
+        });
     let mut matching: Vec<Package> = manifest_meta
         .packages
         .into_iter()

crates/toolchain/instructions/src/instruction.rs

@@ -90,6 +90,10 @@ impl<F: Field> Instruction<F> {
             ..Default::default()
         }
     }
+
+    pub fn operands(&self) -> Vec<F> {
+        vec![self.a, self.b, self.c, self.d, self.e, self.f, self.g]
+    }
 }
 
 impl<T: Default> Default for Instruction<T> {

crates/toolchain/transpiler/src/extension.rs

@@ -8,7 +8,35 @@ pub trait TranspilerExtension<F> {
     /// The `instruction_stream` provides a view of the remaining RISC-V instructions to be processed,
     /// presented as 32-bit chunks. The [process_custom](Self::process_custom) should determine if it knows how to transpile
     /// the next contiguous section of RISC-V instructions into an [`Instruction`].
-    /// It returns `None` if it cannot transpile. Otherwise it returns `(instruction, how_many_u32s)` to indicate that
-    /// `instruction_stream[..how_many_u32s]` should be transpiled into `instruction`.
-    fn process_custom(&self, instruction_stream: &[u32]) -> Option<(Instruction<F>, usize)>;
+    /// It returns `None` if it cannot transpile. Otherwise it returns `TranspilerOutput { instructions, used_u32s }` to indicate that
+    /// `instruction_stream[..used_u32s]` should be transpiled into `instructions`.
+    fn process_custom(&self, instruction_stream: &[u32]) -> Option<TranspilerOutput<F>>;
+}
+
+pub struct TranspilerOutput<F> {
+    pub instructions: Vec<Option<Instruction<F>>>,
+    pub used_u32s: usize,
+}
+
+impl<F> TranspilerOutput<F> {
+    pub fn one_to_one(instruction: Instruction<F>) -> Self {
+        Self {
+            instructions: vec![Some(instruction)],
+            used_u32s: 1,
+        }
+    }
+
+    pub fn many_to_one(instruction: Instruction<F>, used_u32s: usize) -> Self {
+        Self {
+            instructions: vec![Some(instruction)],
+            used_u32s,
+        }
+    }
+
+    pub fn gap(gap_length: usize, used_u32s: usize) -> Self {
+        Self {
+            instructions: (0..gap_length).map(|_| None).collect(),
+            used_u32s,
+        }
+    }
 }

crates/toolchain/transpiler/src/lib.rs

@@ -16,7 +16,7 @@ pub mod transpiler;
 pub mod util;
 
 mod extension;
-pub use extension::TranspilerExtension;
+pub use extension::{TranspilerExtension, TranspilerOutput};
 
 pub trait FromElf {
     type ElfContext;
@@ -29,7 +29,7 @@ impl<F: PrimeField32> FromElf for VmExe<F> {
     type ElfContext = Transpiler<F>;
     fn from_elf(elf: Elf, transpiler: Self::ElfContext) -> Result<Self, TranspilerError> {
         let instructions = transpiler.transpile(&elf.instructions)?;
-        let program = Program::new_without_debug_infos(
+        let program = Program::new_without_debug_infos_with_option(
             &instructions,
             DEFAULT_PC_STEP,
             elf.pc_base,

crates/toolchain/transpiler/src/transpiler.rs

@@ -50,7 +50,7 @@ impl<F: PrimeField32> Transpiler<F> {
     pub fn transpile(
         &self,
         instructions_u32: &[u32],
-    ) -> Result<Vec<Instruction<F>>, TranspilerError> {
+    ) -> Result<Vec<Option<Instruction<F>>>, TranspilerError> {
         let mut instructions = Vec::new();
         let mut ptr = 0;
         while ptr < instructions_u32.len() {
@@ -66,9 +66,9 @@ impl<F: PrimeField32> Transpiler<F> {
             if options.len() > 1 {
                 return Err(TranspilerError::AmbiguousNextInstruction);
             }
-            let (instruction, advance) = options.pop().unwrap().unwrap();
-            instructions.push(instruction);
-            ptr += advance;
+            let transpiler_output = options.pop().unwrap().unwrap();
+            instructions.extend(transpiler_output.instructions);
+            ptr += transpiler_output.used_u32s;
         }
         Ok(instructions)
     }

extensions/algebra/transpiler/src/lib.rs

@@ -7,7 +7,7 @@ use openvm_instructions::{
 };
 use openvm_instructions_derive::UsizeOpcode;
 use openvm_stark_backend::p3_field::PrimeField32;
-use openvm_transpiler::{util::from_r_type, TranspilerExtension};
+use openvm_transpiler::{util::from_r_type, TranspilerExtension, TranspilerOutput};
 use rrs_lib::instruction_formats::RType;
 use strum::{EnumCount, EnumIter, FromRepr};
 
@@ -50,7 +50,7 @@ pub struct ModularTranspilerExtension;
 pub struct Fp2TranspilerExtension;
 
 impl<F: PrimeField32> TranspilerExtension<F> for ModularTranspilerExtension {
-    fn process_custom(&self, instruction_stream: &[u32]) -> Option<(Instruction<F>, usize)> {
+    fn process_custom(&self, instruction_stream: &[u32]) -> Option<TranspilerOutput<F>> {
         if instruction_stream.is_empty() {
             return None;
         }
@@ -122,12 +122,12 @@ impl<F: PrimeField32> TranspilerExtension<F> for ModularTranspilerExtension {
                 Some(from_r_type(global_opcode, 2, &dec_insn))
             }
         };
-        instruction.map(|instruction| (instruction, 1))
+        instruction.map(TranspilerOutput::one_to_one)
     }
 }
 
 impl<F: PrimeField32> TranspilerExtension<F> for Fp2TranspilerExtension {
-    fn process_custom(&self, instruction_stream: &[u32]) -> Option<(Instruction<F>, usize)> {
+    fn process_custom(&self, instruction_stream: &[u32]) -> Option<TranspilerOutput<F>> {
         if instruction_stream.is_empty() {
             return None;
         }
@@ -190,6 +190,6 @@ impl<F: PrimeField32> TranspilerExtension<F> for Fp2TranspilerExtension {
                 Some(from_r_type(global_opcode, 2, &dec_insn))
             }
         };
-        instruction.map(|instruction| (instruction, 1))
+        instruction.map(TranspilerOutput::one_to_one)
     }
 }

extensions/bigint/transpiler/src/lib.rs

@@ -8,7 +8,7 @@ use openvm_rv32im_transpiler::{
     BaseAluOpcode, BranchEqualOpcode, BranchLessThanOpcode, LessThanOpcode, MulOpcode, ShiftOpcode,
 };
 use openvm_stark_backend::p3_field::PrimeField32;
-use openvm_transpiler::{util::from_r_type, TranspilerExtension};
+use openvm_transpiler::{util::from_r_type, TranspilerExtension, TranspilerOutput};
 use rrs_lib::instruction_formats::{BType, RType};
 use strum::IntoEnumIterator;
 
@@ -80,7 +80,7 @@ impl Rv32Mul256Opcode {
 pub struct Int256TranspilerExtension;
 
 impl<F: PrimeField32> TranspilerExtension<F> for Int256TranspilerExtension {
-    fn process_custom(&self, instruction_stream: &[u32]) -> Option<(Instruction<F>, usize)> {
+    fn process_custom(&self, instruction_stream: &[u32]) -> Option<TranspilerOutput<F>> {
         if instruction_stream.is_empty() {
             return None;
         }
@@ -154,6 +154,6 @@ impl<F: PrimeField32> TranspilerExtension<F> for Int256TranspilerExtension {
             }
             _ => None,
         };
-        instruction.map(|instruction| (instruction, 1))
+        instruction.map(TranspilerOutput::one_to_one)
     }
 }

extensions/ecc/transpiler/src/lib.rs

@@ -5,7 +5,7 @@ use openvm_instructions::{
 };
 use openvm_instructions_derive::UsizeOpcode;
 use openvm_stark_backend::p3_field::PrimeField32;
-use openvm_transpiler::{util::from_r_type, TranspilerExtension};
+use openvm_transpiler::{util::from_r_type, TranspilerExtension, TranspilerOutput};
 use rrs_lib::instruction_formats::RType;
 use strum::{EnumCount, EnumIter, FromRepr};
 
@@ -32,7 +32,7 @@ pub enum EccPhantom {
 pub struct EccTranspilerExtension;
 
 impl<F: PrimeField32> TranspilerExtension<F> for EccTranspilerExtension {
-    fn process_custom(&self, instruction_stream: &[u32]) -> Option<(Instruction<F>, usize)> {
+    fn process_custom(&self, instruction_stream: &[u32]) -> Option<TranspilerOutput<F>> {
         if instruction_stream.is_empty() {
             return None;
         }
@@ -59,15 +59,12 @@ impl<F: PrimeField32> TranspilerExtension<F> for EccTranspilerExtension {
             let curve_idx_shift = curve_idx * Rv32WeierstrassOpcode::COUNT;
             if let Some(SwBaseFunct7::HintDecompress) = SwBaseFunct7::from_repr(base_funct7) {
                 assert_eq!(dec_insn.rd, 0);
-                return Some((
-                    Instruction::phantom(
-                        PhantomDiscriminant(EccPhantom::HintDecompress as u16),
-                        F::from_canonical_usize(RV32_REGISTER_NUM_LIMBS * dec_insn.rs1),
-                        F::from_canonical_usize(RV32_REGISTER_NUM_LIMBS * dec_insn.rs2),
-                        curve_idx as u16,
-                    ),
-                    1,
-                ));
+                return Some(TranspilerOutput::one_to_one(Instruction::phantom(
+                    PhantomDiscriminant(EccPhantom::HintDecompress as u16),
+                    F::from_canonical_usize(RV32_REGISTER_NUM_LIMBS * dec_insn.rs1),
+                    F::from_canonical_usize(RV32_REGISTER_NUM_LIMBS * dec_insn.rs2),
+                    curve_idx as u16,
+                )));
             }
             if base_funct7 == SwBaseFunct7::SwSetup as u8 {
                 let local_opcode = match dec_insn.rs2 {
@@ -101,6 +98,6 @@ impl<F: PrimeField32> TranspilerExtension<F> for EccTranspilerExtension {
                 Some(from_r_type(global_opcode, 2, &dec_insn))
             }
         };
-        instruction.map(|instruction| (instruction, 1))
+        instruction.map(TranspilerOutput::one_to_one)
     }
 }

extensions/keccak256/transpiler/src/lib.rs

@@ -1,8 +1,8 @@
-use openvm_instructions::{instruction::Instruction, UsizeOpcode};
+use openvm_instructions::UsizeOpcode;
 use openvm_instructions_derive::UsizeOpcode;
 use openvm_keccak256_guest::{KECCAK256_FUNCT3, KECCAK256_FUNCT7, OPCODE};
 use openvm_stark_backend::p3_field::PrimeField32;
-use openvm_transpiler::{util::from_r_type, TranspilerExtension};
+use openvm_transpiler::{util::from_r_type, TranspilerExtension, TranspilerOutput};
 use rrs_lib::instruction_formats::RType;
 use strum::{EnumCount, EnumIter, FromRepr};
 
@@ -19,7 +19,7 @@ pub enum Rv32KeccakOpcode {
 pub struct Keccak256TranspilerExtension;
 
 impl<F: PrimeField32> TranspilerExtension<F> for Keccak256TranspilerExtension {
-    fn process_custom(&self, instruction_stream: &[u32]) -> Option<(Instruction<F>, usize)> {
+    fn process_custom(&self, instruction_stream: &[u32]) -> Option<TranspilerOutput<F>> {
         if instruction_stream.is_empty() {
             return None;
         }
@@ -39,6 +39,6 @@ impl<F: PrimeField32> TranspilerExtension<F> for Keccak256TranspilerExtension {
             2,
             &dec_insn,
         );
-        Some((instruction, 1))
+        Some(TranspilerOutput::one_to_one(instruction))
     }
 }

extensions/native/circuit/Cargo.toml

@@ -29,7 +29,6 @@ eyre.workspace = true
 serde.workspace = true
 serde-big-array.workspace = true
 serde_with.workspace = true
-rayon.workspace = true
 
 [dev-dependencies]
 openvm-stark-sdk = { workspace = true }