Merge pull request #630 from tlsnotary/notary-server-sgx

feat: intel sgx attestation

crates/notary/server/Cargo.toml

@@ -3,6 +3,17 @@ name = "notary-server"
 version = "0.1.0-alpha.8-pre"
 edition = "2021"
 
+[features]
+tee_quote = [
+	"dep:mc-sgx-dcap-types", 
+	"dep:hex", 
+	"dep:rand_chacha", 
+	"dep:once_cell",
+	"dep:simple_asn1",
+        "dep:pem",
+	"dep:lazy_static",
+]
+
 [dependencies]
 tlsn-core = { workspace = true }
 tlsn-common = { workspace = true }
@@ -46,6 +57,15 @@ uuid = { workspace = true, features = ["v4", "fast-rng"] }
 ws_stream_tungstenite = { workspace = true, features = ["tokio_io"] }
 zeroize = { workspace = true }
 
+mc-sgx-dcap-types = { version = "0.11.0", optional = true }
+hex = { workspace = true, optional = true }
+rand_chacha = { workspace = true, optional = true }
+once_cell = { workspace = true, optional =true }
+simple_asn1 = {version = "0.6.2", optional = true }
+pem = { version =  "1.1.0", optional = true }
+lazy_static = { version = "1.4", optional = true }
+
 [build-dependencies]
 git2 = "0.19.0"
 chrono.workspace = true
+

crates/notary/server/src/domain.rs

@@ -1,7 +1,8 @@
 pub mod auth;
 pub mod cli;
 pub mod notary;
-
+#[cfg(feature = "tee_quote")]
+use crate::tee::Quote;
 use serde::{Deserialize, Serialize};
 
 /// Response object of the /info API
@@ -14,4 +15,7 @@ pub struct InfoResponse {
     pub public_key: String,
     /// Current git commit hash of notary-server
     pub git_commit_hash: String,
+    /// Hardware attestation
+    #[cfg(feature = "tee_quote")]
+    pub quote: Quote,
 }

crates/notary/server/src/lib.rs

@@ -7,6 +7,8 @@ mod server_tracing;
 mod service;
 mod settings;
 mod signing;
+#[cfg(feature = "tee_quote")]
+mod tee;
 mod util;
 
 pub use config::{

crates/notary/server/src/server.rs

@@ -46,11 +46,13 @@ use crate::{
     util::parse_csv_file,
 };
 
+#[cfg(feature = "tee_quote")]
+use crate::tee::{generate_ephemeral_keypair, quote};
+
 /// Start a TCP server (with or without TLS) to accept notarization request for
 /// both TCP and WebSocket clients
 #[tracing::instrument(skip(config))]
 pub async fn run_server(config: &NotaryServerProperties) -> Result<(), NotaryServerError> {
-    // Load the private key for notarized transcript signing
     let attestation_key = load_attestation_key(&config.notary_key).await?;
     let crypto_provider = build_crypto_provider(attestation_key);
 
@@ -139,6 +141,8 @@ pub async fn run_server(config: &NotaryServerProperties) -> Result<(), NotarySer
                         version,
                         public_key,
                         git_commit_hash,
+                        #[cfg(feature = "tee_quote")]
+                        quote: quote().await,
                     }),
                 )
                     .into_response()
@@ -229,6 +233,9 @@ fn build_crypto_provider(attestation_key: AttestationKey) -> CryptoProvider {
 
 /// Load notary signing key for attestations from static file
 async fn load_attestation_key(config: &NotarySigningKeyProperties) -> Result<AttestationKey> {
+    #[cfg(feature = "tee_quote")]
+    generate_ephemeral_keypair(&config.private_key_pem_path, &config.public_key_pem_path);
+
     debug!("Loading notary server's signing key");
 
     let mut file = File::open(&config.private_key_pem_path).await?;

crates/notary/server/src/tee.rs

@@ -0,0 +1,186 @@
+use k256::ecdsa::{SigningKey, VerifyingKey as PublicKey};
+use mc_sgx_dcap_types::{QlError, Quote3};
+use once_cell::sync::OnceCell;
+use pkcs8::{EncodePrivateKey, LineEnding};
+use rand_chacha::{
+    rand_core::{OsRng, SeedableRng},
+    ChaCha20Rng,
+};
+use serde::{Deserialize, Serialize};
+use std::{
+    fs,
+    fs::File,
+    io::{self, Read},
+    path::Path,
+};
+use tracing::{debug, error, instrument};
+
+lazy_static::lazy_static! {
+    static ref SECP256K1_OID: simple_asn1::OID = simple_asn1::oid!(1, 3, 132, 0, 10);
+    static ref ECDSA_OID: simple_asn1::OID = simple_asn1::oid!(1, 2, 840, 10045, 2, 1);
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Quote {
+    raw_quote: Option<String>,
+    mrsigner: Option<String>,
+    mrenclave: Option<String>,
+    error: Option<String>,
+}
+
+impl Default for Quote {
+    fn default() -> Quote {
+        Quote {
+            raw_quote: Some("".to_string()),
+            mrsigner: None,
+            mrenclave: None,
+            error: None,
+        }
+    }
+}
+
+impl std::fmt::Debug for QuoteError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            QuoteError::IoError(err) => write!(f, "IoError: {:?}", err),
+            QuoteError::IntelQuoteLibrary(err) => {
+                write!(f, "IntelQuoteLibrary: {}", err)
+            }
+        }
+    }
+}
+
+impl From<io::Error> for QuoteError {
+    fn from(err: io::Error) -> QuoteError {
+        QuoteError::IoError(err)
+    }
+}
+
+enum QuoteError {
+    IoError(io::Error),
+    IntelQuoteLibrary(QlError),
+}
+
+impl From<QlError> for QuoteError {
+    fn from(src: QlError) -> Self {
+        Self::IntelQuoteLibrary(src)
+    }
+}
+
+static PUBLIC_KEY: OnceCell<PublicKey> = OnceCell::new();
+
+fn pem_der_encode_with_asn1(public_point: &[u8]) -> String {
+    use simple_asn1::*;
+
+    let ecdsa_oid = ASN1Block::ObjectIdentifier(0, ECDSA_OID.clone());
+    let secp256k1_oid = ASN1Block::ObjectIdentifier(0, SECP256K1_OID.clone());
+    let alg_id = ASN1Block::Sequence(0, vec![ecdsa_oid, secp256k1_oid]);
+    let key_bytes = ASN1Block::BitString(0, public_point.len() * 8, public_point.to_vec());
+
+    let blocks = vec![alg_id, key_bytes];
+
+    let der_out = simple_asn1::to_der(&ASN1Block::Sequence(0, blocks))
+        .expect("Failed to encode ECDSA private key as DER");
+
+    pem::encode(&pem::Pem {
+        tag: "PUBLIC KEY".to_string(),
+        contents: der_out,
+    })
+}
+
+#[instrument(level = "debug", skip_all)]
+async fn gramine_quote() -> Result<Quote, QuoteError> {
+    //// Check if the the gramine pseudo-hardware exists
+    if !Path::new("/dev/attestation/quote").exists() {
+        return Ok(Quote::default());
+    }
+
+    // Reading attestation type
+    let mut attestation_file = File::open("/dev/attestation/attestation_type")?;
+    let mut attestation_type = String::new();
+    attestation_file.read_to_string(&mut attestation_type)?;
+    debug!("Detected attestation type: {}", attestation_type);
+
+    // Read `/dev/attestation/my_target_info`
+    let my_target_info = fs::read("/dev/attestation/my_target_info")?;
+
+    // Write to `/dev/attestation/target_info`
+    fs::write("/dev/attestation/target_info", my_target_info)?;
+
+    //// Writing the pubkey to bind the instance to the hw (note: this is not
+    //// mrsigner)
+    fs::write(
+        "/dev/attestation/user_report_data",
+        PUBLIC_KEY
+            .get()
+            .expect("pub_key_get")
+            .to_encoded_point(true)
+            .as_bytes(),
+    )?;
+
+    //// Reading from the gramine quote pseudo-hardware `/dev/attestation/quote`
+    let mut quote_file = File::open("/dev/attestation/quote")?;
+    let mut quote = Vec::new();
+    let _ = quote_file.read_to_end(&mut quote);
+    //// todo: wire up Qlerror and drop .expect()
+    let quote3 = Quote3::try_from(quote.as_ref()).expect("quote3 error");
+    let mrenclave = quote3.app_report_body().mr_enclave().to_string();
+    let mrsigner = quote3.app_report_body().mr_signer().to_string();
+
+    debug!("mrenclave: {}", mrenclave);
+    debug!("mrsigner: {}", mrsigner);
+
+    //// Return the Quote struct with the extracted data
+    Ok(Quote {
+        raw_quote: Some(hex::encode(quote)),
+        mrsigner: Some(mrsigner),
+        mrenclave: Some(mrenclave),
+        error: None,
+    })
+}
+
+pub fn generate_ephemeral_keypair(notary_private: &str, notary_public: &str) {
+    let mut rng = ChaCha20Rng::from_rng(OsRng).expect("os rng err!");
+    let signing_key = SigningKey::random(&mut rng);
+    let pem_string = signing_key
+        .clone()
+        .to_pkcs8_pem(LineEnding::LF)
+        .expect("to pem");
+
+    std::fs::write(notary_private, pem_string).expect("fs::write");
+
+    let der = signing_key
+        .verifying_key()
+        .to_encoded_point(true)
+        .to_bytes();
+    let pem_spki_pub = pem_der_encode_with_asn1(&der);
+    std::fs::write(notary_public, pem_spki_pub).expect("fs::write");
+    let _ = PUBLIC_KEY
+        .set(*signing_key.verifying_key())
+        .map_err(|_| "Public key has already been set");
+}
+
+pub async fn quote() -> Quote {
+    //// tee-detection logic will live here, for now its only gramine-sgx
+    match gramine_quote().await {
+        Ok(quote) => quote,
+        Err(err) => {
+            error!("Failed to retrieve quote: {:?}", err);
+            match err {
+                QuoteError::IoError(_) => Quote {
+                    raw_quote: None,
+                    mrsigner: None,
+                    mrenclave: None,
+                    error: Some("io".to_owned()),
+                },
+                QuoteError::IntelQuoteLibrary(_) => Quote {
+                    raw_quote: None,
+                    mrsigner: None,
+                    mrenclave: None,
+                    error: Some("hw".to_owned()),
+                },
+            }
+        }
+    }
+}

crates/notary/server/tee/Dockerfile

@@ -0,0 +1,25 @@
+#tlsnotary server for testing <> gramine sgx (gramine1.7, g++13, libiomp off :()
+### notaryserverbuilds.azurecr.io/prod/notary-sgx
+
+FROM            notaryserverbuilds.azurecr.io/prod/gramine AS teesdk
+
+ARG             TOOLCHAIN=1.81.0
+ENV             PATH=/root/.cargo/bin:/usr/local/musl/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+
+RUN             set -eux \
+                && curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain=$TOOLCHAIN \
+                && rustup target add \
+                x86_64-unknown-linux-gnu
+
+
+RUN             apt update && apt install -y libssl-dev libclang-dev
+ARG             TLSN_TAG=dev
+ARG             TLSN_FT=tee_quote
+RUN             git clone --depth 1 -b $TLSN_TAG https://github.com/tlsnotary/tlsn /tlsn  && \
+                cargo build --release --bin notary-server --features $TLSN_FT --color always --manifest-path /tlsn/Cargo.toml
+RUN             cd tlsn/crates/notary/server/tee && gramine-sgx-gen-private-key && SGX=1 make
+
+FROM            notaryserverbuilds.azurecr.io/prod/gramine AS teetime
+WORKDIR /tee
+COPY            --from=teesdk tlsn/crates/notary/server/tee .
+ENTRYPOINT      ["gramine-sgx", "notary-server"]

crates/notary/server/tee/Makefile

@@ -0,0 +1,63 @@
+# notary-server testing only
+ROOT_DIR := $(dir $(realpath $(lastword $(MAKEFILE_LIST))))
+ARCH_LIBDIR ?= /lib/$(shell $(CC) -dumpmachine)
+
+SELF_EXE = ./notary-server
+
+.PHONY: all
+all: $(SELF_EXE) notary-server.manifest
+ifeq ($(SGX),1)
+all: notary-server.manifest.sgx notary-server.sig
+endif
+
+ifeq ($(DEBUG),1)
+GRAMINE_LOG_LEVEL = debug
+else
+GRAMINE_LOG_LEVEL = error
+endif
+
+# Note that we're compiling in release mode regardless of the DEBUG setting passed
+# to Make, as compiling in debug mode results in an order of magnitude's difference in
+# performance that makes testing by running a benchmark with ab painful. The primary goal
+# of the DEBUG setting is to control Gramine's loglevel.
+-include $(SELF_EXE).d # See also: .cargo/config.toml
+$(SELF_EXE): $(ROOT_DIR)../Cargo.toml
+	cargo build --bin notary-server  --release --features tee_quote
+
+notary-server.manifest: notary-server.manifest.template
+	cp ../../../../target/release/notary-server . && \
+	gramine-manifest \
+		-Dlog_level=$(GRAMINE_LOG_LEVEL) \
+		-Darch_libdir=$(ARCH_LIBDIR) \
+		-Dself_exe=$(SELF_EXE) \
+		$< $@
+
+# Make on Ubuntu <= 20.04 doesn't support "Rules with Grouped Targets" (`&:`),
+# see the helloworld example for details on this workaround.
+notary-server.manifest.sgx notary-server.sig: sgx_sign
+	@:
+
+.INTERMEDIATE: sgx_sign
+sgx_sign: notary-server.manifest $(SELF_EXE)
+	gramine-sgx-sign \
+		--manifest $< \
+		--output $<.sgx
+
+ifeq ($(SGX),)
+GRAMINE = gramine-direct
+else
+GRAMINE = gramine-sgx
+endif
+
+.PHONY: start-gramine-server
+start-gramine-server: all
+	$(GRAMINE) notary-server
+
+.PHONY: clean
+clean:
+	$(RM) -rf *.token *.sig *.manifest.sgx *.manifest result-* OUTPUT
+
+.PHONY: distclean
+distclean: clean
+	$(RM) -rf $(SELF_EXE) Cargo.lock
+

crates/notary/server/tee/README.md

@@ -0,0 +1,21 @@
+#### gramine with intel SGX
+```bash
+SGX=1 make
+```
+```bash
+SGX=1 make start-gramine-server
+```
+#### gramine emulating SGX
+```
+make
+```
+```
+make start-gramine-server
+```
+#### generate measurement without SGX hardware
+```
+make
+```
+```
+gramine-sgx-sigstruct-view --verbose --output-format=toml notary-server.sig
+```