feat: Interactive verifier (#379) (#401)

* Added necessary state and state transitions

* Move Prover future into its own module

* Put phase-specific prover code into its own modules

* Make `ProverFuture` public again

* Added `Into` from `Closed` to `Verify` for `Prover` state transition

* Added first part of finalize method for `Prover<Verify>`

* Rename `SessionData` to `NotarizedSessionData` and introduce
`SessionData` for interactive verifier flow

* Added first sketches for HttpProver and Prover with Verifier state

* Introduced wrapper `ServerInfo`

`ServerInfo` is generated by `SessionData` and is the non-notarization
version of `SessionProof`

* Crate `ServerInfo` from `SessionData` in Prover<Verify> flow

* Introduced another module for direct substring proofs.

* WIP: Added dirty first version of prover flow...

* Move `RangeCollector` and restore substring module

* Tidy up tlsn-core and finish first version of prover flow for dealing
with a verifier

* Refactored verifier

* Added `Verify` state for `Verifier`

* WIP: Added first draft for verify flow...

* Added more parts of verifier flow

* Adapt tests to new api changes

* Add some logging and improve code here and there

* Added `ProofBuilder` trait and started implementing it for `SubstringsProofBuilder`

* WIP: Tinkering with lifetimes...

* Resolved lifetime issues

* Refactor module `proof` to support another implementor of `SubstringProofBuilder`

* WIP: Adding `LabelProofBuilder`...

* Streamlined api

* Improved decoding flow

* Include lengths in `LabelProof`

* Improved structure of `LabelProof` and finished `verify`

* Added integration test for verify flow

* Add tests for `LabelProof`

* Improve test for `LabelProof::verify`

* Make tlsn compile without `tlsn-formats`

* Restore `tlsn-formats` from `dev` and temporarily remove from workspace

* Add first batch of feedback

* Add further feedback

* Separated decoding from finalization

* Add warning comment to `Verifier::receive`

* Remove unnecessary traits

* Adapt test

* Repair notarize integration test

* Add `decode` call to prover for verify integration test

* Simplified `LabelProof` and renamed to `TranscriptProof`

* Add range check to `reconstruct`

* Roll back changes to `tlsn-prover/src/http`

* Rename `Verify` to `Prove`

* Added more feedback

* Various code improvements

* Remove `SessionData`

* Restore naming of `TlsProof` and `SessionData`

* Improve error handling

* Adapt prove-verify flow to new API

* Finalize VM first

* Fix prover closing connection too early

* Add correct server certificate and assert correct redactions

* Fix imports after rebase

* Fix api test in `tlsn-core`

* Add feedback

* Fix linting in integration test

Co-authored-by: th4s <[email protected]>

notary-server/src/service.rs

@@ -162,10 +162,10 @@ pub async fn notary_service<T: AsyncWrite + AsyncRead + Send + Unpin + 'static>(
 
     let mut config_builder = VerifierConfig::builder();
 
-    config_builder.id(session_id);
+    config_builder = config_builder.id(session_id);
 
     if let Some(max_transcript_size) = max_transcript_size {
-        config_builder.max_transcript_size(max_transcript_size);
+        config_builder = config_builder.max_transcript_size(max_transcript_size);
     }
 
     let config = config_builder.build()?;

tlsn/Cargo.toml

@@ -3,7 +3,7 @@ members = [
     "tlsn-core",
     "tlsn-verifier",
     "tlsn-prover",
-    "tlsn-formats",
+#    "tlsn-formats",
     "tlsn-server-fixture",
     "tests-integration",
     "examples",
@@ -16,7 +16,7 @@ tlsn-prover = { path = "tlsn-prover" }
 tlsn-verifier = { path = "tlsn-verifier" }
 tlsn-server-fixture = { path = "tlsn-server-fixture" }
 
-tlsn-formats = { path = "tlsn-formats" }
+#tlsn-formats = { path = "tlsn-formats" }
 
 tlsn-tls-core = { path = "../components/tls/tls-core" }
 tlsn-tls-mpc = { path = "../components/tls/tls-mpc" }

tlsn/examples/discord/discord_dm_verifier.rs

@@ -31,8 +31,9 @@ fn main() {
     let SessionProof {
         // The session header that was signed by the Notary is a succinct commitment to the TLS transcript.
         header,
-        // This is the server name, checked against the certificate chain shared in the TLS handshake.
-        server_name,
+        // This is the session_info, which contains the server_name, that is checked against the
+        // certificate chain shared in the TLS handshake.
+        session_info,
         ..
     } = session;
 
@@ -51,7 +52,7 @@ fn main() {
     println!("-------------------------------------------------------------------");
     println!(
         "Successfully verified that the bytes below came from a session with {:?} at {}.",
-        server_name, time
+        session_info.server_name, time
     );
     println!("Note that the bytes which the Prover chose not to disclose are shown as X.");
     println!();

tlsn/examples/simple/simple_verifier.rs

@@ -31,8 +31,9 @@ fn main() {
     let SessionProof {
         // The session header that was signed by the Notary is a succinct commitment to the TLS transcript.
         header,
-        // This is the server name, checked against the certificate chain shared in the TLS handshake.
-        server_name,
+        // This is the session_info, which contains the server_name, that is checked against the
+        // certificate chain shared in the TLS handshake.
+        session_info,
         ..
     } = session;
 
@@ -51,7 +52,7 @@ fn main() {
     println!("-------------------------------------------------------------------");
     println!(
         "Successfully verified that the bytes below came from a session with {:?} at {}.",
-        server_name, time
+        session_info.server_name, time
     );
     println!("Note that the bytes which the Prover chose not to disclose are shown as X.");
     println!();

tlsn/tests-integration/Cargo.toml

@@ -7,9 +7,10 @@ publish = false
 [dev-dependencies]
 tlsn-core.workspace = true
 tlsn-tls-core.workspace = true
-tlsn-prover = { workspace = true, features = ["tracing", "formats"] }
+tlsn-prover = { workspace = true, features = ["tracing"] }
 tlsn-verifier = { workspace = true, features = ["tracing"] }
 tlsn-server-fixture.workspace = true
+tlsn-utils.workspace = true
 
 p256 = { workspace = true, features = ["ecdsa"] }
 hyper = { workspace = true, features = ["client", "http1"] }

tlsn/tests-integration/tests/test.rs → tlsn/tests-integration/tests/notarize.rs

@@ -9,7 +9,7 @@ use tracing::instrument;
 
 #[tokio::test]
 #[ignore]
-async fn test() {
+async fn notarize() {
     tracing_subscriber::fmt::init();
 
     let (socket_0, socket_1) = tokio::io::duplex(2 << 23);
@@ -73,19 +73,17 @@ async fn prover<T: AsyncWrite + AsyncRead + Send + Unpin + 'static>(notary_socke
 
     client_socket.close().await.unwrap();
 
-    let mut prover = prover_task
-        .await
-        .unwrap()
-        .unwrap()
-        .to_http()
-        .unwrap()
-        .start_notarize();
+    let mut prover = prover_task.await.unwrap().unwrap().start_notarize();
+    let sent_tx_len = prover.sent_transcript().data().len();
+    let recv_tx_len = prover.recv_transcript().data().len();
 
-    prover.commit().unwrap();
+    let builder = prover.commitment_builder();
 
-    let notarized_session = prover.finalize().await.unwrap();
+    // Commit to everything
+    builder.commit_sent(0..sent_tx_len).unwrap();
+    builder.commit_recv(0..recv_tx_len).unwrap();
 
-    _ = notarized_session.proof_builder().build().unwrap();
+    let _notarized_session = prover.finalize().await.unwrap();
 }
 
 #[instrument(skip(socket))]

tlsn/tests-integration/tests/verify.rs

@@ -0,0 +1,119 @@
+use futures::AsyncWriteExt;
+use hyper::{body::to_bytes, Body, Request, StatusCode};
+use tls_core::{anchors::RootCertStore, verify::WebPkiVerifier};
+use tlsn_core::{proof::SessionInfo, Direction, RedactedTranscript};
+use tlsn_prover::tls::{Prover, ProverConfig};
+use tlsn_server_fixture::{CA_CERT_DER, SERVER_DOMAIN};
+use tlsn_verifier::tls::{Verifier, VerifierConfig};
+use tokio::io::{AsyncRead, AsyncWrite};
+use tokio_util::compat::{FuturesAsyncReadCompatExt, TokioAsyncReadCompatExt};
+use tracing::instrument;
+use utils::range::RangeSet;
+
+#[tokio::test]
+#[ignore]
+async fn verify() {
+    tracing_subscriber::fmt::init();
+
+    let (socket_0, socket_1) = tokio::io::duplex(2 << 23);
+
+    let (_, (sent, received, _session_info)) = tokio::join!(prover(socket_0), verifier(socket_1));
+
+    assert_eq!(sent.authed(), &RangeSet::from(0..sent.data().len() - 1));
+    assert_eq!(
+        sent.redacted(),
+        &RangeSet::from(sent.data().len() - 1..sent.data().len())
+    );
+
+    assert_eq!(received.authed(), &RangeSet::from(2..received.data().len()));
+    assert_eq!(received.redacted(), &RangeSet::from(0..2));
+}
+
+#[instrument(skip(notary_socket))]
+async fn prover<T: AsyncWrite + AsyncRead + Send + Unpin + 'static>(notary_socket: T) {
+    let (client_socket, server_socket) = tokio::io::duplex(2 << 16);
+
+    let server_task = tokio::spawn(tlsn_server_fixture::bind(server_socket.compat()));
+
+    let mut root_store = RootCertStore::empty();
+    root_store
+        .add(&tls_core::key::Certificate(CA_CERT_DER.to_vec()))
+        .unwrap();
+
+    let prover = Prover::new(
+        ProverConfig::builder()
+            .id("test")
+            .server_dns(SERVER_DOMAIN)
+            .root_cert_store(root_store)
+            .build()
+            .unwrap(),
+    )
+    .setup(notary_socket.compat())
+    .await
+    .unwrap();
+
+    let (tls_connection, prover_fut) = prover.connect(client_socket.compat()).await.unwrap();
+
+    let prover_task = tokio::spawn(prover_fut);
+
+    let (mut request_sender, connection) = hyper::client::conn::handshake(tls_connection.compat())
+        .await
+        .unwrap();
+
+    let connection_task = tokio::spawn(connection.without_shutdown());
+
+    let request = Request::builder()
+        .uri(format!("https://{}", SERVER_DOMAIN))
+        .header("Host", SERVER_DOMAIN)
+        .header("Connection", "close")
+        .method("GET")
+        .body(Body::empty())
+        .unwrap();
+
+    let response = request_sender.send_request(request).await.unwrap();
+
+    assert!(response.status() == StatusCode::OK);
+
+    println!(
+        "{:?}",
+        String::from_utf8_lossy(&to_bytes(response.into_body()).await.unwrap())
+    );
+
+    server_task.await.unwrap();
+
+    let mut client_socket = connection_task.await.unwrap().unwrap().io.into_inner();
+
+    client_socket.close().await.unwrap();
+
+    let mut prover = prover_task.await.unwrap().unwrap().start_prove();
+
+    let sent_transcript_len = prover.sent_transcript().data().len();
+    let recv_transcript_len = prover.recv_transcript().data().len();
+
+    // Reveal parts of the transcript
+    _ = prover.reveal(0..sent_transcript_len - 1, Direction::Sent);
+    _ = prover.reveal(2..recv_transcript_len, Direction::Received);
+    prover.prove().await.unwrap();
+
+    prover.finalize().await.unwrap()
+}
+
+#[instrument(skip(socket))]
+async fn verifier<T: AsyncWrite + AsyncRead + Send + Sync + Unpin + 'static>(
+    socket: T,
+) -> (RedactedTranscript, RedactedTranscript, SessionInfo) {
+    let mut root_store = RootCertStore::empty();
+    root_store
+        .add(&tls_core::key::Certificate(CA_CERT_DER.to_vec()))
+        .unwrap();
+
+    let verifier_config = VerifierConfig::builder()
+        .id("test")
+        .cert_verifier(WebPkiVerifier::new(root_store, None))
+        .build()
+        .unwrap();
+    let verifier = Verifier::new(verifier_config);
+
+    let (sent, received, session_info) = verifier.verify(socket.compat()).await.unwrap();
+    (sent, received, session_info)
+}

tlsn/tlsn-core/src/msg.rs

@@ -1,8 +1,9 @@
 //! Protocol message types.
 
 use serde::{Deserialize, Serialize};
+use utils::range::RangeSet;
 
-use crate::{merkle::MerkleRoot, signature::Signature, SessionHeader};
+use crate::{merkle::MerkleRoot, proof::SessionInfo, signature::Signature, SessionHeader};
 
 /// Top-level enum for all messages
 #[derive(Debug, Serialize, Deserialize)]
@@ -13,6 +14,10 @@ pub enum TlsnMessage {
     SignedSessionHeader(SignedSessionHeader),
     /// A session header.
     SessionHeader(SessionHeader),
+    /// Information about the TLS session
+    SessionInfo(SessionInfo),
+    /// Information about the values the prover wants to prove
+    ProvingInfo(ProvingInfo),
 }
 
 /// A signed session header.
@@ -23,3 +28,14 @@ pub struct SignedSessionHeader {
     /// The notary's signature
     pub signature: Signature,
 }
+
+/// Information about the values the prover wants to prove
+#[derive(Debug, Serialize, Deserialize, Default)]
+pub struct ProvingInfo {
+    /// The ids for the sent transcript
+    pub sent_ids: RangeSet<usize>,
+    /// The ids for the received transcript
+    pub recv_ids: RangeSet<usize>,
+    /// Purported cleartext values
+    pub cleartext: Vec<u8>,
+}

tlsn/tlsn-core/src/proof/mod.rs

@@ -1,9 +1,21 @@
 //! Different types of proofs used in the TLSNotary protocol.
 
+mod session;
 mod substrings;
-mod tls;
 
+pub use session::{default_cert_verifier, SessionInfo, SessionProof, SessionProofError};
 pub use substrings::{
     SubstringsProof, SubstringsProofBuilder, SubstringsProofBuilderError, SubstringsProofError,
 };
-pub use tls::{SessionProof, TlsProof};
+
+use serde::{Deserialize, Serialize};
+use std::fmt::Debug;
+
+/// Proof that a transcript of communications took place between a Prover and Server.
+#[derive(Debug, Serialize, Deserialize)]
+pub struct TlsProof {
+    /// Proof of the TLS handshake, server identity, and commitments to the transcript.
+    pub session: SessionProof,
+    /// Proof regarding the contents of the transcript.
+    pub substrings: SubstringsProof,
+}