Add client-side HTTP/3 support.

This also adds TLS key logging.

README.md

@@ -35,6 +35,7 @@ Table of Contents
                * [Building on CentOS 8](#building-on-centos-8)
                * [Building on CentOS 7](#building-on-centos-7)
             * [ASan Instrumentation](#asan-instrumentation)
+            * [QUIC/HTTP3 Support](#quichttp3-support)
          * [Running the Tests](#running-the-tests)
             * [Unit Tests](#unit-tests)
             * [Gold Tests](#gold-tests)
@@ -50,6 +51,8 @@ Table of Contents
             * [--rate <requests/second>](#--rate-requestssecond)
             * [--repeat <number>](#--repeat-number)
             * [--thread-limit <number>](#--thread-limit-number)
+            * [--qlog-dir <directory>](#--qlog-dir-directory)
+            * [--tls-secrets-log-file <secrets_log_file_name>](#--tls-secrets-log-file-secrets_log_file_name)
       * [Contribute](#contribute)
       * [License](#license)
 
@@ -1105,6 +1108,39 @@ pipenv run scons \
     proxy-verifier
 ```
 
+#### QUIC/HTTP3 Support
+
+Proxy Verifier supports HTTP/3. The implemenation of this relies upon the
+following libraries:
+
+* A version of OpenSSL that supports QUIC.
+* ngtcp2 for its QUIC support
+* nghttp3 for its HTTP/3 support.
+
+A tool is provided to build these libraries:
+[build_http3_dependencies.sh](https://github.com/yahoo/proxy-verifier/blob/master/tools/build_http3_dependencies.sh)
+
+Here is an example session, tested on MacOS BugSur and CentOS 7, that builds
+Proxy Verifier with QUIC/HTTP3 support:
+
+```
+# Alter this for your desired library location.
+http3_libs_dir=${HOME}/src/http3_libs
+
+bash ./tools/build_http3_dependencies.sh ${http3_libs_dir}
+
+# Replace '/path/to/nghttp2' to your location of the installed nghttp2
+# location.
+pipenv run scons \
+    -j8 \
+    --cfg=release \
+    --with-ssl=${http3_libs_dir}/openssl_build/ \
+    --with-ngtcp2=${http3_libs_dir}/ngtcp2_build/ \
+    --with-nghttp3=${http3_libs_dir}/nghttp3_build/ \
+    --with-nghttp2=/path/to/nghttp2 \
+    proxy-verifier
+```
+
 ### Running the Tests
 
 #### Unit Tests
@@ -1402,6 +1438,21 @@ these connections to 2,000. This limit can be changed via the `--thread-limit`
 option. Setting a value of 1 on the client will effectively cause sessions
 to be replayed in serial.
 
+#### --qlog-dir \<directory\>
+
+Proxy Verifier supports logging of replayed QUIC traffic information conformant
+to the qlog format. If the `--qlog-dir` option is provided, then qlog files for all
+replayed QUIC traffic will be written into the specified directory.  qlog
+diagnostic logging is disabled by default.
+
+#### --tls-secrets-log-file \<secrets_log_file_name\>
+
+To facilitate debugging, Proxy Verifier supports logging TLS keys for encrypted
+replayed traffic. If this option is used, TLS key logging will be appended to
+the specified filename. This file can then be provided to protocol analyzers
+such as Wireshark to decrypt the traffic. TLS key logging is disabled by
+default.
+
 ## Contribute
 
 Please refer to [CONTRIBUTING](CONTRIBUTING.md) for information about how to get involved. We welcome issues, questions, and pull requests.

Sconstruct

@@ -24,6 +24,26 @@ AddOption("--with-nghttp2",
           help='Optional path to custom build of nghttp2'
          )
 
+AddOption("--with-ngtcp2",
+          dest='with_ngtcp2',
+          nargs=1,
+          type='string',
+          action='store',
+          metavar='DIR',
+          default=None,
+          help='Optional path to custom build of ngtcp2'
+         )
+
+AddOption("--with-nghttp3",
+          dest='with_nghttp3',
+          nargs=1,
+          type='string',
+          action='store',
+          metavar='DIR',
+          default=None,
+          help='Optional path to custom build of nghttp3'
+         )
+
 
 AddOption('--enable-asan',
           dest='enable_asan',
@@ -32,15 +52,27 @@ AddOption('--enable-asan',
 
 path_ssl = GetOption("with_ssl")
 Part("#lib/openssl.part", CUSTOM_PATH=path_ssl)
+
 path_nghttp2 = GetOption("with_nghttp2")
 Part("#lib/nghttp2.part", CUSTOM_PATH=path_nghttp2)
+
+path_ngtcp2 = GetOption("with_ngtcp2")
+Part("#lib/ngtcp2.part", CUSTOM_PATH=path_ngtcp2)
+
+path_nghttp3 = GetOption("with_nghttp3")
+Part("#lib/nghttp3.part", CUSTOM_PATH=path_nghttp3)
+
 Part("#lib/libyaml-cpp.part",vcs_type=VcsGit(server="github.com", repository="jbeder/yaml-cpp.git", tag="yaml-cpp-0.6.3"), package_group="proxy-verifier")
 
 custom_rpath=[]
 if path_ssl is not None:
   custom_rpath.append(path_ssl + "/lib")
 if path_nghttp2 is not None:
   custom_rpath.append(path_nghttp2 + "/lib")
+if path_ngtcp2 is not None:
+  custom_rpath.append(path_ngtcp2 + "/lib")
+if path_nghttp3 is not None:
+  custom_rpath.append(path_nghttp3 + "/lib")
 SetOptionDefault("RPATH",  custom_rpath)
 
 Part("code/libswoc.part",vcs_type=VcsGit(server="github.com", repository="SolidWallOfCode/libswoc", tag="1.2.19"), package_group="proxy-verifier")

lib/nghttp3.part

@@ -0,0 +1,19 @@
+Import('*')
+import os
+PartName("nghttp3")
+path=env.get("CUSTOM_PATH") # passed in from top level SConstruct.
+if path:
+    path = env.subst(path);
+    i_path = os.path.join(path,"include")
+    env.Append(CPPPATH=[i_path])
+    env.ExportCPPPATH([i_path])
+    l_path = os.path.join(path,"lib")
+    env.Append(LIBPATH=[l_path])
+    env.ExportLIBPATH(l_path)
+
+cfg = env.Configure()
+if not cfg.CheckCHeader("nghttp3/nghttp3.h"):
+    env.PrintError("nghttp3/nghttp3.h was not found - install nghttp3 or use --with-nghttp3 to specify a directory.",show_stack=False)
+cfg.Finish()
+
+env.ExportLIBS(["libnghttp3"])

lib/ngtcp2.part

@@ -0,0 +1,19 @@
+Import('*')
+import os
+PartName("ngtcp2")
+path=env.get("CUSTOM_PATH") # passed in from top level SConstruct.
+if path:
+    path = env.subst(path);
+    i_path = os.path.join(path,"include")
+    env.Append(CPPPATH=[i_path])
+    env.ExportCPPPATH([i_path])
+    l_path = os.path.join(path,"lib")
+    env.Append(LIBPATH=[l_path])
+    env.ExportLIBPATH(l_path)
+
+cfg = env.Configure()
+if not cfg.CheckCHeader("ngtcp2/ngtcp2.h"):
+    env.PrintError("ngtcp2/ngtcp2.h was not found - install ngtcp2 or use --with-ngtcp2 to specify a directory.",show_stack=False)
+cfg.Finish()
+
+env.ExportLIBS(["libngtcp2", "libngtcp2_crypto_openssl"])

local/include/core/http.h

@@ -12,7 +12,9 @@
 #include <chrono>
 #include <list>
 #include <map>
+#include <unordered_map>
 #include <nghttp2/nghttp2.h>
+#include <nghttp3/nghttp3.h>
 #include <openssl/ssl.h>
 #include <poll.h>
 #include <string>
@@ -78,6 +80,9 @@ BufferWriter &bwformat(BufferWriter &w, bwf::Spec const &spec, HttpHeader const
 } // namespace SWOC_VERSION_NS
 } // namespace swoc
 
+using memoized_ip_endpoints_t =
+    std::unordered_map<std::string_view, std::unordered_map<int, swoc::IPEndpoint>>;
+
 /** Provide the ability to concert an interface name into an IPEndpoint.
  *
  * This provides RAII for the struct ifaddrs allocated via getifaddrs().
@@ -107,6 +112,9 @@ class InterfaceNameToEndpoint
   struct ifaddrs *_ifaddr_list_head = nullptr;
   const std::string _expected_interface;
   const int _expected_family;
+
+  /// Save previously derived IPEndpoints for efficiency.
+  static memoized_ip_endpoints_t memoized_ip_endpoints;
 };
 
 class HttpFields
@@ -170,7 +178,7 @@ class HttpFields
    */
   void merge(self_type const &other);
 
-  /** Convert _fields into nghttp2_nv and add them to the vector provided
+  /** Convert _fields into nghttp2_nv and add them to the provided vector.
    *
    * This assumes that the pseudo header fields are handled separately.  If
    * such fields are in the _fields container they are not added here to the
@@ -180,9 +188,26 @@ class HttpFields
    */
   void add_fields_to_ngnva(nghttp2_nv *l) const;
 
+  /** Convert _fields into nghttp3_nv and add them to the provided vector.
+   *
+   * This assumes that the pseudo header fields are handled separately.  If
+   * such fields are in the _fields container they are not added here to the
+   * nghttp3_nv vector.
+   *
+   * @param[out] l vector of nghttp3_nv structs to populate from _fields.
+   */
+  void add_fields_to_ngnva(nghttp3_nv *l) const;
+
   friend class HttpHeader;
 };
 
+/// An enumeration of the various protocol types.
+enum class HTTP_PROTOCOL_TYPE {
+  HTTP_1,
+  HTTP_2,
+  HTTP_3,
+};
+
 // TODO: rename to HttpMessage?
 class HttpHeader
 {
@@ -272,14 +297,49 @@ class HttpHeader
    */
   void merge(HttpFields const &other);
 
-  /// Whether this is an HTTP/2 message.
-  bool _is_http2 = false;
+  /// Get the HTTP protocol type of this message.
+  HTTP_PROTOCOL_TYPE get_http_protocol() const;
 
-  /// Whether this is an HTTP request
-  bool _is_request = false;
+  /// Set the HTTP protocol type for this message.
+  void set_http_protocol(HTTP_PROTOCOL_TYPE protocol);
+
+  /// Set that this is an HTTP/1.x message.
+  void set_is_http1();
+
+  /// Return whether this is an HTTP/1.x message.
+  bool is_http1() const;
 
-  /// Whether this is an HTTP response
-  bool _is_response = false;
+  /// Set that this is an HTTP/2 message.
+  void set_is_http2();
+
+  /// Return whether this is an HTTP/2 message.
+  bool is_http2() const;
+
+  /// Set that this is an HTTP/3 message.
+  void set_is_http3();
+
+  /// Return whether this is an HTTP/3 message.
+  bool is_http3() const;
+
+  /// Set this to be state for an HTTP request while also specifying the HTTP
+  /// protocol.
+  void set_is_request(HTTP_PROTOCOL_TYPE protocol);
+
+  /// Set this to be state for an HTTP request.
+  void set_is_request();
+
+  /// Return whether this is an HTTP request.
+  bool is_request() const;
+
+  /// Set this to be state for an HTTP response while also specifying the HTTP
+  /// protocol.
+  void set_is_response(HTTP_PROTOCOL_TYPE protocol);
+
+  /// Set this to be state for an HTTP response.
+  void set_is_response();
+
+  /// Return whether this is an HTTP response.
+  bool is_response() const;
 
   /// Whether the _fields array contains pseudo header fields.
   bool _contains_pseudo_headers_in_fields_array = false;
@@ -384,6 +444,12 @@ class HttpHeader
 private:
   /** The key associated with this HTTP transaction. */
   std::string _key;
+
+  /// The HTTP protocol this message represents.
+  HTTP_PROTOCOL_TYPE _http_protocol = HTTP_PROTOCOL_TYPE::HTTP_1;
+
+  /// Whether this is an HTTP request.
+  bool _is_request = false;
 };
 
 struct Txn
@@ -419,6 +485,7 @@ struct Ssn
   int _client_verify_mode = SSL_VERIFY_NONE;
   bool is_tls = false;
   bool is_h2 = false;
+  bool is_h3 = false;
 
   swoc::Errata post_process_transactions();
 };

local/include/core/http2.h

@@ -45,12 +45,6 @@ class H2StreamState
    */
   swoc::TextView register_rcbuf(nghttp2_rcbuf *rcbuf);
 
-  /** Indicate that the stream has closed. */
-  void set_stream_has_closed();
-
-  /** Return whether this stream has closed. */
-  bool get_stream_has_closed() const;
-
   /** Set the stream_id for this and the appropriate members. */
   void set_stream_id(int32_t id);
 
@@ -64,10 +58,9 @@ class H2StreamState
   void store_nv_request_headers_to_free(nghttp2_nv *hdrs);
 
 public:
-  size_t _send_body_offset = 0;
   char const *_body_to_send = nullptr;
   size_t _send_body_length = 0;
-  size_t _received_body_length = 0;
+  size_t _send_body_offset = 0;
   bool _wait_for_continue = false;
   std::string _key;
   /** The composed URL parts from :method, :authority, and :path pseudo headers
@@ -88,7 +81,6 @@ class H2StreamState
 private:
   int32_t _stream_id = -1;
   std::deque<nghttp2_rcbuf *> _rcbufs_to_free;
-  bool _stream_has_closed = false;
   nghttp2_nv *_request_nv_headers = nullptr;
   nghttp2_nv *_response_nv_headers = nullptr;
 };
@@ -119,10 +111,23 @@ class H2Session : public TLSSession
 
   swoc::Errata accept() override;
   swoc::Errata connect() override;
+
+  /** Perform HTTP/2 global initialization.
+   *
+   * @param[in] process_exit_code: The integer to set to non-zero on failure
+   * conditions. This is necessary because many nghttp2 callbacks do
+   * not have direct returns to their callers.
+   */
   static swoc::Errata init(int *process_exit_code);
+
+  /** Delete global instances. */
   static void terminate();
+
+  /** Perform the HTTP/2 (nghttp2) configuration for a client connection. */
   swoc::Errata client_session_init();
+  /** Perform the HTTP/2 (nghttp2) configuration for a server connection. */
   swoc::Errata server_session_init();
+
   swoc::Errata send_connection_settings();
   swoc::Errata run_transactions(
       std::list<Txn> const &txn,