Added ability to proxy Chrome devtools protocol (fixes #607)

docs/devtools.adoc

@@ -0,0 +1,9 @@
+== Accessing Browser Developer Tools
+
+NOTE: Currently only https://chromedevtools.github.io/devtools-protocol/[Chrome Developer Tools] are supported.
+
+Selenoid is proxying Chrome Developer Tools websocket to browser container. Just use the following URL to access this websocket:
+
+```
+ws://selenoid.example.com:4444/devtools/<session-id>
+```

docs/index.adoc

@@ -43,6 +43,7 @@ include::usage-statistics.adoc[leveloffset=+1]
 include::file-upload.adoc[leveloffset=+1]
 include::file-download.adoc[leveloffset=+1]
 include::clipboard.adoc[leveloffset=+1]
+include::devtools.adoc[leveloffset=+1]
 
 include::contributing.adoc[]
 

docs/log-files.adoc

@@ -53,6 +53,10 @@ The following statuses are available:
 | DEFAULT_VERSION | Selenoid is using default browser version 
 | DELETED_LOG_FILE | Log file was deleted by user 
 | DELETED_VIDEO_FILE | Video file was deleted by user 
+| DEVTOOLS_CLIENT_DISCONNECTED | User devtools client disconnected 
+| DEVTOOLS_DISABLED | An attempt to access browser devtools when it is not enabled with capability 
+| DEVTOOLS_ERROR | An error occurred when trying to send devtools traffic
+| DEVTOOLS_SESSION_CLOSED | Sending devtools traffic was stopped 
 | DOWNLOADING_FILE | User requested to download file from browser container 
 | ENVIRONMENT_NOT_AVAILABLE | Browser with desired name and version does not exist
 | FAILED_TO_KILL_VIDEO_CONTAINER | Failed to kill video container after timeout 
@@ -83,9 +87,9 @@ The following statuses are available:
 | UPLOADING_FILE | An issue occurred while uploading file 
 | UPLOADED_FILE | File successfully uploaded
 | VIDEO_ERROR | An error occurred when post-processing recorded video 
+| VNC_CLIENT_DISCONNECTED | User VNC client disconnected
 | VNC_ENABLED | User requested VNC traffic 
 | VNC_ERROR | An error occurred when trying to send VNC traffic 
 | VNC_SESSION_CLOSED | Sending VNC traffic was stopped 
-| VNC_CLIENT_DISCONNECTED | User VNC client disconnected
 | VNC_NOT_ENABLED | User requested VNC traffic but did not specify `enableVNC` capability
 |===

go.mod

@@ -10,9 +10,11 @@ require (
 	github.com/docker/go-units v0.3.2
 	github.com/go-ini/ini v1.38.1 // indirect
 	github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 // indirect
+	github.com/gorilla/websocket v1.4.0
 	github.com/imdario/mergo v0.3.6
 	github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8 // indirect
 	github.com/jtolds/gls v4.2.1+incompatible // indirect
+	github.com/mafredri/cdp v0.21.0
 	github.com/pborman/uuid v0.0.0-20170612153648-e790cca94e6c
 	github.com/pkg/errors v0.8.0
 	github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d // indirect

go.sum

@@ -46,6 +46,8 @@ github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8
 github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
 github.com/gorilla/mux v1.6.2 h1:Pgr17XVTNXAk3q/r4CpKzC5xBM/qW1uVLV+IhRZpIIk=
 github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
+github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q=
+github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28=
 github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8 h1:12VvqtR6Aowv3l/EQUlocDHW2Cp4G9WJVH7uyH8QFJE=
@@ -55,6 +57,8 @@ github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVY
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/konsorten/go-windows-terminal-sequences v0.0.0-20180402223658-b729f2633dfe h1:CHRGQ8V7OlCYtwaKPJi3iA7J+YdNKdo8j7nG5IgDhjs=
 github.com/konsorten/go-windows-terminal-sequences v0.0.0-20180402223658-b729f2633dfe/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/mafredri/cdp v0.21.0 h1:HHvwNtvQr+6Y91bqnDoO7q9j/KrYudoDvEmvsA34a9M=
+github.com/mafredri/cdp v0.21.0/go.mod h1:hgdiA0yp1uqhSaDOHJWPgXpMbh+LAfUdD9vbN2AM8gE=
 github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ=
 github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
 github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI=

main.go

@@ -284,11 +284,18 @@ func onSIGHUP(fn func()) {
 	}()
 }
 
-func mux() http.Handler {
+var seleniumPaths = struct {
+	CreateSession, ProxySession string
+}{
+	CreateSession: "/session",
+	ProxySession:  "/session/",
+}
+
+func selenium() http.Handler {
 	mux := http.NewServeMux()
-	mux.HandleFunc("/session", queue.Try(queue.Check(queue.Protect(post(create)))))
-	mux.HandleFunc("/session/", proxy)
-	mux.HandleFunc("/status", status)
+	mux.HandleFunc(seleniumPaths.CreateSession, queue.Try(queue.Check(queue.Protect(post(create)))))
+	mux.HandleFunc(seleniumPaths.ProxySession, proxy)
+	mux.HandleFunc(paths.Status, status)
 	return mux
 }
 
@@ -314,10 +321,10 @@ func ping(w http.ResponseWriter, _ *http.Request) {
 
 func video(w http.ResponseWriter, r *http.Request) {
 	if r.Method == http.MethodDelete {
-		deleteFileIfExists(w, r, videoOutputDir, videoPath, "DELETED_VIDEO_FILE")
+		deleteFileIfExists(w, r, videoOutputDir, paths.Video, "DELETED_VIDEO_FILE")
 		return
 	}
-	fileServer := http.StripPrefix(videoPath, http.FileServer(http.Dir(videoOutputDir)))
+	fileServer := http.StripPrefix(paths.Video, http.FileServer(http.Dir(videoOutputDir)))
 	fileServer.ServeHTTP(w, r)
 }
 
@@ -337,36 +344,47 @@ func deleteFileIfExists(w http.ResponseWriter, r *http.Request, dir string, pref
 	log.Printf("[%d] [%s] [%s]", serial(), status, fileName)
 }
 
-const (
-	videoPath = "/video/"
-	logsPath  = "/logs/"
-	errorPath = "/error"
-)
+var paths = struct {
+	Video, VNC, Logs, Devtools, Download, Clipboard, File, Ping, Status, Error, WdHub string
+}{
+	Video:     "/video/",
+	VNC:       "/vnc/",
+	Logs:      "/logs/",
+	Devtools:  "/devtools/",
+	Download:  "/download/",
+	Clipboard: "/clipboard/",
+	Status:    "/status",
+	File:      "/file",
+	Ping:      "/ping",
+	Error:     "/error",
+	WdHub:     "/wd/hub",
+}
 
 func handler() http.Handler {
 	root := http.NewServeMux()
-	root.HandleFunc("/wd/hub/", func(w http.ResponseWriter, r *http.Request) {
+	root.HandleFunc(paths.WdHub+"/", func(w http.ResponseWriter, r *http.Request) {
 		w.Header().Add("Content-Type", "application/json")
 		r.URL.Scheme = "http"
 		r.URL.Host = (&request{r}).localaddr()
-		r.URL.Path = strings.TrimPrefix(r.URL.Path, "/wd/hub")
-		mux().ServeHTTP(w, r)
+		r.URL.Path = strings.TrimPrefix(r.URL.Path, paths.WdHub)
+		selenium().ServeHTTP(w, r)
 	})
-	root.HandleFunc(errorPath, func(w http.ResponseWriter, r *http.Request) {
+	root.HandleFunc(paths.Error, func(w http.ResponseWriter, r *http.Request) {
 		util.JsonError(w, "Session timed out or not found", http.StatusNotFound)
 	})
-	root.HandleFunc("/status", func(w http.ResponseWriter, r *http.Request) {
+	root.HandleFunc(paths.Status, func(w http.ResponseWriter, r *http.Request) {
 		w.Header().Add("Content-Type", "application/json")
 		json.NewEncoder(w).Encode(conf.State(sessions, limit, queue.Queued(), queue.Pending()))
 	})
-	root.HandleFunc("/ping", ping)
-	root.Handle("/vnc/", websocket.Handler(vnc))
-	root.HandleFunc(logsPath, logs)
-	root.HandleFunc(videoPath, video)
-	root.HandleFunc("/download/", reverseProxy(func(sess *session.Session) string { return sess.HostPort.Fileserver }, "DOWNLOADING_FILE"))
-	root.HandleFunc("/clipboard/", reverseProxy(func(sess *session.Session) string { return sess.HostPort.Clipboard }, "CLIPBOARD"))
+	root.HandleFunc(paths.Ping, ping)
+	root.Handle(paths.VNC, websocket.Handler(vnc))
+	root.Handle(paths.Devtools, websocket.Server{Handler: devtools})
+	root.HandleFunc(paths.Logs, logs)
+	root.HandleFunc(paths.Video, video)
+	root.HandleFunc(paths.Download, reverseProxy(func(sess *session.Session) string { return sess.HostPort.Fileserver }, "DOWNLOADING_FILE"))
+	root.HandleFunc(paths.Clipboard, reverseProxy(func(sess *session.Session) string { return sess.HostPort.Clipboard }, "CLIPBOARD"))
 	if enableFileUpload {
-		root.HandleFunc("/file", fileUpload)
+		root.HandleFunc(paths.File, fileUpload)
 	}
 	return root
 }

selenoid.go

@@ -454,7 +454,7 @@ func proxy(w http.ResponseWriter, r *http.Request) {
 				r.URL.Host, r.URL.Path = sess.URL.Host, path.Clean(sess.URL.Path+r.URL.Path)
 				return
 			}
-			r.URL.Path = errorPath
+			r.URL.Path = paths.Error
 		},
 		ErrorHandler: defaultErrorHandler(requestId),
 	}).ServeHTTP(w, r)
@@ -580,13 +580,13 @@ func vnc(wsconn *websocket.Conn) {
 }
 
 func logs(w http.ResponseWriter, r *http.Request) {
-	fileNameOrSessionID := strings.TrimPrefix(r.URL.Path, logsPath)
+	fileNameOrSessionID := strings.TrimPrefix(r.URL.Path, paths.Logs)
 	if logOutputDir != "" && (fileNameOrSessionID == "" || strings.HasSuffix(fileNameOrSessionID, logFileExtension)) {
 		if r.Method == http.MethodDelete {
-			deleteFileIfExists(w, r, logOutputDir, logsPath, "DELETED_LOG_FILE")
+			deleteFileIfExists(w, r, logOutputDir, paths.Logs, "DELETED_LOG_FILE")
 			return
 		}
-		fileServer := http.StripPrefix(logsPath, http.FileServer(http.Dir(logOutputDir)))
+		fileServer := http.StripPrefix(paths.Logs, http.FileServer(http.Dir(logOutputDir)))
 		fileServer.ServeHTTP(w, r)
 		return
 	}
@@ -630,6 +630,33 @@ func status(w http.ResponseWriter, _ *http.Request) {
 		})
 }
 
+func devtools(wsconn *websocket.Conn) {
+	sid, _ := splitRequestPath(wsconn.Request().URL.Path)
+	sess, ok := sessions.Get(sid)
+	requestId := serial()
+	if ok {
+		origin := "http://localhost/"
+		u := fmt.Sprintf("ws://%s/", sess.HostPort.Devtools)
+		conn, err := websocket.Dial(u, "", origin)
+		if err != nil {
+			log.Printf("[%d] [DEVTOOLS_ERROR] [%v]", requestId, err)
+			return
+		}
+		log.Printf("[%d] [DEVTOOLS] [%s]", requestId, sid)
+		defer conn.Close()
+		wsconn.PayloadType = websocket.BinaryFrame
+		go func() {
+			io.Copy(wsconn, conn)
+			wsconn.Close()
+			log.Printf("[%d] [DEVTOOLS_SESSION_CLOSED] [%s]", requestId, sid)
+		}()
+		io.Copy(conn, wsconn)
+		log.Printf("[%d] [DEVTOOLS_CLIENT_DISCONNECTED] [%s]", requestId, sid)
+	} else {
+		log.Printf("[%d] [SESSION_NOT_FOUND] [%s]", requestId, sid)
+	}
+}
+
 func onTimeout(t time.Duration, f func()) chan struct{} {
 	cancel := make(chan struct{})
 	go func(cancel chan struct{}) {

selenoid_test.go

@@ -4,6 +4,8 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"github.com/mafredri/cdp"
+	"github.com/mafredri/cdp/rpcc"
 	"io/ioutil"
 	"log"
 	"net/http"
@@ -748,6 +750,32 @@ func TestClipboardMissingSession(t *testing.T) {
 	AssertThat(t, rsp, Code{http.StatusNotFound})
 }
 
+func TestDevtools(t *testing.T) {
+	manager = &HTTPTest{Handler: Selenium()}
+
+	resp, err := http.Post(With(srv.URL).Path("/wd/hub/session"), "", bytes.NewReader([]byte("{}")))
+	AssertThat(t, err, Is{nil})
+
+	var sess map[string]string
+	AssertThat(t, resp, AllOf{Code{http.StatusOK}, IsJson{&sess}})
+
+	u := fmt.Sprintf("ws://%s/devtools/%s", srv.Listener.Addr().String(), sess["sessionId"])
+
+	ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond)
+	defer cancel()
+
+	conn, err := rpcc.DialContext(ctx, u)
+	AssertThat(t, err, Is{nil})
+	defer conn.Close()
+
+	c := cdp.NewClient(conn)
+	err = c.Page.Enable(ctx)
+	AssertThat(t, err, Is{nil})
+
+	sessions.Remove(sess["sessionId"])
+	queue.Release()
+}
+
 func TestParseGgrHost(t *testing.T) {
 	h := parseGgrHost("some-host.example.com:4444")
 	AssertThat(t, h.Name, EqualTo{"some-host.example.com"})

service/docker.go

@@ -27,6 +27,7 @@ const (
 	sysAdmin               = "SYS_ADMIN"
 	overrideVideoOutputDir = "OVERRIDE_VIDEO_OUTPUT_DIR"
 	vncPort                = "5900"
+	devtoolsPort           = "7070"
 	fileserverPort         = "8080"
 	clipboardPort          = "9090"
 )
@@ -44,6 +45,7 @@ type portConfig struct {
 	SeleniumPort   nat.Port
 	FileserverPort nat.Port
 	ClipboardPort  nat.Port
+	DevtoolsPort   nat.Port
 	VNCPort        nat.Port
 	PortBindings   nat.PortMap
 	ExposedPorts   map[nat.Port]struct{}
@@ -59,6 +61,7 @@ func (d *Docker) StartWithCancel() (*StartedService, error) {
 	fileserver := portConfig.FileserverPort
 	clipboard := portConfig.ClipboardPort
 	vnc := portConfig.VNCPort
+	devtools := portConfig.DevtoolsPort
 	requestId := d.RequestId
 	image := d.Service.Image
 	ctx := context.Background()
@@ -140,6 +143,7 @@ func (d *Docker) StartWithCancel() (*StartedService, error) {
 	pc := map[string]nat.Port{
 		servicePort:    selenium,
 		vncPort:        vnc,
+		devtoolsPort:   devtools,
 		fileserverPort: fileserver,
 		clipboardPort:  clipboard,
 	}
@@ -226,11 +230,18 @@ func getPortConfig(service *config.Browser, caps session.Caps, env Environment)
 		}
 		exposedPorts[vnc] = struct{}{}
 	}
+	devtools, err := nat.NewPort("tcp", devtoolsPort)
+	if err != nil {
+		return nil, fmt.Errorf("new devtools port: %v", err)
+	}
+	exposedPorts[devtools] = struct{}{}
+
 	portBindings := nat.PortMap{}
 	if env.IP != "" || !env.InDocker {
 		portBindings[selenium] = []nat.PortBinding{{HostIP: "0.0.0.0"}}
 		portBindings[fileserver] = []nat.PortBinding{{HostIP: "0.0.0.0"}}
 		portBindings[clipboard] = []nat.PortBinding{{HostIP: "0.0.0.0"}}
+		portBindings[devtools] = []nat.PortBinding{{HostIP: "0.0.0.0"}}
 		if caps.VNC {
 			portBindings[vnc] = []nat.PortBinding{{HostIP: "0.0.0.0"}}
 		}
@@ -240,6 +251,7 @@ func getPortConfig(service *config.Browser, caps session.Caps, env Environment)
 		FileserverPort: fileserver,
 		ClipboardPort:  clipboard,
 		VNCPort:        vnc,
+		DevtoolsPort:   devtools,
 		PortBindings:   portBindings,
 		ExposedPorts:   exposedPorts}, nil
 }
@@ -359,6 +371,7 @@ func getHostPort(env Environment, servicePort string, caps session.Caps, stat ty
 		Selenium:   fn(servicePort, pc[servicePort]),
 		Fileserver: fn(fileserverPort, pc[fileserverPort]),
 		Clipboard:  fn(clipboardPort, pc[clipboardPort]),
+		Devtools:   fn(devtoolsPort, pc[devtoolsPort]),
 	}
 
 	if caps.VNC {

service_test.go

@@ -108,6 +108,12 @@ func testMux() http.Handler {
 						"HostPort": "%s"
 						}
 					],
+					"7070/tcp": [
+						{
+						"HostIp": "0.0.0.0",
+						"HostPort": "%s"
+						}
+					],
 					"8080/tcp": [
 						{
 						"HostIp": "0.0.0.0",
@@ -153,7 +159,7 @@ func testMux() http.Handler {
 			  "State": {},
 			  "Mounts": []
 			}
-			`, p, p, p, p, p)
+			`, p, p, p, p, p, p)
 			w.Write([]byte(output))
 		},
 	))

session/session.go

@@ -78,6 +78,7 @@ type HostPort struct {
 	Fileserver string
 	Clipboard  string
 	VNC        string
+	Devtools   string
 }
 
 // Map - session uuid to sessions mapping