Merge pull request #751 from vania-pooh/master

Better devtools and listing files as JSON

docs/contributing.adoc

@@ -2,7 +2,7 @@
 
 To build Selenoid:
 
-. Install https://golang.org/doc/install[Golang] 1.11.4 and above.
+. Install https://golang.org/doc/install[Golang] 1.12 and above.
 
 . Setup `$GOPATH` https://github.com/golang/go/wiki/GOPATH[properly]
 
@@ -12,8 +12,8 @@ To build Selenoid:
 
 . Go to project directory:
 
-    $ cd selenoid 
-    
+    $ cd selenoid
+
 . Build source:
 
     $ go build

docs/log-files.adoc

@@ -19,61 +19,62 @@ Every line contains:
 
 .Log entry contents
 |===
-| Field | Example | Notes 
- 
-| Time | 2017/11/01 19:12:42 | - 
-| Request counter | [41301] | So far as session ID is unknown when doing attempts this counter is used to find all session attempts for each new session request. 
-| Status | [SESSION_ATTEMPTED] | See table below for complete list of statuses. 
+| Field | Example | Notes
+
+| Time | 2017/11/01 19:12:42 | -
+| Request counter | [41301] | So far as session ID is unknown when doing attempts this counter is used to find all session attempts for each new session request.
+| Status | [SESSION_ATTEMPTED] | See table below for complete list of statuses.
 | Browser | [firefox-45.0] | Name and version. Only present for new session requests.
-| Attempt number | [1] | For SESSION_ATTEMPTED entries means current attempt number. For SESSION_CREATED entries means total number of attempts to create this session. 
-| Session ID | [345bb886-7026-46d7-82d4-4788c0460110] | This value is unique for every browser session 
-| Session start time | [4.15s] | - 
+| Attempt number | [1] | For SESSION_ATTEMPTED entries means current attempt number. For SESSION_CREATED entries means total number of attempts to create this session.
+| Session ID | [345bb886-7026-46d7-82d4-4788c0460110] | This value is unique for every browser session
+| Session start time | [4.15s] | -
 |===
 
 The following statuses are available:
 
 .Log entry statuses
 |===
-| Status | Description 
- 
-| ALLOCATED_PORT | Successfully allocated port for driver process 
-| ALLOCATING_PORT | Trying to allocate random free port for driver process 
-| BAD_JSON_FORMAT | User request does not contain valid Selenium data 
+| Status | Description
+
+| ALLOCATED_PORT | Successfully allocated port for driver process
+| ALLOCATING_PORT | Trying to allocate random free port for driver process
+| BAD_JSON_FORMAT | User request does not contain valid Selenium data
 | BAD_SCREEN_RESOLUTION | User requested to set wrong custom screen resolution
 | BAD_TIMEZONE | User requested to set wrong custom time zone inside container
 | BAD_VIDEO_SCREEN_SIZE | User requested to capture video with wrong screen size
-| CLIENT_DISCONNECTED | User disconnected and session was interrupted 
+| CLIENT_DISCONNECTED | User disconnected and session was interrupted
 | CONTAINER_LOGS | User requested container logs
 | CONTAINER_LOGS_ERROR | User requested container logs
 | CONTAINER_LOGS_DISCONNECTED | User logs client disconnected
-| CONTAINER_REMOVED | Docker container was successfully removed 
-| CONTAINER_STARTED | Docker container has successfully started 
-| FAILED_TO_COPY_LOGS | Failed to copy logs from Docker container  
-| CREATING_CONTAINER | Docker container with browser is creating 
-| DEFAULT_VERSION | Selenoid is using default browser version 
-| DELETED_LOG_FILE | Log file was deleted by user 
-| DELETED_VIDEO_FILE | Video file was deleted by user 
-| DEVTOOLS_CLIENT_DISCONNECTED | User devtools client disconnected 
-| DEVTOOLS_DISABLED | An attempt to access browser devtools when it is not enabled with capability 
+| CONTAINER_REMOVED | Docker container was successfully removed
+| CONTAINER_STARTED | Docker container has successfully started
+| FAILED_TO_COPY_LOGS | Failed to copy logs from Docker container
+| CREATING_CONTAINER | Docker container with browser is creating
+| DEFAULT_VERSION | Selenoid is using default browser version
+| DELETED_LOG_FILE | Log file was deleted by user
+| DELETED_VIDEO_FILE | Video file was deleted by user
+| DEVTOOLS_CLIENT_DISCONNECTED | User devtools client disconnected
+| DEVTOOLS_DISABLED | An attempt to access browser devtools when it is not enabled with capability
 | DEVTOOLS_ERROR | An error occurred when trying to send devtools traffic
-| DEVTOOLS_SESSION_CLOSED | Sending devtools traffic was stopped 
-| DOWNLOADING_FILE | User requested to download file from browser container 
+| DEVTOOLS_SESSION_CLOSED | Sending devtools traffic was stopped
+| DOWNLOADING_FILE | User requested to download file from browser container
 | ENVIRONMENT_NOT_AVAILABLE | Browser with desired name and version does not exist
-| FAILED_TO_KILL_VIDEO_CONTAINER | Failed to kill video container after timeout 
-| FAILED_TO_REMOVE_CONTAINER | Failed to remove Docker container 
-| FAILED_TO_TERMINATE_PROCESS | An error occurred while terminating driver process 
-| INIT | Server is starting 
-| KILLED_VIDEO_CONTAINER | Waiting for video container to stop timed out and it was killed 
-| LOG_ERROR | An error occurred when post-processing session logs 
-| METADATA | Metadata processing messages 
+| FAILED_TO_KILL_VIDEO_CONTAINER | Failed to kill video container after timeout
+| FAILED_TO_REMOVE_CONTAINER | Failed to remove Docker container
+| FAILED_TO_TERMINATE_PROCESS | An error occurred while terminating driver process
+| INIT | Server is starting
+| KILLED_VIDEO_CONTAINER | Waiting for video container to stop timed out and it was killed
+| LOG_LISTING | Received a request to list all log files
+| LOG_ERROR | An error occurred when post-processing session logs
+| METADATA | Metadata processing messages
 | NEW_REQUEST | New user request arrived and was placed to queue
 | NEW_REQUEST_ACCEPTED | Started processing new user request
-| PROCESS_STARTED | Driver process successfully started 
-| PROXY_TO | Starting to proxy requests to running container or driver process 
+| PROCESS_STARTED | Driver process successfully started
+| PROXY_TO | Starting to proxy requests to running container or driver process
 | REMOVING_CONTAINER | Docker container with browser or video recorder is being removed
 | SERVICE_STARTED | Successfully started Docker container or driver binary
 | SERVICE_STARTUP_FAILED | Failed to start Docker container or driver binary
-| SESSION_ATTEMPTED | Started Docker container or driver binary and trying to create a new session with it 
+| SESSION_ATTEMPTED | Started Docker container or driver binary and trying to create a new session with it
 | SESSION_ATTEMPT_TIMED_OUT | An attempt to create a new session timed out
 | SESSION_CREATED | A new session was created and returned to user
 | SESSION_TIMED_OUT | Existing session was terminated by timeout
@@ -82,15 +83,16 @@ The following statuses are available:
 | SESSION_NOT_FOUND | Requested VNC or logs for unknown session.
 | STARTING_CONTAINER | Docker container with browser was created and is starting
 | STARTING_PROCESS | Starting driver process
-| SHUTTING_DOWN | Server is stopping 
-| TERMINATING_PROCESS | Stopping driver process 
-| TERMINATED_PROCESS | Driver process was successfully stopped 
-| UPLOADING_FILE | An issue occurred while uploading file 
+| SHUTTING_DOWN | Server is stopping
+| TERMINATING_PROCESS | Stopping driver process
+| TERMINATED_PROCESS | Driver process was successfully stopped
+| UPLOADING_FILE | An issue occurred while uploading file
 | UPLOADED_FILE | File successfully uploaded
-| VIDEO_ERROR | An error occurred when post-processing recorded video 
+| VIDEO_LISTING | Received a request to list all videos
+| VIDEO_ERROR | An error occurred when post-processing recorded video
 | VNC_CLIENT_DISCONNECTED | User VNC client disconnected
-| VNC_ENABLED | User requested VNC traffic 
-| VNC_ERROR | An error occurred when trying to send VNC traffic 
-| VNC_SESSION_CLOSED | Sending VNC traffic was stopped 
+| VNC_ENABLED | User requested VNC traffic
+| VNC_ERROR | An error occurred when trying to send VNC traffic
+| VNC_SESSION_CLOSED | Sending VNC traffic was stopped
 | VNC_NOT_ENABLED | User requested VNC traffic but did not specify `enableVNC` capability
 |===

main.go

@@ -269,15 +269,23 @@ func ping(w http.ResponseWriter, _ *http.Request) {
 }
 
 func video(w http.ResponseWriter, r *http.Request) {
+	requestId := serial()
 	if r.Method == http.MethodDelete {
-		deleteFileIfExists(w, r, videoOutputDir, paths.Video, "DELETED_VIDEO_FILE")
+		deleteFileIfExists(requestId, w, r, videoOutputDir, paths.Video, "DELETED_VIDEO_FILE")
+		return
+	}
+	user, remote := util.RequestInfo(r)
+	log.Printf("[%d] [LOG_LISTING] [%s] [%s]", requestId, user, remote)
+	if _, ok := r.URL.Query()[jsonParam]; ok {
+		listFilesAsJson(requestId, w, videoOutputDir, "VIDEO_ERROR")
 		return
 	}
 	fileServer := http.StripPrefix(paths.Video, http.FileServer(http.Dir(videoOutputDir)))
 	fileServer.ServeHTTP(w, r)
 }
 
-func deleteFileIfExists(w http.ResponseWriter, r *http.Request, dir string, prefix string, status string) {
+func deleteFileIfExists(requestId uint64, w http.ResponseWriter, r *http.Request, dir string, prefix string, status string) {
+	user, remote := util.RequestInfo(r)
 	fileName := strings.TrimPrefix(r.URL.Path, prefix)
 	filePath := filepath.Join(dir, fileName)
 	_, err := os.Stat(filePath)
@@ -290,7 +298,7 @@ func deleteFileIfExists(w http.ResponseWriter, r *http.Request, dir string, pref
 		http.Error(w, fmt.Sprintf("Failed to delete file %s: %v", filePath, err), http.StatusInternalServerError)
 		return
 	}
-	log.Printf("[%d] [%s] [%s]", serial(), status, fileName)
+	log.Printf("[%d] [%s] [%s] [%s] [%s]", requestId, status, user, remote, fileName)
 }
 
 var paths = struct {
@@ -327,11 +335,11 @@ func handler() http.Handler {
 	})
 	root.HandleFunc(paths.Ping, ping)
 	root.Handle(paths.VNC, websocket.Handler(vnc))
-	root.Handle(paths.Devtools, websocket.Server{Handler: devtools})
 	root.HandleFunc(paths.Logs, logs)
 	root.HandleFunc(paths.Video, video)
 	root.HandleFunc(paths.Download, reverseProxy(func(sess *session.Session) string { return sess.HostPort.Fileserver }, "DOWNLOADING_FILE"))
 	root.HandleFunc(paths.Clipboard, reverseProxy(func(sess *session.Session) string { return sess.HostPort.Clipboard }, "CLIPBOARD"))
+	root.HandleFunc(paths.Devtools, reverseProxy(func(sess *session.Session) string { return sess.HostPort.Devtools }, "DEVTOOLS"))
 	if enableFileUpload {
 		root.HandleFunc(paths.File, fileUpload)
 	}

selenoid.go

@@ -581,11 +581,22 @@ func vnc(wsconn *websocket.Conn) {
 	}
 }
 
+const (
+	jsonParam = "json"
+)
+
 func logs(w http.ResponseWriter, r *http.Request) {
+	requestId := serial()
 	fileNameOrSessionID := strings.TrimPrefix(r.URL.Path, paths.Logs)
 	if logOutputDir != "" && (fileNameOrSessionID == "" || strings.HasSuffix(fileNameOrSessionID, logFileExtension)) {
 		if r.Method == http.MethodDelete {
-			deleteFileIfExists(w, r, logOutputDir, paths.Logs, "DELETED_LOG_FILE")
+			deleteFileIfExists(requestId, w, r, logOutputDir, paths.Logs, "DELETED_LOG_FILE")
+			return
+		}
+		user, remote := util.RequestInfo(r)
+		log.Printf("[%d] [LOG_LISTING] [%s] [%s]", requestId, user, remote)
+		if _, ok := r.URL.Query()[jsonParam]; ok {
+			listFilesAsJson(requestId, w, logOutputDir, "LOG_ERROR")
 			return
 		}
 		fileServer := http.StripPrefix(paths.Logs, http.FileServer(http.Dir(logOutputDir)))
@@ -595,6 +606,21 @@ func logs(w http.ResponseWriter, r *http.Request) {
 	websocket.Handler(streamLogs).ServeHTTP(w, r)
 }
 
+func listFilesAsJson(requestId uint64, w http.ResponseWriter, dir string, errStatus string) {
+	files, err := ioutil.ReadDir(dir)
+	if err != nil {
+		log.Printf("[%d] [%s] [%s]", requestId, errStatus, fmt.Sprintf("Failed to list directory %s: %v", logOutputDir, err))
+		w.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+	var ret []string
+	for _, f := range files {
+		ret = append(ret, f.Name())
+	}
+	w.Header().Add("Content-Type", "application/json")
+	json.NewEncoder(w).Encode(ret)
+}
+
 func streamLogs(wsconn *websocket.Conn) {
 	defer wsconn.Close()
 	requestId := serial()
@@ -632,33 +658,6 @@ func status(w http.ResponseWriter, _ *http.Request) {
 		})
 }
 
-func devtools(wsconn *websocket.Conn) {
-	sid, _ := splitRequestPath(wsconn.Request().URL.Path)
-	sess, ok := sessions.Get(sid)
-	requestId := serial()
-	if ok {
-		origin := "http://localhost/"
-		u := fmt.Sprintf("ws://%s/", sess.HostPort.Devtools)
-		conn, err := websocket.Dial(u, "", origin)
-		if err != nil {
-			log.Printf("[%d] [DEVTOOLS_ERROR] [%v]", requestId, err)
-			return
-		}
-		log.Printf("[%d] [DEVTOOLS] [%s]", requestId, sid)
-		defer conn.Close()
-		wsconn.PayloadType = websocket.BinaryFrame
-		go func() {
-			io.Copy(wsconn, conn)
-			wsconn.Close()
-			log.Printf("[%d] [DEVTOOLS_SESSION_CLOSED] [%s]", requestId, sid)
-		}()
-		io.Copy(conn, wsconn)
-		log.Printf("[%d] [DEVTOOLS_CLIENT_DISCONNECTED] [%s]", requestId, sid)
-	} else {
-		log.Printf("[%d] [SESSION_NOT_FOUND] [%s]", requestId, sid)
-	}
-}
-
 func onTimeout(t time.Duration, f func()) chan struct{} {
 	cancel := make(chan struct{})
 	go func(cancel chan struct{}) {

selenoid_test.go

@@ -675,6 +675,13 @@ func TestServeAndDeleteVideoFile(t *testing.T) {
 	AssertThat(t, err, Is{nil})
 	AssertThat(t, rsp, Code{http.StatusOK})
 
+	rsp, err = http.Get(With(srv.URL).Path("/video/?json"))
+	AssertThat(t, err, Is{nil})
+	AssertThat(t, rsp, Code{http.StatusOK})
+	var files []string
+	AssertThat(t, rsp, IsJson{&files})
+	AssertThat(t, files, EqualTo{[]string{"testfile"}})
+
 	deleteReq, _ := http.NewRequest(http.MethodDelete, With(srv.URL).Path("/video/testfile"), nil)
 	rsp, err = http.DefaultClient.Do(deleteReq)
 	AssertThat(t, err, Is{nil})
@@ -695,6 +702,13 @@ func TestServeAndDeleteLogFile(t *testing.T) {
 	AssertThat(t, err, Is{nil})
 	AssertThat(t, rsp, Code{http.StatusOK})
 
+	rsp, err = http.Get(With(srv.URL).Path("/logs/?json"))
+	AssertThat(t, err, Is{nil})
+	AssertThat(t, rsp, Code{http.StatusOK})
+	var files []string
+	AssertThat(t, rsp, IsJson{&files})
+	AssertThat(t, len(files) > 0, Is{true})
+
 	deleteReq, _ := http.NewRequest(http.MethodDelete, With(srv.URL).Path("/logs/logfile.log"), nil)
 	rsp, err = http.DefaultClient.Do(deleteReq)
 	AssertThat(t, err, Is{nil})