[fs] Add a fs.readAll method to the fs module (3/3)

examples/experimental/fs/bonjour.txt

@@ -0,0 +1 @@
+Bonjour, tout le monde!

examples/experimental/fs/fs.js

@@ -0,0 +1,52 @@
+import { open, SeekMode } from "k6/experimental/fs";
+
+export const options = {
+	vus: 100,
+	iterations: 1000,
+};
+
+// k6 doesn't support async in the init context. We use a top-level async function for `await`.
+//
+// Each Virtual User gets its own `file` copy.
+// So, operations like `seek` or `read` won't impact other VUs.
+let file;
+(async function () {
+	file = await open("bonjour.txt");
+})();
+
+export default async function () {
+	// About information about the file
+	const fileinfo = await file.stat();
+	if (fileinfo.name != "bonjour.txt") {
+		throw new Error("Unexpected file name");
+	}
+
+	const buffer = new Uint8Array(4);
+
+	let totalBytesRead = 0;
+	while (true) {
+		// Read into the buffer
+		const bytesRead = await file.read(buffer);
+		if (bytesRead == null) {
+			// EOF
+			break;
+		}
+
+		// Do something useful with the content of the buffer
+
+		totalBytesRead += bytesRead;
+
+		// If bytesRead is less than the buffer size, we've read the whole file
+		if (bytesRead < buffer.byteLength) {
+			break;
+		}
+	}
+
+	// Check that we read the expected number of bytes
+	if (totalBytesRead != fileinfo.size) {
+		throw new Error("Unexpected number of bytes read");
+	}
+
+	// Seek back to the beginning of the file
+	await file.seek(0, SeekMode.Start);
+}

js/initcontext.go

@@ -4,7 +4,6 @@ import (
 	"errors"
 	"fmt"
 	"net/url"
-	"path/filepath"
 
 	"github.com/dop251/goja"
 	"github.com/sirupsen/logrus"
@@ -20,18 +19,7 @@ const cantBeUsedOutsideInitContextMsg = `the "%s" function is only available in
 // contents of a file. If the second argument is "b" it returns an ArrayBuffer
 // instance, otherwise a string representation.
 func openImpl(rt *goja.Runtime, fs fsext.Fs, basePWD *url.URL, filename string, args ...string) (goja.Value, error) {
-	// Here IsAbs should be enough but unfortunately it doesn't handle absolute paths starting from
-	// the current drive on windows like `\users\noname\...`. Also it makes it more easy to test and
-	// will probably be need for archive execution under windows if always consider '/...' as an
-	// absolute path.
-	if filename[0] != '/' && filename[0] != '\\' && !filepath.IsAbs(filename) {
-		filename = filepath.Join(basePWD.Path, filename)
-	}
-	filename = filepath.Clean(filename)
-
-	if filename[0:1] != fsext.FilePathSeparator {
-		filename = fsext.FilePathSeparator + filename
-	}
+	filename = fsext.Abs(basePWD.Path, filename)
 
 	data, err := readFile(fs, filename)
 	if err != nil {

js/jsmodules.go

@@ -8,6 +8,7 @@ import (
 	"go.k6.io/k6/js/modules/k6/data"
 	"go.k6.io/k6/js/modules/k6/encoding"
 	"go.k6.io/k6/js/modules/k6/execution"
+	"go.k6.io/k6/js/modules/k6/experimental/fs"
 	"go.k6.io/k6/js/modules/k6/experimental/tracing"
 	"go.k6.io/k6/js/modules/k6/grpc"
 	"go.k6.io/k6/js/modules/k6/html"
@@ -38,6 +39,7 @@ func getInternalJSModules() map[string]interface{} {
 		"k6/experimental/timers":     timers.New(),
 		"k6/experimental/tracing":    tracing.New(),
 		"k6/experimental/browser":    browser.New(),
+		"k6/experimental/fs":         fs.New(),
 		"k6/net/grpc":                grpc.New(),
 		"k6/html":                    html.New(),
 		"k6/http":                    http.New(),

js/modules/k6/experimental/fs/cache.go

@@ -0,0 +1,86 @@
+package fs
+
+import (
+	"fmt"
+	"io"
+	"path/filepath"
+	"sync"
+
+	"github.com/spf13/afero"
+)
+
+// cache is a cache of opened files.
+type cache struct {
+	// openedFiles holds a safe for concurrent use map, holding the content
+	// of the files that were opened by the user.
+	//
+	// Keys are expected to be strings holding the openedFiles' path.
+	// Values are expected to be byte slices holding the content of the opened file.
+	//
+	// That way, we can cache the file's content and avoid opening too many
+	// file descriptor, and re-reading its content every time the file is opened.
+	//
+	// Importantly, this also means that if the
+	// file is modified from outside of k6, the changes will not be reflected in the file's data.
+	openedFiles sync.Map
+}
+
+// open retrieves the content of a given file from the specified filesystem (fromFs) and
+// stores it in the registry's internal `openedFiles` map.
+//
+// The function cleans the provided filename using filepath.Clean before using it.
+//
+// If the file was previously "opened" (and thus cached) by the registry, it
+// returns the cached content. Otherwise, it reads the file from the
+// filesystem, caches its content, and then returns it.
+//
+// The function is designed to minimize redundant file reads by leveraging an internal cache (openedFiles).
+// In case the cached value is not a byte slice (which should never occur in regular use), it
+// panics with a descriptive error.
+//
+// Parameters:
+//   - filename: The name of the file to be retrieved. This should be a relative or absolute path.
+//   - fromFs: The filesystem (from the afero package) from which the file should be read if not already cached.
+//
+// Returns:
+//   - A byte slice containing the content of the specified file.
+//   - An error if there's any issue opening or reading the file. If the file content is
+//     successfully cached and returned once, subsequent calls will not produce
+//     file-related errors for the same file, as the cached value will be used.
+func (fr *cache) open(filename string, fromFs afero.Fs) (data []byte, err error) {
+	filename = filepath.Clean(filename)
+
+	if f, ok := fr.openedFiles.Load(filename); ok {
+		data, ok = f.([]byte)
+		if !ok {
+			panic(fmt.Errorf("registry's file %s is not stored as a byte slice", filename))
+		}
+
+		return data, nil
+	}
+
+	// The underlying afero.Fs.Open method will cache the file content during this
+	// operation. Which will lead to effectively holding the content of the file in memory twice.
+	// However, as per #1079, we plan to eventually reduce our dependency on afero, and
+	// expect this issue to be resolved at that point.
+	// TODO: re-evaluate opening from the FS this once #1079 is resolved.
+	f, err := fromFs.Open(filename)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		cerr := f.Close()
+		if cerr != nil {
+			err = fmt.Errorf("failed to close file %s: %w", filename, cerr)
+		}
+	}()
+
+	data, err = io.ReadAll(f)
+	if err != nil {
+		return nil, err
+	}
+
+	fr.openedFiles.Store(filename, data)
+
+	return data, nil
+}

js/modules/k6/experimental/fs/cache_test.go

@@ -0,0 +1,70 @@
+package fs
+
+import (
+	"testing"
+
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFileCacheOpen(t *testing.T) {
+	t.Parallel()
+
+	t.Run("open succeeds", func(t *testing.T) {
+		t.Parallel()
+
+		cache := &cache{}
+		fs := newTestFs(t, func(fs afero.Fs) error {
+			return afero.WriteFile(fs, "bonjour.txt", []byte("Bonjour, le monde"), 0o644)
+		})
+
+		_, gotBeforeOk := cache.openedFiles.Load("bonjour.txt")
+		gotData, gotErr := cache.open("bonjour.txt", fs)
+		_, gotAfterOk := cache.openedFiles.Load("bonjour.txt")
+
+		assert.False(t, gotBeforeOk)
+		assert.NoError(t, gotErr)
+		assert.Equal(t, []byte("Bonjour, le monde"), gotData)
+		assert.True(t, gotAfterOk)
+	})
+
+	t.Run("double open succeeds", func(t *testing.T) {
+		t.Parallel()
+
+		cache := &cache{}
+		fs := newTestFs(t, func(fs afero.Fs) error {
+			return afero.WriteFile(fs, "bonjour.txt", []byte("Bonjour, le monde"), 0o644)
+		})
+
+		firstData, firstErr := cache.open("bonjour.txt", fs)
+		_, gotFirstOk := cache.openedFiles.Load("bonjour.txt")
+		secondData, secondErr := cache.open("bonjour.txt", fs)
+		_, gotSecondOk := cache.openedFiles.Load("bonjour.txt")
+
+		assert.True(t, gotFirstOk)
+		assert.NoError(t, firstErr)
+		assert.Equal(t, []byte("Bonjour, le monde"), firstData)
+		assert.True(t, gotSecondOk)
+		assert.NoError(t, secondErr)
+		assert.True(t, sameUnderlyingArray(firstData, secondData))
+		assert.Equal(t, []byte("Bonjour, le monde"), secondData)
+	})
+}
+
+// sameUnderlyingArray returns true if the underlying array of lhs and rhs are the same.
+//
+// This is done by checking that the two slices have a capacity greater than 0 and that
+// the last element of the underlying array is the same for both slices.
+//
+// Once a slice is created, its starting address can move forward, but can never move
+// behond its starting address + its capacity, which is a fixed value for any Go slice.
+//
+// Hence, if the last element of the underlying array is the same for both slices, it
+// means that the underlying array is the same.
+//
+// See [explanation] for more details.
+//
+// [explanation]: https://groups.google.com/g/golang-nuts/c/ks1jvoyMYuc?pli=1
+func sameUnderlyingArray(lhs, rhs []byte) bool {
+	return cap(lhs) > 0 && cap(rhs) > 0 && &lhs[0:cap(lhs)][cap(lhs)-1] == &rhs[0:cap(rhs)][cap(rhs)-1]
+}

js/modules/k6/experimental/fs/errors.go

@@ -0,0 +1,71 @@
+package fs
+
+// newFsError creates a new Error object of the provided kind and with the
+// provided message.
+func newFsError(k errorKind, message string) *fsError {
+	return &fsError{
+		Name:    k.String(),
+		Message: message,
+		kind:    k,
+	}
+}
+
+// errorKind indicates the kind of file system error that has occurred.
+//
+// Its string representation is generated by the `enumer` tool. The
+// `enumer` tool is run by the `go generate` command. See the `go generate`
+// command documentation.
+// The tool itself is not tracked as part of the k6 go.mod file, and
+// therefore must be installed manually using `go install github.com/dmarkham/enumer`.
+//
+//go:generate enumer -type=errorKind -output errors_gen.go
+type errorKind uint8
+
+const (
+	// NotFoundError is emitted when a file is not found.
+	NotFoundError errorKind = iota + 1
+
+	// InvalidResourceError is emitted when a resource is invalid: for
+	// instance when attempting to open a directory, which is not supported.
+	InvalidResourceError
+
+	// ForbiddenError is emitted when an operation is forbidden.
+	ForbiddenError
+
+	// TypeError is emitted when an incorrect type has been used.
+	TypeError
+
+	// EOFError is emitted when the end of a file has been reached.
+	EOFError
+)
+
+// fsError represents a custom error object emitted by the fs module.
+//
+// It is used to provide a more detailed error message to the user, and
+// provide a concrete error type that can be used to differentiate between
+// different types of errors.
+//
+// Exposing error types to the user in a way that's compatible with some
+// JavaScript error handling constructs such as `instanceof` is still non-trivial
+// in Go. See the [dedicated goja issue] with have opened for more details.
+//
+// [dedicated goja issue]: https://github.com/dop251/goja/issues/529
+type fsError struct {
+	// Name contains the name of the error as formalized by the [ErrorKind]
+	// type.
+	Name string `json:"name"`
+
+	// Message contains the error message as presented to the user.
+	Message string `json:"message"`
+
+	// kind contains the kind of error that has occurred.
+	kind errorKind
+}
+
+// Ensure that the Error type implements the Go `error` interface.
+var _ error = (*fsError)(nil)
+
+// Error implements the Go `error` interface.
+func (e *fsError) Error() string {
+	return e.Name + ": " + e.Message
+}