use faster line counter

README.md

@@ -34,7 +34,7 @@ All funds that are donated to this project will be donated to charity. A full lo
 - get rid of `verbose` flag and introduced `debug` instead
 - the version command now also shows some build variables for more info
 - switched to another pkcs12 library to support p12s generated with openssl3 that use SHA256 HMAC
-- use a more accurate line counter which is also slower as before
+- comments in wordlists (strings starting with #) are no longer ignored
 
 ## 3.6
 

libgobuster/helpers.go

@@ -68,12 +68,11 @@ func (set *Set[T]) Stringify() string {
 	return strings.Join(values, ",")
 }
 
-// nolint:unused
-// this method is much more faster than lineCounter but has the following errors:
-// - empty files are reported as 1 lines
-// - files only containing a newline are reported as 1 lines
+// this method is much more faster than lineCounter_slow but has the following errors:
+// - empty files are reported as 1 line
+// - files only containing a newline are reported as 1 line
 // - also counts lines with comments
-func lineCounter_old(r io.Reader) (int, error) {
+func lineCounter(r io.Reader) (int, error) {
 	buf := make([]byte, 32*1024)
 	count := 1
 	lineSep := []byte{'\n'}
@@ -102,13 +101,13 @@ func lineCounter_old(r io.Reader) (int, error) {
 	}
 }
 
-func lineCounter(r io.Reader) (int, error) {
+func lineCounter_slow(r io.Reader) (int, error) {
 	scanner := bufio.NewScanner(r)
 	scanner.Split(bufio.ScanLines)
 	var count int
 	for scanner.Scan() {
 		w := scanner.Text()
-		if w == "" || strings.HasPrefix(w, "#") {
+		if w == "" {
 			continue
 		}
 

libgobuster/helpers_test.go

@@ -166,12 +166,10 @@ func TestLineCounter(t *testing.T) {
 		{"One Line", "test", 1},
 		{"3 Lines", "TestString\nTest\n1234", 3},
 		{"Trailing newline", "TestString\nTest\n1234\n", 3},
-		{"Trailing newline with comment", "TestString\n# Test\n1234\n", 2},
 		{"3 Lines cr lf", "TestString\r\nTest\r\n1234", 3},
-		{"3 Lines cr lf with comment", "TestString\r\n# Test\r\n1234", 2},
-		{"Empty", "", 0},
-		{"Empty 2", "\n", 0},
-		{"Empty 3", "\r\n", 0},
+		{"Empty", "", 1},       // these are wrong but I've found no good way to handle those
+		{"Empty 2", "\n", 1},   // these are wrong but I've found no good way to handle those
+		{"Empty 3", "\r\n", 1}, // these are wrong but I've found no good way to handle those
 	}
 	for _, x := range tt {
 		x := x // NOTE: https://github.com/golang/go/wiki/CommonMistakes#using-goroutines-on-loop-iterator-variables
@@ -189,7 +187,7 @@ func TestLineCounter(t *testing.T) {
 	}
 }
 
-func TestLineCounterOld(t *testing.T) {
+func TestLineCounterSlow(t *testing.T) {
 	t.Parallel()
 	var tt = []struct {
 		testName string
@@ -200,16 +198,16 @@ func TestLineCounterOld(t *testing.T) {
 		{"3 Lines", "TestString\nTest\n1234", 3},
 		{"Trailing newline", "TestString\nTest\n1234\n", 3},
 		{"3 Lines cr lf", "TestString\r\nTest\r\n1234", 3},
-		{"Empty", "", 1},       // these are wrong but I've found no good way to handle those
-		{"Empty 2", "\n", 1},   // these are wrong but I've found no good way to handle those
-		{"Empty 3", "\r\n", 1}, // these are wrong but I've found no good way to handle those
+		{"Empty", "", 0},
+		{"Empty 2", "\n", 0},
+		{"Empty 3", "\r\n", 0},
 	}
 	for _, x := range tt {
 		x := x // NOTE: https://github.com/golang/go/wiki/CommonMistakes#using-goroutines-on-loop-iterator-variables
 		t.Run(x.testName, func(t *testing.T) {
 			t.Parallel()
 			r := strings.NewReader(x.s)
-			l, err := lineCounter_old(r)
+			l, err := lineCounter_slow(r)
 			if err != nil {
 				t.Fatalf("Got error: %v", err)
 			}
@@ -227,24 +225,30 @@ func BenchmarkLineCounter(b *testing.B) {
 			b.Fatalf("Got error: %v", err)
 		}
 		defer r.Close()
-		_, err = lineCounter(r)
+		c, err := lineCounter(r)
 		if err != nil {
 			b.Fatalf("Got error: %v", err)
 		}
+		if c != 14344391 {
+			b.Errorf("invalid count. Expected 14344391, got %d", c)
+		}
 	}
 }
 
-func BenchmarkLineCounterOld(b *testing.B) {
+func BenchmarkLineCounterSlow(b *testing.B) {
 	for i := 0; i < b.N; i++ {
 		r, err := os.Open("../rockyou.txt")
 		if err != nil {
 			b.Fatalf("Got error: %v", err)
 		}
 		defer r.Close()
-		_, err = lineCounter_old(r)
+		c, err := lineCounter_slow(r)
 		if err != nil {
 			b.Fatalf("Got error: %v", err)
 		}
+		if c != 14336792 {
+			b.Errorf("invalid count. Expected 14336792, got %d", c)
+		}
 	}
 }
 

libgobuster/libgobuster.go

@@ -55,8 +55,8 @@ func (g *Gobuster) worker(ctx context.Context, wordChan <-chan string, wg *sync.
 			g.Progress.incrementRequests()
 
 			wordCleaned := strings.TrimSpace(word)
-			// Skip "comment" (starts with #), as well as empty lines
-			if strings.HasPrefix(wordCleaned, "#") || len(wordCleaned) == 0 {
+			// Skip empty lines
+			if len(wordCleaned) == 0 {
 				break
 			}