Merge pull request #10 from mlr0p/master

Added 6 Importers

README.md

@@ -12,7 +12,7 @@ Install
 -------
 
 - Install Go and MongoDB.
-- `go get gopkg.in/mgo.v2 && go get github.com/gorilla/mux`
+- Dependencies are managed using Go modules in `go.mod`. Therefore, go will automatically download the required modules if it can't find one.
 
 At this point, it is recommended to import one of the more simple breaches that do not require an index to import.
 
@@ -51,6 +51,12 @@ That template is threaded and designed for CSVs. See `./importers/linkedin2016.g
 
 If you write an importer for a public breach, please send a pull request so everyone can import it too. Please note that no public breaches are provided here in the repository itself.
 
+Running an importer
+-----------------
+Running an importer is straightforward. Just change the filename in the source code if needed and run `go run ./importers/<importer_name>.go`. 
+There is also a verbose flag that shows a progress bar, if you want to use it just add the flag `go run importers/<importer_name>.go -v`.
+However, be mindful that enabling it will introduce several minutes of delay on average depending on the size of the file.
+
 Problems?
 ---------
 

go.mod

@@ -0,0 +1,13 @@
+module github.com/zxsecurity/steamer
+
+go 1.15
+
+require (
+	github.com/cheggaaa/pb/v3 v3.0.5
+	github.com/fatih/structs v1.1.0
+	github.com/gorilla/mux v1.8.0
+	github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
+	gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b // indirect
+	gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22
+	gopkg.in/yaml.v2 v2.3.0 // indirect
+)

go.sum

@@ -0,0 +1,32 @@
+github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM=
+github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
+github.com/cheggaaa/pb/v3 v3.0.5 h1:lmZOti7CraK9RSjzExsY53+WWfub9Qv13B5m4ptEoPE=
+github.com/cheggaaa/pb/v3 v3.0.5/go.mod h1:X1L61/+36nz9bjIsrDU52qHKOQukUQe2Ge+YvGuquCw=
+github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
+github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo=
+github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
+github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
+github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU=
+github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
+github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
+github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54=
+github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
+golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42 h1:vEOn+mP2zCOVzKckCZy6YsCtDblrpj/w7B9nxGNELpg=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b h1:QRR6H1YWRnHb4Y/HeNFCTJLFVxaq6wH4YuVdsUOr75U=
+gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
+gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

importers/adultfriendfinder.go

@@ -0,0 +1,281 @@
+package main
+
+/***
+ * Format: SQL
+ * Hashing Method: hashcat mode 100 -> SHA1, sha1($pwd, 'blib')
+ * Hash format to crack: <password_hash>
+ */
+
+import (
+	"bufio"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/cheggaaa/pb/v3"
+	"gopkg.in/mgo.v2"
+	"gopkg.in/mgo.v2/bson"
+)
+
+type AdultFriendFinderData struct {
+	Id           bson.ObjectId `json:"id" bson:"_id,omitempty"`
+	MemberID     int           `bson:"memberid"`
+	Username     string        `bson:"username"`
+	Email        string        `bson:"email"`
+	Liame        string        `bson:"liame"`
+	PasswordHash string        `bson:"passwordhash"`
+	Password     string        `bson:"password"`
+	Breach       string        `bson:"breach"`
+}
+
+func main() {
+	// Get command-line flags
+	verbose := flag.Bool("v", false, "Displays progress bar")
+	flag.Parse()
+
+	// Connect to mongodb
+	mdb, err := mgo.Dial("localhost")
+	defer mdb.Close()
+
+	mdb.SetSocketTimeout(24 * time.Hour)
+
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Could not connect to MongoDB: %v\r\n", err)
+		os.Exit(1)
+	}
+
+	threads := 15
+	threader := make(chan string, threads*20) // buffered to 20 * thread size
+	doner := make(chan bool, threads)
+
+	// Set filename
+	filename := "ffadult_edb_users.sql"
+
+	// Make ProgressBar
+	var bar *pb.ProgressBar
+	if *verbose {
+		// Time it
+		start := time.Now()
+		bar = makePbar(filename)
+		elapsed := time.Since(start)
+		fmt.Printf("Making pbar took %s\n", elapsed)
+		if bar == nil {
+			fmt.Fprintf(os.Stderr, "Could not open file %v\r\n", filename)
+			return
+		}
+	}
+
+	for i := 0; i < threads; i++ {
+		go importLine(threader, mdb, doner, bar)
+	}
+
+	// open the file
+	file, err := os.Open(filename)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error opening file\r\n")
+		return
+	}
+	defer file.Close()
+
+	// Quite large, perhaps not required, but better safe than sorry
+	r := bufio.NewReaderSize(file, 1024*1024)
+
+	line, isPrefix, err := r.ReadLine()
+	for err == nil && !isPrefix {
+		s := string(line)
+
+		// Lines with the INSERT line should be further parsed into individual accounts
+		if strings.Contains(s, "INSERT INTO `users`") {
+			// It's a valid line we should be parsing!
+			threader <- s
+		}
+
+		line, isPrefix, err = r.ReadLine()
+	}
+	if isPrefix {
+		fmt.Println("buffer size to small -- increase it in the NewReaderSize line!")
+		return
+	}
+	if err != io.EOF {
+		fmt.Fprintf(os.Stderr, "Error scanning file - %s\r\n", err)
+		return
+	}
+
+	// close the threader channel
+	close(threader)
+
+	// wait until all threads signal done
+	for i := 0; i < threads; i++ {
+		<-doner
+	}
+	// finish progress bar
+	if bar != nil {
+		bar.Finish()
+	}
+}
+
+func importLine(threader <-chan string, mgoreal *mgo.Session, doner chan<- bool, bar *pb.ProgressBar) {
+	// create our mongodb copy
+	mgo := mgoreal.Copy()
+	c := mgo.DB("steamer").C("dumps")
+
+	bc := 0 // insert in counts of 100
+
+	bulk := c.Bulk()
+	bulk.Unordered()
+
+	for text := range threader {
+		if bc > 10000 {
+			bulk.Run()
+			if bar != nil {
+				bar.Add(bc)
+			}
+			bc = 0
+			bulk = c.Bulk()
+			bulk.Unordered()
+		}
+		// Split out the INSERT statement so we have the tuples we need
+		// Start by removing the first and last characters
+		text2 := text[31 : len(text)-2]
+
+		// Split on "),("
+		users := strings.Split(text2, "),(")
+
+		// Now loop through and process each line and INSERT as required
+		tmpBulkCount := 0
+		for _, row := range users {
+			strippedRow := strings.ReplaceAll(row, "'", "")
+			data := strings.Split(strippedRow, ",")
+
+			// format: pwsid, handle, email, lastvisit, password, send_email, show_lang.....
+			if len(data) < 4 {
+				// problem! error
+				fmt.Println("Invalid data, epxected a len > 4, got ", len(data), strippedRow)
+				continue
+			}
+
+			// We remove the underscore and convert it to member ID
+			memberid, err := strconv.Atoi(strings.Replace(data[0], "_", "", 1))
+			if err != nil {
+				fmt.Println("Invalid data read, conversion failed", data)
+				continue
+			}
+
+			// There may be multiple emails in the email column, get the index of the last email
+			// The default index of email is 2
+			lastEmailIndex := 2
+			for i := 2; i < len(data); i++ {
+				// extremely rough check if it's a valid email
+				if strings.Contains(data[i], "@") {
+					lastEmailIndex = i
+				} else {
+					break
+				}
+			}
+			// We only care about users with password hash
+			if len(data[lastEmailIndex+2]) == 40 {
+				// Import all the emails
+				for i := 2; i <= lastEmailIndex; i++ {
+					// Parse relevant entries
+					entry := AdultFriendFinderData{
+						MemberID:     memberid,
+						Email:        data[i],
+						Liame:        Reverse(data[i]),
+						PasswordHash: data[4],
+						Username:     data[1],
+						Breach:       "AdultFriendFinder2016",
+					}
+					bulk.Insert(entry)
+					tmpBulkCount += 1
+				}
+			} else {
+				// Here we insert those emails with commas
+				if len(data) > 4 {
+					hasPwd := false
+					for i := 4; i < len(data); i++ {
+						if len(data[i]) == 40 {
+							hasPwd = true
+						}
+					}
+					// Check if we skipped over some password hash
+					// This may happen if we have comma in the email
+					if hasPwd {
+						tmpData := strings.Split(row, "','")
+						// insert this entry
+						entry := AdultFriendFinderData{
+							MemberID:     memberid,
+							Email:        tmpData[2],
+							Liame:        Reverse(tmpData[2]),
+							PasswordHash: tmpData[4],
+							Username:     tmpData[1],
+							Breach:       "AdultFriendFinder2016",
+						}
+						bulk.Insert(entry)
+						tmpBulkCount += 1
+					}
+				}
+			}
+		}
+		bc += tmpBulkCount
+		if bar != nil {
+			bar.Add(len(users) - tmpBulkCount)
+		}
+	}
+	// final run to be done
+	bulk.Run()
+	if bar != nil {
+		bar.Add(bc)
+	}
+	doner <- true
+}
+
+func Reverse(s string) string {
+	runes := []rune(s)
+	for i, j := 0, len(runes)-1; i < j; i, j = i+1, j-1 {
+		runes[i], runes[j] = runes[j], runes[i]
+	}
+	return string(runes)
+}
+
+func makePbar(filename string) *pb.ProgressBar {
+	fmt.Println("Making pbar...")
+	// count total number of records we need to process
+	count := 0
+	// open the file
+	file, err := os.Open(filename)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error opening file\r\n")
+		return nil
+	}
+	defer file.Close()
+
+	// Quite large, perhaps not required, but better safe than sorry
+	r := bufio.NewReaderSize(file, 1024*1024)
+
+	line, isPrefix, err := r.ReadLine()
+	for err == nil && !isPrefix {
+		text := string(line)
+
+		// Lines with the INSERT line should be further parsed into individual accounts
+		if strings.Contains(text, "INSERT INTO `users`") {
+			text2 := text[31 : len(text)-2]
+			count += len(strings.Split(text2, "),("))
+		}
+
+		line, isPrefix, err = r.ReadLine()
+	}
+	if isPrefix {
+		fmt.Println("buffer size to small -- increase it in the NewReaderSize line!")
+		return nil
+	}
+	if err != io.EOF {
+		fmt.Fprintf(os.Stderr, "Error scanning file - %s\r\n", err)
+		return nil
+	}
+	fmt.Println("Total number of records: ", count)
+	return pb.StartNew(count)
+}