-
Notifications
You must be signed in to change notification settings - Fork 17
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
20 changed files
with
1,079 additions
and
2,302 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,142 +1,83 @@ | ||
| package main | ||
|
|
||
| import ( | ||
| "bufio" | ||
| "fmt" | ||
| "gopkg.in/mgo.v2" | ||
| "gopkg.in/mgo.v2/bson" | ||
| "os" | ||
| "strconv" | ||
| "strings" | ||
| "time" | ||
|
|
||
| "github.com/zxsecurity/steamer/importers/util" | ||
| "gopkg.in/mgo.v2/bson" | ||
| ) | ||
|
|
||
| type AdobeData struct { | ||
| type GenericData struct { | ||
| Id bson.ObjectId `json:"id" bson:"_id,omitempty"` | ||
| MemberID int `bson:"memberid"` | ||
| Email string `bson:"email"` | ||
| Liame string `bson:"liame"` | ||
| PasswordHash string `bson:"passwordhash"` | ||
| Password string `bson:"password"` | ||
| Username string `bson:"username"` | ||
| Hint string `bson:"hint"` | ||
| Breach string `bson:"breach"` | ||
| } | ||
|
|
||
| func main() { | ||
| // Connect to mongodb | ||
| mdb, err := mgo.Dial("localhost") | ||
| defer mdb.Close() | ||
| type TemplateLineParser struct{} | ||
|
|
||
| mdb.SetSocketTimeout(24 * time.Hour) | ||
| func (t TemplateLineParser) ParseLine(line string) ([]interface{}, error) { | ||
| entries := make([]interface{}, 0) | ||
|
|
||
| if err != nil { | ||
| fmt.Fprintf(os.Stderr, "Could not connect to MongoDB: %v\r\n", err) | ||
| os.Exit(1) | ||
| } | ||
| // parse out the fields we need from this horrible format | ||
| idata := strings.Replace(line, "-", "", -1) | ||
| // use splitN so we don't remove pipes from the hint | ||
| data := strings.SplitN(idata, "|", 5) | ||
|
|
||
| threads := 15 | ||
| threader := make(chan string, threads*20) // buffered to 20 * thread size | ||
| doner := make(chan bool, threads) | ||
| // declare nil entry | ||
| entry := interface{}(nil) | ||
|
|
||
| for i := 0; i < threads; i++ { | ||
| go importLine(threader, mdb, doner) | ||
| // check data here | ||
| if len(data) != 5 { | ||
| entries = append(entries, interface{}(nil)) | ||
| return entries, nil | ||
| } | ||
|
|
||
| // open the file | ||
| file, err := os.Open("cred") | ||
| // validate/convert relevant data | ||
| memberid, err := strconv.Atoi(data[0]) | ||
| if err != nil { | ||
| fmt.Fprintf(os.Stderr, "Error opening file\r\n") | ||
| return | ||
| } | ||
| defer file.Close() | ||
|
|
||
| // TODO: Threaded! | ||
| scanner := bufio.NewScanner(file) | ||
| for scanner.Scan() { | ||
| // For each line of the file write to the channel | ||
| threader <- scanner.Text() | ||
| // returns nil entry if condition fails | ||
| entries = append(entries, entry) | ||
| return entries, nil | ||
| } | ||
|
|
||
| if err := scanner.Err(); err != nil { | ||
| fmt.Fprintf(os.Stderr, "Error scanning file\r\n") | ||
| return | ||
| hint := "" | ||
| if len(data[4]) > 2 { | ||
| hint = data[4][0 : len(data[4])-1] | ||
| } | ||
|
|
||
| // close the threader channel | ||
| close(threader) | ||
|
|
||
| // wait until all threads signal done | ||
| for i := 0; i < threads; i++ { | ||
| <-doner | ||
| fmt.Println("Thread signaled done!") | ||
| // create our struct here | ||
| entry = GenericData{ | ||
| MemberID: memberid, | ||
| Email: data[2], | ||
| Liame: util.Reverse(data[2]), | ||
| PasswordHash: data[3], | ||
| Hint: hint, | ||
| Breach: "AdobeUsers", | ||
| } | ||
| entries = append(entries, entry) | ||
| return entries, nil | ||
| } | ||
|
|
||
| func importLine(threader <-chan string, mgoreal *mgo.Session, doner chan<- bool) { | ||
| // create our mongodb copy | ||
| mgo := mgoreal.Copy() | ||
| c := mgo.DB("steamer").C("dumps") | ||
|
|
||
| bc := 0 // insert in counts of 100 | ||
|
|
||
| bulk := c.Bulk() | ||
| bulk.Unordered() | ||
|
|
||
| for text := range threader { | ||
| if bc > 10000 { | ||
| bc = 0 | ||
| bulk.Run() | ||
| bulk = c.Bulk() | ||
| bulk.Unordered() | ||
| } | ||
| bc += 1 | ||
|
|
||
| // parse out the fields we need from this horrible format | ||
| idata := strings.Replace(text, "-", "", -1) | ||
|
|
||
| // use splitN so we don't remove pipes from the hint | ||
| data := strings.SplitN(idata, "|", 5) | ||
|
|
||
| // validate/convert relevant data | ||
| memberid, err := strconv.Atoi(data[0]) | ||
| if err != nil { | ||
| // invalid line I guess? | ||
| fmt.Println("Invalid data read, conversion failed", data) | ||
| continue | ||
| } | ||
|
|
||
| // remove trailing character from hint | ||
| if len(data) != 5 { | ||
| // problem! error | ||
| fmt.Println("Invalid data, lots failed", data) | ||
| continue | ||
| } | ||
|
|
||
| hint := "" | ||
| if len(data[4]) > 2 { | ||
| hint = data[4][0 : len(data[4])-1] | ||
| } | ||
|
|
||
| // create our struct | ||
| entry := AdobeData{ | ||
| MemberID: memberid, | ||
| Email: data[2], | ||
| Liame: Reverse(data[2]), | ||
| PasswordHash: data[3], | ||
| Hint: hint, | ||
| Breach: "AdobeUsers", | ||
| } | ||
| bulk.Insert(entry) | ||
| } | ||
| // final run to be done | ||
| bulk.Run() | ||
| doner <- true | ||
| // EstimateCount estimates how many entries are in a line (for the progress bar) | ||
| func (t TemplateLineParser) EstimateCount(line string) (int, error) { | ||
| // We have one cred per line by default | ||
| return 1, nil | ||
| } | ||
|
|
||
| func Reverse(s string) string { | ||
| runes := []rune(s) | ||
| for i, j := 0, len(runes)-1; i < j; i, j = i+1, j-1 { | ||
| runes[i], runes[j] = runes[j], runes[i] | ||
| func main() { | ||
| parser := TemplateLineParser{} | ||
| // change filename to the default location of the dump file | ||
| importer, err := util.MakeImporter("/dumps/adobe.txt", parser, 15) | ||
| if err != nil { | ||
| fmt.Println(err) | ||
| return | ||
| } | ||
| return string(runes) | ||
| importer.Run() | ||
| importer.Finish() | ||
| } |
Oops, something went wrong.