From ad5b59e7443adf37f46c281ccae6be9c0d11b0ec Mon Sep 17 00:00:00 2001 From: Dhruv <136118444+slashexx@users.noreply.github.com> Date: Fri, 20 Dec 2024 00:37:29 +0530 Subject: [PATCH 1/3] Rename LICENSE.MD to LICENSE --- LICENSE.MD => LICENSE | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename LICENSE.MD => LICENSE (100%) diff --git a/LICENSE.MD b/LICENSE similarity index 100% rename from LICENSE.MD rename to LICENSE From 81de1e5c65cf95d5cde83917506e3ccd6080d3f0 Mon Sep 17 00:00:00 2001 From: slashexx Date: Sun, 29 Dec 2024 00:40:55 +0530 Subject: [PATCH 2/3] Firebase now mass transfer bois lesgo Signed-off-by: slashexx --- .gitignore | 5 ++- config.yaml | 7 ---- integrations/firebase.go | 71 ++++++++++++++++++---------------------- integrations/mongodb.go | 1 + 4 files changed, 36 insertions(+), 48 deletions(-) delete mode 100644 config.yaml diff --git a/.gitignore b/.gitignore index 3da7a70..c57f8ff 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,7 @@ .env myenv/ firebaseConfig.json -tests/firebaseConfig.json \ No newline at end of file +tests/firebaseConfig.json +config.yaml +pbweb.json +mongo.txt \ No newline at end of file diff --git a/config.yaml b/config.yaml deleted file mode 100644 index c1fb9aa..0000000 --- a/config.yaml +++ /dev/null @@ -1,7 +0,0 @@ -inputconfig: - url: sdfxghjuk -inputmethod: WebSocket -outputconfig: - topic: bfhtd - url: dfghjkl -outputmethod: Kafka diff --git a/integrations/firebase.go b/integrations/firebase.go index 0a9135c..606ec24 100644 --- a/integrations/firebase.go +++ b/integrations/firebase.go @@ -3,13 +3,14 @@ package integrations import ( "context" "encoding/json" - "errors" + _ "errors" "fmt" - "strings" + _ "strings" "sync" "time" firebase "firebase.google.com/go" + _ "google.golang.org/api/iterator" "google.golang.org/api/option" "github.com/SkySingh04/fractal/interfaces" @@ -30,8 +31,7 @@ type FirebaseDestination struct { } func (f FirebaseSource) FetchData(req interfaces.Request) (interface{}, error) { - logger.Infof("Connecting to Firebase Source: Collection=%s, Document=%s, using Service Account=%s", - req.Collection, req.Document, req.CredentialFileAddr) + logger.Infof("Connecting to Firebase Source: Collection=%s, using Service Account=%s", req.Collection, req.CredentialFileAddr) opt := option.WithCredentialsFile(req.CredentialFileAddr) app, err := firebase.NewApp(context.Background(), nil, opt) @@ -45,41 +45,30 @@ func (f FirebaseSource) FetchData(req interfaces.Request) (interface{}, error) { } defer client.Close() - dataChan := make(chan map[string]interface{}, 1) - errChan := make(chan error, 1) - var wg sync.WaitGroup + docs, err := client.Collection(req.Collection).Documents(context.Background()).GetAll() + if err != nil { + return nil, fmt.Errorf("failed to fetch documents: %w", err) + } - wg.Add(1) - go func() { - defer wg.Done() - dsnap, err := client.Collection(req.Collection).Doc(req.Document).Get(context.Background()) - if err != nil { - errChan <- fmt.Errorf("failed to fetch document from Firestore: %w", err) - return - } + logger.Infof("Fetched documents from Firebase: %d documents", len(docs)) + for i, doc := range docs { + logger.Infof("Document %d ID: %s, Data: %v", i, doc.Ref.ID, doc.Data()) + } - if !dsnap.Exists() { - errChan <- fmt.Errorf("document not found: Collection=%s, Document=%s", req.Collection, req.Document) - return - } + var allData []map[string]interface{} + for _, doc := range docs { + data := doc.Data() + logger.Infof("Fetched data from Firebase: %v", data) - dataChan <- dsnap.Data() - }() + data["_id"] = doc.Ref.ID - wg.Wait() - close(dataChan) - close(errChan) + validatedData := data + transformedData := validatedData - select { - case data := <-dataChan: - validatedData, err := validateFirebaseData(data) - if err != nil { - return nil, err - } - return transformFirebaseData(validatedData), nil - case err := <-errChan: - return nil, err + allData = append(allData, transformedData) } + + return allData, nil } func (f FirebaseDestination) SendData(data interface{}, req interfaces.Request) error { @@ -127,6 +116,8 @@ func (f FirebaseDestination) SendData(data interface{}, req interfaces.Request) } func convertToMap(data interface{}, result *map[string]interface{}) error { + logger.Infof("Firebase data to map: %v", data) + temp, err := json.Marshal(data) if err != nil { return fmt.Errorf("failed to marshal data to JSON: %w", err) @@ -140,18 +131,18 @@ func convertToMap(data interface{}, result *map[string]interface{}) error { func validateFirebaseData(data map[string]interface{}) (map[string]interface{}, error) { logger.Infof("Validating Firebase data: %v", data) - message, ok := data["data"].(string) - if !ok || strings.TrimSpace(message) == "" { - return nil, errors.New("invalid or missing 'data' field") - } + // // message, ok := data; + // if !ok || strings.TrimSpace(message) == "" { + // return nil, errors.New("invalid or missing 'data' field") + // } return data, nil } func transformFirebaseData(data map[string]interface{}) map[string]interface{} { logger.Infof("Transforming Firebase data: %v", data) - if message, ok := data["data"].(string); ok { - data["data"] = strings.ToUpper(message) - } + // if message, ok := data["data"].(string); ok { + // data["data"] = strings.ToUpper(message) + // } data["processed"] = time.Now().Format(time.RFC3339) return data } diff --git a/integrations/mongodb.go b/integrations/mongodb.go index 42b7cd7..2d84d32 100644 --- a/integrations/mongodb.go +++ b/integrations/mongodb.go @@ -161,6 +161,7 @@ func init() { } func TransformDataToBSON(data interface{}) ([]bson.M, error) { + logger.Infof("Data received for BSON conversion insertion: %+v", data) switch v := data.(type) { case map[string]interface{}: // Single document return []bson.M{v}, nil From 2607f6647539e162473bce3564989c2fe172179a Mon Sep 17 00:00:00 2001 From: slashexx Date: Sun, 29 Dec 2024 00:48:51 +0530 Subject: [PATCH 3/3] why is config causing conflicts Signed-off-by: slashexx --- .gitignore | 1 - config.yaml | 8 ++++++++ 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 config.yaml diff --git a/.gitignore b/.gitignore index c57f8ff..d83caa2 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,5 @@ myenv/ firebaseConfig.json tests/firebaseConfig.json -config.yaml pbweb.json mongo.txt \ No newline at end of file diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..edb0b55 --- /dev/null +++ b/config.yaml @@ -0,0 +1,8 @@ +inputconfig: + topic: dsfdgfh + url: sdfgbh +inputmethod: Kafka +outputconfig: + queuename: sfdgrfthgyj + url: sefrgdthfy +outputmethod: RabbitMQ