Merge pull request #154 from yaacov/catch-errors-in-storage

catch errors in storage and alerts
MohawkTSDB · Jan 10, 2018 · 7d5885d · 7d5885d
2 parents 65bb9d2 + f01f1f6
commit 7d5885d
Show file tree

Hide file tree

Showing 2 changed files with 66 additions and 34 deletions.
diff --git a/src/alerts/alerts.go b/src/alerts/alerts.go
@@ -155,18 +155,28 @@ func (a *AlertRules) checkAlerts() {
 
 		// add metrics from tags query
 		if alert.Tags != "" {
-			res, _ := a.Storage.GetItemList(tenant, storage.ParseTags(alert.Tags))
-			for _, r := range res {
-				metrics = append(metrics, r.ID)
+			if res, err := a.Storage.GetItemList(tenant, storage.ParseTags(alert.Tags)); err != nil {
+				// add query items, if no errors
+				for _, r := range res {
+					metrics = append(metrics, r.ID)
+				}
+			} else if a.Verbose {
+				// log query errors
+				log.Println(err)
 			}
 		}
 
 		for _, metric := range metrics {
-			rawData, _ := a.Storage.GetRawData(tenant, metric, end, start, limit, "DESC")
+			rawData, err := a.Storage.GetRawData(tenant, metric, end, start, limit, "DESC")
+
+			// log query errors
+			if a.Verbose && err != nil {
+				log.Println(err)
+			}
 
 			// if we have new data check for alert status change
 			// [ if no new data found, leave alert status un changed ]
-			if len(rawData) > 0 {
+			if err == nil && len(rawData) > 0 {
 
 				// update alert state and report to user if changed.
 				newState = alert.alertState(rawData[0].Value)

diff --git a/src/storage/sqlite/sqlite.go b/src/storage/sqlite/sqlite.go
@@ -251,22 +251,28 @@ func (r Storage) GetStatData(tenant string, id string, end int64, start int64, l
 func (r Storage) PostRawData(tenant string, id string, t int64, v float64) error {
 	// check if id exist
 	if !r.IDExist(tenant, id) {
-		r.createID(tenant, id)
+		if err := r.createID(tenant, id); err != nil {
+			return err
+		}
 	}
 
-	r.insertData(tenant, id, t, v)
-	return nil
+	err := r.insertData(tenant, id, t, v)
+	return err
 }
 
 // PutTags handle posting tags to db
 func (r Storage) PutTags(tenant string, id string, tags map[string]string) error {
 	// check if id exist
 	if !r.IDExist(tenant, id) {
-		r.createID(tenant, id)
+		if err := r.createID(tenant, id); err != nil {
+			return err
+		}
 	}
 
 	for k, v := range tags {
-		r.insertTag(tenant, id, k, v)
+		if err := r.insertTag(tenant, id, k, v); err != nil {
+			return err
+		}
 	}
 	return nil
 }
@@ -275,24 +281,26 @@ func (r Storage) PutTags(tenant string, id string, tags map[string]string) error
 func (r Storage) DeleteData(tenant string, id string, end int64, start int64) error {
 	// check if id exist
 	if r.IDExist(tenant, id) {
-		r.deleteData(tenant, id, end, start)
-		return nil
+		err := r.deleteData(tenant, id, end, start)
+		return err
 	}
 
-	return errors.New("ID not found")
+	return errors.New("slite: ID not found")
 }
 
 // DeleteTags handle delete tags fron db
 func (r Storage) DeleteTags(tenant string, id string, tags []string) error {
 	// check if id exist
 	if r.IDExist(tenant, id) {
 		for _, k := range tags {
-			r.deleteTag(tenant, id, k)
+			if err := r.deleteTag(tenant, id, k); err != nil {
+				return err
+			}
 		}
 		return nil
 	}
 
-	return errors.New("ID not found")
+	return errors.New("sqlite: ID not found")
 }
 
 // Helper functions
@@ -309,7 +317,6 @@ func (r *Storage) getTenant(name string) (*sql.DB, error) {
 
 	db, err := sql.Open("sqlite3", filename)
 	if err != nil {
-		log.Printf("%q\n", err)
 		return nil, err
 	}
 	db.SetMaxOpenConns(1)
@@ -326,65 +333,80 @@ func (r *Storage) getTenant(name string) (*sql.DB, error) {
 		`
 
 	_, err = db.Exec(sqlStmt)
-	if err != nil {
-		log.Printf("%q: %s\n", err, sqlStmt)
-		return db, err
+	if err == nil {
+		r.tenant[name] = db
 	}
 
-	r.tenant[name] = db
-	return db, nil
+	return db, err
 }
 
 func (r Storage) IDExist(tenant string, id string) bool {
 	var _id string
-	db, _ := r.getTenant(tenant)
+	db, err := r.getTenant(tenant)
+	if err != nil {
+		return false
+	}
 
 	sqlStmt := fmt.Sprintf("select id from ids where id='%s'", id)
-	err := db.QueryRow(sqlStmt).Scan(&_id)
+	err = db.QueryRow(sqlStmt).Scan(&_id)
 	return err != sql.ErrNoRows
 }
 
 func (r Storage) insertData(tenant string, id string, t int64, v float64) error {
-	db, _ := r.getTenant(tenant)
+	db, err := r.getTenant(tenant)
+	if err != nil {
+		return err
+	}
 
 	sqlStmt := fmt.Sprintf("insert into '%s' values (%d, %f)", id, t, v)
-	_, err := db.Exec(sqlStmt)
+	_, err = db.Exec(sqlStmt)
 
 	return err
 }
 
 func (r Storage) insertTag(tenant string, id string, k string, v string) error {
-	db, _ := r.getTenant(tenant)
-
+	db, err := r.getTenant(tenant)
+	if err != nil {
+		return err
+	}
 	sqlStmt := fmt.Sprintf("insert or replace into tags values ('%s', '%s', '%s')", id, k, v)
-	_, err := db.Exec(sqlStmt)
+	_, err = db.Exec(sqlStmt)
 
 	return err
 }
 
 func (r Storage) deleteData(tenant string, id string, end int64, start int64) error {
-	db, _ := r.getTenant(tenant)
+	db, err := r.getTenant(tenant)
+	if err != nil {
+		return err
+	}
 
 	sqlStmt := fmt.Sprintf("delete from '%s' where timestamp >= %d and timestamp < %d", id, start, end)
-	_, err := db.Exec(sqlStmt)
+	_, err = db.Exec(sqlStmt)
 
 	return err
 }
 
 func (r Storage) deleteTag(tenant string, id string, k string) error {
-	db, _ := r.getTenant(tenant)
+	db, err := r.getTenant(tenant)
+	if err != nil {
+		return err
+	}
 
 	sqlStmt := fmt.Sprintf("delete from tags where id='%s' and tag='%s'", id, k)
-	_, err := db.Exec(sqlStmt)
+	_, err = db.Exec(sqlStmt)
 
 	return err
 }
 
 func (r Storage) createID(tenant string, id string) error {
-	db, _ := r.getTenant(tenant)
+	db, err := r.getTenant(tenant)
+	if err != nil {
+		return err
+	}
 
 	sqlStmt := fmt.Sprintf("insert into ids values ('%s')", id)
-	_, err := db.Exec(sqlStmt)
+	_, err = db.Exec(sqlStmt)
 	if err != nil {
 		return err
 	}