From 43e4180f1453c94859a77e2d5f8c5d4d2b737691 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Thu, 13 Jun 2024 21:12:50 +0800 Subject: [PATCH] feat: updated test cases --- controllers/spider_v2.go | 56 +++++++++++++++++--- controllers/spider_v2_test.go | 96 +++++++++++++++++++++++++---------- controllers/task_v2.go | 1 + 3 files changed, 118 insertions(+), 35 deletions(-) diff --git a/controllers/spider_v2.go b/controllers/spider_v2.go index f4bfbbe..391a938 100644 --- a/controllers/spider_v2.go +++ b/controllers/spider_v2.go @@ -3,6 +3,7 @@ package controllers import ( "errors" "fmt" + log2 "github.com/apex/log" "github.com/crawlab-team/crawlab-core/constants" "github.com/crawlab-team/crawlab-core/entity" "github.com/crawlab-team/crawlab-core/fs" @@ -25,6 +26,7 @@ import ( "os" "path/filepath" "strings" + "sync" ) func GetSpiderById(c *gin.Context) { @@ -46,8 +48,10 @@ func GetSpiderById(c *gin.Context) { // stat s.Stat, err = service.NewModelServiceV2[models.SpiderStatV2]().GetById(s.Id) if err != nil { - HandleErrorInternalServerError(c, err) - return + if !errors.Is(err, mongo2.ErrNoDocuments) { + HandleErrorInternalServerError(c, err) + return + } } // data collection @@ -122,7 +126,7 @@ func GetSpiderList(c *gin.Context) { // cache stat list to dict dict := map[primitive.ObjectID]models.SpiderStatV2{} - var tids []primitive.ObjectID + var taskIds []primitive.ObjectID for _, st := range stats { if st.Tasks > 0 { taskCount := int64(st.Tasks) @@ -133,7 +137,7 @@ func GetSpiderList(c *gin.Context) { dict[st.Id] = st if !st.LastTaskId.IsZero() { - tids = append(tids, st.LastTaskId) + taskIds = append(taskIds, st.LastTaskId) } } @@ -141,11 +145,11 @@ func GetSpiderList(c *gin.Context) { var tasks []models.TaskV2 dictTask := map[primitive.ObjectID]models.TaskV2{} dictTaskStat := map[primitive.ObjectID]models.TaskStatV2{} - if len(tids) > 0 { + if len(taskIds) > 0 { // task list queryTask := bson.M{ "_id": bson.M{ - "$in": tids, + "$in": taskIds, }, } tasks, err = service.NewModelServiceV2[models.TaskV2]().GetMany(queryTask, nil) @@ -177,7 +181,7 @@ func GetSpiderList(c *gin.Context) { } // iterate list again - var data []interface{} + var data []models.SpiderV2 for _, s := range spiders { // spider stat st, ok := dict[s.Id] @@ -307,6 +311,10 @@ func DeleteSpiderById(c *gin.Context) { return err } + if len(tasks) == 0 { + return nil + } + // task ids var taskIds []primitive.ObjectID for _, t := range tasks { @@ -325,6 +333,21 @@ func DeleteSpiderById(c *gin.Context) { return err } + // delete tasks logs + wg := sync.WaitGroup{} + wg.Add(len(taskIds)) + for _, id := range taskIds { + go func(id string) { + // delete task logs + logPath := filepath.Join(viper.GetString("log.path"), id) + if err := os.RemoveAll(logPath); err != nil { + log2.Warnf("failed to remove task log directory: %s", logPath) + } + wg.Done() + }(id.Hex()) + } + wg.Wait() + return nil }); err != nil { HandleErrorInternalServerError(c, err) @@ -368,6 +391,10 @@ func DeleteSpiderList(c *gin.Context) { return err } + if len(tasks) == 0 { + return nil + } + // task ids var taskIds []primitive.ObjectID for _, t := range tasks { @@ -384,6 +411,21 @@ func DeleteSpiderList(c *gin.Context) { return err } + // delete tasks logs + wg := sync.WaitGroup{} + wg.Add(len(taskIds)) + for _, id := range taskIds { + go func(id string) { + // delete task logs + logPath := filepath.Join(viper.GetString("log.path"), id) + if err := os.RemoveAll(logPath); err != nil { + log2.Warnf("failed to remove task log directory: %s", logPath) + } + wg.Done() + }(id.Hex()) + } + wg.Wait() + return nil }); err != nil { HandleErrorInternalServerError(c, err) diff --git a/controllers/spider_v2_test.go b/controllers/spider_v2_test.go index 04b6089..e15e188 100644 --- a/controllers/spider_v2_test.go +++ b/controllers/spider_v2_test.go @@ -15,7 +15,6 @@ import ( "net/http" "net/http/httptest" "testing" - "time" ) func TestCreateSpider(t *testing.T) { @@ -58,16 +57,16 @@ func TestGetSpiderById(t *testing.T) { router.Use(middlewares.AuthorizationMiddlewareV2()) router.GET("/spiders/:id", controllers.GetSpiderById) - id := primitive.NewObjectID() model := models.SpiderV2{ Name: "Test Spider", ColName: "test_spiders", } - model.SetId(id) - jsonValue, _ := json.Marshal(model) - _, err := http.NewRequest("POST", "/spiders", bytes.NewBuffer(jsonValue)) + id, err := service.NewModelServiceV2[models.SpiderV2]().InsertOne(model) + require.Nil(t, err) + ts := models.SpiderStatV2{} + ts.SetId(id) + _, err = service.NewModelServiceV2[models.SpiderStatV2]().InsertOne(ts) require.Nil(t, err) - time.Sleep(100 * time.Millisecond) req, _ := http.NewRequest("GET", "/spiders/"+id.Hex(), nil) req.Header.Set("Authorization", TestToken) @@ -93,21 +92,24 @@ func TestUpdateSpiderById(t *testing.T) { router.Use(middlewares.AuthorizationMiddlewareV2()) router.PUT("/spiders/:id", controllers.PutSpiderById) - id := primitive.NewObjectID() model := models.SpiderV2{ Name: "Test Spider", ColName: "test_spiders", } - model.SetId(id) - jsonValue, _ := json.Marshal(model) - _, err := http.NewRequest("POST", "/spiders", bytes.NewBuffer(jsonValue)) + id, err := service.NewModelServiceV2[models.SpiderV2]().InsertOne(model) + require.Nil(t, err) + ts := models.SpiderStatV2{} + ts.SetId(id) + _, err = service.NewModelServiceV2[models.SpiderStatV2]().InsertOne(ts) require.Nil(t, err) spiderId := id.Hex() payload := models.SpiderV2{ - Name: "Updated Spider", + Name: "Updated Spider", + ColName: "test_spider", } - jsonValue, _ = json.Marshal(payload) + payload.SetId(id) + jsonValue, _ := json.Marshal(payload) req, _ := http.NewRequest("PUT", "/spiders/"+spiderId, bytes.NewBuffer(jsonValue)) req.Header.Set("Authorization", TestToken) resp := httptest.NewRecorder() @@ -137,14 +139,23 @@ func TestDeleteSpiderById(t *testing.T) { router.Use(middlewares.AuthorizationMiddlewareV2()) router.DELETE("/spiders/:id", controllers.DeleteSpiderById) - id := primitive.NewObjectID() model := models.SpiderV2{ Name: "Test Spider", ColName: "test_spiders", } - model.SetId(id) - jsonValue, _ := json.Marshal(model) - _, err := http.NewRequest("POST", "/spiders", bytes.NewBuffer(jsonValue)) + id, err := service.NewModelServiceV2[models.SpiderV2]().InsertOne(model) + require.Nil(t, err) + ts := models.SpiderStatV2{} + ts.SetId(id) + _, err = service.NewModelServiceV2[models.SpiderStatV2]().InsertOne(ts) + require.Nil(t, err) + task := models.TaskV2{} + task.SpiderId = id + taskId, err := service.NewModelServiceV2[models.TaskV2]().InsertOne(task) + require.Nil(t, err) + taskStat := models.TaskStatV2{} + taskStat.SetId(taskId) + _, err = service.NewModelServiceV2[models.TaskStatV2]().InsertOne(taskStat) require.Nil(t, err) req, _ := http.NewRequest("DELETE", "/spiders/"+id.Hex(), nil) @@ -155,9 +166,17 @@ func TestDeleteSpiderById(t *testing.T) { assert.Equal(t, http.StatusOK, resp.Code) - svc := service.NewModelServiceV2[models.SpiderV2]() - _, err = svc.GetById(id) - require.NotNil(t, err) + _, err = service.NewModelServiceV2[models.SpiderV2]().GetById(id) + assert.NotNil(t, err) + _, err = service.NewModelServiceV2[models.SpiderStatV2]().GetById(id) + assert.NotNil(t, err) + taskCount, err := service.NewModelServiceV2[models.TaskV2]().Count(bson.M{"spider_id": id}) + require.Nil(t, err) + assert.Equal(t, 0, taskCount) + taskStatCount, err := service.NewModelServiceV2[models.TaskStatV2]().Count(bson.M{"_id": taskId}) + require.Nil(t, err) + assert.Equal(t, 0, taskStatCount) + } func TestDeleteSpiderList(t *testing.T) { @@ -170,7 +189,6 @@ func TestDeleteSpiderList(t *testing.T) { router.Use(middlewares.AuthorizationMiddlewareV2()) router.DELETE("/spiders", controllers.DeleteSpiderList) - svc := service.NewModelServiceV2[models.SpiderV2]() modelList := []models.SpiderV2{ { Name: "Test Name 1", @@ -181,18 +199,31 @@ func TestDeleteSpiderList(t *testing.T) { }, } var ids []primitive.ObjectID + var taskIds []primitive.ObjectID for _, model := range modelList { - id := primitive.NewObjectID() - model.SetId(id) - jsonValue, _ := json.Marshal(model) - _, err := http.NewRequest("POST", "/spiders", bytes.NewBuffer(jsonValue)) + id, err := service.NewModelServiceV2[models.SpiderV2]().InsertOne(model) + require.Nil(t, err) + ts := models.SpiderStatV2{} + ts.SetId(id) + _, err = service.NewModelServiceV2[models.SpiderStatV2]().InsertOne(ts) + require.Nil(t, err) + task := models.TaskV2{} + task.SpiderId = id + taskId, err := service.NewModelServiceV2[models.TaskV2]().InsertOne(task) + require.Nil(t, err) + taskStat := models.TaskStatV2{} + taskStat.SetId(taskId) + _, err = service.NewModelServiceV2[models.TaskStatV2]().InsertOne(taskStat) require.Nil(t, err) ids = append(ids, id) + taskIds = append(taskIds, taskId) } payload := struct { - Ids []string `json:"ids"` - }{} + Ids []primitive.ObjectID `json:"ids"` + }{ + Ids: ids, + } jsonValue, _ := json.Marshal(payload) req, _ := http.NewRequest("DELETE", "/spiders", bytes.NewBuffer(jsonValue)) req.Header.Set("Authorization", TestToken) @@ -202,7 +233,16 @@ func TestDeleteSpiderList(t *testing.T) { assert.Equal(t, http.StatusOK, resp.Code) - total, err := svc.Count(bson.M{"_id": bson.M{"$in": ids}}) + spiderCount, err := service.NewModelServiceV2[models.SpiderV2]().Count(bson.M{"_id": bson.M{"$in": ids}}) + require.Nil(t, err) + assert.Equal(t, 0, spiderCount) + spiderStatCount, err := service.NewModelServiceV2[models.SpiderStatV2]().Count(bson.M{"_id": bson.M{"$in": ids}}) + require.Nil(t, err) + assert.Equal(t, 0, spiderStatCount) + taskCount, err := service.NewModelServiceV2[models.TaskV2]().Count(bson.M{"_id": bson.M{"$in": taskIds}}) + require.Nil(t, err) + assert.Equal(t, 0, taskCount) + taskStatCount, err := service.NewModelServiceV2[models.TaskStatV2]().Count(bson.M{"_id": bson.M{"$in": taskIds}}) require.Nil(t, err) - require.Equal(t, 0, total) + assert.Equal(t, 0, taskStatCount) } diff --git a/controllers/task_v2.go b/controllers/task_v2.go index b9297dc..04a70e9 100644 --- a/controllers/task_v2.go +++ b/controllers/task_v2.go @@ -216,6 +216,7 @@ func DeleteList(c *gin.Context) { return } + // delete tasks logs wg := sync.WaitGroup{} wg.Add(len(payload.Ids)) for _, id := range payload.Ids {