Skip to content

Commit

Permalink
feat: updated test cases
Browse files Browse the repository at this point in the history
  • Loading branch information
@tikazyq
tikazyq committed Jun 13, 2024
1 parent 754481c commit 43e4180
Show file tree
Hide file tree
Showing 3 changed files with 118 additions and 35 deletions.
56 changes: 49 additions & 7 deletions controllers/spider_v2.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package controllers
import (
"errors"
"fmt"
log2 "github.com/apex/log"
"github.com/crawlab-team/crawlab-core/constants"
"github.com/crawlab-team/crawlab-core/entity"
"github.com/crawlab-team/crawlab-core/fs"
Expand All @@ -25,6 +26,7 @@ import (
"os"
"path/filepath"
"strings"
"sync"
)

func GetSpiderById(c *gin.Context) {
Expand All @@ -46,8 +48,10 @@ func GetSpiderById(c *gin.Context) {
// stat
s.Stat, err = service.NewModelServiceV2[models.SpiderStatV2]().GetById(s.Id)
if err != nil {
HandleErrorInternalServerError(c, err)
return
if !errors.Is(err, mongo2.ErrNoDocuments) {
HandleErrorInternalServerError(c, err)
return
}
}

// data collection
Expand Down Expand Up @@ -122,7 +126,7 @@ func GetSpiderList(c *gin.Context) {

// cache stat list to dict
dict := map[primitive.ObjectID]models.SpiderStatV2{}
var tids []primitive.ObjectID
var taskIds []primitive.ObjectID
for _, st := range stats {
if st.Tasks > 0 {
taskCount := int64(st.Tasks)
Expand All @@ -133,19 +137,19 @@ func GetSpiderList(c *gin.Context) {
dict[st.Id] = st

if !st.LastTaskId.IsZero() {
tids = append(tids, st.LastTaskId)
taskIds = append(taskIds, st.LastTaskId)
}
}

// task list and stats
var tasks []models.TaskV2
dictTask := map[primitive.ObjectID]models.TaskV2{}
dictTaskStat := map[primitive.ObjectID]models.TaskStatV2{}
if len(tids) > 0 {
if len(taskIds) > 0 {
// task list
queryTask := bson.M{
"_id": bson.M{
"$in": tids,
"$in": taskIds,
},
}
tasks, err = service.NewModelServiceV2[models.TaskV2]().GetMany(queryTask, nil)
Expand Down Expand Up @@ -177,7 +181,7 @@ func GetSpiderList(c *gin.Context) {
}

// iterate list again
var data []interface{}
var data []models.SpiderV2
for _, s := range spiders {
// spider stat
st, ok := dict[s.Id]
Expand Down Expand Up @@ -307,6 +311,10 @@ func DeleteSpiderById(c *gin.Context) {
return err
}

if len(tasks) == 0 {
return nil
}

// task ids
var taskIds []primitive.ObjectID
for _, t := range tasks {
Expand All @@ -325,6 +333,21 @@ func DeleteSpiderById(c *gin.Context) {
return err
}

// delete tasks logs
wg := sync.WaitGroup{}
wg.Add(len(taskIds))
for _, id := range taskIds {
go func(id string) {
// delete task logs
logPath := filepath.Join(viper.GetString("log.path"), id)
if err := os.RemoveAll(logPath); err != nil {
log2.Warnf("failed to remove task log directory: %s", logPath)
}
wg.Done()
}(id.Hex())
}
wg.Wait()

return nil
}); err != nil {
HandleErrorInternalServerError(c, err)
Expand Down Expand Up @@ -368,6 +391,10 @@ func DeleteSpiderList(c *gin.Context) {
return err
}

if len(tasks) == 0 {
return nil
}

// task ids
var taskIds []primitive.ObjectID
for _, t := range tasks {
Expand All @@ -384,6 +411,21 @@ func DeleteSpiderList(c *gin.Context) {
return err
}

// delete tasks logs
wg := sync.WaitGroup{}
wg.Add(len(taskIds))
for _, id := range taskIds {
go func(id string) {
// delete task logs
logPath := filepath.Join(viper.GetString("log.path"), id)
if err := os.RemoveAll(logPath); err != nil {
log2.Warnf("failed to remove task log directory: %s", logPath)
}
wg.Done()
}(id.Hex())
}
wg.Wait()

return nil
}); err != nil {
HandleErrorInternalServerError(c, err)
Expand Down
96 changes: 68 additions & 28 deletions controllers/spider_v2_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ import (
"net/http"
"net/http/httptest"
"testing"
"time"
)

func TestCreateSpider(t *testing.T) {
Expand Down Expand Up @@ -58,16 +57,16 @@ func TestGetSpiderById(t *testing.T) {
router.Use(middlewares.AuthorizationMiddlewareV2())
router.GET("/spiders/:id", controllers.GetSpiderById)

id := primitive.NewObjectID()
model := models.SpiderV2{
Name: "Test Spider",
ColName: "test_spiders",
}
model.SetId(id)
jsonValue, _ := json.Marshal(model)
_, err := http.NewRequest("POST", "/spiders", bytes.NewBuffer(jsonValue))
id, err := service.NewModelServiceV2[models.SpiderV2]().InsertOne(model)
require.Nil(t, err)
ts := models.SpiderStatV2{}
ts.SetId(id)
_, err = service.NewModelServiceV2[models.SpiderStatV2]().InsertOne(ts)
require.Nil(t, err)
time.Sleep(100 * time.Millisecond)

req, _ := http.NewRequest("GET", "/spiders/"+id.Hex(), nil)
req.Header.Set("Authorization", TestToken)
Expand All @@ -93,21 +92,24 @@ func TestUpdateSpiderById(t *testing.T) {
router.Use(middlewares.AuthorizationMiddlewareV2())
router.PUT("/spiders/:id", controllers.PutSpiderById)

id := primitive.NewObjectID()
model := models.SpiderV2{
Name: "Test Spider",
ColName: "test_spiders",
}
model.SetId(id)
jsonValue, _ := json.Marshal(model)
_, err := http.NewRequest("POST", "/spiders", bytes.NewBuffer(jsonValue))
id, err := service.NewModelServiceV2[models.SpiderV2]().InsertOne(model)
require.Nil(t, err)
ts := models.SpiderStatV2{}
ts.SetId(id)
_, err = service.NewModelServiceV2[models.SpiderStatV2]().InsertOne(ts)
require.Nil(t, err)

spiderId := id.Hex()
payload := models.SpiderV2{
Name: "Updated Spider",
Name: "Updated Spider",
ColName: "test_spider",
}
jsonValue, _ = json.Marshal(payload)
payload.SetId(id)
jsonValue, _ := json.Marshal(payload)
req, _ := http.NewRequest("PUT", "/spiders/"+spiderId, bytes.NewBuffer(jsonValue))
req.Header.Set("Authorization", TestToken)
resp := httptest.NewRecorder()
Expand Down Expand Up @@ -137,14 +139,23 @@ func TestDeleteSpiderById(t *testing.T) {
router.Use(middlewares.AuthorizationMiddlewareV2())
router.DELETE("/spiders/:id", controllers.DeleteSpiderById)

id := primitive.NewObjectID()
model := models.SpiderV2{
Name: "Test Spider",
ColName: "test_spiders",
}
model.SetId(id)
jsonValue, _ := json.Marshal(model)
_, err := http.NewRequest("POST", "/spiders", bytes.NewBuffer(jsonValue))
id, err := service.NewModelServiceV2[models.SpiderV2]().InsertOne(model)
require.Nil(t, err)
ts := models.SpiderStatV2{}
ts.SetId(id)
_, err = service.NewModelServiceV2[models.SpiderStatV2]().InsertOne(ts)
require.Nil(t, err)
task := models.TaskV2{}
task.SpiderId = id
taskId, err := service.NewModelServiceV2[models.TaskV2]().InsertOne(task)
require.Nil(t, err)
taskStat := models.TaskStatV2{}
taskStat.SetId(taskId)
_, err = service.NewModelServiceV2[models.TaskStatV2]().InsertOne(taskStat)
require.Nil(t, err)

req, _ := http.NewRequest("DELETE", "/spiders/"+id.Hex(), nil)
Expand All @@ -155,9 +166,17 @@ func TestDeleteSpiderById(t *testing.T) {

assert.Equal(t, http.StatusOK, resp.Code)

svc := service.NewModelServiceV2[models.SpiderV2]()
_, err = svc.GetById(id)
require.NotNil(t, err)
_, err = service.NewModelServiceV2[models.SpiderV2]().GetById(id)
assert.NotNil(t, err)
_, err = service.NewModelServiceV2[models.SpiderStatV2]().GetById(id)
assert.NotNil(t, err)
taskCount, err := service.NewModelServiceV2[models.TaskV2]().Count(bson.M{"spider_id": id})
require.Nil(t, err)
assert.Equal(t, 0, taskCount)
taskStatCount, err := service.NewModelServiceV2[models.TaskStatV2]().Count(bson.M{"_id": taskId})
require.Nil(t, err)
assert.Equal(t, 0, taskStatCount)

}

func TestDeleteSpiderList(t *testing.T) {
Expand All @@ -170,7 +189,6 @@ func TestDeleteSpiderList(t *testing.T) {
router.Use(middlewares.AuthorizationMiddlewareV2())
router.DELETE("/spiders", controllers.DeleteSpiderList)

svc := service.NewModelServiceV2[models.SpiderV2]()
modelList := []models.SpiderV2{
{
Name: "Test Name 1",
Expand All @@ -181,18 +199,31 @@ func TestDeleteSpiderList(t *testing.T) {
},
}
var ids []primitive.ObjectID
var taskIds []primitive.ObjectID
for _, model := range modelList {
id := primitive.NewObjectID()
model.SetId(id)
jsonValue, _ := json.Marshal(model)
_, err := http.NewRequest("POST", "/spiders", bytes.NewBuffer(jsonValue))
id, err := service.NewModelServiceV2[models.SpiderV2]().InsertOne(model)
require.Nil(t, err)
ts := models.SpiderStatV2{}
ts.SetId(id)
_, err = service.NewModelServiceV2[models.SpiderStatV2]().InsertOne(ts)
require.Nil(t, err)
task := models.TaskV2{}
task.SpiderId = id
taskId, err := service.NewModelServiceV2[models.TaskV2]().InsertOne(task)
require.Nil(t, err)
taskStat := models.TaskStatV2{}
taskStat.SetId(taskId)
_, err = service.NewModelServiceV2[models.TaskStatV2]().InsertOne(taskStat)
require.Nil(t, err)
ids = append(ids, id)
taskIds = append(taskIds, taskId)
}

payload := struct {
Ids []string `json:"ids"`
}{}
Ids []primitive.ObjectID `json:"ids"`
}{
Ids: ids,
}
jsonValue, _ := json.Marshal(payload)
req, _ := http.NewRequest("DELETE", "/spiders", bytes.NewBuffer(jsonValue))
req.Header.Set("Authorization", TestToken)
Expand All @@ -202,7 +233,16 @@ func TestDeleteSpiderList(t *testing.T) {

assert.Equal(t, http.StatusOK, resp.Code)

total, err := svc.Count(bson.M{"_id": bson.M{"$in": ids}})
spiderCount, err := service.NewModelServiceV2[models.SpiderV2]().Count(bson.M{"_id": bson.M{"$in": ids}})
require.Nil(t, err)
assert.Equal(t, 0, spiderCount)
spiderStatCount, err := service.NewModelServiceV2[models.SpiderStatV2]().Count(bson.M{"_id": bson.M{"$in": ids}})
require.Nil(t, err)
assert.Equal(t, 0, spiderStatCount)
taskCount, err := service.NewModelServiceV2[models.TaskV2]().Count(bson.M{"_id": bson.M{"$in": taskIds}})
require.Nil(t, err)
assert.Equal(t, 0, taskCount)
taskStatCount, err := service.NewModelServiceV2[models.TaskStatV2]().Count(bson.M{"_id": bson.M{"$in": taskIds}})
require.Nil(t, err)
require.Equal(t, 0, total)
assert.Equal(t, 0, taskStatCount)
}
1 change: 1 addition & 0 deletions controllers/task_v2.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -216,6 +216,7 @@ func DeleteList(c *gin.Context) {
return
}

// delete tasks logs
wg := sync.WaitGroup{}
wg.Add(len(payload.Ids))
for _, id := range payload.Ids {
Expand Down

0 comments on commit 43e4180

Please sign in to comment.