feat: added models

crawlab-team · Jun 7, 2024 · 886c1ad · 886c1ad
1 parent 0f6cd4e
commit 886c1ad
Show file tree

Hide file tree

Showing 28 changed files with 667 additions and 97 deletions.
diff --git a/controllers/color.go b/controllers/color.go
diff --git a/controllers/i18n.go b/controllers/i18n.go
diff --git a/controllers/init.go b/controllers/init.go
@@ -19,7 +19,6 @@ func InitControllers() (err error) {
 	TagController = NewListControllerDelegate(ControllerIdTag, modelSvc.GetBaseService(interfaces.ModelIdTag))
 	SettingController = newSettingController()
 	LoginController = NewActionControllerDelegate(ControllerIdLogin, getLoginActions())
-	ColorController = NewActionControllerDelegate(ControllerIdColor, getColorActions())
 	DataCollectionController = newDataCollectionController()
 	ResultController = NewActionControllerDelegate(ControllerIdResult, getResultActions())
 	ScheduleController = newScheduleController()
@@ -28,7 +27,6 @@ func InitControllers() (err error) {
 	FilerController = NewActionControllerDelegate(ControllerIdFiler, getFilerActions())
 	GitController = NewListControllerDelegate(ControllerIdGit, modelSvc.GetBaseService(interfaces.ModelIdGit))
 	VersionController = NewActionControllerDelegate(ControllerIdVersion, getVersionActions())
-	I18nController = NewActionControllerDelegate(ControllerIdI18n, getI18nActions())
 	SystemInfoController = NewActionControllerDelegate(ControllerIdSystemInfo, getSystemInfoActions())
 	DemoController = NewActionControllerDelegate(ControllerIdDemo, getDemoActions())
 	RoleController = NewListControllerDelegate(ControllerIdRole, modelSvc.GetBaseService(interfaces.ModelIdRole))

diff --git a/controllers/project_v2.go b/controllers/project_v2.go
diff --git a/controllers/router_v2.go b/controllers/router_v2.go
@@ -2,7 +2,9 @@ package controllers
 
 import (
 	"github.com/crawlab-team/crawlab-core/middlewares"
+	"github.com/crawlab-team/crawlab-core/models/models"
 	"github.com/gin-gonic/gin"
+	"net/http"
 )
 
 type RouterGroups struct {
@@ -30,6 +32,43 @@ func InitRoutes(app *gin.Engine) {
 	// routes groups
 	groups := NewRouterGroups(app)
 
-	RegisterController(groups.AuthGroup, "/projects", ProjectV2Controller)
-	RegisterController(groups.AuthGroup, "/users", UserControllerV2)
+	RegisterController(groups.AuthGroup, "/data/collections", NewControllerV2[models.DataCollectionV2]())
+	RegisterController(groups.AuthGroup, "/data-sources", NewControllerV2[models.DataSourceV2]())
+	RegisterController(groups.AuthGroup, "/environments", NewControllerV2[models.EnvironmentV2]())
+	RegisterController(groups.AuthGroup, "/gits", NewControllerV2[models.GitV2]())
+	RegisterController(groups.AuthGroup, "/nodes", NewControllerV2[models.NodeV2]())
+	RegisterController(groups.AuthGroup, "/notifications/settings", NewControllerV2[models.SettingV2]())
+	RegisterController(groups.AuthGroup, "/permissions", NewControllerV2[models.PermissionV2]())
+	RegisterController(groups.AuthGroup, "/projects", NewControllerV2[models.ProjectV2]())
+	RegisterController(groups.AuthGroup, "/roles", NewControllerV2[models.RoleV2]())
+	RegisterController(groups.AuthGroup, "/schedules", NewControllerV2[models.ScheduleV2](
+	// TODO: implement actions
+	))
+	RegisterController(groups.AuthGroup, "/settings", NewControllerV2[models.SettingV2]())
+	RegisterController(groups.AuthGroup, "/spiders", NewControllerV2[models.SpiderV2](
+	// TODO: implement actions
+	))
+	RegisterController(groups.AuthGroup, "/tasks", NewControllerV2[models.TaskV2](
+	// TODO: implement actions
+	))
+	RegisterController(groups.AuthGroup, "/tokens", NewControllerV2[models.TokenV2](
+	// TODO: implement actions
+	))
+	RegisterController(groups.AuthGroup, "/users", NewControllerV2[models.UserV2](
+		Action{
+			Method:      http.MethodPost,
+			Path:        "/:id/change-password",
+			HandlerFunc: PostUserChangePassword,
+		},
+		Action{
+			Method:      http.MethodGet,
+			Path:        "/me",
+			HandlerFunc: GetUserMe,
+		},
+		Action{
+			Method:      http.MethodPut,
+			Path:        "/me",
+			HandlerFunc: PutUserById,
+		},
+	))
 }
diff --git a/controllers/schedule_v2.go b/controllers/schedule_v2.go
@@ -0,0 +1 @@
+package controllers
diff --git a/controllers/user_v2.go b/controllers/user_v2.go
@@ -8,25 +8,6 @@ import (
 	"github.com/crawlab-team/crawlab-core/utils"
 	"github.com/gin-gonic/gin"
 	"go.mongodb.org/mongo-driver/bson/primitive"
-	"net/http"
-)
-
-var UserControllerV2 = NewControllerV2[models.UserV2](
-	Action{
-		Method:      http.MethodPost,
-		Path:        "/:id/change-password",
-		HandlerFunc: PostUserChangePassword,
-	},
-	Action{
-		Method:      http.MethodGet,
-		Path:        "/me",
-		HandlerFunc: GetUserMe,
-	},
-	Action{
-		Method:      http.MethodPut,
-		Path:        "/me",
-		HandlerFunc: PutUserById,
-	},
 )
 
 func PostUserChangePassword(c *gin.Context) {

diff --git a/grpc/server/model_base_service_v2_server.go b/grpc/server/model_base_service_v2_server.go
@@ -13,18 +13,29 @@ import (
 )
 
 var (
-	typeNameColNameMap   = make(map[string]string)
-	typeOneNameModelMap  = make(map[string]any)
-	typeManyNameModelMap = make(map[string]any)
-	typeOneInstances     = []any{
-		models.TestModel{},
-		models.NodeV2{},
-		models.UserV2{},
-	}
-	typeManyInstances = []any{
-		[]models.NodeV2{},
-		[]models.TestModel{},
-		[]models.UserV2{},
+	typeNameColNameMap  = make(map[string]string)
+	typeOneNameModelMap = make(map[string]any)
+	typeOneInstances    = []any{
+		*new(models.TestModel),
+		*new(models.DataCollectionV2),
+		*new(models.DataSourceV2),
+		*new(models.DependencySettingV2),
+		*new(models.EnvironmentV2),
+		*new(models.GitV2),
+		*new(models.NodeV2),
+		*new(models.PermissionV2),
+		*new(models.ProjectV2),
+		*new(models.RolePermissionV2),
+		*new(models.RoleV2),
+		*new(models.ScheduleV2),
+		*new(models.SettingV2),
+		*new(models.SpiderV2),
+		*new(models.TaskQueueItemV2),
+		*new(models.TaskStatV2),
+		*new(models.TaskV2),
+		*new(models.TokenV2),
+		*new(models.UserRoleV2),
+		*new(models.UserV2),
 	}
 )
 
@@ -36,21 +47,12 @@ func init() {
 		typeNameColNameMap[typeName] = colName
 		typeOneNameModelMap[typeName] = v
 	}
-	for _, v := range typeManyInstances {
-		t := reflect.TypeOf(v)
-		typeName := t.Elem().Name()
-		typeManyNameModelMap[typeName] = v
-	}
 }
 
 func GetOneInstanceModel(typeName string) any {
 	return typeOneNameModelMap[typeName]
 }
 
-func GetManyInstanceModel(typeName string) any {
-	return typeManyNameModelMap[typeName]
-}
-
 type ModelBaseServiceV2Server struct {
 	grpc.UnimplementedModelBaseServiceV2Server
 }

diff --git a/models/models/data_collection_v2.go b/models/models/data_collection_v2.go
@@ -0,0 +1,18 @@
+package models
+
+import (
+	"github.com/crawlab-team/crawlab-core/entity"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type DataCollectionV2 struct {
+	Id                          primitive.ObjectID `json:"_id" bson:"_id" collection:"data_collections"`
+	BaseModelV2[DataCollection] `bson:",inline"`
+	Name                        string             `json:"name" bson:"name"`
+	Fields                      []entity.DataField `json:"fields" bson:"fields"`
+	Dedup                       struct {
+		Enabled bool     `json:"enabled" bson:"enabled"`
+		Keys    []string `json:"keys" bson:"keys"`
+		Type    string   `json:"type" bson:"type"`
+	} `json:"dedup" bson:"dedup"`
+}
diff --git a/models/models/data_source_v2.go b/models/models/data_source_v2.go
@@ -0,0 +1,24 @@
+package models
+
+import (
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type DataSourceV2 struct {
+	Id                      primitive.ObjectID `json:"_id" bson:"_id" collection:"data_sources"`
+	BaseModelV2[DataSource] `bson:",inline"`
+	Name                    string            `json:"name" bson:"name"`
+	Type                    string            `json:"type" bson:"type"`
+	Description             string            `json:"description" bson:"description"`
+	Host                    string            `json:"host" bson:"host"`
+	Port                    string            `json:"port" bson:"port"`
+	Url                     string            `json:"url" bson:"url"`
+	Hosts                   []string          `json:"hosts" bson:"hosts"`
+	Database                string            `json:"database" bson:"database"`
+	Username                string            `json:"username" bson:"username"`
+	Password                string            `json:"password,omitempty" bson:"-"`
+	ConnectType             string            `json:"connect_type" bson:"connect_type"`
+	Status                  string            `json:"status" bson:"status"`
+	Error                   string            `json:"error" bson:"error"`
+	Extra                   map[string]string `json:"extra,omitempty" bson:"extra,omitempty"`
+}
diff --git a/models/models/dependency_setting_v2.go b/models/models/dependency_setting_v2.go
@@ -0,0 +1,18 @@
+package models
+
+import (
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"time"
+)
+
+type DependencySettingV2 struct {
+	Id                             primitive.ObjectID `json:"_id" bson:"_id" collection:"dependency_settings"`
+	BaseModelV2[DependencySetting] `bson:",inline"`
+	Key                            string    `json:"key" bson:"key"`
+	Name                           string    `json:"name" bson:"name"`
+	Description                    string    `json:"description" bson:"description"`
+	Enabled                        bool      `json:"enabled" bson:"enabled"`
+	Cmd                            string    `json:"cmd" bson:"cmd"`
+	Proxy                          string    `json:"proxy" bson:"proxy"`
+	LastUpdateTs                   time.Time `json:"last_update_ts" bson:"last_update_ts"`
+}
diff --git a/models/models/environment_v2.go b/models/models/environment_v2.go
@@ -0,0 +1,12 @@
+package models
+
+import (
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type EnvironmentV2 struct {
+	Id                         primitive.ObjectID `json:"_id" bson:"_id" collection:"environments"`
+	BaseModelV2[EnvironmentV2] `bson:",inline"`
+	Key                        string `json:"key" bson:"key"`
+	Value                      string `json:"value" bson:"value"`
+}
diff --git a/models/models/git_v2.go b/models/models/git_v2.go
@@ -0,0 +1,16 @@
+package models
+
+import (
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type GitV2 struct {
+	Id                 primitive.ObjectID `json:"_id" bson:"_id" collection:"gits"`
+	BaseModelV2[GitV2] `bson:",inline"`
+	Url                string `json:"url" bson:"url"`
+	AuthType           string `json:"auth_type" bson:"auth_type"`
+	Username           string `json:"username" bson:"username"`
+	Password           string `json:"password" bson:"password"`
+	CurrentBranch      string `json:"current_branch" bson:"current_branch"`
+	AutoPull           bool   `json:"auto_pull" bson:"auto_pull"`
+}
diff --git a/models/models/permission_v2.go b/models/models/permission_v2.go
@@ -0,0 +1,17 @@
+package models
+
+import (
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type PermissionV2 struct {
+	Id                        primitive.ObjectID `json:"_id" bson:"_id" collection:"permissions"`
+	BaseModelV2[PermissionV2] `bson:",inline"`
+	Key                       string   `json:"key" bson:"key"`
+	Name                      string   `json:"name" bson:"name"`
+	Description               string   `json:"description" bson:"description"`
+	Type                      string   `json:"type" bson:"type"`
+	Target                    []string `json:"target" bson:"target"`
+	Allow                     []string `json:"allow" bson:"allow"`
+	Deny                      []string `json:"deny" bson:"deny"`
+}
diff --git a/models/models/role_permission_v2.go b/models/models/role_permission_v2.go
@@ -0,0 +1,12 @@
+package models
+
+import (
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type RolePermissionV2 struct {
+	Id                            primitive.ObjectID `json:"_id" bson:"_id" collection:"role_permissions"`
+	BaseModelV2[RolePermissionV2] `bson:",inline"`
+	RoleId                        primitive.ObjectID `json:"role_id" bson:"role_id"`
+	PermissionId                  primitive.ObjectID `json:"permission_id" bson:"permission_id"`
+}
diff --git a/models/models/role_v2.go b/models/models/role_v2.go
@@ -0,0 +1,13 @@
+package models
+
+import (
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type RoleV2 struct {
+	Id                  primitive.ObjectID `json:"_id" bson:"_id" collection:"roles"`
+	BaseModelV2[RoleV2] `bson:",inline"`
+	Key                 string `json:"key" bson:"key"`
+	Name                string `json:"name" bson:"name"`
+	Description         string `json:"description" bson:"description"`
+}
diff --git a/models/models/schedule_v2.go b/models/models/schedule_v2.go
@@ -0,0 +1,23 @@
+package models
+
+import (
+	"github.com/robfig/cron/v3"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type ScheduleV2 struct {
+	Id                      primitive.ObjectID `json:"_id" bson:"_id" collection:"schedules"`
+	BaseModelV2[ScheduleV2] `bson:",inline"`
+	Name                    string               `json:"name" bson:"name"`
+	Description             string               `json:"description" bson:"description"`
+	SpiderId                primitive.ObjectID   `json:"spider_id" bson:"spider_id"`
+	Cron                    string               `json:"cron" bson:"cron"`
+	EntryId                 cron.EntryID         `json:"entry_id" bson:"entry_id"`
+	Cmd                     string               `json:"cmd" bson:"cmd"`
+	Param                   string               `json:"param" bson:"param"`
+	Mode                    string               `json:"mode" bson:"mode"`
+	NodeIds                 []primitive.ObjectID `json:"node_ids" bson:"node_ids"`
+	Priority                int                  `json:"priority" bson:"priority"`
+	Enabled                 bool                 `json:"enabled" bson:"enabled"`
+	UserId                  primitive.ObjectID   `json:"user_id" bson:"user_id"`
+}
diff --git a/models/models/setting_v2.go b/models/models/setting_v2.go
@@ -0,0 +1,13 @@
+package models
+
+import (
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type SettingV2 struct {
+	Id                     primitive.ObjectID `json:"_id" bson:"_id" collection:"settings"`
+	BaseModelV2[SettingV2] `bson:",inline"`
+	Key                    string `json:"key" bson:"key"`
+	Value                  bson.M `json:"value" bson:"value"`
+}