chore(run): determine view permission by owner, credit owner, view na…

…mespace (#685) - **feat: allow users in organization to have owner permission in run logging** - **chore(run): determine view permission by owner, credit owner, view namespace** Because - run logging view permission updated for user and organization resources by owner, credit owner and view namespace This commit - update view permission business logic --------- Co-authored-by: HR Wu <[email protected]>
instill-ai · Sep 13, 2024 · 579a69c · 579a69c
1 parent e0bd741
commit 579a69c
Show file tree

Hide file tree

Showing 18 changed files with 282 additions and 214 deletions.
diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml
@@ -120,7 +120,8 @@ jobs:
           docker pull instill/dummy-text-to-image:dev
           docker pull instill/dummy-multimodal-chat:dev
           docker pull instill/dummy-text-embedding:dev
-          docker pull instill/dummy-multimodal-embedding:dev
+          # disabled for now: image not found
+          # docker pull instill/dummy-multimodal-embedding:dev
           docker tag instill/dummy-cls:dev localhost:5001/admin/dummy-cls
           docker tag instill/dummy-det:dev localhost:5001/admin/dummy-det
           docker tag instill/dummy-instance-segmentation:dev localhost:5001/admin/dummy-instance-segmentation
@@ -131,7 +132,8 @@ jobs:
           docker tag instill/dummy-text-to-image:dev localhost:5001/admin/dummy-text-to-image
           docker tag instill/dummy-multimodal-chat:dev localhost:5001/admin/dummy-multimodal-chat
           docker tag instill/dummy-text-embedding:dev localhost:5001/admin/dummy-text-embedding
-          docker tag instill/dummy-multimodal-embedding:dev localhost:5001/admin/dummy-multimodal-embedding
+          # disabled for now: image not found
+          # docker tag instill/dummy-multimodal-embedding:dev localhost:5001/admin/dummy-multimodal-embedding
           docker push localhost:5001/admin/dummy-cls
           docker push localhost:5001/admin/dummy-det
           docker push localhost:5001/admin/dummy-instance-segmentation
@@ -142,7 +144,8 @@ jobs:
           docker push localhost:5001/admin/dummy-text-to-image
           docker push localhost:5001/admin/dummy-multimodal-chat
           docker push localhost:5001/admin/dummy-text-embedding
-          docker push localhost:5001/admin/dummy-multimodal-embedding
+          # disabled for now: image not found
+          # docker push localhost:5001/admin/dummy-multimodal-embedding
 
       - name: Launch Init Model Pod
         run: |
@@ -173,7 +176,8 @@ jobs:
         run: while [ -z "$(docker ps -f 'name=model-backend-init-model' -f 'status=exited' -q)" ]; do echo "model init pod still running"; sleep 5; done;
 
       - name: Check test model deployment
-        run: while [ "$(curl -s http://localhost:8265/api/serve/applications/ | jq '.applications | to_entries | map(select(.key | contains("dummy-")) | .value.status) | length == 11 and all(. == "RUNNING")')" != "true" ]; do echo "models still deploying"; sleep 5; done;
+        # disabled 1 for now: image not found
+        run: while [ "$(curl -s http://localhost:8265/api/serve/applications/ | jq '.applications | to_entries | map(select(.key | contains("dummy-")) | .value.status) | length == 10 and all(. == "RUNNING")')" != "true" ]; do echo "models still deploying"; sleep 5; done;
 
       - name: Run integration-test
         run: |

diff --git a/.github/workflows/test-images.yml b/.github/workflows/test-images.yml
@@ -32,7 +32,8 @@ jobs:
             "dummy-text-to-image",
             "dummy-multimodal-chat",
             "dummy-text-embedding",
-            "dummy-multimodal-embedding",
+            # disabled for now: image not found
+            # "dummy-multimodal-embedding",
           ]
     steps:
       - uses: actions/checkout@v4

diff --git a/config/config.yaml b/config/config.yaml
@@ -25,7 +25,7 @@ database:
   host: pg-sql
   port: 5432
   name: model
-  version: 10
+  version: 11
   timezone: Etc/UTC
   pool:
     idleconnections: 5

diff --git a/pkg/acl/acl.go b/pkg/acl/acl.go
@@ -15,10 +15,11 @@ import (
 	openfga "github.com/openfga/api/proto/openfga/v1"
 
 	"github.com/gofrs/uuid"
+	"github.com/redis/go-redis/v9"
+
 	"github.com/instill-ai/model-backend/config"
 	"github.com/instill-ai/model-backend/pkg/constant"
 	"github.com/instill-ai/model-backend/pkg/resource"
-	"github.com/redis/go-redis/v9"
 )
 
 type ACLClientInterface interface {
@@ -47,10 +48,16 @@ type Relation struct {
 }
 
 type Mode string
+type ObjectType string
+type Role string
 
 const (
 	ReadMode  Mode = "read"
 	WriteMode Mode = "write"
+
+	Organization ObjectType = "organization"
+
+	Member Role = "member"
 )
 
 func NewACLClient(wc openfga.OpenFGAServiceClient, rc openfga.OpenFGAServiceClient, redisClient *redis.Client) ACLClient {

diff --git a/pkg/datamodel/modeltrigger.go → pkg/datamodel/modelrun.go b/pkg/datamodel/modeltrigger.go → pkg/datamodel/modelrun.go
@@ -11,38 +11,43 @@ import (
 
 // for saving the protobuf types as string values
 type (
-	TriggerStatus runpb.RunStatus
-	TriggerSource runpb.RunSource
+	RunStatus runpb.RunStatus
+	RunSource runpb.RunSource
 )
 
-func (v *TriggerStatus) Scan(value any) error {
-	*v = TriggerStatus(runpb.RunStatus_value[value.(string)])
+func (v *RunStatus) Scan(value any) error {
+	*v = RunStatus(runpb.RunStatus_value[value.(string)])
 	return nil
 }
 
-func (v TriggerStatus) Value() (driver.Value, error) {
+func (v RunStatus) Value() (driver.Value, error) {
 	return runpb.RunStatus(v).String(), nil
 }
 
-func (v *TriggerSource) Scan(value any) error {
-	*v = TriggerSource(runpb.RunSource_value[value.(string)])
+func (v *RunSource) Scan(value any) error {
+	*v = RunSource(runpb.RunSource_value[value.(string)])
 	return nil
 }
 
-func (v TriggerSource) Value() (driver.Value, error) {
+func (v RunSource) Value() (driver.Value, error) {
 	return runpb.RunSource(v).String(), nil
 }
 
-type ModelTrigger struct {
+type ModelRun struct {
 	BaseStaticHardDelete
 	ModelUID          uuid.UUID
 	ModelVersion      string
-	Status            TriggerStatus
-	Source            TriggerSource
+	Status            RunStatus
+	Source            RunSource
 	TotalDuration     null.Int
 	EndTime           null.Time
 	RequesterUID      uuid.UUID
+	RunnerUID         uuid.UUID
 	InputReferenceID  string
 	OutputReferenceID null.String
 	Error             null.String
 }
+
+func (*ModelRun) TableName() string {
+	return "model_trigger"
+}
diff --git a/pkg/db/migration/000011_run_logging_namespace.down.sql b/pkg/db/migration/000011_run_logging_namespace.down.sql
@@ -0,0 +1,7 @@
+BEGIN;
+
+comment on column model_trigger.requester_uid is null;
+
+alter table model_trigger drop column runner_uid;
+
+COMMIT;
diff --git a/pkg/db/migration/000011_run_logging_namespace.up.sql b/pkg/db/migration/000011_run_logging_namespace.up.sql
@@ -0,0 +1,12 @@
+BEGIN;
+
+alter table model_trigger
+add runner_uid uuid;
+
+comment on column model_trigger.requester_uid is 'run by namespace, which is the credit owner';
+
+update model_trigger
+set runner_uid = requester_uid
+where runner_uid is null;
+
+COMMIT;
diff --git a/pkg/handler/mock_service_test.go b/pkg/handler/mock_service_test.go
diff --git a/pkg/handler/public.go b/pkg/handler/public.go
@@ -1124,7 +1124,7 @@ func (h *PublicHandler) ListModelRuns(ctx context.Context, req *modelpb.ListMode
 		return nil, err
 	}
 
-	resp, err := h.service.ListModelTriggers(ctx, req, filter)
+	resp, err := h.service.ListModelRuns(ctx, req, filter)
 	if err != nil {
 		span.SetStatus(1, err.Error())
 		return nil, err

diff --git a/pkg/handler/trigger.go b/pkg/handler/trigger.go
@@ -218,7 +218,7 @@ func (h *PublicHandler) triggerNamespaceModel(ctx context.Context, req TriggerNa
 		return commonpb.Task_TASK_UNSPECIFIED, nil, status.Error(codes.InvalidArgument, err.Error())
 	}
 
-	runLog, err := h.service.CreateModelTrigger(ctx, logUUID, userUID, modelUID, version.Version, inputJSON)
+	runLog, err := h.service.CreateModelRun(ctx, logUUID, userUID, modelUID, version.Version, inputJSON)
 	if err != nil {
 		usageData.Status = mgmtpb.Status_STATUS_ERRORED
 		return commonpb.Task_TASK_UNSPECIFIED, nil, status.Error(codes.InvalidArgument, err.Error())
@@ -228,7 +228,7 @@ func (h *PublicHandler) triggerNamespaceModel(ctx context.Context, req TriggerNa
 	var triggerErr error
 	defer func(u *utils.UsageMetricData, startTime time.Time) {
 		if err != nil && triggerErr == nil {
-			_ = h.service.UpdateModelTriggerWithError(ctx, runLog, err)
+			_ = h.service.UpdateModelRunWithError(ctx, runLog, err)
 		}
 		u.ComputeTimeDuration = time.Since(startTime).Seconds()
 		if err := h.service.WriteNewDataPoint(ctx, usageData); err != nil {
@@ -456,11 +456,11 @@ func (h *PublicHandler) triggerAsyncNamespaceModel(ctx context.Context, req Trig
 		}
 	}
 
-	userUID := uuid.FromStringOrNil(resource.GetRequestSingleHeader(ctx, constant.HeaderUserUIDKey))
+	requesterUID, userUID := utils.GetRequesterUIDAndUserUID(ctx)
 	usageData := &utils.UsageMetricData{
 		OwnerUID:           ns.NsUID.String(),
 		OwnerType:          mgmtpb.OwnerType_OWNER_TYPE_USER,
-		UserUID:            userUID.String(),
+		UserUID:            userUID,
 		UserType:           mgmtpb.OwnerType_OWNER_TYPE_USER,
 		ModelUID:           pbModel.Uid,
 		Mode:               mgmtpb.Mode_MODE_ASYNC,
@@ -476,7 +476,7 @@ func (h *PublicHandler) triggerAsyncNamespaceModel(ctx context.Context, req Trig
 		return nil, status.Error(codes.InvalidArgument, err.Error())
 	}
 
-	runLog, err := h.service.CreateModelTrigger(ctx, logUUID, userUID, modelUID, version.Version, inputJSON)
+	runLog, err := h.service.CreateModelRun(ctx, logUUID, uuid.FromStringOrNil(userUID), modelUID, version.Version, inputJSON)
 	if err != nil {
 		usageData.Status = mgmtpb.Status_STATUS_ERRORED
 		return nil, status.Error(codes.InvalidArgument, err.Error())
@@ -485,7 +485,7 @@ func (h *PublicHandler) triggerAsyncNamespaceModel(ctx context.Context, req Trig
 	// write usage/metric datapoint
 	defer func(u *utils.UsageMetricData, startTime time.Time) {
 		if err != nil {
-			_ = h.service.UpdateModelTriggerWithError(ctx, runLog, err)
+			_ = h.service.UpdateModelRunWithError(ctx, runLog, err)
 		}
 		if u.Status == mgmtpb.Status_STATUS_ERRORED {
 			u.ComputeTimeDuration = time.Since(startTime).Seconds()
@@ -549,14 +549,14 @@ func (h *PublicHandler) triggerAsyncNamespaceModel(ctx context.Context, req Trig
 	// latest operation
 	h.service.GetRedisClient().Set(
 		ctx,
-		fmt.Sprintf("model_trigger_output_key:%s:%s:%s", userUID, pbModel.Uid, ""),
+		fmt.Sprintf("model_trigger_output_key:%s:%s:%s:%s", userUID, requesterUID, pbModel.Uid, ""),
 		operation.GetName(),
 		time.Duration(config.Config.Server.Workflow.MaxWorkflowTimeout)*time.Second,
 	)
 	// latest version operation
 	h.service.GetRedisClient().Set(
 		ctx,
-		fmt.Sprintf("model_trigger_output_key:%s:%s:%s", userUID, pbModel.Uid, version.Version),
+		fmt.Sprintf("model_trigger_output_key:%s:%s:%s:%s", userUID, requesterUID, pbModel.Uid, version.Version),
 		operation.GetName(),
 		time.Duration(config.Config.Server.Workflow.MaxWorkflowTimeout)*time.Second,
 	)
@@ -763,19 +763,19 @@ func HandleTriggerMultipartForm(s service.Service, _ repository.Repository, w ht
 		return
 	}
 
-	runLog, err := s.CreateModelTrigger(ctx, logUUID, userUID, modelUID, version.Version, inputJSON)
+	runLog, err := s.CreateModelRun(ctx, logUUID, userUID, modelUID, version.Version, inputJSON)
 	if err != nil {
 		usageData.Status = mgmtpb.Status_STATUS_ERRORED
-		logger.Error("CreateModelTrigger in DB failed", zap.String("TriggerUID", logUUID.String()), zap.Error(err))
-		makeJSONResponse(w, 500, "CreateModelTrigger in DB failedd", "CreateModelTrigger in DB failed")
-		span.SetStatus(1, "CreateModelTrigger in DB failed")
+		logger.Error("CreateModelRun in DB failed", zap.String("TriggerUID", logUUID.String()), zap.Error(err))
+		makeJSONResponse(w, 500, "CreateModelRun in DB failedd", "CreateModelRun in DB failed")
+		span.SetStatus(1, "CreateModelRun in DB failed")
 		return
 	}
 
 	// write usage/metric datapoint
 	defer func(u *utils.UsageMetricData, startTime time.Time) {
 		if err != nil {
-			_ = s.UpdateModelTriggerWithError(ctx, runLog, err)
+			_ = s.UpdateModelRunWithError(ctx, runLog, err)
 		}
 		u.ComputeTimeDuration = time.Since(startTime).Seconds()
 		if err := s.WriteNewDataPoint(ctx, usageData); err != nil {