initial commit

rilldata · Jan 14, 2025 · 309cefe · 309cefe
1 parent 21167cf
commit 309cefe
Show file tree

Hide file tree

Showing 3 changed files with 65 additions and 0 deletions.
diff --git a/proto/rill/runtime/v1/api.proto b/proto/rill/runtime/v1/api.proto
@@ -218,6 +218,10 @@ service RuntimeService {
     option (google.api.http) = {get: "/v1/instances/{instance_id}/connectors/notifiers"};
   }
 
+  rpc ListSourceConnectors(ListSourceConnectorsRequest) returns (ListSourceConnectorsResponse) {
+    option (google.api.http) = {get: "/v1/connectors/{olap}/meta"};
+  }
+
   // Access management
 
   // IssueDevJWT issues a JWT for mimicking a user in local development.
@@ -805,6 +809,16 @@ message ListConnectorDriversResponse {
   repeated ConnectorDriver connectors = 1;
 }
 
+// Request message for RuntimeService.ListSourceConnectors
+message ListSourceConnectorsRequest {
+  string olap = 1;
+}
+
+// Response message for RuntimeService.ListSourceConnectors
+message ListSourceConnectorsResponse {
+  repeated ConnectorDriver connectors = 1;
+}
+
 // Request message for RuntimeService.AnalyzeConnectors
 message AnalyzeConnectorsRequest {
   string instance_id = 1;

diff --git a/runtime/drivers/duckdb/olap.go b/runtime/drivers/duckdb/olap.go
@@ -28,6 +28,56 @@ var (
 	connectionsInUse      = observability.Must(meter.Int64ObservableGauge("connections_in_use"))
 )
 
+func (c *connection) IngestionSpec() map[string][]*drivers.PropertySpec {
+	return map[string][]*drivers.PropertySpec{
+		"s3": []*drivers.PropertySpec{
+			{
+				Key:         "path",
+				Type:        drivers.StringPropertyType,
+				DisplayName: "S3 URI",
+				Description: "Path to file on the disk.",
+				Placeholder: "s3://bucket-name/path/to/file.csv",
+				Required:    true,
+				Hint:        "Glob patterns are supported",
+			},
+			{
+				Key:         "region",
+				Type:        drivers.StringPropertyType,
+				DisplayName: "AWS region",
+				Description: "AWS Region for the bucket.",
+				Placeholder: "us-east-1",
+				Required:    false,
+				Hint:        "Rill will use the default region in your local AWS config, unless set here.",
+			},
+			{
+				Key:         "endpoint",
+				Type:        drivers.StringPropertyType,
+				DisplayName: "Endpoint URL",
+				Description: "Override S3 Endpoint URL",
+				Placeholder: "https://my.s3.server.com",
+				Required:    false,
+				Hint:        "Overrides the S3 endpoint to connect to. This should only be used to connect to S3-compatible services, such as Cloudflare R2 or MinIO.",
+			},
+			{
+				Key:         "name",
+				Type:        drivers.StringPropertyType,
+				DisplayName: "Source name",
+				Description: "The name of the source",
+				Placeholder: "my_new_source",
+				Required:    true,
+			},
+			{
+				Key:         "aws.credentials",
+				Type:        drivers.InformationalPropertyType,
+				DisplayName: "AWS credentials",
+				Description: "AWS credentials inferred from your local environment.",
+				Hint:        "Set your local credentials: <code>aws configure</code> Click to learn more.",
+				DocsURL:     "https://docs.rilldata.com/reference/connectors/s3#local-credentials",
+			},
+		},
+	}
+}
+
 func (c *connection) Dialect() drivers.Dialect {
 	return drivers.DialectDuckDB
 }

diff --git a/runtime/drivers/olap.go b/runtime/drivers/olap.go
@@ -44,6 +44,7 @@ type InsertTableOptions struct {
 // OLAPStore is implemented by drivers that are capable of storing, transforming and serving analytical queries.
 // NOTE crud APIs are not safe to be called with `WithConnection`
 type OLAPStore interface {
+	IngestionSpec() map[string][]*PropertySpec
 	Dialect() Dialect
 	WithConnection(ctx context.Context, priority int, longRunning bool, fn WithConnectionFunc) error
 	Exec(ctx context.Context, stmt *Statement) error