diff --git a/Dockerfile b/Dockerfile index a84c69d1d..8bf42ea9f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -125,7 +125,6 @@ RUN curl -o /usr/bin/minio https://dl.min.io/server/minio/release/linux-$TARGETA && mc --version COPY ./scripts /usr/local/bin -COPY ./config/full-stack.json /etc/livepeer/full-stack.json ENV CATALYST_DOWNLOADER_PATH=/usr/local/bin \ CATALYST_DOWNLOADER_MANIFEST=/etc/livepeer/manifest.yaml \ @@ -134,7 +133,7 @@ ENV CATALYST_DOWNLOADER_PATH=/usr/local/bin \ RUN mkdir /data -CMD ["/usr/local/bin/catalyst", "--", "/usr/local/bin/MistController", "-c", "/etc/livepeer/full-stack.json"] +CMD ["/usr/local/bin/catalyst"] FROM ${FROM_LOCAL_PARENT} AS box-local diff --git a/Makefile b/Makefile index 35e74a19e..0d404117d 100644 --- a/Makefile +++ b/Makefile @@ -206,7 +206,7 @@ scripts: cp -Rv ./scripts/* ./bin .PHONY: box-dev -box-dev: scripts +box-dev: scripts catalyst ulimit -c unlimited \ && exec docker run \ -v $$(realpath bin):/usr/local/bin \ @@ -214,6 +214,8 @@ box-dev: scripts -v $$(realpath config):/etc/livepeer:ro \ -v $$(realpath ./coredumps):$$(realpath ./coredumps) \ -e CORE_DUMP_DIR=$$(realpath ./coredumps) \ + -v /home/iameli/.ethereum/keystore:/keystore \ + -e CATALYST_SECRET=f61b3cdb-d173-4a7a-a0d3-547b871a56f9 \ $(shell for line in $$(cat .env 2>/dev/null || echo ''); do printf -- "-e $$line "; done) \ --rm \ -it \ @@ -246,3 +248,12 @@ snapshot: && cd data \ && rm -rf cockroach/auxiliary/EMERGENCY_BALLAST \ && tar czvf ../livepeer-studio-bootstrap.tar.gz cockroach + +.PHONY: sql-schema-dump +sql-schema-dump: + cockroach sql --format=raw \ + --url 'postgresql://root@localhost:5432/defaultdb?sslmode=disable' \ + --execute "show create all tables" \ + | grep -v '#' \ + | grep -v 'Time' \ + > ./config/full-stack.sql \ No newline at end of file diff --git a/cmd/catalyst/catalyst.go b/cmd/catalyst/catalyst.go index 0f388cfe3..da6334f5f 100644 --- a/cmd/catalyst/catalyst.go +++ b/cmd/catalyst/catalyst.go @@ -1,13 +1,15 @@ package main import ( + "flag" "os" + "os/exec" "syscall" - "github.com/livepeer/catalyst/cmd/downloader/cli" - "github.com/livepeer/catalyst/cmd/downloader/downloader" - "github.com/livepeer/catalyst/cmd/downloader/types" - glog "github.com/magicsong/color-glog" + "github.com/golang/glog" + "github.com/livepeer/catalyst/cmd/catalyst/config" + "github.com/livepeer/catalyst/cmd/downloader/constants" + "github.com/peterbourgon/ff/v3" ) var Version = "undefined" @@ -16,26 +18,69 @@ var Version = "undefined" // but that one will stay just a downloader and this binary may gain other functionality func main() { - cliFlags, err := cli.GetCliFlags(types.BuildFlags{Version: Version}) + cli := config.Cli{} + flag.Set("logtostderr", "true") + // vFlag := flag.Lookup("v") + fs := flag.NewFlagSet(constants.AppName, flag.ExitOnError) + // fs.StringVar(&cli.Verbosity, "v", "3", "Log verbosity. Integer value from 0 to 9") + fs.StringVar(&cli.PublicURL, "public-url", "http://localhost:8888", "Public-facing URL of your Catalyst node, including protocol and port") + fs.StringVar(&cli.Secret, "secret", "", "Secret UUID to secure your Catalyst node") + fs.StringVar(&cli.ConfOutput, "conf-output", "/tmp/catalyst-generated.json", "Path where we will place generated MistServer configuration") + fs.StringVar(&cli.SQLOutput, "sql-output", "/tmp/catalyst-fixtures.sql", "Path where we will generate SQL fixtures") + fs.StringVar(&cli.Network, "network", "offchain", "Network to use for transcoding. Allowed values: offchain, arbitrum-one-mainnet") + fs.StringVar(&cli.EthURL, "eth-url", "", "HTTPS URL of an Ethereum RPC provider for your selected network") + fs.StringVar(&cli.EthKeystorePath, "eth-keystore-path", "/keystore", "Path to an Ethereum keystore") + fs.StringVar(&cli.EthPassword, "eth-password", "", "Ethereum password or path to password file") + fs.StringVar(&cli.MaxTicketEV, "max-ticket-ev", "50000000001", "The maximum acceptable expected value for one PM ticket") + fs.StringVar(&cli.MaxTotalEV, "max-total-ev", "20000000000000", "The maximum acceptable expected value for one PM payment") + fs.StringVar(&cli.MaxPricePerUnit, "max-price-per-unit", "700", "The maximum transcoding price (in wei) per 'pixelsPerUnit' a broadcaster is willing to accept. If not set explicitly, broadcaster is willing to accept ANY price") + + ff.Parse( + fs, os.Args[1:], + ff.WithConfigFileFlag("config"), + ff.WithConfigFileParser(ff.PlainParser), + ff.WithEnvVarPrefix("CATALYST"), + ff.WithEnvVarSplit(","), + ) + flag.CommandLine.Parse(nil) + conf, sql, err := config.GenerateConfig(&cli) + if err != nil { + panic(err) + } + err = os.WriteFile(cli.ConfOutput, conf, 0600) if err != nil { - glog.Fatalf("error parsing cli flags: %s", err) - return + panic(err) } - err = downloader.Run(cliFlags) + err = os.WriteFile(cli.SQLOutput, sql, 0600) if err != nil { - glog.Fatalf("error running downloader: %s", err) + panic(err) } - execNext(cliFlags) + execNext(cli) } +// Archiving for when we want to introduce auto-updating: + +// func main() { +// cliFlags, err := cli.GetCliFlags(types.BuildFlags{Version: Version}) +// if err != nil { +// glog.Fatalf("error parsing cli flags: %s", err) +// return +// } +// err = downloader.Run(cliFlags) +// if err != nil { +// glog.Fatalf("error running downloader: %s", err) +// } +// execNext(cliFlags) +// } + // Done! Move on to the provided next application, if it exists. -func execNext(cliFlags types.CliFlags) { - if len(cliFlags.ExecCommand) == 0 { - // Nothing to do. - return +func execNext(cli config.Cli) { + fname, err := exec.LookPath("MistController") + if err != nil { + glog.Fatalf("error finding MistController: %s", fname) } - glog.Infof("downloader complete, now we will exec %v", cliFlags.ExecCommand) - execErr := syscall.Exec(cliFlags.ExecCommand[0], cliFlags.ExecCommand, os.Environ()) + glog.Infof("config file written, now we will exec MistController") + execErr := syscall.Exec(fname, []string{"MistController", "-c", cli.ConfOutput}, os.Environ()) if execErr != nil { glog.Fatalf("error running next command: %s", execErr) } diff --git a/cmd/catalyst/config/config.go b/cmd/catalyst/config/config.go new file mode 100644 index 000000000..496c060d6 --- /dev/null +++ b/cmd/catalyst/config/config.go @@ -0,0 +1,223 @@ +package config + +import ( + _ "embed" + "encoding/json" + "fmt" + "net/url" + "strings" + + "github.com/doug-martin/goqu/v9" +) + +//go:embed full-stack.json +var fullstack []byte + +//go:embed full-stack.sql +var sqlTables string + +var adminID = "00000000-0000-4000-0000-000000000000" +var recordingBucketID = "00000000-0000-4000-0000-000000000001" +var vodBucketID = "00000000-0000-4000-0000-000000000002" +var vodBucketCatalystID = "00000000-0000-4000-0000-000000000003" +var privateBucketID = "00000000-0000-4000-0000-000000000004" + +type Cli struct { + PublicURL string + Secret string + Verbosity string + ConfOutput string + SQLOutput string + Network string + EthURL string + EthKeystorePath string + EthPassword string + MaxTicketEV string + MaxTotalEV string + MaxPricePerUnit string +} + +type DBObject map[string]any + +func (d DBObject) Table() string { + switch d["kind"] { + case "user": + return "users" + case "api-token": + return "api_token" + case "object-store": + return "object_store" + } + panic("table not found") +} + +func GenerateConfig(cli *Cli) ([]byte, []byte, error) { + if cli.Secret == "" { + return []byte{}, []byte{}, fmt.Errorf("CATALYST_SECRET parameter is required") + } + u, err := url.Parse(cli.PublicURL) + if err != nil { + return []byte{}, []byte{}, err + } + var conf MistConfig + err = json.Unmarshal(fullstack, &conf) + if err != nil { + return []byte{}, []byte{}, err + } + + inserts := []DBObject{} + + admin := DBObject{ + "id": adminID, + "firstName": "Root", + "lastName": "User", + "admin": true, + "createdAt": 0, + "email": "admin@example.com", + "emailValid": true, + "emailValidToken": "00000000-0000-4000-0000-000000000000", + "kind": "user", + "lastSeen": 1694546853946, + "password": "0000000000000000000000000000000000000000000000000000000000000000", + "salt": "0000000000000000", + } + apiToken := DBObject{ + "name": "ROOT KEY DON'T DELETE", + "createdAt": 0, + "id": cli.Secret, + "kind": "api-token", + "userId": admin["id"], + } + inserts = append(inserts, admin, apiToken) + + recordingBucket := ObjectStore(adminID, cli.PublicURL, recordingBucketID, "os-recordings") + + vodBucket := ObjectStore(adminID, cli.PublicURL, vodBucketID, "os-vod") + + vodBucketCatalyst := ObjectStore(adminID, cli.PublicURL, vodBucketCatalystID, "os-catalyst-vod") + + privateBucket := ObjectStore(adminID, cli.PublicURL, privateBucketID, "os-vod") + inserts = append(inserts, recordingBucket, vodBucket, vodBucketCatalyst, privateBucket) + + newProtocols := []*Protocol{} + for _, protocol := range conf.Config.Protocols { + ok := tweakProtocol(protocol, cli, u) + if ok { + newProtocols = append(newProtocols, protocol) + } + } + conf.Config.Protocols = newProtocols + + video := conf.Streams["video"] + for _, process := range video.Processes { + if process.Process == "Livepeer" { + process.AccessToken = cli.Secret + } + } + + var out []byte + out, err = json.MarshalIndent(conf, "", " ") + if err != nil { + return []byte{}, []byte{}, err + } + + sql := strings.ReplaceAll(sqlTables, "CREATE TABLE", "CREATE TABLE IF NOT EXISTS") + + for _, insert := range inserts { + obj, err := json.Marshal(insert) + if err != nil { + return []byte{}, []byte{}, err + } + ds := goqu.Insert(insert.Table()).Rows( + goqu.Record{"id": insert["id"], "data": obj}, + ).OnConflict(goqu.DoNothing()) + insertSQL, _, err := ds.ToSQL() + if err != nil { + return []byte{}, []byte{}, err + } + + sql = fmt.Sprintf("%s\n%s;", sql, insertSQL) + } + + return out, []byte(sql), nil +} + +// returns true if this protocol should be included +func tweakProtocol(protocol *Protocol, cli *Cli, u *url.URL) bool { + if protocol.Connector == "livepeer-api" && !protocol.StreamInfoService { + protocol.RecordCatalystObjectStoreID = recordingBucketID + protocol.VODCatalystObjectStoreID = vodBucketCatalystID + protocol.VODCatalystPrivateAssetsObjectStore = privateBucketID + protocol.VODObjectStoreID = vodBucketID + protocol.CORSJWTAllowlist = fmt.Sprintf(`["%s"]`, cli.PublicURL) + protocol.Ingest = fmt.Sprintf( + `[{"ingest":"rtmp://%s/live","ingests":{"rtmp":"rtmp://%s/live","srt":"srt://%s:8889"},"playback":"%s/mist/hls","base":"%s","origin":"%s"}]`, + u.Hostname(), + u.Hostname(), + u.Hostname(), + cli.PublicURL, + cli.PublicURL, + cli.PublicURL, + ) + } else if protocol.Connector == "livepeer-catalyst-api" { + protocol.APIToken = cli.Secret + protocol.Tags = fmt.Sprintf("node=media,http=%s/mist,https=%s/mist", cli.PublicURL, cli.PublicURL) + } else if protocol.Connector == "livepeer-task-runner" { + protocol.CatalystSecret = cli.Secret + protocol.LivepeerAccessToken = cli.Secret + } else if protocol.Connector == "livepeer-analyzer" { + protocol.LivepeerAccessToken = cli.Secret + } else if protocol.Connector == "livepeer" && protocol.Broadcaster { + // both broadcasters + if cli.Network != "offchain" { + protocol.Network = cli.Network + protocol.EthKeystorePath = cli.EthKeystorePath + protocol.EthPassword = cli.EthPassword + protocol.EthURL = cli.EthURL + protocol.MaxPricePerUnit = cli.MaxPricePerUnit + protocol.MaxTicketEV = cli.MaxTicketEV + protocol.MaxTotalEV = cli.MaxTotalEV + } else { + protocol.OrchAddr = "http://127.0.0.1:8936" + } + } else if protocol.Connector == "livepeer" && protocol.Broadcaster && protocol.MetadataQueueURI != "" { + // live broadcaster + protocol.AuthWebhookURL = fmt.Sprintf("http://%s:%s@127.0.0.1:3004/api/stream/hook", adminID, cli.Secret) + } else if protocol.Connector == "livepeer" && protocol.Orchestrator { + // if we're not offchain we shouldn't run a local O + if cli.Network != "offchain" { + return false + } + } else if protocol.Connector == "WebRTC" { + protocol.ICEServers = []ICEServer{ + { + URLs: fmt.Sprintf("stun:%s:3478", u.Hostname()), + }, + { + Credential: "livepeer", + URLs: fmt.Sprintf("turn:%s:3478", u.Hostname()), + Username: "livepeer", + }, + { + URLs: fmt.Sprintf("stun:%s:5349", u.Hostname()), + }, + { + Credential: "livepeer", + URLs: fmt.Sprintf("turn:%s:5349", u.Hostname()), + Username: "livepeer", + }, + } + } + return true +} + +func ObjectStore(userID, publicURL, id, bucket string) DBObject { + return DBObject{ + "createdAt": 0, + "id": id, + "publicUrl": fmt.Sprintf("%s/%s", publicURL, bucket), + "url": fmt.Sprintf("s3+http://admin:password@127.0.0.1:9000/%s", bucket), + "userId": userID, + "kind": "object-store", + } +} diff --git a/cmd/catalyst/config/config_test.go b/cmd/catalyst/config/config_test.go new file mode 100644 index 000000000..326165b48 --- /dev/null +++ b/cmd/catalyst/config/config_test.go @@ -0,0 +1,40 @@ +package config + +import ( + _ "embed" + "encoding/json" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/require" +) + +//go:embed full-stack.spec.json +var spec []byte + +func TestSpecIsValid(t *testing.T) { + err := json.Unmarshal(fullstack, &MistConfig{}) + require.NoError(t, err) + err = json.Unmarshal(spec, &MistConfig{}) + require.NoError(t, err) +} + +func TestItCanPassthroughEmptyConfig(t *testing.T) { + generated, _, err := GenerateConfig(&Cli{ + Secret: "44444444-4444-4444-4444-444444444444", + PublicURL: "https://example.com", + }) + require.NoError(t, err) + require.Empty(t, jsonEQ(spec, generated)) +} + +// returns empty if equal +func jsonEQ(x, y []byte) string { + return cmp.Diff(x, y, cmp.Transformer("ParseJSON", func(in []byte) (out any) { + if err := json.Unmarshal(in, &out); err != nil { + return err + } + return out + })) + +} diff --git a/cmd/catalyst/config/full-stack.json b/cmd/catalyst/config/full-stack.json new file mode 100644 index 000000000..3a3a83321 --- /dev/null +++ b/cmd/catalyst/config/full-stack.json @@ -0,0 +1,253 @@ +{ + "account": { + "test": { + "password": "098f6bcd4621d373cade4e832627b4f6" + } + }, + "autopushes": [ + [ + "video+", + "s3+http://admin:password@localhost:9000/os-recordings/$wildcard/$uuid/source/$segmentCounter.ts?m3u8=../output.m3u8&split=5&video=source&audio=AAC&append=1&waittrackcount=2&recstart=-1" + ] + ], + "config": { + "accesslog": "LOG", + "prometheus": "koekjes", + "protocols": [ + { + "connector": "livepeer-cockroach" + }, + { + "connector": "livepeer-rabbitmq" + }, + { + "connector": "livepeer-nginx" + }, + { + "connector": "livepeer-minio" + }, + { + "connector": "livepeer-core-dump-monitor" + }, + { + "connector": "livepeer-coturn" + }, + { + "connector": "livepeer-api", + "postgres-url": "postgresql://root@localhost:5432/defaultdb?sslmode=disable", + "jwt-secret": "stupidlysecret", + "jwt-audience": "my-node", + "broadcasters": "[{\"address\":\"http://127.0.0.1:8935\"}]", + "base-stream-name": "video", + "amqp-url": "amqp://localhost:5672/livepeer", + "own-region": "box" + }, + { + "connector": "livepeer-api", + "postgres-url": "postgresql://root@localhost:5432/defaultdb?sslmode=disable", + "own-region": "box", + "stream-info-service": true, + "port": "3040" + }, + { + "connector": "livepeer-catalyst-api", + "api-server": "http://127.0.0.1:3004", + "redirect-prefixes": "video,videorec", + "node": "localhost", + "http-internal-addr": "0.0.0.0:7979", + "balancer-args": "-P koekjes", + "amqp-url": "amqp://localhost:5672/livepeer", + "own-region": "box", + "v": "8", + "broadcaster-url": "http://127.0.0.1:8937" + }, + { + "connector": "livepeer", + "broadcaster": true, + "metricsClientIP": true, + "metricsPerStream": true, + "monitor": true, + "cliAddr": "127.0.0.1:7935", + "httpAddr": "127.0.0.1:8935", + "rtmpAddr": "127.0.0.1:1936", + "metadataQueueUri": "amqp://localhost:5672/livepeer", + "v": "2" + }, + { + "connector": "livepeer", + "broadcaster": true, + "metricsClientIP": true, + "metricsPerStream": true, + "monitor": true, + "httpAddr": "127.0.0.1:8937", + "cliAddr": "127.0.0.1:7937", + "rtmpAddr": "127.0.0.1:1937", + "v": "2" + }, + { + "connector": "livepeer", + "orchestrator": true, + "transcoder": true, + "cliAddr": "127.0.0.1:7936", + "metricsClientIP": true, + "metricsPerStream": true, + "monitor": true, + "serviceAddr": "127.0.0.1:8936", + "v": "2" + }, + { + "connector": "livepeer-analyzer", + "port": "3080", + "rabbitmq-uri": "amqp://localhost:5672/livepeer", + "disable-bigquery": true, + "v": "8" + }, + { + "connector": "livepeer-task-runner", + "amqp-uri": "amqp://localhost:5672/livepeer", + "catalyst-url": "http://127.0.0.1:7979", + "own-base-url": "http://127.0.0.1:3060/task-runner", + "port": "3060" + }, + { + "connector": "AAC" + }, + { + "connector": "CMAF" + }, + { + "connector": "DTSC" + }, + { + "connector": "EBML" + }, + { + "connector": "FLV" + }, + { + "connector": "H264" + }, + { + "connector": "HDS" + }, + { + "connector": "HLS" + }, + { + "connector": "HTTP" + }, + { + "connector": "HTTPTS" + }, + { + "connector": "JSON" + }, + { + "connector": "MP3" + }, + { + "connector": "MP4" + }, + { + "connector": "OGG" + }, + { + "connector": "RTMP" + }, + { + "connector": "RTSP" + }, + { + "connector": "SRT" + }, + { + "connector": "TSSRT" + }, + { + "connector": "WAV" + }, + { + "connector": "WebRTC", + "bindhost": "127.0.0.1", + "iceservers": [ + { + "urls": "stun:localhost" + }, + { + "credential": "livepeer", + "urls": "turn:localhost", + "username": "livepeer" + } + ], + "pubhost": "127.0.0.1" + } + ], + "sessionInputMode": 15, + "sessionOutputMode": 15, + "sessionStreamInfoMode": 1, + "sessionUnspecifiedMode": 0, + "sessionViewerMode": 14, + "tknMode": 15, + "triggers": {}, + "trustedproxy": ["0.0.0.0/1", "128.0.0.0/1"] + }, + "extwriters": [ + [ + "s3", + "livepeer-catalyst-uploader -t 2592000s", + ["s3", "s3+http", "s3+https", "ipfs"] + ] + ], + "push_settings": { + "maxspeed": null, + "wait": null + }, + "streams": { + "video": { + "DVR": 25000, + "maxkeepaway": 7500, + "name": "video", + "processes": [ + { + "custom_url": "http://127.0.0.1:3004/api/stream/video", + "debug": 5, + "exit_unmask": false, + "leastlive": true, + "process": "Livepeer", + "target_profiles": [ + { + "x-LSP-name": "144p", + "bitrate": 400000, + "fps": 30, + "height": 144, + "name": "P144p30fps16x9", + "width": 256, + "profile": "H264ConstrainedHigh" + } + ] + }, + { + "exec": "gst-launch-1.0 -q fdsrc fd=0 ! matroskademux ! faad ! audioresample ! opusenc inband-fec=true perfect-timestamp=true ! matroskamux ! fdsink fd=1", + "exit_unmask": false, + "process": "MKVExec", + "track_inhibit": "audio=opus&video=source,|bframes", + "track_select": "video=none&audio=aac,|source,|maxbps", + "x-LSP-name": "AAC to Opus", + "leastlive": true + }, + { + "exec": "gst-launch-1.0 -q fdsrc fd=0 ! matroskademux ! opusdec use-inband-fec=true ! audioresample ! voaacenc perfect-timestamp=true ! matroskamux ! fdsink fd=1", + "exit_unmask": false, + "process": "MKVExec", + "track_inhibit": "audio=aac", + "track_select": "video=none&audio=opus,|source,|maxbps", + "x-LSP-name": "Opus to AAC", + "leastlive": true + } + ], + "segmentsize": 1, + "source": "push://", + "stop_sessions": false + } + } +} diff --git a/config/full-stack.json b/cmd/catalyst/config/full-stack.spec.json similarity index 64% rename from config/full-stack.json rename to cmd/catalyst/config/full-stack.spec.json index 2351995f6..2748e21d8 100644 --- a/config/full-stack.json +++ b/cmd/catalyst/config/full-stack.spec.json @@ -5,10 +5,6 @@ } }, "autopushes": [ - [ - "videorec+", - "s3+http://admin:password@localhost:9000/os-recordings/$wildcard/$uuid/source/$segmentCounter.ts?m3u8=../output.m3u8&split=5&video=source&audio=AAC&append=1&waittrackcount=2&recstart=-1" - ], [ "video+", "s3+http://admin:password@localhost:9000/os-recordings/$wildcard/$uuid/source/$segmentCounter.ts?m3u8=../output.m3u8&split=5&video=source&audio=AAC&append=1&waittrackcount=2&recstart=-1" @@ -16,13 +12,6 @@ ], "config": { "accesslog": "LOG", - "controller": { - "interface": null, - "port": null, - "username": null - }, - "defaultStream": null, - "limits": null, "prometheus": "koekjes", "protocols": [ { @@ -46,17 +35,17 @@ { "connector": "livepeer-api", "postgres-url": "postgresql://root@localhost:5432/defaultdb?sslmode=disable", - "cors-jwt-allowlist": "[\"http://localhost:8080\", \"http://localhost:3000\", \"http://localhost:8888\",\"http://127.0.0.1:8080\", \"http://127.0.0.1:3000\", \"http://127.0.0.1:8888\"]", + "cors-jwt-allowlist": "[\"https://example.com\"]", "jwt-secret": "stupidlysecret", "jwt-audience": "my-node", - "ingest": "[{\"ingest\":\"rtmp://localhost/live\",\"ingests\":{\"rtmp\":\"rtmp://localhost/live\",\"srt\":\"srt://localhost:8889\"},\"playback\":\"http://localhost:8888/hls\",\"base\":\"http://localhost:8888\",\"origin\":\"http://localhost:8888\"}]", + "ingest": "[{\"ingest\":\"rtmp://example.com/live\",\"ingests\":{\"rtmp\":\"rtmp://example.com/live\",\"srt\":\"srt://example.com:8889\"},\"playback\":\"https://example.com/mist/hls\",\"base\":\"https://example.com\",\"origin\":\"https://example.com\"}]", "broadcasters": "[{\"address\":\"http://127.0.0.1:8935\"}]", "base-stream-name": "video", "amqp-url": "amqp://localhost:5672/livepeer", - "vodObjectStoreId": "917a2f18-f7a8-4ae3-a849-6efd4aac8e59", - "vodCatalystObjectStoreId": "517873a4-487c-40ad-872f-027f4bc6bd98", - "vodCatalystPrivateAssetsObjectStore": "cab9266f-5583-4532-9630-7be10d92affe", - "recordCatalystObjectStoreId": "0926e4ba-b726-4386-92ee-5c4583f62f0a", + "vodObjectStoreId": "00000000-0000-4000-0000-000000000002", + "vodCatalystObjectStoreId": "00000000-0000-4000-0000-000000000003", + "vodCatalystPrivateAssetsObjectStore": "00000000-0000-4000-0000-000000000004", + "recordCatalystObjectStoreId": "00000000-0000-4000-0000-000000000001", "own-region": "box" }, { @@ -69,8 +58,8 @@ { "connector": "livepeer-catalyst-api", "api-server": "http://127.0.0.1:3004", - "api-token": "f61b3cdb-d173-4a7a-a0d3-547b871a56f9", - "tags": "node=media,http=http://localhost:8888/mist", + "api-token": "44444444-4444-4444-4444-444444444444", + "tags": "node=media,http=https://example.com/mist,https=https://example.com/mist", "redirect-prefixes": "video,videorec", "node": "localhost", "http-internal-addr": "0.0.0.0:7979", @@ -86,11 +75,11 @@ "metricsClientIP": true, "metricsPerStream": true, "monitor": true, - "cliAddr": "127.0.0.1:7935", + "cliAddr": "127.0.0.1:7935", "httpAddr": "127.0.0.1:8935", "orchAddr": "127.0.0.1:8936", "rtmpAddr": "127.0.0.1:1936", - "authWebhookUrl": "http://9c2936b5-143f-4b10-b302-6a21b5f29c3d:f61b3cdb-d173-4a7a-a0d3-547b871a56f9@127.0.0.1:3004/api/stream/hook", + "authWebhookUrl": "http://00000000-0000-4000-0000-000000000000:44444444-4444-4444-4444-444444444444@127.0.0.1:3004/api/stream/hook", "metadataQueueUri": "amqp://localhost:5672/livepeer", "v": "2" }, @@ -119,7 +108,7 @@ }, { "connector": "livepeer-analyzer", - "livepeer-access-token": "f61b3cdb-d173-4a7a-a0d3-547b871a56f9", + "livepeer-access-token": "44444444-4444-4444-4444-444444444444", "port": "3080", "rabbitmq-uri": "amqp://localhost:5672/livepeer", "disable-bigquery": true, @@ -128,9 +117,9 @@ { "connector": "livepeer-task-runner", "amqp-uri": "amqp://localhost:5672/livepeer", - "catalyst-secret": "f61b3cdb-d173-4a7a-a0d3-547b871a56f9", + "catalyst-secret": "44444444-4444-4444-4444-444444444444", "catalyst-url": "http://127.0.0.1:7979", - "livepeer-access-token": "f61b3cdb-d173-4a7a-a0d3-547b871a56f9", + "livepeer-access-token": "44444444-4444-4444-4444-444444444444", "own-base-url": "http://127.0.0.1:3060/task-runner", "port": "3060" }, @@ -196,11 +185,19 @@ "bindhost": "127.0.0.1", "iceservers": [ { - "urls": "stun:localhost" + "urls": "stun:example.com:3478" }, { "credential": "livepeer", - "urls": "turn:localhost", + "urls": "turn:example.com:3478", + "username": "livepeer" + }, + { + "urls": "stun:example.com:5349" + }, + { + "credential": "livepeer", + "urls": "turn:example.com:5349", "username": "livepeer" } ], @@ -234,11 +231,11 @@ "name": "video", "processes": [ { - "access_token": "f61b3cdb-d173-4a7a-a0d3-547b871a56f9", + "access_token": "44444444-4444-4444-4444-444444444444", "custom_url": "http://127.0.0.1:3004/api/stream/video", "debug": 5, "exit_unmask": false, - "leastlive": "1", + "leastlive": true, "process": "Livepeer", "target_profiles": [ { @@ -259,7 +256,7 @@ "track_inhibit": "audio=opus&video=source,|bframes", "track_select": "video=none&audio=aac,|source,|maxbps", "x-LSP-name": "AAC to Opus", - "leastlive": 1 + "leastlive": true }, { "exec": "gst-launch-1.0 -q fdsrc fd=0 ! matroskademux ! opusdec use-inband-fec=true ! audioresample ! voaacenc perfect-timestamp=true ! matroskamux ! fdsink fd=1", @@ -268,60 +265,12 @@ "track_inhibit": "audio=aac", "track_select": "video=none&audio=opus,|source,|maxbps", "x-LSP-name": "Opus to AAC", - "leastlive": 1 - } - ], - "segmentsize": 1, - "source": "push://", - "stop_sessions": false - }, - "videorec": { - "DVR": 25000, - "maxkeepaway": 7500, - "name": "video", - "processes": [ - { - "access_token": "f61b3cdb-d173-4a7a-a0d3-547b871a56f9", - "custom_url": "http://127.0.0.1:3004/api/stream/video", - "debug": 5, - "exit_unmask": false, - "leastlive": "1", - "process": "Livepeer", - "target_profiles": [ - { - "x-LSP-name": "144p", - "bitrate": 400000, - "fps": 30, - "height": 144, - "name": "P144p30fps16x9", - "width": 256, - "profile": "H264ConstrainedHigh" - } - ] - }, - { - "exec": "gst-launch-1.0 -q fdsrc fd=0 ! matroskademux ! faad ! audioresample ! opusenc inband-fec=true perfect-timestamp=true ! matroskamux ! fdsink fd=1", - "exit_unmask": false, - "process": "MKVExec", - "track_inhibit": "audio=opus&video=source,|bframes", - "track_select": "video=none&audio=aac,|source,|maxbps", - "x-LSP-name": "AAC to Opus", - "leastlive": 1 - }, - { - "exec": "gst-launch-1.0 -q fdsrc fd=0 ! matroskademux ! opusdec use-inband-fec=true ! audioresample ! voaacenc perfect-timestamp=true ! matroskamux ! fdsink fd=1", - "exit_unmask": false, - "process": "MKVExec", - "track_inhibit": "audio=aac", - "track_select": "video=none&audio=opus,|source,|maxbps", - "x-LSP-name": "Opus to AAC", - "leastlive": 1 + "leastlive": true } ], "segmentsize": 1, "source": "push://", "stop_sessions": false } - }, - "variables": null + } } diff --git a/cmd/catalyst/config/full-stack.sql b/cmd/catalyst/config/full-stack.sql new file mode 100644 index 000000000..a2de48dba --- /dev/null +++ b/cmd/catalyst/config/full-stack.sql @@ -0,0 +1,117 @@ +CREATE TABLE public.stream ( + id VARCHAR(128) NOT NULL, + data JSONB NULL, + CONSTRAINT stream_pkey PRIMARY KEY (id ASC) +); +CREATE TABLE public.webhook ( + id VARCHAR(128) NOT NULL, + data JSONB NULL, + CONSTRAINT webhook_pkey PRIMARY KEY (id ASC), + INVERTED INDEX webhook_events ((data->'events':::STRING)), + INDEX "webhook_userId" ((data->>'userId':::STRING) ASC) +); +CREATE TABLE public.session ( + id VARCHAR(128) NOT NULL, + data JSONB NULL, + CONSTRAINT session_pkey PRIMARY KEY (id ASC) +); +CREATE TABLE public.asset ( + id VARCHAR(128) NOT NULL, + data JSONB NULL, + CONSTRAINT asset_pkey PRIMARY KEY (id ASC), + INDEX asset_id ((data->>'id':::STRING) ASC), + INDEX "asset_playbackId" ((data->>'playbackId':::STRING) ASC), + INDEX asset_source_url (((data->'source':::STRING)->>'url':::STRING) ASC), + INDEX "asset_source_sessionId" (((data->'source':::STRING)->>'sessionId':::STRING) ASC), + INDEX "asset_storage_ipfs_nftMetadata_cid" (((((data->'storage':::STRING)->'ipfs':::STRING)->'nftMetadata':::STRING)->>'cid':::STRING) ASC), + INDEX asset_storage_ipfs_cid ((((data->'storage':::STRING)->'ipfs':::STRING)->>'cid':::STRING) ASC), + INDEX "asset_userId" ((data->>'userId':::STRING) ASC), + INDEX "asset_playbackRecordingId" ((data->>'playbackRecordingId':::STRING) ASC), + INDEX "asset_sourceAssetId" ((data->>'sourceAssetId':::STRING) ASC) +); +CREATE TABLE public.users ( + id VARCHAR(128) NOT NULL, + data JSONB NULL, + CONSTRAINT users_pkey PRIMARY KEY (id ASC), + UNIQUE INDEX users_email ((data->>'email':::STRING) ASC), + UNIQUE INDEX "users_stripeCustomerId" ((data->>'stripeCustomerId':::STRING) ASC) +); +CREATE TABLE public.webhook_response ( + id VARCHAR(128) NOT NULL, + data JSONB NULL, + CONSTRAINT webhook_response_pkey PRIMARY KEY (id ASC), + INDEX "webhook_response_webhookId" ((data->>'webhookId':::STRING) ASC), + INDEX "webhook_response_eventId" ((data->>'eventId':::STRING) ASC) +); +CREATE TABLE public.multistream_target ( + id VARCHAR(128) NOT NULL, + data JSONB NULL, + CONSTRAINT multistream_target_pkey PRIMARY KEY (id ASC), + INDEX "multistream_target_userId" ((data->>'userId':::STRING) ASC) +); +CREATE TABLE public.task ( + id VARCHAR(128) NOT NULL, + data JSONB NULL, + CONSTRAINT task_pkey PRIMARY KEY (id ASC), + INDEX task_id ((data->>'id':::STRING) ASC), + INDEX "task_inputAssetId" ((data->>'inputAssetId':::STRING) ASC), + INDEX "task_outputAssetId" ((data->>'outputAssetId':::STRING) ASC), + INDEX "task_userId" ((data->>'userId':::STRING) ASC), + INDEX "task_pendingTasks" ((data->>'userId':::STRING) ASC, (COALESCE(((data->'status':::STRING)->>'updatedAt':::STRING)::INT8, 0:::INT8)) ASC, ((data->'status':::STRING)->>'phase':::STRING) ASC) +); +CREATE TABLE public.usage ( + id VARCHAR(128) NOT NULL, + data JSONB NULL, + CONSTRAINT usage_pkey PRIMARY KEY (id ASC), + UNIQUE INDEX usage_id ((data->>'id':::STRING) ASC) +); +CREATE TABLE public.signing_key ( + id VARCHAR(128) NOT NULL, + data JSONB NULL, + CONSTRAINT signing_key_pkey PRIMARY KEY (id ASC), + INDEX signing_key_id ((data->>'id':::STRING) ASC), + INDEX "signing_key_userId" ((data->>'userId':::STRING) ASC) +); +CREATE TABLE public.room ( + id VARCHAR(128) NOT NULL, + data JSONB NULL, + CONSTRAINT room_pkey PRIMARY KEY (id ASC), + INDEX "room_userId" ((data->>'userId':::STRING) ASC) +); +CREATE TABLE public.attestation ( + id VARCHAR(128) NOT NULL, + data JSONB NULL, + CONSTRAINT attestation_pkey PRIMARY KEY (id ASC) +); +CREATE TABLE public.object_store ( + id VARCHAR(128) NOT NULL, + data JSONB NULL, + CONSTRAINT object_store_pkey PRIMARY KEY (id ASC), + INDEX "object_store_userId" ((data->>'userId':::STRING) ASC) +); +CREATE TABLE public.password_reset_token ( + id VARCHAR(128) NOT NULL, + data JSONB NULL, + CONSTRAINT password_reset_token_pkey PRIMARY KEY (id ASC), + INDEX "password_reset_token_userId" ((data->>'userId':::STRING) ASC) +); +CREATE TABLE public.experiment ( + id VARCHAR(128) NOT NULL, + data JSONB NULL, + CONSTRAINT experiment_pkey PRIMARY KEY (id ASC), + UNIQUE INDEX experiment_name ((data->>'name':::STRING) ASC), + INDEX "experiment_userId" ((data->>'userId':::STRING) ASC), + INVERTED INDEX "experiment_audienceUserIds" ((data->'audienceUserIds':::STRING)) +); +CREATE TABLE public.regions ( + id VARCHAR(128) NOT NULL, + data JSONB NULL, + CONSTRAINT regions_pkey PRIMARY KEY (id ASC), + UNIQUE INDEX regions_region ((data->>'region':::STRING) ASC) +); +CREATE TABLE public.api_token ( + id VARCHAR(128) NOT NULL, + data JSONB NULL, + CONSTRAINT api_token_pkey PRIMARY KEY (id ASC), + INDEX "api_token_userId" ((data->>'userId':::STRING) ASC) +); diff --git a/cmd/catalyst/config/mist_config.go b/cmd/catalyst/config/mist_config.go new file mode 100644 index 000000000..5438bc27d --- /dev/null +++ b/cmd/catalyst/config/mist_config.go @@ -0,0 +1,342 @@ +package config + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" +) + +const ( + advertisePort = "9935" + catalystAPIPort = "7979" + catalystAPIInternalPort = "7878" + boxPort = "8888" +) + +type Account struct { + Password string `json:"password"` +} + +type Bandwidth struct { + Exceptions []string `json:"exceptions"` + Limit int `json:"limit"` +} + +type ICEServer struct { + URLs string `json:"urls,omitempty"` + Username string `json:"username,omitempty"` + Credential string `json:"credential,omitempty"` +} +type Protocol struct { + Connector string `json:"connector"` + RetryJoin string `json:"retry-join,omitempty"` + Advertise string `json:"advertise,omitempty"` + RPCAddr string `json:"rpc-addr,omitempty"` + RedirectPrefixes string `json:"redirect-prefixes,omitempty"` + Debug string `json:"debug,omitempty"` + HTTPAddr string `json:"http-addr,omitempty"` + HTTPAddrInternal string `json:"http-internal-addr,omitempty"` + Broadcaster bool `json:"broadcaster,omitempty"` + Orchestrator bool `json:"orchestrator,omitempty"` + Transcoder bool `json:"transcoder,omitempty"` + HTTPRPCAddr string `json:"httpAddr,omitempty"` + OrchAddr string `json:"orchAddr,omitempty"` + ServiceAddr string `json:"serviceAddr,omitempty"` + CliAddr string `json:"cliAddr,omitempty"` + RtmpAddr string `json:"rtmpAddr,omitempty"` + SourceOutput string `json:"source-output,omitempty"` + Catabalancer string `json:"catabalancer,omitempty"` + BaseStreamName string `json:"base-stream-name,omitempty"` + Broadcasters string `json:"broadcasters,omitempty"` + JWTAudience string `json:"jwt-audience,omitempty"` + JWTSecret string `json:"jwt-secret,omitempty"` + OwnRegion string `json:"own-region,omitempty"` + PostgresURL string `json:"postgres-url,omitempty"` + RecordCatalystObjectStoreID string `json:"recordCatalystObjectStoreId,omitempty"` + VODCatalystObjectStoreID string `json:"vodCatalystObjectStoreId,omitempty"` + VODCatalystPrivateAssetsObjectStore string `json:"vodCatalystPrivateAssetsObjectStore,omitempty"` + VODObjectStoreID string `json:"vodObjectStoreId,omitempty"` + Port string `json:"port,omitempty"` + StreamInfoService bool `json:"stream-info-service,omitempty"` + V string `json:"v,omitempty"` + OwnBaseURL string `json:"own-base-url,omitempty"` + Node string `json:"node,omitempty"` + Monitor bool `json:"monitor,omitempty"` + MetricsPerStream bool `json:"metricsPerStream,omitempty"` + MetricsClientIP bool `json:"metricsClientIP,omitempty"` + CatalystURL string `json:"catalyst-url,omitempty"` + BroadcasterURL string `json:"broadcaster-url,omitempty"` + DisableBigquery bool `json:"disable-bigquery,omitempty"` + BindHost string `json:"bindhost,omitempty"` + BalancerArgs string `json:"balancer-args,omitempty"` + APIServer string `json:"api-server,omitempty"` + APIToken string `json:"api-token,omitempty"` + CatalystSecret string `json:"catalyst-secret,omitempty"` + PubHost string `json:"pubhost,omitempty"` + Tags string `json:"tags,omitempty"` + Ingest string `json:"ingest,omitempty"` + CORSJWTAllowlist string `json:"cors-jwt-allowlist,omitempty"` + LivepeerAccessToken string `json:"livepeer-access-token,omitempty"` + AuthWebhookURL string `json:"authWebhookUrl,omitempty"` + Network string `json:"network,omitempty"` + EthURL string `json:"ethUrl,omitempty"` + EthKeystorePath string `json:"ethKeystorePath,omitempty"` + EthPassword string `json:"ethPassword,omitempty"` + MaxTicketEV string `json:"maxTicketEV,omitempty"` + MaxTotalEV string `json:"maxTotalEV,omitempty"` + MaxPricePerUnit string `json:"maxPricePerUnit,omitempty"` + + ICEServers []ICEServer `json:"iceservers,omitempty"` + // And finally, four ways to spell the same thing: + AMQPURL string `json:"amqp-url,omitempty"` + AMQPURI string `json:"amqp-uri,omitempty"` + MetadataQueueURI string `json:"metadataQueueUri,omitempty"` + RabbitMQURI string `json:"rabbitmq-uri,omitempty"` +} + +type Config struct { + Accesslog string `json:"accesslog,omitempty"` + Controller *struct { + Interface *string `json:"interface,omitempty"` + Port *string `json:"port,omitempty"` + Username *string `json:"username,omitempty"` + } `json:"controller,omitempty"` + Debug interface{} `json:"debug,omitempty"` + DefaultStream string `json:"defaultStream,omitempty"` + Limits interface{} `json:"limits,omitempty"` + Location *struct { + Lat float64 `json:"lat,omitempty"` + Lon float64 `json:"lon,omitempty"` + Name string `json:"name,omitempty"` + } `json:"location,omitempty"` + Prometheus string `json:"prometheus,omitempty"` + Protocols []*Protocol `json:"protocols,omitempty"` + ServerID interface{} `json:"serverid,omitempty"` + SessionInputMode int `json:"sessionInputMode"` + SessionOutputMode int `json:"sessionOutputMode"` + SessionStreamInfoMode int `json:"sessionStreamInfoMode"` + SessionUnspecifiedMode int `json:"sessionUnspecifiedMode"` + SessionViewerMode int `json:"sessionViewerMode"` + SidMode int `json:"sidMode,omitempty"` + TknMode int `json:"tknMode,omitempty"` + Triggers map[string][]Trigger `json:"triggers"` + Trustedproxy []string `json:"trustedproxy,omitempty"` +} + +type Stream struct { + Name string `json:"name"` + Processes []*Process `json:"processes,omitempty"` + Realtime bool `json:"realtime,omitempty"` + Source string `json:"source"` + StopSessions bool `json:"stop_sessions"` + DVR int `json:"DVR"` + MaxKeepAway int `json:"maxkeepaway"` + SegmentSize int `json:"segmentsize"` +} + +type Process struct { + Debug int `json:"debug,omitempty"` + HardcodedBroadcasters string `json:"hardcoded_broadcasters,omitempty"` + Exec string `json:"exec,omitempty"` + Leastlive bool `json:"leastlive,omitempty"` + Process string `json:"process,omitempty"` + TargetProfiles []TargetProfile `json:"target_profiles,omitempty"` + AccessToken string `json:"access_token,omitempty"` + CustomURL string `json:"custom_url,omitempty"` + ExitUnmask bool `json:"exit_unmask"` + TrackInhibit string `json:"track_inhibit,omitempty"` + TrackSelect string `json:"track_select,omitempty"` + XLSPName string `json:"x-LSP-name,omitempty"` +} + +type TargetProfile struct { + Bitrate int `json:"bitrate"` + Fps int `json:"fps"` + Height int `json:"height"` + Name string `json:"name"` + Profile string `json:"profile"` + Width int `json:"width"` + XLSPName string `json:"x-LSP-name"` +} + +type Trigger struct { + Handler string `json:"handler"` + Sync bool `json:"sync"` + Default string `json:"default"` + Streams []string `json:"streams"` +} + +type MistConfig struct { + Account map[string]Account `json:"account"` + Autopushes interface{} `json:"autopushes"` + Bandwidth *Bandwidth `json:"bandwidth,omitempty"` + Config Config `json:"config"` + PushSettings struct { + Maxspeed interface{} `json:"maxspeed"` + Wait interface{} `json:"wait"` + } `json:"push_settings"` + Streams map[string]*Stream `json:"streams"` + UISettings interface{} `json:"ui_settings,omitempty"` + ExtWriters []any `json:"extwriters"` +} + +func DefaultMistConfig(host, sourceOutput string) MistConfig { + return MistConfig{ + Account: map[string]Account{ + "test": { + Password: "098f6bcd4621d373cade4e832627b4f6", + }, + }, + Bandwidth: &Bandwidth{ + Exceptions: []string{}, + }, + Config: Config{ + Accesslog: "LOG", + Prometheus: "koekjes", + Protocols: []*Protocol{ + {Connector: "AAC"}, + {Connector: "CMAF"}, + {Connector: "DTSC"}, + {Connector: "EBML"}, + {Connector: "FLV"}, + {Connector: "H264"}, + {Connector: "HDS"}, + {Connector: "HLS"}, + {Connector: "HTTP"}, + {Connector: "HTTPTS"}, + {Connector: "JSON"}, + {Connector: "MP3"}, + {Connector: "MP4"}, + {Connector: "OGG"}, + {Connector: "RTMP"}, + {Connector: "RTSP"}, + {Connector: "SDP"}, + {Connector: "SRT"}, + {Connector: "TSSRT"}, + {Connector: "WAV"}, + {Connector: "WebRTC"}, + { + Connector: "livepeer", + Broadcaster: true, + CliAddr: "127.0.0.1:7935", + HTTPRPCAddr: "127.0.0.1:8935", + OrchAddr: "127.0.0.1:8936", + RtmpAddr: "127.0.0.1:1936", + }, + { + Connector: "livepeer", + Orchestrator: true, + Transcoder: true, + CliAddr: "127.0.0.1:7936", + ServiceAddr: "127.0.0.1:8936", + }, + { + Connector: "livepeer-catalyst-api", + SourceOutput: sourceOutput, + Advertise: fmt.Sprintf("%s:%s", host, advertisePort), + HTTPAddr: fmt.Sprintf("0.0.0.0:%s", catalystAPIPort), + HTTPAddrInternal: fmt.Sprintf("0.0.0.0:%s", catalystAPIInternalPort), + RedirectPrefixes: "stream", + Debug: "6", + Catabalancer: "enabled", + }, + }, + SessionInputMode: 14, + SessionOutputMode: 14, + SessionStreamInfoMode: 1, + SessionUnspecifiedMode: 0, + SessionViewerMode: 14, + SidMode: 0, + Trustedproxy: []string{}, + Triggers: map[string][]Trigger{ + "STREAM_SOURCE": { + { + Handler: "http://127.0.0.1:7878/STREAM_SOURCE", + Sync: true, + Default: "push://", + Streams: []string{}, + }, + }, + "PUSH_END": { + { + Handler: "http://127.0.0.1:7878/api/mist/trigger", + Sync: false, + }, + }, + "RECORDING_END": { + { + Handler: "http://127.0.0.1:7878/api/mist/trigger", + Sync: false, + }, + }, + }, + }, + Streams: map[string]*Stream{ + "stream": { + Name: "stream", + Realtime: false, + Source: "push://", + StopSessions: false, + }, + }, + } +} + +func DefaultMistConfigWithLivepeerProcess(host, sourceOutput string) MistConfig { + mc := DefaultMistConfig(host, sourceOutput) + s := mc.Streams["stream"] + s.Processes = []*Process{ + { + Debug: 5, + HardcodedBroadcasters: "[{\"address\":\"http://127.0.0.1:8935\"}]", + Leastlive: true, + Process: "livepeer", + TargetProfiles: []TargetProfile{ + {Bitrate: 400000, + Fps: 30, + Height: 144, + Name: "P144p30fps16x9", + Width: 256, + XLSPName: "", + }, + }, + }, + } + mc.Streams["stream"] = s + return mc +} + +func (m *MistConfig) string() (string, error) { + s, err := json.Marshal(m) + if err != nil { + return "", err + } + return string(s), nil +} + +func (m *MistConfig) toFile(file *os.File) error { + str, err := m.string() + if err != nil { + return err + } + if _, err = file.Write([]byte(str)); err != nil { + return err + } + return nil +} + +func (m *MistConfig) ToTmpFile(dir string) (string, error) { + tmpFile, err := ioutil.TempFile(dir, "mist-config-*.json") + if err != nil { + return "", err + } + defer tmpFile.Close() + + err = m.toFile(tmpFile) + if err != nil { + os.Remove(tmpFile.Name()) + return "", err + } + return tmpFile.Name(), nil +} diff --git a/go.mod b/go.mod index 6eb38aebb..1e3139908 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,9 @@ replace github.com/testcontainers/testcontainers-go v0.26.0 => github.com/lefina require ( github.com/ProtonMail/gopenpgp/v2 v2.4.10 + github.com/doug-martin/goqu/v9 v9.19.0 github.com/golang/glog v1.1.2 + github.com/google/go-cmp v0.6.0 github.com/livepeer/stream-tester v0.12.30-0.20230823234013-5cfb4bbcf27d github.com/magicsong/color-glog v0.0.1 github.com/minio/minio-go/v7 v7.0.46 diff --git a/go.sum b/go.sum index 3e3026c13..8231e56d9 100644 --- a/go.sum +++ b/go.sum @@ -66,6 +66,8 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= +github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/Microsoft/hcsshim v0.11.4 h1:68vKo2VN8DE9AdN4tnkWnmdhqdbpUFM8OF3Airm7fz8= @@ -129,6 +131,7 @@ github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/denisenkom/go-mssqldb v0.10.0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0= github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk= @@ -139,6 +142,8 @@ github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKoh github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/doug-martin/goqu/v9 v9.19.0 h1:PD7t1X3tRcUiSdc5TEyOFKujZA5gs3VSA7wxSvBx7qo= +github.com/doug-martin/goqu/v9 v9.19.0/go.mod h1:nf0Wc2/hV3gYK9LiyqIrzBEVGlI8qW3GuDCEobC4wBQ= github.com/dsnet/golib/memfile v1.0.0 h1:J9pUspY2bDCbF9o+YGwcf3uG6MdyITfh/Fk3/CaEiFs= github.com/dsnet/golib/memfile v1.0.0/go.mod h1:tXGNW9q3RwvWt1VV2qrRKlSSz0npnh12yftCSCy2T64= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= @@ -175,11 +180,13 @@ github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= +github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= github.com/golang/glog v1.1.2 h1:DVjP2PbBOzHyzA+dn3WhHIq4NdVu3Q+pvivFICf/7fo= @@ -338,6 +345,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lefinal/testcontainers-go v0.0.0-20231107224233-ca049655293f h1:D+6lSv7Hss2GEpjVYHg+v4Mabr8VXqJ49zJvsUV/x0Q= github.com/lefinal/testcontainers-go v0.0.0-20231107224233-ca049655293f/go.mod h1:C1b+AP3uPSlG6aNY8icPePlED0bHR7sSTKQDh23oCdA= +github.com/lib/pq v1.10.1/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/livepeer/catalyst-api v0.1.1 h1:WP4rHH88b+lsxo33wPCjl0yvqVDNyxkleZH1sA0M5GE= github.com/livepeer/catalyst-api v0.1.1/go.mod h1:d6XPE9ehhCutWhCqqcmlYqQa+e9bf3Ke92x+gRZlzoQ= github.com/livepeer/go-api-client v0.4.7 h1:pJd0Ba7TtDJDUEOiPBx9KmLm+fIB8GRbAurd3lsZUVY= @@ -374,6 +383,7 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-sqlite3 v1.14.7/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= @@ -502,6 +512,7 @@ github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVs github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -548,6 +559,7 @@ go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= diff --git a/manifest.yaml b/manifest.yaml index 0a239bc27..d45184e08 100644 --- a/manifest.yaml +++ b/manifest.yaml @@ -16,10 +16,15 @@ box: windows-arm64: livepeer-analyzer-windows-arm64.zip - name: api strategy: - download: github - project: livepeer/studio - commit: f51f0e62cf50a9d54b01c265da4106d45d78df4b - release: v0.16.2 + download: bucket + project: studio + commit: 085d87dec71bee6a30c15356b24ece9e6c8cdbf9 + release: master + srcFilenames: + darwin-amd64: livepeer-api-darwin-amd64.tar.gz + darwin-arm64: livepeer-api-darwin-arm64.tar.gz + linux-amd64: livepeer-api-linux-amd64.tar.gz + linux-arm64: livepeer-api-linux-arm64.tar.gz - name: catalyst-api strategy: download: bucket diff --git a/scripts/cockroach-dump b/scripts/cockroach-dump new file mode 100755 index 000000000..9ac9e94fe --- /dev/null +++ b/scripts/cockroach-dump @@ -0,0 +1,17 @@ +#!/bin/bash + +set -euo pipefail + +workdir="$(mktemp -d)" +cockroach sql \ + --url 'postgresql://root@localhost:5432/defaultdb?sslmode=disable' \ + --execute "select format('copy %I to stdout with csv',tablename) from pg_tables where schemaname='public';" \ + --format=raw \ + | grep copy \ + > "$workdir/commands" + +while IFS="" read -r line || [ -n "$line" ]; do + cockroach sql \ + --url 'postgresql://root@localhost:5432/defaultdb?sslmode=disable' \ + --execute "$line" +done < "$workdir/commands" diff --git a/scripts/livepeer-cockroach b/scripts/livepeer-cockroach index c3493a5ed..4f189052c 100755 --- a/scripts/livepeer-cockroach +++ b/scripts/livepeer-cockroach @@ -11,14 +11,10 @@ if [[ $* == *-j* ]]; then exit 0 fi -if [[ ! -d /data/cockroach ]]; then - ( - echo "no database detected, pulling snapshot" - cd /data - curl -L $COCKROACH_DB_SNAPSHOT -o snapshot.tar.gz - tar --no-same-owner -xzf snapshot.tar.gz - rm snapshot.tar.gz - ) -fi +bash <