feat: support atproto source and bluesky worker (#662)

Co-authored-by: brucexc <[email protected]>

cmd/main.go

@@ -324,6 +324,7 @@ func findModuleByID(configFile *config.File, workerID string) (*config.Module, e
 		return module, nil
 	}
 
+	// Search in RSS components
 	if module, found := findInComponent([]*config.Module{configFile.Component.RSS}); found {
 		return module, nil
 	}

config/config_test.go

@@ -70,11 +70,17 @@ component:
           access_key: abc
           access_code: def
   federated:
-      network: mastodon
-      worker: core
+    - network: mastodon
+      worker: mastodon
       endpoint: mastodon
       parameters:
         relay_url_list: ["https://relay.fedi.buzz/instance/mastodon.social"]
+    - network: bluesky
+      worker: bluesky
+      endpoint: bluesky
+      parameters:
+        username: user
+        password: pass
 
   decentralized:
     - network: ethereum
@@ -118,7 +124,7 @@ component:
     }
   },
   "database": {
-	"coverage_period": 3,
+    "coverage_period": 3,
     "uri": "postgres://postgres@localhost:5432/postgres"
   },
   "stream": {
@@ -160,12 +166,21 @@ component:
   "federated": [
     {
       "network": "mastodon",
-      "worker": "core",
+      "worker": "mastodon",
       "endpoint": "mastodon",
       "parameters": {
-		  "relay_url_list": [
-			"https://relay.fedi.buzz/instance/mastodon.social"
-		  ]
+          "relay_url_list": [
+            "https://relay.fedi.buzz/instance/mastodon.social"
+          ]
+      }
+    },
+    {
+      "network": "bluesky",
+      "worker": "bluesky",
+      "endpoint": "bluesky",
+      "parameters": {
+        "username": "user",
+        "password": "pass"
       }
     }
   ],
@@ -249,12 +264,21 @@ access_code = "def"
 
 [[component.federated]]
 network = "mastodon"
-worker = "core"
+worker = "mastodon"
 endpoint = "mastodon"
 
 [component.federated.parameters]
 relay_url_list = ["https://relay.fedi.buzz/instance/mastodon.social"]
 
+[[component.federated]]
+network = "bluesky"
+worker = "bluesky"
+endpoint = "bluesky"
+
+[component.federated.parameters]
+username = "user"
+password = "pass"
+
 [[component.decentralized]]
 network = "ethereum"
 worker = "core"
@@ -323,7 +347,7 @@ var configFileExpected = &File{
 		Federated: []*Module{
 			{
 				Network:    network.Mastodon,
-				Worker:     federated.Core,
+				Worker:     federated.Mastodon,
 				EndpointID: "mastodon",
 				Endpoint: Endpoint{
 					URL: "https://newdomain.ngrok.app",
@@ -336,6 +360,18 @@ var configFileExpected = &File{
 					// "port":           8181, //ToDo: add port here and other places
 				},
 			},
+			{
+				Network:    network.Bluesky,
+				Worker:     federated.Bluesky,
+				EndpointID: "bluesky",
+				Endpoint: Endpoint{
+					URL: "bluesky",
+				},
+				Parameters: &Parameters{
+					"username": "user",
+					"password": "pass",
+				},
+			},
 		},
 		Decentralized: []*Module{
 			{

deploy/config.example.yaml

@@ -85,13 +85,21 @@ component:
       parameters:
         # `concurrent_block_requests` is used to specify the number of concurrent block requests.
         concurrent_block_requests: 2
-  # `federated` network type includes workers indexing data from federated networks such as ActivityPub.
+  # `federated` network type includes workers indexing data from federated networks such as ActivityPub, Atprotocol.
   federated:
     # mastodon
-    - id: mastodon-core
+    - id: mastodon-mastodon
       network: mastodon
-      worker: core
+      worker: mastodon
       endpoint: mastodon
       parameters:
         relay_url_list: [ "https://relay.fedi.buzz/instance/mastodon.social"]
         port: 8181
+    # bluesky
+    - id: bluesky-bluesky
+      network: bluesky
+      worker: bluesky
+      endpoint:
+      parameters:
+        username:
+        password:

docs/api.yaml

@@ -7,7 +7,7 @@ info:
     name: MIT
     url: https://opensource.org/licenses/MIT
 servers:
-  - url: http://localhost:8080
+  - url: http://localhost:80
     description: Local development server
 tags:
   - name: RSS

docs/schemas/FederatedPlatform.yaml

@@ -2,6 +2,7 @@ type: string
 enum:
   - Unknown
   - Mastodon
+  - Bluesky
 x-go-type: federated.Platform
 x-go-type-skip-optional-pointer: true
 x-go-type-import:

docs/schemas/FederatedWorker.yaml

@@ -1,6 +1,7 @@
 type: string
 enum:
-  - core
+  - mastodon
+  - bluesky
 x-go-type: federated.Worker
 x-go-type-skip-optional-pointer: true
 x-go-type-import:

go.mod

@@ -1,10 +1,11 @@
 module github.com/rss3-network/node
 
-go 1.22
+go 1.22.4
 
-toolchain go1.22.1
-
-replace github.com/chenzhuoyu/iasm => github.com/cloudwego/iasm v0.2.0
+replace (
+	github.com/chenzhuoyu/iasm => github.com/cloudwego/iasm v0.2.0
+	github.com/cockroachdb/pebble => github.com/cockroachdb/pebble v0.0.0-20230928194634-aa077af62593
+)
 
 require (
 	github.com/JohannesKaufmann/html-to-markdown v1.6.0
@@ -57,22 +58,27 @@ require (
 
 require (
 	github.com/adrianbrad/psqldocker v1.2.1
+	github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de
+	github.com/bluesky-social/indigo v0.0.0-20241212185500-749cc07bf4eb
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0
 	github.com/emirpasic/gods v1.18.1
 	github.com/getkin/kin-openapi v0.128.0
 	github.com/go-fed/httpsig v1.1.0
 	github.com/go-redsync/redsync/v4 v4.13.0
 	github.com/google/uuid v1.6.0
+	github.com/gorilla/websocket v1.5.3
 	github.com/grafana/pyroscope-go v1.2.0
 	github.com/jaytaylor/html2text v0.0.0-20230321000545-74c2419ad056
 	github.com/mitchellh/mapstructure v1.5.0
 	github.com/oapi-codegen/oapi-codegen/v2 v2.4.1
 	github.com/redis/rueidis v1.0.51
 	github.com/redis/rueidis/rueidiscompat v1.0.51
-	github.com/rss3-network/protocol-go v0.5.14
+	github.com/reiver/go-bsky v0.0.0-20240906205655-8c7fadb4f3bb
+	github.com/rss3-network/protocol-go v0.5.16
 	github.com/spf13/afero v1.11.0
 	github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7
 	github.com/vincent-petithory/dataurl v1.0.0
+	github.com/whyrusleeping/cbor-gen v0.2.1-0.20241030202151-b7a6831be65e
 	golang.org/x/net v0.32.0
 )
 
@@ -83,12 +89,19 @@ require (
 	github.com/Microsoft/go-winio v0.6.2 // indirect
 	github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect
 	github.com/PuerkitoBio/goquery v1.9.2 // indirect
+	github.com/RussellLuo/slidingwindow v0.0.0-20200528002341-535bb99d338b // indirect
 	github.com/andybalholm/cascadia v1.3.2 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bits-and-blooms/bitset v1.10.0 // indirect
 	github.com/btcsuite/btcd/btcec/v2 v2.3.2 // indirect
 	github.com/btcsuite/btcd/chaincfg/chainhash v1.0.2 // indirect
+	github.com/carlmjohnson/versioninfo v0.22.5 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/cockroachdb/errors v1.11.3 // indirect
+	github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
+	github.com/cockroachdb/pebble v1.1.2 // indirect
+	github.com/cockroachdb/redact v1.1.5 // indirect
+	github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect
 	github.com/consensys/bavard v0.1.13 // indirect
 	github.com/consensys/gnark-crypto v0.12.1 // indirect
 	github.com/containerd/continuity v0.4.3 // indirect
@@ -104,6 +117,7 @@ require (
 	github.com/ethereum/c-kzg-4844 v0.4.0 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
+	github.com/getsentry/sentry-go v0.27.0 // indirect
 	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-ole/go-ole v1.3.0 // indirect
@@ -117,25 +131,53 @@ require (
 	github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f // indirect
 	github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
-	github.com/gorilla/websocket v1.5.0 // indirect
 	github.com/grafana/pyroscope-go/godeltaprof v0.1.8 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.23.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
+	github.com/hashicorp/go-retryablehttp v0.7.5 // indirect
+	github.com/hashicorp/golang-lru v1.0.2 // indirect
+	github.com/hashicorp/golang-lru/arc/v2 v2.0.6 // indirect
+	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/holiman/uint256 v1.2.4 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/invopop/yaml v0.3.1 // indirect
+	github.com/ipfs/bbloom v0.0.4 // indirect
+	github.com/ipfs/go-block-format v0.2.0 // indirect
+	github.com/ipfs/go-blockservice v0.5.2 // indirect
+	github.com/ipfs/go-datastore v0.6.0 // indirect
+	github.com/ipfs/go-ipfs-blockstore v1.3.1 // indirect
+	github.com/ipfs/go-ipfs-ds-help v1.1.1 // indirect
+	github.com/ipfs/go-ipfs-exchange-interface v0.2.1 // indirect
+	github.com/ipfs/go-ipfs-util v0.0.3 // indirect
+	github.com/ipfs/go-ipld-cbor v0.1.0 // indirect
+	github.com/ipfs/go-ipld-format v0.6.0 // indirect
+	github.com/ipfs/go-ipld-legacy v0.2.1 // indirect
+	github.com/ipfs/go-libipfs v0.7.0 // indirect
+	github.com/ipfs/go-log v1.0.5 // indirect
+	github.com/ipfs/go-log/v2 v2.5.1 // indirect
+	github.com/ipfs/go-merkledag v0.11.0 // indirect
+	github.com/ipfs/go-metrics-interface v0.0.1 // indirect
+	github.com/ipfs/go-verifcid v0.0.3 // indirect
+	github.com/ipld/go-car v0.6.2 // indirect
+	github.com/ipld/go-car/v2 v2.13.1 // indirect
+	github.com/ipld/go-codec-dagpb v1.6.0 // indirect
+	github.com/ipld/go-ipld-prime v0.21.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
 	github.com/jackc/pgx/v5 v5.7.1 // indirect
 	github.com/jackc/puddle/v2 v2.2.2 // indirect
+	github.com/jbenet/goprocess v0.1.4 // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/jinzhu/now v1.1.5 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/compress v1.17.11 // indirect
-	github.com/klauspost/cpuid/v2 v2.2.5 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.7 // indirect
+	github.com/kr/pretty v0.3.1 // indirect
+	github.com/kr/text v0.2.0 // indirect
 	github.com/labstack/gommon v0.4.2 // indirect
 	github.com/leodido/go-urn v1.4.0 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
@@ -162,16 +204,26 @@ require (
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.1.0 // indirect
 	github.com/opencontainers/runc v1.1.12 // indirect
+	github.com/opentracing/opentracing-go v1.2.0 // indirect
 	github.com/ory/dockertest/v3 v3.10.0 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.2 // indirect
 	github.com/perimeterx/marshmallow v1.1.5 // indirect
+	github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 // indirect
 	github.com/pierrec/lz4/v4 v4.1.21 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+	github.com/polydawn/refmt v0.89.1-0.20221221234430-40501e09de1f // indirect
 	github.com/prometheus/client_model v0.6.1 // indirect
 	github.com/prometheus/common v0.60.1 // indirect
 	github.com/prometheus/procfs v0.15.1 // indirect
+	github.com/reiver/go-atproto v0.0.0-20240906195827-a1832b0df821 // indirect
+	github.com/reiver/go-cast v0.0.0-20240904124821-be0103637c3f // indirect
+	github.com/reiver/go-erorr v0.0.0-20240801233437-8cbde6d1fa3f // indirect
+	github.com/reiver/go-fallback v0.0.0-20240906145154-1ce9eadf06a8 // indirect
+	github.com/reiver/go-maps v0.0.0-20240906190342-93be57f28be1 // indirect
+	github.com/reiver/go-reg v0.0.0-20240906195701-6e62f43c2835 // indirect
 	github.com/rivo/uniseg v0.2.0 // indirect
+	github.com/rogpeppe/go-internal v1.13.1 // indirect
 	github.com/sagikazarmark/locafero v0.4.0 // indirect
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
 	github.com/sethvargo/go-retry v0.3.0 // indirect
@@ -193,12 +245,16 @@ require (
 	github.com/valyala/fasttemplate v1.2.2 // indirect
 	github.com/vektah/gqlparser/v2 v2.5.11 // indirect
 	github.com/vmware-labs/yaml-jsonpath v0.3.2 // indirect
+	github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 // indirect
 	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
 	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
 	github.com/yusufpapurcu/wmi v1.2.3 // indirect
+	gitlab.com/yawning/secp256k1-voi v0.0.0-20230925100816-f2616030848b // indirect
+	gitlab.com/yawning/tuplehash v0.0.0-20230713102510-df83abbf9a02 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.50.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.3.1 // indirect
+	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/crypto v0.30.0 // indirect
 	golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
@@ -207,6 +263,7 @@ require (
 	golang.org/x/text v0.21.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
 	golang.org/x/tools v0.24.0 // indirect
+	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20241104194629-dd2ea8efbc28 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28 // indirect
 	google.golang.org/grpc v1.67.1 // indirect