use cluster name instead of stand alone region config in object node

Signed-off-by: Mofei Zhang <[email protected]>
shuoranliu · Jan 10, 2020 · 0a53ac8 · 0a53ac8
1 parent 455a23d
commit 0a53ac8
Show file tree

Hide file tree

Showing 16 changed files with 98 additions and 65 deletions.
diff --git a/client/fuse.go b/client/fuse.go
@@ -277,7 +277,9 @@ func parseMountOption(cfg *config.Config) (*cfs.MountOption, error) {
 	opt.Authenticate = cfg.GetBool(proto.Authenticate)
 	if opt.Authenticate {
 		opt.TicketMess.ClientKey = cfg.GetString(proto.ClientKey)
-		opt.TicketMess.TicketHost = cfg.GetString(proto.TicketHost)
+		ticketHostConfig := cfg.GetString(proto.TicketHost)
+		ticketHosts := strings.Split(ticketHostConfig, ",")
+		opt.TicketMess.TicketHosts = ticketHosts
 		opt.TicketMess.EnableHTTPS = cfg.GetBool(proto.EnableHTTPS)
 		if opt.TicketMess.EnableHTTPS {
 			opt.TicketMess.CertFile = cfg.GetString(proto.CertFile)

diff --git a/clientv2/main.go b/clientv2/main.go
@@ -269,7 +269,9 @@ func parseMountOption(cfg *config.Config) (*cfs.MountOption, error) {
 	opt.Authenticate = cfg.GetBool(proto.Authenticate)
 	if opt.Authenticate {
 		opt.TicketMess.ClientKey = cfg.GetString(proto.ClientKey)
-		opt.TicketMess.TicketHost = cfg.GetString(proto.TicketHost)
+		ticketHostConfig := cfg.GetString(proto.TicketHost)
+		ticketHosts := strings.Split(ticketHostConfig, ",")
+		opt.TicketMess.TicketHosts = ticketHosts
 		opt.TicketMess.EnableHTTPS = cfg.GetBool(proto.EnableHTTPS)
 		if opt.TicketMess.EnableHTTPS {
 			opt.TicketMess.CertFile = cfg.GetString(proto.CertFile)

diff --git a/cmd/cfg/objectnode.json b/cmd/cfg/objectnode.json
@@ -1,15 +1,19 @@
 {
   "role": "objectnode",
-  "domains": [
-    "object.cfs.local"
-  ],
+  "hosts": "127.0.0.1",
+  "logDir": "/cfs/log/",
+  "logLevel": "debug",
   "listen": 80,
   "masters": [
-    "master1.cfs.local:80",
-    "master2.cfs.local:80",
-    "master3.cfs.local:80"
+    "192.168.0.11:17010",
+    "192.168.0.12:17010",
+    "192.168.0.13:17010"
+  ],
+  "authNodes": [
+    "192.168.0.14:8080",
+    "192.168.0.15:8081",
+    "192.168.0.16:8082"
   ],
-  "logLevel": "info",
-  "logDir": "/export/Logs/objectnode",
-  "region": "cn_bj"
+  "enableHTTPS": "false",
+  "certFile": ""
 }
diff --git a/datanode/server.go b/datanode/server.go
@@ -191,11 +191,12 @@ func (s *DataNode) parseConfig(cfg *config.Config) (err error) {
 		return fmt.Errorf("Err:port must string")
 	}
 	s.port = port
-	if len(cfg.GetArray(ConfigKeyMasterAddr)) == 0 {
-		return fmt.Errorf("Err:masterAddr unavalid")
+	masters := cfg.GetStringSlice(ConfigKeyMasterAddr)
+	if len(masters) == 0 {
+		return fmt.Errorf("invliad config %s", ConfigKeyMasterAddr)
 	}
-	for _, ip := range cfg.GetArray(ConfigKeyMasterAddr) {
-		MasterClient.AddNode(ip.(string))
+	for _, master := range masters {
+		MasterClient.AddNode(master)
 	}
 	s.cellName = cfg.GetString(ConfigKeyCell)
 	if s.cellName == "" {
@@ -219,7 +220,7 @@ func (s *DataNode) startSpaceManager(cfg *config.Config) (err error) {
 	s.space.SetClusterID(s.clusterID)
 
 	var wg sync.WaitGroup
-	for _, d := range cfg.GetArray(ConfigKeyDisks) {
+	for _, d := range cfg.GetSlice(ConfigKeyDisks) {
 		log.LogDebugf("action[startSpaceManager] load disk raw config(%v).", d)
 
 		// format "PATH:RESET_SIZE

diff --git a/docker/conf/metanode.json b/docker/conf/metanode.json
@@ -16,10 +16,5 @@
         "192.168.0.11:17010",
         "192.168.0.12:17010",
         "192.168.0.13:17010"
-    ],
-    "masterAddr": [
-        "192.168.0.11:17010",
-        "192.168.0.12:17010",
-        "192.168.0.13:17010"
     ]
 }
diff --git a/docker/conf/objectnode.json b/docker/conf/objectnode.json
@@ -5,11 +5,15 @@
   "logLevel": "debug",
   "listen": 80,
   "masters": [
-      "192.168.0.11:17010",
-      "192.168.0.12:17010",
-      "192.168.0.13:17010"
+    "192.168.0.11:17010",
+    "192.168.0.12:17010",
+    "192.168.0.13:17010"
+  ],
+  "authNodes": [
+    "192.168.0.14:8080",
+    "192.168.0.15:8081",
+    "192.168.0.16:8082"
   ],
-  "authNodes": "192.168.0.14:8080,192.168.0.15:8081,192.168.0.16:8082",
   "enableHTTPS": "false",
   "certFile": ""
 }
diff --git a/docs/source/design/objectnode.rst b/docs/source/design/objectnode.rst
@@ -31,6 +31,7 @@ Based on our POSIX-compatible design. Every file operate request comes from obje
 .. csv-table::
     :header: "POSIX", "Object Storage"
 
+    "``Cluster``", "``Region``"
     "``Volume``", "``Bucket``"
     "``Path``", "``Key``"
 

diff --git a/docs/source/user-guide/objectnode.rst b/docs/source/user-guide/objectnode.rst
@@ -23,7 +23,6 @@ Object Node using `JSON` format configuration file.
 
    "role", "string", "Role of process and must be set to *objectnode*", "Yes"
    "listen", "string", "Listen and accept port of the server. Default: 80", "Yes"
-   "region", "string", "Region of this gateway. Used by S3-like interface signature validation. Default: cfs_default", "No"
    "domains", "string slice", "
    | Format: *DOMAIN*.
    | DOMAIN: Domain of S3-like interface which makes wildcard domain support", "No"
@@ -33,6 +32,10 @@ Object Node using `JSON` format configuration file.
    | Format: *HOST:PORT*.
    | HOST: Hostname, domain or IP address of master (resource manager).
    | PORT: port number which listened by this master", "Yes"
+   "authNodes", "string slice", "
+   | Format: *HOST:PORT*.
+   | HOST: Hostname, domain or IP address of AuthNode.
+   | PORT: port number which listened by this AuthNode", "Yes"
    "exporterPort", "string", "Port for monitor system", "No"
    "prof", "string", "Pprof port", "Yes"
 
@@ -44,7 +47,6 @@ Object Node using `JSON` format configuration file.
    {
         "role": "objectnode",
         "listen": 80,
-        "region": "test",
         "domains": [
             "object.cfs.local"
         ],

diff --git a/metanode/const.go b/metanode/const.go
@@ -132,7 +132,6 @@ const (
 	cfgMetadataDir       = "metadataDir"
 	cfgRaftDir           = "raftDir"
 	cfgMasterAddrs       = "masterAddrs" // will be deprecated
-	cfgMasterAddr        = "masterAddr"
 	cfgRaftHeartbeatPort = "raftHeartbeatPort"
 	cfgRaftReplicaPort   = "raftReplicaPort"
 	cfgTotalMem          = "totalMem"

diff --git a/metanode/metanode.go b/metanode/metanode.go
@@ -211,10 +211,12 @@ func (m *MetaNode) parseConfig(cfg *config.Config) (err error) {
 	log.LogInfof("[parseConfig] load raftHeartbeatPort[%v].", m.raftHeartbeatPort)
 	log.LogInfof("[parseConfig] load raftReplicatePort[%v].", m.raftReplicatePort)
 
-	addrs := cfg.GetArray(cfgMasterAddrs)
-	masters := make([]string, 0, len(addrs))
-	for _, addr := range addrs {
-		masters = append(masters, addr.(string))
+	masters := cfg.GetStringSlice(cfgMasterAddrs)
+	if len(masters) == 0 {
+		return config.NewIllegalConfigError(cfgMasterAddrs)
+	}
+	for _, master := range masters {
+		masters = append(masters, master)
 	}
 	masterClient = masterSDK.NewMasterClient(masters, false)
 	err = m.validConfig()

diff --git a/objectnode/fs_store_authnode.go b/objectnode/fs_store_authnode.go
@@ -37,7 +37,7 @@ type authnodeStore struct {
 	closeOnce   sync.Once
 }
 
-func newAuthStore(authKey, authNodes, certFile string, enableHTTPS bool) *authnodeStore {
+func newAuthStore(authNodes []string, authKey, certFile string, enableHTTPS bool) *authnodeStore {
 	authClient := authSDK.NewAuthClient(authNodes, enableHTTPS, certFile)
 	as := &authnodeStore{
 		authKey:     authKey,

diff --git a/objectnode/server.go b/objectnode/server.go
@@ -16,6 +16,8 @@ package objectnode
 
 import (
 	"context"
+	"github.com/chubaofs/chubaofs/proto"
+	"github.com/chubaofs/chubaofs/sdk/master"
 	"net/http"
 	"regexp"
 	"sync"
@@ -41,7 +43,6 @@ const (
 	configListen      = "listen"
 	configDomains     = "domains"
 	configMasters     = "masters"
-	configRegion      = "region"
 	configAuthnodes   = "authNodes"
 	configAuthkey     = "authKey"
 	configEnableHTTPS = "enableHTTPS"
@@ -51,7 +52,6 @@ const (
 // Default of configuration value
 const (
 	defaultListen = ":80"
-	defaultRegion = "cfs_default"
 )
 
 var (
@@ -64,6 +64,7 @@ type ObjectNode struct {
 	region     string
 	httpServer *http.Server
 	vm         VolumeManager
+	mc         *master.MasterClient
 	state      uint32
 	wg         sync.WaitGroup
 	authStore  *authnodeStore
@@ -100,7 +101,7 @@ func (o *ObjectNode) Sync() {
 	}
 }
 
-func (o *ObjectNode) parseConfig(cfg *config.Config) (err error) {
+func (o *ObjectNode) loadConfig(cfg *config.Config) (err error) {
 	// parse listen
 	listen := cfg.GetString(configListen)
 	if len(listen) == 0 {
@@ -111,53 +112,62 @@ func (o *ObjectNode) parseConfig(cfg *config.Config) (err error) {
 		return
 	}
 	o.listen = listen
+	log.LogInfof("loadConfig: setup config: %v(%v)", configListen, listen)
 
 	// parse domain
-	domainCfgs := cfg.GetArray(configDomains)
-	domains := make([]string, len(domainCfgs))
-	for i, domainCfg := range domainCfgs {
-		domains[i] = domainCfg.(string)
-	}
+	domains := cfg.GetStringSlice(configDomains)
 	o.domains = domains
+	log.LogInfof("loadConfig: setup config: %v(%v)", configDomains, domains)
 
 	// parse master config
-	masterCfgs := cfg.GetArray(configMasters)
-	masters := make([]string, len(masterCfgs))
-	for i, masterCfg := range masterCfgs {
-		masters[i] = masterCfg.(string)
+	masters := cfg.GetStringSlice(configMasters)
+	if len(masters) == 0 {
+		return config.NewIllegalConfigError(configMasters)
 	}
+	log.LogInfof("loadConfig: setup config: %v(%v)", configMasters, masters)
+
+	o.mc = master.NewMasterClient(masters, false)
 	o.vm = NewVolumeManager(masters)
 	o.vm.InitStore(new(xattrStore))
 
-	// parse region
-	region := cfg.GetString(configRegion)
-	if len(region) == 0 {
-		region = defaultRegion
+	//parse AuthNode info
+	authNodes := cfg.GetStringSlice(configAuthnodes)
+	if len(authNodes) == 0 {
+		return config.NewIllegalConfigError(configAuthnodes)
 	}
-	o.region = region
+	log.LogInfof("loadConfig: setup config: %v(%v)", configAuthnodes, authNodes)
 
-	//parse authnode info
-	authNodes := cfg.GetString(configAuthnodes)
 	enableHTTPS := cfg.GetBool(configEnableHTTPS)
+
 	certFile := cfg.GetString(configCertFile)
 	authKey := cfg.GetString(configAuthkey)
-	o.authStore = newAuthStore(authKey, authNodes, certFile, enableHTTPS)
+	log.LogInfof("loadConfig: setup config: %v(%v)", configAuthkey, authKey)
+
+	o.authStore = newAuthStore(authNodes, authKey, certFile, enableHTTPS)
 
 	return
 }
 
 func (o *ObjectNode) handleStart(cfg *config.Config) (err error) {
 	// parse config
-	if err = o.parseConfig(cfg); err != nil {
+	if err = o.loadConfig(cfg); err != nil {
+		return
+	}
+
+	// Get cluster info from master
+	var ci *proto.ClusterInfo
+	if ci, err = o.mc.AdminAPI().GetClusterInfo(); err != nil {
 		return
 	}
+	o.region = ci.Cluster
+	log.LogInfof("handleStart: get cluster information: region(%v)", o.region)
 
 	// start rest api
 	if err = o.startMuxRestAPI(); err != nil {
-		log.LogInfof("handleStart: start mux rest api fail, err(%v)", err)
+		log.LogInfof("handleStart: start rest api fail: err(%v)", err)
 		return
 	}
-	log.LogInfo("s3node start success")
+	log.LogInfo("object subsystem start success")
 	return
 }
 

diff --git a/sdk/auth/client.go b/sdk/auth/client.go
@@ -23,7 +23,6 @@ import (
 	"github.com/chubaofs/chubaofs/util/keystore"
 	"io/ioutil"
 	"net/http"
-	"strings"
 	"sync"
 	"time"
 
@@ -32,7 +31,6 @@ import (
 
 const (
 	requestTimeout       = 30 * time.Second
-	HostsSeparator       = ","
 	RequestMaxRetry      = 5
 	RequestSleepInterval = 100 * time.Millisecond
 )
@@ -52,9 +50,8 @@ func (c *AuthClient) API() *API {
 	}
 }
 
-func NewAuthClient(authnodeStr string, enableHTTPS bool, certFile string) *AuthClient {
-	authnodes := strings.Split(authnodeStr, HostsSeparator)
-	return &AuthClient{authnodes: authnodes, enableHTTPS: enableHTTPS, certFile: certFile}
+func NewAuthClient(authNodes []string, enableHTTPS bool, certFile string) *AuthClient {
+	return &AuthClient{authnodes: authNodes, enableHTTPS: enableHTTPS, certFile: certFile}
 }
 
 func (c *AuthClient) request(clientID, clientKey string, key []byte, data interface{}, path, serviceID string) (respData []byte, err error) {

diff --git a/sdk/meta/meta.go b/sdk/meta/meta.go
@@ -94,7 +94,7 @@ func NewMetaWrapper(volname, owner, masterHosts string, authenticate, validateOw
 	mw := new(MetaWrapper)
 	mw.closeCh = make(chan struct{}, 1)
 	if authenticate {
-		mw.ac = authSDK.NewAuthClient(ticketMess.TicketHost, ticketMess.EnableHTTPS, ticketMess.CertFile)
+		mw.ac = authSDK.NewAuthClient(ticketMess.TicketHosts, ticketMess.EnableHTTPS, ticketMess.CertFile)
 		ticket, err := mw.ac.API().GetTicket(owner, ticketMess.ClientKey, proto.MasterServiceID)
 		if err != nil {
 			return nil, errors.Trace(err, "Get ticket from authnode failed!")

diff --git a/util/auth/auth.go b/util/auth/auth.go
@@ -2,7 +2,7 @@ package auth
 
 type TicketMess struct {
 	ClientKey   string
-	TicketHost  string
+	TicketHosts []string
 	EnableHTTPS bool
 	CertFile    string
 }

diff --git a/util/config/config.go b/util/config/config.go
@@ -16,6 +16,7 @@ package config
 
 import (
 	"encoding/json"
+	"fmt"
 	"io/ioutil"
 	"log"
 	"strconv"
@@ -136,11 +137,24 @@ func (c *Config) GetInt64(key string) int64 {
 	return 0
 }
 
-// GetArray returns an array for the config key.
-func (c *Config) GetArray(key string) []interface{} {
+// GetSlice returns an array for the config key.
+func (c *Config) GetSlice(key string) []interface{} {
 	result, present := c.data[key]
 	if !present {
 		return []interface{}(nil)
 	}
 	return result.([]interface{})
 }
+
+func (c *Config) GetStringSlice(key string) []string {
+	s := c.GetSlice(key)
+	result := make([]string, 0, len(s))
+	for _, item := range s {
+		result = append(result, item.(string))
+	}
+	return result
+}
+
+func NewIllegalConfigError(configKey string) error {
+	return fmt.Errorf("illegal config %s", configKey)
+}