From 2f02a60b545ca140647ff00f5b649e76d36bdd73 Mon Sep 17 00:00:00 2001 From: ligeng1 Date: Fri, 15 Nov 2024 11:19:40 +0800 Subject: [PATCH 1/7] fix: viper sup more namespace --- viper-remote/remote.go | 130 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 116 insertions(+), 14 deletions(-) diff --git a/viper-remote/remote.go b/viper-remote/remote.go index 0324495..25eb3c2 100644 --- a/viper-remote/remote.go +++ b/viper-remote/remote.go @@ -3,9 +3,11 @@ package remote import ( "bytes" "errors" + "fmt" "io" "os" "path/filepath" + "strings" "sync" crypt "github.com/bketelsen/crypt/config" @@ -49,7 +51,9 @@ func SetAgolloOptions(opts ...agollo.Option) { type viperConfigManager interface { Get(key string) ([]byte, error) + GetMultipleNamespaces(keys []string) (map[string][]byte, error) Watch(key string, stop chan bool) <-chan *viper.RemoteResponse + WatchMultipleNamespaces(keys []string, stop chan bool) <-chan *viper.RemoteResponse } type apolloConfigManager struct { @@ -99,6 +103,21 @@ func (cm apolloConfigManager) Get(namespace string) ([]byte, error) { return marshalConfigs(getConfigType(namespace), configs) } +func (cm apolloConfigManager) GetMultipleNamespaces(namespaces []string) (map[string][]byte, error) { + configs := make(map[string][]byte) + var err error + + for _, namespace := range namespaces { + singleConfig, singleErr := cm.Get(namespace) + if singleErr != nil { + err = singleErr // 仅记录错误 + } + configs[namespace] = singleConfig + } + + return configs, err +} + func marshalConfigs(configType string, configs map[string]interface{}) ([]byte, error) { var bts []byte var err error @@ -140,6 +159,36 @@ func (cm apolloConfigManager) Watch(namespace string, stop chan bool) <-chan *vi }() return resp } +func (cm apolloConfigManager) WatchMultipleNamespaces(namespaces []string, stop chan bool) <-chan *viper.RemoteResponse { + combinedResp := make(chan *viper.RemoteResponse) + + // 创建一个监听每个namespace变化的goroutine + for _, namespace := range namespaces { + go func(ns string) { + backendResp := cm.agollo.WatchNamespace(ns, stop) + for { + select { + case <-stop: + return + case r := <-backendResp: + if r.Error != nil { + combinedResp <- &viper.RemoteResponse{ + Value: nil, + Error: r.Error, + } + continue + } + + configType := getConfigType(ns) + value, err := marshalConfigs(configType, r.NewValue) + combinedResp <- &viper.RemoteResponse{Value: value, Error: err} + } + } + }(namespace) + } + + return combinedResp +} type configProvider struct { } @@ -149,13 +198,24 @@ func (rc configProvider) Get(rp viper.RemoteProvider) (io.Reader, error) { if err != nil { return nil, err } - + namespaces := strings.Split(rp.Path(), ",") var b []byte - switch cm := cmt.(type) { - case viperConfigManager: - b, err = cm.Get(rp.Path()) - case crypt.ConfigManager: - b, err = cm.Get(rp.Path()) + if len(namespaces) == 1 { + switch cm := cmt.(type) { + case viperConfigManager: + b, err = cm.Get(namespaces[0]) + case crypt.ConfigManager: + b, err = cm.Get(namespaces[0]) + } + } else if len(namespaces) > 1 { + switch cm := cmt.(type) { + case viperConfigManager: + configs, err := cm.GetMultipleNamespaces(namespaces) + if err != nil { + return nil, err + } + b, err = mergeConfigs(configs) + } } if err != nil { @@ -163,19 +223,53 @@ func (rc configProvider) Get(rp viper.RemoteProvider) (io.Reader, error) { } return bytes.NewReader(b), nil } - +func mergeConfigs(configs map[string][]byte) ([]byte, error) { + finalConfig := make(map[string]string) + + for _, config := range configs { + lines := strings.Split(string(config), "\n") + for _, line := range lines { + if line == "" { + continue + } + kv := strings.SplitN(line, "=", 2) + if len(kv) != 2 { + return nil, fmt.Errorf("invalid config line: %s", line) + } + key := strings.TrimSpace(kv[0]) + value := strings.TrimSpace(kv[1]) + finalConfig[key] = value + } + } + var b bytes.Buffer + for k, v := range finalConfig { + fmt.Fprintf(&b, "%s = %s\n", k, v) + } + return b.Bytes(), nil +} func (rc configProvider) Watch(rp viper.RemoteProvider) (io.Reader, error) { cmt, err := getConfigManager(rp) if err != nil { return nil, err } - + namespaces := strings.Split(rp.Path(), ",") var resp []byte - switch cm := cmt.(type) { - case viperConfigManager: - resp, err = cm.Get(rp.Path()) - case crypt.ConfigManager: - resp, err = cm.Get(rp.Path()) + if len(namespaces) == 1 { + switch cm := cmt.(type) { + case viperConfigManager: + resp, err = cm.Get(namespaces[0]) + case crypt.ConfigManager: + resp, err = cm.Get(namespaces[0]) + } + } else if len(namespaces) > 1 { + switch cm := cmt.(type) { + case viperConfigManager: + configs, err := cm.GetMultipleNamespaces(namespaces) + if err != nil { + return nil, err + } + resp, err = mergeConfigs(configs) + } } if err != nil { @@ -190,7 +284,15 @@ func (rc configProvider) WatchChannel(rp viper.RemoteProvider) (<-chan *viper.Re if err != nil { return nil, nil } - + namespaces := strings.Split(rp.Path(), ",") + if len(namespaces) > 1 { + switch cm := cmt.(type) { + case viperConfigManager: + quitwc := make(chan bool) + viperResponsCh := cm.WatchMultipleNamespaces(namespaces, quitwc) + return viperResponsCh, quitwc + } + } switch cm := cmt.(type) { case viperConfigManager: quitwc := make(chan bool) From fe6001d593530fd4c5cc609e36fac6cb058ad256 Mon Sep 17 00:00:00 2001 From: ligeng1 Date: Fri, 15 Nov 2024 12:28:08 +0800 Subject: [PATCH 2/7] fix: viper sup more namespace --- go.mod | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index c195e29..f1fcd7a 100644 --- a/go.mod +++ b/go.mod @@ -1,10 +1,11 @@ -module github.com/shima-park/agollo +module github.com/shima-park-fork/agollo go 1.13 require ( github.com/bketelsen/crypt v0.0.4 github.com/magiconair/properties v1.8.5 + github.com/shima-park/agollo v1.2.14 github.com/spf13/viper v1.8.1 github.com/stretchr/testify v1.7.0 gopkg.in/go-playground/assert.v1 v1.2.1 From 1c692df955305c65c512a1a46b5b6035edc1eda2 Mon Sep 17 00:00:00 2001 From: ligeng1 Date: Fri, 15 Nov 2024 12:36:44 +0800 Subject: [PATCH 3/7] fix: viper sup more namespace --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index f1fcd7a..1580334 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/shima-park-fork/agollo +module github.com/ligeng8848/agollo go 1.13 From e756ca260a9c7206ea4c24493b1ef6d05dd76bb6 Mon Sep 17 00:00:00 2001 From: ligeng1 Date: Fri, 15 Nov 2024 13:37:25 +0800 Subject: [PATCH 4/7] fix: mod --- go.mod | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 1580334..c195e29 100644 --- a/go.mod +++ b/go.mod @@ -1,11 +1,10 @@ -module github.com/ligeng8848/agollo +module github.com/shima-park/agollo go 1.13 require ( github.com/bketelsen/crypt v0.0.4 github.com/magiconair/properties v1.8.5 - github.com/shima-park/agollo v1.2.14 github.com/spf13/viper v1.8.1 github.com/stretchr/testify v1.7.0 gopkg.in/go-playground/assert.v1 v1.2.1 From 29572935c60ac748d0251e511581074defa17cb8 Mon Sep 17 00:00:00 2001 From: ligeng1 Date: Mon, 18 Nov 2024 15:15:17 +0800 Subject: [PATCH 5/7] fix: merge configs --- go.mod | 5 ++ viper-remote/remote.go | 121 +++++++++++++++++++++++++++-------------- 2 files changed, 84 insertions(+), 42 deletions(-) diff --git a/go.mod b/go.mod index c195e29..d6ea4cd 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,13 @@ go 1.13 require ( github.com/bketelsen/crypt v0.0.4 + github.com/hashicorp/hcl v1.0.0 github.com/magiconair/properties v1.8.5 + github.com/pelletier/go-toml v1.9.3 + github.com/spf13/cast v1.3.1 github.com/spf13/viper v1.8.1 github.com/stretchr/testify v1.7.0 + github.com/subosito/gotenv v1.2.0 gopkg.in/go-playground/assert.v1 v1.2.1 + gopkg.in/yaml.v2 v2.4.0 ) diff --git a/viper-remote/remote.go b/viper-remote/remote.go index 25eb3c2..255752f 100644 --- a/viper-remote/remote.go +++ b/viper-remote/remote.go @@ -2,6 +2,8 @@ package remote import ( "bytes" + "encoding/json" + "encoding/xml" "errors" "fmt" "io" @@ -13,6 +15,7 @@ import ( crypt "github.com/bketelsen/crypt/config" "github.com/shima-park/agollo" "github.com/spf13/viper" + "gopkg.in/yaml.v2" ) var ( @@ -51,7 +54,7 @@ func SetAgolloOptions(opts ...agollo.Option) { type viperConfigManager interface { Get(key string) ([]byte, error) - GetMultipleNamespaces(keys []string) (map[string][]byte, error) + GetMultipleNamespaces(keys []string) ([]byte, error) Watch(key string, stop chan bool) <-chan *viper.RemoteResponse WatchMultipleNamespaces(keys []string, stop chan bool) <-chan *viper.RemoteResponse } @@ -103,19 +106,65 @@ func (cm apolloConfigManager) Get(namespace string) ([]byte, error) { return marshalConfigs(getConfigType(namespace), configs) } -func (cm apolloConfigManager) GetMultipleNamespaces(namespaces []string) (map[string][]byte, error) { - configs := make(map[string][]byte) - var err error - +func (cm apolloConfigManager) GetMultipleNamespaces(namespaces []string) ([]byte, error) { + newConfigs := make(map[string]interface{}) + var configType string + var b []byte for _, namespace := range namespaces { - singleConfig, singleErr := cm.Get(namespace) - if singleErr != nil { - err = singleErr // 仅记录错误 + configs := cm.agollo.GetNameSpace(namespace) + configType = getConfigType(namespace) + // 根据类型进行合并 + switch configType { + case "json", "yml", "yaml", "xml": + content := configs["content"] + if content != nil { + var tempConfig map[string]interface{} + err := unmarshalConfig(configType, []byte(content.(string)), &tempConfig) + if err != nil { + return nil, err + } + // 执行合并 + viper.MergeConfigMap(tempConfig) + mergedConfigs := viper.AllSettings() + b, err = marshalConfig(configType, mergedConfigs) + if err != nil { + return nil, err + } + newConfigs["content"] = string(b) + } + case "properties": + viper.MergeConfigMap(configs) + newConfigs = viper.AllSettings() } - configs[namespace] = singleConfig } - return configs, err + return marshalConfigs(configType, newConfigs) +} + +func unmarshalConfig(configType string, data []byte, config *map[string]interface{}) error { + switch configType { + case "json": + return json.Unmarshal(data, config) + case "yml", "yaml": + return yaml.Unmarshal(data, config) + case "xml": + return xml.Unmarshal(data, config) + default: + return fmt.Errorf("unsupported config type: %s", configType) + } +} + +func marshalConfig(configType string, config map[string]interface{}) ([]byte, error) { + switch configType { + case "json": + return json.Marshal(config) + case "yml", "yaml": + return yaml.Marshal(config) + case "xml": + return xml.Marshal(config) + default: + return nil, fmt.Errorf("unsupported config type: %s", configType) + } } func marshalConfigs(configType string, configs map[string]interface{}) ([]byte, error) { @@ -180,7 +229,23 @@ func (cm apolloConfigManager) WatchMultipleNamespaces(namespaces []string, stop } configType := getConfigType(ns) - value, err := marshalConfigs(configType, r.NewValue) + switch configType { + case "json", "yml", "yaml", "xml": + content := r.NewValue["content"] + if content != nil { + var tempConfig map[string]interface{} + err := unmarshalConfig(configType, []byte(content.(string)), &tempConfig) + if err != nil { + combinedResp <- &viper.RemoteResponse{Value: nil, Error: err} + continue + } + viper.MergeConfigMap(tempConfig) + } + case "properties": + viper.MergeConfigMap(r.NewValue) + } + mergedConfigs := viper.AllSettings() + value, err := marshalConfig(configType, mergedConfigs) combinedResp <- &viper.RemoteResponse{Value: value, Error: err} } } @@ -210,11 +275,10 @@ func (rc configProvider) Get(rp viper.RemoteProvider) (io.Reader, error) { } else if len(namespaces) > 1 { switch cm := cmt.(type) { case viperConfigManager: - configs, err := cm.GetMultipleNamespaces(namespaces) + b, err = cm.GetMultipleNamespaces(namespaces) if err != nil { return nil, err } - b, err = mergeConfigs(configs) } } @@ -223,30 +287,7 @@ func (rc configProvider) Get(rp viper.RemoteProvider) (io.Reader, error) { } return bytes.NewReader(b), nil } -func mergeConfigs(configs map[string][]byte) ([]byte, error) { - finalConfig := make(map[string]string) - - for _, config := range configs { - lines := strings.Split(string(config), "\n") - for _, line := range lines { - if line == "" { - continue - } - kv := strings.SplitN(line, "=", 2) - if len(kv) != 2 { - return nil, fmt.Errorf("invalid config line: %s", line) - } - key := strings.TrimSpace(kv[0]) - value := strings.TrimSpace(kv[1]) - finalConfig[key] = value - } - } - var b bytes.Buffer - for k, v := range finalConfig { - fmt.Fprintf(&b, "%s = %s\n", k, v) - } - return b.Bytes(), nil -} + func (rc configProvider) Watch(rp viper.RemoteProvider) (io.Reader, error) { cmt, err := getConfigManager(rp) if err != nil { @@ -264,11 +305,7 @@ func (rc configProvider) Watch(rp viper.RemoteProvider) (io.Reader, error) { } else if len(namespaces) > 1 { switch cm := cmt.(type) { case viperConfigManager: - configs, err := cm.GetMultipleNamespaces(namespaces) - if err != nil { - return nil, err - } - resp, err = mergeConfigs(configs) + resp, err = cm.GetMultipleNamespaces(namespaces) } } From 5fe6d21e73f1c928c0165718da33119a3d262390 Mon Sep 17 00:00:00 2001 From: ligeng1 Date: Mon, 18 Nov 2024 17:36:10 +0800 Subject: [PATCH 6/7] fix: merge configs --- viper-remote/remote.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/viper-remote/remote.go b/viper-remote/remote.go index 255752f..454b5d4 100644 --- a/viper-remote/remote.go +++ b/viper-remote/remote.go @@ -133,8 +133,9 @@ func (cm apolloConfigManager) GetMultipleNamespaces(namespaces []string) ([]byte newConfigs["content"] = string(b) } case "properties": - viper.MergeConfigMap(configs) - newConfigs = viper.AllSettings() + for k, v := range configs { + newConfigs[k] = v + } } } @@ -229,6 +230,7 @@ func (cm apolloConfigManager) WatchMultipleNamespaces(namespaces []string, stop } configType := getConfigType(ns) + var mergedConfigs map[string]interface{} switch configType { case "json", "yml", "yaml", "xml": content := r.NewValue["content"] @@ -240,12 +242,13 @@ func (cm apolloConfigManager) WatchMultipleNamespaces(namespaces []string, stop continue } viper.MergeConfigMap(tempConfig) + mergedConfigs = viper.AllSettings() } case "properties": - viper.MergeConfigMap(r.NewValue) + mergedConfigs = r.NewValue } - mergedConfigs := viper.AllSettings() - value, err := marshalConfig(configType, mergedConfigs) + + value, err := marshalConfigs(configType, mergedConfigs) combinedResp <- &viper.RemoteResponse{Value: value, Error: err} } } From 65ffa06f0c5204cf0313386416b6f4ea79cfbdd9 Mon Sep 17 00:00:00 2001 From: ligeng1 Date: Fri, 29 Nov 2024 14:39:25 +0800 Subject: [PATCH 7/7] fix: config the read priority --- viper-remote/remote.go | 27 ++++++--------------------- 1 file changed, 6 insertions(+), 21 deletions(-) diff --git a/viper-remote/remote.go b/viper-remote/remote.go index 454b5d4..fb65561 100644 --- a/viper-remote/remote.go +++ b/viper-remote/remote.go @@ -228,28 +228,13 @@ func (cm apolloConfigManager) WatchMultipleNamespaces(namespaces []string, stop } continue } - - configType := getConfigType(ns) - var mergedConfigs map[string]interface{} - switch configType { - case "json", "yml", "yaml", "xml": - content := r.NewValue["content"] - if content != nil { - var tempConfig map[string]interface{} - err := unmarshalConfig(configType, []byte(content.(string)), &tempConfig) - if err != nil { - combinedResp <- &viper.RemoteResponse{Value: nil, Error: err} - continue - } - viper.MergeConfigMap(tempConfig) - mergedConfigs = viper.AllSettings() - } - case "properties": - mergedConfigs = r.NewValue + // 重载所有配置以确保多配置的优先级 + allConfigs, err := cm.GetMultipleNamespaces(namespaces) + if err != nil { + combinedResp <- &viper.RemoteResponse{Value: nil, Error: err} + continue } - - value, err := marshalConfigs(configType, mergedConfigs) - combinedResp <- &viper.RemoteResponse{Value: value, Error: err} + combinedResp <- &viper.RemoteResponse{Value: allConfigs, Error: err} } } }(namespace)