diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..f980ab9 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,7 @@ +{ + // 使用 IntelliSense 了解相关属性。 + // 悬停以查看现有属性的描述。 + // 欲了解更多信息,请访问: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [] +} \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..56e19b1 --- /dev/null +++ b/README.md @@ -0,0 +1,3 @@ +# remote-log-go + +remote-log sdk go 版本。将日志内容按照统一格式通过 http 发送到日志采集层,支持缓存和压缩上传。 diff --git a/console_transport.go b/console_transport.go new file mode 100644 index 0000000..674c58e --- /dev/null +++ b/console_transport.go @@ -0,0 +1,59 @@ +package remote_log_go + +import ( + "fmt" + "os" +) + +type ConsoleTransport struct { + allowLevel []Level // 允许使用此方式的日志级别 +} + +/** + * @description: 创建ConsoleTransport类 + * @param {...Level} allowLevel + * @return {*} + */ +func NewConsoleTransport(allowLevel ...Level) *ConsoleTransport { + return &ConsoleTransport{ + allowLevel: allowLevel, + } +} + +/** + * @description: 是否允许此方式记录日志 + * @param {Level} level + * @return {*} + */ +func (c *ConsoleTransport) ShouldLog(level Level) bool { + for _, v := range c.allowLevel { + if v == level { + return true + } + } + + return false +} + +/** + * @description: 记录日志 + * @param {*LogInfo} log + * @return {*} + */ +func (c *ConsoleTransport) Log(log *LogInfo) { + logStr := formatConsole(log) + if log.Level == string(Error) { + fmt.Fprintln(os.Stderr, logStr) + } else { + fmt.Println(logStr) + } +} + +/** + * @description: 日志格式化 + * @param {*logger.LogInfo} log + * @return {*} + */ +func formatConsole(log *LogInfo) string { + return fmt.Sprintf("%v %v %v %v %v", log.LogTime, log.Level, log.ServiceName, log.AppName, log.Message) +} diff --git a/consts.go b/consts.go new file mode 100644 index 0000000..41de7ef --- /dev/null +++ b/consts.go @@ -0,0 +1,11 @@ +package remote_log_go + +type Level string + +const ( + Debug Level = "debug" + Info Level = "info" + Warn Level = "warn" + Error Level = "error" + Access Level = "access" +) diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..c750f32 --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module github.com/doubility/remote-log-go + +go 1.16 diff --git a/http_transport.go b/http_transport.go new file mode 100644 index 0000000..3490e10 --- /dev/null +++ b/http_transport.go @@ -0,0 +1,235 @@ +package remote_log_go + +import ( + "bufio" + "bytes" + "compress/zlib" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "net/http" + "os" + "time" + "unsafe" +) + +type HttpTransport struct { + allowLevel []Level // 允许使用此方式的日志级别 + maxBufferLength int64 // 最大缓存字符串长度 + maxBufferSize int64 // 最大日志条数 + bufferLog []unsafe.Pointer // 缓存日志 + bufferLength int64 // 缓存日志长度 + bufferChan chan string + t *time.Ticker +} + +/** + * @description: 创建HttpTransport类 + * @param {...string} allowLevel + * @return {*} + */ +func NewHttpTransport(allowLevel ...Level) *HttpTransport { + h := &HttpTransport{ + allowLevel: allowLevel, + maxBufferLength: 50000, + maxBufferSize: 100, + bufferLog: make([]unsafe.Pointer, 0, 200), + bufferLength: 0, + bufferChan: make(chan string, 10000), + t: time.NewTicker(time.Millisecond * 1000), + } + // 定时执行任务、接受chan中的日志 + go h.createInterval() + + return h +} + +func (h *HttpTransport) createInterval() { + for { + select { + case <-h.t.C: + h.flush() + case logStr := <-h.bufferChan: + h.bufferLength += int64(len(logStr)) + h.bufferLog = append(h.bufferLog, unsafe.Pointer(&logStr)) + if len(h.bufferLog) >= int(h.maxBufferSize) || h.bufferLength >= h.maxBufferLength { + h.flush() + } + } + } +} + +/** + * @description: 设置自动上传间隔 + * @param {int64} ms + * @return {*} + */ +func (h *HttpTransport) SetFlushInterval(ms int64) { + h.t.Reset(time.Millisecond * time.Duration(ms)) +} + +/** + * @description: 设置最大缓存字符串长度 + * @param {int64} length + * @return {*} + */ +func (h *HttpTransport) SetMaxBufferLength(length int64) { + h.maxBufferLength = length +} + +/** + * @description: 设置最大缓存条数 + * @param {int64} size + * @return {*} + */ +func (h *HttpTransport) SetMaxBufferSize(size int64) { + h.maxBufferSize = size + h.bufferLog = make([]unsafe.Pointer, 0, size*2) +} + +/** + * @description: 是否允许此方式记录日志 + * @param {Level} level + * @return {*} + */ +func (h *HttpTransport) ShouldLog(level Level) bool { + for _, v := range h.allowLevel { + if v == level { + return true + } + } + + return false +} + +/** + * @description: 记录日志 + * @param {*LogInfo} log + * @return {*} + */ +func (h *HttpTransport) Log(log *LogInfo) { + logStr := formatHttp(log) + h.bufferChan <- logStr +} + +/** + * @description: 处理日志 + * @param {*} + * @return {*} + */ +func (h *HttpTransport) flush() { + if len(h.bufferLog) > 0 { + arrStrBufferLog := []string{} + for _, v := range h.bufferLog { + arrStrBufferLog = append(arrStrBufferLog, *(*string)(v)) + } + // 长度大于1000时压缩上传 + // 压缩失败时,原字符串上传 + if h.bufferLength > 1000 { + bytesData, err := json.Marshal(arrStrBufferLog) + if err != nil { + sendLog(1, arrStrBufferLog, "") + } + strLog, err := doZlibCompress(bytesData) + if err != nil { + sendLog(1, arrStrBufferLog, "") + } else { + sendLog(2, []string{}, strLog) + } + + } else { + sendLog(1, arrStrBufferLog, "") + } + + h.bufferLog = h.bufferLog[:0] + h.bufferLength = 0 + } +} + +/** + * @description: 请求接口 上传日志 + * @param {int32} _type + * @param {[]string} data1 + * @param {string} data2 + * @return {*} + */ +func sendLog(_type int32, data1 []string, data2 string) { + defer func() { + if err := recover(); err != nil { + data := make(map[string]interface{}) + data["type"] = _type + data["data1"] = data1 + data["data2"] = data2 + bytesData, _ := json.Marshal(data) + + httpErrorLog(string(bytesData)) + + fmt.Println(err) + } + }() + + data := make(map[string]interface{}) + data["type"] = _type + data["data1"] = data1 + data["data2"] = data2 + bytesData, _ := json.Marshal(data) + + res, err := http.Post(RemoteLogApiUrl+"/api/collectLog?pwd=b3981ef7-694b-11ec-a673-00163e1357b3", "application/json", bytes.NewBuffer(bytesData)) + if err != nil { + panic(err) + } + body, err := ioutil.ReadAll(res.Body) + if err != nil { + panic(err) + } + + var response Response + err = json.Unmarshal(body, &response) + if err != nil { + panic(err) + } + + if response.Code != 200 { + panic(errors.New(response.Message)) + } +} + +// 记录上传失败的日志 +func httpErrorLog(log string) { + file, _ := os.OpenFile(fmt.Sprintf("%v/error_log_%v.log", ErrorLogPath, time.Now().Format("2006-01-02")), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) + defer file.Close() + write := bufio.NewWriter(file) + write.WriteString(log + "\n") + write.Flush() +} + +/** + * @description: 日志格式化 + * @param {*LogInfo} log + * @return {*} + */ +func formatHttp(log *LogInfo) string { + return fmt.Sprintf("%v|**|%v|**|%v|**|%v|**|%v", log.LogTime, log.Level, log.ServiceName, log.AppName, log.Message) +} + +/** + * @description: 压缩字符串 + * @param {[]byte} src + * @return {*} + */ +func doZlibCompress(src []byte) (string, error) { + var in bytes.Buffer + w := zlib.NewWriter(&in) + n, err := w.Write(src) + if err != nil || n == 0 { + return "", err + } + err = w.Close() + if err != nil { + return "", err + } + + return base64.StdEncoding.EncodeToString(in.Bytes()), nil +} diff --git a/log_info.go b/log_info.go new file mode 100644 index 0000000..af5d20e --- /dev/null +++ b/log_info.go @@ -0,0 +1,9 @@ +package remote_log_go + +type LogInfo struct { + ServiceName string + AppName string + Level string + LogTime string + Message string +} diff --git a/logger.go b/logger.go new file mode 100644 index 0000000..ae2d6b1 --- /dev/null +++ b/logger.go @@ -0,0 +1,142 @@ +package remote_log_go + +import ( + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "os" + "time" +) + +type Logger struct { + appName string + storageDays int32 + serviceName string + transport []interface{} +} + +/** + * @description: 创建Logger类 + * @param {string} appName + * @param {int32} storageDays + * @param {[]interface{}} transport + * @return {*} + */ +func NewLogger(appName string, storageDays int32, transport ...interface{}) *Logger { + hostname, _ := os.Hostname() + + if appName == "" { + panic(errors.New("appname cannot be empty")) + } + + RemoteLogApiUrl = os.Getenv("REMOTE_LOG_API_URL") + if RemoteLogApiUrl == "" { + panic(errors.New("invalid env REMOTE_LOG_API_URL")) + } + + goPath := os.Getenv("GO_APP_DATA") + if goPath != "" { + ErrorLogPath = fmt.Sprintf("%v/%v/remote_logs", goPath, appName) + os.MkdirAll(ErrorLogPath, os.ModePerm) + } else { + panic(errors.New("invalid env GO_APP_DATA")) + } + + return &Logger{ + appName: appName, + storageDays: storageDays, + transport: transport, + serviceName: hostname, + } +} + +func (l *Logger) Init() error { + params := url.Values{} + params.Add("app", l.appName) + params.Add("storageDays", fmt.Sprintf("%v", l.storageDays)) + params.Add("pwd", "b3981ef7-694b-11ec-a673-00163e1357b3") + baseUrl, _ := url.Parse(RemoteLogApiUrl) + baseUrl.Path = "api/appStorageDays" + baseUrl.RawQuery = params.Encode() + + res, err := http.Get(baseUrl.String()) + if err != nil { + return err + } + + body, err := ioutil.ReadAll(res.Body) + if err != nil { + return err + } + + var response Response + err = json.Unmarshal(body, &response) + if err != nil { + return err + } + + if response.Code != 200 { + return errors.New(response.Message) + } + + return nil +} + +func (l *Logger) Debug(message string) { + l.log(Debug, message) +} + +func (l *Logger) Info(message string) { + l.log(Info, message) +} + +func (l *Logger) Warn(message string) { + l.log(Warn, message) +} + +func (l *Logger) Error(message error) { + l.log(Error, message.Error()) +} + +func (l *Logger) Access(message string) { + l.log(Access, message) +} + +func (l *Logger) log(level Level, message string) { + if message == "" { + return + } + now := time.Now().Format("2006-01-02 15:04:05") + logInfo := &LogInfo{ + ServiceName: l.serviceName, + AppName: l.appName, + Level: string(level), + LogTime: now, + Message: message, + } + + isLog := false // // 是否日志已记录,未记录的日志console打印 + + for _, item := range l.transport { + switch v := item.(type) { + case *HttpTransport: + if v.ShouldLog(level) { + isLog = true + v.Log(logInfo) + } + case *ConsoleTransport: + if v.ShouldLog(level) { + isLog = true + v.Log(logInfo) + } + } + } + + if !isLog { + consoleTransport := NewConsoleTransport() + consoleTransport.Log(logInfo) + } +} diff --git a/logger_test.go b/logger_test.go new file mode 100644 index 0000000..7aa356f --- /dev/null +++ b/logger_test.go @@ -0,0 +1,58 @@ +package remote_log_go + +import ( + "sync" + "testing" + "time" +) + +func TestMany(t *testing.T) { + + httpTransport := NewHttpTransport(Info, Warn, Error, Access) + consoleTransport := NewConsoleTransport(Debug) + log := NewLogger("go_app", 40, httpTransport, consoleTransport) + + err := log.Init() + + if err != nil { + t.Errorf("初始化错误:%v", err.Error()) + } + + var wait sync.WaitGroup + wait.Add(100) + workM(log, &wait) + wait.Wait() + + t.Log("执行完成") + + time.Sleep(time.Second * 20) +} + +func workM(log *Logger, w *sync.WaitGroup) { + for i := 0; i < 100; i++ { + go func() { + defer w.Done() + for i := 0; i < 100; i++ { + log.Info("go应用消息测试") + } + }() + } +} + +func TestOne(t *testing.T) { + httpTransport := NewHttpTransport(Info, Warn, Error, Access) + consoleTransport := NewConsoleTransport(Debug) + log := NewLogger("go_app", 40, httpTransport, consoleTransport) + + // err := log.Init() + + // if err != nil { + // t.Errorf("初始化错误:%v", err.Error()) + // } + + log.Info("go消息测试2") + + log.Info("go消息测试3") + + time.Sleep(time.Second * 5) +} diff --git a/response.go b/response.go new file mode 100644 index 0000000..ab40408 --- /dev/null +++ b/response.go @@ -0,0 +1,7 @@ +package remote_log_go + +type Response struct { + Code int `json:"code"` + Data interface{} `json:"data"` + Message string `json:"message"` +} diff --git a/variable.go b/variable.go new file mode 100644 index 0000000..a146749 --- /dev/null +++ b/variable.go @@ -0,0 +1,5 @@ +package remote_log_go + +var ErrorLogPath string + +var RemoteLogApiUrl string