From 017b1f9fa1c5642db02161c836d76338cc86f4f5 Mon Sep 17 00:00:00 2001 From: Fergal Mc Carthy Date: Fri, 2 Aug 2024 08:39:23 -0400 Subject: [PATCH] Eliminate unnecessary JSON blob processing Ensure that provided telemetry data JSOB blob data is not processed unless needed, avoiding unmarshaling and re-marshaling of the field. Depends up breaking changes in SUSE/telemetry#42 Fixes: #27 --- app/telemetry_default.go | 17 ++--------------- app/telemetry_scchwinfo.go | 18 +++++++++++++++--- server/telemetry-admin/.gitignore | 2 +- server/telemetry-server/app_test.go | 28 ++++++++++++---------------- 4 files changed, 30 insertions(+), 35 deletions(-) diff --git a/app/telemetry_default.go b/app/telemetry_default.go index deb494f..bfeeda8 100644 --- a/app/telemetry_default.go +++ b/app/telemetry_default.go @@ -25,25 +25,12 @@ type DefaultTelemetryDataRow struct { // Embed the common rows TelemetryDataCommon - DataItem any `json:"dataItem"` + DataItem []byte `json:"dataItem"` } func (t *DefaultTelemetryDataRow) Init(dItm *telemetrylib.TelemetryDataItem, bHdr *telemetrylib.TelemetryBundleHeader, tagSetId int64) (err error) { t.TelemetryDataCommon.Init(dItm, bHdr, tagSetId) - - // marshal telemetry data as JSON - jsonData, err := json.Marshal(dItm.TelemetryData) - if err != nil { - slog.Error( - "JSON marshal failed", - slog.Int64("clientId", t.ClientId), - slog.String("telemetryId", t.TelemetryId), - slog.String("timestamp", t.Timestamp), - slog.String("error", err.Error()), - ) - return - } - t.DataItem = jsonData + t.DataItem = []byte(dItm.TelemetryData) return } diff --git a/app/telemetry_scchwinfo.go b/app/telemetry_scchwinfo.go index 5e48dc0..7c30798 100644 --- a/app/telemetry_scchwinfo.go +++ b/app/telemetry_scchwinfo.go @@ -84,13 +84,25 @@ func int64Conv(value any) (outValue int64, err error) { func (t *SccHwInfoTelemetryDataRow) Init(dItm *telemetrylib.TelemetryDataItem, bHdr *telemetrylib.TelemetryBundleHeader, tagSetId int64) (err error) { t.TelemetryDataCommon.Init(dItm, bHdr, tagSetId) + // unmarshal the provided telemetry JSON blob + var tData map[string]any + err = json.Unmarshal([]byte(dItm.TelemetryData), &tData) + if err != nil { + slog.Error( + "Failed to unmarshal telemetry data JSON blob", + slog.String("telemetryType", t.TelemetryType), + slog.String("error", err.Error()), + ) + return + } + hwiName := "hwinfo" - err = checkRequiredMapFieldsExist(dItm.TelemetryData, hwiName, "distro_target") + err = checkRequiredMapFieldsExist(tData, hwiName, "distro_target") if err != nil { slog.Error("required data fields missing", slog.String("telemetryType", t.TelemetryType), slog.String("error", err.Error())) return } - hwi, ok := dItm.TelemetryData[hwiName].(map[string]any) + hwi, ok := tData[hwiName].(map[string]any) if !ok { err := fmt.Errorf("field %q in telemetryType %q data is not a map", hwiName, t.TelemetryType) return err @@ -102,7 +114,7 @@ func (t *SccHwInfoTelemetryDataRow) Init(dItm *telemetrylib.TelemetryDataItem, b } t.Hostname = hwi["hostname"].(string) - t.DistroTarget = dItm.TelemetryData["distro_target"].(string) + t.DistroTarget = tData["distro_target"].(string) t.Cpus, err = int64Conv(hwi["cpus"]) if err != nil { diff --git a/server/telemetry-admin/.gitignore b/server/telemetry-admin/.gitignore index be339fd..bf488fc 100644 --- a/server/telemetry-admin/.gitignore +++ b/server/telemetry-admin/.gitignore @@ -1 +1 @@ -telemetry-server +telemetry-admin diff --git a/server/telemetry-server/app_test.go b/server/telemetry-server/app_test.go index 3890506..05e22fd 100644 --- a/server/telemetry-server/app_test.go +++ b/server/telemetry-server/app_test.go @@ -114,7 +114,7 @@ func (t *AppTestSuite) TestReportTelemetry() { // Simulated request handled via the router's ServeHTTP // Response recorded via the httptest.HttpRecorder - body := createReportPayload(t.T()) + body := createReportPayload() rr, err := postToReportTelemetryHandler(body, "", true, t) assert.NoError(t.T(), err) @@ -136,7 +136,7 @@ func (t *AppTestSuite) TestReportTelemetryCompressedPayloadGZIP() { // Simulated request handled via the router's ServeHTTP // Response recorded via the httptest.HttpRecorder - body := createReportPayload(t.T()) + body := createReportPayload() //Compress payload cbody, err := compressedData([]byte(body), "gzip") @@ -161,7 +161,7 @@ func (t *AppTestSuite) TestReportTelemetryCompressedPayloadDeflate() { // Simulated request handled via the router's ServeHTTP // Response recorded via the httptest.HttpRecorder - body := createReportPayload(t.T()) + body := createReportPayload() //Compress payload cbody, err := compressedData([]byte(body), "deflate") @@ -447,23 +447,19 @@ func TestAppTestSuite(t *testing.T) { suite.Run(t, new(AppTestSuite)) } -func createReportPayload(t *testing.T) (reportPayload string) { +func createReportPayload() (reportPayload string) { // Create 2 dataitems telemetryType := types.TelemetryType("SLE-SERVER-Test") itags1 := types.Tags{types.Tag("ikey1=ivalue1"), types.Tag("ikey2")} itags2 := types.Tags{types.Tag("ikey1=ivalue1")} - payload := ` - { - "ItemA": 1, - "ItemB": "b", - "ItemC": "c" - } - ` - - item1, err := telemetrylib.NewTelemetryDataItem(telemetryType, itags1, []byte(payload)) - assert.NoError(t, err) - item2, err := telemetrylib.NewTelemetryDataItem(telemetryType, itags2, []byte(payload)) - assert.NoError(t, err) + payload := types.NewTelemetryBlob([]byte(`{ + "ItemA": 1, + "ItemB": "b", + "ItemC": "c" + }`)) + + item1 := telemetrylib.NewTelemetryDataItem(telemetryType, itags1, payload) + item2 := telemetrylib.NewTelemetryDataItem(telemetryType, itags2, payload) client_id := int64(12345)