Skip to content

Commit

Permalink
feat: add unknown fields in mesgdef structs (#468)
Browse files Browse the repository at this point in the history 
* feat: add unknown fields in mesgdef structs

* fitgen: generate mesgdef

* test: add record test for unknown fields

* feat: aggregator now support aggregating unknown fields

* feat: add sport messages and update split summary

* test: fix combiner test
  • Loading branch information
@muktihari
muktihari authored Sep 24, 2024
1 parent e344207 commit c943dbf
Show file tree
Hide file tree
Showing 125 changed files with 2,895 additions and 477 deletions.
18 changes: 18 additions & 0 deletions cmd/fitactivity/aggregator/aggregator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"time"

"github.com/muktihari/fit/profile/basetype"
"github.com/muktihari/fit/proto"
)

// Aggregate aggregates src and dst into dst using reflection where T
Expand Down Expand Up @@ -47,6 +48,22 @@ func Aggregate[T any](dst, src T) {
min(dv.Field(i), sv.Field(i)) // MinHeartRate, MinCadence, EnhancedMinAltitude, etc.
case strings.HasPrefix(f.Name, "Avg") || strings.HasPrefix(f.Name, "EnhancedAvg"):
avg(dv.Field(i), sv.Field(i)) // AvgHeartRate, AvgCadence, EnhancedAvgSpeed, etc..
case f.Type == reflect.TypeOf([]proto.Field{}):
dFields, sFields := dv.Field(i), sv.Field(i) // UnknownFields
for j := 0; j < sFields.Len(); j++ {
sNum := sFields.Index(j).FieldByName("Num").Uint()
var ok bool
for k := 0; k < dFields.Len(); k++ {
dNum := dFields.Index(k).FieldByName("Num").Uint()
if sNum == dNum {
ok = true
break
}
}
if !ok {
dFields.Set(reflect.AppendSlice(dFields, sFields.Slice(j, j+1)))
}
}
default:
fill(dv.Field(i), sv.Field(i)) // Timestamp, Sport, Event, etc.
}
Expand Down Expand Up @@ -301,6 +318,7 @@ func fill(dst, src reflect.Value) {
case reflect.Struct:
if dst.IsZero() && dst.Type() == reflect.TypeOf(time.Time{}) {
dst.Set(src)
return
}
}
}
17 changes: 17 additions & 0 deletions cmd/fitactivity/aggregator/aggregator_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (
"github.com/muktihari/fit/profile/basetype"
"github.com/muktihari/fit/profile/mesgdef"
"github.com/muktihari/fit/profile/typedef"
"github.com/muktihari/fit/proto"
)

func BenchmarkAggregate(b *testing.B) {
Expand Down Expand Up @@ -149,6 +150,22 @@ func TestAggregate(t *testing.T) {
Total: 3,
},
},
{
name: "unknown fields",
dst: mesgdef.NewSession(nil).SetUnknownFields(
proto.Field{FieldBase: &proto.FieldBase{Num: 1}, Value: proto.Uint8(1)},
proto.Field{FieldBase: &proto.FieldBase{Num: 2}, Value: proto.Uint8(2)},
),
src: mesgdef.NewSession(nil).SetUnknownFields(
proto.Field{FieldBase: &proto.FieldBase{Num: 2}, Value: proto.Uint8(22)},
proto.Field{FieldBase: &proto.FieldBase{Num: 3}, Value: proto.Uint8(3)},
),
exp: mesgdef.NewSession(nil).SetUnknownFields(
proto.Field{FieldBase: &proto.FieldBase{Num: 1}, Value: proto.Uint8(1)},
proto.Field{FieldBase: &proto.FieldBase{Num: 2}, Value: proto.Uint8(2)},
proto.Field{FieldBase: &proto.FieldBase{Num: 3}, Value: proto.Uint8(3)},
),
},
}

for i, tc := range tt {
Expand Down
51 changes: 46 additions & 5 deletions cmd/fitactivity/combiner/combiner.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ func Combine(fits []*proto.FIT) (result *proto.FIT, err error) {
})

var (
sports []*mesgdef.Sport
splitSummaries []*mesgdef.SplitSummary
sessionsByIndex = make([][]*mesgdef.Session, len(fits))
activities = make([]*mesgdef.Activity, 0, len(fits))
Expand Down Expand Up @@ -79,6 +80,19 @@ func Combine(fits []*proto.FIT) (result *proto.FIT, err error) {
case mesgnum.Activity:
activities = append(activities, mesgdef.NewActivity(&mesg))
continue
case mesgnum.Sport:
m := mesgdef.NewSport(&mesg)
var ok bool
for _, v := range sports {
if v.Sport == m.Sport {
ok = true
break
}
}
if !ok {
sports = append(sports, m)
}
continue
}
if j != valid {
fit.Messages[valid], fit.Messages[j] = fit.Messages[j], fit.Messages[valid]
Expand Down Expand Up @@ -154,16 +168,43 @@ func Combine(fits []*proto.FIT) (result *proto.FIT, err error) {

// Summarize

for _, ses := range sessions {
var ok bool
for _, v := range sports {
if v.Sport == ses.Sport {
ok = true
break
}
}
if !ok {
sports = append(sports, mesgdef.NewSport(nil).
SetSport(ses.Sport).
SetSubSport(ses.SubSport).
SetName(ses.SportProfileName))
}
}

for _, v := range sports {
result.Messages = append(result.Messages, v.ToMesg(nil))
}

firstTimestamp := getFirstTimestamp(result.Messages)
lastTimestamp := getLastTimestamp(result.Messages)

for _, v := range splitSummaries {
mesg := v.ToMesg(nil)
// Split Summary does not have timestamp, but we found a case where it may contains
// timestamp on FIT files produced by Garmin Devices, so let's create one.
mesg.Fields = append([]proto.Field{
factory.CreateField(mesgnum.Session, proto.FieldNumTimestamp).WithValue(lastTimestamp),
}, mesg.Fields...)

// Split Summary does not have timestamp, but Garmin devices produce timestamp for this message
// and Garmin Connect will reject our files if we don't include it.
// Discussion: https://forums.garmin.com/developer/fit-sdk/f/discussion/385625/timestamp-field-in-split_summary-messages

mesg.RemoveFieldByNum(proto.FieldNumTimestamp)

field := factory.CreateField(mesgnum.Session, proto.FieldNumTimestamp).WithValue(lastTimestamp)
mesg.Fields = append(mesg.Fields, proto.Field{})
copy(mesg.Fields[1:], mesg.Fields)
mesg.Fields[0] = field // Put timestamp as first field

result.Messages = append(result.Messages, mesg)
}

Expand Down
36 changes: 36 additions & 0 deletions cmd/fitactivity/combiner/combiner_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,8 @@ func TestCombine(t *testing.T) {
}},
{Num: mesgnum.Session, Fields: []proto.Field{
factory.CreateField(mesgnum.Session, fieldnum.SessionSport).WithValue(typedef.SportCycling),
factory.CreateField(mesgnum.Session, fieldnum.SessionSubSport).WithValue(typedef.SubSportGeneric),
factory.CreateField(mesgnum.Session, fieldnum.SessionSportProfileName).WithValue("Cycling"),
factory.CreateField(mesgnum.Session, fieldnum.SessionTimestamp).WithValue(datetime.ToUint32(now) + 2),
factory.CreateField(mesgnum.Session, fieldnum.SessionStartTime).WithValue(datetime.ToUint32(now)),
factory.CreateField(mesgnum.Session, fieldnum.SessionTotalDistance).WithValue(uint32(200)),
Expand Down Expand Up @@ -93,6 +95,8 @@ func TestCombine(t *testing.T) {
}},
{Num: mesgnum.Session, Fields: []proto.Field{
factory.CreateField(mesgnum.Session, fieldnum.SessionSport).WithValue(typedef.SportCycling),
factory.CreateField(mesgnum.Session, fieldnum.SessionSubSport).WithValue(typedef.SubSportGeneric),
factory.CreateField(mesgnum.Session, fieldnum.SessionSportProfileName).WithValue("Cycling"),
factory.CreateField(mesgnum.Session, fieldnum.SessionTimestamp).WithValue(datetime.ToUint32(now) + 20),
factory.CreateField(mesgnum.Session, fieldnum.SessionStartTime).WithValue(datetime.ToUint32(now) + 10),
factory.CreateField(mesgnum.Session, fieldnum.SessionTotalDistance).WithValue(uint32(200)),
Expand Down Expand Up @@ -133,6 +137,11 @@ func TestCombine(t *testing.T) {
factory.CreateField(mesgnum.Record, fieldnum.RecordDistance).WithValue(uint32(400)),
factory.CreateField(mesgnum.Record, fieldnum.RecordAccumulatedPower).WithValue(uint32(400)),
}},
{Num: mesgnum.Sport, Fields: []proto.Field{
factory.CreateField(mesgnum.Sport, fieldnum.SportSport).WithValue(typedef.SportCycling),
factory.CreateField(mesgnum.Sport, fieldnum.SportSubSport).WithValue(typedef.SubSportGeneric),
factory.CreateField(mesgnum.Sport, fieldnum.SportName).WithValue("Cycling"),
}},
{Num: mesgnum.SplitSummary, Fields: []proto.Field{
factory.CreateField(mesgnum.Session, proto.FieldNumTimestamp).WithValue(datetime.ToUint32(now) + 20), // Additional
factory.CreateField(mesgnum.SplitSummary, fieldnum.SplitSummarySplitType).WithValue(typedef.SplitTypeRunActive),
Expand All @@ -145,6 +154,8 @@ func TestCombine(t *testing.T) {
}},
{Num: mesgnum.Session, Fields: []proto.Field{
factory.CreateField(mesgnum.Session, fieldnum.SessionSport).WithValue(typedef.SportCycling),
factory.CreateField(mesgnum.Session, fieldnum.SessionSubSport).WithValue(typedef.SubSportGeneric),
factory.CreateField(mesgnum.Session, fieldnum.SessionSportProfileName).WithValue("Cycling"),
factory.CreateField(mesgnum.Session, fieldnum.SessionTimestamp).WithValue(datetime.ToUint32(now) + 20),
factory.CreateField(mesgnum.Session, fieldnum.SessionStartTime).WithValue(datetime.ToUint32(now)),
factory.CreateField(mesgnum.Session, fieldnum.SessionTotalDistance).WithValue(uint32(400)),
Expand Down Expand Up @@ -186,8 +197,15 @@ func TestCombine(t *testing.T) {
factory.CreateField(mesgnum.Record, fieldnum.RecordTimestamp).WithValue(datetime.ToUint32(now) + 2),
factory.CreateField(mesgnum.Record, fieldnum.RecordDistance).WithValue(uint32(200)),
}},
{Num: mesgnum.Sport, Fields: []proto.Field{
factory.CreateField(mesgnum.Sport, fieldnum.SportSport).WithValue(typedef.SportCycling),
factory.CreateField(mesgnum.Sport, fieldnum.SportSubSport).WithValue(typedef.SubSportGeneric),
factory.CreateField(mesgnum.Sport, fieldnum.SportName).WithValue("Cycling"),
}},
{Num: mesgnum.Session, Fields: []proto.Field{
factory.CreateField(mesgnum.Session, fieldnum.SessionSport).WithValue(typedef.SportCycling),
factory.CreateField(mesgnum.Session, fieldnum.SessionSubSport).WithValue(typedef.SubSportGeneric),
factory.CreateField(mesgnum.Session, fieldnum.SessionSportProfileName).WithValue("Cycling"),
factory.CreateField(mesgnum.Session, fieldnum.SessionTimestamp).WithValue(datetime.ToUint32(now) + 2),
factory.CreateField(mesgnum.Session, fieldnum.SessionStartTime).WithValue(datetime.ToUint32(now)),
factory.CreateField(mesgnum.Session, fieldnum.SessionTotalDistance).WithValue(uint32(200)),
Expand Down Expand Up @@ -221,6 +239,8 @@ func TestCombine(t *testing.T) {
}},
{Num: mesgnum.Session, Fields: []proto.Field{
factory.CreateField(mesgnum.Session, fieldnum.SessionSport).WithValue(typedef.SportCycling),
factory.CreateField(mesgnum.Session, fieldnum.SessionSubSport).WithValue(typedef.SubSportGeneric),
factory.CreateField(mesgnum.Session, fieldnum.SessionSportProfileName).WithValue("Cycling"),
factory.CreateField(mesgnum.Session, fieldnum.SessionTimestamp).WithValue(datetime.ToUint32(now) + 20),
factory.CreateField(mesgnum.Session, fieldnum.SessionStartTime).WithValue(datetime.ToUint32(now) + 10),
factory.CreateField(mesgnum.Session, fieldnum.SessionTotalDistance).WithValue(uint32(200)),
Expand Down Expand Up @@ -250,6 +270,8 @@ func TestCombine(t *testing.T) {
}},
{Num: mesgnum.Session, Fields: []proto.Field{
factory.CreateField(mesgnum.Session, fieldnum.SessionSport).WithValue(typedef.SportRunning),
factory.CreateField(mesgnum.Session, fieldnum.SessionSubSport).WithValue(typedef.SubSportTrail),
factory.CreateField(mesgnum.Session, fieldnum.SessionSportProfileName).WithValue("Running"),
factory.CreateField(mesgnum.Session, fieldnum.SessionTimestamp).WithValue(datetime.ToUint32(now) + 200),
factory.CreateField(mesgnum.Session, fieldnum.SessionStartTime).WithValue(datetime.ToUint32(now) + 100),
factory.CreateField(mesgnum.Session, fieldnum.SessionTotalDistance).WithValue(uint32(200)),
Expand Down Expand Up @@ -294,6 +316,16 @@ func TestCombine(t *testing.T) {
factory.CreateField(mesgnum.Record, fieldnum.RecordTimestamp).WithValue(datetime.ToUint32(now) + 200),
factory.CreateField(mesgnum.Record, fieldnum.RecordDistance).WithValue(uint32(600)),
}},
{Num: mesgnum.Sport, Fields: []proto.Field{
factory.CreateField(mesgnum.Sport, fieldnum.SportSport).WithValue(typedef.SportCycling),
factory.CreateField(mesgnum.Sport, fieldnum.SportSubSport).WithValue(typedef.SubSportGeneric),
factory.CreateField(mesgnum.Sport, fieldnum.SportName).WithValue("Cycling"),
}},
{Num: mesgnum.Sport, Fields: []proto.Field{
factory.CreateField(mesgnum.Sport, fieldnum.SportSport).WithValue(typedef.SportRunning),
factory.CreateField(mesgnum.Sport, fieldnum.SportSubSport).WithValue(typedef.SubSportTrail),
factory.CreateField(mesgnum.Sport, fieldnum.SportName).WithValue("Running"),
}},
{Num: mesgnum.SplitSummary, Fields: []proto.Field{
factory.CreateField(mesgnum.Session, proto.FieldNumTimestamp).WithValue(datetime.ToUint32(now) + 200), // Additional
factory.CreateField(mesgnum.SplitSummary, fieldnum.SplitSummarySplitType).WithValue(typedef.SplitTypeRunActive),
Expand All @@ -306,6 +338,8 @@ func TestCombine(t *testing.T) {
}},
{Num: mesgnum.Session, Fields: []proto.Field{
factory.CreateField(mesgnum.Session, fieldnum.SessionSport).WithValue(typedef.SportCycling),
factory.CreateField(mesgnum.Session, fieldnum.SessionSubSport).WithValue(typedef.SubSportGeneric),
factory.CreateField(mesgnum.Session, fieldnum.SessionSportProfileName).WithValue("Cycling"),
factory.CreateField(mesgnum.Session, fieldnum.SessionTimestamp).WithValue(datetime.ToUint32(now) + 200),
factory.CreateField(mesgnum.Session, fieldnum.SessionStartTime).WithValue(datetime.ToUint32(now)),
factory.CreateField(mesgnum.Session, fieldnum.SessionTotalDistance).WithValue(uint32(400)),
Expand All @@ -315,6 +349,8 @@ func TestCombine(t *testing.T) {
}},
{Num: mesgnum.Session, Fields: []proto.Field{
factory.CreateField(mesgnum.Session, fieldnum.SessionSport).WithValue(typedef.SportRunning),
factory.CreateField(mesgnum.Session, fieldnum.SessionSubSport).WithValue(typedef.SubSportTrail),
factory.CreateField(mesgnum.Session, fieldnum.SessionSportProfileName).WithValue("Running"),
factory.CreateField(mesgnum.Session, fieldnum.SessionTimestamp).WithValue(datetime.ToUint32(now) + 200),
factory.CreateField(mesgnum.Session, fieldnum.SessionStartTime).WithValue(datetime.ToUint32(now) + 100),
factory.CreateField(mesgnum.Session, fieldnum.SessionTotalDistance).WithValue(uint32(200)),
Expand Down
29 changes: 24 additions & 5 deletions internal/cmd/fitgen/profile/mesgdef/mesgdef.tmpl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,11 +31,10 @@ type {{ .Name }} struct {
state [{{ byteDiv .MaxFieldExpandNum 8 | byteAdd 1 }}]uint8 // Used for tracking expanded fields.
{{ end }}

UnknownFields []proto.Field // UnknownFields are fields that are exist but they are not defined in Profile.xlsx
{{- if and (not (eq .Name "FileId")) (not (eq .Name "DeveloperDataId")) (not (eq .Name "FieldDescription")) }}
// Developer Fields are dynamic, can't be mapped as struct's fields.
// [Added since protocol version 2.0]
DeveloperFields []proto.DeveloperField
{{ end -}}
DeveloperFields []proto.DeveloperField // DeveloperFields are custom data fields [Added since protocol version 2.0]
{{- end }}
}

// New{{ .Name }} creates new {{ .Name }} struct based on given mesg.
Expand All @@ -47,12 +46,16 @@ func New{{ .Name }}(mesg *proto.Message) *{{ .Name }} {
var state [{{ byteDiv .MaxFieldExpandNum 8 | byteAdd 1 }}]uint8
{{ end -}}

var unknownFields []proto.Field
{{ if and (not (eq .Name "FileId")) (not (eq .Name "DeveloperDataId")) (not (eq .Name "FieldDescription")) -}}
var developerFields []proto.DeveloperField
{{ end -}}
if mesg != nil {
arr := pool.Get().(*[poolsize]proto.Field)
unknownFields = arr[:0]
for i := range mesg.Fields {
if mesg.Fields[i].Num > {{ byteSub .MaxFieldNum 1 }} {
if mesg.Fields[i].Num > {{ byteSub .MaxFieldNum 1 }} || mesg.Fields[i].Name == factory.NameUnknown {
unknownFields = append(unknownFields, mesg.Fields[i])
continue
}
{{ if gt $.MaxFieldExpandNum 1 -}}
Expand All @@ -63,6 +66,11 @@ func New{{ .Name }}(mesg *proto.Message) *{{ .Name }} {
{{ end -}}
vals[mesg.Fields[i].Num] = mesg.Fields[i].Value
}
if len(unknownFields) == 0 {
unknownFields = nil
}
unknownFields = append(unknownFields[:0:0], unknownFields...)
pool.Put(arr)
{{- if and (not (eq .Name "FileId")) (not (eq .Name "DeveloperDataId")) (not (eq .Name "FieldDescription")) }}
developerFields = mesg.DeveloperFields
{{ end -}}
Expand All @@ -76,6 +84,7 @@ func New{{ .Name }}(mesg *proto.Message) *{{ .Name }} {
{{ if gt .MaxFieldExpandNum 1 -}}
state: state,
{{ end }}
UnknownFields: unknownFields,
{{- if and (not (eq .Name "FileId")) (not (eq .Name "DeveloperDataId")) (not (eq .Name "FieldDescription")) }}
DeveloperFields: developerFields,
{{ end }}
Expand Down Expand Up @@ -119,6 +128,10 @@ func (m *{{ .Name }}) ToMesg(options *Options) proto.Message {
}
{{ end }}

for i := range m.UnknownFields {
fields = append(fields, m.UnknownFields[i])
}

mesg.Fields = make([]proto.Field, len(fields))
copy(mesg.Fields, fields)
pool.Put(arr)
Expand Down Expand Up @@ -271,6 +284,12 @@ func (m *{{ $.Name }}) Set{{ .Name }}Degrees(degrees float64) *{{ $.Name }} {

{{ end }}

// SetDeveloperFields {{ $.Name }}'s UnknownFields (fields that are exist but they are not defined in Profile.xlsx)
func (m *{{ $.Name }}) SetUnknownFields(unknownFields ...proto.Field) *{{ $.Name }} {
m.UnknownFields = unknownFields
return m
}

{{- if and (not (eq .Name "FileId")) (not (eq .Name "DeveloperDataId")) (not (eq .Name "FieldDescription")) }}
// SetDeveloperFields {{ $.Name }}'s DeveloperFields.
func (m *{{ $.Name }}) SetDeveloperFields(developerFields ...proto.DeveloperField) *{{ $.Name }} {
Expand Down
Loading

0 comments on commit c943dbf

Please sign in to comment.