Skip to content

Commit

Permalink
CLI: accept RFC3339-formatted dates in relevant arguments (#1263)
Browse files Browse the repository at this point in the history
### Changelog
#### Added
CLI: `mcap add attachment` accepts RFC3339-formatted dates to specify
the log and creation times for the new attachment.
CLI: `mcap filter` accepts RFC3339-formatted dates for start and end
times.
### Docs

None.

### Description

Adds the ability for users to specify a date when filtering MCAP files,
or adding attachments.

I did a quick pass to see if there are any other arguments where we
accept a date or date-like value, I can't see any more.
  • Loading branch information
james-rms authored Nov 4, 2024
1 parent 693030f commit ba3c9c0
Show file tree
Hide file tree
Showing 3 changed files with 102 additions and 26 deletions.
36 changes: 26 additions & 10 deletions go/cli/mcap/cmd/attachment.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,9 @@ import (
)

var (
addAttachmentLogTime uint64
addAttachmentLogTime string
addAttachmentName string
addAttachmentCreationTime uint64
addAttachmentCreationTime string
addAttachmentFilename string
addAttachmentMediaType string
)
Expand Down Expand Up @@ -155,12 +155,20 @@ var addAttachmentCmd = &cobra.Command{
die("failed to stat file %s", addAttachmentFilename)
}
createTime := uint64(fi.ModTime().UTC().UnixNano())
if addAttachmentCreationTime > 0 {
createTime = addAttachmentCreationTime
if addAttachmentCreationTime != "" {
date, err := parseDateOrNanos(addAttachmentCreationTime)
if err != nil {
die("failed to parse creation date: %s", err)
}
createTime = date
}
logTime := uint64(time.Now().UTC().UnixNano())
if addAttachmentLogTime > 0 {
logTime = addAttachmentLogTime
if addAttachmentLogTime != "" {
date, err := parseDateOrNanos(addAttachmentCreationTime)
if err != nil {
die("failed to parse log date: %s", err)
}
logTime = date
}
err = utils.AmendMCAP(f, []*mcap.Attachment{
{
Expand All @@ -187,11 +195,19 @@ func init() {
addAttachmentCmd.PersistentFlags().StringVarP(
&addAttachmentMediaType, "content-type", "", "application/octet-stream", "content type of attachment",
)
addAttachmentCmd.PersistentFlags().Uint64VarP(
&addAttachmentLogTime, "log-time", "", 0, "attachment log time in nanoseconds (defaults to current timestamp)",
addAttachmentCmd.PersistentFlags().StringVarP(
&addAttachmentLogTime,
"log-time",
"",
"",
"attachment log time in nanoseconds or RFC3339 format (defaults to current timestamp)",
)
addAttachmentCmd.PersistentFlags().Uint64VarP(
&addAttachmentLogTime, "creation-time", "", 0, "attachment creation time in nanoseconds (defaults to ctime)",
addAttachmentCmd.PersistentFlags().StringVarP(
&addAttachmentLogTime,
"creation-time",
"",
"",
"attachment creation time in nanoseconds or RFC3339 format (defaults to ctime)",
)
err := addAttachmentCmd.MarkPersistentFlagRequired("file")
if err != nil {
Expand Down
82 changes: 66 additions & 16 deletions go/cli/mcap/cmd/filter.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ import (
"math"
"os"
"regexp"
"strconv"
"time"

"github.com/foxglove/mcap/go/cli/mcap/utils"
"github.com/foxglove/mcap/go/mcap"
Expand All @@ -22,6 +24,8 @@ type filterFlags struct {
endSec uint64
startNano uint64
endNano uint64
start string
end string
includeMetadata bool
includeAttachments bool
outputCompression string
Expand All @@ -41,20 +45,51 @@ type filterOpts struct {
chunkSize int64
}

// parseDateOrNanos parses a string containing either an RFC3339-formatted date with timezone
// or a decimal number of nanoseconds. It returns a uint64 timestamp in nanoseconds.
func parseDateOrNanos(dateOrNanos string) (uint64, error) {
intNanos, err := strconv.ParseUint(dateOrNanos, 10, 64)
if err == nil {
return intNanos, nil
}
date, err := time.Parse(time.RFC3339, dateOrNanos)
if err != nil {
return 0, err
}
return uint64(date.UnixNano()), nil
}

// parseTimestampArgs implements the semantics for setting start and end times in the CLI.
// a non-default value in `dateOrNanos` overrides `nanoseconds`, which overrides `seconds`.
func parseTimestampArgs(dateOrNanos string, nanoseconds uint64, seconds uint64) (uint64, error) {
if dateOrNanos != "" {
return parseDateOrNanos(dateOrNanos)
}
if nanoseconds != 0 {
return nanoseconds, nil
}
return seconds * 1e9, nil
}

func buildFilterOptions(flags *filterFlags) (*filterOpts, error) {
opts := &filterOpts{
output: flags.output,
includeMetadata: flags.includeMetadata,
includeAttachments: flags.includeAttachments,
}
opts.start = flags.startNano
if flags.startSec > 0 {
opts.start = flags.startSec * 1e9
}
opts.end = flags.endNano
if flags.endSec > 0 {
opts.end = flags.endSec * 1e9
start, err := parseTimestampArgs(flags.start, flags.startNano, flags.startSec)
if err != nil {
return nil, fmt.Errorf("invalid start: %w", err)
}
opts.start = start
end, err := parseTimestampArgs(flags.end, flags.endSec, flags.endNano)
if err != nil {
return nil, fmt.Errorf("invalid end: %w", err)
}
opts.end = end
if opts.end == 0 {
opts.end = math.MaxUint64
}
Expand Down Expand Up @@ -392,30 +427,43 @@ usage:
[]string{},
"messages with topic names matching this regex will be excluded, can be supplied multiple times",
)
start := filterCmd.PersistentFlags().StringP(
"start",
"S",
"",
"only include messages logged at or after this time. Accepts integer nanoseconds or RFC3339-formatted date.",
)
startSec := filterCmd.PersistentFlags().Uint64P(
"start-secs",
"s",
0,
"messages with log times after or equal to this timestamp will be included.",
"only include messages logged at or after this time. Accepts integer seconds."+
"Ignored if `--start` or `--start-nsecs` are used.",
)
startNano := filterCmd.PersistentFlags().Uint64(
"start-nsecs",
0,
"deprecated, use --start. Only include messages logged at or after this time. Accepts integer nanoseconds.",
)
end := filterCmd.PersistentFlags().StringP(
"end",
"E",
"",
"Only include messages logged before this time. Accepts integer nanoseconds or RFC3339-formatted date.",
)
endSec := filterCmd.PersistentFlags().Uint64P(
"end-secs",
"e",
0,
"messages with log times before timestamp will be included.",
"only include messages logged before this time. Accepts integer seconds."+
"Ignored if `--end` or `--end-nsecs` are used.",
)
startNano := filterCmd.PersistentFlags().Uint64P(
"start-nsecs",
"S",
0,
"messages with log times after or equal to this nanosecond-precision timestamp will be included.",
)
endNano := filterCmd.PersistentFlags().Uint64P(
endNano := filterCmd.PersistentFlags().Uint64(
"end-nsecs",
"E",
0,
"messages with log times before nanosecond-precision timestamp will be included.",
"(Deprecated, use --end) Only include messages logged before this time. Accepts integer nanosconds.",
)

filterCmd.MarkFlagsMutuallyExclusive("start-secs", "start-nsecs")
filterCmd.MarkFlagsMutuallyExclusive("end-secs", "end-nsecs")
chunkSize := filterCmd.PersistentFlags().Int64P("chunk-size", "", 4*1024*1024, "chunk size of output file")
Expand All @@ -439,9 +487,11 @@ usage:
output: *output,
includeTopics: *includeTopics,
excludeTopics: *excludeTopics,
start: *start,
startSec: *startSec,
endSec: *endSec,
startNano: *startNano,
end: *end,
endSec: *endSec,
endNano: *endNano,
chunkSize: *chunkSize,
includeMetadata: *includeMetadata,
Expand Down
10 changes: 10 additions & 0 deletions go/cli/mcap/cmd/filter_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -368,3 +368,13 @@ func TestCompileMatchers(t *testing.T) {
assert.True(t, matchers[0].MatchString("camera"))
assert.True(t, matchers[1].MatchString("lights"))
}

func TestParseDateOrNanos(t *testing.T) {
expected := uint64(1690298850132545471)
zulu, err := parseDateOrNanos("2023-07-25T15:27:30.132545471Z")
require.NoError(t, err)
assert.Equal(t, expected, zulu)
withTimezone, err := parseDateOrNanos("2023-07-26T01:27:30.132545471+10:00")
require.NoError(t, err)
assert.Equal(t, expected, withTimezone)
}

0 comments on commit ba3c9c0

Please sign in to comment.