From 09b4b3eae8826fac0fcc4d1505eb00179d508cf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Senart?= Date: Wed, 22 Jun 2022 17:26:02 +0200 Subject: [PATCH] ulid: add DefaultEntropy() and Make() (#81) Users of this library have opened many issues regarding the difficulty of choosing an entropy source that is safe for concurrent use. This commit introduces a thread safe per process monotonically increase `DefaultEntropy()` function as well as an easy to use `Make()` function, aimed at users that want safe defaults chosen for them. Co-authored-by: Peter Bourgon --- README.md | 70 ++++++++++++++++++++++++++++++++++++++++++++-------- ulid.go | 48 +++++++++++++++++++++++++++++++++-- ulid_test.go | 46 ++++++++++++++++------------------ 3 files changed, 127 insertions(+), 37 deletions(-) diff --git a/README.md b/README.md index 630e0d2..c0094ce 100644 --- a/README.md +++ b/README.md @@ -39,21 +39,71 @@ go get github.com/oklog/ulid/v2 ## Usage -An ULID is constructed with a `time.Time` and an `io.Reader` entropy source. -This design allows for greater flexibility in choosing your trade-offs. +ULIDs are constructed from two things: a timestamp with millisecond precision, +and some random data. + +Timestamps are modeled as uint64 values representing a Unix time in milliseconds. +They can be produced by passing a [time.Time](https://pkg.go.dev/time#Time) to +[ulid.Timestamp](https://pkg.go.dev/github.com/oklog/ulid/v2#Timestamp), +or by calling [time.Time.UnixMilli](https://pkg.go.dev/time#Time.UnixMilli) +and converting the returned value to `uint64`. + +Random data is taken from a provided [io.Reader](https://pkg.go.dev/io#Reader). +This design allows for greater flexibility when choosing trade-offs, but can be +a bit confusing to newcomers. + +If you just want to generate a ULID and don't (yet) care about details like +performance, cryptographic security, monotonicity, etc., use the +[ulid.Make](https://pkg.go.dev/github.com/oklog/ulid/v2#Make) helper function. +This function calls [time.Now](https://pkg.go.dev/time#Now) to get a timestamp, +and uses a source of entropy which is process-global, +[pseudo-random](https://pkg.go.dev/math/rand)), and +[monotonic](https://pkg.go.dev/oklog/ulid/v2#LockedMonotonicReader)). -Please note that `rand.Rand` from the `math` package is *not* safe for concurrent use. -Instantiate one per long living go-routine or use a `sync.Pool` if you want to avoid the potential contention of a locked `rand.Source` as its been frequently observed in the package level functions. +```go +println(ulid.Make()) +// 01G65Z755AFWAKHE12NY0CQ9FH +``` + +More advanced use cases should utilize +[ulid.New](https://pkg.go.dev/github.com/oklog/ulid/v2#New). ```go -func ExampleULID() { - t := time.Unix(1000000, 0) - entropy := ulid.Monotonic(rand.New(rand.NewSource(t.UnixNano())), 0) - fmt.Println(ulid.MustNew(ulid.Timestamp(t), entropy)) - // Output: 0000XSNJG0MQJHBF4QX1EFD6Y3 -} +entropy := rand.New(rand.NewSource(time.Now().UnixNano())) +ms := ulid.Timestamp(time.Now()) +println(ulid.New(ms, entropy)) +// 01G65Z755AFWAKHE12NY0CQ9FH ``` +Care should be taken when providing a source of entropy. + +The above example utilizes [math/rand.Rand](https://pkg.go.dev/math/rand#Rand), +which is not safe for concurrent use by multiple goroutines. Consider +alternatives such as +[x/exp/rand](https://pkg.go.dev/golang.org/x/exp/rand#LockedSource). +Security-sensitive use cases should always use cryptographically secure entropy +provided by [crypto/rand](https://pkg.go.dev/crypto/rand). + +Performance-sensitive use cases should avoid synchronization when generating +IDs. One option is to use a unique source of entropy for each concurrent +goroutine, which results in no lock contention, but cannot provide strong +guarantees about the random data, and does not provide monotonicity within a +given millisecond. One common performance optimization is to pool sources of +entropy using a [sync.Pool](https://pkg.go.dev/sync#Pool). + +Monotonicity is a property that says each ULID is "bigger than" the previous +one. ULIDs are automatically monotonic, but only to millisecond precision. ULIDs +generated within the same millisecond are ordered by their random component, +which means they are by default un-ordered. You can use +[ulid.MonotonicEntropy](https://pkg.go.dev/oklog/ulid/v2#MonotonicEntropy) or +[ulid.LockedMonotonicEntropy](https://pkg.go.dev/oklog/ulid/v2#LockedMonotonicEntropy) +to create ULIDs that are monotonic within a given millisecond, with caveats. See +the documentation for details. + +If you don't care about time-based ordering of generated IDs, then there's no +reason to use ULIDs! There are many other kinds of IDs that are easier, faster, +smaller, etc. Consider UUIDs. + ## Commandline tool This repo also provides a tool to generate and parse ULIDs at the command line. diff --git a/ulid.go b/ulid.go index 9f98159..0cb258d 100644 --- a/ulid.go +++ b/ulid.go @@ -23,6 +23,7 @@ import ( "math" "math/bits" "math/rand" + "sync" "time" ) @@ -121,6 +122,32 @@ func MustNew(ms uint64, entropy io.Reader) ULID { return id } +var ( + entropy io.Reader + entropyOnce sync.Once +) + +// DefaultEntropy returns a thread-safe per process monotonically increasing +// entropy source. +func DefaultEntropy() io.Reader { + entropyOnce.Do(func() { + rng := rand.New(rand.NewSource(time.Now().UnixNano())) + entropy = &LockedMonotonicReader{ + MonotonicReader: Monotonic(rng, 0), + } + }) + return entropy +} + +// Make returns an ULID with the current time in Unix milliseconds and +// monotonically increasing entropy for the same millisecond. +// It is safe for concurrent use, leveraging a sync.Pool underneath for minimal +// contention. +func Make() (id ULID) { + // NOTE: MustNew can't panic since DefaultEntropy never returns an error. + return MustNew(Now(), DefaultEntropy()) +} + // Parse parses an encoded ULID, returning an error in case of failure. // // ErrDataSize is returned if the len(ulid) is different from an encoded @@ -531,13 +558,30 @@ func Monotonic(entropy io.Reader, inc uint64) *MonotonicEntropy { m.inc = math.MaxUint32 } - if rng, ok := entropy.(*rand.Rand); ok { + if rng, ok := entropy.(rng); ok { m.rng = rng } return &m } +type rng interface{ Int63n(n int64) int64 } + +// LockedMonotonicReader wraps a MonotonicReader with a sync.Mutex for +// safe concurrent use. +type LockedMonotonicReader struct { + mu sync.Mutex + MonotonicReader +} + +// MonotonicRead synchronizes calls to the wrapped MonotonicReader. +func (r *LockedMonotonicReader) MonotonicRead(ms uint64, p []byte) (err error) { + r.mu.Lock() + err = r.MonotonicReader.MonotonicRead(ms, p) + r.mu.Unlock() + return err +} + // MonotonicEntropy is an opaque type that provides monotonic entropy. type MonotonicEntropy struct { io.Reader @@ -545,7 +589,7 @@ type MonotonicEntropy struct { inc uint64 entropy uint80 rand [8]byte - rng *rand.Rand + rng rng } // MonotonicRead implements the MonotonicReader interface. diff --git a/ulid_test.go b/ulid_test.go index e6c044a..2a9476d 100644 --- a/ulid_test.go +++ b/ulid_test.go @@ -21,7 +21,6 @@ import ( "math" "math/rand" "strings" - "sync" "testing" "testing/iotest" "testing/quick" @@ -61,6 +60,18 @@ func TestNew(t *testing.T) { }) } +func TestMake(t *testing.T) { + t.Parallel() + id := ulid.Make() + rt, err := ulid.Parse(id.String()) + if err != nil { + t.Fatalf("parse %q: %v", id.String(), err) + } + if id != rt { + t.Fatalf("%q != %q", id.String(), rt.String()) + } +} + func TestMustNew(t *testing.T) { t.Parallel() @@ -142,7 +153,7 @@ func TestRoundTrips(t *testing.T) { id == ulid.MustParseStrict(id.String()) } - err := quick.Check(prop, &quick.Config{MaxCount: 1E5}) + err := quick.Check(prop, &quick.Config{MaxCount: 1e5}) if err != nil { t.Fatal(err) } @@ -229,7 +240,7 @@ func TestEncoding(t *testing.T) { return true } - if err := quick.Check(prop, &quick.Config{MaxCount: 1E5}); err != nil { + if err := quick.Check(prop, &quick.Config{MaxCount: 1e5}); err != nil { t.Fatal(err) } } @@ -258,7 +269,7 @@ func TestLexicographicalOrder(t *testing.T) { top = next } - if err := quick.Check(prop, &quick.Config{MaxCount: 1E6}); err != nil { + if err := quick.Check(prop, &quick.Config{MaxCount: 1e6}); err != nil { t.Fatal(err) } } @@ -316,7 +327,7 @@ func TestParseRobustness(t *testing.T) { return err == nil } - err := quick.Check(prop, &quick.Config{MaxCount: 1E4}) + err := quick.Check(prop, &quick.Config{MaxCount: 1e4}) if err != nil { t.Fatal(err) } @@ -368,7 +379,7 @@ func TestTimestampRoundTrips(t *testing.T) { return ts == ulid.Timestamp(ulid.Time(ts)) } - err := quick.Check(prop, &quick.Config{MaxCount: 1E5}) + err := quick.Check(prop, &quick.Config{MaxCount: 1e5}) if err != nil { t.Fatal(err) } @@ -447,7 +458,7 @@ func TestEntropyRead(t *testing.T) { return eq } - if err := quick.Check(prop, &quick.Config{MaxCount: 1E4}); err != nil { + if err := quick.Check(prop, &quick.Config{MaxCount: 1e4}); err != nil { t.Fatal(err) } } @@ -463,7 +474,7 @@ func TestCompare(t *testing.T) { return a.Compare(b) } - err := quick.CheckEqual(a, b, &quick.Config{MaxCount: 1E5}) + err := quick.CheckEqual(a, b, &quick.Config{MaxCount: 1e5}) if err != nil { t.Error(err) } @@ -586,11 +597,8 @@ func TestMonotonicSafe(t *testing.T) { t.Parallel() var ( - src = rand.NewSource(time.Now().UnixNano()) - entropy = rand.New(src) - monotonic = ulid.Monotonic(entropy, 0) - safe = &safeMonotonicReader{MonotonicReader: monotonic} - t0 = ulid.Timestamp(time.Now()) + safe = ulid.DefaultEntropy() + t0 = ulid.Timestamp(time.Now()) ) errs := make(chan error, 100) @@ -630,18 +638,6 @@ func TestULID_Bytes(t *testing.T) { } } -type safeMonotonicReader struct { - mtx sync.Mutex - ulid.MonotonicReader -} - -func (r *safeMonotonicReader) MonotonicRead(ms uint64, p []byte) (err error) { - r.mtx.Lock() - err = r.MonotonicReader.MonotonicRead(ms, p) - r.mtx.Unlock() - return err -} - func BenchmarkNew(b *testing.B) { benchmarkMakeULID(b, func(timestamp uint64, entropy io.Reader) { _, _ = ulid.New(timestamp, entropy)