Skip to content

Commit

Permalink
Change benchmark to do multiple runs with increasing data sizes
Browse files Browse the repository at this point in the history
  • Loading branch information
@samwillis
samwillis committed Dec 21, 2024
1 parent 0ccdc5a commit 51bc937
Show file tree
Hide file tree
Showing 2 changed files with 78 additions and 75 deletions.
8 changes: 4 additions & 4 deletions packages/d2ts-benchmark/src/base.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,11 +111,11 @@ export class Suite {
// Calculate data for the table
const headers = [
'Benchmark',
'Startup (ms)',
// 'Startup (ms)',
'First Run (ms)',
'Incremental (ms)',
'ops/sec',
'Teardown (ms)',
// 'Teardown (ms)',
]
const rows = Array.from(resultsByBenchmark.entries()).map(
([name, results]) => {
Expand All @@ -127,11 +127,11 @@ export class Suite {

return [
name,
formatAverage(results.map((r) => r.startupTime)),
// formatAverage(results.map((r) => r.startupTime)),
formatAverage(results.map((r) => r.firstRunTime)),
formatAverage(avgIncrementalMs),
formatOpsPerSec(avgIncrementalMs.map((ms) => 1000 / ms)),
formatAverage(results.map((r) => r.teardownTime)),
// formatAverage(results.map((r) => r.teardownTime)),
]
},
)
Expand Down
145 changes: 74 additions & 71 deletions packages/d2ts-benchmark/src/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,32 +69,35 @@ function run({
})

// Add naive join benchmark
joinSuite.add({
name: 'Naive Join',
setup: () => ({
currentUsers: [...initialUsers],
currentPosts: [...initialPosts],
result: [] as { userName: string; postTitle: string }[],
}),
firstRun: (ctx) => {
ctx.result = ctx.currentUsers.flatMap((user) =>
ctx.currentPosts
.filter((post) => post.userId === user.id)
.map((post) => ({ userName: user.name, postTitle: post.title })),
)
},
incrementalRun: (ctx, i) => {
ctx.currentUsers.push(incrementalUsers[i])
ctx.currentPosts.push(incrementalPosts[i * 2])
ctx.currentPosts.push(incrementalPosts[i * 2 + 1])

ctx.result = ctx.currentUsers.flatMap((user) =>
ctx.currentPosts
.filter((post) => post.userId === user.id)
.map((post) => ({ userName: user.name, postTitle: post.title })),
)
},
})
// Skip for large datasets
if (initialSize <= 9000) {
joinSuite.add({
name: 'Naive Join',
setup: () => ({
currentUsers: [...initialUsers],
currentPosts: [...initialPosts],
result: [] as { userName: string; postTitle: string }[],
}),
firstRun: (ctx) => {
ctx.result = ctx.currentUsers.flatMap((user) =>
ctx.currentPosts
.filter((post) => post.userId === user.id)
.map((post) => ({ userName: user.name, postTitle: post.title })),
)
},
incrementalRun: (ctx, i) => {
ctx.currentUsers.push(incrementalUsers[i])
ctx.currentPosts.push(incrementalPosts[i * 2])
ctx.currentPosts.push(incrementalPosts[i * 2 + 1])

ctx.result = ctx.currentUsers.flatMap((user) =>
ctx.currentPosts
.filter((post) => post.userId === user.id)
.map((post) => ({ userName: user.name, postTitle: post.title })),
)
},
})
}

// Add naive indexed join benchmark
joinSuite.add({
Expand Down Expand Up @@ -203,53 +206,53 @@ function run({
},
})

// Add D2TS join with frontier benchmark
joinSuite.add({
name: 'D2TS Join with Frontier',
setup: () => {
const graph = new D2({ initialFrontier: v([0]) })
const usersStream = graph.newInput<[number, (typeof allUsers)[0]]>()
const postsStream = graph.newInput<[number, (typeof allPosts)[0]]>()
// // Add D2TS join with frontier benchmark
// joinSuite.add({
// name: 'D2TS Join with Frontier',
// setup: () => {
// const graph = new D2({ initialFrontier: v([0]) })
// const usersStream = graph.newInput<[number, (typeof allUsers)[0]]>()
// const postsStream = graph.newInput<[number, (typeof allPosts)[0]]>()

const joined = usersStream.pipe(
join(postsStream),
map(([_key, [user, post]]) => ({
userName: user.name,
postTitle: post.title,
})),
output((_data) => {
// do nothing
}),
)
// const joined = usersStream.pipe(
// join(postsStream),
// map(([_key, [user, post]]) => ({
// userName: user.name,
// postTitle: post.title,
// })),
// output((_data) => {
// // do nothing
// }),
// )

graph.finalize()
return { graph, usersStream, postsStream, joined }
},
firstRun: (ctx) => {
ctx.usersStream.sendData(v([1]), initialUsersSet)
ctx.postsStream.sendData(v([1]), initialPostsSet)
ctx.usersStream.sendFrontier(v([2]))
ctx.postsStream.sendFrontier(v([2]))
ctx.graph.step()
},
incrementalRun: (ctx, i) => {
const user = incrementalUsers[i]
const post1 = incrementalPosts[i * 2]
const post2 = incrementalPosts[i * 2 + 1]
// graph.finalize()
// return { graph, usersStream, postsStream, joined }
// },
// firstRun: (ctx) => {
// ctx.usersStream.sendData(v([1]), initialUsersSet)
// ctx.postsStream.sendData(v([1]), initialPostsSet)
// ctx.usersStream.sendFrontier(v([2]))
// ctx.postsStream.sendFrontier(v([2]))
// ctx.graph.step()
// },
// incrementalRun: (ctx, i) => {
// const user = incrementalUsers[i]
// const post1 = incrementalPosts[i * 2]
// const post2 = incrementalPosts[i * 2 + 1]

ctx.usersStream.sendData(v([i + 2]), new MultiSet([[[user.id, user], 1]]))
ctx.postsStream.sendData(
v([i + 2]),
new MultiSet([
[[post1.userId, post1], 1],
[[post2.userId, post2], 1],
]),
)
ctx.usersStream.sendFrontier(v([i + 3]))
ctx.postsStream.sendFrontier(v([i + 3]))
ctx.graph.step()
},
})
// ctx.usersStream.sendData(v([i + 2]), new MultiSet([[[user.id, user], 1]]))
// ctx.postsStream.sendData(
// v([i + 2]),
// new MultiSet([
// [[post1.userId, post1], 1],
// [[post2.userId, post2], 1],
// ]),
// )
// ctx.usersStream.sendFrontier(v([i + 3]))
// ctx.postsStream.sendFrontier(v([i + 3]))
// ctx.graph.step()
// },
// })

// Add SQLite join benchmark
joinSuite.add({
Expand Down Expand Up @@ -482,4 +485,4 @@ function run({
run({ initialSize: 90, incrementalRuns: 1000 })
run({ initialSize: 900, incrementalRuns: 1000 })
run({ initialSize: 9000, incrementalRuns: 1000 })
// run({ initialSize: 90000, incrementalRuns: 1000 })
run({ initialSize: 90000, incrementalRuns: 1000 })

0 comments on commit 51bc937

Please sign in to comment.