From 602a4cc71fa2b3db17c209950d5f7f38044c0ec4 Mon Sep 17 00:00:00 2001 From: Tim Barry <21149133+tim-barry@users.noreply.github.com> Date: Fri, 20 Dec 2024 10:04:49 -0800 Subject: [PATCH] Apply suggestions from code review Channel size of 20000 and worker count of 100 suggested by Leo. 20000 is approximately equal to 4 hours of execution results. Co-authored-by: Leo Zhang --- engine/execution/ingestion/uploader/uploader.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/engine/execution/ingestion/uploader/uploader.go b/engine/execution/ingestion/uploader/uploader.go index d1dcabb0cbc..d29a0d3a69c 100644 --- a/engine/execution/ingestion/uploader/uploader.go +++ b/engine/execution/ingestion/uploader/uploader.go @@ -35,10 +35,10 @@ func NewAsyncUploader(uploader Uploader, maxRetryNumber: maxRetryNumber, // we use a channel rather than a Fifoqueue here because a Fifoqueue might drop items when full, // but it is not acceptable to skip uploading an execution result - queue: make(chan *execution.ComputationResult, 100), + queue: make(chan *execution.ComputationResult, 20000), } builder := component.NewComponentManagerBuilder() - for i := 0; i < 3; i++ { + for i := 0; i < 10; i++ { builder.AddWorker(a.UploadWorker) } a.cm = builder.Build()