Release 1.17.11

- Prevent divide-by-zero error while loading csv files. ([GitHub #89](#89))
- Handle panics during processor executions.

CHANGELOG.md

@@ -1,5 +1,12 @@
 # Change Log
 
+## Version 1.17.11
+
+Released on Nov 5, 2022
+
+- Prevent divide-by-zero error while loading csv files. ([GitHub #89](https://github.com/mithrandie/csvq/issues/89))
+- Handle panics during processor executions.
+
 ## Version 1.17.10
 
 Released on Aug 14, 2022

README.md

@@ -42,12 +42,12 @@ This tool may be useful for those who want to handle data easily and roughly, wi
 * Transaction Management
 * Support loading data from Standard Input
 * Support following file formats
-  * CSV
+  * [CSV](https://datatracker.ietf.org/doc/html/rfc4180)
   * TSV
-  * LTSV
+  * [LTSV](http://ltsv.org)
   * Fixed-Length Format
-  * JSON
-  * JSON Lines
+  * [JSON](https://datatracker.ietf.org/doc/html/rfc8259)
+  * [JSON Lines](https://jsonlines.org)
 * Support following file encodings
   * UTF-8
   * UTF-16

docs/changelog.md

@@ -5,6 +5,13 @@ title: Change Log - csvq
 
 # Change Log
 
+## Version 1.17.11
+
+Released on Nov 5, 2022
+
+- Prevent divide-by-zero error while loading csv files. ([GitHub #89](https://github.com/mithrandie/csvq/issues/89))
+- Handle panics during processor executions.
+
 ## Version 1.17.10
 
 Released on Aug 14, 2022

docs/index.md

@@ -13,10 +13,10 @@ In the multiple operations, you can use variables, cursors, temporary tables, an
 
 ## Latest Release
 
-Version 1.17.10
-: Released on Aug 14, 2022
+Version 1.17.11
+: Released on Nov 5, 2022
 
-  <a class="waves-effect waves-light btn" href="https://github.com/mithrandie/csvq/releases/tag/v1.17.10">
+  <a class="waves-effect waves-light btn" href="https://github.com/mithrandie/csvq/releases/tag/v1.17.11">
     <i class="material-icons left">file_download</i>download
   </a>
 
@@ -46,12 +46,12 @@ This tool may be useful for those who want to handle data easily and roughly, wi
 * [Transaction Management]({{ '/reference/transaction.html' | relative_url }})
 * Support loading data from Standard Input
 * Support following file formats
-  * CSV
+  * [CSV](https://datatracker.ietf.org/doc/html/rfc4180)
   * TSV
-  * LTSV
+  * [LTSV](http://ltsv.org)
   * Fixed-Length Format
-  * [JSON]({{ '/reference/json.html' | relative_url }})
-  * JSON Lines
+  * [JSON](https://datatracker.ietf.org/doc/html/rfc8259)
+  * [JSON Lines](https://jsonlines.org)
 * Support following file encodings
   * UTF-8
   * UTF-16

docs/sitemap.xml

@@ -6,7 +6,7 @@
             http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd">
     <url>
         <loc>https://mithrandie.github.io/csvq/</loc>
-        <lastmod>2022-08-14T07:45:53+00:00</lastmod>
+        <lastmod>2022-11-05T14:32:12+00:00</lastmod>
     </url>
     <url>
         <loc>https://mithrandie.github.io/csvq/reference.html</loc>
@@ -182,7 +182,7 @@
     </url>
     <url>
         <loc>https://mithrandie.github.io/csvq/changelog.html</loc>
-        <lastmod>2022-08-14T07:45:53+00:00</lastmod>
+        <lastmod>2022-11-05T14:32:12+00:00</lastmod>
     </url>
     <url>
         <loc>https://mithrandie.github.io/csvq/license.html</loc>

lib/cli/help.go

@@ -3,7 +3,7 @@ package cli
 var appHHelpTemplate = `Name:
    {{.Name}}{{if .Usage}} - {{.Usage}}{{end}}
 
-     https://mithrandie.github.io/csvq
+     https://mithrandie.github.io/csvq/
 
 Usage:
    {{if .UsageText}}{{.UsageText}}{{else}}{{.HelpName}} {{if .VisibleFlags}}[options]{{end}}{{if .Commands}} [subcommand]{{end}} {{if .ArgsUsage}}{{.ArgsUsage}}{{else}}[arguments...]{{end}}{{end}}{{if .Version}}{{if not .HideVersion}}

lib/query/analytic_function.go

@@ -130,6 +130,18 @@ func Analyze(ctx context.Context, scope *ReferenceScope, view *View, fn parser.A
 	gm := NewGoroutineTaskManager(len(partitionMapKeys), minReq, scope.Tx.Flags.CPU)
 
 	var analyzeFn = func(thIdx int) {
+		defer func() {
+			if !gm.HasError() {
+				if panicReport := recover(); panicReport != nil {
+					gm.SetError(NewFatalError(panicReport))
+				}
+			}
+
+			if 1 < gm.Number {
+				gm.Done()
+			}
+		}()
+
 		start, end := gm.RecordRange(thIdx)
 		seqScope := scope.CreateScopeForSequentialEvaluation(view)
 
@@ -197,10 +209,6 @@ func Analyze(ctx context.Context, scope *ReferenceScope, view *View, fn parser.A
 				}
 			}
 		}
-
-		if 1 < gm.Number {
-			gm.Done()
-		}
 	}
 
 	if 1 < gm.Number {

lib/query/error.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"os"
 	"reflect"
+	"runtime"
 	"strconv"
 	"strings"
 
@@ -266,6 +267,34 @@ func NewBaseErrorWithPrefix(prefix string, message string, code int, number int)
 	}
 }
 
+type FatalError struct {
+	*BaseError
+}
+
+func NewFatalError(panicReport interface{}) error {
+	stacks := make([]string, 0, 30)
+	for depth := 0; ; depth++ {
+		pc, src, line, ok := runtime.Caller(depth)
+		if !ok {
+			break
+		}
+		if depth == 0 {
+			continue
+		}
+		stacks = append(stacks, fmt.Sprintf("  %d: %s [%s:%d]", depth-1, runtime.FuncForPC(pc).Name(), src, line))
+	}
+
+	message := fmt.Sprintf("%v\n", panicReport) +
+		"An unexpected error has occurred. Please report this problem to: https://github.com/mithrandie/csvq/issues\n" +
+		"\n" +
+		"Stack:\n" +
+		strings.Join(stacks, "\n")
+
+	return &FatalError{
+		NewBaseErrorWithPrefix("Fatal Error", message, ReturnCodeApplicationError, ErrorFatal),
+	}
+}
+
 type SystemError struct {
 	*BaseError
 }

lib/query/error_code.go

@@ -15,6 +15,7 @@ const (
 
 const (
 	//Application Error
+	ErrorFatal                                = 1
 	ErrorCannotDetectFileEncoding             = 10001
 	ErrorFieldAmbiguous                       = 10101
 	ErrorFieldNotExist                        = 10102

lib/query/eval.go

@@ -102,6 +102,18 @@ func Evaluate(ctx context.Context, scope *ReferenceScope, expr parser.QueryExpre
 }
 
 func evaluateSequentialRoutine(ctx context.Context, scope *ReferenceScope, view *View, fn func(*ReferenceScope, int) error, thIdx int, gm *GoroutineTaskManager) {
+	defer func() {
+		if !gm.HasError() {
+			if panicReport := recover(); panicReport != nil {
+				gm.SetError(NewFatalError(panicReport))
+			}
+		}
+
+		if 1 < gm.Number {
+			gm.Done()
+		}
+	}()
+
 	start, end := gm.RecordRange(thIdx)
 	seqScope := scope.CreateScopeForSequentialEvaluation(
 		&View{
@@ -127,10 +139,6 @@ func evaluateSequentialRoutine(ctx context.Context, scope *ReferenceScope, view
 
 		i++
 	}
-
-	if 1 < gm.Number {
-		gm.Done()
-	}
 }
 
 func EvaluateSequentially(ctx context.Context, scope *ReferenceScope, view *View, fn func(*ReferenceScope, int) error) error {

lib/query/goroutine_manager.go

@@ -143,6 +143,18 @@ func (m *GoroutineTaskManager) Wait() {
 }
 
 func (m *GoroutineTaskManager) run(ctx context.Context, fn func(int) error, thIdx int) {
+	defer func() {
+		if !m.HasError() {
+			if panicReport := recover(); panicReport != nil {
+				m.SetError(NewFatalError(panicReport))
+			}
+		}
+
+		if 1 < m.Number {
+			m.Done()
+		}
+	}()
+
 	start, end := m.RecordRange(thIdx)
 
 	for i := start; i < end; i++ {
@@ -158,10 +170,6 @@ func (m *GoroutineTaskManager) run(ctx context.Context, fn func(int) error, thId
 			break
 		}
 	}
-
-	if 1 < m.Number {
-		m.Done()
-	}
 }
 
 func (m *GoroutineTaskManager) Run(ctx context.Context, fn func(int) error) error {

lib/query/join.go

@@ -142,6 +142,18 @@ func InnerJoin(ctx context.Context, scope *ReferenceScope, view *View, joinView
 	recordsList := make([]RecordSet, gm.Number)
 
 	var joinFn = func(thIdx int) {
+		defer func() {
+			if !gm.HasError() {
+				if panicReport := recover(); panicReport != nil {
+					gm.SetError(NewFatalError(panicReport))
+				}
+			}
+
+			if 1 < gm.Number {
+				gm.Done()
+			}
+		}()
+
 		ctx := ctx
 		start, end := gm.RecordRange(thIdx)
 		records := make(RecordSet, 0, end-start)
@@ -183,10 +195,6 @@ func InnerJoin(ctx context.Context, scope *ReferenceScope, view *View, joinView
 		}
 
 		recordsList[thIdx] = records
-
-		if 1 < gm.Number {
-			gm.Done()
-		}
 	}
 
 	if 1 < gm.Number {
@@ -235,6 +243,18 @@ func OuterJoin(ctx context.Context, scope *ReferenceScope, view *View, joinView
 	joinViewMatchesList := make([][]bool, gm.Number)
 
 	var joinFn = func(thIdx int) {
+		defer func() {
+			if !gm.HasError() {
+				if panicReport := recover(); panicReport != nil {
+					gm.SetError(NewFatalError(panicReport))
+				}
+			}
+
+			if 1 < gm.Number {
+				gm.Done()
+			}
+		}()
+
 		ctx := ctx
 		start, end := gm.RecordRange(thIdx)
 		records := make(RecordSet, 0, end-start)
@@ -318,10 +338,6 @@ func OuterJoin(ctx context.Context, scope *ReferenceScope, view *View, joinView
 
 		recordsList[thIdx] = records
 		joinViewMatchesList[thIdx] = joinViewMatches
-
-		if 1 < gm.Number {
-			gm.Done()
-		}
 	}
 
 	if 1 < gm.Number {