close #84: separate reference frame selection for alignment and histo…

…gram matching

cmd/nightlight/main.go

@@ -57,7 +57,7 @@ var job = flag.String("job", "", "JSON job specification to run")
 var out = flag.String("out", "out.fits", "save output to `file`")
 var jpg = flag.String("jpg", "%auto", "save 8bit preview of output as JPEG to `file`. `%auto` replaces suffix of output file with .jpg")
 var jpgGamma = flag.Float64("jpgGamma", 1.0, "gamma correction for JPG output, 1.0=off")
-var tiff = flag.String("tiff", "", "save 16bit preview of output as TIFF to `file`. `%auto` replaces suffix of output file with .tiff")
+var tiff = flag.String("tiff", "", "save 16bit preview of output as TIFF to `file`. `%auto` replaces suffix of output file with .tif")
 var log = flag.String("log", "%auto", "save log output to `file`. `%auto` replaces suffix of output file with .log")
 var pPre = flag.String("pre", "", "save pre-processed frames with given filename pattern, e.g. `pre%04d.fits`")
 var stars = flag.String("stars", "", "save star detections with given filename pattern, e.g. `stars%04d.fits`")
@@ -112,7 +112,8 @@ var stSigHigh = flag.Float64("stSigHigh", -1, "high sigma for stacking as multip
 var stWeight = flag.Int64("stWeight", 0, "weights for stacking. 0=unweighted (default), 1=by exposure, 2=by inverse noise")
 var stMemory = flag.Int64("stMemory", int64((totalMiBs*7)/10), "total MiB of memory to use for stacking, default=0.7x physical memory")
 
-var refSelMode = flag.Int64("refSelMode", 0, "reference frame selection mode, 0=best #stars/HFR (default), 1=median HFR (for master flats)")
+var histoRef = flag.String("histoRef", "%starsHFR", "histogram reference, %starsHFR= best #stars/HFR (default), %location=median location, any int=image ID, filename=image filename")
+var alignRef = flag.String("alignRef", "%starsHFR", "alignment reference, %starsHFR= best #stars/HFR (default), %location=median location, any int=image ID, filename=image filename")
 
 var neutSigmaLow = flag.Float64("neutSigmaLow", -1, "neutralize background color below this threshold, <0 = no op")
 var neutSigmaHigh = flag.Float64("neutSigmaHigh", -1, "keep background color above this threshold, interpolate in between, <0 = no op")
@@ -201,7 +202,7 @@ Flags:
 
 	// auto-fill filenames for secondary targets
 	autoFill(jpg, *out, ".jpg")
-	autoFill(tiff, *out, ".tiff")
+	autoFill(tiff, *out, ".tif")
 
 	// Enable CPU profiling if flagged
 	if *cpuprofile != "" {
@@ -307,7 +308,8 @@ Flags:
 			stack.NewOpStackBatches(
 				ops.NewOpSequence(
 					opPreProc,
-					ref.NewOpSelectReference(ref.RefSelMode(*refSelMode), *alignTo, 0, opStarDetect),
+					ref.NewOpSelectReference(ref.SRHisto, *histoRef, opStarDetect),
+					ref.NewOpSelectReference(ref.SRAlign, *alignRef, opStarDetect),
 					post.NewOpMatchHistogram(post.HistoNormMode(*normHist)),
 					post.NewOpAlign(int32(*alignK), float32(*alignT), post.OOBModeNaN),
 					ops.NewOpSave(*pPost, ops.EMMinMax, 1),
@@ -338,7 +340,7 @@ Flags:
 			stretch.NewOpGammaPP(float32(*ppGamma), float32(*ppSigma)),
 			stretch.NewOpScaleBlack(float32(*scaleBlack/100)),
 			opStarDetect,
-			ref.NewOpSelectReference(ref.RFMFileName, *alignTo, 0, opStarDetect),
+			ref.NewOpSelectReference(ref.SRAlign, *alignRef, opStarDetect),
 			post.NewOpAlign(int32(*alignK), float32(*alignT), post.OOBModeOwnLocation),
 			stretch.NewOpUnsharpMask(float32(*usmSigma), float32(*usmGain), float32(*usmThresh)),
 			ops.NewOpSave(*out, ops.EMMinMax, 1),
@@ -351,7 +353,7 @@ Flags:
 		opSeq := ops.NewOpSequence(
 			opLoadMany,
 			opStarDetect,
-			ref.NewOpSelectReference(ref.RFMLRGB, "", 0, opStarDetect),
+			ref.NewOpSelectReference(ref.SRAlign, "%rgb", opStarDetect),
 
 			rgb.NewOpRGBCombine(),
 			rgb.NewOpRGBBalance(int32(*balBlock), float32(*balBorder), float32(*balSkipBright), float32(*balSkipDim),

internal/fits/rgb.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"math"
 	"github.com/mlnoga/nightlight/internal/stats"
+	"github.com/mlnoga/nightlight/internal/star"
 )
 
 // A RGB color in float32
@@ -39,14 +40,14 @@ func (rgb RGB) String() string {
 
 // Combine single color images into one multi-channel image.
 // All images must have the same dimensions, or undefined results occur. 
-func NewRGBFromChannels(chans []*Image, ref *Image, logWriter io.Writer) *Image {
+func NewRGBFromChannels(chans []*Image, alignStars []star.Star, alignHFR float32, logWriter io.Writer) *Image {
 	naxisn:=make([]int32, len(chans[0].Naxisn)+1)
 	copy(naxisn, chans[0].Naxisn)
 	naxisn[len(chans[0].Naxisn)]=int32(len(chans))
 
 	rgb:=NewImageFromNaxisn(naxisn, nil)
 	rgb.Exposure=chans[0].Exposure+chans[1].Exposure+chans[2].Exposure
-	if ref!=nil { rgb.Stars, rgb.HFR=ref.Stars, ref.HFR }
+	if alignStars!=nil { rgb.Stars, rgb.HFR=alignStars, alignHFR }
 
 	pixelsOrig:=chans[0].Pixels
 	min, mult:=getCommonNormalizationFactors(chans)

internal/ops/operator.go

@@ -27,6 +27,7 @@ import (
 
 	"github.com/mlnoga/nightlight/internal/fits"
 	"github.com/mlnoga/nightlight/internal/stats"
+	"github.com/mlnoga/nightlight/internal/star"
 	"github.com/pbnjay/memory"
 )
 
@@ -39,8 +40,11 @@ type Context struct {
 	MaxThreads      int `json:"maxThreads"`
 	DarkFrame       *fits.Image
 	FlatFrame       *fits.Image
-	RefFrame        *fits.Image
-	RefFrameError   error
+	AlignNaxisn     []int32
+	AlignStars      []star.Star
+	AlignHFR        float32
+	MatchHisto      *stats.Stats
+	RefFrameError 	error
 	LumFrame        *fits.Image
 }
 

internal/ops/post/postprocess.go

@@ -71,16 +71,16 @@ func (op *OpMatchHistogram) UnmarshalJSON(data []byte) error {
 
 func (op *OpMatchHistogram) Apply(f *fits.Image, c *ops.Context) (result *fits.Image, err error) {
 	if op.Mode==HNMNone { return f, nil }
-	if c.RefFrame==nil { return nil, errors.New("missing histogram reference frame")}
+	if c.MatchHisto==nil { return nil, errors.New("missing histogram reference")}
 	switch op.Mode {
 		case HNMLocation:
-			f.MatchLocation(c.RefFrame.Stats.Location())
+			f.MatchLocation(c.MatchHisto.Location())
 		case HNMLocScale:
-			f.MatchHistogram(c.RefFrame.Stats)
+			f.MatchHistogram(c.MatchHisto)
 		case HNMLocBlack:
-	    	f.ShiftBlackToMove(f.Stats.Location(), c.RefFrame.Stats.Location())
+	    	f.ShiftBlackToMove(f.Stats.Location(), c.MatchHisto.Location())
 	}
-	fmt.Fprintf(c.Log, "%d: %s after matching histogram of reference %d\n", f.ID, f.Stats, c.RefFrame.ID)
+	fmt.Fprintf(c.Log, "%d: %s after matching reference histogram %v\n", f.ID, f.Stats, c.MatchHisto)
 	return f, nil
 }
 
@@ -147,8 +147,8 @@ func (op *OpAlign) Apply(f *fits.Image, c *ops.Context) (result *fits.Image, err
 		var outOfBounds float32
 		switch(op.OobMode) {
 			case OOBModeNaN:         outOfBounds=float32(math.NaN())
-			case OOBModeRefLocation: outOfBounds=c.RefFrame.Stats.Location()
-			case OOBModeOwnLocation: outOfBounds=f         .Stats.Location()
+			case OOBModeRefLocation: outOfBounds=c.MatchHisto.Location()
+			case OOBModeOwnLocation: outOfBounds=f     .Stats.Location()
 		}
 
 		// Determine alignment of the image to the reference frame
@@ -172,12 +172,12 @@ func (op *OpAlign) init(c *ops.Context) error {
 	defer op.mutex.Unlock()
 	if op.K<=0 || op.Aligner!=nil { return nil }
 
-	if c.RefFrame==nil {
+	if c.AlignNaxisn==nil || c.AlignStars==nil {
 		return errors.New("Unable to align without reference frame")
-	} else if len(c.RefFrame.Stars)==0 {
+	} else if len(c.AlignStars)==0 {
 		return errors.New("Unable to align without star detections in reference frame")
 	}
-	op.Aligner=star.NewAligner(c.RefFrame.Naxisn, c.RefFrame.Stars, op.K)
+	op.Aligner=star.NewAligner(c.AlignNaxisn, c.AlignStars, op.K)
 	return nil
 }
 

internal/ops/ref/refframe.go

@@ -21,6 +21,7 @@ import (
 	"errors"
 	"fmt"
 	"math"
+	"strconv"
 	"sync"
 	"github.com/mlnoga/nightlight/internal/fits"
 	"github.com/mlnoga/nightlight/internal/qsort"
@@ -29,38 +30,34 @@ import (
 )
 
 
-
-// Reference frame selection mode
-type RefSelMode int
+// Reference frame selection target
+type SelRefTarget int
 const (
-	RFMStarsOverHFR RefSelMode = iota // Pick frame with highest ratio of stars over HFR (for lights)
-	RFMMedianLoc                      // Pick frame with median location (for multiplicative correction when integrating master flats)
-	RFMFileName                       // Load from given filename
-	RFMFileID                         // Load from given frame ID
-	RFMLRGB                           // (L)RGB mode. Uses luminance if present (id=3), else the RGB frame with the best stars/HFR ratio
+	SRAlign SelRefTarget = iota // Select star alignment frame
+	SRHisto                     // Selct histogram matching frame
 )
 
+var srTargetStrings=[]string{"alignemnt", "histogram"}
+
 type OpSelectReference struct {
 	ops.OpBase
-	Mode            RefSelMode         `json:"mode"`
-	FileName        string             `json:"fileName"`
-	FileID          int                `json:"fileID"`
+	Target          SelRefTarget       `json:"target"`
+	Mode            string             `json:"mode"`
 	StarDetect     *pre.OpStarDetect   `json:"starDetect"`
 	mutex           sync.Mutex         `json:"-"`
 	materialized    []*fits.Image      `json:"-"` 
 }
 
 func init() { ops.SetOperatorFactory(func() ops.Operator { return NewOpSelectReferenceDefault() })} // register the operator for JSON decoding
 
-func NewOpSelectReferenceDefault() *OpSelectReference { return NewOpSelectReference(RFMStarsOverHFR, "", 0, pre.NewOpStarDetectDefault() )}
+func NewOpSelectReferenceDefault() *OpSelectReference { return NewOpSelectReference(SRAlign, "%starsHFR", pre.NewOpStarDetectDefault() )}
 
 // Preprocess all light frames with given global settings, limiting concurrency to the number of available CPUs
-func NewOpSelectReference(mode RefSelMode, fileName string, fileID int, opStarDetect *pre.OpStarDetect) *OpSelectReference {
+func NewOpSelectReference(target SelRefTarget, mode string,opStarDetect *pre.OpStarDetect) *OpSelectReference {
 	op:=OpSelectReference{
 		OpBase    : ops.OpBase{Type:"selectRef"},
+		Target: 	target,
 		Mode:       mode,
-		FileName:   fileName,
-		FileID:     fileID,
 		StarDetect: opStarDetect,
 	}
 	return &op	
@@ -97,7 +94,8 @@ func (op *OpSelectReference) applySingle(i int, ins []ops.Promise, c *ops.Contex
 			op.mutex.Unlock()           // return immediately with the same error
 			return nil, errors.New("same error") 
 		}
-		if c.RefFrame!=nil {            // if a reference frame already exists
+		if (op.Target==SRAlign && c.AlignStars!=nil) ||
+		   (op.Target==SRHisto && c.MatchHisto!=nil) {            // if a reference frame already exists
 			op.mutex.Unlock()           // unlock immediately to allow ...
 			if op.materialized==nil || op.materialized[i]==nil {
 				return ins[i]()         // ... materializations to be parallelized by the caller
@@ -109,12 +107,16 @@ func (op *OpSelectReference) applySingle(i int, ins []ops.Promise, c *ops.Contex
 		}
 		defer op.mutex.Unlock()		    // else release lock later when reference frame is computed
 
+		mode:=op.Mode
+		fileID, atoiErr:=strconv.Atoi(op.Mode) // reference frame ID, if given 
+
 		// if reference image is given in a file, load it and detect stars w/o materializing all input promises
-		if op.Mode==RFMFileName {
-			if op.FileName=="" { return ins[i]() }
+		if mode!="%starsHFR" && mode!="%location" && mode!="%rgb" && atoiErr!=nil {
+			refFileName:=op.Mode
+			if refFileName=="" { return ins[i]() }
 
 			var promises []ops.Promise
-			promises, c.RefFrameError=ops.NewOpLoad(-3, op.FileName).MakePromises(nil, c)
+			promises, c.RefFrameError=ops.NewOpLoad(-3, refFileName).MakePromises(nil, c)
 			if c.RefFrameError!=nil { return nil, c.RefFrameError }
 			if len(promises)!=1 {
 				c.RefFrameError=errors.New("load operator did not create exactly one promise") 
@@ -127,9 +129,12 @@ func (op *OpSelectReference) applySingle(i int, ins []ops.Promise, c *ops.Contex
 				c.RefFrameError=errors.New("star detect did not return exactly one promise") 
 				return nil, c.RefFrameError
 			} 
-			c.RefFrame, c.RefFrameError=promises[0]()
+			var refFrame *fits.Image
+			refFrame, c.RefFrameError = promises[0]()
 			if c.RefFrameError!=nil { return nil, c.RefFrameError }
+			op.assignResults(c, refFrame)
 
+			fmt.Fprintf(c.Log, "using loaded image %d as %s reference\n", refFrame.ID, srTargetStrings[op.Target])
 			return ins[i]()
 		}
 
@@ -138,33 +143,34 @@ func (op *OpSelectReference) applySingle(i int, ins []ops.Promise, c *ops.Contex
 		if c.RefFrameError!=nil { return nil, c.RefFrameError }
 
 		// Auto-select mode for (L)RGB
-		mode, fileID :=op.Mode, op.FileID
-		if mode == RFMLRGB {
+		if mode == "%rgb" {
 			if len(op.materialized)>3 {
-				mode, fileID=RFMFileID, 3
+				mode, fileID, atoiErr="3", 3, nil
 			} else {
-				mode=RFMStarsOverHFR
+				mode="%starsHFR"
 			}
 		}
 
 		// select reference with given mode
 		var refScore float32
-		if mode==RFMStarsOverHFR {
-			c.RefFrame, refScore, c.RefFrameError=selectReferenceStarsOverHFR(op.materialized)
-		} else if mode==RFMMedianLoc {
-			c.RefFrame, refScore, c.RefFrameError=selectReferenceMedianLoc(op.materialized, c)
-		} else if mode==RFMFileID {
+		var refFrame *fits.Image
+		if mode=="%starsHFR" {
+			refFrame, refScore, c.RefFrameError=selectReferenceStarsOverHFR(op.materialized)
+		} else if mode=="%location" {
+			refFrame, refScore, c.RefFrameError=selectReferenceMedianLoc(op.materialized, c)
+		} else if atoiErr==nil {
 			if fileID<0 || fileID>=len(op.materialized) {
 				c.RefFrameError=errors.New(fmt.Sprintf("invalid reference file ID %d", fileID))
 				return nil, c.RefFrameError
 			}
-			c.RefFrame=op.materialized[fileID]
+			refFrame=op.materialized[fileID]
 		} else {
 			c.RefFrameError=errors.New(fmt.Sprintf("Unknown refrence selection mode %d", op.Mode))
 		}
-		if c.RefFrame==nil { c.RefFrameError=errors.New("Unable to select reference image.") }
+		if refFrame==nil { c.RefFrameError=errors.New("Unable to select reference image.") }
 		if c.RefFrameError!=nil { return nil, c.RefFrameError }
-		fmt.Fprintf(c.Log, "Using image %d with score %.4g as reference frame.\n", c.RefFrame.ID, refScore)
+		fmt.Fprintf(c.Log, "Using image %d with score %.4g as %s reference.\n", refFrame.ID, refScore, srTargetStrings[op.Target])
+		op.assignResults(c, refFrame)
 
 		// return promise for the materialized image of this instance
 		mat:=op.materialized[i]
@@ -173,6 +179,18 @@ func (op *OpSelectReference) applySingle(i int, ins []ops.Promise, c *ops.Contex
 	}
 }
 
+func (op * OpSelectReference) assignResults(c *ops.Context, refFrame *fits.Image) {
+		if op.Target==SRAlign {
+			c.AlignNaxisn=refFrame.Naxisn
+			c.AlignStars =refFrame.Stars
+			c.AlignHFR   =refFrame.HFR
+		}  else if op.Target==SRHisto {
+			c.MatchHisto=refFrame.Stats
+		} else {
+			fmt.Fprintf(c.Log, "Invalid reference selection target %d, skipping.\n", op.Target)
+		}
+}
+
 func selectReferenceStarsOverHFR(lights []*fits.Image) (refFrame *fits.Image, refScore float32, err error) {
 	refFrame, refScore=nil, -1
 	for _, lightP:=range lights {

internal/ops/rgb/rgb.go

@@ -73,7 +73,7 @@ func (op *OpRGBCombine) Apply(fs []*fits.Image, c *ops.Context) (fOut *fits.Imag
 		c.LumFrame=fs[3]
 	}
 	fmt.Fprintf(c.Log, "\nCombining RGB color channels...\n")
-	fOut=fits.NewRGBFromChannels(fs[:3], c.RefFrame, c.Log)
+	fOut=fits.NewRGBFromChannels(fs[:3], c.AlignStars, c.AlignHFR, c.Log)
 	return fOut, nil
 }
 

internal/ops/stack/stackbatches.go

@@ -92,7 +92,7 @@ func (op *OpStackBatches) Apply(ins []ops.Promise, c *ops.Context) (fOut *fits.I
 	}
 
 	// Free more memory; primary frames already freed after stacking
-	c.DarkFrame, c.FlatFrame, c.RefFrame = nil, nil, nil
+	c.DarkFrame, c.FlatFrame = nil, nil
 	debug.FreeOSMemory()
 
 	if numBatches>1 {