Merge pull request #33 from tilezen/fix-metadata

Save metadata as part of closing mbtiles

cmd/build/main.go

@@ -30,11 +30,6 @@ func processResults(results chan *tilepack.TileResponse, processor tilepack.Tile
 		progress.Add(1)
 	}
 
-	err := processor.Close()
-	if err != nil {
-		log.Printf("Error closing processor: %+v", err)
-	}
-
 	progress.Finish()
 	log.Printf("Processed %d tiles", tileCount)
 }
@@ -48,6 +43,8 @@ func main() {
 	zoomsStr := flag.String("zooms", "0,1,2,3,4,5,6,7,8,9,10", "Comma-separated list of zoom levels or a '{MIN_ZOOM}-{MAX_ZOOM}' range string.")
 	numTileFetchWorkers := flag.Int("workers", 25, "Number of tile fetch workers to use.")
 	mbtilesBatchSize := flag.Int("batch-size", 50, "(For mbtiles outputter) Number of tiles to batch together before writing to mbtiles")
+	mbtilesTilesetName := flag.String("tileset-name", "tileset", "(For mbtiles outputter) Name of the tileset to write to the mbtiles file metadata.")
+	mbtilesFormat := flag.String("mbtiles-format", "pbf", "(For mbtiles outputter) Format of the tiles in the mbtiles file metadata.")
 	requestTimeout := flag.Int("timeout", 60, "HTTP client timeout for tile requests.")
 	cpuProfile := flag.String("cpuprofile", "", "Enables CPU profiling. Saves the dump to the given path.")
 	invertedY := flag.Bool("inverted-y", false, "Invert the Y-value of tiles to match the TMS (as opposed to ZXY) tile format.")
@@ -236,7 +233,23 @@ func main() {
 	case "disk":
 		outputter, outputterErr = tilepack.NewDiskOutputter(*outputDSN)
 	case "mbtiles":
-		outputter, outputterErr = tilepack.NewMbtilesOutputter(*outputDSN, *mbtilesBatchSize)
+		metadata := tilepack.NewMbtilesMetadata(map[string]string{})
+
+		if *mbtilesFormat == "" {
+			log.Fatalf("--mbtiles-format is required for mbtiles output")
+		}
+		metadata.Set("format", *mbtilesFormat)
+
+		if *mbtilesFormat != "pbf" && *ensureGzip {
+			log.Printf("Warning: gzipping is only required for PBF tiles. You may want to disable it for other formats with --ensure-gzip=false")
+		}
+
+		if *mbtilesTilesetName == "" {
+			log.Fatalf("--tileset-name is required for mbtiles output")
+		}
+		metadata.Set("name", *mbtilesTilesetName)
+
+		outputter, outputterErr = tilepack.NewMbtilesOutputter(*outputDSN, *mbtilesBatchSize, metadata)
 	default:
 		log.Fatalf("Unknown outputter: %s", *outputMode)
 	}
@@ -296,10 +309,14 @@ func main() {
 	log.Print("Finished processing tiles")
 
 	err = outputter.AssignSpatialMetadata(bounds, zooms[0], zooms[len(zooms)-1])
-
 	if err != nil {
 		log.Printf("Wrote tiles but failed to assign spatial metadata, %v", err)
 	}
+
+	err = outputter.Close()
+	if err != nil {
+		log.Printf("Error closing processor: %+v", err)
+	}
 }
 
 func calculateExpectedTiles(bounds orb.Bound, zooms []maptile.Zoom) uint32 {

cmd/mbtiles-assign-metadata/main.go

@@ -50,7 +50,9 @@ func main() {
 
 		mbtilesReader.Close()
 
-		mbtilesWriter, err := tilepack.NewMbtilesOutputter(path, 0)
+		metadata := tilepack.NewMbtilesMetadata(map[string]string{})
+
+		mbtilesWriter, err := tilepack.NewMbtilesOutputter(path, 0, metadata)
 
 		if err != nil {
 			log.Fatalf("Couldn't read input mbtiles %s: %+v", path, err)

cmd/merge/main.go

@@ -42,6 +42,8 @@ func main() {
 	var outputBounds orb.Bound
 	var outputMinZoom uint
 	var outputMaxZoom uint
+	var outputFormat string
+	var outputTilesetName string
 
 	inputReaders := make([]tilepack.MbtilesReader, len(inputFilenames))
 
@@ -58,6 +60,28 @@ func main() {
 			log.Fatalf("Unable to read metadata for %s, %v", inputFilename, err)
 		}
 
+		thisFormat, err := metadata.Format()
+		if err != nil {
+			log.Fatalf("Unable to read format for %s, %v", inputFilename, err)
+		}
+
+		if outputFormat == "" {
+			outputFormat = thisFormat
+		} else if outputFormat != thisFormat {
+			log.Fatalf("Input %s has format %s, but consensus output format is %s", inputFilename, thisFormat, outputFormat)
+		}
+
+		thisTilesetName, err := metadata.Name()
+		if err != nil {
+			log.Fatalf("Unable to read name for %s, %v", inputFilename, err)
+		}
+
+		if outputTilesetName == "" {
+			outputTilesetName = thisTilesetName
+		} else {
+			outputTilesetName += "," + thisTilesetName
+		}
+
 		bounds, err := metadata.Bounds()
 
 		if err != nil {
@@ -88,8 +112,13 @@ func main() {
 		inputReaders[i] = mbtilesReader
 	}
 
+	metadata := tilepack.NewMbtilesMetadata(map[string]string{
+		"name":   outputTilesetName,
+		"format": outputFormat,
+	})
+
 	// Create the output mbtiles
-	outputMbtiles, err := tilepack.NewMbtilesOutputter(*outputFilename, 1000)
+	outputMbtiles, err := tilepack.NewMbtilesOutputter(*outputFilename, 1000, metadata)
 	if err != nil {
 		log.Fatalf("Couldn't create output mbtiles: %+v", err)
 	}

tilepack/mbtiles_metadata.go

@@ -153,3 +153,15 @@ func (m *MbtilesMetadata) MaxZoom() (uint, error) {
 
 	return uint(i), nil
 }
+
+func (m *MbtilesMetadata) Set(key string, value string) {
+	m.metadata[key] = value
+}
+
+func (m *MbtilesMetadata) Format() (string, error) {
+	return m.metadata["format"], nil
+}
+
+func (m *MbtilesMetadata) Name() (string, error) {
+	return m.metadata["name"], nil
+}

tilepack/mbtiles_outputter.go

@@ -13,13 +13,13 @@ import (
 	"github.com/paulmach/orb/maptile"
 )
 
-func NewMbtilesOutputter(dsn string, batchSize int) (*mbtilesOutputter, error) {
+func NewMbtilesOutputter(dsn string, batchSize int, metadata *MbtilesMetadata) (*mbtilesOutputter, error) {
 	db, err := sql.Open("sqlite3", dsn)
 	if err != nil {
 		return nil, err
 	}
 
-	return &mbtilesOutputter{db: db, batchSize: batchSize}, nil
+	return &mbtilesOutputter{db: db, batchSize: batchSize, metadata: metadata}, nil
 }
 
 type mbtilesOutputter struct {
@@ -29,15 +29,28 @@ type mbtilesOutputter struct {
 	hasTiles   bool
 	batchCount int
 	batchSize  int
+	metadata   *MbtilesMetadata
 }
 
 func (o *mbtilesOutputter) Close() error {
 	var err error
 
+	// Save the metadata
+	for name, value := range o.metadata.metadata {
+		q := "INSERT OR REPLACE INTO metadata (name, value) VALUES(?, ?)"
+		_, err = o.txn.Exec(q, name, value)
+
+		if err != nil {
+			return fmt.Errorf("failed to add %s metadata key: %w", name, err)
+		}
+	}
+
+	// Commit the transaction
 	if o.txn != nil {
 		err = o.txn.Commit()
 	}
 
+	// Close the database
 	if o.db != nil {
 		if err2 := o.db.Close(); err2 != nil {
 			err = err2
@@ -93,27 +106,21 @@ func (o *mbtilesOutputter) AssignSpatialMetadata(bounds orb.Bound, minZoom mapti
 
 	center := bounds.Center()
 
-	str_bounds := fmt.Sprintf("%f,%f,%f,%f", bounds.Min[0], bounds.Min[1], bounds.Max[0], bounds.Max[1])
-	str_center := fmt.Sprintf("%f,%f", center[0], center[1])
+	strBounds := fmt.Sprintf("%f,%f,%f,%f", bounds.Min[0], bounds.Min[1], bounds.Max[0], bounds.Max[1])
+	strCenter := fmt.Sprintf("%f,%f", center[0], center[1])
 
-	str_minzoom := strconv.Itoa(int(minZoom))
-	str_maxzoom := strconv.Itoa(int(maxZoom))
+	strMinzoom := strconv.Itoa(int(minZoom))
+	strMaxzoom := strconv.Itoa(int(maxZoom))
 
 	metadata := map[string]string{
-		"bounds":  str_bounds,
-		"center":  str_center,
-		"minzoom": str_minzoom,
-		"maxzoom": str_maxzoom,
+		"bounds":  strBounds,
+		"center":  strCenter,
+		"minzoom": strMinzoom,
+		"maxzoom": strMaxzoom,
 	}
 
 	for name, value := range metadata {
-
-		q := "INSERT OR REPLACE INTO metadata (name, value) VALUES(?, ?)"
-		_, err := o.db.Exec(q, name, value)
-
-		if err != nil {
-			return fmt.Errorf("Failed to add %s metadata key, %w", name, err)
-		}
+		o.metadata.Set(name, value)
 	}
 
 	return nil

tilepack/mbtiles_reader.go

@@ -20,10 +20,6 @@ type MbtilesReader interface {
 	Metadata() (*MbtilesMetadata, error)
 }
 
-type tileDataFromDatabase struct {
-	Data *[]byte
-}
-
 func NewMbtilesReader(dsn string) (MbtilesReader, error) {
 	db, err := sql.Open("sqlite3", dsn)
 	if err != nil {