Add an option to write tiles to the standard output in tar format

Makefile

@@ -140,7 +140,7 @@ parallel-test:
 
 raw-tiles-test:
 	./tippecanoe -q -f -e tests/raw-tiles/raw-tiles -r1 -pC tests/raw-tiles/hackspots.geojson
-	./tippecanoe-decode -x generator tests/raw-tiles/raw-tiles > tests/raw-tiles/raw-tiles.json.check
+	./tippecanoe-decode -x generator_options -x generator tests/raw-tiles/raw-tiles > tests/raw-tiles/raw-tiles.json.check
 	cmp tests/raw-tiles/raw-tiles.json.check tests/raw-tiles/raw-tiles.json
 	# Test that -z and -Z work in tippecanoe-decode
 	./tippecanoe-decode -x generator -Z6 -z7 tests/raw-tiles/raw-tiles > tests/raw-tiles/raw-tiles-z67.json.check
@@ -149,6 +149,12 @@ raw-tiles-test:
 	./tile-join -q -f -Z6 -z7 -e tests/raw-tiles/raw-tiles-z67 tests/raw-tiles/raw-tiles
 	./tippecanoe-decode -x generator tests/raw-tiles/raw-tiles-z67 > tests/raw-tiles/raw-tiles-z67-join.json.check
 	cmp tests/raw-tiles/raw-tiles-z67-join.json.check tests/raw-tiles/raw-tiles-z67-join.json
+        # Test that writing to tar does basically the same thing as -e
+	rm -r tests/raw-tiles/raw-tiles
+	mkdir -p tests/raw-tiles/raw-tiles
+	./tippecanoe -q --output-to-tar -r1 -pC --name tests/raw-tiles/raw-tiles --description tests/raw-tiles/raw-tiles tests/raw-tiles/hackspots.geojson | (cd tests/raw-tiles/raw-tiles && tar xf -)
+	./tippecanoe-decode -x generator_options -x generator tests/raw-tiles/raw-tiles > tests/raw-tiles/raw-tiles.json.check
+	cmp tests/raw-tiles/raw-tiles.json.check tests/raw-tiles/raw-tiles.json
 	rm -rf tests/raw-tiles/raw-tiles tests/raw-tiles/raw-tiles-z67 tests/raw-tiles/raw-tiles.json.check raw-tiles-z67.json.check tests/raw-tiles/raw-tiles-z67-join.json.check
 	# Test that metadata.json is created even if all features are clipped away
 	./tippecanoe -q -f -e tests/raw-tiles/nothing tests/raw-tiles/nothing.geojson

README.md

@@ -293,6 +293,7 @@ If your input is formatted as newline-delimited GeoJSON, use `-P` to make input
 
  * `-o` _file_`.mbtiles` or `--output=`_file_`.mbtiles`: Name the output file.
  * `-e` _directory_ or `--output-to-directory`=_directory_: Write tiles to the specified *directory* instead of to an mbtiles file.
+ * `--output-to-tar`: Write tiles in `tar` format to the standard output instead of to an mbtiles file.
  * `-f` or `--force`: Delete the mbtiles file if it already exists instead of giving an error
  * `-F` or `--allow-existing`: Proceed (without deleting existing data) if the metadata or tiles table already exists
    or if metadata fields can't be set. You probably don't want to use this.

dirtiles.cpp

@@ -2,12 +2,14 @@
 #include <fstream>
 #include <sstream>
 #include <string>
+#include <set>
 #include <algorithm>
 #include <stdio.h>
 #include <string.h>
 #include <unistd.h>
 #include <dirent.h>
 #include <limits.h>
+#include <time.h>
 #include <sys/stat.h>
 #include <sqlite3.h>
 #include "jsonpull/jsonpull.h"
@@ -43,6 +45,81 @@ void dir_write_tile(const char *outdir, int z, int tx, int ty, std::string const
 	pbfFile.close();
 }
 
+void tar_write(std::string const &name, std::string const &data, bool dir) {
+	time_t now = time(NULL);
+
+	int mode = 0644;
+	if (dir) {
+		mode = 0755;
+	}
+
+	char buf[512];
+	memset(buf, ' ', 512);
+	sprintf(buf + 0, "%s", name.c_str()); // name
+	sprintf(buf + 100, "%o", mode); // mode
+	sprintf(buf + 108, "%o", 0); // uid
+	sprintf(buf + 116, "%o", 0); // gid
+	sprintf(buf + 124, "%lo", (long) data.size()); // size
+	sprintf(buf + 136, "%lo", (long) now); // mtime
+
+	if (dir) {
+		buf[156] = '5'; // typeflag
+	} else {
+		buf[156] = '0'; // typeflag
+	}
+
+	sprintf(buf + 157, "%s", ""); // linkname
+	sprintf(buf + 257, "%s", "ustar"); // magic
+	buf[263] = '0'; // version
+	buf[264] = '0'; // version
+	sprintf(buf + 265, "%s", "root"); // uname
+	sprintf(buf + 297, "%s", "wheel"); // gname
+	sprintf(buf + 345, "%s", ""); // prefix
+
+	long sum = 0;
+	for (size_t i = 0; i < 512; i++) {
+		sum += (unsigned char) buf[i];
+	}
+
+	sprintf(buf + 148, "%lo", sum); // chksum
+	fwrite(buf, sizeof(char), 512, stdout);
+
+	for (size_t i = 0; i < data.size(); i += 512) {
+		for (size_t j = 0; j < 512; j++) {
+			if (i + j < data.size()) {
+				buf[j] = data[i + j];
+			} else {
+				buf[j] = '\0';
+			}
+		}
+
+		fwrite(buf, sizeof(char), 512, stdout);
+	}
+}
+
+void tar_mkdir(std::string const &dir) {
+	static std::set<std::string> existing;
+	if (existing.count(dir) == 0) {
+		existing.insert(dir);
+		tar_write(dir + "/", "", true);
+	}
+}
+
+void tar_write_tile(int z, int tx, int ty, std::string const &pbf) {
+	tar_mkdir(std::to_string(z));
+	tar_mkdir(std::to_string(z) + "/" + std::to_string(tx));
+
+	std::string fname = std::to_string(z) + "/" + std::to_string(tx) + "/" + std::to_string(ty) + ".pbf";
+	tar_write(fname, pbf, false);
+}
+
+void tar_close() {
+	char buf[512];
+	memset(buf, '\0', 512);
+	fwrite(buf, sizeof(char), 512, stdout);
+	fwrite(buf, sizeof(char), 512, stdout);
+}
+
 static bool numeric(const char *s) {
 	if (*s == '\0') {
 		return false;

dirtiles.hpp

@@ -6,6 +6,9 @@
 #define DIRTILES_HPP
 
 void dir_write_tile(const char *outdir, int z, int tx, int ty, std::string const &pbf);
+void tar_write_tile(int z, int tx, int ty, std::string const &pbf);
+void tar_write(std::string const &name, std::string const &data, bool dir);
+void tar_close();
 
 void check_dir(const char *d, char **argv, bool force, bool forcetable);
 

main.cpp

@@ -1135,7 +1135,7 @@ void choose_first_zoom(long long *file_bbox, std::vector<struct reader> &readers
 	}
 }
 
-int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzoom, int basezoom, double basezoom_marker_width, sqlite3 *outdb, const char *outdir, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, json_object *filter, double droprate, int buffer, const char *tmpdir, double gamma, int read_parallel, int forcetable, const char *attribution, bool uses_gamma, long long *file_bbox, const char *prefilter, const char *postfilter, const char *description, bool guess_maxzoom, std::map<std::string, int> const *attribute_types, const char *pgm, std::map<std::string, attribute_op> const *attribute_accum, std::map<std::string, std::string> const &attribute_descriptions, std::string const &commandline) {
+int read_input(std::vector<source> &sources, const char *fname, int maxzoom, int minzoom, int basezoom, double basezoom_marker_width, sqlite3 *outdb, const char *outdir, bool out_tar, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, json_object *filter, double droprate, int buffer, const char *tmpdir, double gamma, int read_parallel, int forcetable, const char *attribution, bool uses_gamma, long long *file_bbox, const char *prefilter, const char *postfilter, const char *description, bool guess_maxzoom, std::map<std::string, int> const *attribute_types, const char *pgm, std::map<std::string, attribute_op> const *attribute_accum, std::map<std::string, std::string> const &attribute_descriptions, std::string const &commandline) {
 	int ret = EXIT_SUCCESS;
 
 	std::vector<struct reader> readers;
@@ -2267,7 +2267,7 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
 
 	std::atomic<unsigned> midx(0);
 	std::atomic<unsigned> midy(0);
-	int written = traverse_zooms(fd, size, meta, stringpool, &midx, &midy, maxzoom, minzoom, outdb, outdir, buffer, fname, tmpdir, gamma, full_detail, low_detail, min_detail, meta_off, pool_off, initial_x, initial_y, simplification, layermaps, prefilter, postfilter, attribute_accum, filter);
+	int written = traverse_zooms(fd, size, meta, stringpool, &midx, &midy, maxzoom, minzoom, outdb, outdir, out_tar, buffer, fname, tmpdir, gamma, full_detail, low_detail, min_detail, meta_off, pool_off, initial_x, initial_y, simplification, layermaps, prefilter, postfilter, attribute_accum, filter);
 
 	if (maxzoom != written) {
 		if (written > minzoom) {
@@ -2329,7 +2329,7 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
 		ai->second.maxzoom = maxzoom;
 	}
 
-	mbtiles_write_metadata(outdb, outdir, fname, minzoom, maxzoom, minlat, minlon, maxlat, maxlon, midlat, midlon, forcetable, attribution, merged_lm, true, description, !prevent[P_TILE_STATS], attribute_descriptions, "tippecanoe", commandline);
+	mbtiles_write_metadata(outdb, outdir, out_tar, fname, minzoom, maxzoom, minlat, minlon, maxlat, maxlon, midlat, midlon, forcetable, attribution, merged_lm, true, description, !prevent[P_TILE_STATS], attribute_descriptions, "tippecanoe", commandline);
 
 	return ret;
 }
@@ -2463,6 +2463,7 @@ int main(int argc, char **argv) {
 	char *out_mbtiles = NULL;
 	char *out_dir = NULL;
 	sqlite3 *outdb = NULL;
+	bool out_tar = false;
 	int maxzoom = 14;
 	int minzoom = 0;
 	int basezoom = -1;
@@ -2497,6 +2498,7 @@ int main(int argc, char **argv) {
 		{"Output tileset", 0, 0, 0},
 		{"output", required_argument, 0, 'o'},
 		{"output-to-directory", required_argument, 0, 'e'},
+		{"output-to-tar", no_argument, 0, '~'},
 		{"force", no_argument, 0, 'f'},
 		{"allow-existing", no_argument, 0, 'F'},
 
@@ -2697,6 +2699,8 @@ int main(int argc, char **argv) {
 				}
 			} else if (strcmp(opt, "use-attribute-for-id") == 0) {
 				attribute_for_id = optarg;
+			} else if (strcmp(opt, "output-to-tar") == 0) {
+				out_tar = true;
 			} else {
 				fprintf(stderr, "%s: Unrecognized option --%s\n", argv[0], opt);
 				exit(EXIT_FAILURE);
@@ -3135,13 +3139,13 @@ int main(int argc, char **argv) {
 		fprintf(stderr, "Forcing -g0 since -B or -r is not known\n");
 	}
 
-	if (out_mbtiles == NULL && out_dir == NULL) {
-		fprintf(stderr, "%s: must specify -o out.mbtiles or -e directory\n", argv[0]);
+	if (out_mbtiles == NULL && out_dir == NULL && out_tar == false) {
+		fprintf(stderr, "%s: must specify -o out.mbtiles or -e directory or --output-to-tar\n", argv[0]);
 		exit(EXIT_FAILURE);
 	}
 
-	if (out_mbtiles != NULL && out_dir != NULL) {
-		fprintf(stderr, "%s: Options -o and -e cannot be used together\n", argv[0]);
+	if ((out_mbtiles != NULL) + (out_dir != NULL) + (out_tar != false) > 1) {
+		fprintf(stderr, "%s: Options -o and -e and --output-to-tar cannot be used together\n", argv[0]);
 		exit(EXIT_FAILURE);
 	}
 
@@ -3180,11 +3184,14 @@ int main(int argc, char **argv) {
 
 	long long file_bbox[4] = {UINT_MAX, UINT_MAX, 0, 0};
 
-	ret = read_input(sources, name ? name : out_mbtiles ? out_mbtiles : out_dir, maxzoom, minzoom, basezoom, basezoom_marker_width, outdb, out_dir, &exclude, &include, exclude_all, filter, droprate, buffer, tmpdir, gamma, read_parallel, forcetable, attribution, gamma != 0, file_bbox, prefilter, postfilter, description, guess_maxzoom, &attribute_types, argv[0], &attribute_accum, attribute_descriptions, commandline);
+	ret = read_input(sources, name ? name : out_mbtiles ? out_mbtiles : out_dir ? out_dir : "unknown", maxzoom, minzoom, basezoom, basezoom_marker_width, outdb, out_dir, out_tar, &exclude, &include, exclude_all, filter, droprate, buffer, tmpdir, gamma, read_parallel, forcetable, attribution, gamma != 0, file_bbox, prefilter, postfilter, description, guess_maxzoom, &attribute_types, argv[0], &attribute_accum, attribute_descriptions, commandline);
 
 	if (outdb != NULL) {
 		mbtiles_close(outdb, argv[0]);
 	}
+	if (out_tar) {
+		tar_close();
+	}
 
 #ifdef MTRACE
 	muntrace();

mbtiles.cpp

@@ -270,7 +270,7 @@ void tilestats(std::map<std::string, layermap_entry> const &layermap1, size_t el
 	state.json_end_hash();
 }
 
-void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fname, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, int forcetable, const char *attribution, std::map<std::string, layermap_entry> const &layermap, bool vector, const char *description, bool do_tilestats, std::map<std::string, std::string> const &attribute_descriptions, std::string const &program, std::string const &commandline) {
+void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, bool out_tar, const char *fname, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, int forcetable, const char *attribution, std::map<std::string, layermap_entry> const &layermap, bool vector, const char *description, bool do_tilestats, std::map<std::string, std::string> const &attribute_descriptions, std::string const &program, std::string const &commandline) {
 	char *sql, *err;
 
 	sqlite3 *db = outdb;
@@ -490,49 +490,56 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fnam
 		sqlite3_free(sql);
 	}
 
-	if (outdir != NULL) {
-		std::string metadata = std::string(outdir) + "/metadata.json";
+	if (outdir != NULL || out_tar) {
+		std::string out;
+		json_writer state(&out);
 
-		struct stat st;
-		if (stat(metadata.c_str(), &st) == 0) {
-			// Leave existing metadata in place with --allow-existing
-		} else {
-			FILE *fp = fopen(metadata.c_str(), "w");
-			if (fp == NULL) {
-				perror(metadata.c_str());
-				exit(EXIT_FAILURE);
+		state.json_write_hash();
+		state.json_write_newline();
+
+		sqlite3_stmt *stmt;
+		bool first = true;
+		if (sqlite3_prepare_v2(db, "SELECT name, value from metadata;", -1, &stmt, NULL) == SQLITE_OK) {
+			while (sqlite3_step(stmt) == SQLITE_ROW) {
+				std::string key, value;
+
+				const char *k = (const char *) sqlite3_column_text(stmt, 0);
+				const char *v = (const char *) sqlite3_column_text(stmt, 1);
+				if (k == NULL || v == NULL) {
+					fprintf(stderr, "Corrupt mbtiles file: null metadata\n");
+					exit(EXIT_FAILURE);
+				}
+
+				state.json_comma_newline();
+				state.json_write_string(k);
+				state.json_write_string(v);
+				first = false;
 			}
+			sqlite3_finalize(stmt);
+		}
 
-			json_writer state(fp);
+		state.json_write_newline();
+		state.json_end_hash();
+		state.json_write_newline();
 
-			state.json_write_hash();
-			state.json_write_newline();
-
-			sqlite3_stmt *stmt;
-			bool first = true;
-			if (sqlite3_prepare_v2(db, "SELECT name, value from metadata;", -1, &stmt, NULL) == SQLITE_OK) {
-				while (sqlite3_step(stmt) == SQLITE_ROW) {
-					std::string key, value;
-
-					const char *k = (const char *) sqlite3_column_text(stmt, 0);
-					const char *v = (const char *) sqlite3_column_text(stmt, 1);
-					if (k == NULL || v == NULL) {
-						fprintf(stderr, "Corrupt mbtiles file: null metadata\n");
-						exit(EXIT_FAILURE);
-					}
+		if (outdir != NULL) {
+			std::string metadata = std::string(outdir) + "/metadata.json";
 
-					state.json_comma_newline();
-					state.json_write_string(k);
-					state.json_write_string(v);
-					first = false;
+			struct stat st;
+			if (stat(metadata.c_str(), &st) == 0) {
+				// Leave existing metadata in place with --allow-existing
+			} else {
+				FILE *fp = fopen(metadata.c_str(), "w");
+				if (fp == NULL) {
+					perror(metadata.c_str());
+					exit(EXIT_FAILURE);
 				}
-				sqlite3_finalize(stmt);
-			}
 
-			state.json_write_newline();
-			state.json_end_hash();
-			state.json_write_newline();
-			fclose(fp);
+				fprintf(fp, "%s", out.c_str());
+				fclose(fp);
+			}
+		} else if (out_tar) {
+			tar_write("metadata.json", out, false);
 		}
 	}
 

mbtiles.hpp

@@ -4,6 +4,7 @@
 #include <math.h>
 #include <map>
 #include "mvt.hpp"
+#include "dirtiles.hpp"
 
 extern size_t max_tilestats_attributes;
 extern size_t max_tilestats_sample_values;
@@ -45,7 +46,7 @@ sqlite3 *mbtiles_open(char *dbname, char **argv, int forcetable);
 
 void mbtiles_write_tile(sqlite3 *outdb, int z, int tx, int ty, const char *data, int size);
 
-void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fname, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, int forcetable, const char *attribution, std::map<std::string, layermap_entry> const &layermap, bool vector, const char *description, bool do_tilestats, std::map<std::string, std::string> const &attribute_descriptions, std::string const &program, std::string const &commandline);
+void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, bool out_tar, const char *fname, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, int forcetable, const char *attribution, std::map<std::string, layermap_entry> const &layermap, bool vector, const char *description, bool do_tilestats, std::map<std::string, std::string> const &attribute_descriptions, std::string const &program, std::string const &commandline);
 
 void mbtiles_close(sqlite3 *outdb, const char *pgm);
 

tests/raw-tiles/raw-tiles.json

@@ -3,7 +3,6 @@
 "center": "-122.662354,45.514045,14",
 "description": "tests/raw-tiles/raw-tiles",
 "format": "pbf",
-"generator_options": "./tippecanoe -q -f -e tests/raw-tiles/raw-tiles -r1 -pC tests/raw-tiles/hackspots.geojson",
 "json": "{\"vector_layers\": [ { \"id\": \"hackspots\", \"description\": \"\", \"minzoom\": 0, \"maxzoom\": 14, \"fields\": {\"Address\": \"String\", \"Name\": \"String\", \"Notes\": \"String\"} } ],\"tilestats\": {\"layerCount\": 1,\"layers\": [{\"layer\": \"hackspots\",\"count\": 4,\"geometry\": \"Point\",\"attributeCount\": 3,\"attributes\": [{\"attribute\": \"Address\",\"count\": 4,\"type\": \"string\",\"values\": [\"1507 N Rosa Parks Way  Portland, OR 97217\",\"201 SE 12th Ave, Portland, OR 97214\",\"4637 N Albina Ave  Portland, OR 97217\",\"915 SE Hawthorne Blvd. Portland, OR 97214\"]},{\"attribute\": \"Name\",\"count\": 4,\"type\": \"string\",\"values\": [\"Albina Press\",\"Arbor Lodge\",\"Lucky Labrador Brew Pub\",\"Three Friends Coffeehouse\"]},{\"attribute\": \"Notes\",\"count\": 3,\"type\": \"string\",\"values\": [\"\",\"Dog friendly\",\"usually busy, outlets on side wall only\"]}]}]}}",
 "maxzoom": "14",
 "minzoom": "0",

tile-join.cpp

@@ -1124,7 +1124,7 @@ int main(int argc, char **argv) {
 		}
 	}
 
-	mbtiles_write_metadata(outdb, out_dir, name.c_str(), st.minzoom, st.maxzoom, st.minlat, st.minlon, st.maxlat, st.maxlon, st.midlat, st.midlon, 0, attribution.size() != 0 ? attribution.c_str() : NULL, layermap, true, description.c_str(), !pg, attribute_descriptions, "tile-join", generator_options);
+	mbtiles_write_metadata(outdb, out_dir, false, name.c_str(), st.minzoom, st.maxzoom, st.minlat, st.minlon, st.maxlat, st.maxlon, st.midlat, st.midlon, 0, attribution.size() != 0 ? attribution.c_str() : NULL, layermap, true, description.c_str(), !pg, attribute_descriptions, "tile-join", generator_options);
 
 	if (outdb != NULL) {
 		mbtiles_close(outdb, argv[0]);