small improvements and fixes

src/downloader.jl

@@ -30,7 +30,7 @@ struct PathDownloader <: AbstractDownloader
     lru::LRU{String, Int}
 end
 
-function PathDownloader(cache_dir; timeout=10, cache_size_gb=5)
+function PathDownloader(cache_dir; timeout=10, cache_size_gb=50)
     isdir(cache_dir) || mkpath(cache_dir)
     lru = LRU{String, Int}(maxsize=cache_size_gb * 10^9, by=identity)
     return PathDownloader(timeout, Downloads.Downloader(), cache_dir, lru)

src/provider/elevation/elevation-provider.jl

@@ -21,7 +21,7 @@ TileProviders.max_zoom(::ElevationProvider) = 16
 function ElevationProvider(provider=TileProviders.Esri(:WorldImagery); cache_size_gb=5)
     TileFormat = get_tile_format(provider)
     fetched_tiles = LRU{String,TileFormat}(; maxsize=cache_size_gb * 10^9, by=Base.sizeof)
-    downloader = [get_downloader(provider) for i in 1:Threads.nthreads()]
+    downloader = [get_downloader(provider) for i in 1:Threads.maxthreadid()]
     ElevationProvider(provider, fetched_tiles, downloader)
 end
 

src/provider/pointclouds/geotiles-pointcloud-provider.jl

@@ -46,7 +46,7 @@ function get_ahn_sub_mapping()
 end
 
 function GeoTilePointCloudProvider(; baseurl="https://geotiles.citg.tudelft.nl", subset="AHN1_T")
-    projs = [Proj.Transformation(GFT.EPSG(28992), GFT.EPSG(3857)) for i in 1:Threads.nthreads()]
+    projs = [Proj.Transformation(GFT.EPSG(28992), GFT.EPSG(3857)) for i in 1:Threads.maxthreadid()]
     return GeoTilePointCloudProvider(baseurl, subset, get_ahn_sub_mapping(), projs)
 end
 

src/tile-plotting.jl

@@ -73,54 +73,45 @@ get_preprocess(config::PlotConfig) = config.preprocess
 get_postprocess(config) = identity
 get_postprocess(config::PlotConfig) = config.postprocess
 
-function create_tile_plot!(m::AbstractMap, tile::Tile, data)
-    # For providers which have to map the same data to different tiles
-    # Or providers that have e.g. additional parameters like a date
-    # the url is a much better key than the tile itself
-    # TODO, should we instead have custom tiles for certain provider?
-    key = tile_key(m.provider, tile)
-    # This can happen for tile providers with overlapping data that doesn't map 1:1 to tiles
-    if haskey(m.plots, key)
-        delete!(m.should_get_plotted, key)
-        return
-    end
 
-    cfg = m.plot_config
-    data_processed = get_preprocess(cfg)(data)
-    bounds = get_bounds(tile, data_processed, m.crs)
-    if bounds isa Rect3
-        # for 3d meshes, we need to remove any plot in the same 2d area
-        bounds2d = Rect2d(bounds)
-        for (other_key, (plot, other_tile, other_bounds)) in copy(m.plots)
-            other_bounds2d = Rect2d(other_bounds)
-            # If overlap
-            if bounds2d in other_bounds2d || other_bounds2d in bounds2d
-                if haskey(m.current_tiles, tile)
-                    # the new plot has priority since it's in the newest current tile set
+function filter_overlapping!(m::Map, bounds::Rect2, tile, key)
+    # dont filter for 2d plots
+end
+
+function filter_overlapping!(m::Map, bounds::Rect3, tile, key)
+    # for 3d meshes, we need to remove any plot in the same 2d area
+    bounds2d = Rect2d(bounds)
+    for (other_key, (plot, other_tile, other_bounds)) in copy(m.plots)
+        other_bounds2d = Rect2d(other_bounds)
+        # If overlap
+        if bounds2d in other_bounds2d || other_bounds2d in bounds2d
+            if haskey(m.current_tiles, tile)
+                # the new plot has priority since it's in the newest current tile set
+                remove_plot!(m, other_key)
+            elseif haskey(m.current_tiles, other_tile)
+                delete!(m.should_get_plotted, key)
+                # the existing plot has priority so we skip the new plot
+                return true
+            else
+                # If both are not in current_tiles, we remove the plot farthest away from the current zoom level
+                if abs(tile.z - m.zoom[]) <= abs(other_tile.z - m.zoom[])
                     remove_plot!(m, other_key)
-                elseif haskey(m.current_tiles, other_tile)
-                    delete!(m.should_get_plotted, key)
-                    # the existing plot has priority so we skip the new plot
-                    return
                 else
-                    # If both are not in current_tiles, we remove the plot farthest away from the current zoom level
-                    if abs(tile.z - m.zoom[]) <= abs(other_tile.z - m.zoom[])
-                        remove_plot!(m, other_key)
-                    else
-                        delete!(m.should_get_plotted, key)
-                        return
-                    end
+                    delete!(m.should_get_plotted, key)
+                    return true
                 end
             end
         end
     end
+    return false
+end
 
-    # Cull unused plots
+function cull_plots!(m::Map)
     if length(m.plots) >= (m.max_plots - 1)
         # remove the oldest plot
         p_tiles = plotted_tiles(m)
         available_to_remove = setdiff(p_tiles, keys(m.current_tiles))
-        sort!(available_to_remove, by=tile-> abs(tile.z - m.zoom[]))
+        sort!(available_to_remove, by=tile -> abs(tile.z - m.zoom[]))
         n_avail = length(available_to_remove)
         need_to_remove = min(n_avail, length(m.plots) - m.max_plots)
         to_remove = available_to_remove[1:need_to_remove]
@@ -131,13 +122,36 @@ function create_tile_plot!(m::AbstractMap, tile::Tile, data)
             end
         end
     end
+end
+
+function create_tile_plot!(m::AbstractMap, tile::Tile, data)
+    # For providers which have to map the same data to different tiles
+    # Or providers that have e.g. additional parameters like a date
+    # the url is a much better key than the tile itself
+    # TODO, should we instead have custom tiles for certain provider?
+    key = tile_key(m.provider, tile)
+    # This can happen for tile providers with overlapping data that doesn't map 1:1 to tiles
+    if haskey(m.plots, key)
+        delete!(m.should_get_plotted, key)
+        return
+    end
+
+    cfg = m.plot_config
+    data_processed = get_preprocess(cfg)(data)
+    bounds = get_bounds(tile, data_processed, m.crs)
+
+    this_got_filtered = filter_overlapping!(m, bounds, tile, key)
+    this_got_filtered && return # skip plotting if it overlaps with a more important plot
+
+    # Cull plots over plot limit
+    cull_plots!(m)
 
-    # if isempty(m.unused_plots)
+    if isempty(m.unused_plots)
         mplot = create_tileplot!(cfg, m.axis, data_processed, bounds, (tile, m.crs))
-    # else
-    #     mplot = pop!(m.unused_plots)
-    #     update_tile_plot!(mplot, cfg, m.axis, data_processed, bounds, (tile, m.crs))
-    # end
+    else
+        mplot = pop!(m.unused_plots)
+        update_tile_plot!(mplot, cfg, m.axis, data_processed, bounds, (tile, m.crs))
+    end
 
     if haskey(m.current_tiles, tile)
         move_in_front!(mplot, abs(m.zoom[] - tile.z), bounds)

src/tiles.jl

@@ -1,7 +1,10 @@
 struct TileCache{TileFormat,Downloader}
     provider::AbstractProvider
-    fetched_tiles::LRU{String,TileFormat}
+    # Nothing for unavailable tiles, so we don't download them again
+    fetched_tiles::LRU{String,Union{Nothing, TileFormat}}
     tile_queue::Channel{Tile}
+    # We also need to put! nothing for unavailable tiles into downloaded_tiles,
+    # so `Map` can clean up the expected tiles
     downloaded_tiles::Channel{Tuple{Tile,Union{Nothing, TileFormat}}}
     downloader::Vector{Downloader}
 end
@@ -28,8 +31,7 @@ function take_last!(c::Channel)
 end
 
 
-function run_loop(thread_id, tile_queue, fetched_tiles, downloader, provider, downloaded_tiles)
-    dl = downloader[thread_id]
+function run_loop(dl, tile_queue, fetched_tiles, provider, downloaded_tiles)
     while isopen(tile_queue) || isready(tile_queue)
         tile = take_last!(tile_queue) # priorize newly arrived tiles
         result = nothing
@@ -42,11 +44,21 @@ function run_loop(thread_id, tile_queue, fetched_tiles, downloader, provider, do
             # if the provider knows it doesn't have a tile, it can return nothing
             isnothing(key) && continue
             result = get!(fetched_tiles, key) do
-                fetch_tile(provider, dl, tile)
+                try
+                    return fetch_tile(provider, dl, tile)
+                catch e
+                    if isa(e, RequestError)
+                        status = e.response.status
+                        if (status == 404 || status == 500)
+                            @warn "tile $(tile) not available, will not download again" maxlog = 10
+                            return nothing
+                        end
+                    end
+                    rethrow(e)
+                end
             end
         catch e
             @warn "Error while fetching tile on thread $(Threads.threadid())" exception = (e, catch_backtrace())
-            # put!(tile_queue, tile)  # retry (not implemented, should have a max retry count and some error handling)
             nothing
         end
         put!(downloaded_tiles, (tile, result))
@@ -57,21 +69,22 @@ end
 function TileCache(provider; cache_size_gb=5, max_parallel_downloads=max(1, Threads.nthreads() ÷ 3))
     TileFormat = get_tile_format(provider)
     downloader = [get_downloader(provider) for i in 1:max_parallel_downloads]
-    fetched_tiles = LRU{String,TileFormat}(; maxsize=cache_size_gb * 10^9, by=Base.summarysize)
+    fetched_tiles = LRU{String,Union{Nothing, TileFormat}}(; maxsize=cache_size_gb * 10^9, by=Base.summarysize)
     downloaded_tiles = Channel{Tuple{Tile,Union{Nothing, TileFormat}}}(Inf)
     tile_queue = Channel{Tile}(Inf)
-    async = Threads.nthreads() <= 1
+    async = Threads.nthreads(:default) <= 1
 
     if async && max_parallel_downloads > 1
         @warn "Multiple download threads are not supported with Threads.nthreads()==1, falling back to async. Start Julia with more threads for parallel downloads."
         async = true
     end
     @assert max_parallel_downloads > 0
     for thread in 1:max_parallel_downloads
+        dl = downloader[thread]
         if async
-            @async run_loop(thread, tile_queue, fetched_tiles, downloader, provider, downloaded_tiles)
+            @async run_loop(dl, tile_queue, fetched_tiles, provider, downloaded_tiles)
         else
-            Threads.@spawn run_loop(thread, tile_queue, fetched_tiles, downloader, provider, downloaded_tiles)
+            Threads.@spawn run_loop(dl, tile_queue, fetched_tiles, provider, downloaded_tiles)
         end
     end
     return TileCache{TileFormat,eltype(downloader)}(provider, fetched_tiles, tile_queue, downloaded_tiles, downloader)