diff --git a/src/downloader.jl b/src/downloader.jl index 71562310..a1a26287 100644 --- a/src/downloader.jl +++ b/src/downloader.jl @@ -30,7 +30,7 @@ struct PathDownloader <: AbstractDownloader lru::LRU{String, Int} end -function PathDownloader(cache_dir; timeout=10, cache_size_gb=5) +function PathDownloader(cache_dir; timeout=10, cache_size_gb=50) isdir(cache_dir) || mkpath(cache_dir) lru = LRU{String, Int}(maxsize=cache_size_gb * 10^9, by=identity) return PathDownloader(timeout, Downloads.Downloader(), cache_dir, lru) diff --git a/src/provider/elevation/elevation-provider.jl b/src/provider/elevation/elevation-provider.jl index c66f8a28..e359ed61 100644 --- a/src/provider/elevation/elevation-provider.jl +++ b/src/provider/elevation/elevation-provider.jl @@ -21,7 +21,7 @@ TileProviders.max_zoom(::ElevationProvider) = 16 function ElevationProvider(provider=TileProviders.Esri(:WorldImagery); cache_size_gb=5) TileFormat = get_tile_format(provider) fetched_tiles = LRU{String,TileFormat}(; maxsize=cache_size_gb * 10^9, by=Base.sizeof) - downloader = [get_downloader(provider) for i in 1:Threads.nthreads()] + downloader = [get_downloader(provider) for i in 1:Threads.maxthreadid()] ElevationProvider(provider, fetched_tiles, downloader) end diff --git a/src/provider/pointclouds/geotiles-pointcloud-provider.jl b/src/provider/pointclouds/geotiles-pointcloud-provider.jl index d6657523..4dc2d781 100644 --- a/src/provider/pointclouds/geotiles-pointcloud-provider.jl +++ b/src/provider/pointclouds/geotiles-pointcloud-provider.jl @@ -46,7 +46,7 @@ function get_ahn_sub_mapping() end function GeoTilePointCloudProvider(; baseurl="https://geotiles.citg.tudelft.nl", subset="AHN1_T") - projs = [Proj.Transformation(GFT.EPSG(28992), GFT.EPSG(3857)) for i in 1:Threads.nthreads()] + projs = [Proj.Transformation(GFT.EPSG(28992), GFT.EPSG(3857)) for i in 1:Threads.maxthreadid()] return GeoTilePointCloudProvider(baseurl, subset, get_ahn_sub_mapping(), projs) end diff --git a/src/tile-plotting.jl b/src/tile-plotting.jl index 16f8095d..ce36cf94 100644 --- a/src/tile-plotting.jl +++ b/src/tile-plotting.jl @@ -73,54 +73,45 @@ get_preprocess(config::PlotConfig) = config.preprocess get_postprocess(config) = identity get_postprocess(config::PlotConfig) = config.postprocess -function create_tile_plot!(m::AbstractMap, tile::Tile, data) - # For providers which have to map the same data to different tiles - # Or providers that have e.g. additional parameters like a date - # the url is a much better key than the tile itself - # TODO, should we instead have custom tiles for certain provider? - key = tile_key(m.provider, tile) - # This can happen for tile providers with overlapping data that doesn't map 1:1 to tiles - if haskey(m.plots, key) - delete!(m.should_get_plotted, key) - return - end - cfg = m.plot_config - data_processed = get_preprocess(cfg)(data) - bounds = get_bounds(tile, data_processed, m.crs) - if bounds isa Rect3 - # for 3d meshes, we need to remove any plot in the same 2d area - bounds2d = Rect2d(bounds) - for (other_key, (plot, other_tile, other_bounds)) in copy(m.plots) - other_bounds2d = Rect2d(other_bounds) - # If overlap - if bounds2d in other_bounds2d || other_bounds2d in bounds2d - if haskey(m.current_tiles, tile) - # the new plot has priority since it's in the newest current tile set +function filter_overlapping!(m::Map, bounds::Rect2, tile, key) + # dont filter for 2d plots +end + +function filter_overlapping!(m::Map, bounds::Rect3, tile, key) + # for 3d meshes, we need to remove any plot in the same 2d area + bounds2d = Rect2d(bounds) + for (other_key, (plot, other_tile, other_bounds)) in copy(m.plots) + other_bounds2d = Rect2d(other_bounds) + # If overlap + if bounds2d in other_bounds2d || other_bounds2d in bounds2d + if haskey(m.current_tiles, tile) + # the new plot has priority since it's in the newest current tile set + remove_plot!(m, other_key) + elseif haskey(m.current_tiles, other_tile) + delete!(m.should_get_plotted, key) + # the existing plot has priority so we skip the new plot + return true + else + # If both are not in current_tiles, we remove the plot farthest away from the current zoom level + if abs(tile.z - m.zoom[]) <= abs(other_tile.z - m.zoom[]) remove_plot!(m, other_key) - elseif haskey(m.current_tiles, other_tile) - delete!(m.should_get_plotted, key) - # the existing plot has priority so we skip the new plot - return else - # If both are not in current_tiles, we remove the plot farthest away from the current zoom level - if abs(tile.z - m.zoom[]) <= abs(other_tile.z - m.zoom[]) - remove_plot!(m, other_key) - else - delete!(m.should_get_plotted, key) - return - end + delete!(m.should_get_plotted, key) + return true end end end end + return false +end - # Cull unused plots +function cull_plots!(m::Map) if length(m.plots) >= (m.max_plots - 1) # remove the oldest plot p_tiles = plotted_tiles(m) available_to_remove = setdiff(p_tiles, keys(m.current_tiles)) - sort!(available_to_remove, by=tile-> abs(tile.z - m.zoom[])) + sort!(available_to_remove, by=tile -> abs(tile.z - m.zoom[])) n_avail = length(available_to_remove) need_to_remove = min(n_avail, length(m.plots) - m.max_plots) to_remove = available_to_remove[1:need_to_remove] @@ -131,13 +122,36 @@ function create_tile_plot!(m::AbstractMap, tile::Tile, data) end end end +end + +function create_tile_plot!(m::AbstractMap, tile::Tile, data) + # For providers which have to map the same data to different tiles + # Or providers that have e.g. additional parameters like a date + # the url is a much better key than the tile itself + # TODO, should we instead have custom tiles for certain provider? + key = tile_key(m.provider, tile) + # This can happen for tile providers with overlapping data that doesn't map 1:1 to tiles + if haskey(m.plots, key) + delete!(m.should_get_plotted, key) + return + end + + cfg = m.plot_config + data_processed = get_preprocess(cfg)(data) + bounds = get_bounds(tile, data_processed, m.crs) + + this_got_filtered = filter_overlapping!(m, bounds, tile, key) + this_got_filtered && return # skip plotting if it overlaps with a more important plot + + # Cull plots over plot limit + cull_plots!(m) - # if isempty(m.unused_plots) + if isempty(m.unused_plots) mplot = create_tileplot!(cfg, m.axis, data_processed, bounds, (tile, m.crs)) - # else - # mplot = pop!(m.unused_plots) - # update_tile_plot!(mplot, cfg, m.axis, data_processed, bounds, (tile, m.crs)) - # end + else + mplot = pop!(m.unused_plots) + update_tile_plot!(mplot, cfg, m.axis, data_processed, bounds, (tile, m.crs)) + end if haskey(m.current_tiles, tile) move_in_front!(mplot, abs(m.zoom[] - tile.z), bounds) diff --git a/src/tiles.jl b/src/tiles.jl index 6ef06190..b5641e16 100644 --- a/src/tiles.jl +++ b/src/tiles.jl @@ -1,7 +1,10 @@ struct TileCache{TileFormat,Downloader} provider::AbstractProvider - fetched_tiles::LRU{String,TileFormat} + # Nothing for unavailable tiles, so we don't download them again + fetched_tiles::LRU{String,Union{Nothing, TileFormat}} tile_queue::Channel{Tile} + # We also need to put! nothing for unavailable tiles into downloaded_tiles, + # so `Map` can clean up the expected tiles downloaded_tiles::Channel{Tuple{Tile,Union{Nothing, TileFormat}}} downloader::Vector{Downloader} end @@ -28,8 +31,7 @@ function take_last!(c::Channel) end -function run_loop(thread_id, tile_queue, fetched_tiles, downloader, provider, downloaded_tiles) - dl = downloader[thread_id] +function run_loop(dl, tile_queue, fetched_tiles, provider, downloaded_tiles) while isopen(tile_queue) || isready(tile_queue) tile = take_last!(tile_queue) # priorize newly arrived tiles result = nothing @@ -42,11 +44,21 @@ function run_loop(thread_id, tile_queue, fetched_tiles, downloader, provider, do # if the provider knows it doesn't have a tile, it can return nothing isnothing(key) && continue result = get!(fetched_tiles, key) do - fetch_tile(provider, dl, tile) + try + return fetch_tile(provider, dl, tile) + catch e + if isa(e, RequestError) + status = e.response.status + if (status == 404 || status == 500) + @warn "tile $(tile) not available, will not download again" maxlog = 10 + return nothing + end + end + rethrow(e) + end end catch e @warn "Error while fetching tile on thread $(Threads.threadid())" exception = (e, catch_backtrace()) - # put!(tile_queue, tile) # retry (not implemented, should have a max retry count and some error handling) nothing end put!(downloaded_tiles, (tile, result)) @@ -57,10 +69,10 @@ end function TileCache(provider; cache_size_gb=5, max_parallel_downloads=max(1, Threads.nthreads() รท 3)) TileFormat = get_tile_format(provider) downloader = [get_downloader(provider) for i in 1:max_parallel_downloads] - fetched_tiles = LRU{String,TileFormat}(; maxsize=cache_size_gb * 10^9, by=Base.summarysize) + fetched_tiles = LRU{String,Union{Nothing, TileFormat}}(; maxsize=cache_size_gb * 10^9, by=Base.summarysize) downloaded_tiles = Channel{Tuple{Tile,Union{Nothing, TileFormat}}}(Inf) tile_queue = Channel{Tile}(Inf) - async = Threads.nthreads() <= 1 + async = Threads.nthreads(:default) <= 1 if async && max_parallel_downloads > 1 @warn "Multiple download threads are not supported with Threads.nthreads()==1, falling back to async. Start Julia with more threads for parallel downloads." @@ -68,10 +80,11 @@ function TileCache(provider; cache_size_gb=5, max_parallel_downloads=max(1, Thre end @assert max_parallel_downloads > 0 for thread in 1:max_parallel_downloads + dl = downloader[thread] if async - @async run_loop(thread, tile_queue, fetched_tiles, downloader, provider, downloaded_tiles) + @async run_loop(dl, tile_queue, fetched_tiles, provider, downloaded_tiles) else - Threads.@spawn run_loop(thread, tile_queue, fetched_tiles, downloader, provider, downloaded_tiles) + Threads.@spawn run_loop(dl, tile_queue, fetched_tiles, provider, downloaded_tiles) end end return TileCache{TileFormat,eltype(downloader)}(provider, fetched_tiles, tile_queue, downloaded_tiles, downloader)