tweak tile load order

src/map.jl

@@ -41,7 +41,7 @@ struct Map{Ax<:Makie.AbstractAxis} <: AbstractMap
     # The tile downloader + cacher
     tiles::TileCache
     # The tiles for the current zoom level - we may plot many more than this
-    current_tiles::ThreadSafeDict{Tile,Bool}
+    foreground_tiles::ThreadSafeDict{Tile,Bool}
     # The plots we have created but are not currently visible and can be reused
     unused_plots::Vector{Makie.Plot}
     # All tile plots we're currently plotting
@@ -113,7 +113,7 @@ toggle_visibility!(m::Map) = m.axis.scene.visible[] = !m.axis.scene.visible[]
 
 function Base.close(m::Map)
     cleanup_queue!(m, OrderedSet{Tile}())
-    empty!(m.current_tiles)
+    empty!(m.foreground_tiles)
     empty!(m.unused_plots)
     empty!(m.plots)
     empty!(m.should_get_plotted)
@@ -128,7 +128,7 @@ function Map(extent, extent_crs=wgs84;
     provider=TileProviders.OpenStreetMap(:Mapnik),
     crs=MapTiles.web_mercator,
     cache_size_gb=5,
-    max_parallel_downloads=1,
+    max_parallel_downloads=10,
     fetching_scheme=Halo2DTiling(),
     max_zoom=TileProviders.max_zoom(provider),
     max_plots=400,
@@ -148,7 +148,7 @@ function Map(extent, extent_crs=wgs84;
 
     plots = ThreadSafeDict{String,Tuple{Makie.Plot,Tile,Rect}}()
     should_get_plotted = ThreadSafeDict{String,Tile}()
-    current_tiles = ThreadSafeDict{Tile,Bool}()
+    foreground_tiles = ThreadSafeDict{Tile,Bool}()
     unused_plots = Makie.Plot[]
     display_task = Base.RefValue{Task}()
 
@@ -157,7 +157,7 @@ function Map(extent, extent_crs=wgs84;
         axis,
         plot_config,
         tiles,
-        current_tiles,
+        foreground_tiles,
         unused_plots,
         plots,
         should_get_plotted,
@@ -207,7 +207,7 @@ function Base.wait(m::AbstractMap; timeout=50)
     wait(m.tiles; timeout=timeout)
     start = time()
     while true
-        tile_keys = Set(tile_key.((m.provider,), keys(m.current_tiles)))
+        tile_keys = Set(tile_key.((m.provider,), keys(m.foreground_tiles)))
         if all(k -> haskey(m.plots, k), tile_keys)
             break
         end

src/tile-fetching.jl

@@ -30,54 +30,55 @@ end
 
 function update_tiles!(m::Map, arealike)
     # Get the tiles to be plotted from the fetching scheme and arealike
-    new_tiles_set, background_tiles = get_tiles_for_area(m, m.fetching_scheme, arealike)
-    if length(new_tiles_set) > m.max_plots
+    tiles = get_tiles_for_area(m, m.fetching_scheme, arealike)
+    if length(tiles.foreground) > m.max_plots
         @warn "Too many tiles to plot, which means zoom level is not supported. Plotting no tiles for this zoomlevel." maxlog = 1
-        new_tiles_set = OrderedSet{Tile}()
-        background_tiles = OrderedSet{Tile}()
+        tiles.foreground = OrderedSet{Tile}()
     end
     queued_or_plotted = values(m.should_get_plotted)
-    to_add = setdiff(new_tiles_set, queued_or_plotted)
+    # Queue tiles to be downloaded & displayed
+    to_add = map(t -> setdiff(t, queued_or_plotted), tiles)
 
-    # We don't add any background tile to the current_tiles, so they stay shifted to the back
-    # They get added async, so at this point `to_add` won't be in currently_plotted yet
-    will_be_plotted = union(new_tiles_set, queued_or_plotted)
     # replace
-    empty!(m.current_tiles)
-    for tile in new_tiles_set
-        m.current_tiles[tile] = true
+    empty!(m.foreground_tiles)
+    for tile in tiles.foreground
+        m.foreground_tiles[tile] = true
     end
 
     # Move all plots to the back, that aren't in the newest tileset anymore
     for (key, (plot, tile, bounds)) in m.plots
         dist = abs(m.zoom[] - tile.z)
-        if haskey(m.current_tiles, tile)
+        if haskey(m.foreground_tiles, tile)
             move_in_front!(plot, dist, bounds)
         else
             move_to_back!(plot, dist, bounds)
         end
     end
 
-    # Queue tiles to be downloaded & displayed
-    to_add_background = setdiff(background_tiles, will_be_plotted)
     # Remove any item from queue, that isn't in the new set
-    to_keep = union(background_tiles, will_be_plotted)
+    to_keep_queued = union(tiles...)
     # Remove all tiles that are not in the new set from the queue
-    cleanup_queue!(m, to_keep)
+    cleanup_queue!(m, to_keep_queued)
 
     # The unique is needed to avoid tiles referencing the same tile
     # TODO, we should really consider to disallow this for tile providers,
     # This is currently only allowed because of the PointCloudProvider
-    background = unique(t -> tile_key(m.provider, t), to_add_background)
-    foreground = unique(t -> tile_key(m.provider, t), to_add)
+    to_add_keys = map(to_add) do ta
+        unique(t -> tile_key(m.provider, t), ta)
+    end
 
     # We lock the queue, to put all tiles in one go into the tile queue
-    # Since download workers take the last tiles first, foreground tiles go last
-    # Without the lock, a few (n_download_threads) background tiles would be downloaded first,
-    # since they will be the last in the queue until we add the foreground tiles
+    # Without the lock, a few (n_download_threads) old tiles will be downloaded first
+    # since they will be the last in the queue until we add the new tiles
     lock(m.tiles.tile_queue) do
-        foreach(tile -> queue_plot!(m, tile), background)
-        foreach(tile -> queue_plot!(m, tile), foreground)
+        # Offscreen tiles show last, so scroll and zoom don't show
+        # empty white areas or low resolution tiles
+        foreach(tile -> queue_plot!(m, tile), to_add_keys.offscreen)
+        # Foreground tiles show in the middle, filling out details
+        foreach(tile -> queue_plot!(m, tile), to_add_keys.foreground)
+        # Lower-resolution background tiles show first
+        # Its quick to get them and they immediately fill the plot
+        foreach(tile -> queue_plot!(m, tile), to_add_keys.background)
     end
 end
 
@@ -110,31 +111,36 @@ function get_tiles_for_area(m::Map{Axis}, scheme::Halo2DTiling, area::Union{Rect
     xspan = (area.X[2] - area.X[1]) * 0.01
     yspan = (area.Y[2] - area.Y[1]) * 0.01
     mouse_area = Extents.Extent(; X=(xpos - xspan, xpos + xspan), Y=(ypos - yspan, ypos + yspan))
-    # Make a halo around the mouse tile to load next, intersecting area so we don't download outside the plot
-    mouse_halo_area = grow_extent(mouse_area, 10)
     # Define a halo around the area to download last, so pan/zoom are filled already
     halo_area = grow_extent(area, scheme.halo) # We don't mind that the middle tiles are the same, the OrderedSet will remove them
-    # Define all the tiles in the order they will load in
-    background_areas = if Extents.intersects(mouse_halo_area, area)
-        mha = Extents.intersection(mouse_halo_area, area)
-        if Extents.intersects(mouse_area, area)
-            [Extents.intersection(mouse_area, area), mha, halo_area]
-        else
-            [mha, halo_area]
-        end
-    else
-        [halo_area]
-    end
 
-    foreground = OrderedSet{Tile}(MapTiles.TileGrid(area, zoom, m.crs))
+    foreground = OrderedSet{Tile}()
     background = OrderedSet{Tile}()
+    offscreen = OrderedSet{Tile}()
     for z in layer_range
-        z == zoom && continue
-        for ext in background_areas
-            union!(background, MapTiles.TileGrid(ext, z, m.crs))
+        # Make a halo around the mouse tile to load next,
+        # intersecting area so we don't download outside the plot
+        for ext_scale in 1:2:100
+            mouse_halo_area = grow_extent(mouse_area, ext_scale)
+            ext = Extents.intersection(mouse_halo_area, area)
+            isnothing(ext) && continue
+            tilegrid = collect(MapTiles.TileGrid(ext, z, m.crs))
+            if z == zoom
+                union!(foreground, OrderedSet(tilegrid))
+            else
+                union!(background, tilegrid)
+            end
         end
+        # Get just the halo ring tiles to load offscreen
+        area_grid = MapTiles.TileGrid(area, z, m.crs)
+        halo_grid = MapTiles.TileGrid(halo_area, z, m.crs)
+        halo_tiles = setdiff(halo_grid, area_grid)
+        union!(offscreen, halo_tiles)
     end
-    return foreground, background
+    tiles = (; foreground, background, offscreen)
+    # Reverse the order of the groups. Reversing the ranges 
+    # above doesn't have the same effect due to then unions
+    return map(OrderedSet ∘ reverse ∘ collect, tiles)
 end
 
 #########################################################################################
@@ -149,7 +155,10 @@ function get_tiles_for_area(m::Map, ::SimpleTiling, area::Union{Rect,Extent})
     # Calculate the zoom level
     ideal_zoom, zoom, approx_ntiles = optimal_zoom(m, diag)
     m.zoom[] = zoom
-    return OrderedSet{Tile}(MapTiles.TileGrid(area, zoom, m.crs)), OrderedSet{Tile}()
+    foreground = OrderedSet{Tile}(MapTiles.TileGrid(area, zoom, m.crs))
+    background = OrderedSet{Tile}()
+    offscreen = OrderedSet{Tile}()
+    return (; foreground, background, offscreen)
 end
 
 #########################################################################################
@@ -165,7 +174,7 @@ function get_tiles_for_area(m::Map{LScene}, ::Tiling3D, (cam, camc)::Tuple{Camer
     camc.far[] = maxdist
     camc.near[] = eyepos[3] * 0.01
     update_cam!(m.axis.scene)
-    return tiles_from_poly(m, points), OrderedSet{Tile}()
+    return tiles_from_poly(m, points), OrderedSet{Tile}(), OrderedSet{Tile}()
 end
 
 function get_tiles_for_area(m::Map{LScene}, s::SimpleTiling, (cam, camc)::Tuple{Camera,Camera3D})

src/tile-plotting.jl

@@ -86,15 +86,15 @@ function filter_overlapping!(m::Map, bounds::Rect3, tile, key)
         other_bounds2d = Rect2d(other_bounds)
         # If overlap
         if bounds2d in other_bounds2d || other_bounds2d in bounds2d
-            if haskey(m.current_tiles, tile)
+            if haskey(m.foreground_tiles, tile)
                 # the new plot has priority since it's in the newest current tile set
                 remove_plot!(m, other_key)
-            elseif haskey(m.current_tiles, other_tile)
+            elseif haskey(m.foreground_tiles, other_tile)
                 delete!(m.should_get_plotted, key)
                 # the existing plot has priority so we skip the new plot
                 return true
             else
-                # If both are not in current_tiles, we remove the plot farthest away from the current zoom level
+                # If both are not in foreground_tiles, we remove the plot farthest away from the current zoom level
                 if abs(tile.z - m.zoom[]) <= abs(other_tile.z - m.zoom[])
                     remove_plot!(m, other_key)
                 else
@@ -111,7 +111,7 @@ function cull_plots!(m::Map)
     if length(m.plots) >= (m.max_plots - 1)
         # remove the oldest plot
         p_tiles = plotted_tiles(m)
-        available_to_remove = setdiff(p_tiles, keys(m.current_tiles))
+        available_to_remove = setdiff(p_tiles, keys(m.foreground_tiles))
         sort!(available_to_remove, by=tile -> abs(tile.z - m.zoom[]))
         n_avail = length(available_to_remove)
         need_to_remove = min(n_avail, length(m.plots) - m.max_plots)
@@ -154,7 +154,7 @@ function create_tile_plot!(m::AbstractMap, tile::Tile, data)
         update_tile_plot!(mplot, cfg, m.axis, data_processed, bounds, (tile, m.crs))
     end
 
-    if haskey(m.current_tiles, tile)
+    if haskey(m.foreground_tiles, tile)
         move_in_front!(mplot, abs(m.zoom[] - tile.z), bounds)
     else
         move_to_back!(mplot, abs(m.zoom[] - tile.z), bounds)

src/tiles.jl

@@ -34,6 +34,8 @@ end
 function run_loop(dl, tile_queue, fetched_tiles, provider, downloaded_tiles)
     while isopen(tile_queue) || isready(tile_queue)
         tile = take_last!(tile_queue) # priorize newly arrived tiles
+        @show (tile.x, tile.y, tile.z)
+        #sleep()
         result = nothing
         try
             @debug("downloading tile on thread $(Threads.threadid())")
@@ -78,7 +80,9 @@ function TileCache(provider; cache_size_gb=5, max_parallel_downloads=1)
     fetched_tiles = LRU{String,Union{Nothing, TileFormat}}(; maxsize=cache_size_gb * 10^9, by=Base.summarysize)
     downloaded_tiles = Channel{Tuple{Tile,Union{Nothing, TileFormat}}}(Inf)
     tile_queue = Channel{Tile}(Inf)
+
     async = Threads.nthreads(:default) <= 1
+    async = true # TODO remove 
     if async && max_parallel_downloads > 1
         @warn "Multiple download threads are not supported with Threads.nthreads()==1, falling back to async. Start Julia with more threads for parallel downloads."
         async = true

test/runtests.jl

@@ -9,22 +9,22 @@ london = Rect2f(-0.0921, 51.5, 0.04, 0.025)
 m = Tyler.Map(london); m.figure.scene
 s = display(m) # waits until all tiles are displayed
 @test isempty(m.tiles.tile_queue)
-@test length(m.current_tiles) == 25
+@test length(m.foreground_tiles) == 25
 @test length(m.tiles.fetched_tiles) == 48
 
 london = Rect2f(-0.0921, 51.5, 0.04, 0.025)
 m = Tyler.Map(london; scale=1, provider=Tyler.TileProviders.Google(), crs=Tyler.MapTiles.WGS84()) # waits until all tiles are displayed
 s = display(m) # waits until all tiles are displayed
 @test isempty(m.tiles.tile_queue)
-@test length(m.current_tiles) == 35
+@test length(m.foreground_tiles) == 35
 @test length(m.tiles.fetched_tiles) == 66
 
 # test Extent input
 london = Extents.Extent(X=(-0.0921, -0.0521), Y=(51.5, 51.525))
 m = Tyler.Map(london; scale=1) # waits until all tiles are displayed
 display(m)
 @test isempty(m.tiles.tile_queue)
-@test length(m.current_tiles) == 25
+@test length(m.foreground_tiles) == 25
 @test length(m.tiles.fetched_tiles) == 48
 
 @testset "Interfaces" begin

test/test-providers.jl

@@ -87,10 +87,8 @@ begin
     m2 = Tyler.Map3D(ext; provider=ElevationProvider(), figure=m1.figure, axis=m1.axis, max_parallel_downloads=1, plot_config=cfg)
     m1
 end
-max_zoom(m1)
-Tyler.approx_tiles(m1, , 1000)
 
-m1.current_tiles
+m1.foreground_tiles
 m1.tiles.tile_queue
 m1.plots
 m1.should_get_plotted

test/tests.jl

@@ -7,6 +7,6 @@ begin
     provider = GeoTilePointCloudProvider(subset=subset)
     m1 = Tyler.Map3D(ext; provider=provider)
     wait(m1)
-    unique_plots = unique(Tyler.tile_key.((m1.provider,), keys(m1.current_tiles)))
+    unique_plots = unique(Tyler.tile_key.((m1.provider,), keys(m1.foreground_tiles)))
     @test length(unique_plots) == length(m1.plots)
 end