Parallel process and load plans

DynmapCore/src/main/java/org/dynmap/common/chunk/GenericMapChunkCache.java

@@ -1,8 +1,9 @@
 package org.dynmap.common.chunk;
 
-import java.util.ArrayList;
+import java.util.*;
-import java.util.List;
+import java.util.function.BiConsumer;
-import java.util.ListIterator;
+import java.util.function.Consumer;
+import java.util.function.Supplier;
 
 import org.dynmap.DynmapChunk;
 import org.dynmap.DynmapCore;
@@ -697,6 +698,14 @@ public abstract class GenericMapChunkCache extends MapChunkCache {
 	protected abstract GenericChunk getLoadedChunk(DynmapChunk ch);
 	// Load generic chunk from unloaded chunk
 	protected abstract GenericChunk loadChunk(DynmapChunk ch);
+	// Load generic chunk from existing and already loaded chunk
+	protected Supplier<GenericChunk> getLoadedChunkAsync(DynmapChunk ch) {
+		throw new RuntimeException("Not implemeted");
+	}
+	// Load generic chunks from unloaded chunk async
+	protected Supplier<GenericChunk> loadChunkAsync(DynmapChunk ch){
+		throw new RuntimeException("Not implemeted");
+	}
 	
 	/**
 	 * Read NBT data from loaded chunks - needs to be called from server/world
@@ -753,11 +762,75 @@ public abstract class GenericMapChunkCache extends MapChunkCache {
 		}
 		return cnt;
 	}
+	public void getLoadedChunksAsync() {
+		class SimplePair{ //simple pair of the supplier that finishes read async, and a consumer that also finish his work async
+			final Supplier<GenericChunk> supplier;
+			final BiConsumer<GenericChunk, Long> consumer;
+
+			SimplePair(Supplier<GenericChunk> supplier, BiConsumer<GenericChunk, Long> consumer) {
+				this.supplier = supplier;
+				this.consumer = consumer;
+			}
+		}
+		if (!dw.isLoaded()) {
+			isempty = true;
+			unloadChunks();
+			return;
+		}
+		List<SimplePair> lastApply = new ArrayList<>();
+		for (DynmapChunk dynmapChunk : chunks) {
+			long startTime = System.nanoTime();
+			int chunkIndex = (dynmapChunk.x - x_min) + (dynmapChunk.z - z_min) * x_dim;
+			if (snaparray[chunkIndex] != null)
+				continue; // Skip if already processed
+
+			boolean vis = isChunkVisible(dynmapChunk);
+
+			/* Check if cached chunk snapshot found */
+			if (tryChunkCache(dynmapChunk, vis)) {
+				endChunkLoad(startTime, ChunkStats.CACHED_SNAPSHOT_HIT);
+			}
+			// If chunk is loaded and not being unloaded, we're grabbing its NBT data
+			else {
+				// Get generic chunk from already loaded chunk, if we can
+				Supplier<GenericChunk> supplier = getLoadedChunkAsync(dynmapChunk);
+				long startPause = System.nanoTime();
+				BiConsumer<GenericChunk, Long> consumer = (ss, reloadTime) -> {
+					if (ss == null) return;
+					long pause = reloadTime - startPause;
+					if (vis) { // If visible
+						prepChunkSnapshot(dynmapChunk, ss);
+					} else {
+						if (hidestyle == HiddenChunkStyle.FILL_STONE_PLAIN) {
+							ss = getStone();
+						} else if (hidestyle == HiddenChunkStyle.FILL_OCEAN) {
+							ss = getOcean();
+						} else {
+							ss = getEmpty();
+						}
+					}
+					snaparray[chunkIndex] = ss;
+					endChunkLoad(startTime - pause, ChunkStats.LOADED_CHUNKS);
+
+				};
+				lastApply.add(new SimplePair(supplier, consumer));
+			}
+		}
+		//impact on the main thread should be minimal, so we plan and finish the work after main thread finished it's part
+		lastApply.forEach(simplePair -> {
+			long reloadWork = System.nanoTime();
+			simplePair.consumer.accept(simplePair.supplier.get(), reloadWork);
+		});
+	}
 
 	@Override
 	public int loadChunks(int max_to_load) {
 		return getLoadedChunks() + readChunks(max_to_load);
+	}
 
+	public void loadChunksAsync() {
+		getLoadedChunksAsync();
+		readChunksAsync();
 	}
 
 	public int readChunks(int max_to_load) {
@@ -840,6 +913,81 @@ public abstract class GenericMapChunkCache extends MapChunkCache {
 		return cnt;
 	}
 
+	public void readChunksAsync() {
+		if (!dw.isLoaded()) {
+			isempty = true;
+			unloadChunks();
+			return;
+		}
+
+		List<DynmapChunk> chunks;
+		if (iterator == null) {
+			iterator = Collections.emptyListIterator();
+			chunks = new ArrayList<>(this.chunks);
+		} else {
+			chunks = new ArrayList<>();
+			iterator.forEachRemaining(chunks::add);
+		}
+//		DynmapCore.setIgnoreChunkLoads(true);
+
+		List<DynmapChunk> cached = new ArrayList<>();
+		List<Map.Entry<Supplier<GenericChunk>,DynmapChunk>> notCached = new ArrayList<>();
+
+		iterator.forEachRemaining(chunks::add);
+		chunks.stream()
+				.filter(chunk -> snaparray[(chunk.x - x_min) + (chunk.z - z_min) * x_dim] == null)
+				.forEach(chunk -> {
+					if (cache.getSnapshot(dw.getName(),chunk.x, chunk.z) == null) {
+						notCached.add(new AbstractMap.SimpleEntry<>(loadChunkAsync(chunk),chunk));
+					} else {
+						cached.add(chunk);
+					}
+				});
+
+		cached.forEach(chunk -> {
+			long startTime = System.nanoTime();
+			tryChunkCache(chunk, isChunkVisible(chunk));
+			endChunkLoad(startTime, ChunkStats.CACHED_SNAPSHOT_HIT);
+		});
+		notCached.forEach(chunkSupplier -> {
+			long startTime = System.nanoTime();
+			GenericChunk chunk = chunkSupplier.getKey().get();
+			DynmapChunk dynmapChunk = chunkSupplier.getValue();
+			if (chunk != null) {
+				// If hidden
+				if (isChunkVisible(dynmapChunk)) {
+					// Prep snapshot
+					prepChunkSnapshot(dynmapChunk, chunk);
+				} else {
+					if (hidestyle == HiddenChunkStyle.FILL_STONE_PLAIN) {
+						chunk = getStone();
+					} else if (hidestyle == HiddenChunkStyle.FILL_OCEAN) {
+						chunk = getOcean();
+					} else {
+						chunk = getEmpty();
+					}
+				}
+				snaparray[(dynmapChunk.x - x_min) + (dynmapChunk.z - z_min) * x_dim] = chunk;
+				endChunkLoad(startTime, ChunkStats.UNLOADED_CHUNKS);
+			} else {
+				endChunkLoad(startTime, ChunkStats.UNGENERATED_CHUNKS);
+			}
+		});
+
+//		DynmapCore.setIgnoreChunkLoads(false);
+
+		isempty = true;
+		/* Fill missing chunks with empty dummy chunk */
+		for (int i = 0; i < snaparray.length; i++) {
+			if (snaparray[i] == null) {
+				snaparray[i] = getEmpty();
+			} else if (!snaparray[i].isEmpty) {
+				isempty = false;
+			}
+		}
+
+	}
+
 	/**
 	 * Test if done loading
 	 */

bukkit-helper-118-2/src/main/java/org/dynmap/bukkit/helper/v118_2/AsyncChunkProvider118_2.java

@@ -45,12 +45,17 @@ public class AsyncChunkProvider118_2 {
             throw new RuntimeException(e);
         }
     }
-    public NBTTagCompound getChunk(WorldServer world, int x, int y) throws InvocationTargetException, IllegalAccessException, NoSuchFieldException {
+    public CompletableFuture<NBTTagCompound> getChunk(WorldServer world, int x, int y) throws InvocationTargetException, IllegalAccessException {
         CompletableFuture<Object> future = new CompletableFuture<>();
         getChunk.invoke(ioThread,world,x,y,5,(Consumer<Object>) future::complete, false, true, true);
-        Object resultFuture = future.join();
+        return future.thenApply((resultFuture) -> {
-        if (resultFuture == null) return null;
+            if (resultFuture == null) return null;
-        NBTTagCompound result = (NBTTagCompound) resultFuture.getClass().getField("chunkData").get(resultFuture);
+            try {
-        return ifFailed.test(result) ? null : result;
+                return (NBTTagCompound) resultFuture.getClass().getField("chunkData").get(resultFuture);
+            } catch (IllegalAccessException | NoSuchFieldException e) {
+                e.printStackTrace();
+            }
+            return null;
+        });
     }
 }

bukkit-helper-118-2/src/main/java/org/dynmap/bukkit/helper/v118_2/MapChunkCache118_2.java

@@ -20,6 +20,7 @@ import java.io.IOException;
 import java.lang.reflect.InvocationTargetException;
 import java.util.List;
 import java.util.concurrent.CompletableFuture;
+import java.util.function.Supplier;
 
 /**
  * Container for managing chunks - dependent upon using chunk snapshots, since rendering is off server thread
@@ -36,23 +37,43 @@ public class MapChunkCache118_2 extends GenericMapChunkCache {
 
     // Load generic chunk from existing and already loaded chunk
     protected GenericChunk getLoadedChunk(DynmapChunk chunk) {
-        CraftWorld cw = (CraftWorld) w;
+        return getLoadedChunk(chunk, false).get();
-        NBTTagCompound nbt = null;
+    }
-        GenericChunk gc = null;
+    @Override
-        if (cw.isChunkLoaded(chunk.x, chunk.z)) {
+    protected Supplier<GenericChunk> getLoadedChunkAsync(DynmapChunk ch) {
-            Chunk c = cw.getHandle().getChunkIfLoaded(chunk.x, chunk.z); //already safe async on vanilla
+        return getLoadedChunk(ch, true);
-            if ((c != null) && c.o) {    // c.loaded
+    }
-                if (provider == null) { //idk why, but paper uses this only sync, so I won't be smarter
+
-                    nbt = ChunkRegionLoader.a(cw.getHandle(), c);
+    @Override
-                } else {
+    protected Supplier<GenericChunk> loadChunkAsync(DynmapChunk chunk){
-                    nbt = CompletableFuture.supplyAsync(() -> ChunkRegionLoader.a(cw.getHandle(), c), ((CraftServer) Bukkit.getServer()).getServer()).join();
+        try {
-                }
+            CompletableFuture<NBTTagCompound> nbt = provider.getChunk(((CraftWorld) w).getHandle(), chunk.x, chunk.z);
-            }
+            return () -> {
-            if (nbt != null) {
+                NBTTagCompound compound = nbt.join();
-                gc = parseChunkFromNBT(new NBT.NBTCompound(nbt));
+                return compound == null ? null : parseChunkFromNBT(new NBT.NBTCompound(compound));
-            }
+            };
+        } catch (InvocationTargetException | IllegalAccessException e) {
+            e.printStackTrace();
         }
-        return gc;
+        return () -> null;
+    }
+
+    private Supplier<GenericChunk> getLoadedChunk(DynmapChunk chunk, boolean async) {
+        CraftWorld cw = (CraftWorld) w;
+        if (!cw.isChunkLoaded(chunk.x, chunk.z)) return () -> null;
+        Chunk c = cw.getHandle().getChunkIfLoaded(chunk.x, chunk.z); //already safe async on vanilla
+        if ((c == null) || c.o) return () -> null;    // c.loaded
+        if (async) { //idk why, but paper uses this only sync, so I won't be smarter
+            CompletableFuture<NBTTagCompound> nbt = CompletableFuture.supplyAsync(() -> ChunkRegionLoader.a(cw.getHandle(), c), ((CraftServer) Bukkit.getServer()).getServer());
+            return () -> parseChunkFromNBT(new NBT.NBTCompound(nbt.join()));
+        } else {
+            NBTTagCompound nbt = ChunkRegionLoader.a(cw.getHandle(), c);
+            GenericChunk genericChunk;
+            if (nbt != null) genericChunk = parseChunkFromNBT(new NBT.NBTCompound(nbt));
+            else genericChunk = null;
+            return () -> genericChunk;
+        }
+
     }
     // Load generic chunk from unloaded chunk
     protected GenericChunk loadChunk(DynmapChunk chunk) {
@@ -61,12 +82,9 @@ public class MapChunkCache118_2 extends GenericMapChunkCache {
         ChunkCoordIntPair cc = new ChunkCoordIntPair(chunk.x, chunk.z);
         GenericChunk gc = null;
         try {
-            if (provider == null){
+            nbt = cw.getHandle().k().a.f(cc);	// playerChunkMap
-                nbt = cw.getHandle().k().a.f(cc);	// playerChunkMap
+        } catch (IOException iox) {
-            } else {
+        }
-                nbt = provider.getChunk(cw.getHandle(),chunk.x, chunk.z);
-            }
-        } catch (IOException | InvocationTargetException | IllegalAccessException | NoSuchFieldException ignored) {}
         if (nbt != null) {
             gc = parseChunkFromNBT(new NBT.NBTCompound(nbt));
         }

spigot/src/main/java/org/dynmap/bukkit/DynmapPlugin.java

@@ -104,6 +104,7 @@ import org.dynmap.common.DynmapPlayer;
 import org.dynmap.common.DynmapServerInterface;
 import org.dynmap.common.chunk.GenericChunkCache;
 import org.dynmap.common.DynmapListenerManager.EventType;
+import org.dynmap.common.chunk.GenericMapChunkCache;
 import org.dynmap.hdmap.HDMap;
 import org.dynmap.markers.MarkerAPI;
 import org.dynmap.modsupport.ModSupportImpl;
@@ -562,17 +563,17 @@ public class DynmapPlugin extends JavaPlugin implements DynmapAPI {
                         }
                     }
                 } else {
-                    try {
+//                    synchronized (lock) {
-                        synchronized (lock) {
+                    if (prev_tick != cur_tick) {
-                            if (prev_tick != cur_tick) {
+                        prev_tick = cur_tick;
-                                prev_tick = cur_tick;
+                        cur_tick_starttime = System.nanoTime();
-                                cur_tick_starttime = System.nanoTime();
-                            }
-                            cc.loadChunks(Integer.MAX_VALUE);
-                        }
-                    } catch (Exception e) {
-                        e.printStackTrace();
                     }
+                    if (cc instanceof GenericMapChunkCache) {
+                        ((GenericMapChunkCache) cc).loadChunksAsync();
+                    } else {
+                        cc.loadChunks(Integer.MAX_VALUE);
+                    }
+//                    }
                 }
             }
             /* If cancelled due to world unload return nothing */