From 247e81bc61cb660043731075984bc019720e06cb Mon Sep 17 00:00:00 2001 From: stormboomer Date: Mon, 28 Aug 2023 08:03:43 +0200 Subject: [PATCH 1/3] Drasticly improve zoom tile calculation for larger maps when using MySQL storage engine. For Larger Tables doing Limit and Offset can have a big Impact on Statement execution because it is an IO heavy task. This fixes the issue by not doing any limit / offset on the SQL statement but instead query for all the data at once, and let the JDBC handler do the resultset handling. This can probably be adapted for MSSQL, PostgreSQL and SQLLite. --- .../dynmap/storage/mysql/MySQLMapStorage.java | 36 ++++++++----------- 1 file changed, 14 insertions(+), 22 deletions(-) diff --git a/DynmapCore/src/main/java/org/dynmap/storage/mysql/MySQLMapStorage.java b/DynmapCore/src/main/java/org/dynmap/storage/mysql/MySQLMapStorage.java index ec9c701b..c8c017b8 100644 --- a/DynmapCore/src/main/java/org/dynmap/storage/mysql/MySQLMapStorage.java +++ b/DynmapCore/src/main/java/org/dynmap/storage/mysql/MySQLMapStorage.java @@ -809,29 +809,21 @@ public class MySQLMapStorage extends MapStorage { } try { c = getConnection(); - boolean done = false; - int limit = 100; - int offset = 0; - while (!done) { - // Query tiles for given mapkey - Statement stmt = c.createStatement(); - ResultSet rs = stmt.executeQuery(String.format("SELECT x,y,zoom,Format FROM %s WHERE MapID=%d LIMIT %d OFFSET %d;", tableTiles, mapkey, limit, offset)); - int cnt = 0; - while (rs.next()) { - StorageTile st = new StorageTile(world, map, rs.getInt("x"), rs.getInt("y"), rs.getInt("zoom"), var); - final MapType.ImageEncoding encoding = MapType.ImageEncoding.fromOrd(rs.getInt("Format")); - if(cb != null) - cb.tileFound(st, encoding); - if(cbBase != null && st.zoom == 0) - cbBase.tileFound(st, encoding); - st.cleanup(); - cnt++; - } - rs.close(); - stmt.close(); - if (cnt < limit) done = true; - offset += cnt; + Statement stmt = c.createStatement(java.sql.ResultSet.TYPE_FORWARD_ONLY, //we want to stream our resultset one row at a time, we are not interessted in going back + java.sql.ResultSet.CONCUR_READ_ONLY); //since we do not handle the entire resultset in memory -> tell the statement that we are going to work read only + stmt.setFetchSize(100); //we can change the jdbc "retrieval chunk size". Basicly we limit how much rows are kept in memory. Bigger value = less network calls to DB, but more memory consumption + ResultSet rs = stmt.executeQuery(String.format("SELECT x,y,zoom,Format FROM %s WHERE MapID=%d;", tableTiles, mapkey)); //we do the query, but do not set any limit / offset. Since data is not kept in memory, just streamed from DB this should not be a problem, only the rows from setFetchSize are kept in memory. + while (rs.next()) { + StorageTile st = new StorageTile(world, map, rs.getInt("x"), rs.getInt("y"), rs.getInt("zoom"), var); + final MapType.ImageEncoding encoding = MapType.ImageEncoding.fromOrd(rs.getInt("Format")); + if(cb != null) + cb.tileFound(st, encoding); + if(cbBase != null && st.zoom == 0) + cbBase.tileFound(st, encoding); + st.cleanup(); } + rs.close(); + stmt.close(); if(cbEnd != null) cbEnd.searchEnded(); } catch (SQLException x) { From b8166a912214b4e116707e6c08b2547de48e2e35 Mon Sep 17 00:00:00 2001 From: stormboomer Date: Tue, 29 Aug 2023 13:23:54 +0200 Subject: [PATCH 2/3] Remove thread safety implementation for java 17, since it seems to not be needed anymore and improves render time by 8-10% --- .../java/org/dynmap/utils/ImageIOManager.java | 158 ++++++++++++------ 1 file changed, 108 insertions(+), 50 deletions(-) diff --git a/DynmapCore/src/main/java/org/dynmap/utils/ImageIOManager.java b/DynmapCore/src/main/java/org/dynmap/utils/ImageIOManager.java index e3fd9123..19fc12f1 100644 --- a/DynmapCore/src/main/java/org/dynmap/utils/ImageIOManager.java +++ b/DynmapCore/src/main/java/org/dynmap/utils/ImageIOManager.java @@ -107,68 +107,126 @@ public class ImageIOManager { } public static BufferOutputStream imageIOEncode(BufferedImage img, ImageFormat fmt) { - BufferOutputStream bos = new BufferOutputStream(); - + if(isRequiredJDKVersion(17,-1,-1)){ + return imageIOEncodeUnsafe(img, fmt); //we can skip Thread safety for more performance + } synchronized(imageioLock) { - try { - ImageIO.setUseCache(false); /* Don't use file cache - too small to be worth it */ - - fmt = validateFormat(fmt); - - if(fmt.getEncoding() == ImageEncoding.JPG) { - WritableRaster raster = img.getRaster(); - WritableRaster newRaster = raster.createWritableChild(0, 0, img.getWidth(), - img.getHeight(), 0, 0, new int[] {0, 1, 2}); - DirectColorModel cm = (DirectColorModel)img.getColorModel(); - DirectColorModel newCM = new DirectColorModel(cm.getPixelSize(), - cm.getRedMask(), cm.getGreenMask(), cm.getBlueMask()); - // now create the new buffer that is used ot write the image: - BufferedImage rgbBuffer = new BufferedImage(newCM, newRaster, false, null); + return imageIOEncodeUnsafe(img, fmt); + } + } + private static BufferOutputStream imageIOEncodeUnsafe(BufferedImage img, ImageFormat fmt) { + BufferOutputStream bos = new BufferOutputStream(); + try { + ImageIO.setUseCache(false); /* Don't use file cache - too small to be worth it */ - // Find a jpeg writer - ImageWriter writer = null; - Iterator iter = ImageIO.getImageWritersByFormatName("jpg"); - if (iter.hasNext()) { - writer = iter.next(); - } - if(writer == null) { - Log.severe("No JPEG ENCODER - Java VM does not support JPEG encoding"); - return null; - } - ImageWriteParam iwp = writer.getDefaultWriteParam(); - iwp.setCompressionMode(ImageWriteParam.MODE_EXPLICIT); - iwp.setCompressionQuality(fmt.getQuality()); + fmt = validateFormat(fmt); - ImageOutputStream ios; - ios = ImageIO.createImageOutputStream(bos); - writer.setOutput(ios); + if(fmt.getEncoding() == ImageEncoding.JPG) { + WritableRaster raster = img.getRaster(); + WritableRaster newRaster = raster.createWritableChild(0, 0, img.getWidth(), + img.getHeight(), 0, 0, new int[] {0, 1, 2}); + DirectColorModel cm = (DirectColorModel)img.getColorModel(); + DirectColorModel newCM = new DirectColorModel(cm.getPixelSize(), + cm.getRedMask(), cm.getGreenMask(), cm.getBlueMask()); + // now create the new buffer that is used ot write the image: + BufferedImage rgbBuffer = new BufferedImage(newCM, newRaster, false, null); - writer.write(null, new IIOImage(rgbBuffer, null, null), iwp); - writer.dispose(); - - rgbBuffer.flush(); + // Find a jpeg writer + ImageWriter writer = null; + Iterator iter = ImageIO.getImageWritersByFormatName("jpg"); + if (iter.hasNext()) { + writer = iter.next(); } - else if (fmt.getEncoding() == ImageEncoding.WEBP) { - doWEBPEncode(img, fmt, bos); + if(writer == null) { + Log.severe("No JPEG ENCODER - Java VM does not support JPEG encoding"); + return null; } - else { - ImageIO.write(img, fmt.getFileExt(), bos); /* Write to byte array stream - prevent bogus I/O errors */ - } - } catch (IOException iox) { - Log.info("Error encoding image - " + iox.getMessage()); - return null; + ImageWriteParam iwp = writer.getDefaultWriteParam(); + iwp.setCompressionMode(ImageWriteParam.MODE_EXPLICIT); + iwp.setCompressionQuality(fmt.getQuality()); + + ImageOutputStream ios; + ios = ImageIO.createImageOutputStream(bos); + writer.setOutput(ios); + + writer.write(null, new IIOImage(rgbBuffer, null, null), iwp); + writer.dispose(); + + rgbBuffer.flush(); } + else if (fmt.getEncoding() == ImageEncoding.WEBP) { + doWEBPEncode(img, fmt, bos); + } + else { + ImageIO.write(img, fmt.getFileExt(), bos); /* Write to byte array stream - prevent bogus I/O errors */ + } + } catch (IOException iox) { + Log.info("Error encoding image - " + iox.getMessage()); + return null; } return bos; } - + public static BufferedImage imageIODecode(MapStorageTile.TileRead tr) throws IOException { + if(isRequiredJDKVersion(17,-1,-1)){ + return imageIODecodeUnsafe(tr); //we can skip Thread safety for more performance + } synchronized(imageioLock) { - ImageIO.setUseCache(false); /* Don't use file cache - too small to be worth it */ - if (tr.format == ImageEncoding.WEBP) { - return doWEBPDecode(tr.image); - } - return ImageIO.read(tr.image); + return imageIODecodeUnsafe(tr); } } + + private static BufferedImage imageIODecodeUnsafe(MapStorageTile.TileRead tr) throws IOException { + ImageIO.setUseCache(false); /* Don't use file cache - too small to be worth it */ + if (tr.format == ImageEncoding.WEBP) { + return doWEBPDecode(tr.image); + } + return ImageIO.read(tr.image); + } + + /** + * Checks if the current JDK is running at least a specific version + * targetMinor and targetBuild can be set to -1, if the java.version only provides a Major release this will then only check for the major release + * @param targetMajor the required minimum major version + * @param targetMinor the required minimum minor version + * @param targetBuild the required minimum build version + * @return true if the current JDK version is the required minimum version + */ + private static boolean isRequiredJDKVersion(int targetMajor, int targetMinor, int targetBuild){ + String javaVersion = System.getProperty("java.version"); + String[] versionParts = javaVersion.split("\\."); + if(versionParts.length < 3){ + if(versionParts.length == 1 + && targetMinor == -1 + && targetBuild == -1 + && parseInt(versionParts[0], -1) >= targetMajor){ + return true;//we only have a major version and thats ok + } + return false; //can not evaluate + } + int major = parseInt(versionParts[0], -1); + int minor = parseInt(versionParts[1], -1); + int build = parseInt(versionParts[2], -1); + if(major != -1 && major >= targetMajor && + minor != -1 && minor >= targetMinor && + build != -1 && build >= targetBuild + ){ + return true; + } + return false; + } + + /** + * Parses a string to int, with a dynamic fallback value if not parsable + * @param input the String to parse + * @param fallback the Fallback value to use + * @return the parsed integer or the fallback value if unparsable + */ + private static int parseInt(String input, int fallback){ + int output = fallback; + try{ + output = Integer.parseInt(input); + } catch (NumberFormatException e) {} + return output; + } } From 0120c135c498564e827de2330e47d1b667ceff85 Mon Sep 17 00:00:00 2001 From: stormboomer Date: Tue, 29 Aug 2023 20:07:22 +0200 Subject: [PATCH 3/3] Added JDK 8 version compatiblity for DynmapCoreAPI. It seems that this is missing from latest release and causes incompatiblity with older versions. This should resolve this --- DynmapCoreAPI/build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/DynmapCoreAPI/build.gradle b/DynmapCoreAPI/build.gradle index 6003b05c..15771378 100644 --- a/DynmapCoreAPI/build.gradle +++ b/DynmapCoreAPI/build.gradle @@ -6,6 +6,7 @@ eclipse { name = "Dynmap(DynmapCoreAPI)" } } +sourceCompatibility = targetCompatibility = compileJava.sourceCompatibility = compileJava.targetCompatibility = '1.8' // Need this here so eclipse task generates correctly. description = "DynmapCoreAPI"