From 45bc02cf3a44cf0e6af60dd2904c9a0bf7d42fef Mon Sep 17 00:00:00 2001 From: Mike Primm Date: Fri, 18 Feb 2022 00:40:55 -0600 Subject: [PATCH] Switch to sdk v2 --- DynmapCore/build.gradle | 37 ++-- .../storage/aws_s3/AWSS3MapStorage.java | 164 +++++++++--------- .../.settings/org.eclipse.jdt.core.prefs | 2 +- spigot/src/main/resources/configuration.txt | 2 - 4 files changed, 105 insertions(+), 100 deletions(-) diff --git a/DynmapCore/build.gradle b/DynmapCore/build.gradle index cdd7e06c..f6b85116 100644 --- a/DynmapCore/build.gradle +++ b/DynmapCore/build.gradle @@ -17,12 +17,17 @@ dependencies { implementation 'org.yaml:snakeyaml:1.23' // DON'T UPDATE - NEWER ONE TRIPS ON WINDOWS ENCODED FILES implementation 'com.googlecode.owasp-java-html-sanitizer:owasp-java-html-sanitizer:20180219.1' implementation 'org.postgresql:postgresql:42.2.18' - implementation platform('com.amazonaws:aws-java-sdk-bom:1.11.1000') - implementation 'com.amazonaws:aws-java-sdk-core' - implementation 'com.amazonaws:aws-java-sdk-s3' - implementation 'com.fasterxml.jackson.core:jackson-core' - implementation 'com.fasterxml.jackson.core:jackson-databind' - implementation 'com.fasterxml.jackson.core:jackson-annotations' + implementation 'software.amazon.awssdk:s3:2.17.132' + implementation 'software.amazon.awssdk:aws-core:2.17.132' + implementation 'software.amazon.awssdk:sdk-core:2.17.132' + implementation 'software.amazon.awssdk:utils:2.17.132' + implementation 'software.amazon.awssdk:http-client-spi:2.17.132' + implementation 'software.amazon.awssdk:profiles:2.17.132' + implementation 'software.amazon.awssdk:regions:2.17.132' + implementation 'software.amazon.awssdk:auth:2.17.132' + implementation 'software.amazon.awssdk:metrics-spi:2.17.132' + implementation 'software.amazon.awssdk:aws-xml-protocol:2.17.132' + implementation 'software.amazon.awssdk:protocol-core:2.17.132' } processResources { @@ -57,12 +62,17 @@ shadowJar { include(dependency('org.eclipse.jetty::')) include(dependency('org.eclipse.jetty.orbit:javax.servlet:')) include(dependency('org.postgresql:postgresql:')) - include(dependency('com.amazonaws:aws-java-sdk-bom:')) - include(dependency('com.amazonaws:aws-java-sdk-core:')) - include(dependency('com.amazonaws:aws-java-sdk-s3:')) - include(dependency('com.fasterxml.jackson.core:jackson-core:')) - include(dependency('com.fasterxml.jackson.core:jackson-databind:')) - include(dependency('com.fasterxml.jackson.core:jackson-annotations:')) + include(dependency('software.amazon.awssdk:s3:')) + include(dependency('software.amazon.awssdk:aws-core:')) + include(dependency('software.amazon.awssdk:sdk-core:')) + include(dependency('software.amazon.awssdk:utils:')) + include(dependency('software.amazon.awssdk:http-client-spi:')) + include(dependency('software.amazon.awssdk:profiles:')) + include(dependency('software.amazon.awssdk:regions:')) + include(dependency('software.amazon.awssdk:auth:')) + include(dependency('software.amazon.awssdk:metrics-spi:')) + include(dependency('software.amazon.awssdk:aws-xml-protocol:')) + include(dependency('software.amazon.awssdk:protocol-core:')) include(dependency(':DynmapCoreAPI')) exclude("META-INF/maven/**") exclude("META-INF/services/**") @@ -73,8 +83,7 @@ shadowJar { relocate('org.owasp.html', 'org.dynmap.org.owasp.html') relocate('javax.servlet', 'org.dynmap.javax.servlet' ) relocate('org.postgresql', 'org.dynmap.org.postgresql') - relocate('com.amazonaws', 'org.dynmap.com.amazonaws') - relocate('com.fasterxml', 'org.dynmap.com.fasterxml') + relocate('software.amazon.awssdk', 'org.dynmap.software.amazon.awssdk') destinationDir = file '../target' classifier = '' } diff --git a/DynmapCore/src/main/java/org/dynmap/storage/aws_s3/AWSS3MapStorage.java b/DynmapCore/src/main/java/org/dynmap/storage/aws_s3/AWSS3MapStorage.java index 055fa52b..38ffeafb 100644 --- a/DynmapCore/src/main/java/org/dynmap/storage/aws_s3/AWSS3MapStorage.java +++ b/DynmapCore/src/main/java/org/dynmap/storage/aws_s3/AWSS3MapStorage.java @@ -1,13 +1,6 @@ package org.dynmap.storage.aws_s3; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; import java.nio.charset.StandardCharsets; -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -28,19 +21,25 @@ import org.dynmap.storage.MapStorageTileSearchEndCB; import org.dynmap.utils.BufferInputStream; import org.dynmap.utils.BufferOutputStream; -import com.amazonaws.AmazonServiceException; -import com.amazonaws.auth.AWSStaticCredentialsProvider; -import com.amazonaws.auth.BasicAWSCredentials; -import com.amazonaws.auth.profile.ProfileCredentialsProvider; -import com.amazonaws.internal.StaticCredentialsProvider; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3ClientBuilder; -import com.amazonaws.services.s3.model.AccessControlList; -import com.amazonaws.services.s3.model.DeleteObjectsRequest; -import com.amazonaws.services.s3.model.ListObjectsV2Result; -import com.amazonaws.services.s3.model.ObjectMetadata; -import com.amazonaws.services.s3.model.S3Object; -import com.amazonaws.services.s3.model.S3ObjectSummary; +import software.amazon.awssdk.awscore.exception.AwsServiceException; +import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.Delete; +import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest; +import software.amazon.awssdk.services.s3.model.GetBucketAclRequest; +import software.amazon.awssdk.services.s3.model.GetBucketAclResponse; +import software.amazon.awssdk.services.s3.model.GetObjectAclRequest; +import software.amazon.awssdk.services.s3.model.GetObjectAclResponse; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Response; +import software.amazon.awssdk.services.s3.model.ObjectIdentifier; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.S3Object; public class AWSS3MapStorage extends MapStorage { public class StorageTile extends MapStorageTile { @@ -62,10 +61,11 @@ public class AWSS3MapStorage extends MapStorage { public boolean exists() { boolean exists = false; try { - AccessControlList rslt = s3.getObjectAcl(bucketname, baseKey); + GetObjectAclRequest req = GetObjectAclRequest.builder().bucket(bucketname).key(baseKey).build(); + GetObjectAclResponse rslt = s3.getObjectAcl(req); if (rslt != null) exists = true; - } catch (AmazonServiceException x) { + } catch (AwsServiceException x) { Log.severe("AWS Exception", x); } return exists; @@ -80,23 +80,20 @@ public class AWSS3MapStorage extends MapStorage { public TileRead read() { AWSS3MapStorage.this.getWriteLock(baseKey); try { - S3Object obj = s3.getObject(bucketname, baseKey); + GetObjectRequest req = GetObjectRequest.builder().bucket(bucketname).key(baseKey).build(); + ResponseBytes obj = s3.getObjectAsBytes(req); if (obj != null) { - ObjectMetadata md = obj.getObjectMetadata(); + GetObjectResponse rsp = obj.response(); TileRead tr = new TileRead(); - byte[] buf = new byte[(int) md.getContentLength()]; - InputStream fis = obj.getObjectContent(); - fis.read(buf, 0, buf.length); // Read whole thing + byte[] buf = obj.asByteArray(); tr.image = new BufferInputStream(buf); - tr.format = ImageEncoding.fromContentType(md.getContentType()); - tr.hashCode = md.getContentMD5().hashCode(); - tr.lastModified = md.getLastModified().getTime(); + tr.format = ImageEncoding.fromContentType(rsp.contentType()); + tr.hashCode = rsp.eTag().hashCode(); + tr.lastModified = rsp.lastModified().toEpochMilli(); return tr; } - } catch (IOException x) { - Log.severe("AWS Exception", x); - } catch (AmazonServiceException x) { + } catch (AwsServiceException x) { Log.severe("AWS Exception", x); } finally { AWSS3MapStorage.this.releaseWriteLock(baseKey); @@ -110,15 +107,15 @@ public class AWSS3MapStorage extends MapStorage { AWSS3MapStorage.this.getWriteLock(baseKey); try { if (encImage == null) { // Delete? - s3.deleteObject(bucketname, baseKey); + DeleteObjectRequest req = DeleteObjectRequest.builder().bucket(bucketname).key(baseKey).build(); + s3.deleteObject(req); } else { - ObjectMetadata md = new ObjectMetadata(); - md.setContentType(map.getImageFormat().getEncoding().getContentType()); - s3.putObject(bucketname, baseKey, new ByteArrayInputStream(encImage.buf), md); + PutObjectRequest req = PutObjectRequest.builder().bucket(bucketname).key(baseKey).contentType(map.getImageFormat().getEncoding().getContentType()).build(); + s3.putObject(req, RequestBody.fromBytes(encImage.buf)); } done = true; - } catch (AmazonServiceException x) { + } catch (AwsServiceException x) { Log.severe("AWS Exception", x); } finally { AWSS3MapStorage.this.releaseWriteLock(baseKey); @@ -200,7 +197,7 @@ public class AWSS3MapStorage extends MapStorage { private String bucketname; private String region; private String profile_id; - private AmazonS3 s3; + private S3Client s3; public AWSS3MapStorage() { } @@ -217,24 +214,22 @@ public class AWSS3MapStorage extends MapStorage { // Get our settings bucketname = core.configuration.getString("storage/bucketname", "dynmap"); region = core.configuration.getString("storage/region", "us-east-1"); - profile_id = core.configuration.getString("storage/aws_profile_id", System.getenv("AWS_PROFILE")); try { // Now creste the access client for the S3 service Log.info("Using AWS S3 storage: web site at S3 bucket " + bucketname + " in region " + region + " using AWS_PROFILE_ID=" + profile_id); - s3 = AmazonS3ClientBuilder.standard().withRegion(region) - //.withCredentials(new ProfileCredentialsProvider("profile " + profile_id)) - .build(); + s3 = S3Client.builder().region(Region.of(region)).build(); if (s3 == null) { Log.severe("Error creating S3 access client"); return false; } // Make sure bucket exists and get ACL - AccessControlList bucketACL = s3.getBucketAcl(bucketname); + GetBucketAclRequest baclr = GetBucketAclRequest.builder().bucket(bucketname).build(); + GetBucketAclResponse bucketACL = s3.getBucketAcl(baclr); if (bucketACL == null) { Log.severe("Error: cannot find or access S3 bucket"); return false; } - } catch (AmazonServiceException x) { + } catch (AwsServiceException x) { Log.severe("AWS Exception", x); return false; } @@ -296,10 +291,11 @@ public class AWSS3MapStorage extends MapStorage { private void processEnumMapTiles(DynmapWorld world, MapType map, ImageVariant var, MapStorageTileEnumCB cb, MapStorageBaseTileEnumCB cbBase, MapStorageTileSearchEndCB cbEnd) { String basekey = "tiles/" + world.getName() + "/" + map.getPrefix() + var.variantSuffix + "/"; try { - ListObjectsV2Result result = s3.listObjectsV2(bucketname, basekey); - List objects = result.getObjectSummaries(); - for (S3ObjectSummary os : objects) { - String key = os.getKey(); + ListObjectsV2Request req = ListObjectsV2Request.builder().bucket(bucketname).prefix(basekey).build(); + ListObjectsV2Response result = s3.listObjectsV2(req); + List objects = result.contents(); + for (S3Object os : objects) { + String key = os.key(); key = key.substring(basekey.length()); // Strip off base // Parse the extension String ext = null; @@ -341,7 +337,7 @@ public class AWSS3MapStorage extends MapStorage { } } } - } catch (AmazonServiceException x) { + } catch (AwsServiceException x) { Log.severe("AWS Exception", x); } if(cbEnd != null) { @@ -388,25 +384,26 @@ public class AWSS3MapStorage extends MapStorage { private void processPurgeMapTiles(DynmapWorld world, MapType map, ImageVariant var) { String basekey = "tiles/" + world.getName() + "/" + map.getPrefix() + var.variantSuffix + "/"; try { - ListObjectsV2Result result = s3.listObjectsV2(bucketname, basekey); - List objects = result.getObjectSummaries(); - ArrayList keys = new ArrayList(); - for (S3ObjectSummary os : objects) { - String key = os.getKey(); - keys.add(key); + ListObjectsV2Request req = ListObjectsV2Request.builder().bucket(bucketname).prefix(basekey).build(); + ListObjectsV2Response result = s3.listObjectsV2(req); + List objects = result.contents(); + ArrayList keys = new ArrayList(); + for (S3Object os : objects) { + String key = os.key(); + keys.add(ObjectIdentifier.builder().key(key).build()); if (keys.size() >= 100) { - DeleteObjectsRequest dor = new DeleteObjectsRequest(bucketname).withKeys(keys.toArray(new String[0])); - s3.deleteObjects(dor); + DeleteObjectsRequest delreq = DeleteObjectsRequest.builder().bucket(bucketname).delete(Delete.builder().objects(keys).build()).build(); + s3.deleteObjects(delreq); keys.clear(); } } // Any left? if (keys.size() > 0) { - DeleteObjectsRequest dor = new DeleteObjectsRequest(bucketname).withKeys(keys.toArray(new String[0])); - s3.deleteObjects(dor); + DeleteObjectsRequest delreq = DeleteObjectsRequest.builder().bucket(bucketname).delete(Delete.builder().objects(keys).build()).build(); + s3.deleteObjects(delreq); keys.clear(); } - } catch (AmazonServiceException x) { + } catch (AwsServiceException x) { Log.severe("AWS Exception", x); } } @@ -437,15 +434,15 @@ public class AWSS3MapStorage extends MapStorage { getWriteLock(baseKey); try { if (encImage == null) { // Delete? - s3.deleteObject(bucketname, baseKey); + DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucket(bucketname).key(baseKey).build(); + s3.deleteObject(delreq); } else { - ObjectMetadata md = new ObjectMetadata(); - md.setContentType("image/png"); - s3.putObject(bucketname, baseKey, new ByteArrayInputStream(encImage.buf), md); + PutObjectRequest req = PutObjectRequest.builder().bucket(bucketname).key(baseKey).contentType("image/png").build(); + s3.putObject(req, RequestBody.fromBytes(encImage.buf)); } done = true; - } catch (AmazonServiceException x) { + } catch (AwsServiceException x) { Log.severe("AWS Exception", x); } finally { releaseWriteLock(baseKey); @@ -464,10 +461,11 @@ public class AWSS3MapStorage extends MapStorage { String baseKey = "faces/" + facetype.id + "/" + playername + ".png"; boolean exists = false; try { - AccessControlList rslt = s3.getObjectAcl(bucketname, baseKey); + GetObjectAclRequest req = GetObjectAclRequest.builder().bucket(bucketname).key(baseKey).build(); + GetObjectAclResponse rslt = s3.getObjectAcl(req); if (rslt != null) exists = true; - } catch (AmazonServiceException x) { + } catch (AwsServiceException x) { Log.severe("AWS Exception", x); } return exists; @@ -480,15 +478,15 @@ public class AWSS3MapStorage extends MapStorage { getWriteLock(baseKey); try { if (encImage == null) { // Delete? - s3.deleteObject(bucketname, baseKey); + DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucket(bucketname).key(baseKey).build(); + s3.deleteObject(delreq); } else { - ObjectMetadata md = new ObjectMetadata(); - md.setContentType("image/png"); - s3.putObject(bucketname, baseKey, new ByteArrayInputStream(encImage.buf), md); + PutObjectRequest req = PutObjectRequest.builder().bucket(bucketname).key(baseKey).contentType("image/png").build(); + s3.putObject(req, RequestBody.fromBytes(encImage.buf)); } done = true; - } catch (AmazonServiceException x) { + } catch (AwsServiceException x) { Log.severe("AWS Exception", x); } finally { releaseWriteLock(baseKey); @@ -508,15 +506,15 @@ public class AWSS3MapStorage extends MapStorage { getWriteLock(baseKey); try { if (content == null) { // Delete? - s3.deleteObject(bucketname, baseKey); + DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucket(bucketname).key(baseKey).build(); + s3.deleteObject(delreq); } else { - ObjectMetadata md = new ObjectMetadata(); - md.setContentType("application/json"); - s3.putObject(bucketname, baseKey, new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)), md); + PutObjectRequest req = PutObjectRequest.builder().bucket(bucketname).key(baseKey).contentType("application/json").build(); + s3.putObject(req, RequestBody.fromBytes(content.getBytes(StandardCharsets.UTF_8))); } done = true; - } catch (AmazonServiceException x) { + } catch (AwsServiceException x) { Log.severe("AWS Exception", x); } finally { releaseWriteLock(baseKey); @@ -571,15 +569,15 @@ public class AWSS3MapStorage extends MapStorage { getWriteLock(baseKey); try { if (content == null) { // Delete? - s3.deleteObject(bucketname, baseKey); + DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucket(bucketname).key(baseKey).build(); + s3.deleteObject(delreq); } else { - ObjectMetadata md = new ObjectMetadata(); - md.setContentType("text/plain"); - s3.putObject(bucketname, baseKey, new ByteArrayInputStream(content.buf), md); + PutObjectRequest req = PutObjectRequest.builder().bucket(bucketname).key(baseKey).contentType("text/plain").build(); + s3.putObject(req, RequestBody.fromBytes(content.buf)); } done = true; - } catch (AmazonServiceException x) { + } catch (AwsServiceException x) { Log.severe("AWS Exception", x); } finally { releaseWriteLock(baseKey); diff --git a/bukkit-helper/.settings/org.eclipse.jdt.core.prefs b/bukkit-helper/.settings/org.eclipse.jdt.core.prefs index c8f0f61d..8f00b17b 100644 --- a/bukkit-helper/.settings/org.eclipse.jdt.core.prefs +++ b/bukkit-helper/.settings/org.eclipse.jdt.core.prefs @@ -1,5 +1,5 @@ # -#Thu Feb 17 18:10:31 CST 2022 +#Thu Feb 17 22:40:49 CST 2022 org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning org.eclipse.jdt.core.compiler.problem.assertIdentifier=error org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 diff --git a/spigot/src/main/resources/configuration.txt b/spigot/src/main/resources/configuration.txt index d70a9f51..7968a104 100644 --- a/spigot/src/main/resources/configuration.txt +++ b/spigot/src/main/resources/configuration.txt @@ -45,8 +45,6 @@ storage: #type: aws_s3 #bucketname: dynmap #region: us-east-1 - #aws_access_key_id: - #aws_secret_access_key: components: