Sulkta-OSS/dynmap-neoforge
3
0
Fork 0
Code Issues 1 Pull requests 10 Projects Releases Packages Wiki Activity Actions

Switch to sdk v2

Browse source
This commit is contained in:
Mike Primm 2022-02-18 00:40:55 -06:00
parent b963cfe5d5
commit c4dc5777c7
4 changed files with 105 additions and 100 deletions
Download patch file Download diff file Expand all files Collapse all files

37
DynmapCore/build.gradle
View file

@ -17,12 +17,17 @@ dependencies {
implementation 'org.yaml:snakeyaml:1.23' // DON'T UPDATE - NEWER ONE TRIPS ON WINDOWS ENCODED FILES implementation 'org.yaml:snakeyaml:1.23' // DON'T UPDATE - NEWER ONE TRIPS ON WINDOWS ENCODED FILES
implementation 'com.googlecode.owasp-java-html-sanitizer:owasp-java-html-sanitizer:20180219.1' implementation 'com.googlecode.owasp-java-html-sanitizer:owasp-java-html-sanitizer:20180219.1'
implementation 'org.postgresql:postgresql:42.2.18' implementation 'org.postgresql:postgresql:42.2.18'
implementation platform('com.amazonaws:aws-java-sdk-bom:1.11.1000') implementation 'software.amazon.awssdk:s3:2.17.132'
implementation 'com.amazonaws:aws-java-sdk-core' implementation 'software.amazon.awssdk:aws-core:2.17.132'
implementation 'com.amazonaws:aws-java-sdk-s3' implementation 'software.amazon.awssdk:sdk-core:2.17.132'
implementation 'com.fasterxml.jackson.core:jackson-core' implementation 'software.amazon.awssdk:utils:2.17.132'
implementation 'com.fasterxml.jackson.core:jackson-databind' implementation 'software.amazon.awssdk:http-client-spi:2.17.132'
implementation 'com.fasterxml.jackson.core:jackson-annotations' implementation 'software.amazon.awssdk:profiles:2.17.132'
implementation 'software.amazon.awssdk:regions:2.17.132'
implementation 'software.amazon.awssdk:auth:2.17.132'
implementation 'software.amazon.awssdk:metrics-spi:2.17.132'
implementation 'software.amazon.awssdk:aws-xml-protocol:2.17.132'
implementation 'software.amazon.awssdk:protocol-core:2.17.132'
} }
processResources { processResources {
@ -57,12 +62,17 @@ shadowJar {
include(dependency('org.eclipse.jetty::')) include(dependency('org.eclipse.jetty::'))
include(dependency('org.eclipse.jetty.orbit:javax.servlet:')) include(dependency('org.eclipse.jetty.orbit:javax.servlet:'))
include(dependency('org.postgresql:postgresql:')) include(dependency('org.postgresql:postgresql:'))
include(dependency('com.amazonaws:aws-java-sdk-bom:')) include(dependency('software.amazon.awssdk:s3:'))
include(dependency('com.amazonaws:aws-java-sdk-core:')) include(dependency('software.amazon.awssdk:aws-core:'))
include(dependency('com.amazonaws:aws-java-sdk-s3:')) include(dependency('software.amazon.awssdk:sdk-core:'))
include(dependency('com.fasterxml.jackson.core:jackson-core:')) include(dependency('software.amazon.awssdk:utils:'))
include(dependency('com.fasterxml.jackson.core:jackson-databind:')) include(dependency('software.amazon.awssdk:http-client-spi:'))
include(dependency('com.fasterxml.jackson.core:jackson-annotations:')) include(dependency('software.amazon.awssdk:profiles:'))
include(dependency('software.amazon.awssdk:regions:'))
include(dependency('software.amazon.awssdk:auth:'))
include(dependency('software.amazon.awssdk:metrics-spi:'))
include(dependency('software.amazon.awssdk:aws-xml-protocol:'))
include(dependency('software.amazon.awssdk:protocol-core:'))
include(dependency(':DynmapCoreAPI')) include(dependency(':DynmapCoreAPI'))
exclude("META-INF/maven/**") exclude("META-INF/maven/**")
exclude("META-INF/services/**") exclude("META-INF/services/**")
@ -73,8 +83,7 @@ shadowJar {
relocate('org.owasp.html', 'org.dynmap.org.owasp.html') relocate('org.owasp.html', 'org.dynmap.org.owasp.html')
relocate('javax.servlet', 'org.dynmap.javax.servlet' ) relocate('javax.servlet', 'org.dynmap.javax.servlet' )
relocate('org.postgresql', 'org.dynmap.org.postgresql') relocate('org.postgresql', 'org.dynmap.org.postgresql')
relocate('com.amazonaws', 'org.dynmap.com.amazonaws') relocate('software.amazon.awssdk', 'org.dynmap.software.amazon.awssdk')
relocate('com.fasterxml', 'org.dynmap.com.fasterxml')
destinationDir = file '../target' destinationDir = file '../target'
classifier = '' classifier = ''
} }

164
DynmapCore/src/main/java/org/dynmap/storage/aws_s3/AWSS3MapStorage.java
View file

@ -1,13 +1,6 @@
package org.dynmap.storage.aws_s3; package org.dynmap.storage.aws_s3;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
@ -28,19 +21,25 @@ import org.dynmap.storage.MapStorageTileSearchEndCB;
import org.dynmap.utils.BufferInputStream; import org.dynmap.utils.BufferInputStream;
import org.dynmap.utils.BufferOutputStream; import org.dynmap.utils.BufferOutputStream;
import com.amazonaws.AmazonServiceException; import software.amazon.awssdk.awscore.exception.AwsServiceException;
import com.amazonaws.auth.AWSStaticCredentialsProvider; import software.amazon.awssdk.core.ResponseBytes;
import com.amazonaws.auth.BasicAWSCredentials; import software.amazon.awssdk.core.sync.RequestBody;
import com.amazonaws.auth.profile.ProfileCredentialsProvider; import software.amazon.awssdk.regions.Region;
import com.amazonaws.internal.StaticCredentialsProvider; import software.amazon.awssdk.services.s3.S3Client;
import com.amazonaws.services.s3.AmazonS3; import software.amazon.awssdk.services.s3.model.Delete;
import com.amazonaws.services.s3.AmazonS3ClientBuilder; import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
import com.amazonaws.services.s3.model.AccessControlList; import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest;
import com.amazonaws.services.s3.model.DeleteObjectsRequest; import software.amazon.awssdk.services.s3.model.GetBucketAclRequest;
import com.amazonaws.services.s3.model.ListObjectsV2Result; import software.amazon.awssdk.services.s3.model.GetBucketAclResponse;
import com.amazonaws.services.s3.model.ObjectMetadata; import software.amazon.awssdk.services.s3.model.GetObjectAclRequest;
import com.amazonaws.services.s3.model.S3Object; import software.amazon.awssdk.services.s3.model.GetObjectAclResponse;
import com.amazonaws.services.s3.model.S3ObjectSummary; import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
import software.amazon.awssdk.services.s3.model.ObjectIdentifier;
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
import software.amazon.awssdk.services.s3.model.S3Object;
public class AWSS3MapStorage extends MapStorage { public class AWSS3MapStorage extends MapStorage {
public class StorageTile extends MapStorageTile { public class StorageTile extends MapStorageTile {
@ -62,10 +61,11 @@ public class AWSS3MapStorage extends MapStorage {
public boolean exists() { public boolean exists() {
boolean exists = false; boolean exists = false;
try { try {
AccessControlList rslt = s3.getObjectAcl(bucketname, baseKey); GetObjectAclRequest req = GetObjectAclRequest.builder().bucket(bucketname).key(baseKey).build();
GetObjectAclResponse rslt = s3.getObjectAcl(req);
if (rslt != null) if (rslt != null)
exists = true; exists = true;
} catch (AmazonServiceException x) { } catch (AwsServiceException x) {
Log.severe("AWS Exception", x); Log.severe("AWS Exception", x);
} }
return exists; return exists;
@ -80,23 +80,20 @@ public class AWSS3MapStorage extends MapStorage {
public TileRead read() { public TileRead read() {
AWSS3MapStorage.this.getWriteLock(baseKey); AWSS3MapStorage.this.getWriteLock(baseKey);
try { try {
S3Object obj = s3.getObject(bucketname, baseKey); GetObjectRequest req = GetObjectRequest.builder().bucket(bucketname).key(baseKey).build();
ResponseBytes<GetObjectResponse> obj = s3.getObjectAsBytes(req);
if (obj != null) { if (obj != null) {
ObjectMetadata md = obj.getObjectMetadata(); GetObjectResponse rsp = obj.response();
TileRead tr = new TileRead(); TileRead tr = new TileRead();
byte[] buf = new byte[(int) md.getContentLength()]; byte[] buf = obj.asByteArray();
InputStream fis = obj.getObjectContent();
fis.read(buf, 0, buf.length); // Read whole thing
tr.image = new BufferInputStream(buf); tr.image = new BufferInputStream(buf);
tr.format = ImageEncoding.fromContentType(md.getContentType()); tr.format = ImageEncoding.fromContentType(rsp.contentType());
tr.hashCode = md.getContentMD5().hashCode(); tr.hashCode = rsp.eTag().hashCode();
tr.lastModified = md.getLastModified().getTime(); tr.lastModified = rsp.lastModified().toEpochMilli();
return tr; return tr;
} }
} catch (IOException x) { } catch (AwsServiceException x) {
Log.severe("AWS Exception", x);
} catch (AmazonServiceException x) {
Log.severe("AWS Exception", x); Log.severe("AWS Exception", x);
} finally { } finally {
AWSS3MapStorage.this.releaseWriteLock(baseKey); AWSS3MapStorage.this.releaseWriteLock(baseKey);
@ -110,15 +107,15 @@ public class AWSS3MapStorage extends MapStorage {
AWSS3MapStorage.this.getWriteLock(baseKey); AWSS3MapStorage.this.getWriteLock(baseKey);
try { try {
if (encImage == null) { // Delete? if (encImage == null) { // Delete?
s3.deleteObject(bucketname, baseKey); DeleteObjectRequest req = DeleteObjectRequest.builder().bucket(bucketname).key(baseKey).build();
s3.deleteObject(req);
} }
else { else {
ObjectMetadata md = new ObjectMetadata(); PutObjectRequest req = PutObjectRequest.builder().bucket(bucketname).key(baseKey).contentType(map.getImageFormat().getEncoding().getContentType()).build();
md.setContentType(map.getImageFormat().getEncoding().getContentType()); s3.putObject(req, RequestBody.fromBytes(encImage.buf));
s3.putObject(bucketname, baseKey, new ByteArrayInputStream(encImage.buf), md);
} }
done = true; done = true;
} catch (AmazonServiceException x) { } catch (AwsServiceException x) {
Log.severe("AWS Exception", x); Log.severe("AWS Exception", x);
} finally { } finally {
AWSS3MapStorage.this.releaseWriteLock(baseKey); AWSS3MapStorage.this.releaseWriteLock(baseKey);
@ -200,7 +197,7 @@ public class AWSS3MapStorage extends MapStorage {
private String bucketname; private String bucketname;
private String region; private String region;
private String profile_id; private String profile_id;
private AmazonS3 s3; private S3Client s3;
public AWSS3MapStorage() { public AWSS3MapStorage() {
} }
@ -217,24 +214,22 @@ public class AWSS3MapStorage extends MapStorage {
// Get our settings // Get our settings
bucketname = core.configuration.getString("storage/bucketname", "dynmap"); bucketname = core.configuration.getString("storage/bucketname", "dynmap");
region = core.configuration.getString("storage/region", "us-east-1"); region = core.configuration.getString("storage/region", "us-east-1");
profile_id = core.configuration.getString("storage/aws_profile_id", System.getenv("AWS_PROFILE"));
try { try {
// Now creste the access client for the S3 service // Now creste the access client for the S3 service
Log.info("Using AWS S3 storage: web site at S3 bucket " + bucketname + " in region " + region + " using AWS_PROFILE_ID=" + profile_id); Log.info("Using AWS S3 storage: web site at S3 bucket " + bucketname + " in region " + region + " using AWS_PROFILE_ID=" + profile_id);
s3 = AmazonS3ClientBuilder.standard().withRegion(region) s3 = S3Client.builder().region(Region.of(region)).build();
//.withCredentials(new ProfileCredentialsProvider("profile " + profile_id))
.build();
if (s3 == null) { if (s3 == null) {
Log.severe("Error creating S3 access client"); Log.severe("Error creating S3 access client");
return false; return false;
} }
// Make sure bucket exists and get ACL // Make sure bucket exists and get ACL
AccessControlList bucketACL = s3.getBucketAcl(bucketname); GetBucketAclRequest baclr = GetBucketAclRequest.builder().bucket(bucketname).build();
GetBucketAclResponse bucketACL = s3.getBucketAcl(baclr);
if (bucketACL == null) { if (bucketACL == null) {
Log.severe("Error: cannot find or access S3 bucket"); Log.severe("Error: cannot find or access S3 bucket");
return false; return false;
} }
} catch (AmazonServiceException x) { } catch (AwsServiceException x) {
Log.severe("AWS Exception", x); Log.severe("AWS Exception", x);
return false; return false;
} }
@ -296,10 +291,11 @@ public class AWSS3MapStorage extends MapStorage {
private void processEnumMapTiles(DynmapWorld world, MapType map, ImageVariant var, MapStorageTileEnumCB cb, MapStorageBaseTileEnumCB cbBase, MapStorageTileSearchEndCB cbEnd) { private void processEnumMapTiles(DynmapWorld world, MapType map, ImageVariant var, MapStorageTileEnumCB cb, MapStorageBaseTileEnumCB cbBase, MapStorageTileSearchEndCB cbEnd) {
String basekey = "tiles/" + world.getName() + "/" + map.getPrefix() + var.variantSuffix + "/"; String basekey = "tiles/" + world.getName() + "/" + map.getPrefix() + var.variantSuffix + "/";
try { try {
ListObjectsV2Result result = s3.listObjectsV2(bucketname, basekey); ListObjectsV2Request req = ListObjectsV2Request.builder().bucket(bucketname).prefix(basekey).build();
List<S3ObjectSummary> objects = result.getObjectSummaries(); ListObjectsV2Response result = s3.listObjectsV2(req);
for (S3ObjectSummary os : objects) { List<S3Object> objects = result.contents();
String key = os.getKey(); for (S3Object os : objects) {
String key = os.key();
key = key.substring(basekey.length()); // Strip off base key = key.substring(basekey.length()); // Strip off base
// Parse the extension // Parse the extension
String ext = null; String ext = null;
@ -341,7 +337,7 @@ public class AWSS3MapStorage extends MapStorage {
} }
} }
} }
} catch (AmazonServiceException x) { } catch (AwsServiceException x) {
Log.severe("AWS Exception", x); Log.severe("AWS Exception", x);
} }
if(cbEnd != null) { if(cbEnd != null) {
@ -388,25 +384,26 @@ public class AWSS3MapStorage extends MapStorage {
private void processPurgeMapTiles(DynmapWorld world, MapType map, ImageVariant var) { private void processPurgeMapTiles(DynmapWorld world, MapType map, ImageVariant var) {
String basekey = "tiles/" + world.getName() + "/" + map.getPrefix() + var.variantSuffix + "/"; String basekey = "tiles/" + world.getName() + "/" + map.getPrefix() + var.variantSuffix + "/";
try { try {
ListObjectsV2Result result = s3.listObjectsV2(bucketname, basekey); ListObjectsV2Request req = ListObjectsV2Request.builder().bucket(bucketname).prefix(basekey).build();
List<S3ObjectSummary> objects = result.getObjectSummaries(); ListObjectsV2Response result = s3.listObjectsV2(req);
ArrayList<String> keys = new ArrayList<String>(); List<S3Object> objects = result.contents();
for (S3ObjectSummary os : objects) { ArrayList<ObjectIdentifier> keys = new ArrayList<ObjectIdentifier>();
String key = os.getKey(); for (S3Object os : objects) {
keys.add(key); String key = os.key();
keys.add(ObjectIdentifier.builder().key(key).build());
if (keys.size() >= 100) { if (keys.size() >= 100) {
DeleteObjectsRequest dor = new DeleteObjectsRequest(bucketname).withKeys(keys.toArray(new String[0])); DeleteObjectsRequest delreq = DeleteObjectsRequest.builder().bucket(bucketname).delete(Delete.builder().objects(keys).build()).build();
s3.deleteObjects(dor); s3.deleteObjects(delreq);
keys.clear(); keys.clear();
} }
} }
// Any left? // Any left?
if (keys.size() > 0) { if (keys.size() > 0) {
DeleteObjectsRequest dor = new DeleteObjectsRequest(bucketname).withKeys(keys.toArray(new String[0])); DeleteObjectsRequest delreq = DeleteObjectsRequest.builder().bucket(bucketname).delete(Delete.builder().objects(keys).build()).build();
s3.deleteObjects(dor); s3.deleteObjects(delreq);
keys.clear(); keys.clear();
} }
} catch (AmazonServiceException x) { } catch (AwsServiceException x) {
Log.severe("AWS Exception", x); Log.severe("AWS Exception", x);
} }
} }
@ -437,15 +434,15 @@ public class AWSS3MapStorage extends MapStorage {
getWriteLock(baseKey); getWriteLock(baseKey);
try { try {
if (encImage == null) { // Delete? if (encImage == null) { // Delete?
s3.deleteObject(bucketname, baseKey); DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucket(bucketname).key(baseKey).build();
s3.deleteObject(delreq);
} }
else { else {
ObjectMetadata md = new ObjectMetadata(); PutObjectRequest req = PutObjectRequest.builder().bucket(bucketname).key(baseKey).contentType("image/png").build();
md.setContentType("image/png"); s3.putObject(req, RequestBody.fromBytes(encImage.buf));
s3.putObject(bucketname, baseKey, new ByteArrayInputStream(encImage.buf), md);
} }
done = true; done = true;
} catch (AmazonServiceException x) { } catch (AwsServiceException x) {
Log.severe("AWS Exception", x); Log.severe("AWS Exception", x);
} finally { } finally {
releaseWriteLock(baseKey); releaseWriteLock(baseKey);
@ -464,10 +461,11 @@ public class AWSS3MapStorage extends MapStorage {
String baseKey = "faces/" + facetype.id + "/" + playername + ".png"; String baseKey = "faces/" + facetype.id + "/" + playername + ".png";
boolean exists = false; boolean exists = false;
try { try {
AccessControlList rslt = s3.getObjectAcl(bucketname, baseKey); GetObjectAclRequest req = GetObjectAclRequest.builder().bucket(bucketname).key(baseKey).build();
GetObjectAclResponse rslt = s3.getObjectAcl(req);
if (rslt != null) if (rslt != null)
exists = true; exists = true;
} catch (AmazonServiceException x) { } catch (AwsServiceException x) {
Log.severe("AWS Exception", x); Log.severe("AWS Exception", x);
} }
return exists; return exists;
@ -480,15 +478,15 @@ public class AWSS3MapStorage extends MapStorage {
getWriteLock(baseKey); getWriteLock(baseKey);
try { try {
if (encImage == null) { // Delete? if (encImage == null) { // Delete?
s3.deleteObject(bucketname, baseKey); DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucket(bucketname).key(baseKey).build();
s3.deleteObject(delreq);
} }
else { else {
ObjectMetadata md = new ObjectMetadata(); PutObjectRequest req = PutObjectRequest.builder().bucket(bucketname).key(baseKey).contentType("image/png").build();
md.setContentType("image/png"); s3.putObject(req, RequestBody.fromBytes(encImage.buf));
s3.putObject(bucketname, baseKey, new ByteArrayInputStream(encImage.buf), md);
} }
done = true; done = true;
} catch (AmazonServiceException x) { } catch (AwsServiceException x) {
Log.severe("AWS Exception", x); Log.severe("AWS Exception", x);
} finally { } finally {
releaseWriteLock(baseKey); releaseWriteLock(baseKey);
@ -508,15 +506,15 @@ public class AWSS3MapStorage extends MapStorage {
getWriteLock(baseKey); getWriteLock(baseKey);
try { try {
if (content == null) { // Delete? if (content == null) { // Delete?
s3.deleteObject(bucketname, baseKey); DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucket(bucketname).key(baseKey).build();
s3.deleteObject(delreq);
} }
else { else {
ObjectMetadata md = new ObjectMetadata(); PutObjectRequest req = PutObjectRequest.builder().bucket(bucketname).key(baseKey).contentType("application/json").build();
md.setContentType("application/json"); s3.putObject(req, RequestBody.fromBytes(content.getBytes(StandardCharsets.UTF_8)));
s3.putObject(bucketname, baseKey, new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)), md);
} }
done = true; done = true;
} catch (AmazonServiceException x) { } catch (AwsServiceException x) {
Log.severe("AWS Exception", x); Log.severe("AWS Exception", x);
} finally { } finally {
releaseWriteLock(baseKey); releaseWriteLock(baseKey);
@ -571,15 +569,15 @@ public class AWSS3MapStorage extends MapStorage {
getWriteLock(baseKey); getWriteLock(baseKey);
try { try {
if (content == null) { // Delete? if (content == null) { // Delete?
s3.deleteObject(bucketname, baseKey); DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucket(bucketname).key(baseKey).build();
s3.deleteObject(delreq);
} }
else { else {
ObjectMetadata md = new ObjectMetadata(); PutObjectRequest req = PutObjectRequest.builder().bucket(bucketname).key(baseKey).contentType("text/plain").build();
md.setContentType("text/plain"); s3.putObject(req, RequestBody.fromBytes(content.buf));
s3.putObject(bucketname, baseKey, new ByteArrayInputStream(content.buf), md);
} }
done = true; done = true;
} catch (AmazonServiceException x) { } catch (AwsServiceException x) {
Log.severe("AWS Exception", x); Log.severe("AWS Exception", x);
} finally { } finally {
releaseWriteLock(baseKey); releaseWriteLock(baseKey);

2
bukkit-helper/.settings/org.eclipse.jdt.core.prefs
View file

@ -1,5 +1,5 @@
# #
#Thu Feb 17 18:10:31 CST 2022 #Thu Feb 17 22:40:49 CST 2022
org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
org.eclipse.jdt.core.compiler.problem.assertIdentifier=error org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8

2
spigot/src/main/resources/configuration.txt
View file

@ -45,8 +45,6 @@ storage:
#type: aws_s3 #type: aws_s3
#bucketname: dynmap #bucketname: dynmap
#region: us-east-1 #region: us-east-1
#aws_access_key_id: <access-key>
#aws_secret_access_key: <secret-access-key>
components: components: