From a5b7adb32a099bee3ade2b7a4bcad977a72143c6 Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Mon, 17 Jul 2023 15:30:56 +0800 Subject: [PATCH 01/28] version to 7.14.0 --- CHANGELOG.md | 9 +++++++-- README.md | 4 ++-- src/main/java/com/qiniu/common/Constants.java | 2 +- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 88fc00ac8..85e69e3ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,12 @@ # Changelog +## 7.14.0（2023-07-18) +* Region 移除亚太首尔区域 +* BucketManager 的部分 Bucket 级别接口支持主备重试 + ## 7.13.1（2023-05-25) -* GetBucketQuota -* ChangeApiHost +* 优化 BucketManager 的 getBucketQuota 方法 +* 废弃 pu.qbox.me 域名 +* 废弃 BucketManager 的 listFilesV2 方法 * PlayExpire ## 7.13.0（2023-03-30) diff --git a/README.md b/README.md index 0950f6efb..d30d2398c 100644 --- a/README.md +++ b/README.md @@ -13,12 +13,12 @@ com.qiniu qiniu-java-sdk - [7.13.0, 7.13.99] + [7.14.0, 7.14.99] ``` 或者 Gradle: ```groovy -implementation 'com.qiniu:qiniu-java-sdk:7.13.+' +implementation 'com.qiniu:qiniu-java-sdk:7.14.+' ``` ## 运行环境 diff --git a/src/main/java/com/qiniu/common/Constants.java b/src/main/java/com/qiniu/common/Constants.java index a1cd6655d..83447124f 100644 --- a/src/main/java/com/qiniu/common/Constants.java +++ b/src/main/java/com/qiniu/common/Constants.java @@ -9,7 +9,7 @@ public final class Constants { /** * 版本号 */ - public static final String VERSION = "7.13.1"; + public static final String VERSION = "7.14.0"; /** * 块大小，不能改变 */ From e47717965fab7603d11853f8d289dd636772e85e Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Fri, 27 Oct 2023 11:35:39 +0800 Subject: [PATCH 02/28] Region support create by region id --- .../storage/ApiInterceptorRetryHosts.java | 4 ++ .../storage/ApiInterceptorRetrySimple.java | 4 ++ src/main/java/com/qiniu/storage/Region.java | 49 +++++++++++-- src/test/java/test/com/qiniu/TestConfig.java | 4 +- .../test/com/qiniu/storage/RegionTest.java | 70 +++++++++++++++++++ 5 files changed, 125 insertions(+), 6 deletions(-) create mode 100644 src/test/java/test/com/qiniu/storage/RegionTest.java diff --git a/src/main/java/com/qiniu/storage/ApiInterceptorRetryHosts.java b/src/main/java/com/qiniu/storage/ApiInterceptorRetryHosts.java index 92878e65e..343ac2b5f 100644 --- a/src/main/java/com/qiniu/storage/ApiInterceptorRetryHosts.java +++ b/src/main/java/com/qiniu/storage/ApiInterceptorRetryHosts.java @@ -84,6 +84,10 @@ Api.Response intercept(Api.Request request, Api.Handler handler) throws QiniuExc request.setHost(newHost); } + if (response != null && response.getResponse() != null) { + response.getResponse().close(); + } + int interval = retryInterval.interval(); if (interval <= 0) { continue; diff --git a/src/main/java/com/qiniu/storage/ApiInterceptorRetrySimple.java b/src/main/java/com/qiniu/storage/ApiInterceptorRetrySimple.java index befd92c67..64768bb4d 100644 --- a/src/main/java/com/qiniu/storage/ApiInterceptorRetrySimple.java +++ b/src/main/java/com/qiniu/storage/ApiInterceptorRetrySimple.java @@ -50,6 +50,10 @@ Api.Response intercept(Api.Request request, Api.Handler handler) throws QiniuExc } request = cloneRequest; + if (response != null && response.getResponse() != null) { + response.getResponse().close(); + } + int interval = retryInterval.interval(); if (interval <= 0) { continue; diff --git a/src/main/java/com/qiniu/storage/Region.java b/src/main/java/com/qiniu/storage/Region.java index e9d75becb..35fcb3f86 100644 --- a/src/main/java/com/qiniu/storage/Region.java +++ b/src/main/java/com/qiniu/storage/Region.java @@ -57,9 +57,11 @@ public class Region implements Cloneable { /** * 华东机房相关域名 + * 废弃，使用 {@link Region#createWithRegionId} 替换，regionId：z0 * * @return 区域信息 */ + @Deprecated public static Region region0() { return new Builder(). region("z0"). @@ -74,9 +76,11 @@ public static Region region0() { /** * 华东机房相关域名 + * 废弃，使用 {@link Region#createWithRegionId} 替换，regionId：z0 * * @return 区域信息 */ + @Deprecated public static Region huadong() { return region0(); } @@ -84,9 +88,11 @@ public static Region huadong() { /** * 华东浙江 2 机房相关域名 + * 废弃，使用 {@link Region#createWithRegionId} 替换，regionId：cn-east-2 * * @return 区域信息 */ + @Deprecated public static Region regionCnEast2() { return new Builder(). region("cn-east-2"). @@ -101,9 +107,11 @@ public static Region regionCnEast2() { /** * 华东浙江 2 机房相关域名 + * 废弃，使用 {@link Region#createWithRegionId} 替换，regionId：cn-east-2 * * @return 区域信息 */ + @Deprecated public static Region huadongZheJiang2() { return regionCnEast2(); } @@ -137,9 +145,11 @@ public static Region qvmHuadong() { /** * 华北机房相关域名 + * 废弃，使用 {@link Region#createWithRegionId} 替换，regionId：z1 * * @return 区域信息 */ + @Deprecated public static Region region1() { return new Builder(). region("z1"). @@ -154,9 +164,11 @@ public static Region region1() { /** * 华北机房相关域名 + * 废弃，使用 {@link Region#createWithRegionId} 替换，regionId：z1 * * @return 区域信息 */ + @Deprecated public static Region huabei() { return region1(); } @@ -189,9 +201,11 @@ public static Region qvmHuabei() { /** * 华南机房相关域名 + * 废弃，使用 {@link Region#createWithRegionId} 替换，regionId：z2 * * @return 区域信息 */ + @Deprecated public static Region region2() { return new Builder(). region("z2"). @@ -206,18 +220,22 @@ public static Region region2() { /** * 华南机房相关域名 + * 废弃，使用 {@link Region#createWithRegionId} 替换，regionId：z2 * * @return 区域信息 */ + @Deprecated public static Region huanan() { return region2(); } /** * 北美机房相关域名 + * 废弃，使用 {@link Region#createWithRegionId} 替换，regionId：na0 * * @return 区域信息 */ + @Deprecated public static Region regionNa0() { return new Builder(). region("na0"). @@ -232,21 +250,25 @@ public static Region regionNa0() { /** * 北美机房相关域名 + * 废弃，使用 {@link Region#createWithRegionId} 替换，regionId：na0 * * @return 区域信息 */ + @Deprecated public static Region beimei() { return regionNa0(); } /** * 新加坡相关域名 + * 废弃，使用 {@link Region#createWithRegionId} 替换，regionId：as0 * * @return 区域信息 */ + @Deprecated public static Region regionAs0() { return new Builder(). - region("na0"). + region("as0"). srcUpHost("up-as0.qiniup.com"). accUpHost("upload-as0.qiniup.com"). iovipHost("iovip-as0.qbox.me"). @@ -256,6 +278,25 @@ public static Region regionAs0() { build(); } + /** + * Region 构造方法 + * regionId 参考链接： + * + * @param regionId 根据区域 ID 创建 Region + * @return Region 实例 + */ + public static Region createWithRegionId(String regionId) { + return new Builder() + .region(regionId) + .srcUpHost("up-" + regionId + ".qiniup.com") + .accUpHost("upload-" + regionId + ".qiniup.com") + .iovipHost("iovip-" + regionId + ".qiniuio.com") + .rsHost("rs-" + regionId + ".qiniuapi.com") + .rsfHost("rsf-" + regionId + ".qiniuapi.com") + .apiHost("api-" + regionId + ".qiniuapi.com") + .build(); + } + /** * 新加坡机房相关域名 * @@ -434,10 +475,10 @@ public Region autoRegion(String... ucServers) { /** * 自动选择,其它参数设置无效 * - * @param retryMax 单个域名最大重试次数 - * @param retryInterval 重试间隔，单位：毫秒 + * @param retryMax 单个域名最大重试次数 + * @param retryInterval 重试间隔，单位：毫秒 * @param hostFreezeDuration 冻结时间，单位：毫秒；域名请求失败会被冻结，冻结后域名在冻结时间内不会被使用 - * @param ucServers uc host + * @param ucServers uc host * @return 区域信息 **/ public Region autoRegion(int retryMax, int retryInterval, int hostFreezeDuration, String... ucServers) { diff --git a/src/test/java/test/com/qiniu/TestConfig.java b/src/test/java/test/com/qiniu/TestConfig.java index e7162328b..788268613 100644 --- a/src/test/java/test/com/qiniu/TestConfig.java +++ b/src/test/java/test/com/qiniu/TestConfig.java @@ -98,7 +98,7 @@ public static TestFile[] getTestFileArray(String fileSaveKey, String fileMimeTyp na0.testDomainTimeStamp = testDomain_na0_timeStamp; na0.testUrlTimeStamp = "http://" + testDomain_na0_timeStamp + "/" + fileSaveKey; na0.regionId = "na0"; - na0.region = Region.regionNa0(); + na0.region = Region.createWithRegionId("na0"); TestFile z0 = new TestFile(); z0.key = fileSaveKey; @@ -109,7 +109,7 @@ public static TestFile[] getTestFileArray(String fileSaveKey, String fileMimeTyp z0.testDomainTimeStamp = testDomain_z0_timeStamp; z0.testUrlTimeStamp = "http://" + testDomain_z0_timeStamp + "/" + fileSaveKey; z0.regionId = "z0"; - z0.region = Region.region0(); + z0.region = Region.createWithRegionId("z0"); TestFile z0_auto = new TestFile(); z0_auto.key = fileSaveKey; diff --git a/src/test/java/test/com/qiniu/storage/RegionTest.java b/src/test/java/test/com/qiniu/storage/RegionTest.java new file mode 100644 index 000000000..9a63b3ef4 --- /dev/null +++ b/src/test/java/test/com/qiniu/storage/RegionTest.java @@ -0,0 +1,70 @@ +package test.com.qiniu.storage; + +import com.qiniu.storage.Region; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Constructor; +import java.lang.reflect.Method; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class RegionTest { + + @Test + @Tag("UnitTest") + public void testCreateWithRegionId() throws Exception { + Class regionReqInfoClass = Class.forName("com.qiniu.storage.RegionReqInfo"); + Constructor regionReqInfoConstructor = regionReqInfoClass.getDeclaredConstructor(String.class, String.class); + regionReqInfoConstructor.setAccessible(true); + + Object info = regionReqInfoConstructor.newInstance("a", "b"); + Region na0 = Region.createWithRegionId("na0"); + + // region id + Method getRegion = Region.class.getDeclaredMethod("getRegion", regionReqInfoClass); + getRegion.setAccessible(true); + String regionId = (String) getRegion.invoke(na0, info); + assertEquals("na0", regionId); + + // up + // up src + Method getSrcUpHost = Region.class.getDeclaredMethod("getSrcUpHost", regionReqInfoClass); + getSrcUpHost.setAccessible(true); + List srcHosts = (List) getSrcUpHost.invoke(na0, info); + assertTrue(srcHosts.contains("up-na0.qiniup.com"), "check up src error"); + + // up acc + Method getAccUpHost = Region.class.getDeclaredMethod("getAccUpHost", regionReqInfoClass); + getAccUpHost.setAccessible(true); + List accHosts = (List) getAccUpHost.invoke(na0, info); + assertTrue(accHosts.contains("upload-na0.qiniup.com"), "check up acc error"); + + // io + Method getIoHost = Region.class.getDeclaredMethod("getIovipHost", regionReqInfoClass); + getIoHost.setAccessible(true); + String ioHost = (String) getIoHost.invoke(na0, info); + assertTrue(ioHost.equals("iovip-na0.qiniuio.com"), "check io error"); + + // rs + Method getRsHost = Region.class.getDeclaredMethod("getRsHost", regionReqInfoClass); + getRsHost.setAccessible(true); + String rsHost = (String) getRsHost.invoke(na0, info); + assertTrue(rsHost.equals("rs-na0.qiniuapi.com"), "check rs error"); + + // rsf + Method getRsfHost = Region.class.getDeclaredMethod("getRsfHost", regionReqInfoClass); + getRsfHost.setAccessible(true); + String rsfHost = (String) getRsfHost.invoke(na0, info); + assertTrue(rsfHost.equals("rsf-na0.qiniuapi.com"), "check rsf error"); + + // api + Method getApiHost = Region.class.getDeclaredMethod("getApiHost", regionReqInfoClass); + getApiHost.setAccessible(true); + String apiHost = (String) getApiHost.invoke(na0, info); + assertTrue(apiHost.equals("api-na0.qiniuapi.com"), "check api error"); + } + +} From ba3073bf1a93ec101e83b1227cd9d20aca5120a4 Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Fri, 27 Oct 2023 15:20:47 +0800 Subject: [PATCH 03/28] test case: exception don't print --- src/main/java/com/qiniu/http/Response.java | 2 +- src/main/java/com/qiniu/storage/BucketManager.java | 1 + src/test/java/test/com/qiniu/sms/SmsTest.java | 1 - .../java/test/com/qiniu/storage/ApiUploadV2Test.java | 5 ++--- src/test/java/test/com/qiniu/storage/BucketTest.java | 2 +- src/test/java/test/com/qiniu/storage/BucketTest2.java | 11 ++++------- .../test/com/qiniu/storage/FixBlockUploaderTest.java | 3 +-- .../java/test/com/qiniu/storage/FormUploadTest.java | 2 -- .../java/test/com/qiniu/storage/RecordUploadTest.java | 3 +-- 9 files changed, 11 insertions(+), 19 deletions(-) diff --git a/src/main/java/com/qiniu/http/Response.java b/src/main/java/com/qiniu/http/Response.java index 9461f9fec..00d2da9e5 100644 --- a/src/main/java/com/qiniu/http/Response.java +++ b/src/main/java/com/qiniu/http/Response.java @@ -248,7 +248,7 @@ public synchronized InputStream bodyStream() throws QiniuException { } public synchronized void close() { - if (this.response != null) { + if (this.response != null && this.response.body() != null) { this.response.close(); } } diff --git a/src/main/java/com/qiniu/storage/BucketManager.java b/src/main/java/com/qiniu/storage/BucketManager.java index d3100be0d..91eeba1fc 100644 --- a/src/main/java/com/qiniu/storage/BucketManager.java +++ b/src/main/java/com/qiniu/storage/BucketManager.java @@ -238,6 +238,7 @@ public FileListing listFiles(String bucket, String prefix, String marker, int li throws QiniuException { Response response = listV1(bucket, prefix, marker, limit, delimiter); if (!response.isOK()) { + response.close(); throw new QiniuException(response); } FileListing fileListing = response.jsonToObject(FileListing.class); diff --git a/src/test/java/test/com/qiniu/sms/SmsTest.java b/src/test/java/test/com/qiniu/sms/SmsTest.java index 9b8616be8..a41054c1e 100644 --- a/src/test/java/test/com/qiniu/sms/SmsTest.java +++ b/src/test/java/test/com/qiniu/sms/SmsTest.java @@ -102,7 +102,6 @@ public void sendFulltextMessage() { Response response = smsManager.sendFulltextMessage(mobiles, "【七牛云】尊敬的用户你好，您的验证码是 38232"); assertNotNull(response); } catch (QiniuException e) { - e.printStackTrace(); assertTrue(ResCode.find(e.code(), ResCode.getPossibleResCode(401))); } } diff --git a/src/test/java/test/com/qiniu/storage/ApiUploadV2Test.java b/src/test/java/test/com/qiniu/storage/ApiUploadV2Test.java index e315171b7..dc336084b 100644 --- a/src/test/java/test/com/qiniu/storage/ApiUploadV2Test.java +++ b/src/test/java/test/com/qiniu/storage/ApiUploadV2Test.java @@ -323,8 +323,7 @@ public void testAbortUpload() { assertTrue(initUploadResponse.isOK(), initUploadResponse.getResponse() + ""); assertNotNull(initUploadResponse.getUploadId(), initUploadResponse.getUploadId() + ""); assertNotNull(initUploadResponse.getExpireAt(), initUploadResponse.getExpireAt() + ""); - } catch (QiniuException e) { - e.printStackTrace(); + } catch (QiniuException ignore) { } // 2. 上传文件数据 @@ -393,7 +392,7 @@ public void testAbortUpload() { // 列举结束 break; } catch (QiniuException e) { - e.printStackTrace(); +// e.printStackTrace(); assertTrue(e.response.statusCode == 612, e.response + ""); break; } diff --git a/src/test/java/test/com/qiniu/storage/BucketTest.java b/src/test/java/test/com/qiniu/storage/BucketTest.java index 5f58384e5..f6d50ad10 100644 --- a/src/test/java/test/com/qiniu/storage/BucketTest.java +++ b/src/test/java/test/com/qiniu/storage/BucketTest.java @@ -220,7 +220,7 @@ public void testListIterator() throws Exception { testFileWithHandler(new TestFileHandler() { @Override public void testFile(TestConfig.TestFile file, BucketManager bucketManager) throws IOException { - BucketManager.FileListIterator it = bucketManager.createFileListIterator(file.getBucketName(), "", 20, + BucketManager.FileListIterator it = bucketManager.createFileListIterator(file.getBucketName(), "", 500, null); assertTrue(it.hasNext()); FileInfo[] items0 = it.next(); diff --git a/src/test/java/test/com/qiniu/storage/BucketTest2.java b/src/test/java/test/com/qiniu/storage/BucketTest2.java index bc01a89a4..bfa247cac 100644 --- a/src/test/java/test/com/qiniu/storage/BucketTest2.java +++ b/src/test/java/test/com/qiniu/storage/BucketTest2.java @@ -139,7 +139,7 @@ public void testListUseDelimiter() { public void testListIterator() { String[] buckets = new String[]{TestConfig.testBucket_z0}; for (String bucket : buckets) { - BucketManager.FileListIterator it = bucketManager.createFileListIterator(bucket, "", 300, null); + BucketManager.FileListIterator it = bucketManager.createFileListIterator(bucket, "", 500, null); assertTrue(it.hasNext()); FileInfo[] items0 = it.next(); @@ -230,8 +230,7 @@ public void testStat() { try { bucketManager.deleteBucketLifecycleRule(TestConfig.testBucket_z0, ruleName); - } catch (QiniuException e) { - e.printStackTrace(); + } catch (QiniuException ignore) { } try { @@ -265,8 +264,7 @@ public void testStat() { } finally { try { bucketManager.deleteBucketLifecycleRule(TestConfig.testBucket_z0, ruleName); - } catch (QiniuException e) { - e.printStackTrace(); + } catch (QiniuException ignore) { } } @@ -1470,8 +1468,7 @@ public void testIndexPage() throws QiniuException { try { bucketManager.setIndexPage(TestConfig.dummyBucket, IndexPageType.HAS); - } catch (QiniuException e) { - e.printStackTrace(); + } catch (QiniuException ignore) { } } diff --git a/src/test/java/test/com/qiniu/storage/FixBlockUploaderTest.java b/src/test/java/test/com/qiniu/storage/FixBlockUploaderTest.java index f3e7b2372..b02e4f1a9 100644 --- a/src/test/java/test/com/qiniu/storage/FixBlockUploaderTest.java +++ b/src/test/java/test/com/qiniu/storage/FixBlockUploaderTest.java @@ -232,8 +232,7 @@ public void testKey2() throws IOException { try { Response res = up.upload(f, token, null); fail("key not match, should failed"); - } catch (QiniuException e) { - e.printStackTrace(); + } catch (QiniuException ignore) { // TODO } } diff --git a/src/test/java/test/com/qiniu/storage/FormUploadTest.java b/src/test/java/test/com/qiniu/storage/FormUploadTest.java index 1948c839a..35d0756f1 100644 --- a/src/test/java/test/com/qiniu/storage/FormUploadTest.java +++ b/src/test/java/test/com/qiniu/storage/FormUploadTest.java @@ -146,8 +146,6 @@ public void testInvalidToken() { if (e.code() != -1) { assertEquals(401, e.code()); assertNotNull(e.response.reqId); - } else { - e.printStackTrace(); } } } diff --git a/src/test/java/test/com/qiniu/storage/RecordUploadTest.java b/src/test/java/test/com/qiniu/storage/RecordUploadTest.java index 6c0be4f9b..0cd9314ee 100644 --- a/src/test/java/test/com/qiniu/storage/RecordUploadTest.java +++ b/src/test/java/test/com/qiniu/storage/RecordUploadTest.java @@ -87,9 +87,8 @@ public void run() { System.out.println("UP: " + i + ", enter run"); response = up.up(region); System.out.println("UP: " + i + ", left run"); - } catch (Exception e) { + } catch (Exception ignore) { System.out.println("UP: " + i + ", exception run"); - e.printStackTrace(); } complete.isComplete.set(true); } From 915e96c3f6b70e27058f88c81c8a61db3ffbec05 Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Fri, 27 Oct 2023 18:29:29 +0800 Subject: [PATCH 04/28] change test case --- src/test/java/test/com/qiniu/storage/BucketTest.java | 1 - src/test/java/test/com/qiniu/storage/BucketTest2.java | 1 - src/test/java/test/com/qiniu/storage/DownloadUrlTest.java | 3 +++ 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/test/java/test/com/qiniu/storage/BucketTest.java b/src/test/java/test/com/qiniu/storage/BucketTest.java index f6d50ad10..5dabf6170 100644 --- a/src/test/java/test/com/qiniu/storage/BucketTest.java +++ b/src/test/java/test/com/qiniu/storage/BucketTest.java @@ -525,7 +525,6 @@ public void testFile(TestConfig.TestFile file, BucketManager bucketManager) thro fRet = bucketManager.fetch(resUrl, file.getBucketName()); assertEquals(resHash, fRet.hash); } catch (QiniuException e) { - e.printStackTrace(); // use e.response.toString() may get NullPointException // when java.net.SocketTimeoutException: timeout fail(e.getMessage()); diff --git a/src/test/java/test/com/qiniu/storage/BucketTest2.java b/src/test/java/test/com/qiniu/storage/BucketTest2.java index bfa247cac..a1498b16a 100644 --- a/src/test/java/test/com/qiniu/storage/BucketTest2.java +++ b/src/test/java/test/com/qiniu/storage/BucketTest2.java @@ -499,7 +499,6 @@ public void testFetch() { fRet = bucketManager.fetch(resUrl, bucket); assertEquals(resHash, fRet.hash); } catch (QiniuException e) { - e.printStackTrace(); // use e.response.toString() may get NullPointException // when java.net.SocketTimeoutException: timeout fail(e.getMessage()); diff --git a/src/test/java/test/com/qiniu/storage/DownloadUrlTest.java b/src/test/java/test/com/qiniu/storage/DownloadUrlTest.java index 567f7e7a4..5afac57e9 100644 --- a/src/test/java/test/com/qiniu/storage/DownloadUrlTest.java +++ b/src/test/java/test/com/qiniu/storage/DownloadUrlTest.java @@ -100,6 +100,7 @@ public void testUrlWithDeadline() { System.out.println("create url:" + url); Client client = new Client(); Response response = client.get(url); + response.close(); assertTrue(response.isOK(), response.toString()); try { @@ -120,6 +121,7 @@ private void testNoAuthority(String url) { try { Client client = new Client(); Response response = client.get(url); + response.close(); assertFalse(response.isOK(), url); } catch (QiniuException e) { assertNotNull(e.response, "except no authority:" + url + "\n but no response:" + e); @@ -131,6 +133,7 @@ private void testHasAuthority(String url) { try { Client client = new Client(); Response response = client.get(url); + response.close(); assertTrue(response.isOK(), url); } catch (QiniuException e) { fail("except has authority:" + url + "\n response:" + e.response); From f4a48e2dec805a7f14d8fd73e9f2c4bb436b5182 Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Mon, 30 Oct 2023 10:08:37 +0800 Subject: [PATCH 05/28] change test case --- src/test/java/test/com/qiniu/CdnTest.java | 4 +++- src/test/java/test/com/qiniu/streaming/StreamingTest.java | 3 +-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/test/java/test/com/qiniu/CdnTest.java b/src/test/java/test/com/qiniu/CdnTest.java index 92852bcbb..4007965d2 100644 --- a/src/test/java/test/com/qiniu/CdnTest.java +++ b/src/test/java/test/com/qiniu/CdnTest.java @@ -176,7 +176,9 @@ public void testCreateTimestampAntiLeechUrlSimple() { String testUrl_z0_timeStamp_outdate = "http://javasdk-timestamp.peterpy.cn/do_not_delete/1.png?sign=14f48f829b78d5c9a34eb77e9a13f1b6&t=5890f13a"; try { URL url = new URL(TestConfig.testUrl_z0_timeStamp); - assertEquals(403, getResponse(url.toString()).statusCode, msg); + Response response = getResponse(url.toString()); + response.close(); + assertEquals(403, response.statusCode, msg); String signedUrl1 = CdnManager.createTimestampAntiLeechUrl(host, fileName, queryStringMap, encryptKey1, deadline1); String signedUrl2 = CdnManager.createTimestampAntiLeechUrl(url, encryptKey2, deadline2); diff --git a/src/test/java/test/com/qiniu/streaming/StreamingTest.java b/src/test/java/test/com/qiniu/streaming/StreamingTest.java index 9b515c8af..4aae60eb0 100644 --- a/src/test/java/test/com/qiniu/streaming/StreamingTest.java +++ b/src/test/java/test/com/qiniu/streaming/StreamingTest.java @@ -56,8 +56,7 @@ public void testStreamOperation() throws QiniuException { try { // 确保流存在 // manager.create(stream); - } catch (Exception e) { - e.printStackTrace(); + } catch (Exception ignore) { } StreamAttribute attr = manager.attribute(stream); assertEquals(0, attr.disabledTill); From e95a91fca02285f8db2d61a79c5f532e88ecd387 Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Mon, 30 Oct 2023 11:04:55 +0800 Subject: [PATCH 06/28] change test case --- src/test/java/test/com/qiniu/CdnTest.java | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/test/java/test/com/qiniu/CdnTest.java b/src/test/java/test/com/qiniu/CdnTest.java index 4007965d2..b5dbc6f14 100644 --- a/src/test/java/test/com/qiniu/CdnTest.java +++ b/src/test/java/test/com/qiniu/CdnTest.java @@ -186,9 +186,16 @@ public void testCreateTimestampAntiLeechUrlSimple() { System.out.println(signedUrl1); System.out.println(signedUrl2); System.out.println(signedUrl3); - assertEquals(200, getResponse(signedUrl1).statusCode, msg); - assertEquals(200, getResponse(signedUrl2).statusCode, msg); - assertEquals(403, getResponse(signedUrl3).statusCode, msg); + + response = getResponse(signedUrl1); + response.close(); + assertEquals(200, response.statusCode, msg); + response = getResponse(signedUrl2); + response.close(); + assertEquals(200, response.statusCode, msg); + response = getResponse(signedUrl3); + response.close(); + assertEquals(403, response.statusCode, msg); assertEquals(testUrl_z0_timeStamp_outdate, signedUrl3); } catch (Exception ex) { ex.printStackTrace(); From 18ba83ba7b110300baa7ce0d3d1e8fdbe7650074 Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Thu, 2 Nov 2023 14:29:09 +0800 Subject: [PATCH 07/28] uc query cache support disk --- src/main/java/com/qiniu/common/Constants.java | 22 +++ .../java/com/qiniu/storage/AutoRegion.java | 31 ++- src/main/java/com/qiniu/util/Cache.java | 179 ++++++++++++++++++ .../com/qiniu/storage/FormUploadTest.java | 2 +- .../java/test/com/qiniu/util/CacheTest.java | 50 +++++ 5 files changed, 276 insertions(+), 8 deletions(-) create mode 100644 src/main/java/com/qiniu/util/Cache.java create mode 100644 src/test/java/test/com/qiniu/util/CacheTest.java diff --git a/src/main/java/com/qiniu/common/Constants.java b/src/main/java/com/qiniu/common/Constants.java index 83447124f..e97887afa 100644 --- a/src/main/java/com/qiniu/common/Constants.java +++ b/src/main/java/com/qiniu/common/Constants.java @@ -1,5 +1,6 @@ package com.qiniu.common; +import java.io.File; import java.nio.charset.Charset; /** @@ -47,7 +48,28 @@ public final class Constants { */ public static final int CONNECTION_POOL_MAX_IDLE_MINUTES = 5; + public static final String CACHE_DIR = getCacheDir(); + private Constants() { } + + private static String getCacheDir() { + String tmpDir = System.getProperty("java.io.tmpdir"); + if (tmpDir == null || tmpDir.isEmpty()) { + return null; + } + + String qiniuDir = tmpDir + "com.qiniu.java-sdk"; + File dir = new File(qiniuDir); + if (!dir.exists()) { + return dir.mkdirs() ? qiniuDir : null; + } + + if (dir.isDirectory()) { + return qiniuDir; + } + + return null; + } } diff --git a/src/main/java/com/qiniu/storage/AutoRegion.java b/src/main/java/com/qiniu/storage/AutoRegion.java index e458215ef..bf3d16df9 100644 --- a/src/main/java/com/qiniu/storage/AutoRegion.java +++ b/src/main/java/com/qiniu/storage/AutoRegion.java @@ -3,8 +3,7 @@ import com.qiniu.common.QiniuException; import com.qiniu.http.Client; import com.qiniu.http.Response; -import com.qiniu.util.StringUtils; -import com.qiniu.util.UrlUtils; +import com.qiniu.util.*; import java.util.ArrayList; import java.util.Arrays; @@ -27,7 +26,9 @@ class AutoRegion extends Region { /** * 全局空间信息缓存，此缓存绑定了 token、bucket，全局有效。 */ - private static Map globalRegionCache = new ConcurrentHashMap<>(); + private static final Cache globalRegionCache = new Cache.Builder<>(UCRet.class) + .setVersion("v1") + .builder(); /** * 定义HTTP请求管理相关方法 @@ -75,8 +76,8 @@ private AutoRegion() { * 通过 API 接口查询上传域名 */ private UCRet queryRegionInfoFromServerIfNeeded(RegionIndex index) throws QiniuException { - String cacheKey = index.accessKey + index.bucket; - UCRet ret = globalRegionCache.get(cacheKey); + String cacheKey = getCacheId(index); + UCRet ret = globalRegionCache.cacheForKey(cacheKey); if (ret != null && ret.isValid()) { return ret; } @@ -94,7 +95,7 @@ private UCRet queryRegionInfoFromServerIfNeeded(RegionIndex index) throws QiniuE ret = r.jsonToObject(UCRet.class); if (ret != null) { ret.setupDeadline(); - globalRegionCache.put(cacheKey, ret); + globalRegionCache.cache(cacheKey, ret); } return ret; } @@ -122,7 +123,7 @@ static Region regionGroup(UCRet ret) { */ private Region queryRegionInfo(String accessKey, String bucket) throws QiniuException { RegionIndex index = new RegionIndex(accessKey, bucket); - String cacheKey = index.accessKey + "::" + index.bucket; + String cacheKey = getCacheId(index); Region region = regions.get(cacheKey); Exception ex = null; @@ -311,6 +312,22 @@ String[] getUcHostArray() throws QiniuException { return getUcHosts(null).toArray(new String[0]); } + private String getCacheId(RegionIndex index) { + StringBuilder builder = new StringBuilder() + .append(index.accessKey) + .append("-") + .append(index.bucket); + + if (ucServers != null && !ucServers.isEmpty()) { + for (String host : ucServers) { + if (host != null && !host.isEmpty()) { + builder.append(host); + } + } + } + + return UrlSafeBase64.encodeToString(builder.toString()); + } public Object clone() { AutoRegion newRegion = new AutoRegion(); diff --git a/src/main/java/com/qiniu/util/Cache.java b/src/main/java/com/qiniu/util/Cache.java new file mode 100644 index 000000000..0cc297487 --- /dev/null +++ b/src/main/java/com/qiniu/util/Cache.java @@ -0,0 +1,179 @@ +package com.qiniu.util; + +import com.qiniu.common.Constants; +import com.qiniu.storage.persistent.FileRecorder; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + + +/** + * 包含内存缓存和磁盘缓存 + * 磁盘缓存会被缓存在一个文件中 + */ +public class Cache { + + // 当 cache 修改数量到达这个值时，就会 flush，默认是 1 + private final int flushCount; + + // 缓存被持久化为一个文件，此文件的文件名为 version，version 默认为：v1 + private final String version; + + // 存储对象的类型 + private final Class objectClass; + + // 内部 + private boolean isFlushing = false; + private int needFlushCount = 0; + + private final ConcurrentHashMap memCache = new ConcurrentHashMap<>(); + private final FileRecorder diskCache; + + private Cache(Class objectClass, String cacheDir, int flushCount, String version) { + this.objectClass = objectClass; + this.flushCount = flushCount; + this.version = version; + + FileRecorder fileRecorder = null; + try { + if (objectClass != null && cacheDir != null && !cacheDir.isEmpty()) { + fileRecorder = new FileRecorder(cacheDir + "/" + objectClass.getName()); + } + } catch (Exception e) { + e.printStackTrace(); + } + this.diskCache = fileRecorder; + + this.load(); + } + + private void load() { + if (this.diskCache == null || objectClass == null) { + return; + } + + byte[] cacheData = this.diskCache.get(this.version); + if (cacheData == null || cacheData.length == 0) { + return; + } + + try { + HashMap cacheJson = Json.decode(new String(cacheData), HashMap.class); + for (String key : cacheJson.keySet()) { + try { + Object jsonMap = cacheJson.get(key); + String jsonString = Json.encode(jsonMap); + T object = Json.decode(jsonString, this.objectClass); + this.memCache.put(key, object); + } catch (Exception e) { + e.printStackTrace(); + break; + } + } + } catch (Exception e) { + e.printStackTrace(); + this.diskCache.del(this.version); + } + } + + public T cacheForKey(String cacheKey) { + return this.memCache.get(cacheKey); + } + + public void cache(String cacheKey, T object) { + if (StringUtils.isNullOrEmpty(cacheKey) || object == null) { + return; + } + + synchronized (this) { + this.needFlushCount++; + this.memCache.put(cacheKey, object); + + if (this.needFlushCount < this.flushCount) { + return; + } + + this.needFlushCount = 0; + } + + this.flush(); + } + + private void flush() { + if (this.diskCache == null) { + return; + } + + Map flushCache = null; + synchronized (this) { + if (this.isFlushing) { + return; + } + + this.isFlushing = true; + flushCache = new HashMap<>(this.memCache); + } + + if (flushCache.isEmpty()) { + return; + } + + String jsonString = Json.encode(flushCache); + if (jsonString == null || jsonString.isEmpty()) { + return; + } + + byte[] cacheData = jsonString.getBytes(); + if (cacheData.length == 0) { + return; + } + + this.diskCache.set(this.version, cacheData); + + synchronized (this) { + isFlushing = false; + } + } + + public void clearMemoryCache() { + this.memCache.clear(); + } + + public static class Builder { + // 当 cache 修改数量到达这个值时，就会 flush，默认是 1 + private int flushCount = 1; + + // 缓存被持久化为一个文件，此文件的文件名为 version，version 默认为：v1 + private String version = "v1"; + + // 存储路径 + private String cacheDir = Constants.CACHE_DIR; + + // 存储对象的类型 + private final Class objectClass; + + public Builder(Class objectClass) { + this.objectClass = objectClass; + } + + public Builder setFlushCount(int flushCount) { + this.flushCount = flushCount; + return this; + } + + public Builder setCacheDir(String cacheDir) { + this.cacheDir = cacheDir; + return this; + } + + public Builder setVersion(String version) { + this.version = version; + return this; + } + + public Cache builder() { + return new Cache<>(this.objectClass, cacheDir, this.flushCount, this.version); + } + } +} diff --git a/src/test/java/test/com/qiniu/storage/FormUploadTest.java b/src/test/java/test/com/qiniu/storage/FormUploadTest.java index 35d0756f1..62b14f310 100644 --- a/src/test/java/test/com/qiniu/storage/FormUploadTest.java +++ b/src/test/java/test/com/qiniu/storage/FormUploadTest.java @@ -60,7 +60,7 @@ public void testSyncRetry() { public void testHello2() { TestConfig.TestFile[] files = TestConfig.getTestFileArray(); for (TestConfig.TestFile file : files) { - Configuration config = new Configuration(file.getRegion()); + Configuration config = new Configuration(); config.useHttpsDomains = true; UploadManager uploadManager = new UploadManager(config); hello(uploadManager, file.getBucketName()); diff --git a/src/test/java/test/com/qiniu/util/CacheTest.java b/src/test/java/test/com/qiniu/util/CacheTest.java new file mode 100644 index 000000000..fa3c10945 --- /dev/null +++ b/src/test/java/test/com/qiniu/util/CacheTest.java @@ -0,0 +1,50 @@ +package test.com.qiniu.util; + +import com.qiniu.util.Cache; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class CacheTest { + + @Test + public void testCache() { + Info info = new Info(); + info.foo = "foo"; + info.bar = 1; + + String key = "info_key"; + Cache cache = new Cache.Builder(Info.class) + .setVersion("v1") + .setFlushCount(1) + .builder(); + + cache.cache(key, info); + + + // 1. 测试内存缓存 + Info memInfo = (Info) cache.cacheForKey(key); + assertEquals("foo", memInfo.foo); + + // 2. 测试删除内存缓存 + cache.clearMemoryCache(); + memInfo = (Info) cache.cacheForKey(key); + assertEquals(null, memInfo); + + // 3. 测试 load + cache = new Cache.Builder(Info.class) + .setVersion("v1") + .setFlushCount(1) + .builder(); + memInfo = (Info) cache.cacheForKey(key); + assertEquals("foo", memInfo.foo); + } + + static class Info { + String foo; + int bar; + + Info(){ + } + } +} From a73fb1768a58c45a0f737eaeaaa2aa233c07b273 Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Thu, 2 Nov 2023 14:37:30 +0800 Subject: [PATCH 08/28] change test case --- src/test/java/test/com/qiniu/util/CacheTest.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/test/java/test/com/qiniu/util/CacheTest.java b/src/test/java/test/com/qiniu/util/CacheTest.java index fa3c10945..eca8dfa45 100644 --- a/src/test/java/test/com/qiniu/util/CacheTest.java +++ b/src/test/java/test/com/qiniu/util/CacheTest.java @@ -43,8 +43,5 @@ public void testCache() { static class Info { String foo; int bar; - - Info(){ - } } } From 34fbd2c1c2ab8041d51630714691b9ecfb3bebda Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Fri, 3 Nov 2023 10:53:45 +0800 Subject: [PATCH 09/28] region create by id handle z0 --- src/main/java/com/qiniu/storage/Region.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/main/java/com/qiniu/storage/Region.java b/src/main/java/com/qiniu/storage/Region.java index 35fcb3f86..8765c868c 100644 --- a/src/main/java/com/qiniu/storage/Region.java +++ b/src/main/java/com/qiniu/storage/Region.java @@ -286,6 +286,14 @@ public static Region regionAs0() { * @return Region 实例 */ public static Region createWithRegionId(String regionId) { + if (regionId == null || regionId.isEmpty()) { + return null; + } + + if (regionId.equals("z0")) { + return Region.region0(); + } + return new Builder() .region(regionId) .srcUpHost("up-" + regionId + ".qiniup.com") From a1981208b70bcfafae59848d4381598d708a5557 Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Mon, 6 Nov 2023 16:14:39 +0800 Subject: [PATCH 10/28] optimize api content-type --- src/main/java/com/qiniu/storage/Api.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/qiniu/storage/Api.java b/src/main/java/com/qiniu/storage/Api.java index b21608395..b43efaf66 100644 --- a/src/main/java/com/qiniu/storage/Api.java +++ b/src/main/java/com/qiniu/storage/Api.java @@ -612,13 +612,12 @@ public StringMap getHeader() throws QiniuException { header.put(key, this.header.get(key)); } - if (header.keySet().contains("Content-Type")) { + if (body == null || body.contentType == null + || header.keySet().contains("Content-Type")) { return header; } - if (body != null) { - header.put("Content-Type", body.contentType.toString()); - } + header.put("Content-Type", body.contentType.toString()); return header; } From e56e19445774d32b7fafa7295fb02cd8d4c46b05 Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Mon, 6 Nov 2023 16:52:25 +0800 Subject: [PATCH 11/28] optimize test case --- src/test/java/test/com/qiniu/storage/BucketTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/test/com/qiniu/storage/BucketTest.java b/src/test/java/test/com/qiniu/storage/BucketTest.java index ec29f5e96..bccc64288 100644 --- a/src/test/java/test/com/qiniu/storage/BucketTest.java +++ b/src/test/java/test/com/qiniu/storage/BucketTest.java @@ -1757,7 +1757,7 @@ public void testFile(TestConfig.TestFile file, BucketManager bucketManager) thro } long current = new Date().getTime(); - if (current - checkStart > 1000 * 60 * 5.5) { + if (current - checkStart > 1000 * 120 * 5.5) { shouldCheck = false; } From c6c65853fbf50c925ea1dd88416b61682b4097f4 Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Tue, 7 Nov 2023 16:12:54 +0800 Subject: [PATCH 12/28] cache remove flushcount --- src/main/java/com/qiniu/util/Cache.java | 23 ++--------------------- 1 file changed, 2 insertions(+), 21 deletions(-) diff --git a/src/main/java/com/qiniu/util/Cache.java b/src/main/java/com/qiniu/util/Cache.java index 0cc297487..bd76e5adc 100644 --- a/src/main/java/com/qiniu/util/Cache.java +++ b/src/main/java/com/qiniu/util/Cache.java @@ -14,9 +14,6 @@ */ public class Cache { - // 当 cache 修改数量到达这个值时，就会 flush，默认是 1 - private final int flushCount; - // 缓存被持久化为一个文件，此文件的文件名为 version，version 默认为：v1 private final String version; @@ -25,14 +22,12 @@ public class Cache { // 内部 private boolean isFlushing = false; - private int needFlushCount = 0; private final ConcurrentHashMap memCache = new ConcurrentHashMap<>(); private final FileRecorder diskCache; - private Cache(Class objectClass, String cacheDir, int flushCount, String version) { + private Cache(Class objectClass, String cacheDir, String version) { this.objectClass = objectClass; - this.flushCount = flushCount; this.version = version; FileRecorder fileRecorder = null; @@ -87,14 +82,7 @@ public void cache(String cacheKey, T object) { } synchronized (this) { - this.needFlushCount++; this.memCache.put(cacheKey, object); - - if (this.needFlushCount < this.flushCount) { - return; - } - - this.needFlushCount = 0; } this.flush(); @@ -141,8 +129,6 @@ public void clearMemoryCache() { } public static class Builder { - // 当 cache 修改数量到达这个值时，就会 flush，默认是 1 - private int flushCount = 1; // 缓存被持久化为一个文件，此文件的文件名为 version，version 默认为：v1 private String version = "v1"; @@ -157,11 +143,6 @@ public Builder(Class objectClass) { this.objectClass = objectClass; } - public Builder setFlushCount(int flushCount) { - this.flushCount = flushCount; - return this; - } - public Builder setCacheDir(String cacheDir) { this.cacheDir = cacheDir; return this; @@ -173,7 +154,7 @@ public Builder setVersion(String version) { } public Cache builder() { - return new Cache<>(this.objectClass, cacheDir, this.flushCount, this.version); + return new Cache<>(this.objectClass, cacheDir, this.version); } } } From 36d4607138d4b659fcf96a802babb4bf0b044727 Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Tue, 7 Nov 2023 16:25:25 +0800 Subject: [PATCH 13/28] optimize response --- src/main/java/com/qiniu/http/Response.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/main/java/com/qiniu/http/Response.java b/src/main/java/com/qiniu/http/Response.java index 00d2da9e5..e09ee5ed8 100644 --- a/src/main/java/com/qiniu/http/Response.java +++ b/src/main/java/com/qiniu/http/Response.java @@ -105,7 +105,18 @@ public static Response create(okhttp3.Response response, String address, double error = e.getMessage(); } } + } else if (response.body() != null) { + // 处理其他 body 非预期情况 + if (response.code() >= 300) { + try { + body = response.body().bytes(); + error = new String(body, Constants.UTF_8); + } catch (Exception e) { + error = e.getMessage(); + } + } } + return new Response(response, code, reqId, response.header("X-Log"), via(response), address, duration, error, body); } From cca65c5cf8188f92762eb66968b0908247845449 Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Tue, 7 Nov 2023 17:42:59 +0800 Subject: [PATCH 14/28] change cache test case --- src/test/java/test/com/qiniu/util/CacheTest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/test/java/test/com/qiniu/util/CacheTest.java b/src/test/java/test/com/qiniu/util/CacheTest.java index eca8dfa45..9ef3816d5 100644 --- a/src/test/java/test/com/qiniu/util/CacheTest.java +++ b/src/test/java/test/com/qiniu/util/CacheTest.java @@ -16,7 +16,6 @@ public void testCache() { String key = "info_key"; Cache cache = new Cache.Builder(Info.class) .setVersion("v1") - .setFlushCount(1) .builder(); cache.cache(key, info); @@ -34,7 +33,6 @@ public void testCache() { // 3. 测试 load cache = new Cache.Builder(Info.class) .setVersion("v1") - .setFlushCount(1) .builder(); memInfo = (Info) cache.cacheForKey(key); assertEquals("foo", memInfo.foo); From 789cdef3e1e004c0e42695ea7fb3345a3596110d Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Thu, 9 Nov 2023 19:05:55 +0800 Subject: [PATCH 15/28] add condition --- .../java/com/qiniu/storage/BucketManager.java | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/src/main/java/com/qiniu/storage/BucketManager.java b/src/main/java/com/qiniu/storage/BucketManager.java index 91eeba1fc..ec07922be 100644 --- a/src/main/java/com/qiniu/storage/BucketManager.java +++ b/src/main/java/com/qiniu/storage/BucketManager.java @@ -1427,6 +1427,76 @@ public String execBucket() { public int size() { return ops.size(); } + + public final static class Condition { + private final String hash; + private final String mime; + private final Long fSize; + private final Long putTime; + + public Condition(String hash, String mime, Long fSize, Long putTime) { + this.hash = hash; + this.mime = mime; + this.fSize = fSize; + this.putTime = putTime; + } + + private String encodedString() { + List values = new ArrayList<>(); + if (hash != null && !hash.isEmpty()) { + values.add("hash="+hash); + } + if (mime != null && !mime.isEmpty()) { + values.add("mime="+mime); + } + if (fSize != null) { + values.add("fsize="+fSize); + } + if (putTime != null) { + values.add("putTime="+putTime); + } + + if (values.isEmpty()) { + return null; + } + + return UrlSafeBase64.encodeToString(String.join("&", values)); + } + + public final static class Builder { + private String hash; + private String mime; + private Long fileSize; + private Long putTime; + + public Builder() { + } + + public Builder setHash(String hash) { + this.hash = hash; + return this; + } + + public Builder setMime(String mime) { + this.mime = mime; + return this; + } + + public Builder setFileSize(Long fileSize) { + this.fileSize = fileSize; + return this; + } + + public Builder setPutTime(Long putTime) { + this.putTime = putTime; + return this; + } + + public Condition build() { + return new Condition(hash, mime, fileSize, putTime); + } + } + } } /** From 8b13d71b24fb4853a6fd6237c31dfa02b14533bf Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Fri, 10 Nov 2023 14:21:49 +0800 Subject: [PATCH 16/28] batch add some operation --- .../java/com/qiniu/storage/BucketManager.java | 96 +++++++++++++++++-- .../java/com/qiniu/storage/BatchTest.java | 48 ++++++++++ 2 files changed, 135 insertions(+), 9 deletions(-) create mode 100644 src/test/java/com/qiniu/storage/BatchTest.java diff --git a/src/main/java/com/qiniu/storage/BucketManager.java b/src/main/java/com/qiniu/storage/BucketManager.java index ec07922be..11cae941b 100644 --- a/src/main/java/com/qiniu/storage/BucketManager.java +++ b/src/main/java/com/qiniu/storage/BucketManager.java @@ -1258,8 +1258,26 @@ public BatchOperations addChgmOp(String bucket, String key, String newMimeType) return this; } + /** + * 添加chgm指令 + * + * @param bucket 空间名 + * @param key 文件的 key + * @param newMimeType 修改后的 MimeType + * @param condition 自定义条件信息；只有条件匹配才会执行修改操作 + * @return BatchOperations + */ + public BatchOperations addChgmOp(String bucket, String key, String newMimeType, Condition condition) { + String resource = encodedEntry(bucket, key); + String encodedMime = UrlSafeBase64.encodeToString(newMimeType); + ops.add(String.format("/chgm/%s/mime/%s/cond/%s", resource, encodedMime, condition.encodedString())); + setExecBucket(bucket); + return this; + } + /** * 添加 copy 指令 + * 如果目标文件名已被占用，则返回错误码 614，且不做任何覆盖操作； * * @param fromBucket 源空间名 * @param fromFileKey 源文件的 key @@ -1275,6 +1293,26 @@ public BatchOperations addCopyOp(String fromBucket, String fromFileKey, String t return this; } + /** + * 添加 copy 指令 + * + * @param fromBucket 源空间名 + * @param fromFileKey 源文件的 key + * @param toBucket 目标空间名 + * @param toFileKey 目标文件的 key + * @param force 当目标文件已存在时，是否木盖目标文件 + * false: 如果目标文件名已被占用，则返回错误码 614，且不做任何覆盖操作； + * true: 如果目标文件名已被占用，会强制覆盖目标文件 + * @return BatchOperations + */ + public BatchOperations addCopyOp(String fromBucket, String fromFileKey, String toBucket, String toFileKey, boolean force) { + String from = encodedEntry(fromBucket, fromFileKey); + String to = encodedEntry(toBucket, toFileKey); + ops.add(String.format("copy/%s/%s/force/%s", from, to, force)); + setExecBucket(fromBucket); + return this; + } + /** * 添加重命名指令 * @@ -1287,6 +1325,21 @@ public BatchOperations addRenameOp(String fromBucket, String fromFileKey, String return addMoveOp(fromBucket, fromFileKey, fromBucket, toFileKey); } + /** + * 添加重命名指令 + * + * @param fromBucket 源空间名 + * @param fromFileKey 源文件的 key + * @param toFileKey 目标文件的 key + * @param force 当目标文件已存在时，是否木盖目标文件 + * false: 如果目标文件名已被占用，则返回错误码 614，且不做任何覆盖操作； + * true: 如果目标文件名已被占用，会强制覆盖目标文件 + * @return BatchOperations + */ + public BatchOperations addRenameOp(String fromBucket, String fromFileKey, String toFileKey, boolean force) { + return addMoveOp(fromBucket, fromFileKey, fromBucket, toFileKey, force); + } + /** * 添加move指令 * @@ -1304,6 +1357,26 @@ public BatchOperations addMoveOp(String fromBucket, String fromKey, String toBuc return this; } + /** + * 添加move指令 + * + * @param fromBucket 源空间名 + * @param fromKey 源文件的 keys + * @param toBucket 目标空间名 + * @param toKey 目标文件的 keys + * @param force 当目标文件已存在时，是否木盖目标文件 + * false: 如果目标文件名已被占用，则返回错误码 614，且不做任何覆盖操作； + * true: 如果目标文件名已被占用，会强制覆盖目标文件 + * @return BatchOperations + */ + public BatchOperations addMoveOp(String fromBucket, String fromKey, String toBucket, String toKey, boolean force) { + String from = encodedEntry(fromBucket, fromKey); + String to = encodedEntry(toBucket, toKey); + ops.add(String.format("move/%s/%s/force/%s", from, to, force)); + setExecBucket(fromBucket); + return this; + } + /** * 添加delete指令 * @@ -1434,33 +1507,38 @@ public final static class Condition { private final Long fSize; private final Long putTime; - public Condition(String hash, String mime, Long fSize, Long putTime) { + private Condition(String hash, String mime, Long fSize, Long putTime) { this.hash = hash; this.mime = mime; this.fSize = fSize; this.putTime = putTime; } - private String encodedString() { - List values = new ArrayList<>(); + String encodedString() { + StringBuilder builder = new StringBuilder(); if (hash != null && !hash.isEmpty()) { - values.add("hash="+hash); + builder.append("hash=" + hash + "&"); } if (mime != null && !mime.isEmpty()) { - values.add("mime="+mime); + builder.append("mime=" + mime + "&"); } if (fSize != null) { - values.add("fsize="+fSize); + builder.append("fsize=" + fSize + "&"); } if (putTime != null) { - values.add("putTime="+putTime); + builder.append("putTime=" + putTime + "&"); } - if (values.isEmpty()) { + String encoded = builder.toString(); + if (encoded.isEmpty()) { return null; } - return UrlSafeBase64.encodeToString(String.join("&", values)); + if (encoded.endsWith("&")) { + encoded = encoded.substring(0, encoded.length() - 1); + } + + return UrlSafeBase64.encodeToString(encoded); } public final static class Builder { diff --git a/src/test/java/com/qiniu/storage/BatchTest.java b/src/test/java/com/qiniu/storage/BatchTest.java new file mode 100644 index 000000000..e6cc3cc3e --- /dev/null +++ b/src/test/java/com/qiniu/storage/BatchTest.java @@ -0,0 +1,48 @@ +package com.qiniu.storage; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +public class BatchTest { + + @Test + @Tag("UnitTest") + public void testBatchCondition() { + + BucketManager.BatchOperations.Condition condition = new BucketManager.BatchOperations.Condition.Builder() + .build(); + String encodedString = condition.encodedString(); + Assertions.assertNull(encodedString); + + condition = new BucketManager.BatchOperations.Condition.Builder() + .setHash("hash") + .build(); + encodedString = condition.encodedString(); + Assertions.assertEquals(encodedString, "aGFzaD1oYXNo"); + + condition = new BucketManager.BatchOperations.Condition.Builder() + .setHash("hash") + .setPutTime(1993232L) + .build(); + encodedString = condition.encodedString(); + Assertions.assertEquals(encodedString, "aGFzaD1oYXNoJnB1dFRpbWU9MTk5MzIzMg=="); + + condition = new BucketManager.BatchOperations.Condition.Builder() + .setHash("hash") + .setMime("application/txt") + .setPutTime(1993232L) + .build(); + encodedString = condition.encodedString(); + Assertions.assertEquals(encodedString, "aGFzaD1oYXNoJm1pbWU9YXBwbGljYXRpb24vdHh0JnB1dFRpbWU9MTk5MzIzMg=="); + + condition = new BucketManager.BatchOperations.Condition.Builder() + .setHash("hash") + .setMime("application/txt") + .setFileSize(100L) + .setPutTime(1993232L) + .build(); + encodedString = condition.encodedString(); + Assertions.assertEquals(encodedString, "aGFzaD1oYXNoJm1pbWU9YXBwbGljYXRpb24vdHh0JmZzaXplPTEwMCZwdXRUaW1lPTE5OTMyMzI="); + } +} From 1e22059e56f1361b44023652f3720d03ae9d47c5 Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Mon, 13 Nov 2023 17:40:19 +0800 Subject: [PATCH 17/28] batch optimize some code --- .../java/com/qiniu/storage/BucketManager.java | 164 ++++++++++-------- .../java/com/qiniu/storage/BatchTest.java | 10 +- .../test/com/qiniu/storage/BucketTest2.java | 123 +++++++++++-- 3 files changed, 201 insertions(+), 96 deletions(-) diff --git a/src/main/java/com/qiniu/storage/BucketManager.java b/src/main/java/com/qiniu/storage/BucketManager.java index 11cae941b..bed2b5e01 100644 --- a/src/main/java/com/qiniu/storage/BucketManager.java +++ b/src/main/java/com/qiniu/storage/BucketManager.java @@ -1251,26 +1251,46 @@ public BatchOperations() { * @return BatchOperations */ public BatchOperations addChgmOp(String bucket, String key, String newMimeType) { - String resource = encodedEntry(bucket, key); - String encodedMime = UrlSafeBase64.encodeToString(newMimeType); - ops.add(String.format("/chgm/%s/mime/%s", resource, encodedMime)); - setExecBucket(bucket); - return this; + return addChgmOp(bucket, key, newMimeType, null, null); } /** - * 添加chgm指令 + * 添加 chgm 指令 + * 相关链接 + * newMimeType 和 metaData 必须有其一 * * @param bucket 空间名 * @param key 文件的 key - * @param newMimeType 修改后的 MimeType - * @param condition 自定义条件信息；只有条件匹配才会执行修改操作 + * @param newMimeType 修改后的 MimeType [可选] + * @param metas 需要修改的 metas，只包含需要更改的 metas，可增加 [可选] + * 服务接口中 key 必须包含 x-qn-meta- 前缀，SDK 会对 metas 中的 key 进行检测 + * - key 如果包含了 x-qn-meta- 前缀，则直接使用 key + * - key 如果不包含了 x-qn-meta- 前缀，则内部会为 key 拼接 x-qn-meta- 前缀 + * @param condition 自定义条件信息；只有条件匹配才会执行修改操作 [可选] * @return BatchOperations */ - public BatchOperations addChgmOp(String bucket, String key, String newMimeType, Condition condition) { - String resource = encodedEntry(bucket, key); - String encodedMime = UrlSafeBase64.encodeToString(newMimeType); - ops.add(String.format("/chgm/%s/mime/%s/cond/%s", resource, encodedMime, condition.encodedString())); + public BatchOperations addChgmOp(String bucket, String key, String newMimeType, Map metas, Condition condition) { + StringBuilder builder = new StringBuilder() + .append("/chgm/").append(encodedEntry(bucket, key)); + if (newMimeType != null && !newMimeType.isEmpty()) { + builder.append("/mime/").append(UrlSafeBase64.encodeToString(newMimeType)); + } + + if (metas != null) { + for (String k : metas.keySet()) { + if (k.startsWith("x-qn-meta-")) { + builder.append("/").append(k); + } else { + builder.append("/x-qn-meta-").append(k); + } + builder.append("/").append(UrlSafeBase64.encodeToString(metas.get(k))); + } + } + + if (condition != null && condition.encodedString() != null) { + builder.append("/cond/").append(condition.encodedString()); + } + ops.add(builder.toString()); setExecBucket(bucket); return this; } @@ -1500,79 +1520,79 @@ public String execBucket() { public int size() { return ops.size(); } + } - public final static class Condition { - private final String hash; - private final String mime; - private final Long fSize; - private final Long putTime; + public final static class Condition { + private final String hash; + private final String mime; + private final Long fSize; + private final Long putTime; - private Condition(String hash, String mime, Long fSize, Long putTime) { - this.hash = hash; - this.mime = mime; - this.fSize = fSize; - this.putTime = putTime; - } - - String encodedString() { - StringBuilder builder = new StringBuilder(); - if (hash != null && !hash.isEmpty()) { - builder.append("hash=" + hash + "&"); - } - if (mime != null && !mime.isEmpty()) { - builder.append("mime=" + mime + "&"); - } - if (fSize != null) { - builder.append("fsize=" + fSize + "&"); - } - if (putTime != null) { - builder.append("putTime=" + putTime + "&"); - } + private Condition(String hash, String mime, Long fSize, Long putTime) { + this.hash = hash; + this.mime = mime; + this.fSize = fSize; + this.putTime = putTime; + } - String encoded = builder.toString(); - if (encoded.isEmpty()) { - return null; - } + String encodedString() { + StringBuilder builder = new StringBuilder(); + if (hash != null && !hash.isEmpty()) { + builder.append("hash=" + hash + "&"); + } + if (mime != null && !mime.isEmpty()) { + builder.append("mime=" + mime + "&"); + } + if (fSize != null) { + builder.append("fsize=" + fSize + "&"); + } + if (putTime != null) { + builder.append("putTime=" + putTime + "&"); + } - if (encoded.endsWith("&")) { - encoded = encoded.substring(0, encoded.length() - 1); - } + String encoded = builder.toString(); + if (encoded.isEmpty()) { + return null; + } - return UrlSafeBase64.encodeToString(encoded); + if (encoded.endsWith("&")) { + encoded = encoded.substring(0, encoded.length() - 1); } - public final static class Builder { - private String hash; - private String mime; - private Long fileSize; - private Long putTime; + return UrlSafeBase64.encodeToString(encoded); + } - public Builder() { - } + public final static class Builder { + private String hash; + private String mime; + private Long fileSize; + private Long putTime; - public Builder setHash(String hash) { - this.hash = hash; - return this; - } + public Builder() { + } - public Builder setMime(String mime) { - this.mime = mime; - return this; - } + public Builder setHash(String hash) { + this.hash = hash; + return this; + } - public Builder setFileSize(Long fileSize) { - this.fileSize = fileSize; - return this; - } + public Builder setMime(String mime) { + this.mime = mime; + return this; + } - public Builder setPutTime(Long putTime) { - this.putTime = putTime; - return this; - } + public Builder setFileSize(Long fileSize) { + this.fileSize = fileSize; + return this; + } - public Condition build() { - return new Condition(hash, mime, fileSize, putTime); - } + public Builder setPutTime(Long putTime) { + this.putTime = putTime; + return this; + } + + public Condition build() { + return new Condition(hash, mime, fileSize, putTime); } } } diff --git a/src/test/java/com/qiniu/storage/BatchTest.java b/src/test/java/com/qiniu/storage/BatchTest.java index e6cc3cc3e..906c2ab5b 100644 --- a/src/test/java/com/qiniu/storage/BatchTest.java +++ b/src/test/java/com/qiniu/storage/BatchTest.java @@ -10,25 +10,25 @@ public class BatchTest { @Tag("UnitTest") public void testBatchCondition() { - BucketManager.BatchOperations.Condition condition = new BucketManager.BatchOperations.Condition.Builder() + BucketManager.Condition condition = new BucketManager.Condition.Builder() .build(); String encodedString = condition.encodedString(); Assertions.assertNull(encodedString); - condition = new BucketManager.BatchOperations.Condition.Builder() + condition = new BucketManager.Condition.Builder() .setHash("hash") .build(); encodedString = condition.encodedString(); Assertions.assertEquals(encodedString, "aGFzaD1oYXNo"); - condition = new BucketManager.BatchOperations.Condition.Builder() + condition = new BucketManager.Condition.Builder() .setHash("hash") .setPutTime(1993232L) .build(); encodedString = condition.encodedString(); Assertions.assertEquals(encodedString, "aGFzaD1oYXNoJnB1dFRpbWU9MTk5MzIzMg=="); - condition = new BucketManager.BatchOperations.Condition.Builder() + condition = new BucketManager.Condition.Builder() .setHash("hash") .setMime("application/txt") .setPutTime(1993232L) @@ -36,7 +36,7 @@ public void testBatchCondition() { encodedString = condition.encodedString(); Assertions.assertEquals(encodedString, "aGFzaD1oYXNoJm1pbWU9YXBwbGljYXRpb24vdHh0JnB1dFRpbWU9MTk5MzIzMg=="); - condition = new BucketManager.BatchOperations.Condition.Builder() + condition = new BucketManager.Condition.Builder() .setHash("hash") .setMime("application/txt") .setFileSize(100L) diff --git a/src/test/java/test/com/qiniu/storage/BucketTest2.java b/src/test/java/test/com/qiniu/storage/BucketTest2.java index a1498b16a..f4ff534de 100644 --- a/src/test/java/test/com/qiniu/storage/BucketTest2.java +++ b/src/test/java/test/com/qiniu/storage/BucketTest2.java @@ -27,6 +27,7 @@ import static org.junit.jupiter.api.Assertions.fail; import java.io.ByteArrayInputStream; +import java.io.File; import java.io.IOException; import java.text.SimpleDateFormat; import java.util.*; @@ -1140,6 +1141,26 @@ public void testBatchMove() { String moveToKey = key + "to"; BucketManager.BatchOperations ops = new BucketManager.BatchOperations().addMoveOp(bucket, moveFromKey, bucket, moveToKey); + try { + Response r = bucketManager.batch(ops); + BatchStatus[] bs = r.jsonToObject(BatchStatus[].class); + assertTrue(bs[0].code == 614 || bs[0].code == 200, "200 or 614"); + } catch (QiniuException e) { + fail(e.response.toString()); + } + + ops = new BucketManager.BatchOperations().addMoveOp(bucket, moveFromKey, + bucket, moveToKey, false); + try { + Response r = bucketManager.batch(ops); + BatchStatus[] bs = r.jsonToObject(BatchStatus[].class); + assertTrue(bs[0].code == 614, "614"); + } catch (QiniuException e) { + fail(e.response.toString()); + } + + ops = new BucketManager.BatchOperations().addMoveOp(bucket, moveFromKey, + bucket, moveToKey, true); try { Response r = bucketManager.batch(ops); BatchStatus[] bs = r.jsonToObject(BatchStatus[].class); @@ -1147,6 +1168,7 @@ public void testBatchMove() { } catch (QiniuException e) { fail(e.response.toString()); } + try { bucketManager.delete(bucket, moveToKey); } catch (QiniuException e) { @@ -1276,37 +1298,100 @@ public void testBatchCopyChgmDelete() { for (Map.Entry entry : bucketKeyMap.entrySet()) { String bucket = entry.getKey(); - String key = entry.getValue(); + String oKey = entry.getValue(); - // make 100 copies - String[] keyArray = new String[100]; - for (int i = 0; i < keyArray.length; i++) { - keyArray[i] = String.format("%s-copy-%d", key, i); - } + // 待操作的 key + String key = String.format("%s-copy-%d", oKey, new Date().getTime()); BucketManager.BatchOperations ops = new BucketManager.BatchOperations(); - for (int i = 0; i < keyArray.length; i++) { - ops.addCopyOp(bucket, key, bucket, keyArray[i]); - } + ops.addCopyOp(bucket, oKey, bucket, key); try { + // batch copy Response response = bucketManager.batch(ops); - assertTrue(batchStatusCode.contains(response.statusCode), "200 or 298"); + BatchStatus[] bs = response.jsonToObject(BatchStatus[].class); + assertTrue(bs[0].code == 200, "200"); + + + // batch copy: 非强制已存在不能成功 + ops = new BucketManager.BatchOperations(); + ops.addCopyOp(bucket, oKey, bucket, key, false); + response = bucketManager.batch(ops); + bs = response.jsonToObject(BatchStatus[].class); + assertTrue(bs[0].code == 614, "already exist, can't copy"); + + + // batch copy: 已存在，强制需成功 + ops = new BucketManager.BatchOperations(); + ops.addCopyOp(bucket, oKey, bucket, key, true); + response = bucketManager.batch(ops); + bs = response.jsonToObject(BatchStatus[].class); + assertTrue(bs[0].code == 200, "already exist, force should copy"); // clear ops ops.clearOps(); - // batch chane mimetype - for (int i = 0; i < keyArray.length; i++) { - ops.addChgmOp(bucket, keyArray[i], "image/png"); - } + // batch change mimetype + ops = new BucketManager.BatchOperations(); + ops.addChgmOp(bucket, key, "image/png"); response = bucketManager.batch(ops); - assertTrue(batchStatusCode.contains(response.statusCode), "200 or 298"); + bs = response.jsonToObject(BatchStatus[].class); + assertTrue(bs[0].code == 200, "200"); + + // 根据错误条件修改 + ops = new BucketManager.BatchOperations(); + ops.addChgmOp(bucket, key, "image/jpg", null, new BucketManager.Condition.Builder() + .setHash("ijdew[qoewqewwe") + .build()); + response = bucketManager.batch(ops); + bs = response.jsonToObject(BatchStatus[].class); + assertTrue(bs[0].code == 613, "298"); - // clear ops - for (int i = 0; i < keyArray.length; i++) { - ops.addDeleteOp(bucket, keyArray[i]); - } + + // 根据正确条件修改 + ops = new BucketManager.BatchOperations(); + ops.addChgmOp(bucket, key, "image/jpg", null, new BucketManager.Condition.Builder() + .setMime("image/png") + .build()); + response = bucketManager.batch(ops); + bs = response.jsonToObject(BatchStatus[].class); + assertTrue(bs[0].code == 200, "200"); + + // 增加 meta data + HashMap metas = new HashMap<>(); + metas.put("haha", "value"); + + // 根据错误条件增加 + ops = new BucketManager.BatchOperations(); + ops.addChgmOp(bucket, key, "", metas, new BucketManager.Condition.Builder() + .setHash("ijdew[qoewqewwe") + .build()); + response = bucketManager.batch(ops); + bs = response.jsonToObject(BatchStatus[].class); + assertTrue(bs[0].code == 613, "298"); + + + // 根据正确条件增加 + FileInfo fileInfo = bucketManager.stat(bucket, key); + ops = new BucketManager.BatchOperations(); + ops.addChgmOp(bucket, key, "", metas, new BucketManager.Condition.Builder() + .setMime(fileInfo.mimeType) + .setHash(fileInfo.hash) + .setFileSize(fileInfo.fsize) + .setPutTime(fileInfo.putTime) + .build()); + response = bucketManager.batch(ops); + bs = response.jsonToObject(BatchStatus[].class); + assertTrue(bs[0].code == 200, "298"); + + + fileInfo = bucketManager.stat(bucket, key); + Map meta = fileInfo.meta; + assertTrue(meta != null && meta.get("haha").equals("value"), "meta value"); + + // 删除文件列表 + ops = new BucketManager.BatchOperations(); + ops.addDeleteOp(bucket, key); response = bucketManager.batch(ops); assertTrue(batchStatusCode.contains(response.statusCode), "200 or 298"); From 3fe2c7ab5d7cca21ebee55c95e4aab9aa1d6b9a6 Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Tue, 14 Nov 2023 14:53:37 +0800 Subject: [PATCH 18/28] region host support contain scheme --- .../java/com/qiniu/storage/ConfigHelper.java | 35 +++++++++++++------ .../java/com/qiniu/storage/UpHostHelper.java | 3 +- src/main/java/com/qiniu/util/UrlUtils.java | 17 +++++++++ 3 files changed, 43 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/qiniu/storage/ConfigHelper.java b/src/main/java/com/qiniu/storage/ConfigHelper.java index 4858e93a8..cd4ebf21f 100644 --- a/src/main/java/com/qiniu/storage/ConfigHelper.java +++ b/src/main/java/com/qiniu/storage/ConfigHelper.java @@ -39,33 +39,44 @@ public String tryChangeUpHost(String upToken, String lastUsedHost) throws QiniuE private String upHost(String upToken, String lastUsedHost, boolean changeHost, boolean mustReturnUpHost) throws QiniuException { - return getScheme() - + getHelper().upHost(config.region, upToken, UrlUtils.removeHostScheme(lastUsedHost), changeHost, mustReturnUpHost); + String host = getHelper().upHost(config.region, upToken, UrlUtils.removeHostScheme(lastUsedHost), changeHost, mustReturnUpHost); + host = UrlUtils.setHostScheme(host, config.useHttpsDomains); + return host; } public String ioHost(String ak, String bucket) throws QiniuException { RegionReqInfo regionReqInfo = new RegionReqInfo(ak, bucket); - return getScheme() + config.region.getIovipHost(regionReqInfo); + String host = config.region.getIovipHost(regionReqInfo); + host = UrlUtils.setHostScheme(host, config.useHttpsDomains); + return host; } public String ioSrcHost(String ak, String bucket) throws QiniuException { RegionReqInfo regionReqInfo = new RegionReqInfo(ak, bucket); - return config.region.getIoSrcHost(regionReqInfo); + String host = config.region.getIoSrcHost(regionReqInfo); + host = UrlUtils.setHostScheme(host, config.useHttpsDomains); + return host; } public String apiHost(String ak, String bucket) throws QiniuException { RegionReqInfo regionReqInfo = new RegionReqInfo(ak, bucket); - return getScheme() + config.region.getApiHost(regionReqInfo); + String host = config.region.getApiHost(regionReqInfo); + host = UrlUtils.setHostScheme(host, config.useHttpsDomains); + return host; } public String rsHost(String ak, String bucket) throws QiniuException { RegionReqInfo regionReqInfo = new RegionReqInfo(ak, bucket); - return getScheme() + config.region.getRsHost(regionReqInfo); + String host = config.region.getRsHost(regionReqInfo); + host = UrlUtils.setHostScheme(host, config.useHttpsDomains); + return host; } public String rsfHost(String ak, String bucket) throws QiniuException { RegionReqInfo regionReqInfo = new RegionReqInfo(ak, bucket); - return getScheme() + config.region.getRsfHost(regionReqInfo); + String host = config.region.getRsfHost(regionReqInfo); + host = UrlUtils.setHostScheme(host, config.useHttpsDomains); + return host; } public String rsHost() { @@ -78,7 +89,8 @@ public String rsHost() { if (host == null || host.length() == 0) { host = Configuration.defaultRsHost; } - return getScheme() + host; + host = UrlUtils.setHostScheme(host, config.useHttpsDomains); + return host; } public String apiHost() { @@ -91,7 +103,8 @@ public String apiHost() { if (host == null || host.length() == 0) { host = Configuration.defaultApiHost; } - return getScheme() + host; + host = UrlUtils.setHostScheme(host, config.useHttpsDomains); + return host; } public String ucHost() { @@ -104,7 +117,9 @@ public String ucHost() { if (host == null || host.length() == 0) { host = Configuration.defaultUcHost; } - return getScheme() + host; + + host = UrlUtils.setHostScheme(host, config.useHttpsDomains); + return host; } List ucHostsWithoutScheme() { diff --git a/src/main/java/com/qiniu/storage/UpHostHelper.java b/src/main/java/com/qiniu/storage/UpHostHelper.java index 67a783624..70f3185e4 100644 --- a/src/main/java/com/qiniu/storage/UpHostHelper.java +++ b/src/main/java/com/qiniu/storage/UpHostHelper.java @@ -47,8 +47,7 @@ String upHost(Region region, String upToken, String lastUsedHost, boolean change regionHostsLRU.put(regionKey, regionHost); } - String host = regionHost.upHost(accHosts, srcHosts, lastUsedHost, changeHost); - return host; + return regionHost.upHost(accHosts, srcHosts, lastUsedHost, changeHost); } private String failedUpHost(String regionKey) { diff --git a/src/main/java/com/qiniu/util/UrlUtils.java b/src/main/java/com/qiniu/util/UrlUtils.java index ceff8acf9..a2892cf8d 100644 --- a/src/main/java/com/qiniu/util/UrlUtils.java +++ b/src/main/java/com/qiniu/util/UrlUtils.java @@ -190,4 +190,21 @@ public static String removeHostScheme(String host) { host = host.replace("https://", ""); return host; } + + + /** + * 如果 host 包含 scheme 则优先使用 host 中包含的 scheme + * 如果 host 不包含 scheme 则按照 useHttps 增加 scheme + */ + public static String setHostScheme(String host, boolean useHttps) { + if (host == null || StringUtils.isNullOrEmpty(host)) { + return null; + } + + if (host.startsWith("http://") || host.startsWith("https://") ) { + return host; + } + + return useHttps ? "https://" : "http://" + host; + } } From cba5907e5f70b9972475397b1b285da422cb6894 Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Tue, 14 Nov 2023 14:57:56 +0800 Subject: [PATCH 19/28] remove useless code --- src/main/java/com/qiniu/storage/ConfigHelper.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/main/java/com/qiniu/storage/ConfigHelper.java b/src/main/java/com/qiniu/storage/ConfigHelper.java index cd4ebf21f..6b57b396b 100644 --- a/src/main/java/com/qiniu/storage/ConfigHelper.java +++ b/src/main/java/com/qiniu/storage/ConfigHelper.java @@ -166,10 +166,6 @@ List upHostsWithoutScheme() { return new ArrayList<>(hosts); } - private String getScheme() { - return config.useHttpsDomains ? "https://" : "http://"; - } - private void makeSureRegion() { if (config.region == null) { if (config.zone != null) { From c6e3c2987d3c764bd6a524bed41fbda3c986a909 Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Tue, 14 Nov 2023 16:42:34 +0800 Subject: [PATCH 20/28] optimize some code --- src/main/java/com/qiniu/storage/BucketManager.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/qiniu/storage/BucketManager.java b/src/main/java/com/qiniu/storage/BucketManager.java index bed2b5e01..5d573b540 100644 --- a/src/main/java/com/qiniu/storage/BucketManager.java +++ b/src/main/java/com/qiniu/storage/BucketManager.java @@ -1522,7 +1522,7 @@ public int size() { } } - public final static class Condition { + public static final class Condition { private final String hash; private final String mime; private final Long fSize; @@ -1562,7 +1562,7 @@ String encodedString() { return UrlSafeBase64.encodeToString(encoded); } - public final static class Builder { + public static final class Builder { private String hash; private String mime; private Long fileSize; From 2238f6f8a3ae380e31981f0df911ddeb63a0da11 Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Tue, 14 Nov 2023 17:56:02 +0800 Subject: [PATCH 21/28] change test case --- src/test/java/test/com/qiniu/storage/BucketTest2.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/test/java/test/com/qiniu/storage/BucketTest2.java b/src/test/java/test/com/qiniu/storage/BucketTest2.java index f4ff534de..41c36788e 100644 --- a/src/test/java/test/com/qiniu/storage/BucketTest2.java +++ b/src/test/java/test/com/qiniu/storage/BucketTest2.java @@ -27,7 +27,6 @@ import static org.junit.jupiter.api.Assertions.fail; import java.io.ByteArrayInputStream; -import java.io.File; import java.io.IOException; import java.text.SimpleDateFormat; import java.util.*; From 76c8940fe60e12beba46e5263df6671043c77f52 Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Wed, 15 Nov 2023 11:24:02 +0800 Subject: [PATCH 22/28] handle url setHostScheme --- src/main/java/com/qiniu/util/UrlUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/qiniu/util/UrlUtils.java b/src/main/java/com/qiniu/util/UrlUtils.java index a2892cf8d..f5b8e83e9 100644 --- a/src/main/java/com/qiniu/util/UrlUtils.java +++ b/src/main/java/com/qiniu/util/UrlUtils.java @@ -205,6 +205,6 @@ public static String setHostScheme(String host, boolean useHttps) { return host; } - return useHttps ? "https://" : "http://" + host; + return (useHttps ? "https://" : "http://") + host; } } From a5b22e5f74cb781716f4bbf65be927a81054d0de Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Wed, 15 Nov 2023 12:15:19 +0800 Subject: [PATCH 23/28] change download url --- src/main/java/com/qiniu/storage/DownloadUrl.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/main/java/com/qiniu/storage/DownloadUrl.java b/src/main/java/com/qiniu/storage/DownloadUrl.java index 5df7e3735..f2851e679 100644 --- a/src/main/java/com/qiniu/storage/DownloadUrl.java +++ b/src/main/java/com/qiniu/storage/DownloadUrl.java @@ -204,11 +204,7 @@ protected void didBuildUrl() throws QiniuException { } private String getUrlPrefix() throws QiniuException { - if (useHttps) { - return "https://" + domain; - } else { - return "http://" + domain; - } + return UrlUtils.setHostScheme(domain, useHttps); } /** From 4c8963c372ee579eedb0fd41648075c3f5219dde Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Wed, 15 Nov 2023 12:18:29 +0800 Subject: [PATCH 24/28] change test case --- src/test/java/test/com/qiniu/storage/BucketTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test/java/test/com/qiniu/storage/BucketTest.java b/src/test/java/test/com/qiniu/storage/BucketTest.java index bccc64288..85634a378 100644 --- a/src/test/java/test/com/qiniu/storage/BucketTest.java +++ b/src/test/java/test/com/qiniu/storage/BucketTest.java @@ -1165,10 +1165,10 @@ public void testFile(TestConfig.TestFile file, BucketManager bucketManager) thro assertTrue(ResCode.find(e.code(), ResCode.getPossibleResCode(400))); } try { - testBucketQuota(bucketManager, bucket, 0, 0); - testBucketQuota(bucketManager, bucket, 100, 100); - testBucketQuota(bucketManager, bucket, 0, 100); - testBucketQuota(bucketManager, bucket, 100, -1); +// testBucketQuota(bucketManager, bucket, 0, 0); + testBucketQuota(bucketManager, bucket, 10000, 10000); +// testBucketQuota(bucketManager, bucket, 0, 10000); + testBucketQuota(bucketManager, bucket, 10000, -1); testBucketQuota(bucketManager, bucket, -1, -1); } catch (QiniuException e) { assertTrue(ResCode.find(e.code(), ResCode.getPossibleResCode())); From 38942b7de02c2f496339bdb6539d9dbf67707fb4 Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Fri, 17 Nov 2023 15:43:15 +0800 Subject: [PATCH 25/28] change test case --- src/test/java/test/com/qiniu/TestConfig.java | 32 +++++++++++++++++-- .../test/com/qiniu/storage/BucketTest2.java | 7 +++- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/src/test/java/test/com/qiniu/TestConfig.java b/src/test/java/test/com/qiniu/TestConfig.java index 788268613..8278f7748 100644 --- a/src/test/java/test/com/qiniu/TestConfig.java +++ b/src/test/java/test/com/qiniu/TestConfig.java @@ -22,6 +22,9 @@ public final class TestConfig { // test: ak, sk, auth public static final String testAccessKey = System.getenv("QINIU_ACCESS_KEY"); public static final String testSecretKey = System.getenv("QINIU_SECRET_KEY"); + public static final String testBucket = System.getenv("BUCKET"); + public static final String testBucketDomain = System.getenv("BUCKET_DOMAIN"); + // 内部测试环境 AK/SK public static final String innerAccessKey = System.getenv("testAK"); public static final String innerSecretKey = System.getenv("testSK"); @@ -46,7 +49,10 @@ public final class TestConfig { public static final String testPipeline = "sdktest"; // z1 - public static final String testBucket_z1 = "sdk-z1"; + public static final String testBucket_z1 = "javasdk-z1"; + public static final String testKey_z1 = "do_not_delete/1.png"; + public static final String testDomain_z1 = "javasdk-z1.qiniupkg.com"; + public static final String testDomain_z1_timeStamp = "javasdk-timestamp.peterpy.cn"; // na0 public static final String testBucket_na0 = "java-sdk-na0"; @@ -111,6 +117,17 @@ public static TestFile[] getTestFileArray(String fileSaveKey, String fileMimeTyp z0.regionId = "z0"; z0.region = Region.createWithRegionId("z0"); + TestFile z1 = new TestFile(); + z1.key = fileSaveKey; + z1.mimeType = fileMimeType; + z1.bucketName = testBucket_z1; + z1.testDomain = testDomain_z1; + z1.testUrl = "http://" + z1.testDomain + "/" + fileSaveKey; + z1.testDomainTimeStamp = testDomain_z0_timeStamp; + z1.testUrlTimeStamp = "http://" + testDomain_z0_timeStamp + "/" + fileSaveKey; + z1.regionId = "z1"; + z1.region = Region.createWithRegionId("z1"); + TestFile z0_auto = new TestFile(); z0_auto.key = fileSaveKey; z0_auto.mimeType = fileMimeType; @@ -122,7 +139,18 @@ public static TestFile[] getTestFileArray(String fileSaveKey, String fileMimeTyp z0_auto.regionId = "z0"; z0_auto.region = Region.region0(); - return new TestFile[]{z0}; + TestFile file = new TestFile(); + file.key = fileSaveKey; + file.mimeType = fileMimeType; + file.bucketName = (testBucket != null && !testBucket.isEmpty())? testBucket : testBucket_z0; + file.testDomain = (testBucketDomain != null && !testBucketDomain.isEmpty())? testBucketDomain : testDomain_z0; + file.testUrl = "http://" + file.testDomain + "/" + fileSaveKey; + file.testDomainTimeStamp = testDomain_z0_timeStamp; + file.testUrlTimeStamp = "http://" + testDomain_z0_timeStamp + "/" + fileSaveKey; + file.regionId = "z0"; + file.region = Region.autoRegion(); + + return new TestFile[]{file}; } public static TestFile[] getAllRegionTestFileArray() { diff --git a/src/test/java/test/com/qiniu/storage/BucketTest2.java b/src/test/java/test/com/qiniu/storage/BucketTest2.java index 41c36788e..8721b2837 100644 --- a/src/test/java/test/com/qiniu/storage/BucketTest2.java +++ b/src/test/java/test/com/qiniu/storage/BucketTest2.java @@ -1148,12 +1148,17 @@ public void testBatchMove() { fail(e.response.toString()); } + try { + bucketManager.copy(bucket, key, bucket, moveFromKey, true); + } catch (QiniuException ignored) { + } + ops = new BucketManager.BatchOperations().addMoveOp(bucket, moveFromKey, bucket, moveToKey, false); try { Response r = bucketManager.batch(ops); BatchStatus[] bs = r.jsonToObject(BatchStatus[].class); - assertTrue(bs[0].code == 614, "614"); + assertEquals(bs[0].code, 614); } catch (QiniuException e) { fail(e.response.toString()); } From b1a222938e532529be831e60663ecd66f33a43af Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Fri, 17 Nov 2023 16:14:07 +0800 Subject: [PATCH 26/28] format --- src/test/java/test/com/qiniu/TestConfig.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/java/test/com/qiniu/TestConfig.java b/src/test/java/test/com/qiniu/TestConfig.java index 8278f7748..2343fa888 100644 --- a/src/test/java/test/com/qiniu/TestConfig.java +++ b/src/test/java/test/com/qiniu/TestConfig.java @@ -142,8 +142,8 @@ public static TestFile[] getTestFileArray(String fileSaveKey, String fileMimeTyp TestFile file = new TestFile(); file.key = fileSaveKey; file.mimeType = fileMimeType; - file.bucketName = (testBucket != null && !testBucket.isEmpty())? testBucket : testBucket_z0; - file.testDomain = (testBucketDomain != null && !testBucketDomain.isEmpty())? testBucketDomain : testDomain_z0; + file.bucketName = (testBucket != null && !testBucket.isEmpty()) ? testBucket : testBucket_z0; + file.testDomain = (testBucketDomain != null && !testBucketDomain.isEmpty()) ? testBucketDomain : testDomain_z0; file.testUrl = "http://" + file.testDomain + "/" + fileSaveKey; file.testDomainTimeStamp = testDomain_z0_timeStamp; file.testUrlTimeStamp = "http://" + testDomain_z0_timeStamp + "/" + fileSaveKey; @@ -249,7 +249,7 @@ public static TestFile[] getRetryTestFileArray() { regionGroup.addRegion(region00); regionGroup.addRegion(region01); na0.region = regionGroup; - return new TestFile[] { na0 }; + return new TestFile[]{na0}; } private static Region toRegion(Zone zone) { From 9823f9930f029aad0ce6f5c29fa7e1ab9ff0160b Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Mon, 20 Nov 2023 14:51:08 +0800 Subject: [PATCH 27/28] change test case --- src/test/java/test/com/qiniu/TestConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/test/com/qiniu/TestConfig.java b/src/test/java/test/com/qiniu/TestConfig.java index 2343fa888..26aaf5937 100644 --- a/src/test/java/test/com/qiniu/TestConfig.java +++ b/src/test/java/test/com/qiniu/TestConfig.java @@ -148,7 +148,7 @@ public static TestFile[] getTestFileArray(String fileSaveKey, String fileMimeTyp file.testDomainTimeStamp = testDomain_z0_timeStamp; file.testUrlTimeStamp = "http://" + testDomain_z0_timeStamp + "/" + fileSaveKey; file.regionId = "z0"; - file.region = Region.autoRegion(); + file.region = testBucket_z1.equals(testBucket) ? Region.region1() : Region.region0(); return new TestFile[]{file}; } From 0ea1d3624d035c95bf8bf1979019f1b812871708 Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Tue, 21 Nov 2023 16:30:37 +0800 Subject: [PATCH 28/28] region support set uc host --- src/main/java/com/qiniu/storage/Region.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/qiniu/storage/Region.java b/src/main/java/com/qiniu/storage/Region.java index 8765c868c..4d93959cc 100644 --- a/src/main/java/com/qiniu/storage/Region.java +++ b/src/main/java/com/qiniu/storage/Region.java @@ -371,7 +371,7 @@ String getApiHost(RegionReqInfo regionReqInfo) throws QiniuException { } String getUcHost(RegionReqInfo regionReqInfo) throws QiniuException { - if (ucHosts == null || ucHosts.size() == 0) { + if (ucHosts == null || ucHosts.isEmpty()) { return ""; } return ucHosts.get(0); @@ -470,6 +470,13 @@ public Builder apiHost(String apiHost) { return this; } + public Builder ucHost(String... ucHosts) { + if (ucHosts.length > 0) { + this.region.ucHosts = Arrays.asList(ucHosts); + } + return this; + } + /** * 自动选择,其它参数设置无效 *