|
| 1 | +From 699f329ba20d938004fc2b983b7b225de36ecf88 Mon Sep 17 00:00:00 2001 |
| 2 | +From: =?UTF-8?q?=E5=91=A8=E7=BF=B1?= < [email protected]> |
| 3 | +Date: Tue, 18 Feb 2025 14:16:37 +0800 |
| 4 | +Subject: HADOOP-19352. Hadoop OSS Connector adds support for V4 signatures. |
| 5 | + (#7205) |
| 6 | + |
| 7 | +* Aliyun oss connector support v4 signature |
| 8 | +--- |
| 9 | + hadoop-project/pom.xml | 2 +- |
| 10 | + hadoop-tools/hadoop-aliyun/pom.xml | 26 +++++ |
| 11 | + .../aliyun/oss/AliyunOSSFileSystemStore.java | 16 +++ |
| 12 | + .../hadoop/fs/aliyun/oss/Constants.java | 15 +++ |
| 13 | + .../fs/aliyun/oss/ITAliyunOSSSignatureV4.java | 98 +++++++++++++++++++ |
| 14 | + .../src/test/resources/log4j.properties | 3 + |
| 15 | + 6 files changed, 159 insertions(+), 1 deletion(-) |
| 16 | + create mode 100644 hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/ITAliyunOSSSignatureV4.java |
| 17 | + |
| 18 | +diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml |
| 19 | +index 3426a76c12..3a812ebc64 100644 |
| 20 | +--- a/hadoop-project/pom.xml |
| 21 | ++++ b/hadoop-project/pom.xml |
| 22 | +@@ -1620,7 +1620,7 @@ |
| 23 | + <dependency> |
| 24 | + <groupId>com.aliyun.oss</groupId> |
| 25 | + <artifactId>aliyun-sdk-oss</artifactId> |
| 26 | +- <version>3.13.2</version> |
| 27 | ++ <version>3.18.1</version> |
| 28 | + <exclusions> |
| 29 | + <exclusion> |
| 30 | + <groupId>org.apache.httpcomponents</groupId> |
| 31 | +diff --git a/hadoop-tools/hadoop-aliyun/pom.xml b/hadoop-tools/hadoop-aliyun/pom.xml |
| 32 | +index 5678cff642..3db04ce913 100644 |
| 33 | +--- a/hadoop-tools/hadoop-aliyun/pom.xml |
| 34 | ++++ b/hadoop-tools/hadoop-aliyun/pom.xml |
| 35 | +@@ -165,5 +165,31 @@ |
| 36 | + <scope>test</scope> |
| 37 | + <type>jar</type> |
| 38 | + </dependency> |
| 39 | ++ |
| 40 | ++ <dependency> |
| 41 | ++ <groupId>org.junit.jupiter</groupId> |
| 42 | ++ <artifactId>junit-jupiter-api</artifactId> |
| 43 | ++ <scope>test</scope> |
| 44 | ++ </dependency> |
| 45 | ++ <dependency> |
| 46 | ++ <groupId>org.junit.jupiter</groupId> |
| 47 | ++ <artifactId>junit-jupiter-engine</artifactId> |
| 48 | ++ <scope>test</scope> |
| 49 | ++ </dependency> |
| 50 | ++ <dependency> |
| 51 | ++ <groupId>org.junit.jupiter</groupId> |
| 52 | ++ <artifactId>junit-jupiter-params</artifactId> |
| 53 | ++ <scope>test</scope> |
| 54 | ++ </dependency> |
| 55 | ++ <dependency> |
| 56 | ++ <groupId>org.junit.platform</groupId> |
| 57 | ++ <artifactId>junit-platform-launcher</artifactId> |
| 58 | ++ <scope>test</scope> |
| 59 | ++ </dependency> |
| 60 | ++ <dependency> |
| 61 | ++ <groupId>org.junit.vintage</groupId> |
| 62 | ++ <artifactId>junit-vintage-engine</artifactId> |
| 63 | ++ <scope>test</scope> |
| 64 | ++ </dependency> |
| 65 | + </dependencies> |
| 66 | + </project> |
| 67 | +diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java |
| 68 | +index 6e0c7dc7e4..dba267b04c 100644 |
| 69 | +--- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java |
| 70 | ++++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java |
| 71 | +@@ -73,6 +73,7 @@ import java.util.List; |
| 72 | + import java.util.ListIterator; |
| 73 | + import java.util.NoSuchElementException; |
| 74 | + import java.util.stream.Collectors; |
| 75 | ++import com.aliyun.oss.common.comm.SignVersion; |
| 76 | + |
| 77 | + import static org.apache.hadoop.fs.aliyun.oss.Constants.*; |
| 78 | + |
| 79 | +@@ -113,6 +114,16 @@ public class AliyunOSSFileSystemStore { |
| 80 | + conf.get(USER_AGENT_PREFIX, USER_AGENT_PREFIX_DEFAULT) + ", Hadoop/" |
| 81 | + + VersionInfo.getVersion()); |
| 82 | + |
| 83 | ++ String region = conf.get(REGION_KEY, ""); |
| 84 | ++ String signatureVersion = conf.get(SIGNATURE_VERSION_KEY, SIGNATURE_VERSION_DEFAULT); |
| 85 | ++ if ("V4".equalsIgnoreCase(signatureVersion)) { |
| 86 | ++ clientConf.setSignatureVersion(SignVersion.V4); |
| 87 | ++ if (StringUtils.isEmpty(region)) { |
| 88 | ++ LOG.error("Signature version is V4 ,but region is empty."); |
| 89 | ++ throw new IOException("SignVersion is V4 but region is empty"); |
| 90 | ++ } |
| 91 | ++ } |
| 92 | ++ |
| 93 | + String proxyHost = conf.getTrimmed(PROXY_HOST_KEY, ""); |
| 94 | + int proxyPort = conf.getInt(PROXY_PORT_KEY, -1); |
| 95 | + if (StringUtils.isNotEmpty(proxyHost)) { |
| 96 | +@@ -171,6 +182,11 @@ public class AliyunOSSFileSystemStore { |
| 97 | + statistics.incrementWriteOps(1); |
| 98 | + } |
| 99 | + |
| 100 | ++ if (StringUtils.isNotEmpty(region)) { |
| 101 | ++ ossClient.setRegion(region); |
| 102 | ++ LOG.debug("ossClient setRegion {}", region); |
| 103 | ++ } |
| 104 | ++ |
| 105 | + maxKeys = conf.getInt(MAX_PAGING_KEYS_KEY, MAX_PAGING_KEYS_DEFAULT); |
| 106 | + int listVersion = conf.getInt(LIST_VERSION, DEFAULT_LIST_VERSION); |
| 107 | + if (listVersion < 1 || listVersion > 2) { |
| 108 | +diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java |
| 109 | +index baeb919937..176669ed15 100644 |
| 110 | +--- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java |
| 111 | ++++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java |
| 112 | +@@ -211,4 +211,19 @@ public final class Constants { |
| 113 | + public static final String LIST_VERSION = "fs.oss.list.version"; |
| 114 | + |
| 115 | + public static final int DEFAULT_LIST_VERSION = 2; |
| 116 | ++ |
| 117 | ++ /** |
| 118 | ++ * OSS signature version. |
| 119 | ++ */ |
| 120 | ++ public static final String SIGNATURE_VERSION_KEY = "fs.oss.signatureversion"; |
| 121 | ++ |
| 122 | ++ /** |
| 123 | ++ * OSS signature version DEFAULT {@value}. |
| 124 | ++ */ |
| 125 | ++ public static final String SIGNATURE_VERSION_DEFAULT = "V1"; |
| 126 | ++ |
| 127 | ++ /** |
| 128 | ++ * OSS region {@value}. |
| 129 | ++ */ |
| 130 | ++ public static final String REGION_KEY = "fs.oss.region"; |
| 131 | + } |
| 132 | +diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/ITAliyunOSSSignatureV4.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/ITAliyunOSSSignatureV4.java |
| 133 | +new file mode 100644 |
| 134 | +index 0000000000..5070f2a581 |
| 135 | +--- /dev/null |
| 136 | ++++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/ITAliyunOSSSignatureV4.java |
| 137 | +@@ -0,0 +1,98 @@ |
| 138 | ++/** |
| 139 | ++ * Licensed to the Apache Software Foundation (ASF) under one |
| 140 | ++ * or more contributor license agreements. See the NOTICE file |
| 141 | ++ * distributed with this work for additional information |
| 142 | ++ * regarding copyright ownership. The ASF licenses this file |
| 143 | ++ * to you under the Apache License, Version 2.0 (the |
| 144 | ++ * "License"); you may not use this file except in compliance |
| 145 | ++ * with the License. You may obtain a copy of the License at |
| 146 | ++ * |
| 147 | ++ * http://www.apache.org/licenses/LICENSE-2.0 |
| 148 | ++ * |
| 149 | ++ * Unless required by applicable law or agreed to in writing, software |
| 150 | ++ * distributed under the License is distributed on an "AS IS" BASIS, |
| 151 | ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 152 | ++ * See the License for the specific language governing permissions and |
| 153 | ++ * limitations under the License. |
| 154 | ++ */ |
| 155 | ++ |
| 156 | ++package org.apache.hadoop.fs.aliyun.oss; |
| 157 | ++ |
| 158 | ++import org.apache.hadoop.conf.Configuration; |
| 159 | ++import org.apache.hadoop.fs.FileStatus; |
| 160 | ++import org.apache.hadoop.fs.Path; |
| 161 | ++import org.junit.Before; |
| 162 | ++import org.junit.Test; |
| 163 | ++import org.slf4j.Logger; |
| 164 | ++import org.slf4j.LoggerFactory; |
| 165 | ++ |
| 166 | ++import java.io.IOException; |
| 167 | ++import java.net.URI; |
| 168 | ++ |
| 169 | ++import static org.apache.hadoop.fs.aliyun.oss.Constants.REGION_KEY; |
| 170 | ++import static org.apache.hadoop.fs.aliyun.oss.Constants.SIGNATURE_VERSION_KEY; |
| 171 | ++import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile; |
| 172 | ++import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset; |
| 173 | ++import static org.junit.Assert.*; |
| 174 | ++import static org.junit.Assume.assumeNotNull; |
| 175 | ++ |
| 176 | ++/** |
| 177 | ++ * Tests Aliyun OSS system. |
| 178 | ++ */ |
| 179 | ++public class ITAliyunOSSSignatureV4 { |
| 180 | ++ private static final Logger LOG = LoggerFactory.getLogger(ITAliyunOSSSignatureV4.class); |
| 181 | ++ private Configuration conf; |
| 182 | ++ private URI testURI; |
| 183 | ++ private Path testFile = new Path("ITAliyunOSSSignatureV4/atestr"); |
| 184 | ++ |
| 185 | ++ @Before |
| 186 | ++ public void setUp() throws Exception { |
| 187 | ++ conf = new Configuration(); |
| 188 | ++ String bucketUri = conf.get("test.fs.oss.name"); |
| 189 | ++ LOG.debug("bucketUri={}", bucketUri); |
| 190 | ++ testURI = URI.create(bucketUri); |
| 191 | ++ } |
| 192 | ++ |
| 193 | ++ @Test |
| 194 | ++ public void testV4() throws IOException { |
| 195 | ++ conf.set(SIGNATURE_VERSION_KEY, "V4"); |
| 196 | ++ conf.set(REGION_KEY, "cn-hongkong"); |
| 197 | ++ AliyunOSSFileSystem fs = new AliyunOSSFileSystem(); |
| 198 | ++ fs.initialize(testURI, conf); |
| 199 | ++ assumeNotNull(fs); |
| 200 | ++ |
| 201 | ++ createFile(fs, testFile, true, dataset(256, 0, 255)); |
| 202 | ++ FileStatus status = fs.getFileStatus(testFile); |
| 203 | ++ fs.delete(testFile); |
| 204 | ++ fs.close(); |
| 205 | ++ } |
| 206 | ++ |
| 207 | ++ @Test |
| 208 | ++ public void testDefaultSignatureVersion() throws IOException { |
| 209 | ++ AliyunOSSFileSystem fs = new AliyunOSSFileSystem(); |
| 210 | ++ fs.initialize(testURI, conf); |
| 211 | ++ assumeNotNull(fs); |
| 212 | ++ |
| 213 | ++ Path testFile2 = new Path("/test/atestr"); |
| 214 | ++ createFile(fs, testFile2, true, dataset(256, 0, 255)); |
| 215 | ++ FileStatus status = fs.getFileStatus(testFile2); |
| 216 | ++ fs.delete(testFile2); |
| 217 | ++ fs.close(); |
| 218 | ++ } |
| 219 | ++ |
| 220 | ++ @Test |
| 221 | ++ public void testV4WithoutRegion() throws IOException { |
| 222 | ++ conf.set(SIGNATURE_VERSION_KEY, "V4"); |
| 223 | ++ AliyunOSSFileSystem fs = new AliyunOSSFileSystem(); |
| 224 | ++ IOException expectedException = null; |
| 225 | ++ try { |
| 226 | ++ fs.initialize(testURI, conf); |
| 227 | ++ } catch (IOException e) { |
| 228 | ++ LOG.warn("use V4 , but do not set region, get exception={}", e); |
| 229 | ++ expectedException = e; |
| 230 | ++ assertEquals("use V4 , but do not set region", e.getMessage(), |
| 231 | ++ "SignVersion is V4 but region is empty"); |
| 232 | ++ } |
| 233 | ++ assertNotNull(expectedException); |
| 234 | ++ } |
| 235 | ++} |
| 236 | +diff --git a/hadoop-tools/hadoop-aliyun/src/test/resources/log4j.properties b/hadoop-tools/hadoop-aliyun/src/test/resources/log4j.properties |
| 237 | +index bb5cbe5ec3..2167f68811 100644 |
| 238 | +--- a/hadoop-tools/hadoop-aliyun/src/test/resources/log4j.properties |
| 239 | ++++ b/hadoop-tools/hadoop-aliyun/src/test/resources/log4j.properties |
| 240 | +@@ -21,3 +21,6 @@ log4j.threshold=ALL |
| 241 | + log4j.appender.stdout=org.apache.log4j.ConsoleAppender |
| 242 | + log4j.appender.stdout.layout=org.apache.log4j.PatternLayout |
| 243 | + log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n |
| 244 | ++ |
| 245 | ++# Log all oss classes |
| 246 | ++log4j.logger.org.apache.hadoop.fs.aliyun.oss=DEBUG |
| 247 | +\ No newline at end of file |
0 commit comments