From 03d9cb37726a6ea26c099df592e4644fd66941ec Mon Sep 17 00:00:00 2001 From: cxorm Date: Thu, 10 Oct 2019 22:08:30 +0800 Subject: [PATCH 1/3] HADOOP-16637. Fix findbugs warnings in hadoop-cos. --- .../org/apache/hadoop/fs/cosn/BufferPool.java | 5 ----- .../apache/hadoop/fs/cosn/CosNInputStream.java | 11 ++++++++++- .../hadoop/fs/cosn/CosNativeFileSystemStore.java | 15 ++++++++++++--- 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/BufferPool.java b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/BufferPool.java index a4ee4d5be9ac8..5a7d4347b8e39 100644 --- a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/BufferPool.java +++ b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/BufferPool.java @@ -63,7 +63,6 @@ private BufferPool() { private File createDir(String dirPath) throws IOException { File dir = new File(dirPath); - if (null != dir) { if (!dir.exists()) { LOG.debug("Buffer dir: [{}] does not exists. create it first.", dirPath); @@ -86,10 +85,6 @@ private File createDir(String dirPath) throws IOException { } else { LOG.debug("buffer dir: {} already exists.", dirPath); } - } else { - throw new IOException("creating buffer dir: " + dir.getAbsolutePath() - + "unsuccessfully."); - } return dir; } diff --git a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNInputStream.java b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNInputStream.java index e759b55a559ad..37ea2954a4c6a 100644 --- a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNInputStream.java +++ b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNInputStream.java @@ -33,6 +33,8 @@ import org.apache.hadoop.fs.FSInputStream; import org.apache.hadoop.fs.FileSystem; +import javax.annotation.Nullable; + /** * The input stream for the COS blob store. * Optimized sequential read flow based on a forward read-ahead queue @@ -83,8 +85,15 @@ public void signalAll() { readyCondition.signalAll(); } + @Nullable public byte[] getBuffer() { - return this.buffer; + if (this.buffer.length == 0) return null; + + byte[] tempBuffer = new byte[this.buffer.length]; + for (int index = 0; index < this.buffer.length; index++) { + tempBuffer[index] = this.buffer[index]; + } + return tempBuffer; } public int getStatus() { diff --git a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNativeFileSystemStore.java b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNativeFileSystemStore.java index 833f42d7be6e7..69eaa256d0876 100644 --- a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNativeFileSystemStore.java +++ b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNativeFileSystemStore.java @@ -24,6 +24,7 @@ import java.io.IOException; import java.io.InputStream; import java.net.URI; +import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -175,7 +176,7 @@ private void storeFileWithRetry(String key, InputStream inputStream, PutObjectResult putObjectResult = (PutObjectResult) callCOSClientWithRetry(putObjectRequest); LOG.debug("Store file successfully. COS key: [{}], ETag: [{}], " - + "MD5: [{}].", key, putObjectResult.getETag(), new String(md5Hash)); + + "MD5: [{}].", key, putObjectResult.getETag(), Charset.defaultCharset()); } catch (Exception e) { String errMsg = String.format("Store file failed. COS key: [%s], " + "exception: [%s]", key, e.toString()); @@ -197,7 +198,7 @@ public void storeFile(String key, File file, byte[] md5Hash) throws IOException { LOG.info("Store file from local path: [{}]. file length: [{}] COS key: " + "[{}] MD5: [{}].", file.getCanonicalPath(), file.length(), key, - new String(md5Hash)); + Charset.defaultCharset()); storeFileWithRetry(key, new BufferedInputStream(new FileInputStream(file)), md5Hash, file.length()); } @@ -249,7 +250,15 @@ public void storeEmptyFile(String key) throws IOException { public PartETag uploadPart(File file, String key, String uploadId, int partNum) throws IOException { - InputStream inputStream = new FileInputStream(file); + InputStream inputStream = null; + try { + inputStream = new FileInputStream(file); + } + finally { + if (inputStream != null) { + inputStream.close(); + } + } return uploadPart(inputStream, key, uploadId, partNum, file.length()); } From cc35eb9b62d06a55f0d40a25fe219fe0b93bc20b Mon Sep 17 00:00:00 2001 From: cxorm Date: Thu, 24 Oct 2019 01:32:45 +0800 Subject: [PATCH 2/3] Fix the CosNativeFileSystemStore.java --- .../apache/hadoop/fs/cosn/CosNativeFileSystemStore.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNativeFileSystemStore.java b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNativeFileSystemStore.java index 69eaa256d0876..3509902e07483 100644 --- a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNativeFileSystemStore.java +++ b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNativeFileSystemStore.java @@ -69,6 +69,8 @@ import org.apache.hadoop.util.VersionInfo; import org.apache.http.HttpStatus; +import javax.annotation.Nullable; + /** * The class actually performs access operation to the COS blob store. * It provides the bridging logic for the Hadoop's abstract filesystem and COS. @@ -248,6 +250,7 @@ public void storeEmptyFile(String key) throws IOException { } } + @Nullable public PartETag uploadPart(File file, String key, String uploadId, int partNum) throws IOException { InputStream inputStream = null; @@ -256,10 +259,10 @@ public PartETag uploadPart(File file, String key, String uploadId, } finally { if (inputStream != null) { - inputStream.close(); + return uploadPart(inputStream, key, uploadId, partNum, file.length()); } } - return uploadPart(inputStream, key, uploadId, partNum, file.length()); + return null; } @Override From 12436aab380642df2df49b12234d5bd66c7828ad Mon Sep 17 00:00:00 2001 From: cxorm Date: Fri, 13 Mar 2020 20:30:28 +0800 Subject: [PATCH 3/3] Fix indent --- .../org/apache/hadoop/fs/cosn/BufferPool.java | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/BufferPool.java b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/BufferPool.java index 5a7d4347b8e39..409c9cb42f966 100644 --- a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/BufferPool.java +++ b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/BufferPool.java @@ -63,28 +63,28 @@ private BufferPool() { private File createDir(String dirPath) throws IOException { File dir = new File(dirPath); - if (!dir.exists()) { - LOG.debug("Buffer dir: [{}] does not exists. create it first.", - dirPath); - if (dir.mkdirs()) { - if (!dir.setWritable(true) || !dir.setReadable(true) - || !dir.setExecutable(true)) { - LOG.warn("Set the buffer dir: [{}]'s permission [writable," - + "readable, executable] failed.", dir.getAbsolutePath()); - } - LOG.debug("Buffer dir: [{}] is created successfully.", - dir.getAbsolutePath()); - } else { - // Once again, check if it has been created successfully. - // Prevent problems created by multiple processes at the same time. - if (!dir.exists()) { - throw new IOException("buffer dir:" + dir.getAbsolutePath() - + " is created unsuccessfully"); - } + if (!dir.exists()) { + LOG.debug("Buffer dir: [{}] does not exists. create it first.", + dirPath); + if (dir.mkdirs()) { + if (!dir.setWritable(true) || !dir.setReadable(true) + || !dir.setExecutable(true)) { + LOG.warn("Set the buffer dir: [{}]'s permission [writable," + + "readable, executable] failed.", dir.getAbsolutePath()); } + LOG.debug("Buffer dir: [{}] is created successfully.", + dir.getAbsolutePath()); } else { - LOG.debug("buffer dir: {} already exists.", dirPath); + // Once again, check if it has been created successfully. + // Prevent problems created by multiple processes at the same time. + if (!dir.exists()) { + throw new IOException("buffer dir:" + dir.getAbsolutePath() + + " is created unsuccessfully"); + } } + } else { + LOG.debug("buffer dir: {} already exists.", dirPath); + } return dir; }