8000 HADOOP-19072 Override fs.s3a.performance.flags for tests (ADDENDUM) by virajjasani · Pull Request #6993 · apache/hadoop · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

HADOOP-19072 Override fs.s3a.performance.flags for tests (ADDENDUM) #6993

New issue
8000

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Aug 14, 2024
+69 −74
Merged

HADOOP-19072 Override fs.s3a.performance.flags for tests (ADDENDUM) #6993

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
a3cf48e
HADOOP-19072 Override fs.s3a.performance.flags for tests (ADDENDUM)
virajjasani Aug 12, 2024
b20db54
refactor
virajjasani Aug 12, 2024
456e0ac
common usage
virajjasani Aug 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 4 additions & 8 deletions ...ls/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractCreate.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,7 @@
import org.apache.hadoop.fs.contract.AbstractFSContract;
import org.apache.hadoop.fs.s3a.S3ATestUtils;

import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE;
import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_PERFORMANCE_FLAGS;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags;

/**
* S3A contract tests creating files.
Expand Down Expand Up @@ -70,11 +68,9 @@ protected AbstractFSContract createContract(Configuration conf) {

@Override
protected Configuration createConfiguration() {
final Configuration conf = super.createConfiguration();
removeBaseAndBucketOverrides(conf,
FS_S3A_CREATE_PERFORMANCE,
FS_S3A_PERFORMANCE_FLAGS);
conf.setBoolean(FS_S3A_CREATE_PERFORMANCE, createPerformance);
final Configuration conf = setPerformanceFlags(
super.createConfiguration(),
createPerformance ? "create" : "");
S3ATestUtils.disableFilesystemCaching(conf);
return conf;
}
Expand Down
14 changes: 4 additions & 10 deletions ...ols/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMkdir.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,7 @@
import org.apache.hadoop.fs.contract.AbstractContractMkdirTest;
import org.apache.hadoop.fs.contract.AbstractFSContract;

import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE;
import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_PERFORMANCE_FLAGS;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags;

/**
* Test dir operations on S3A.
Expand All @@ -33,13 +31,9 @@ public class ITestS3AContractMkdir extends AbstractContractMkdirTest {

@Override
protected Configuration createConfiguration() {
Configuration conf = super.createConfiguration();
removeBaseAndBucketOverrides(
conf,
FS_S3A_CREATE_PERFORMANCE,
FS_S3A_PERFORMANCE_FLAGS);
conf.set(FS_S3A_PERFORMANCE_FLAGS, "");
return conf;
return setPerformanceFlags(
super.createConfiguration(),
"");
}

@Override
Expand Down
13 changes: 3 additions & 10 deletions .../src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMkdirWithCreatePerf.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,7 @@

import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile;
import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE;
import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_PERFORMANCE_FLAGS;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags;

/**
* Test mkdir operations on S3A with create performance mode.
Expand All @@ -40,14 +38,9 @@ public class ITestS3AContractMkdirWithCreatePerf extends AbstractContractMkdirTe

@Override
protected Configuration createConfiguration() {
Configuration conf = super.createConfiguration();
removeBaseAndBucketOverrides(
conf,
FS_S3A_CREATE_PERFORMANCE,
FS_S3A_PERFORMANCE_FLAGS);
conf.setStrings(FS_S3A_PERFORMANCE_FLAGS,
return setPerformanceFlags(
super.createConfiguration(),
"create,mkdir");
return conf;
}

@Override
Expand Down
6 changes: 5 additions & 1 deletion hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFSMainOperations.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@

import static org.apache.hadoop.fs.s3a.S3ATestUtils.createTestPath;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.isCreatePerformanceEnabled;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags;

/**
* S3A Test suite for the FSMainOperationsBaseTest tests.
Expand All @@ -46,7 +47,10 @@ public ITestS3AFSMainOperations() {

@Override
protected FileSystem createFileSystem() throws Exception {
contract = new S3AContract(new Configuration());
Configuration conf = setPerformanceFlags(
new Configuration(),
"");
contract = new S3AContract(conf);
contract.init();
return contract.getTestFileSystem();
}
Expand Down
13 changes: 4 additions & 9 deletions ...op-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,9 +40,7 @@


import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE;
import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_PERFORMANCE_FLAGS;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags;
import static org.apache.hadoop.fs.s3a.Statistic.*;
import static org.apache.hadoop.fs.s3a.performance.OperationCost.*;
import static org.apache.hadoop.test.GenericTestUtils.getTestDir;
Expand Down Expand Up @@ -80,12 +78,9 @@ public ITestS3AFileOperationCost(

@Override
public Configuration createConfiguration() {
final Configuration conf = super.createConfiguration();
removeBaseAndBucketOverrides(conf,
FS_S3A_CREATE_PERFORMANCE,
FS_S3A_PERFORMANCE_FLAGS);
conf.setBoolean(FS_S3A_CREATE_PERFORMANCE, isKeepingMarkers());
return conf;
return setPerformanceFlags(
super.createConfiguration(),
isKeepingMarkers() ? "create" : "");
}

/**
Expand Down
5 changes: 4 additions & 1 deletion ...p-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@

import static org.apache.hadoop.fs.contract.ContractTestUtils.skip;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.isCreatePerformanceEnabled;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags;
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
import static org.junit.Assume.*;
import static org.junit.Assert.*;
Expand Down Expand Up @@ -65,7 +66,9 @@ protected int getGlobalTimeout() {
@Before
public void setUp() throws Exception {
nameThread();
Configuration conf = new Configuration();
Configuration conf = setPerformanceFlags(
new Configuration(),
"");

fs = S3ATestUtils.createTestFileSystem(conf);
assumeNotNull(fs);
Expand Down
19 changes: 19 additions & 0 deletions hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1127,6 +1127,25 @@ public static void assumeStoreAwsHosted(final FileSystem fs) {
.getTrimmed(ENDPOINT, DEFAULT_ENDPOINT)));
}

/**
* Modify the config by setting the performance flags and return the modified config.
*
* @param conf The configuration object.
* @param flagStr The performance flag string.
* @return The modified configuration object.
*/
public static Configuration setPerformanceFlags(final Configuration conf,
final String flagStr) {
removeBaseAndBucketOverrides(
conf,
FS_S3A_CREATE_PERFORMANCE,
FS_S3A_PERFORMANCE_FLAGS);
if (flagStr != null) {
conf.set(FS_S3A_PERFORMANCE_FLAGS, flagStr);
}
return conf;
}

/**
* Helper class to do diffs of metrics.
*/
Expand Down
12 changes: 4 additions & 8 deletions ...ws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextCreateMkdir.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,7 @@
import org.apache.hadoop.fs.s3a.S3ATestUtils;
import org.junit.Before;

import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE;
import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_PERFORMANCE_FLAGS;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags;

/**
* Extends FileContextCreateMkdirBaseTest for a S3a FileContext.
Expand All @@ -30,11 +28,9 @@ public class ITestS3AFileContextCreateMkdir

@Before
public void setUp() throws Exception {
Configuration conf = new Configuration();
removeBaseAndBucketOverrides(
conf,
FS_S3A_CREATE_PERFORMANCE,
FS_S3A_PERFORMANCE_FLAGS);
Configuration conf = setPerformanceFlags(
new Configuration(),
null);
fc = S3ATestUtils.createTestFileContext(conf);
super.setUp();
}
Expand Down
12 changes: 3 additions & 9 deletions ...t/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextCreateMkdirCreatePerf.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,7 @@
import org.apache.hadoop.fs.FileContextCreateMkdirBaseTest;
import org.apache.hadoop.fs.s3a.S3ATestUtils;

import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE;
import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_PERFORMANCE_FLAGS;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags;
import static org.apache.hadoop.test.LambdaTestUtils.intercept;

/**
Expand All @@ -34,12 +32,8 @@ public class ITestS3AFileContextCreateMkdirCreatePerf

@Before
public void setUp() throws Exception {
Configuration conf = new Configuration();
removeBaseAndBucketOverrides(
conf,
FS_S3A_CREATE_PERFORMANCE,
FS_S3A_PERFORMANCE_FLAGS);
conf.setStrings(FS_S3A_PERFORMANCE_FLAGS,
Configuration conf = setPerformanceFlags(
new Configuration(),
"mkdir");
fc = S3ATestUtils.createTestFileContext(conf);
super.setUp();
Expand Down
7 changes: 6 additions & 1 deletion ...src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextMainOperations.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.s3a.S3ATestUtils;

import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags;

/**
* S3A implementation of FileContextMainOperationsBaseTest.
*/
Expand All @@ -36,7 +38,10 @@ public class ITestS3AFileContextMainOperations

@Before
public void setUp() throws IOException, Exception {
Configuration conf = new Configuration();
Configuration conf = setPerformanceFlags(
new Configuration(),
"");

fc = S3ATestUtils.createTestFileContext(conf);
super.setUp();
}
Expand Down
6 changes: 5 additions & 1 deletion ...hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextURI.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@
import org.junit.Ignore;
import org.junit.Test;

import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags;

/**
* S3a implementation of FileContextURIBase.
*/
Expand All @@ -30,7 +32,9 @@ public class ITestS3AFileContextURI extends FileContextURIBase {

@Before
public void setUp() throws IOException, Exception {
conf = new Configuration();
conf = setPerformanceFlags(
new Configuration(),
"");

fc1 = S3ATestUtils.createTestFileContext(conf);
fc2 = S3ATestUtils.createTestFileContext(conf); //different object, same FS
Expand Down
11 changes: 4 additions & 7 deletions ...ls/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestCreateFileCost.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,8 @@
import static org.apache.hadoop.fs.contract.ContractTestUtils.toChar;
import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_HEADER;
import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE;
import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_PERFORMANCE_FLAGS;
import static org.apache.hadoop.fs.s3a.Constants.XA_HEADER_PREFIX;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags;
import static org.apache.hadoop.fs.s3a.Statistic.OBJECT_BULK_DELETE_REQUEST;
import static org.apache.hadoop.fs.s3a.Statistic.OBJECT_DELETE_REQUEST;
import static org.apache.hadoop.fs.s3a.performance.OperationCost.CREATE_FILE_NO_OVERWRITE;
Expand Down Expand Up @@ -105,11 +104,9 @@ private OperationCost expected(OperationCost source) {

@Override
public Configuration createConfiguration() {
final Configuration conf = super.createConfiguration();
removeBaseAndBucketOverrides(conf,
FS_S3A_CREATE_PERFORMANCE,
FS_S3A_PERFORMANCE_FLAGS);
conf.setBoolean(FS_S3A_CREATE_PERFORMANCE, createPerformance);
final Configuration conf = setPerformanceFlags(
super.createConfiguration(),
createPerformance ? "create" : "");
S3ATestUtils.disableFilesystemCaching(conf);
return conf;
}
Expand Down
13 changes: 4 additions & 9 deletions ...ols/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3ADeleteCost.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,7 @@
import org.apache.hadoop.fs.s3a.Tristate;
import org.apache.hadoop.fs.s3a.impl.StatusProbeEnum;

import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE;
import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_PERFORMANCE_FLAGS;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags;
import static org.apache.hadoop.fs.s3a.Statistic.*;
import static org.apache.hadoop.fs.s3a.performance.OperationCost.*;
import static org.apache.hadoop.fs.s3a.performance.OperationCostValidator.probe;
Expand Down Expand Up @@ -80,12 +78,9 @@ public ITestS3ADeleteCost(final String name,

@Override
public Configuration createConfiguration() {
Configuration conf = super.createConfiguration();
removeBaseAndBucketOverrides(conf,
FS_S3A_CREATE_PERFORMANCE,
FS_S3A_PERFORMANCE_FLAGS);
conf.setBoolean(FS_S3A_CREATE_PERFORMANCE, false);
return conf;
return setPerformanceFlags(
super.createConfiguration(),
"");
}

@Override
Expand Down
Loading
0