apache · avijayanhwx · Mar 7, 2019 · Mar 7, 2019 · Mar 7, 2019 · Mar 7, 2019
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/LevelDBStoreIterator.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/LevelDBStoreIterator.java
@@ -61,4 +61,9 @@ public void seekToFirst() {
   public void seekToLast() {
     levelDBIterator.seekToLast();
   }
+
+  @Override
+  public void prefixSeek(byte[] prefix) {
+    levelDBIterator.seek(prefix);
+  }
 }
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/MetaStoreIterator.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/MetaStoreIterator.java
@@ -36,4 +36,9 @@ public interface MetaStoreIterator<T> extends Iterator<T> {
    */
   void seekToLast();
 
+  /**
+   * seek with prefix.
+   */
+  void prefixSeek(byte[] prefix);
+
 }
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/RocksDBStoreIterator.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/RocksDBStoreIterator.java
@@ -63,4 +63,9 @@ public void seekToLast() {
     rocksDBIterator.seekToLast();
   }
 
+  @Override
+  public void prefixSeek(byte[] prefix) {
+    rocksDBIterator.seek(prefix);
+  }
+
 }
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/TestMetadataStore.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/TestMetadataStore.java
@@ -163,6 +163,58 @@ public void testIterator() throws Exception {
 
   }
 
+
+  @Test
+  public void testIteratorPrefixSeek() throws Exception {
+    Configuration conf = new OzoneConfiguration();
+    conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL, storeImpl);
+    File dbDir = GenericTestUtils.getRandomizedTestDir();
+    MetadataStore dbStore = MetadataStoreBuilder.newBuilder()
+        .setConf(conf)
+        .setCreateIfMissing(true)
+        .setDbFile(dbDir)
+        .build();
+
+    for (int i = 0; i < 5; i++) {
+      dbStore.put(getBytes("a" + i), getBytes("a-value" + i));
+    }
+
+    for (int i = 0; i < 5; i++) {
+      dbStore.put(getBytes("b" + i), getBytes("b-value" + i));
+    }
+
+    for (int i = 0; i < 5; i++) {
+      dbStore.put(getBytes("c" + i), getBytes("c-value" + i));
+    }
+
+    for (int i = 5; i < 10; i++) {
+      dbStore.put(getBytes("b" + i), getBytes("b-value" + i));
+    }
+
+    for (int i = 5; i < 10; i++) {
+      dbStore.put(getBytes("a" + i), getBytes("a-value" + i));
+    }
+
+
+    MetaStoreIterator<KeyValue> metaStoreIterator = dbStore.iterator();
+    metaStoreIterator.prefixSeek(getBytes("b"));
+    int i = 0;
+    while (metaStoreIterator.hasNext()) {
+      KeyValue val = metaStoreIterator.next();
+      String key = getString(val.getKey());
+      if (key.startsWith("b")) {
+        assertEquals("b-value" + i, getString(val.getValue()));
+      } else {
+        break;
+      }
+      i++;
+    }
+    assertTrue(i == 10);
+    dbStore.close();
+    dbStore.destroy();
+    FileUtils.deleteDirectory(dbDir);
+  }
+
   @Test
   public void testMetaStoreConfigDifferentFromType() throws IOException {
 

diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/ServerUtils.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/ServerUtils.java
@@ -125,13 +125,29 @@ public static void releaseConnection(HttpRequestBase request) {
    * @return
    */
   public static File getScmDbDir(Configuration conf) {
-    final Collection<String> metadirs = conf.getTrimmedStringCollection(
-        ScmConfigKeys.OZONE_SCM_DB_DIRS);
+
+    File metadataDir = getDirWithFallBackToOzoneMetadata(conf, ScmConfigKeys
+        .OZONE_SCM_DB_DIRS, "SCM");
+    if (metadataDir != null) {
+      return metadataDir;
+    }
+
+    LOG.warn("{} is not configured. We recommend adding this setting. " +
+        "Falling back to {} instead.",
+        ScmConfigKeys.OZONE_SCM_DB_DIRS, HddsConfigKeys.OZONE_METADATA_DIRS);
+    return getOzoneMetaDirPath(conf);
+  }
+
+  public static File getDirWithFallBackToOzoneMetadata(Configuration conf,
+                                                       String key,
+                                                       String componentName) {
+    final Collection<String> metadirs = conf.getTrimmedStringCollection(key);
 
     if (metadirs.size() > 1) {
       throw new IllegalArgumentException(
-          "Bad config setting " + ScmConfigKeys.OZONE_SCM_DB_DIRS +
-          ". SCM does not support multiple metadata dirs currently");
+          "Bad config setting " + key +
+              ". " + componentName +
+              " does not support multiple metadata dirs currently");
     }
 
6D4E    if (metadirs.size() == 1) {
@@ -143,11 +159,7 @@ public static File getScmDbDir(Configuration conf) {
       }
       return dbDirPath;
     }
-
-    LOG.warn("{} is not configured. We recommend adding this setting. " +
-        "Falling back to {} instead.",
-        ScmConfigKeys.OZONE_SCM_DB_DIRS, HddsConfigKeys.OZONE_METADATA_DIRS);
-    return getOzoneMetaDirPath(conf);
+    return null;
   }
 
   /**

diff --git a/hadoop-ozone/ozone-recon/pom.xml b/hadoop-ozone/ozone-recon/pom.xml
@@ -22,7 +22,7 @@
   </parent>
   <name>Apache Hadoo
9E7A
p Ozone Recon</name>
   <modelVersion>4.0.0</modelVersion>
-  <artifactId>ozone-recon</artifactId>
+  <artifactId>hadoop-ozone-ozone-recon</artifactId>
   <dependencies>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
@@ -50,5 +50,16 @@
       <artifactId>guice-assistedinject</artifactId>
       <version>4.1.0</version>
     </dependency>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.mockito</groupId>
+      <artifactId>mockito-all</artifactId>
+      <version>1.10.19</version>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 </project>
diff --git a/...-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/ReconControllerModule.java b/...-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/ReconControllerModule.java
@@ -18,6 +18,10 @@
 package org.apache.hadoop.ozone.recon;
 
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.recon.spi.MetadataStoreProvider;
+import org.apache.hadoop.ozone.recon.spi.ContainerDBServiceProvider;
+import org.apache.hadoop.ozone.recon.spi.impl.ContainerDBServiceProviderImpl;
+import org.apache.hadoop.utils.MetadataStore;
 
 import com.google.inject.AbstractModule;
 import com.google.inject.Singleton;
@@ -30,6 +34,9 @@ public class ReconControllerModule extends AbstractModule {
   protected void configure() {
     bind(OzoneConfiguration.class).toProvider(OzoneConfigurationProvider.class);
     bind(ReconHttpServer.class).in(Singleton.class);
+    bind(MetadataStore.class).toProvider(MetadataStoreProvider.class);
+    bind(ContainerDBServiceProvider.class)
+        .to(ContainerDBServiceProviderImpl.class);
   }
 
 

diff --git a/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/ReconHttpServer.java b/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/ReconHttpServer.java
@@ -37,52 +37,52 @@ public class ReconHttpServer extends BaseHttpServer {
 
   @Override
   protected String getHttpAddressKey() {
-    return ReconServerConfiguration.OZONE_RECON_HTTP_ADDRESS_KEY;
+    return ReconServerConfigKeys.OZONE_RECON_HTTP_ADDRESS_KEY;
   }
 
   @Override
   protected String getHttpsAddressKey() {
-    return ReconServerConfiguration.OZONE_RECON_HTTPS_ADDRESS_KEY;
+    return ReconServerConfigKeys.OZONE_RECON_HTTPS_ADDRESS_KEY;
   }
 
   @Override
   protected String getHttpBindHostKey() {
-    return ReconServerConfiguration.OZONE_RECON_HTTP_BIND_HOST_KEY;
+    return ReconServerConfigKeys.OZONE_RECON_HTTP_BIND_HOST_KEY;
   }
 
   @Override
   protected String getHttpsBindHostKey() {
-    return ReconServerConfiguration.OZONE_RECON_HTTPS_BIND_HOST_KEY;
+    return ReconServerConfigKeys.OZONE_RECON_HTTPS_BIND_HOST_KEY;
   }
 
   @Override
   protected String getBindHostDefault() {
-    return ReconServerConfiguration.OZONE_RECON_HTTP_BIND_HOST_DEFAULT;
+    return ReconServerConfigKeys.OZONE_RECON_HTTP_BIND_HOST_DEFAULT;
   }
 
   @Override
   protected int getHttpBindPortDefault() {
-    return ReconServerConfiguration.OZONE_RECON_HTTP_BIND_PORT_DEFAULT;
+    return ReconServerConfigKeys.OZONE_RECON_HTTP_BIND_PORT_DEFAULT;
   }
 
   @Override
   protected int getHttpsBindPortDefault() {
-    return ReconServerConfiguration.OZONE_RECON_HTTPS_BIND_PORT_DEFAULT;
+    return ReconServerConfigKeys.OZONE_RECON_HTTPS_BIND_PORT_DEFAULT;
   }
 
   @Override
   protected String getKeytabFile() {
-    return ReconServerConfiguration.OZONE_RECON_KEYTAB_FILE;
+    return ReconServerConfigKeys.OZONE_RECON_KEYTAB_FILE;
   }
 
   @Override
   protected String getSpnegoPrincipal() {
-    return ReconServerConfiguration
+    return ReconServerConfigKeys
         .OZONE_RECON_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL;
   }
 
   @Override
   protected String getEnabledKey() {
-    return ReconServerConfiguration.OZONE_RECON_HTTP_ENABLED_KEY;
+    return ReconServerConfigKeys.OZONE_RECON_HTTP_ENABLED_KEY;
   }
 }
diff --git a/...ozone/recon/ReconServerConfiguration.java → ...op/ozone/recon/ReconServerConfigKeys.java b/...ozone/recon/ReconServerConfiguration.java → ...op/ozone/recon/ReconServerConfigKeys.java
@@ -17,6 +17,8 @@
  */
 package org.apache.hadoop.ozone.recon;
 
+import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_DB_SUFFIX;
+
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 
@@ -25,7 +27,7 @@
  */
 @InterfaceAudience.Public
 @InterfaceStability.Unstable
-public final class ReconServerConfiguration {
+public final class ReconServerConfigKeys {
 
   public static final String OZONE_RECON_HTTP_ENABLED_KEY =
       "ozone.recon.http.enabled";
@@ -48,9 +50,17 @@ public final class ReconServerConfiguration {
   public static final String OZONE_RECON_DOMAIN_NAME =
       "ozone.recon.domain.name";
 
+  public static final String OZONE_RECON_CONTAINER_DB_CACHE_SIZE_MB =
+      "ozone.recon.container.db.cache.size.mb";
+  public static final int OZONE_RECON_CONTAINER_DB_CACHE_SIZE_DEFAULT = 128;
+
+  public static final String OZONE_RECON_DB_DIRS = "ozone.recon.db.dirs";
+  public static final String RECON_CONTAINER_DB = "recon-" +
+      CONTAINER_DB_SUFFIX;
+
   /**
    * Private constructor for utility class.
    */
-  private ReconServerConfiguration() {
+  private ReconServerConfigKeys() {
   }
 }
diff --git a/...ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ContainerKeyPrefix.java b/...ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ContainerKeyPrefix.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.recon.api.types;
+
+/**
+ * Class to encapsulate the Key information needed for the Recon container DB.
+ * Currently, it is containerId and key prefix.
+ */
+public class ContainerKeyPrefix {
+
+  private long containerId;
+  private String keyPrefix;
+
+  public ContainerKeyPrefix(long containerId, String keyPrefix) {
+    this.containerId = containerId;
+    this.keyPrefix = keyPrefix;
+  }
+
+  public long getContainerId() {
+    return containerId;
+  }
+
+  public void setContainerId(long containerId) {
+    this.containerId = containerId;
+  }
+
+  public String getKeyPrefix() {
+    return keyPrefix;
+  }
+
+  public void setKeyPrefix(String keyPrefix) {
+    this.keyPrefix = keyPrefix;
+  }
+}
diff --git a/...one-recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ContainerDBServiceProvider.java b/...one-recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ContainerDBServiceProvider.java
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.recon.spi;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.ozone.recon.api.types.ContainerKeyPrefix;
+
+/**
+ * The Recon Container DB Service interface.
+ */
+@InterfaceStability.Evolving
+public interface ContainerDBServiceProvider {
+
+  /**
+   * Store the container to Key prefix mapping into the Recon Container DB.
+   *
+   * @param containerKeyPrefix the containerId, key-prefix tuple.
+   * @param count              Count of Keys with that prefix.
+   */
+  void storeContainerKeyMapping(ContainerKeyPrefix containerKeyPrefix,
+                                Integer count) throws IOException;
+
+  /**
+   * Get the stored key prefix count for the given containerId, key prefix.
+   *
+   * @param containerKeyPrefix the containerId, key-prefix tuple.
+   * @return count of keys with that prefix.
+   */
+  Integer getCountForForContainerKeyPrefix(
+      ContainerKeyPrefix containerKeyPrefix) throws IOException;
+
+  /**
+   * Get the stored key prefixes for the given containerId.
+   *
+   * @param containerId the given containerId.
+   * @return Map of Key prefix -> count.
+   */
+  Map<String, Integer> getKeyPrefixesForContainer(long containerId);
+}