HBASE-29842 Add Ribbon Filter as an alternative to Bloom Filter

hbase-client/src/main/java/org/apache/hadoop/hbase/client/ColumnFamilyDescriptor.java

@@ -25,6 +25,7 @@
 import org.apache.hadoop.hbase.io.compress.Compression;
 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
 import org.apache.hadoop.hbase.io.encoding.IndexBlockEncoding;
+import org.apache.hadoop.hbase.regionserver.BloomFilterImpl;
 import org.apache.hadoop.hbase.regionserver.BloomType;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.yetus.audience.InterfaceAudience;
@@ -84,6 +85,9 @@ public interface ColumnFamilyDescriptor {
   /** Returns bloom filter type used for new StoreFiles in ColumnFamily */
   BloomType getBloomFilterType();
 
+  /** Returns bloom filter implementation used for new StoreFiles in ColumnFamily */
+  BloomFilterImpl getBloomFilterImpl();
+
   /** Returns Compression type setting. */
   Compression.Algorithm getCompactionCompressionType();
 

hbase-client/src/main/java/org/apache/hadoop/hbase/client/ColumnFamilyDescriptorBuilder.java

@@ -32,6 +32,7 @@
 import org.apache.hadoop.hbase.io.compress.Compression;
 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
 import org.apache.hadoop.hbase.io.encoding.IndexBlockEncoding;
+import org.apache.hadoop.hbase.regionserver.BloomFilterImpl;
 import org.apache.hadoop.hbase.regionserver.BloomType;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.PrettyPrinter;
@@ -142,6 +143,9 @@ public class ColumnFamilyDescriptorBuilder {
   public static final String BLOOMFILTER = "BLOOMFILTER";
   private static final Bytes BLOOMFILTER_BYTES = new Bytes(Bytes.toBytes(BLOOMFILTER));
   @InterfaceAudience.Private
+  public static final String BLOOMFILTER_IMPL = "BLOOMFILTER_IMPL";
+  private static final Bytes BLOOMFILTER_IMPL_BYTES = new Bytes(Bytes.toBytes(BLOOMFILTER_IMPL));
+  @InterfaceAudience.Private
   public static final String REPLICATION_SCOPE = "REPLICATION_SCOPE";
   @InterfaceAudience.Private
   public static final String MAX_VERSIONS = HConstants.VERSIONS;
@@ -258,6 +262,11 @@ public class ColumnFamilyDescriptorBuilder {
    */
   public static final BloomType DEFAULT_BLOOMFILTER = BloomType.ROW;
 
+  /**
+   * Default bloom filter implementation.
+   */
+  public static final BloomFilterImpl DEFAULT_BLOOMFILTER_IMPL = BloomFilterImpl.BLOOM;
+
   /**
    * Default setting for whether to cache bloom filter blocks on write if block caching is enabled.
    */
@@ -305,6 +314,7 @@ public static Map<String, String> getDefaultValues() {
 
   static {
     DEFAULT_VALUES.put(BLOOMFILTER, DEFAULT_BLOOMFILTER.name());
+    DEFAULT_VALUES.put(BLOOMFILTER_IMPL, DEFAULT_BLOOMFILTER_IMPL.name());
     DEFAULT_VALUES.put(REPLICATION_SCOPE, String.valueOf(DEFAULT_REPLICATION_SCOPE));
     DEFAULT_VALUES.put(MAX_VERSIONS, String.valueOf(DEFAULT_MAX_VERSIONS));
     DEFAULT_VALUES.put(MIN_VERSIONS, String.valueOf(DEFAULT_MIN_VERSIONS));
@@ -456,6 +466,11 @@ public ColumnFamilyDescriptorBuilder setBloomFilterType(final BloomType value) {
     return this;
   }
 
+  public ColumnFamilyDescriptorBuilder setBloomFilterImpl(final BloomFilterImpl value) {
+    desc.setBloomFilterImpl(value);
+    return this;
+  }
+
   public ColumnFamilyDescriptorBuilder setCacheBloomsOnWrite(boolean value) {
     desc.setCacheBloomsOnWrite(value);
     return this;
@@ -1053,6 +1068,16 @@ public ModifyableColumnFamilyDescriptor setBloomFilterType(final BloomType bt) {
       return setValue(BLOOMFILTER_BYTES, bt.name());
     }
 
+    @Override
+    public BloomFilterImpl getBloomFilterImpl() {
+      return getStringOrDefault(BLOOMFILTER_IMPL_BYTES,
+        n -> BloomFilterImpl.valueOf(n.toUpperCase()), DEFAULT_BLOOMFILTER_IMPL);
+    }
+
+    public ModifyableColumnFamilyDescriptor setBloomFilterImpl(final BloomFilterImpl impl) {
+      return setValue(BLOOMFILTER_IMPL_BYTES, impl.name());
+    }
+
     @Override
     public int getScope() {
       return getStringOrDefault(REPLICATION_SCOPE_BYTES, Integer::valueOf,

hbase-client/src/main/java/org/apache/hadoop/hbase/regionserver/BloomFilterImpl.java

@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import org.apache.yetus.audience.InterfaceAudience;
+
+/**
+ * Specifies the filter implementation to use for probabilistic key lookups in store files.
+ */
+@InterfaceAudience.Public
+public enum BloomFilterImpl {
+  /**
+   * Traditional Bloom filter implementation
+   */
+  BLOOM,
+  /**
+   * Ribbon filter implementation (more space-efficient than Bloom filters)
+   */
+  RIBBON
+}

hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestColumnFamilyDescriptorBuilder.java

@@ -32,6 +32,7 @@
 import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
 import org.apache.hadoop.hbase.io.encoding.IndexBlockEncoding;
+import org.apache.hadoop.hbase.regionserver.BloomFilterImpl;
 import org.apache.hadoop.hbase.regionserver.BloomType;
 import org.apache.hadoop.hbase.testclassification.MiscTests;
 import org.apache.hadoop.hbase.testclassification.SmallTests;
@@ -220,9 +221,11 @@ public void testSetBlocksize() throws HBaseException {
   @Test
   public void testDefaultBuilder() {
     final Map<String, String> defaultValueMap = ColumnFamilyDescriptorBuilder.getDefaultValues();
-    assertEquals(defaultValueMap.size(), 12);
+    assertEquals(defaultValueMap.size(), 13);
     assertEquals(defaultValueMap.get(ColumnFamilyDescriptorBuilder.BLOOMFILTER),
       BloomType.ROW.toString());
+    assertEquals(defaultValueMap.get(ColumnFamilyDescriptorBuilder.BLOOMFILTER_IMPL),
+      BloomFilterImpl.BLOOM.toString());
     assertEquals(defaultValueMap.get(ColumnFamilyDescriptorBuilder.REPLICATION_SCOPE), "0");
     assertEquals(defaultValueMap.get(ColumnFamilyDescriptorBuilder.MAX_VERSIONS), "1");
     assertEquals(defaultValueMap.get(ColumnFamilyDescriptorBuilder.MIN_VERSIONS), "0");

hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CompoundRibbonFilter.java

@@ -0,0 +1,238 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io.hfile;
+
+import java.io.DataInput;
+import java.io.IOException;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.nio.ByteBuff;
+import org.apache.hadoop.hbase.regionserver.BloomType;
+import org.apache.hadoop.hbase.util.BloomFilter;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.ribbon.InterleavedRibbonSolution;
+import org.apache.hadoop.hbase.util.ribbon.RibbonFilterUtil;
+import org.apache.hadoop.hbase.util.ribbon.RibbonHasher;
+import org.apache.yetus.audience.InterfaceAudience;
+
+/**
+ * Reader for compound Ribbon filters stored in HFiles.
+ * <p>
+ * This class provides on-demand loading of Ribbon filter chunks, similar to
+ * {@link CompoundBloomFilter}. Each chunk is loaded from the HFile block cache when needed for a
+ * query.
+ * <p>
+ */
+@InterfaceAudience.Private
+public class CompoundRibbonFilter extends CompoundRibbonFilterBase implements BloomFilter {
+
+  /** HFile reader for loading chunks on demand */
+  private final HFile.Reader reader;
+
+  /** Metrics collector */
+  private final BloomFilterMetrics metrics;
+
+  /** Block index for locating chunks */
+  private final HFileBlockIndex.BlockIndexReader index;
+
+  /** Per-chunk metadata for queries */
+  private final int[] chunkNumSlots;
+
+  /** Per-chunk upperNumColumns for ICML mode */
+  private final int[] chunkUpperNumColumns;
+
+  /** Per-chunk upperStartBlock for ICML mode */
+  private final int[] chunkUpperStartBlock;
+
+  /**
+   * Deserializes a CompoundRibbonFilter from HFile metadata.
+   * @param meta    DataInput positioned at the start of Ribbon filter metadata (after version)
+   * @param reader  HFile reader for loading chunks
+   * @param metrics Metrics collector (may be null)
+   * @throws IOException If an I/O error occurs
+   */
+  public CompoundRibbonFilter(DataInput meta, HFile.Reader reader, BloomFilterMetrics metrics)
+    throws IOException {
+    this.reader = reader;
+    this.metrics = metrics;
+
+    // Read metadata (must match CompoundRibbonFilterWriter.MetaWriter.write())
+    totalByteSize = meta.readLong();
+    bandwidth = meta.readInt();
+    hashType = meta.readInt();
+    overheadRatio = meta.readDouble();
+    totalKeyCount = meta.readLong();
+    totalNumSlots = meta.readLong();
+    numChunks = meta.readInt();
+
+    // Read comparator class name
+    byte[] comparatorClassName = Bytes.readByteArray(meta);
+    if (comparatorClassName.length != 0) {
+      comparator = FixedFileTrailer.createComparator(Bytes.toString(comparatorClassName));
+    }
+
+    // Read per-chunk numSlots array
+    chunkNumSlots = new int[numChunks];
+    for (int i = 0; i < numChunks; i++) {
+      chunkNumSlots[i] = meta.readInt();
+    }
+
+    // Read ICML per-chunk metadata
+    chunkUpperNumColumns = new int[numChunks];
+    chunkUpperStartBlock = new int[numChunks];
+    for (int i = 0; i < numChunks; i++) {
+      chunkUpperNumColumns[i] = meta.readInt();
+      chunkUpperStartBlock[i] = meta.readInt();
+    }
+
+    // Initialize block index reader
+    if (comparator == null) {
+      index = new HFileBlockIndex.ByteArrayKeyBlockIndexReader(1);
+    } else {
+      index = new HFileBlockIndex.CellBasedKeyBlockIndexReader(comparator, 1);
+    }
+    index.readRootIndex(meta, numChunks);
+  }
+
+  @Override
+  public boolean contains(Cell keyCell, ByteBuff bloom, BloomType type) {
+    boolean result = containsInternal(keyCell, type);
+    if (metrics != null) {
+      metrics.incrementRequests(result);
+    }
+    return result;
+  }
+
+  private boolean containsInternal(Cell keyCell, BloomType type) {
+    byte[] key = RibbonFilterUtil.extractKeyFromCell(keyCell, type);
+
+    // Find block using appropriate index type
+    int block;
+    if (comparator != null) {
+      block = index.rootBlockContainingKey(keyCell);
+    } else {
+      block = index.rootBlockContainingKey(key, 0, key.length);
+    }
+
+    return containsInternal(block, key, 0, key.length);
+  }
+
+  @Override
+  public boolean contains(byte[] buf, int offset, int length, ByteBuff bloom) {
+    boolean result = containsInternal(buf, offset, length);
+    if (metrics != null) {
+      metrics.incrementRequests(result);
+    }
+    return result;
+  }
+
+  private boolean containsInternal(byte[] key, int keyOffset, int keyLength) {
+    int block = index.rootBlockContainingKey(key, keyOffset, keyLength);
+    return containsInternal(block, key, keyOffset, keyLength);
+  }
+
+  private boolean containsInternal(int block, byte[] key, int keyOffset, int keyLength) {
+    if (block < 0) {
+      return false;
+    }
+
+    HFileBlock ribbonBlock = loadRibbonBlock(block);
+    try {
+      ByteBuff buf = ribbonBlock.getBufferReadOnly();
+      int headerSize = ribbonBlock.headerSize();
+      int numSlots = getChunkNumSlots(block, ribbonBlock);
+
+      RibbonHasher hasher = new RibbonHasher(numSlots, bandwidth, hashType);
+
+      RibbonHasher.RibbonHashResult hashResult = hasher.hash(key, keyOffset, keyLength);
+
+      return InterleavedRibbonSolution.contains(hashResult.start(), hashResult.coeffRow(),
+        hashResult.resultRow(), buf, headerSize, numSlots, chunkUpperNumColumns[block],
+        chunkUpperStartBlock[block]);
+    } finally {
+      ribbonBlock.release();
+    }
+  }
+
+  /**
+   * Loads a Ribbon filter block from the HFile.
+   * @param blockIndex Index of the block to load
+   * @return The loaded HFile block containing Ribbon filter data
+   */
+  private HFileBlock loadRibbonBlock(int blockIndex) {
+    try {
+      return reader.readBlock(index.getRootBlockOffset(blockIndex),
+        index.getRootBlockDataSize(blockIndex), true, // cacheBlock
+        true, // pread
+        false, // isCompaction
+        true, // updateCacheMetrics
+        BlockType.BLOOM_CHUNK, null // expectedDataBlockEncoding
+      );
+    } catch (IOException e) {
+      throw new IllegalArgumentException("Failed to load Ribbon block", e);
+    }
+  }
+
+  /**
+   * Gets the number of slots for a chunk.
+   * <p>
+   * With byte-based format (1 byte per slot), the relationship is: dataSize = numSlots bytes.
+   * <p>
+   * Uses precomputed chunkNumSlots array from metadata if available, otherwise uses block size
+   * directly.
+   * @param blockIndex Index of the block
+   * @param block      The HFile block (used as fallback if metadata unavailable)
+   * @return Number of slots in this chunk
+   */
+  private int getChunkNumSlots(int blockIndex, HFileBlock block) {
+    // Use precomputed value if available
+    if (blockIndex < chunkNumSlots.length && chunkNumSlots[blockIndex] > 0) {
+      return chunkNumSlots[blockIndex];
+    }
+
+    // Fallback: dataSize equals numSlots (1 byte per slot)
+    return block.getUncompressedSizeWithoutHeader();
+  }
+
+  @Override
+  public boolean supportsAutoLoading() {
+    return true;
+  }
+
+  @Override
+  public String toString() {
+    StringBuilder sb = new StringBuilder();
+    sb.append("CompoundRibbonFilter");
+    sb.append(RibbonFilterUtil.STATS_RECORD_SEP);
+    sb.append("Keys: ").append(totalKeyCount);
+    sb.append(RibbonFilterUtil.STATS_RECORD_SEP);
+    sb.append("Slots: ").append(totalNumSlots);
+    sb.append(RibbonFilterUtil.STATS_RECORD_SEP);
+    sb.append("Chunks: ").append(numChunks);
+    sb.append(RibbonFilterUtil.STATS_RECORD_SEP);
+    sb.append("Bandwidth: ").append(bandwidth);
+    sb.append(RibbonFilterUtil.STATS_RECORD_SEP);
+    sb.append("Byte size: ").append(totalByteSize);
+    sb.append(RibbonFilterUtil.STATS_RECORD_SEP);
+    sb.append("Overhead: ").append(String.format("%.2f%%", overheadRatio * 100));
+    if (comparator != null) {
+      sb.append(RibbonFilterUtil.STATS_RECORD_SEP);
+      sb.append("Comparator: ").append(comparator.getClass().getSimpleName());
+    }
+    return sb.toString();
+  }
+}