refactor(xor): remove persisted item counts

src/main/java/org/apache/datasketches/common/Family.java

@@ -167,7 +167,7 @@ public enum Family {
   /**
    * Xor Filter
    */
-  XORFILTER(22, "XORFILTER", 3, 3);
+  XORFILTER(22, "XORFILTER", 2, 2);
 
   private static final Map<Integer, Family> lookupID = new HashMap<>();
   private static final Map<String, Family> lookupFamName = new HashMap<>();

src/main/java/org/apache/datasketches/filters/xorfilter/XorFilter.java

@@ -61,7 +61,7 @@
  */
 public final class XorFilter implements MemorySegmentStatus {
   private static final int SER_VER = 1;
-  private static final int PREAMBLE_LONGS = 3;
+  private static final int PREAMBLE_LONGS = 2;
   private static final long PREAMBLE_SIZE_BYTES = (long) PREAMBLE_LONGS * Long.BYTES;
 
   // number of hash functions / segments used by the three-partite construction
@@ -84,7 +84,6 @@ public final class XorFilter implements MemorySegmentStatus {
 
   private final int bitsPerFingerprint_;    // fingerprint width in bits, 8 or 16
   private final int segmentLength_;          // number of fingerprint slots per hash segment
-  private final int numKeys_;                // number of distinct keys used to build the filter
   private final long seed_;                  // winning construction seed for the murmur mix
   private final MemorySegment fpSeg_;        // holds the raw fingerprint payload
   private final MemorySegment wseg_;         // used only when wrapping a serialized image
@@ -100,7 +99,6 @@ public final class XorFilter implements MemorySegmentStatus {
    */
   XorFilter(final int bitsPerFingerprint, final long seed, final long[] keys, final int numKeys) {
     bitsPerFingerprint_ = bitsPerFingerprint;
-    numKeys_ = numKeys;
 
     final int capacity = computeCapacity(numKeys);
     segmentLength_ = capacity / NUM_HASHES;
@@ -142,11 +140,10 @@ public final class XorFilter implements MemorySegmentStatus {
   }
 
   // Constructor used with internalHeapifyOrWrap()
-  XorFilter(final int bitsPerFingerprint, final int segmentLength, final int numKeys, final long seed,
+  XorFilter(final int bitsPerFingerprint, final int segmentLength, final long seed,
       final MemorySegment fpSeg, final MemorySegment wseg) {
     bitsPerFingerprint_ = bitsPerFingerprint;
     segmentLength_ = segmentLength;
-    numKeys_ = numKeys;
     seed_ = seed;
     fpSeg_ = fpSeg;
     wseg_ = wseg;
@@ -179,7 +176,6 @@ private static XorFilter internalHeapifyOrWrap(final MemorySegment seg, final bo
     final int preLongs = posSeg.getByte();
     final int serVer = posSeg.getByte();
     final int familyID = posSeg.getByte();
-    posSeg.getByte(); // flags, reserved
 
     checkArgument((preLongs < Family.XORFILTER.getMinPreLongs()) || (preLongs > Family.XORFILTER.getMaxPreLongs()),
         "Possible corruption: Incorrect number of preamble bytes specified in header");
@@ -189,44 +185,25 @@ private static XorFilter internalHeapifyOrWrap(final MemorySegment seg, final bo
     final int bitsPerFingerprint = posSeg.getByte();
     checkArgument((bitsPerFingerprint != 8) && (bitsPerFingerprint != 16),
         "Possible corruption: Fingerprint width must be 8 or 16. Found: " + bitsPerFingerprint);
-    final int numHashes = posSeg.getByte();
-    checkArgument(numHashes != NUM_HASHES, "Possible corruption: Number of hash functions must be " + NUM_HASHES
-        + ". Found: " + numHashes);
-    posSeg.getShort(); // unused
-
-    final long seed = posSeg.getLong();
     final int segmentLength = posSeg.getInt();
-    final int numKeys = posSeg.getInt();
     checkArgument(segmentLength < 1, "Possible corruption: Segment length must be strictly positive. Found: " + segmentLength);
-    checkArgument(numKeys < 0, "Possible corruption: Number of keys must be non-negative. Found: " + numKeys);
+    final long seed = posSeg.getLong();
 
-    final long fingerprintBytes = (long) numHashes * segmentLength * (bitsPerFingerprint >>> 3);
+    final long fingerprintBytes = (long) NUM_HASHES * segmentLength * (bitsPerFingerprint >>> 3);
     checkArgument(seg.byteSize() < (PREAMBLE_SIZE_BYTES + fingerprintBytes),
         "Possible corruption: MemorySegment capacity insufficient to hold the fingerprint array");
 
     final MemorySegment payload = seg.asSlice(PREAMBLE_SIZE_BYTES, fingerprintBytes);
     if (isWrap) {
-      return new XorFilter(bitsPerFingerprint, segmentLength, numKeys, seed, payload, seg);
+      return new XorFilter(bitsPerFingerprint, segmentLength, seed, payload, seg);
     }
     // heapify copies the payload onto the Java heap
     checkArgument(fingerprintBytes > Integer.MAX_VALUE, "Filter is too large to heapify; use wrap() instead");
     final byte[] fingerprints = new byte[(int) fingerprintBytes];
     MemorySegment.copy(payload, JAVA_BYTE, 0, fingerprints, 0, (int) fingerprintBytes);
-    return new XorFilter(bitsPerFingerprint, segmentLength, numKeys, seed, MemorySegment.ofArray(fingerprints), null);
+    return new XorFilter(bitsPerFingerprint, segmentLength, seed, MemorySegment.ofArray(fingerprints), null);
   }
 
-  /**
-   * Checks if the XorFilter was built from any items.
-   * @return True if the XorFilter holds no items, otherwise False
-   */
-  public boolean isEmpty() { return numKeys_ == 0; }
-
-  /**
-   * Returns the number of distinct items used to build this XorFilter.
-   * @return The number of distinct items in this filter
-   */
-  public int getNumItems() { return numKeys_; }
-
   /**
    * Returns the fixed number of hash functions used by the three-partite xor construction.
    * @return The number of hash functions used by this filter
@@ -251,16 +228,6 @@ private static XorFilter internalHeapifyOrWrap(final MemorySegment seg, final bo
    */
   public long getSeed() { return seed_; }
 
-  /**
-   * Returns the average number of bits stored per item by this XorFilter, a measure of its
-   * space efficiency. This is approximately 1.23 times the fingerprint width.
-   * @return The number of bits used per item, or 0 for an empty filter
-   */
-  public double getBitsPerItem() {
-    if (numKeys_ == 0) { return 0.0; }
-    return ((double) getCapacity() * bitsPerFingerprint_) / numKeys_;
-  }
-
   @Override
   public boolean hasMemorySegment() { return wseg_ != null; }
 
@@ -597,23 +564,19 @@ public long getSerializedSizeBytes() {
   }
 
 /*
- * An Xor Filter's serialized image always uses 3 longs of preamble, as the filter is
- * immutable and therefore has no empty or growable state:
+ * An Xor Filter's serialized image uses 2 longs of preamble:
  *
  * <pre>
  * Long || Start Byte Adr:
  * Adr:
  *      ||       0        |    1   |    2   |    3   |    4   |    5   |    6   |    7   |
- *  0   || Preamble_Longs | SerVer | FamID  |  Flags | BitsPFP|NumHash |-----Unused------|
+ *  0   || Preamble_Longs | SerVer | FamID  |BitsPFP |--------Segment Length (int)------|
  *
  *      ||       8        |    9   |   10   |   11   |   12   |   13   |   14   |   15   |
  *  1   ||-------------------------------Construction Seed-------------------------------|
- *
- *      ||      16        |   17   |   18   |   19   |   20   |   21   |   22   |   23   |
- *  2   ||----------Segment Length (int)-------------|-----------Num Keys (int)----------|
  *  </pre>
  *
- * The raw fingerprint array starts at byte 24.
+ * The raw fingerprint array starts at byte 16. The layout always has three segments.
  */
 
   /**
@@ -654,13 +617,9 @@ private void writeToSegment(final MemorySegment seg) {
     posSeg.setByte((byte) PREAMBLE_LONGS);
     posSeg.setByte((byte) SER_VER);
     posSeg.setByte((byte) Family.XORFILTER.getID());
-    posSeg.setByte((byte) 0); // flags
     posSeg.setByte((byte) bitsPerFingerprint_);
-    posSeg.setByte((byte) NUM_HASHES);
-    posSeg.setShort((short) 0); // unused
-    posSeg.setLong(seed_);
     posSeg.setInt(segmentLength_);
-    posSeg.setInt(numKeys_);
+    posSeg.setLong(seed_);
     MemorySegment.copy(fpSeg_, JAVA_BYTE, 0, seg, JAVA_BYTE, PREAMBLE_SIZE_BYTES, fpSeg_.byteSize());
   }
 
@@ -676,12 +635,10 @@ public String toString() {
     sb.append(LS);
     final String thisSimpleName = this.getClass().getSimpleName();
     sb.append("### ").append(thisSimpleName).append(" SUMMARY: ").append(LS);
-    sb.append("   numItems     : ").append(numKeys_).append(LS);
     sb.append("   numHashes    : ").append(NUM_HASHES).append(LS);
     sb.append("   bitsPerFP    : ").append(bitsPerFingerprint_).append(LS);
     sb.append("   capacity     : ").append(getCapacity()).append(LS);
     sb.append("   seed         : ").append(seed_).append(LS);
-    sb.append("   bits/item    : ").append(getBitsPerItem()).append(LS);
     sb.append("### END SKETCH SUMMARY").append(LS);
 
     return sb.toString();

src/test/java/org/apache/datasketches/filters/xorfilter/XorFilterBenchmark.java

@@ -90,7 +90,7 @@ private static Result measure(final int numKeys, final int bits) {
     final Result r = new Result();
     r.buildNanosPerKey = (double) buildElapsed / numKeys;
     r.queryNanosPerOp = (double) queryElapsed / numKeys;
-    r.bitsPerItem = filter.getBitsPerItem();
+    r.bitsPerItem = (double) filter.getCapacity() * bits / numKeys;
     r.observedFpp = (double) falsePositives / numKeys;
     return r;
   }

src/test/java/org/apache/datasketches/filters/xorfilter/XorFilterTest.java

@@ -26,6 +26,7 @@
 
 import java.lang.foreign.Arena;
 import java.lang.foreign.MemorySegment;
+import java.util.Arrays;
 import java.util.HexFormat;
 
 import org.apache.datasketches.common.SketchesArgumentException;
@@ -137,6 +138,7 @@ public void duplicateItemsTest() {
     for (int i = 0; i < 1000; ++i) {
       builder.update(i % 100); // only 100 distinct values, each added 10 times
     }
+    assertEquals(builder.getNumItems(), 1000L);
     final XorFilter filter = builder.build();
     for (int i = 0; i < 100; ++i) {
       assertTrue(filter.query(i));
@@ -148,8 +150,6 @@ public void duplicateItemsTest() {
   @Test
   public void emptyFilterTest() {
     final XorFilter filter = new XorFilterBuilder(8, 7L).build();
-    assertTrue(filter.isEmpty());
-    assertEquals(filter.getNumItems(), 0);
 
     long positives = 0;
     for (long i = 0; i < 100_000; ++i) {
@@ -187,7 +187,6 @@ public void heapifyRoundTripTest() {
 
     final XorFilter rebuilt = XorFilter.heapify(MemorySegment.ofArray(bytes));
     assertEquals(rebuilt.getBitsPerFingerprint(), filter.getBitsPerFingerprint());
-    assertEquals(rebuilt.getNumItems(), filter.getNumItems());
     assertEquals(rebuilt.getSeed(), filter.getSeed());
     for (long i = 0; i < numItems; ++i) {
       assertTrue(rebuilt.query(i));
@@ -378,43 +377,25 @@ public void smallSetsTest() {
         builder.update(i);
       }
       final XorFilter filter = builder.build();
-      assertEquals(filter.getNumItems(), n);
       for (long i = 0; i < n; ++i) {
         assertTrue(filter.query(i));
       }
     }
   }
 
-  // the accessors report the expected metadata
+  // the serialized footprint stays near ten bits per input for an 8-bit filter
   @Test
-  public void gettersTest() {
+  public void spaceUsageTest() {
     final int numItems = 10_000;
     final XorFilterBuilder builder = new XorFilterBuilder(8, 2L);
     for (long i = 0; i < numItems; ++i) {
       builder.update(i);
     }
     final XorFilter filter = builder.build();
-    assertEquals(filter.getNumHashes(), 3);
-    assertEquals(filter.getBitsPerFingerprint(), 8);
-    assertEquals(filter.getNumItems(), numItems);
-    final long capacity = filter.getCapacity();
-    assertEquals(capacity, 3L * (capacity / 3)); // a multiple of the segment count
-    assertTrue(capacity >= numItems);            // roughly 1.23 times the item count
-    final double bitsPerItem = filter.getBitsPerItem();
+    final double bitsPerItem = (double) filter.getSerializedSizeBytes() * Byte.SIZE / numItems;
     assertTrue((bitsPerItem > 9.5) && (bitsPerItem < 10.5), "unexpected bits/item: " + bitsPerItem);
   }
 
-  // the builder counts every item added while the filter counts only the distinct ones
-  @Test
-  public void builderNumItemsTest() {
-    final XorFilterBuilder builder = new XorFilterBuilder(8, 4L);
-    for (int i = 0; i < 1000; ++i) {
-      builder.update(i % 100);
-    }
-    assertEquals(builder.getNumItems(), 1000L);
-    assertEquals(builder.build().getNumItems(), 100);
-  }
-
   // 16-bit construction must also be deterministic for a fixed seed and item set
   @Test
   public void determinism16Test() {
@@ -452,29 +433,42 @@ public void heapifyCorruptionBranchesTest() {
     }
     final byte[] valid = builder.build().toByteArray();
 
+    assertThrows(SketchesArgumentException.class, () -> XorFilter.heapify(corrupt(valid, 0, (byte) 3)));  // preamble
     assertThrows(SketchesArgumentException.class, () -> XorFilter.heapify(corrupt(valid, 1, (byte) 99))); // serVer
     assertThrows(SketchesArgumentException.class, () -> XorFilter.heapify(corrupt(valid, 2, (byte) 7)));  // familyID
-    assertThrows(SketchesArgumentException.class, () -> XorFilter.heapify(corrupt(valid, 4, (byte) 5)));  // bitsPerFP
-    assertThrows(SketchesArgumentException.class, () -> XorFilter.heapify(corrupt(valid, 5, (byte) 4)));  // numHashes
-    assertThrows(SketchesArgumentException.class, () -> XorFilter.heapify(MemorySegment.ofArray(new byte[16])));
+    assertThrows(SketchesArgumentException.class, () -> XorFilter.heapify(corrupt(valid, 3, (byte) 5)));  // bitsPerFP
+    assertThrows(SketchesArgumentException.class, () -> XorFilter.heapify(corrupt(valid, 7, (byte) 0x80))); // negative length
+    for (int length = 0; length < valid.length; ++length) {
+      final MemorySegment truncated = MemorySegment.ofArray(Arrays.copyOf(valid, length));
+      assertThrows(SketchesArgumentException.class, () -> XorFilter.heapify(truncated));
+    }
   }
 
-  // Images built by datasketches-rust for the same items and base seed. Both libraries peel through
-  // one queue per segment, so their images must be identical.
+  // Fingerprint payloads generated by datasketches-rust for the same items and base seed.
+  // Both libraries peel through one queue per segment.
   private static final String REFERENCE_XOR8_IMAGE =
-      "0301160008030000956eeb2f2632d7bd0c00000004000000"
+      "020116080c000000956eeb2f2632d7bd"
       + "000000000000000400dd0000000000000000ce0000c50000000000000000000000000000";
   private static final String REFERENCE_XOR16_IMAGE =
-      "0301160010030000956eeb2f2632d7bd0c00000004000000"
+      "020116100c000000956eeb2f2632d7bd"
       + "000000000000000000000000000004e80000dd6e00000000000000000000000000000000"
       + "ce9500000000c5b600000000000000000000000000000000000000000000000000000000";
 
   // the peeling order determines the fingerprint payload, not just the header
   @Test
   public void referenceImplementationImageTest() {
     final long[] items = { 1, 2, 3, Long.MIN_VALUE };
-    assertEquals(XorFilterBuilder.create(items, 8, 42L).toByteArray(), HexFormat.of().parseHex(REFERENCE_XOR8_IMAGE));
-    assertEquals(XorFilterBuilder.create(items, 16, 42L).toByteArray(), HexFormat.of().parseHex(REFERENCE_XOR16_IMAGE));
+    for (final int bits : new int[] { 8, 16 }) {
+      final byte[] image = HexFormat.of().parseHex(bits == 8 ? REFERENCE_XOR8_IMAGE : REFERENCE_XOR16_IMAGE);
+      assertEquals(XorFilterBuilder.create(items, bits, 42L).toByteArray(), image);
+      final MemorySegment seg = MemorySegment.ofArray(image);
+      for (final XorFilter restored : new XorFilter[] { XorFilter.heapify(seg), XorFilter.wrap(seg) }) {
+        assertEquals(restored.toByteArray(), image);
+        for (final long item : items) {
+          assertTrue(restored.query(item));
+        }
+      }
+    }
   }
 
   // returns a copy of the image with a single byte replaced