From a038a8b298e3580da73f4c8ee9d0c8e33f77a607 Mon Sep 17 00:00:00 2001 From: tison Date: Thu, 5 Nov 2026 23:22:06 +0800 Subject: [PATCH 1/2] refactor(xor): remove persisted item counts --- .../apache/datasketches/common/Family.java | 2 +- .../filters/xorfilter/XorFilter.java | 65 ++++--------------- .../filters/xorfilter/XorFilterBenchmark.java | 2 +- .../filters/xorfilter/XorFilterTest.java | 62 ++++++++---------- 4 files changed, 41 insertions(+), 90 deletions(-) diff --git a/src/main/java/org/apache/datasketches/common/Family.java b/src/main/java/org/apache/datasketches/common/Family.java index fef730d2a..b8d90d586 100644 --- a/src/main/java/org/apache/datasketches/common/Family.java +++ b/src/main/java/org/apache/datasketches/common/Family.java @@ -167,7 +167,7 @@ public enum Family { /** * Xor Filter */ - XORFILTER(22, "XORFILTER", 3, 3); + XORFILTER(22, "XORFILTER", 2, 2); private static final Map lookupID = new HashMap<>(); private static final Map lookupFamName = new HashMap<>(); diff --git a/src/main/java/org/apache/datasketches/filters/xorfilter/XorFilter.java b/src/main/java/org/apache/datasketches/filters/xorfilter/XorFilter.java index ae0923f5d..11daeb9a4 100644 --- a/src/main/java/org/apache/datasketches/filters/xorfilter/XorFilter.java +++ b/src/main/java/org/apache/datasketches/filters/xorfilter/XorFilter.java @@ -60,8 +60,8 @@ * ACM Journal of Experimental Algorithmics, 2020.

*/ public final class XorFilter implements MemorySegmentStatus { - private static final int SER_VER = 1; - private static final int PREAMBLE_LONGS = 3; + private static final int SER_VER = 2; + private static final int PREAMBLE_LONGS = 2; private static final long PREAMBLE_SIZE_BYTES = (long) PREAMBLE_LONGS * Long.BYTES; // number of hash functions / segments used by the three-partite construction @@ -84,7 +84,6 @@ public final class XorFilter implements MemorySegmentStatus { private final int bitsPerFingerprint_; // fingerprint width in bits, 8 or 16 private final int segmentLength_; // number of fingerprint slots per hash segment - private final int numKeys_; // number of distinct keys used to build the filter private final long seed_; // winning construction seed for the murmur mix private final MemorySegment fpSeg_; // holds the raw fingerprint payload private final MemorySegment wseg_; // used only when wrapping a serialized image @@ -100,7 +99,6 @@ public final class XorFilter implements MemorySegmentStatus { */ XorFilter(final int bitsPerFingerprint, final long seed, final long[] keys, final int numKeys) { bitsPerFingerprint_ = bitsPerFingerprint; - numKeys_ = numKeys; final int capacity = computeCapacity(numKeys); segmentLength_ = capacity / NUM_HASHES; @@ -142,11 +140,10 @@ public final class XorFilter implements MemorySegmentStatus { } // Constructor used with internalHeapifyOrWrap() - XorFilter(final int bitsPerFingerprint, final int segmentLength, final int numKeys, final long seed, + XorFilter(final int bitsPerFingerprint, final int segmentLength, final long seed, final MemorySegment fpSeg, final MemorySegment wseg) { bitsPerFingerprint_ = bitsPerFingerprint; segmentLength_ = segmentLength; - numKeys_ = numKeys; seed_ = seed; fpSeg_ = fpSeg; wseg_ = wseg; @@ -179,7 +176,6 @@ private static XorFilter internalHeapifyOrWrap(final MemorySegment seg, final bo final int preLongs = posSeg.getByte(); final int serVer = posSeg.getByte(); final int familyID = posSeg.getByte(); - posSeg.getByte(); // flags, reserved checkArgument((preLongs < Family.XORFILTER.getMinPreLongs()) || (preLongs > Family.XORFILTER.getMaxPreLongs()), "Possible corruption: Incorrect number of preamble bytes specified in header"); @@ -189,44 +185,25 @@ private static XorFilter internalHeapifyOrWrap(final MemorySegment seg, final bo final int bitsPerFingerprint = posSeg.getByte(); checkArgument((bitsPerFingerprint != 8) && (bitsPerFingerprint != 16), "Possible corruption: Fingerprint width must be 8 or 16. Found: " + bitsPerFingerprint); - final int numHashes = posSeg.getByte(); - checkArgument(numHashes != NUM_HASHES, "Possible corruption: Number of hash functions must be " + NUM_HASHES - + ". Found: " + numHashes); - posSeg.getShort(); // unused - - final long seed = posSeg.getLong(); final int segmentLength = posSeg.getInt(); - final int numKeys = posSeg.getInt(); checkArgument(segmentLength < 1, "Possible corruption: Segment length must be strictly positive. Found: " + segmentLength); - checkArgument(numKeys < 0, "Possible corruption: Number of keys must be non-negative. Found: " + numKeys); + final long seed = posSeg.getLong(); - final long fingerprintBytes = (long) numHashes * segmentLength * (bitsPerFingerprint >>> 3); + final long fingerprintBytes = (long) NUM_HASHES * segmentLength * (bitsPerFingerprint >>> 3); checkArgument(seg.byteSize() < (PREAMBLE_SIZE_BYTES + fingerprintBytes), "Possible corruption: MemorySegment capacity insufficient to hold the fingerprint array"); final MemorySegment payload = seg.asSlice(PREAMBLE_SIZE_BYTES, fingerprintBytes); if (isWrap) { - return new XorFilter(bitsPerFingerprint, segmentLength, numKeys, seed, payload, seg); + return new XorFilter(bitsPerFingerprint, segmentLength, seed, payload, seg); } // heapify copies the payload onto the Java heap checkArgument(fingerprintBytes > Integer.MAX_VALUE, "Filter is too large to heapify; use wrap() instead"); final byte[] fingerprints = new byte[(int) fingerprintBytes]; MemorySegment.copy(payload, JAVA_BYTE, 0, fingerprints, 0, (int) fingerprintBytes); - return new XorFilter(bitsPerFingerprint, segmentLength, numKeys, seed, MemorySegment.ofArray(fingerprints), null); + return new XorFilter(bitsPerFingerprint, segmentLength, seed, MemorySegment.ofArray(fingerprints), null); } - /** - * Checks if the XorFilter was built from any items. - * @return True if the XorFilter holds no items, otherwise False - */ - public boolean isEmpty() { return numKeys_ == 0; } - - /** - * Returns the number of distinct items used to build this XorFilter. - * @return The number of distinct items in this filter - */ - public int getNumItems() { return numKeys_; } - /** * Returns the fixed number of hash functions used by the three-partite xor construction. * @return The number of hash functions used by this filter @@ -251,16 +228,6 @@ private static XorFilter internalHeapifyOrWrap(final MemorySegment seg, final bo */ public long getSeed() { return seed_; } - /** - * Returns the average number of bits stored per item by this XorFilter, a measure of its - * space efficiency. This is approximately 1.23 times the fingerprint width. - * @return The number of bits used per item, or 0 for an empty filter - */ - public double getBitsPerItem() { - if (numKeys_ == 0) { return 0.0; } - return ((double) getCapacity() * bitsPerFingerprint_) / numKeys_; - } - @Override public boolean hasMemorySegment() { return wseg_ != null; } @@ -597,23 +564,19 @@ public long getSerializedSizeBytes() { } /* - * An Xor Filter's serialized image always uses 3 longs of preamble, as the filter is - * immutable and therefore has no empty or growable state: + * An Xor Filter's version 2 serialized image uses 2 longs of preamble: * * 
  * Long || Start Byte Adr:
  * Adr:
  *      ||       0        |    1   |    2   |    3   |    4   |    5   |    6   |    7   |
- *  0   || Preamble_Longs | SerVer | FamID  |  Flags | BitsPFP|NumHash |-----Unused------|
+ *  0   || Preamble_Longs | SerVer | FamID  |BitsPFP |--------Segment Length (int)------|
  *
  *      ||       8        |    9   |   10   |   11   |   12   |   13   |   14   |   15   |
  *  1   ||-------------------------------Construction Seed-------------------------------|
- *
- *      ||      16        |   17   |   18   |   19   |   20   |   21   |   22   |   23   |
- *  2   ||----------Segment Length (int)-------------|-----------Num Keys (int)----------|
  *
* - * The raw fingerprint array starts at byte 24. + * The raw fingerprint array starts at byte 16. The layout always has three segments. */ /** @@ -654,13 +617,9 @@ private void writeToSegment(final MemorySegment seg) { posSeg.setByte((byte) PREAMBLE_LONGS); posSeg.setByte((byte) SER_VER); posSeg.setByte((byte) Family.XORFILTER.getID()); - posSeg.setByte((byte) 0); // flags posSeg.setByte((byte) bitsPerFingerprint_); - posSeg.setByte((byte) NUM_HASHES); - posSeg.setShort((short) 0); // unused - posSeg.setLong(seed_); posSeg.setInt(segmentLength_); - posSeg.setInt(numKeys_); + posSeg.setLong(seed_); MemorySegment.copy(fpSeg_, JAVA_BYTE, 0, seg, JAVA_BYTE, PREAMBLE_SIZE_BYTES, fpSeg_.byteSize()); } @@ -676,12 +635,10 @@ public String toString() { sb.append(LS); final String thisSimpleName = this.getClass().getSimpleName(); sb.append("### ").append(thisSimpleName).append(" SUMMARY: ").append(LS); - sb.append(" numItems : ").append(numKeys_).append(LS); sb.append(" numHashes : ").append(NUM_HASHES).append(LS); sb.append(" bitsPerFP : ").append(bitsPerFingerprint_).append(LS); sb.append(" capacity : ").append(getCapacity()).append(LS); sb.append(" seed : ").append(seed_).append(LS); - sb.append(" bits/item : ").append(getBitsPerItem()).append(LS); sb.append("### END SKETCH SUMMARY").append(LS); return sb.toString(); diff --git a/src/test/java/org/apache/datasketches/filters/xorfilter/XorFilterBenchmark.java b/src/test/java/org/apache/datasketches/filters/xorfilter/XorFilterBenchmark.java index af6ae0122..1846d744c 100644 --- a/src/test/java/org/apache/datasketches/filters/xorfilter/XorFilterBenchmark.java +++ b/src/test/java/org/apache/datasketches/filters/xorfilter/XorFilterBenchmark.java @@ -90,7 +90,7 @@ private static Result measure(final int numKeys, final int bits) { final Result r = new Result(); r.buildNanosPerKey = (double) buildElapsed / numKeys; r.queryNanosPerOp = (double) queryElapsed / numKeys; - r.bitsPerItem = filter.getBitsPerItem(); + r.bitsPerItem = (double) filter.getCapacity() * bits / numKeys; r.observedFpp = (double) falsePositives / numKeys; return r; } diff --git a/src/test/java/org/apache/datasketches/filters/xorfilter/XorFilterTest.java b/src/test/java/org/apache/datasketches/filters/xorfilter/XorFilterTest.java index 453393eb8..d89adf86f 100644 --- a/src/test/java/org/apache/datasketches/filters/xorfilter/XorFilterTest.java +++ b/src/test/java/org/apache/datasketches/filters/xorfilter/XorFilterTest.java @@ -26,6 +26,7 @@ import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; +import java.util.Arrays; import java.util.HexFormat; import org.apache.datasketches.common.SketchesArgumentException; @@ -137,6 +138,7 @@ public void duplicateItemsTest() { for (int i = 0; i < 1000; ++i) { builder.update(i % 100); // only 100 distinct values, each added 10 times } + assertEquals(builder.getNumItems(), 1000L); final XorFilter filter = builder.build(); for (int i = 0; i < 100; ++i) { assertTrue(filter.query(i)); @@ -148,8 +150,6 @@ public void duplicateItemsTest() { @Test public void emptyFilterTest() { final XorFilter filter = new XorFilterBuilder(8, 7L).build(); - assertTrue(filter.isEmpty()); - assertEquals(filter.getNumItems(), 0); long positives = 0; for (long i = 0; i < 100_000; ++i) { @@ -187,7 +187,6 @@ public void heapifyRoundTripTest() { final XorFilter rebuilt = XorFilter.heapify(MemorySegment.ofArray(bytes)); assertEquals(rebuilt.getBitsPerFingerprint(), filter.getBitsPerFingerprint()); - assertEquals(rebuilt.getNumItems(), filter.getNumItems()); assertEquals(rebuilt.getSeed(), filter.getSeed()); for (long i = 0; i < numItems; ++i) { assertTrue(rebuilt.query(i)); @@ -378,43 +377,25 @@ public void smallSetsTest() { builder.update(i); } final XorFilter filter = builder.build(); - assertEquals(filter.getNumItems(), n); for (long i = 0; i < n; ++i) { assertTrue(filter.query(i)); } } } - // the accessors report the expected metadata + // the serialized footprint stays near ten bits per input for an 8-bit filter @Test - public void gettersTest() { + public void spaceUsageTest() { final int numItems = 10_000; final XorFilterBuilder builder = new XorFilterBuilder(8, 2L); for (long i = 0; i < numItems; ++i) { builder.update(i); } final XorFilter filter = builder.build(); - assertEquals(filter.getNumHashes(), 3); - assertEquals(filter.getBitsPerFingerprint(), 8); - assertEquals(filter.getNumItems(), numItems); - final long capacity = filter.getCapacity(); - assertEquals(capacity, 3L * (capacity / 3)); // a multiple of the segment count - assertTrue(capacity >= numItems); // roughly 1.23 times the item count - final double bitsPerItem = filter.getBitsPerItem(); + final double bitsPerItem = (double) filter.getSerializedSizeBytes() * Byte.SIZE / numItems; assertTrue((bitsPerItem > 9.5) && (bitsPerItem < 10.5), "unexpected bits/item: " + bitsPerItem); } - // the builder counts every item added while the filter counts only the distinct ones - @Test - public void builderNumItemsTest() { - final XorFilterBuilder builder = new XorFilterBuilder(8, 4L); - for (int i = 0; i < 1000; ++i) { - builder.update(i % 100); - } - assertEquals(builder.getNumItems(), 1000L); - assertEquals(builder.build().getNumItems(), 100); - } - // 16-bit construction must also be deterministic for a fixed seed and item set @Test public void determinism16Test() { @@ -452,20 +433,24 @@ public void heapifyCorruptionBranchesTest() { } final byte[] valid = builder.build().toByteArray(); - assertThrows(SketchesArgumentException.class, () -> XorFilter.heapify(corrupt(valid, 1, (byte) 99))); // serVer + assertThrows(SketchesArgumentException.class, () -> XorFilter.heapify(corrupt(valid, 0, (byte) 3))); // preamble + assertThrows(SketchesArgumentException.class, () -> XorFilter.heapify(corrupt(valid, 1, (byte) 1))); // old serVer assertThrows(SketchesArgumentException.class, () -> XorFilter.heapify(corrupt(valid, 2, (byte) 7))); // familyID - assertThrows(SketchesArgumentException.class, () -> XorFilter.heapify(corrupt(valid, 4, (byte) 5))); // bitsPerFP - assertThrows(SketchesArgumentException.class, () -> XorFilter.heapify(corrupt(valid, 5, (byte) 4))); // numHashes - assertThrows(SketchesArgumentException.class, () -> XorFilter.heapify(MemorySegment.ofArray(new byte[16]))); + assertThrows(SketchesArgumentException.class, () -> XorFilter.heapify(corrupt(valid, 3, (byte) 5))); // bitsPerFP + assertThrows(SketchesArgumentException.class, () -> XorFilter.heapify(corrupt(valid, 7, (byte) 0x80))); // negative length + for (int length = 0; length < valid.length; ++length) { + final MemorySegment truncated = MemorySegment.ofArray(Arrays.copyOf(valid, length)); + assertThrows(SketchesArgumentException.class, () -> XorFilter.heapify(truncated)); + } } - // Images built by datasketches-rust for the same items and base seed. Both libraries peel through - // one queue per segment, so their images must be identical. + // Version 2 headers with fingerprint payloads generated by datasketches-rust for the same items + // and base seed. Both libraries peel through one queue per segment. private static final String REFERENCE_XOR8_IMAGE = - "0301160008030000956eeb2f2632d7bd0c00000004000000" + "020216080c000000956eeb2f2632d7bd" + "000000000000000400dd0000000000000000ce0000c50000000000000000000000000000"; private static final String REFERENCE_XOR16_IMAGE = - "0301160010030000956eeb2f2632d7bd0c00000004000000" + "020216100c000000956eeb2f2632d7bd" + "000000000000000000000000000004e80000dd6e00000000000000000000000000000000" + "ce9500000000c5b600000000000000000000000000000000000000000000000000000000"; @@ -473,8 +458,17 @@ public void heapifyCorruptionBranchesTest() { @Test public void referenceImplementationImageTest() { final long[] items = { 1, 2, 3, Long.MIN_VALUE }; - assertEquals(XorFilterBuilder.create(items, 8, 42L).toByteArray(), HexFormat.of().parseHex(REFERENCE_XOR8_IMAGE)); - assertEquals(XorFilterBuilder.create(items, 16, 42L).toByteArray(), HexFormat.of().parseHex(REFERENCE_XOR16_IMAGE)); + for (final int bits : new int[] { 8, 16 }) { + final byte[] image = HexFormat.of().parseHex(bits == 8 ? REFERENCE_XOR8_IMAGE : REFERENCE_XOR16_IMAGE); + assertEquals(XorFilterBuilder.create(items, bits, 42L).toByteArray(), image); + final MemorySegment seg = MemorySegment.ofArray(image); + for (final XorFilter restored : new XorFilter[] { XorFilter.heapify(seg), XorFilter.wrap(seg) }) { + assertEquals(restored.toByteArray(), image); + for (final long item : items) { + assertTrue(restored.query(item)); + } + } + } } // returns a copy of the image with a single byte replaced From c179c5bc00d5aadaed8c68ce11d14d812abe08d6 Mon Sep 17 00:00:00 2001 From: tison Date: Thu, 5 Nov 2026 23:42:08 +0800 Subject: [PATCH 2/2] fix(xor): retain the unreleased serialization version --- .../datasketches/filters/xorfilter/XorFilter.java | 4 ++-- .../datasketches/filters/xorfilter/XorFilterTest.java | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/apache/datasketches/filters/xorfilter/XorFilter.java b/src/main/java/org/apache/datasketches/filters/xorfilter/XorFilter.java index 11daeb9a4..e00d9e71d 100644 --- a/src/main/java/org/apache/datasketches/filters/xorfilter/XorFilter.java +++ b/src/main/java/org/apache/datasketches/filters/xorfilter/XorFilter.java @@ -60,7 +60,7 @@ * ACM Journal of Experimental Algorithmics, 2020.

*/ public final class XorFilter implements MemorySegmentStatus { - private static final int SER_VER = 2; + private static final int SER_VER = 1; private static final int PREAMBLE_LONGS = 2; private static final long PREAMBLE_SIZE_BYTES = (long) PREAMBLE_LONGS * Long.BYTES; @@ -564,7 +564,7 @@ public long getSerializedSizeBytes() { } /* - * An Xor Filter's version 2 serialized image uses 2 longs of preamble: + * An Xor Filter's serialized image uses 2 longs of preamble: * * 
  * Long || Start Byte Adr:
diff --git a/src/test/java/org/apache/datasketches/filters/xorfilter/XorFilterTest.java b/src/test/java/org/apache/datasketches/filters/xorfilter/XorFilterTest.java
index d89adf86f..4fe5f11ad 100644
--- a/src/test/java/org/apache/datasketches/filters/xorfilter/XorFilterTest.java
+++ b/src/test/java/org/apache/datasketches/filters/xorfilter/XorFilterTest.java
@@ -434,7 +434,7 @@ public void heapifyCorruptionBranchesTest() {
     final byte[] valid = builder.build().toByteArray();
 
     assertThrows(SketchesArgumentException.class, () -> XorFilter.heapify(corrupt(valid, 0, (byte) 3)));  // preamble
-    assertThrows(SketchesArgumentException.class, () -> XorFilter.heapify(corrupt(valid, 1, (byte) 1)));  // old serVer
+    assertThrows(SketchesArgumentException.class, () -> XorFilter.heapify(corrupt(valid, 1, (byte) 99))); // serVer
     assertThrows(SketchesArgumentException.class, () -> XorFilter.heapify(corrupt(valid, 2, (byte) 7)));  // familyID
     assertThrows(SketchesArgumentException.class, () -> XorFilter.heapify(corrupt(valid, 3, (byte) 5)));  // bitsPerFP
     assertThrows(SketchesArgumentException.class, () -> XorFilter.heapify(corrupt(valid, 7, (byte) 0x80))); // negative length
@@ -444,13 +444,13 @@ public void heapifyCorruptionBranchesTest() {
     }
   }
 
-  // Version 2 headers with fingerprint payloads generated by datasketches-rust for the same items
-  // and base seed. Both libraries peel through one queue per segment.
+  // Fingerprint payloads generated by datasketches-rust for the same items and base seed.
+  // Both libraries peel through one queue per segment.
   private static final String REFERENCE_XOR8_IMAGE =
-      "020216080c000000956eeb2f2632d7bd"
+      "020116080c000000956eeb2f2632d7bd"
       + "000000000000000400dd0000000000000000ce0000c50000000000000000000000000000";
   private static final String REFERENCE_XOR16_IMAGE =
-      "020216100c000000956eeb2f2632d7bd"
+      "020116100c000000956eeb2f2632d7bd"
       + "000000000000000000000000000004e80000dd6e00000000000000000000000000000000"
       + "ce9500000000c5b600000000000000000000000000000000000000000000000000000000";