From 6c0d679c7e74c0ac281f35285c7cbae4b41989ae Mon Sep 17 00:00:00 2001 From: Konstantin Date: Fri, 24 Jul 2026 15:55:10 +0200 Subject: [PATCH 1/3] Add scale_offset array->array codec Implements the zarr-extensions scale_offset codec: encode applies (in - offset) * scale, decode applies (in / scale) + offset, in the input data type's arithmetic. Integer arithmetic is exact with a representability check at each step (out-of-range or non-exact division is an error); floating-point uses native float/double ops. Supports the 10 real-number data types this library models. The data type is unchanged; only the fill value is transformed and propagated downstream. Registered in the v3 CodecRegistry and exposed via CodecBuilder.withScaleOffset(...). Tests in the new dev.zarr.zarrjava.codec test package. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../zarr/zarrjava/v3/codec/CodecBuilder.java | 10 + .../zarr/zarrjava/v3/codec/CodecRegistry.java | 1 + .../v3/codec/core/ScaleOffsetCodec.java | 379 ++++++++++++++++++ .../zarrjava/codec/ScaleOffsetCodecTest.java | 123 ++++++ 4 files changed, 513 insertions(+) create mode 100644 src/main/java/dev/zarr/zarrjava/v3/codec/core/ScaleOffsetCodec.java create mode 100644 src/test/java/dev/zarr/zarrjava/codec/ScaleOffsetCodecTest.java diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java b/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java index 5c3487ce..5c0018f3 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java @@ -64,6 +64,16 @@ public CodecBuilder withTranspose(int[] order) { return this; } + public CodecBuilder withScaleOffset(Object offset, Object scale) { + codecs.add(new ScaleOffsetCodec(new ScaleOffsetCodec.Configuration(offset, scale))); + return this; + } + + public CodecBuilder withScaleOffset() { + codecs.add(new ScaleOffsetCodec(null)); + return this; + } + public CodecBuilder withBytes(Endian endian) { if (dataType.getByteCount() <= 1) codecs.add(new BytesCodec()); diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecRegistry.java b/src/main/java/dev/zarr/zarrjava/v3/codec/CodecRegistry.java index ad249e08..341961bf 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecRegistry.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/CodecRegistry.java @@ -18,6 +18,7 @@ public class CodecRegistry { addType("zstd", ZstdCodec.class); addType("crc32c", Crc32cCodec.class); addType("sharding_indexed", ShardingIndexedCodec.class); + addType("scale_offset", ScaleOffsetCodec.class); } public static void addType(String name, Class codecClass) { diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/ScaleOffsetCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/ScaleOffsetCodec.java new file mode 100644 index 00000000..227e5b46 --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/ScaleOffsetCodec.java @@ -0,0 +1,379 @@ +package dev.zarr.zarrjava.v3.codec.core; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.core.ArrayMetadata.CoreArrayMetadata; +import dev.zarr.zarrjava.core.codec.ArrayArrayCodec; +import dev.zarr.zarrjava.v3.ArrayMetadata; +import dev.zarr.zarrjava.v3.DataType; +import dev.zarr.zarrjava.v3.codec.Codec; +import ucar.ma2.Array; +import ucar.ma2.IndexIterator; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.math.BigInteger; + +/** + * The {@code scale_offset} codec applies the affine transformation {@code (in - offset) * scale} to + * every array element on encode, and inverts it with {@code (in / scale) + offset} on decode. It is + * an {@code array -> array} codec: it does not change the data type or shape, it only rescales the + * stored values. It is typically followed by a narrowing codec (e.g. {@code cast_value}) that + * converts the rescaled array to a smaller data type to achieve (lossy) compression. + * + *

The arithmetic is performed using the semantics of the input array's data type. For integral + * data types the computation is exact: if any intermediate or final value is not representable in + * the data type (e.g. an unsigned subtraction going negative, an overflow, or a non-exact division + * on decode), the codec fails with a {@link ZarrException}. For floating-point data types the + * computation uses native {@code float}/{@code double} arithmetic, whose results are always + * representable (including {@code NaN} and {@code +-Infinity}). + * + *

The {@code offset} and {@code scale} configuration values are scalars encoded with the input + * array's data type using the Zarr V3 fill value encoding. A missing {@code offset} defaults to the + * additive identity (0); a missing {@code scale} defaults to the multiplicative identity (1). When + * both are absent the codec is a no-op. + * + *

Supported data types are the real-number types this library models: {@code int8/16/32/64}, + * {@code uint8/16/32/64}, {@code float32} and {@code float64}. Other data types from the codec + * specification (e.g. {@code float8_*}, {@code bfloat16}, {@code int2}) are not modelled here. + */ +public class ScaleOffsetCodec extends ArrayArrayCodec implements Codec { + + @JsonIgnore + @Nonnull + public final String name = "scale_offset"; + @Nullable + public final Configuration configuration; + + @JsonCreator + public ScaleOffsetCodec( + @Nullable @JsonProperty(value = "configuration") Configuration configuration + ) { + this.configuration = configuration; + } + + // ===== Codec pipeline integration ======================================================== + + @Override + public Array encode(Array chunkArray) throws ZarrException { + return transform(chunkArray, true); + } + + @Override + public Array decode(Array chunkArray) throws ZarrException { + return transform(chunkArray, false); + } + + @Override + public long computeEncodedSize(long inputByteLength, ArrayMetadata.CoreArrayMetadata arrayMetadata) + throws ZarrException { + // The data type and shape are unchanged, so the encoded chunk has the same byte length. + return inputByteLength; + } + + @Override + public CoreArrayMetadata resolveArrayMetadata() throws ZarrException { + super.resolveArrayMetadata(); + DataType type = arrayDataType(); + requireSupported(type); + // The data type stays the same; only the fill value is transformed (encode direction) so that + // fill-value-aware downstream codecs stay aligned with the rescaled data. + Object transformedFillValue = transformFillValue(arrayMetadata.parsedFillValue, type); + return new CoreArrayMetadata( + arrayMetadata.shape, arrayMetadata.chunkShape, type, transformedFillValue); + } + + private DataType arrayDataType() throws ZarrException { + if (!(arrayMetadata.dataType instanceof DataType)) { + throw new ZarrException("The scale_offset codec requires a Zarr v3 data type."); + } + return (DataType) arrayMetadata.dataType; + } + + // ===== Element transformation ============================================================ + + private Array transform(Array input, boolean encode) throws ZarrException { + DataType type = arrayDataType(); + requireSupported(type); + int[] shape = input.getShape(); + Array output = Array.factory(type.getMA2DataType(), shape); + IndexIterator in = input.getIndexIterator(); + IndexIterator out = output.getIndexIterator(); + + if (type == DataType.FLOAT32) { + float offset = floatParam(offsetConfig(), 0.0f); + float scale = floatParam(scaleConfig(), 1.0f); + while (in.hasNext()) { + float x = in.getFloatNext(); + out.setFloatNext(encode ? (x - offset) * scale : (x / scale) + offset); + } + } else if (type == DataType.FLOAT64) { + double offset = doubleParam(offsetConfig(), 0.0); + double scale = doubleParam(scaleConfig(), 1.0); + while (in.hasNext()) { + double x = in.getDoubleNext(); + out.setDoubleNext(encode ? (x - offset) * scale : (x / scale) + offset); + } + } else { + BigInteger offset = intParam(offsetConfig(), BigInteger.ZERO, type); + BigInteger scale = intParam(scaleConfig(), BigInteger.ONE, type); + BigInteger min = integerMin(type); + BigInteger max = integerMax(type); + while (in.hasNext()) { + BigInteger x = readInt(in, type); + BigInteger r = encode + ? encodeInt(x, offset, scale, type, min, max) + : decodeInt(x, offset, scale, type, min, max); + writeInt(out, type, r); + } + } + return output; + } + + private Object transformFillValue(Object fillValue, DataType type) throws ZarrException { + if (fillValue == null) { + return null; + } + if (type == DataType.FLOAT32) { + float offset = floatParam(offsetConfig(), 0.0f); + float scale = floatParam(scaleConfig(), 1.0f); + float x = ((Number) fillValue).floatValue(); + return (x - offset) * scale; + } + if (type == DataType.FLOAT64) { + double offset = doubleParam(offsetConfig(), 0.0); + double scale = doubleParam(scaleConfig(), 1.0); + double x = ((Number) fillValue).doubleValue(); + return (x - offset) * scale; + } + BigInteger offset = intParam(offsetConfig(), BigInteger.ZERO, type); + BigInteger scale = intParam(scaleConfig(), BigInteger.ONE, type); + BigInteger r = encodeInt(toBigInteger(fillValue, type), offset, scale, type, + integerMin(type), integerMax(type)); + return boxInt(r, type); + } + + // ===== Integer arithmetic (exact, with representability checks) ========================== + + private static BigInteger encodeInt(BigInteger x, BigInteger offset, BigInteger scale, + DataType type, BigInteger min, BigInteger max) + throws ZarrException { + BigInteger shifted = x.subtract(offset); + requireInRange(shifted, min, max, type, "intermediate value (in - offset)"); + BigInteger scaled = shifted.multiply(scale); + requireInRange(scaled, min, max, type, "result (in - offset) * scale"); + return scaled; + } + + private static BigInteger decodeInt(BigInteger x, BigInteger offset, BigInteger scale, + DataType type, BigInteger min, BigInteger max) + throws ZarrException { + if (scale.signum() == 0) { + throw new ZarrException("The scale_offset codec cannot decode with a scale of 0."); + } + BigInteger[] quotientRemainder = x.divideAndRemainder(scale); + if (quotientRemainder[1].signum() != 0) { + throw new ZarrException( + "The scale_offset codec cannot decode the value " + x + " because it is not exactly " + + "divisible by the scale " + scale + " in the '" + type.getValue() + "' data type."); + } + BigInteger divided = quotientRemainder[0]; + requireInRange(divided, min, max, type, "intermediate value (in / scale)"); + BigInteger result = divided.add(offset); + requireInRange(result, min, max, type, "result (in / scale) + offset"); + return result; + } + + private static void requireInRange(BigInteger value, BigInteger min, BigInteger max, + DataType type, String label) throws ZarrException { + if (value.compareTo(min) < 0 || value.compareTo(max) > 0) { + throw new ZarrException( + "The scale_offset " + label + " (" + value + ") is not representable in the '" + + type.getValue() + "' data type."); + } + } + + // ===== Configuration parameter parsing =================================================== + + @Nullable + private Object offsetConfig() { + return configuration == null ? null : configuration.offset; + } + + @Nullable + private Object scaleConfig() { + return configuration == null ? null : configuration.scale; + } + + private static float floatParam(@Nullable Object raw, float identity) throws ZarrException { + if (raw == null) { + return identity; + } + return ((Number) ArrayMetadata.parseFillValue(raw, DataType.FLOAT32)).floatValue(); + } + + private static double doubleParam(@Nullable Object raw, double identity) throws ZarrException { + if (raw == null) { + return identity; + } + return ((Number) ArrayMetadata.parseFillValue(raw, DataType.FLOAT64)).doubleValue(); + } + + private static BigInteger intParam(@Nullable Object raw, BigInteger identity, DataType type) + throws ZarrException { + if (raw == null) { + return identity; + } + return toBigInteger(ArrayMetadata.parseFillValue(raw, type), type); + } + + // ===== Data type facts and element reading/writing ======================================= + + private static void requireSupported(DataType type) throws ZarrException { + if (type == DataType.BOOL) { + throw new ZarrException( + "The scale_offset codec does not support the data type '" + type.getValue() + + "'. Supported types are the integral and floating-point real-number types."); + } + } + + private static BigInteger readInt(IndexIterator it, DataType type) { + switch (type) { + case INT8: + return BigInteger.valueOf(it.getByteNext()); + case UINT8: + return BigInteger.valueOf(it.getByteNext() & 0xFFL); + case INT16: + return BigInteger.valueOf(it.getShortNext()); + case UINT16: + return BigInteger.valueOf(it.getShortNext() & 0xFFFFL); + case INT32: + return BigInteger.valueOf(it.getIntNext()); + case UINT32: + return BigInteger.valueOf(it.getIntNext() & 0xFFFFFFFFL); + case INT64: + return BigInteger.valueOf(it.getLongNext()); + case UINT64: + return new BigInteger(Long.toUnsignedString(it.getLongNext())); + default: + throw new IllegalStateException("Unsupported scale_offset data type: " + type); + } + } + + private static void writeInt(IndexIterator it, DataType type, BigInteger value) { + switch (type) { + case INT8: + case UINT8: + it.setByteNext(value.byteValue()); + break; + case INT16: + case UINT16: + it.setShortNext(value.shortValue()); + break; + case INT32: + case UINT32: + it.setIntNext(value.intValue()); + break; + case INT64: + case UINT64: + it.setLongNext(value.longValue()); + break; + default: + throw new IllegalStateException("Unsupported scale_offset data type: " + type); + } + } + + private static BigInteger toBigInteger(Object boxed, DataType type) { + Number number = (Number) boxed; + switch (type) { + case INT8: + return BigInteger.valueOf(number.byteValue()); + case UINT8: + return BigInteger.valueOf(number.longValue() & 0xFFL); + case INT16: + return BigInteger.valueOf(number.shortValue()); + case UINT16: + return BigInteger.valueOf(number.longValue() & 0xFFFFL); + case INT32: + return BigInteger.valueOf(number.intValue()); + case UINT32: + return BigInteger.valueOf(number.longValue() & 0xFFFFFFFFL); + case INT64: + return BigInteger.valueOf(number.longValue()); + case UINT64: + return new BigInteger(Long.toUnsignedString(number.longValue())); + default: + throw new IllegalStateException("Unsupported scale_offset data type: " + type); + } + } + + private static Object boxInt(BigInteger value, DataType type) { + switch (type) { + case INT8: + case UINT8: + return value.byteValue(); + case INT16: + case UINT16: + return value.shortValue(); + case INT32: + case UINT32: + return value.intValue(); + case INT64: + case UINT64: + return value.longValue(); + default: + throw new IllegalStateException("Unsupported scale_offset data type: " + type); + } + } + + private static int integerBits(DataType type) { + return type.getByteCount() * 8; + } + + private static boolean isUnsigned(DataType type) { + return type == DataType.UINT8 || type == DataType.UINT16 || type == DataType.UINT32 + || type == DataType.UINT64; + } + + private static BigInteger integerMin(DataType type) { + if (isUnsigned(type)) { + return BigInteger.ZERO; + } + return BigInteger.ONE.shiftLeft(integerBits(type) - 1).negate(); + } + + private static BigInteger integerMax(DataType type) { + if (isUnsigned(type)) { + return BigInteger.ONE.shiftLeft(integerBits(type)).subtract(BigInteger.ONE); + } + return BigInteger.ONE.shiftLeft(integerBits(type) - 1).subtract(BigInteger.ONE); + } + + // ===== Configuration ===================================================================== + + public static final class Configuration { + + /** The offset subtracted on encode, as a JSON scalar in the input array's data type. */ + @Nullable + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonProperty("offset") + public final Object offset; + + /** The scale multiplied on encode, as a JSON scalar in the input array's data type. */ + @Nullable + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonProperty("scale") + public final Object scale; + + @JsonCreator + public Configuration( + @Nullable @JsonProperty("offset") Object offset, + @Nullable @JsonProperty("scale") Object scale) { + this.offset = offset; + this.scale = scale; + } + } +} diff --git a/src/test/java/dev/zarr/zarrjava/codec/ScaleOffsetCodecTest.java b/src/test/java/dev/zarr/zarrjava/codec/ScaleOffsetCodecTest.java new file mode 100644 index 00000000..359df11f --- /dev/null +++ b/src/test/java/dev/zarr/zarrjava/codec/ScaleOffsetCodecTest.java @@ -0,0 +1,123 @@ +package dev.zarr.zarrjava.codec; + +import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.ZarrTest; +import dev.zarr.zarrjava.store.FilesystemStore; +import dev.zarr.zarrjava.store.StoreHandle; +import dev.zarr.zarrjava.v3.Array; +import dev.zarr.zarrjava.v3.ArrayMetadata; +import dev.zarr.zarrjava.v3.ArrayMetadataBuilder; +import dev.zarr.zarrjava.v3.DataType; +import dev.zarr.zarrjava.v3.codec.core.ScaleOffsetCodec; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import ucar.ma2.MAMath; + +import java.io.IOException; + +import static dev.zarr.zarrjava.core.ArrayMetadata.parseFillValue; +import static dev.zarr.zarrjava.utils.Utils.toLongArray; +import static org.junit.Assert.assertThrows; + +public class ScaleOffsetCodecTest extends ZarrTest { + + private static ScaleOffsetCodec scaleOffsetCodec(Object offset, Object scale, DataType dataType, + Object fillValue, int[] shape) throws ZarrException { + ScaleOffsetCodec codec = new ScaleOffsetCodec(new ScaleOffsetCodec.Configuration(offset, scale)); + codec.setCoreArrayMetadata(new ArrayMetadata.CoreArrayMetadata( + toLongArray(shape), shape, dataType, parseFillValue(fillValue, dataType))); + return codec; + } + + @Test + public void testScaleOffsetCodecFloat() throws ZarrException { + // scale 0.5 is exactly representable in float32, so the round-trip is lossless here. + ucar.ma2.Array in = ucar.ma2.Array.factory(ucar.ma2.DataType.FLOAT, new int[]{4}, + new float[]{4.0f, 5.0f, 6.0f, 8.0f}); + // (x - 5) * 0.5 + ucar.ma2.Array encoded = ucar.ma2.Array.factory(ucar.ma2.DataType.FLOAT, new int[]{4}, + new float[]{-0.5f, 0.0f, 0.5f, 1.5f}); + + ScaleOffsetCodec codec = scaleOffsetCodec(5, 0.5, DataType.FLOAT32, null, new int[]{4}); + assert MAMath.equals(encoded, codec.encode(in.copy())); + assert MAMath.equals(in, codec.decode(encoded.copy())); + } + + @Test + public void testScaleOffsetCodecUintOffsetOnly() throws ZarrException { + // Range reduction: subtract 1000, leaving values that fit in a byte. scale defaults to 1. + ucar.ma2.Array in = ucar.ma2.Array.factory(ucar.ma2.DataType.USHORT, new int[]{4}, + new short[]{1000, 1001, 1050, 1255}); + ucar.ma2.Array encoded = ucar.ma2.Array.factory(ucar.ma2.DataType.USHORT, new int[]{4}, + new short[]{0, 1, 50, 255}); + + ScaleOffsetCodec codec = scaleOffsetCodec(1000, null, DataType.UINT16, null, new int[]{4}); + assert MAMath.equals(encoded, codec.encode(in.copy())); + assert MAMath.equals(in, codec.decode(encoded.copy())); + } + + @Test + public void testScaleOffsetCodecNoOp() throws ZarrException { + ucar.ma2.Array in = ucar.ma2.Array.factory(ucar.ma2.DataType.INT, new int[]{3}, + new int[]{-7, 0, 42}); + ScaleOffsetCodec codec = new ScaleOffsetCodec(null); + codec.setCoreArrayMetadata(new ArrayMetadata.CoreArrayMetadata( + new long[]{3}, new int[]{3}, DataType.INT32, null)); + assert MAMath.equals(in, codec.encode(in.copy())); + assert MAMath.equals(in, codec.decode(in.copy())); + } + + @Test + public void testScaleOffsetCodecFillValueTransform() throws ZarrException { + // The fill value is transformed with the encode formula and reported downstream. + ScaleOffsetCodec codec = scaleOffsetCodec(5, 0.5, DataType.FLOAT32, 5.0f, new int[]{4}); + Object resolvedFill = codec.resolveArrayMetadata().parsedFillValue; + Assertions.assertEquals(0.0f, resolvedFill); + + ScaleOffsetCodec uintCodec = scaleOffsetCodec(1000, null, DataType.UINT16, 1000, new int[]{4}); + Assertions.assertEquals((short) 0, uintCodec.resolveArrayMetadata().parsedFillValue); + } + + @Test + public void testScaleOffsetCodecIntegerOutOfRangeIsError() throws ZarrException { + // 500 - 1000 = -500 is not representable in uint16 -> hard error (no numpy-style wraparound). + ucar.ma2.Array in = ucar.ma2.Array.factory(ucar.ma2.DataType.USHORT, new int[]{1}, + new short[]{500}); + ScaleOffsetCodec codec = scaleOffsetCodec(1000, null, DataType.UINT16, null, new int[]{1}); + assertThrows(ZarrException.class, () -> codec.encode(in)); + } + + @Test + public void testScaleOffsetCodecNonExactDivisionIsError() throws ZarrException { + // Decoding requires in / scale to be an exact integer for integral data types. + ucar.ma2.Array stored = ucar.ma2.Array.factory(ucar.ma2.DataType.INT, new int[]{1}, + new int[]{5}); + ScaleOffsetCodec codec = scaleOffsetCodec(0, 10, DataType.INT32, null, new int[]{1}); + assertThrows(ZarrException.class, () -> codec.decode(stored)); + } + + @Test + public void testScaleOffsetCodecReadWrite() throws IOException, ZarrException { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testScaleOffsetCodecReadWrite"); + float[] values = new float[16 * 16 * 16]; + for (int i = 0; i < values.length; i++) { + // multiples of 0.5, all exactly representable and exactly recoverable with scale 0.5 + values[i] = (i % 32) * 0.5f + 3.0f; + } + ucar.ma2.Array testData = ucar.ma2.Array.factory(ucar.ma2.DataType.FLOAT, + new int[]{16, 16, 16}, values); + + ArrayMetadataBuilder builder = Array.metadataBuilder() + .withShape(16, 16, 16) + .withDataType(DataType.FLOAT32) + .withChunkShape(4, 8, 16) + .withFillValue(3.0f) + .withCodecs(c -> c.withScaleOffset(3.0, 0.5)); + Array writeArray = Array.create(storeHandle, builder.build()); + writeArray.write(testData); + + Array readArray = Array.open(storeHandle); + ucar.ma2.Array result = readArray.read(); + assert MAMath.equals(testData, result); + } +} From 77b448da285d65f093ba26f385ed5c97b74ec886 Mon Sep 17 00:00:00 2001 From: Konstantin Date: Tue, 4 Aug 2026 15:29:55 +0200 Subject: [PATCH 2/3] add dropped lines --- src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java b/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java index 8d8547f9..4e7c567d 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java @@ -71,6 +71,9 @@ public CodecBuilder withScaleOffset(Object offset, Object scale) { public CodecBuilder withScaleOffset() { codecs.add(new ScaleOffsetCodec(null)); + return this; + } + /** * Adds a {@code reshape} codec. Each entry of {@code shape} must be a positive {@link Integer}, the * special value {@code -1} (at most once), or an {@code int[]} / array of input dimension indices. From 7b8820806a49f940b4623e5b3ca206a137ae2f38 Mon Sep 17 00:00:00 2001 From: Norman Rzepka Date: Fri, 4 Sep 2026 13:57:38 +0200 Subject: [PATCH 3/3] Address review findings on the scale_offset codec - Do not fail the codec pipeline when the fill value is not representable after the encode transformation. The fill value is metadata, not stored data, so an unsigned array with a fill value below the offset (e.g. uint16 with fill_value 0 and offset 1000) previously could not be created or opened at all. It is now kept untransformed in that case. - Reject a scale of 0 when the configuration is parsed. Encoding with a scale of 0 mapped every value to 0 without an error, while decoding either threw or produced NaN, so the data was unrecoverable. - Use long arithmetic with Math.*Exact and explicit bound checks for every integral type except uint64, instead of allocating several BigIntegers per array element, and skip the element-by-element rebuild entirely when neither offset nor scale is configured. Co-Authored-By: Claude Opus 5 --- .../v3/codec/core/ScaleOffsetCodec.java | 160 +++++++++++++++++- 1 file changed, 153 insertions(+), 7 deletions(-) diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/ScaleOffsetCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/ScaleOffsetCodec.java index 227e5b46..dccb82ff 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/ScaleOffsetCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/ScaleOffsetCodec.java @@ -79,6 +79,7 @@ public CoreArrayMetadata resolveArrayMetadata() throws ZarrException { super.resolveArrayMetadata(); DataType type = arrayDataType(); requireSupported(type); + requireNonZeroScale(type); // The data type stays the same; only the fill value is transformed (encode direction) so that // fill-value-aware downstream codecs stay aligned with the rescaled data. Object transformedFillValue = transformFillValue(arrayMetadata.parsedFillValue, type); @@ -98,6 +99,11 @@ private DataType arrayDataType() throws ZarrException { private Array transform(Array input, boolean encode) throws ZarrException { DataType type = arrayDataType(); requireSupported(type); + requireNonZeroScale(type); + if (offsetConfig() == null && scaleConfig() == null) { + // Identity transformation: avoid rebuilding the array element by element. + return input; + } int[] shape = input.getShape(); Array output = Array.factory(type.getMA2DataType(), shape); IndexIterator in = input.getIndexIterator(); @@ -117,7 +123,8 @@ private Array transform(Array input, boolean encode) throws ZarrException { double x = in.getDoubleNext(); out.setDoubleNext(encode ? (x - offset) * scale : (x / scale) + offset); } - } else { + } else if (type == DataType.UINT64) { + // uint64 is the only supported type whose values do not fit into a long. BigInteger offset = intParam(offsetConfig(), BigInteger.ZERO, type); BigInteger scale = intParam(scaleConfig(), BigInteger.ONE, type); BigInteger min = integerMin(type); @@ -129,6 +136,18 @@ private Array transform(Array input, boolean encode) throws ZarrException { : decodeInt(x, offset, scale, type, min, max); writeInt(out, type, r); } + } else { + long offset = intParam(offsetConfig(), BigInteger.ZERO, type).longValueExact(); + long scale = intParam(scaleConfig(), BigInteger.ONE, type).longValueExact(); + long min = integerMin(type).longValueExact(); + long max = integerMax(type).longValueExact(); + while (in.hasNext()) { + long x = readLong(in, type); + long r = encode + ? encodeLong(x, offset, scale, type, min, max) + : decodeLong(x, offset, scale, type, min, max); + writeLong(out, type, r); + } } return output; } @@ -151,9 +170,16 @@ private Object transformFillValue(Object fillValue, DataType type) throws ZarrEx } BigInteger offset = intParam(offsetConfig(), BigInteger.ZERO, type); BigInteger scale = intParam(scaleConfig(), BigInteger.ONE, type); - BigInteger r = encodeInt(toBigInteger(fillValue, type), offset, scale, type, - integerMin(type), integerMax(type)); - return boxInt(r, type); + try { + BigInteger r = encodeInt(toBigInteger(fillValue, type), offset, scale, type, + integerMin(type), integerMax(type)); + return boxInt(r, type); + } catch (ZarrException e) { + // The fill value is metadata, not stored data. If it is not representable after the + // encode transformation (e.g. an unsigned fill value below the offset), keep it + // untransformed instead of making the array impossible to create or open. + return fillValue; + } } // ===== Integer arithmetic (exact, with representability checks) ========================== @@ -187,12 +213,70 @@ private static BigInteger decodeInt(BigInteger x, BigInteger offset, BigInteger return result; } + private static long encodeLong(long x, long offset, long scale, DataType type, long min, + long max) throws ZarrException { + long shifted; + try { + shifted = Math.subtractExact(x, offset); + } catch (ArithmeticException e) { + throw outOfRange(BigInteger.valueOf(x).subtract(BigInteger.valueOf(offset)), type, + "intermediate value (in - offset)"); + } + requireInRange(shifted, min, max, type, "intermediate value (in - offset)"); + long scaled; + try { + scaled = Math.multiplyExact(shifted, scale); + } catch (ArithmeticException e) { + throw outOfRange(BigInteger.valueOf(shifted).multiply(BigInteger.valueOf(scale)), type, + "result (in - offset) * scale"); + } + requireInRange(scaled, min, max, type, "result (in - offset) * scale"); + return scaled; + } + + private static long decodeLong(long x, long offset, long scale, DataType type, long min, + long max) throws ZarrException { + if (scale == 0) { + throw new ZarrException("The scale_offset codec cannot decode with a scale of 0."); + } + if (x % scale != 0) { + throw new ZarrException( + "The scale_offset codec cannot decode the value " + x + " because it is not exactly " + + "divisible by the scale " + scale + " in the '" + type.getValue() + "' data type."); + } + if (x == Long.MIN_VALUE && scale == -1) { + throw outOfRange(BigInteger.valueOf(x).negate(), type, "intermediate value (in / scale)"); + } + long divided = x / scale; + requireInRange(divided, min, max, type, "intermediate value (in / scale)"); + long result; + try { + result = Math.addExact(divided, offset); + } catch (ArithmeticException e) { + throw outOfRange(BigInteger.valueOf(divided).add(BigInteger.valueOf(offset)), type, + "result (in / scale) + offset"); + } + requireInRange(result, min, max, type, "result (in / scale) + offset"); + return result; + } + + private static void requireInRange(long value, long min, long max, DataType type, String label) + throws ZarrException { + if (value < min || value > max) { + throw outOfRange(BigInteger.valueOf(value), type, label); + } + } + + private static ZarrException outOfRange(BigInteger value, DataType type, String label) { + return new ZarrException( + "The scale_offset " + label + " (" + value + ") is not representable in the '" + + type.getValue() + "' data type."); + } + private static void requireInRange(BigInteger value, BigInteger min, BigInteger max, DataType type, String label) throws ZarrException { if (value.compareTo(min) < 0 || value.compareTo(max) > 0) { - throw new ZarrException( - "The scale_offset " + label + " (" + value + ") is not representable in the '" - + type.getValue() + "' data type."); + throw outOfRange(value, type, label); } } @@ -232,6 +316,25 @@ private static BigInteger intParam(@Nullable Object raw, BigInteger identity, Da // ===== Data type facts and element reading/writing ======================================= + private void requireNonZeroScale(DataType type) throws ZarrException { + if (scaleConfig() == null) { + return; + } + boolean zero; + if (type == DataType.FLOAT32) { + zero = floatParam(scaleConfig(), 1.0f) == 0.0f; + } else if (type == DataType.FLOAT64) { + zero = doubleParam(scaleConfig(), 1.0) == 0.0; + } else { + zero = intParam(scaleConfig(), BigInteger.ONE, type).signum() == 0; + } + if (zero) { + throw new ZarrException( + "The scale_offset codec requires a non-zero scale, because a scale of 0 maps every " + + "value to 0 and cannot be inverted on decode."); + } + } + private static void requireSupported(DataType type) throws ZarrException { if (type == DataType.BOOL) { throw new ZarrException( @@ -240,6 +343,49 @@ private static void requireSupported(DataType type) throws ZarrException { } } + private static long readLong(IndexIterator it, DataType type) { + switch (type) { + case INT8: + return it.getByteNext(); + case UINT8: + return it.getByteNext() & 0xFFL; + case INT16: + return it.getShortNext(); + case UINT16: + return it.getShortNext() & 0xFFFFL; + case INT32: + return it.getIntNext(); + case UINT32: + return it.getIntNext() & 0xFFFFFFFFL; + case INT64: + return it.getLongNext(); + default: + throw new IllegalStateException("Unsupported scale_offset data type: " + type); + } + } + + private static void writeLong(IndexIterator it, DataType type, long value) { + switch (type) { + case INT8: + case UINT8: + it.setByteNext((byte) value); + break; + case INT16: + case UINT16: + it.setShortNext((short) value); + break; + case INT32: + case UINT32: + it.setIntNext((int) value); + break; + case INT64: + it.setLongNext(value); + break; + default: + throw new IllegalStateException("Unsupported scale_offset data type: " + type); + } + } + private static BigInteger readInt(IndexIterator it, DataType type) { switch (type) { case INT8: