From 0a33ae5009b56ff87002be295fcb46e4655e26cb Mon Sep 17 00:00:00 2001 From: Konstantin Date: Thu, 13 Aug 2026 16:59:32 +0200 Subject: [PATCH] implemented zfp codec --- USERGUIDE.md | 15 +- pom.xml | 6 + .../zarr/zarrjava/v3/codec/CodecBuilder.java | 60 +++ .../zarr/zarrjava/v3/codec/CodecRegistry.java | 1 + .../zarr/zarrjava/v3/codec/core/ZfpCodec.java | 479 ++++++++++++++++++ .../java/dev/zarr/zarrjava/ZarrV3Test.java | 239 +++++++++ 6 files changed, 799 insertions(+), 1 deletion(-) create mode 100644 src/main/java/dev/zarr/zarrjava/v3/codec/core/ZfpCodec.java diff --git a/USERGUIDE.md b/USERGUIDE.md index 65bf9944..8916feee 100644 --- a/USERGUIDE.md +++ b/USERGUIDE.md @@ -20,7 +20,7 @@ zarr-java is a Java implementation of the [Zarr specification](https://zarr.dev/ ### Key Features - **Full Zarr v2 and v3 support**: Read and write arrays in both formats - **Multiple storage backends**: Filesystem, HTTP, S3, ZIP, and in-memory storage -- **Compression codecs**: Blosc, Gzip, Zstd, and more +- **Compression codecs**: Blosc, Gzip, Zstd, Zfp, and more - **Sharding support**: Efficient storage for many small chunks (v3) - **Parallel I/O**: Optional parallel reading and writing for performance - **Type-safe API**: Strong typing with covariant return types @@ -448,6 +448,19 @@ Array array = Array.create( ```java .withCodecs(c -> c.withZstd(3)) // Level 1-22 ``` +#### Zfp Compression +Compresses numerical chunks with [zfp](https://zfp.io), lossless or with a chosen error bound: +```java +.withCodecs(c -> c.withZfpReversible()) // Lossless +.withCodecs(c -> c.withZfpFixedAccuracy(0.05)) // Absolute error bound +.withCodecs(c -> c.withZfpFixedRate(8)) // Compressed bits per value +.withCodecs(c -> c.withZfpFixedPrecision(19)) // Bit planes retained +.withCodecs(c -> c.withZfpExpert(1, 13, 19, -2)) // zfp's expert mode parameters +``` +Zfp replaces the `bytes` codec, so it cannot be combined with it. Chunks may have at most four +dimensions, and `bool` is not supported. Data types narrower than 32 bits are promoted to `int32`; +`uint32` and `uint64` values beyond the signed range are clamped, so they do not survive a round trip +even in reversible mode. #### Transpose Codec ```java .withCodecs(c -> c diff --git a/pom.xml b/pom.xml index 36eb31ca..20a49c71 100644 --- a/pom.xml +++ b/pom.xml @@ -48,6 +48,7 @@ 2.34.6 5.9.1 1.5.5-7 + 0.1-1.0.1 5.14.0 3.0.2 @@ -104,6 +105,11 @@ blosc-java 0.3-1.21.6 + + com.scalableminds + zfp-java + ${zfpVersion} + com.github.luben zstd-jni diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java b/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java index 5c3487ce..d995a703 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java @@ -110,6 +110,66 @@ public CodecBuilder withZstd(int level) { return withZstd(level, true); } + public CodecBuilder withZfp(ZfpCodec.Configuration configuration) { + codecs.add(new ZfpCodec(configuration)); + return this; + } + + /** + * Compresses chunks losslessly with zfp. + */ + public CodecBuilder withZfpReversible() { + try { + return withZfp(ZfpCodec.Configuration.reversible()); + } catch (ZarrException e) { + throw new RuntimeException(e); + } + } + + /** + * Compresses chunks with zfp, with a guaranteed absolute error bound. + */ + public CodecBuilder withZfpFixedAccuracy(double tolerance) { + try { + return withZfp(ZfpCodec.Configuration.fixedAccuracy(tolerance)); + } catch (ZarrException e) { + throw new RuntimeException(e); + } + } + + /** + * Compresses chunks with zfp, at a fixed number of compressed bits per value. + */ + public CodecBuilder withZfpFixedRate(double rate) { + try { + return withZfp(ZfpCodec.Configuration.fixedRate(rate)); + } catch (ZarrException e) { + throw new RuntimeException(e); + } + } + + /** + * Compresses chunks with zfp, retaining a fixed number of bit planes. + */ + public CodecBuilder withZfpFixedPrecision(int precision) { + try { + return withZfp(ZfpCodec.Configuration.fixedPrecision(precision)); + } catch (ZarrException e) { + throw new RuntimeException(e); + } + } + + /** + * Compresses chunks with zfp, setting all four of zfp's expert mode parameters directly. + */ + public CodecBuilder withZfpExpert(int minbits, int maxbits, int maxprec, int minexp) { + try { + return withZfp(ZfpCodec.Configuration.expert(minbits, maxbits, maxprec, minexp)); + } catch (ZarrException e) { + throw new RuntimeException(e); + } + } + public CodecBuilder withSharding(int[] chunkShape) { try { codecs.add( diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecRegistry.java b/src/main/java/dev/zarr/zarrjava/v3/codec/CodecRegistry.java index ad249e08..4187bbcb 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecRegistry.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/CodecRegistry.java @@ -16,6 +16,7 @@ public class CodecRegistry { addType("blosc", BloscCodec.class); addType("gzip", GzipCodec.class); addType("zstd", ZstdCodec.class); + addType("zfp", ZfpCodec.class); addType("crc32c", Crc32cCodec.class); addType("sharding_indexed", ShardingIndexedCodec.class); } diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZfpCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZfpCodec.java new file mode 100644 index 00000000..80ebb155 --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZfpCodec.java @@ -0,0 +1,479 @@ +package dev.zarr.zarrjava.v3.codec.core; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonValue; +import com.scalableminds.zfpjava.Zfp; +import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.core.codec.ArrayBytesCodec; +import dev.zarr.zarrjava.utils.Utils; +import dev.zarr.zarrjava.v3.ArrayMetadata; +import dev.zarr.zarrjava.v3.codec.Codec; +import ucar.ma2.Array; +import ucar.ma2.DataType; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.Arrays; + +/** + * Compresses chunks with the zfp algorithm, as specified by the + * zfp codec + * extension. + * + *

zfp itself only handles 1- to 4-dimensional arrays of {@code int32}, {@code int64}, + * {@code float32} and {@code float64}. Narrower integer data types are promoted to {@code int32} and + * demoted back with the shifts given in the specification; unsigned 32- and 64-bit values are + * clamped into the signed range, matching the reference implementation in zarrs. Clamping is lossy + * for values above {@code 2^31 - 1} respectively {@code 2^63 - 1}, even in reversible mode. + */ +public class ZfpCodec extends ArrayBytesCodec implements Codec { + + @JsonIgnore + @Nonnull + public final String name = "zfp"; + @Nonnull + public final Configuration configuration; + + @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) + public ZfpCodec( + @Nonnull @JsonProperty(value = "configuration", required = true) Configuration configuration + ) { + this.configuration = configuration; + } + + private static int clamp(int value, int min, int max) { + return Math.min(Math.max(value, min), max); + } + + @Override + public ByteBuffer encode(Array chunkArray) throws ZarrException { + final DataType dataType = arrayMetadata.dataType.getMA2DataType(); + final Zfp.Type zfpType = zfpType(dataType); + final int[] zfpShape = zfpShape(); + + final ByteBuffer promoted = + ByteBuffer.allocate((int) Zfp.countValues(zfpShape) * zfpType.getByteCount()) + .order(ByteOrder.nativeOrder()); + promote(chunkArray, dataType, promoted); + promoted.rewind(); + + try { + return ByteBuffer.wrap( + Zfp.compress(promoted.array(), zfpType, zfpShape, params(zfpType, zfpShape.length))); + } catch (RuntimeException ex) { + throw new ZarrException("Error in encoding zfp.", ex); + } + } + + @Override + public Array decode(ByteBuffer chunkBytes) throws ZarrException { + final DataType dataType = arrayMetadata.dataType.getMA2DataType(); + final Zfp.Type zfpType = zfpType(dataType); + final int[] zfpShape = zfpShape(); + + final byte[] promoted; + try { + promoted = Zfp.decompress(Utils.toArray(chunkBytes), zfpType, zfpShape, + params(zfpType, zfpShape.length)); + } catch (RuntimeException ex) { + throw new ZarrException("Error in decoding zfp.", ex); + } + return demote(ByteBuffer.wrap(promoted).order(ByteOrder.nativeOrder()), dataType, + (int) Zfp.countValues(zfpShape)); + } + + @Override + public long computeEncodedSize(long inputByteLength, + ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException { + throw new ZarrException("Not implemented for Zfp codec."); + } + + /** + * The zfp scalar type the chunk's values are compressed as. Data types narrower than 32 bits are + * compressed as {@code int32}, unsigned types as their signed counterpart. + */ + private Zfp.Type zfpType(DataType dataType) throws ZarrException { + switch (dataType) { + case BYTE: + case UBYTE: + case SHORT: + case USHORT: + case INT: + case UINT: + return Zfp.Type.INT32; + case LONG: + case ULONG: + return Zfp.Type.INT64; + case FLOAT: + return Zfp.Type.FLOAT; + case DOUBLE: + return Zfp.Type.DOUBLE; + default: + throw new ZarrException( + "The zfp codec does not support the data type '" + arrayMetadata.dataType + "'."); + } + } + + /** + * The chunk shape as a zfp field shape, in row-major order. The chunk of a zero-dimensional array + * is a 1D field holding a single value. + */ + private int[] zfpShape() throws ZarrException { + final int[] chunkShape = arrayMetadata.chunkShape; + if (chunkShape.length == 0) { + return new int[]{1}; + } + if (chunkShape.length > Zfp.MAX_DIMS) { + throw new ZarrException( + "The zfp codec supports at most " + Zfp.MAX_DIMS + " dimensions, but the chunk shape " + + Arrays.toString(chunkShape) + " has " + chunkShape.length + "."); + } + return chunkShape; + } + + /** + * Resolves the configured compression mode to zfp's expert mode parameters. Fixed-rate mode + * depends on the zfp type and the dimensionality, which is why this cannot happen while parsing + * the configuration. + */ + private Zfp.Params params(Zfp.Type zfpType, int dims) throws ZarrException { + try { + switch (configuration.mode) { + case FIXED_RATE: + return Zfp.Params.fixedRate(configuration.rate, zfpType, dims); + default: + return configuration.resolveParams(); + } + } catch (IllegalArgumentException ex) { + throw new ZarrException("Invalid zfp codec configuration.", ex); + } + } + + /** + * Writes the chunk's values into {@code out} as zfp scalars, promoting narrower integers into + * {@code int32} as specified. + */ + private void promote(Array chunkArray, DataType dataType, ByteBuffer out) { + switch (dataType) { + case BYTE: { + final byte[] values = (byte[]) chunkArray.copyTo1DJavaArray(); + for (byte value : values) { + out.putInt(value << 23); + } + break; + } + case UBYTE: { + final byte[] values = (byte[]) chunkArray.copyTo1DJavaArray(); + for (byte value : values) { + out.putInt(((value & 0xFF) - 0x80) << 23); + } + break; + } + case SHORT: { + final short[] values = (short[]) chunkArray.copyTo1DJavaArray(); + for (short value : values) { + out.putInt(value << 15); + } + break; + } + case USHORT: { + final short[] values = (short[]) chunkArray.copyTo1DJavaArray(); + for (short value : values) { + out.putInt(((value & 0xFFFF) - 0x8000) << 15); + } + break; + } + case INT: { + final int[] values = (int[]) chunkArray.copyTo1DJavaArray(); + for (int value : values) { + out.putInt(value); + } + break; + } + case UINT: { + final int[] values = (int[]) chunkArray.copyTo1DJavaArray(); + for (int value : values) { + // Values above Integer.MAX_VALUE have their sign bit set and are clamped + out.putInt(value < 0 ? Integer.MAX_VALUE : value); + } + break; + } + case LONG: { + final long[] values = (long[]) chunkArray.copyTo1DJavaArray(); + for (long value : values) { + out.putLong(value); + } + break; + } + case ULONG: { + final long[] values = (long[]) chunkArray.copyTo1DJavaArray(); + for (long value : values) { + // Values above Long.MAX_VALUE have their sign bit set and are clamped + out.putLong(value < 0 ? Long.MAX_VALUE : value); + } + break; + } + case FLOAT: { + final float[] values = (float[]) chunkArray.copyTo1DJavaArray(); + for (float value : values) { + out.putFloat(value); + } + break; + } + case DOUBLE: { + final double[] values = (double[]) chunkArray.copyTo1DJavaArray(); + for (double value : values) { + out.putDouble(value); + } + break; + } + default: + throw new IllegalStateException("Unsupported data type: " + dataType); + } + } + + /** + * Reads {@code valueCount} zfp scalars from {@code in} and demotes them back into the chunk's data + * type. + */ + private Array demote(ByteBuffer in, DataType dataType, int valueCount) { + final int[] shape = arrayMetadata.chunkShape; + switch (dataType) { + case BYTE: + case UBYTE: { + final byte[] values = new byte[valueCount]; + for (int i = 0; i < valueCount; i++) { + final int value = in.getInt() >> 23; + values[i] = dataType == DataType.BYTE + ? (byte) clamp(value, -0x80, 0x7F) + : (byte) clamp(value + 0x80, 0x00, 0xFF); + } + return Array.factory(dataType, shape, values); + } + case SHORT: + case USHORT: { + final short[] values = new short[valueCount]; + for (int i = 0; i < valueCount; i++) { + final int value = in.getInt() >> 15; + values[i] = dataType == DataType.SHORT + ? (short) clamp(value, -0x8000, 0x7FFF) + : (short) clamp(value + 0x8000, 0x0000, 0xFFFF); + } + return Array.factory(dataType, shape, values); + } + case INT: + case UINT: { + final int[] values = new int[valueCount]; + for (int i = 0; i < valueCount; i++) { + final int value = in.getInt(); + values[i] = dataType == DataType.INT ? value : Math.max(value, 0); + } + return Array.factory(dataType, shape, values); + } + case LONG: + case ULONG: { + final long[] values = new long[valueCount]; + for (int i = 0; i < valueCount; i++) { + final long value = in.getLong(); + values[i] = dataType == DataType.LONG ? value : Math.max(value, 0); + } + return Array.factory(dataType, shape, values); + } + case FLOAT: { + final float[] values = new float[valueCount]; + for (int i = 0; i < valueCount; i++) { + values[i] = in.getFloat(); + } + return Array.factory(dataType, shape, values); + } + case DOUBLE: { + final double[] values = new double[valueCount]; + for (int i = 0; i < valueCount; i++) { + values[i] = in.getDouble(); + } + return Array.factory(dataType, shape, values); + } + default: + throw new IllegalStateException("Unsupported data type: " + dataType); + } + } + + /** + * The zfp compression modes. + */ + public enum Mode { + REVERSIBLE("reversible"), + EXPERT("expert"), + FIXED_ACCURACY("fixed_accuracy"), + FIXED_RATE("fixed_rate"), + FIXED_PRECISION("fixed_precision"); + + private final String mode; + + Mode(String mode) { + this.mode = mode; + } + + @JsonCreator + public static Mode fromValue(String value) { + for (Mode mode : values()) { + if (mode.mode.equals(value)) { + return mode; + } + } + throw new IllegalArgumentException("Unknown zfp mode '" + value + "'."); + } + + @JsonValue + public String getValue() { + return mode; + } + } + + public static final class Configuration { + + @Nonnull + public final Mode mode; + @Nullable + public final Integer minbits; + @Nullable + public final Integer maxbits; + @Nullable + public final Integer maxprec; + @Nullable + public final Integer minexp; + @Nullable + public final Double tolerance; + @Nullable + public final Double rate; + @Nullable + public final Integer precision; + + @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) + public Configuration( + @Nonnull @JsonProperty(value = "mode", required = true) Mode mode, + @Nullable @JsonProperty("minbits") Integer minbits, + @Nullable @JsonProperty("maxbits") Integer maxbits, + @Nullable @JsonProperty("maxprec") Integer maxprec, + @Nullable @JsonProperty("minexp") Integer minexp, + @Nullable @JsonProperty("tolerance") Double tolerance, + @Nullable @JsonProperty("rate") Double rate, + @Nullable @JsonProperty("precision") Integer precision + ) throws ZarrException { + if (mode == null) { + throw new ZarrException("'mode' is required for the zfp codec."); + } + this.mode = mode; + this.minbits = minbits; + this.maxbits = maxbits; + this.maxprec = maxprec; + this.minexp = minexp; + this.tolerance = tolerance; + this.rate = rate; + this.precision = precision; + + switch (mode) { + case REVERSIBLE: + requireAbsent("minbits", minbits, "maxbits", maxbits, "maxprec", maxprec, "minexp", + minexp, "tolerance", tolerance, "rate", rate, "precision", precision); + break; + case EXPERT: + requirePresent("minbits", minbits, "maxbits", maxbits, "maxprec", maxprec, "minexp", + minexp); + requireAbsent("tolerance", tolerance, "rate", rate, "precision", precision); + break; + case FIXED_ACCURACY: + requirePresent("tolerance", tolerance); + requireAbsent("minbits", minbits, "maxbits", maxbits, "maxprec", maxprec, "minexp", + minexp, "rate", rate, "precision", precision); + break; + case FIXED_RATE: + requirePresent("rate", rate); + requireAbsent("minbits", minbits, "maxbits", maxbits, "maxprec", maxprec, "minexp", + minexp, "tolerance", tolerance, "precision", precision); + if (!(rate > 0)) { + throw new ZarrException("'rate' needs to be positive, got " + rate + "."); + } + break; + case FIXED_PRECISION: + requirePresent("precision", precision); + requireAbsent("minbits", minbits, "maxbits", maxbits, "maxprec", maxprec, "minexp", + minexp, "tolerance", tolerance, "rate", rate); + break; + } + if (mode != Mode.FIXED_RATE) { + // Fails fast on out-of-range parameters. Fixed-rate mode needs the data type and the + // dimensionality, so it can only be resolved once the array metadata is known. + try { + resolveParams(); + } catch (IllegalArgumentException ex) { + throw new ZarrException("Invalid zfp codec configuration.", ex); + } + } + } + + public static Configuration reversible() throws ZarrException { + return new Configuration(Mode.REVERSIBLE, null, null, null, null, null, null, null); + } + + public static Configuration expert(int minbits, int maxbits, int maxprec, int minexp) + throws ZarrException { + return new Configuration(Mode.EXPERT, minbits, maxbits, maxprec, minexp, null, null, null); + } + + public static Configuration fixedAccuracy(double tolerance) throws ZarrException { + return new Configuration(Mode.FIXED_ACCURACY, null, null, null, null, tolerance, null, null); + } + + public static Configuration fixedRate(double rate) throws ZarrException { + return new Configuration(Mode.FIXED_RATE, null, null, null, null, null, rate, null); + } + + public static Configuration fixedPrecision(int precision) throws ZarrException { + return new Configuration(Mode.FIXED_PRECISION, null, null, null, null, null, null, precision); + } + + private static void requirePresent(Object... namesAndValues) throws ZarrException { + for (int i = 0; i < namesAndValues.length; i += 2) { + if (namesAndValues[i + 1] == null) { + throw new ZarrException( + "'" + namesAndValues[i] + "' is required for the zfp codec."); + } + } + } + + private static void requireAbsent(Object... namesAndValues) throws ZarrException { + for (int i = 0; i < namesAndValues.length; i += 2) { + if (namesAndValues[i + 1] != null) { + throw new ZarrException( + "'" + namesAndValues[i] + "' is not a parameter of this zfp mode."); + } + } + } + + /** + * Resolves this configuration to zfp's expert mode parameters. Not supported for fixed-rate + * mode, which additionally depends on the data type and the dimensionality of the chunk. + */ + @JsonIgnore + Zfp.Params resolveParams() { + switch (mode) { + case REVERSIBLE: + return Zfp.Params.reversible(); + case EXPERT: + return Zfp.Params.expert(minbits, maxbits, maxprec, minexp); + case FIXED_ACCURACY: + return Zfp.Params.fixedAccuracy(tolerance); + case FIXED_PRECISION: + return Zfp.Params.fixedPrecision(precision); + default: + throw new IllegalStateException( + "The zfp parameters of mode '" + mode.getValue() + "' depend on the array metadata."); + } + } + } +} diff --git a/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java b/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java index 614a2b90..036f410a 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java @@ -1,5 +1,7 @@ package dev.zarr.zarrjava; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -12,10 +14,12 @@ import dev.zarr.zarrjava.v3.*; import dev.zarr.zarrjava.v3.codec.Codec; import dev.zarr.zarrjava.v3.codec.CodecBuilder; +import dev.zarr.zarrjava.v3.codec.CodecRegistry; import dev.zarr.zarrjava.v3.codec.core.BloscCodec; import dev.zarr.zarrjava.v3.codec.core.BytesCodec; import dev.zarr.zarrjava.v3.codec.core.ShardingIndexedCodec; import dev.zarr.zarrjava.v3.codec.core.TransposeCodec; +import dev.zarr.zarrjava.v3.codec.core.ZfpCodec; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -27,6 +31,7 @@ import java.io.BufferedReader; import java.io.IOException; +import java.nio.ByteBuffer; import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.nio.file.Path; @@ -1052,4 +1057,238 @@ public void testEndianness(DataType dataType, BytesCodec.Endian endian) throws I ucar.ma2.Array readData = reopenedArray.read(); assertIsTestdata(readData, dataType); } + + static Stream zfpDataTypeProvider() { + return dataTypeProviderV3().filter(dataType -> dataType != DataType.BOOL); + } + + static Stream zfpConfigurationJsonProvider() { + // The examples of the zfp codec specification + return Stream.of( + "{\"name\":\"zfp\",\"configuration\":{\"mode\":\"reversible\"}}", + "{\"name\":\"zfp\",\"configuration\":{\"mode\":\"expert\",\"minbits\":1,\"maxbits\":13,\"maxprec\":19,\"minexp\":-2}}", + "{\"name\":\"zfp\",\"configuration\":{\"mode\":\"fixed_accuracy\",\"tolerance\":0.05}}", + "{\"name\":\"zfp\",\"configuration\":{\"mode\":\"fixed_rate\",\"rate\":10.5}}", + "{\"name\":\"zfp\",\"configuration\":{\"mode\":\"fixed_precision\",\"precision\":19}}" + ); + } + + static Stream invalidZfpConfigurationJsonProvider() { + return Stream.of( + // Unknown mode + "{\"name\":\"zfp\",\"configuration\":{\"mode\":\"fixed_nonsense\"}}", + // Missing mode + "{\"name\":\"zfp\",\"configuration\":{\"tolerance\":0.05}}", + // Missing required parameters + "{\"name\":\"zfp\",\"configuration\":{\"mode\":\"fixed_accuracy\"}}", + "{\"name\":\"zfp\",\"configuration\":{\"mode\":\"fixed_rate\"}}", + "{\"name\":\"zfp\",\"configuration\":{\"mode\":\"fixed_precision\"}}", + "{\"name\":\"zfp\",\"configuration\":{\"mode\":\"expert\",\"minbits\":1,\"maxbits\":13,\"maxprec\":19}}", + // Parameters that do not belong to the mode + "{\"name\":\"zfp\",\"configuration\":{\"mode\":\"reversible\",\"tolerance\":0.05}}", + "{\"name\":\"zfp\",\"configuration\":{\"mode\":\"fixed_rate\",\"rate\":10.5,\"precision\":19}}", + // Parameters out of range + "{\"name\":\"zfp\",\"configuration\":{\"mode\":\"expert\",\"minbits\":13,\"maxbits\":1,\"maxprec\":19,\"minexp\":-2}}", + "{\"name\":\"zfp\",\"configuration\":{\"mode\":\"expert\",\"minbits\":1,\"maxbits\":13,\"maxprec\":65,\"minexp\":-2}}", + "{\"name\":\"zfp\",\"configuration\":{\"mode\":\"fixed_rate\",\"rate\":0}}" + ); + } + + static ObjectMapper zfpObjectMapper() { + // Mirrors dev.zarr.zarrjava.v3.Node#makeObjectMapper, which is not visible here + ObjectMapper objectMapper = new ObjectMapper(); + objectMapper.registerSubtypes(CodecRegistry.getNamedTypes()); + objectMapper.setDefaultPropertyInclusion(JsonInclude.Include.NON_NULL); + return objectMapper; + } + + static ZfpCodec zfpCodec(ZfpCodec.Configuration configuration, DataType dataType, + int[] chunkShape) throws ZarrException { + ZfpCodec codec = new ZfpCodec(configuration); + codec.setCoreArrayMetadata(new ArrayMetadata.CoreArrayMetadata( + toLongArray(chunkShape), chunkShape, dataType, null)); + return codec; + } + + static ucar.ma2.Array sineTestData(ucar.ma2.DataType dataType, int[] shape) { + ucar.ma2.Array array = ucar.ma2.Array.factory(dataType, shape); + for (int i = 0; i < array.getSize(); i++) { + array.setDouble(i, Math.sin(i * 0.01) * 100.0); + } + return array; + } + + @ParameterizedTest + @MethodSource("zfpDataTypeProvider") + public void testZfpCodecReversibleReadWrite(DataType dataType) throws ZarrException, IOException { + ucar.ma2.Array testData = testdata(dataType); + + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testZfpCodecReversibleReadWrite", dataType.name()); + Array writeArray = Array.create(storeHandle, Array.metadataBuilder() + .withShape(toLongArray(testData.getShape())) + .withDataType(dataType) + .withChunkShape(8, 8, 8) + .withFillValue(0) + .withCodecs(CodecBuilder::withZfpReversible) + .build()); + writeArray.write(testData); + + ucar.ma2.Array readData = Array.open(storeHandle).read(); + assertIsTestdata(readData, dataType); + } + + @Test + public void testZfpCodecShardingReadWrite() throws ZarrException, IOException { + ucar.ma2.Array testData = testdata(DataType.FLOAT32); + + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testZfpCodecShardingReadWrite"); + Array writeArray = Array.create(storeHandle, Array.metadataBuilder() + .withShape(toLongArray(testData.getShape())) + .withDataType(DataType.FLOAT32) + .withChunkShape(16, 16, 16) + .withFillValue(0) + .withCodecs(c -> c.withSharding(new int[]{4, 8, 8}, CodecBuilder::withZfpReversible)) + .build()); + writeArray.write(testData); + + ucar.ma2.Array readData = Array.open(storeHandle).read(); + assertIsTestdata(readData, DataType.FLOAT32); + } + + @ParameterizedTest + @CsvSource({"FLOAT32, 0.05", "FLOAT32, 0.5", "FLOAT64, 0.05", "FLOAT64, 0.5"}) + public void testZfpCodecFixedAccuracy(DataType dataType, double tolerance) throws ZarrException { + int[] chunkShape = new int[]{8, 16, 32}; + ucar.ma2.Array testData = sineTestData(dataType.getMA2DataType(), chunkShape); + + ZfpCodec codec = zfpCodec(ZfpCodec.Configuration.fixedAccuracy(tolerance), dataType, chunkShape); + ByteBuffer encoded = codec.encode(testData); + ucar.ma2.Array decoded = codec.decode(encoded.duplicate()); + + Assertions.assertTrue(encoded.remaining() < testData.getSize() * dataType.getByteCount(), + "zfp should compress smooth data"); + for (int i = 0; i < testData.getSize(); i++) { + Assertions.assertEquals(testData.getDouble(i), decoded.getDouble(i), tolerance); + } + } + + @Test + public void testZfpCodecFixedRate() throws ZarrException { + int[] chunkShape = new int[]{8, 8, 8}; + ucar.ma2.Array testData = sineTestData(ucar.ma2.DataType.DOUBLE, chunkShape); + + // Every one of the 2*2*2 blocks of 4*4*4 values takes exactly 8 * 64 bits + ZfpCodec codec = zfpCodec(ZfpCodec.Configuration.fixedRate(8), DataType.FLOAT64, chunkShape); + ByteBuffer encoded = codec.encode(testData); + Assertions.assertEquals(8 * 8 * 64 / 8, encoded.remaining()); + + ucar.ma2.Array decoded = codec.decode(encoded.duplicate()); + for (int i = 0; i < testData.getSize(); i++) { + Assertions.assertEquals(testData.getDouble(i), decoded.getDouble(i), 1.0); + } + } + + @Test + public void testZfpCodecFixedPrecision() throws ZarrException { + int[] chunkShape = new int[]{8, 16, 32}; + ucar.ma2.Array testData = sineTestData(ucar.ma2.DataType.FLOAT, chunkShape); + + ZfpCodec codec = zfpCodec(ZfpCodec.Configuration.fixedPrecision(24), DataType.FLOAT32, chunkShape); + ByteBuffer encoded = codec.encode(testData); + ucar.ma2.Array decoded = codec.decode(encoded.duplicate()); + + Assertions.assertTrue(encoded.remaining() < testData.getSize() * Float.BYTES, + "zfp should compress smooth data"); + for (int i = 0; i < testData.getSize(); i++) { + Assertions.assertEquals(testData.getDouble(i), decoded.getDouble(i), 0.01); + } + } + + @Test + public void testZfpCodecExpert() throws ZarrException { + int[] chunkShape = new int[]{8, 16, 32}; + ucar.ma2.Array testData = sineTestData(ucar.ma2.DataType.DOUBLE, chunkShape); + + // Reversible mode expressed through its expert mode parameters + ZfpCodec codec = zfpCodec(ZfpCodec.Configuration.expert(1, 16658, 64, -1075), DataType.FLOAT64, + chunkShape); + ucar.ma2.Array decoded = codec.decode(codec.encode(testData)); + for (int i = 0; i < testData.getSize(); i++) { + Assertions.assertEquals(testData.getDouble(i), decoded.getDouble(i)); + } + } + + @Test + public void testZfpCodecOneDimensionalAndZeroDimensional() throws ZarrException { + ucar.ma2.Array oneDimensional = sineTestData(ucar.ma2.DataType.DOUBLE, new int[]{64}); + ZfpCodec oneDimensionalCodec = zfpCodec(ZfpCodec.Configuration.reversible(), DataType.FLOAT64, + new int[]{64}); + ucar.ma2.Array decoded = oneDimensionalCodec.decode(oneDimensionalCodec.encode(oneDimensional)); + for (int i = 0; i < oneDimensional.getSize(); i++) { + Assertions.assertEquals(oneDimensional.getDouble(i), decoded.getDouble(i)); + } + + // The chunk of a zero-dimensional array is a 1D zfp field holding a single value + ucar.ma2.Array zeroDimensional = ucar.ma2.Array.factory(ucar.ma2.DataType.DOUBLE, new int[0], + new double[]{42.5}); + ZfpCodec zeroDimensionalCodec = zfpCodec(ZfpCodec.Configuration.reversible(), DataType.FLOAT64, + new int[0]); + ucar.ma2.Array decodedScalar = + zeroDimensionalCodec.decode(zeroDimensionalCodec.encode(zeroDimensional)); + Assertions.assertEquals(42.5, decodedScalar.getDouble(0)); + } + + @Test + public void testZfpCodecClampsLargeUnsignedValues() throws ZarrException { + // uint32 and uint64 values beyond the signed range are clamped, matching zarrs + int[] chunkShape = new int[]{4}; + ucar.ma2.Array uint32Data = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, chunkShape, + new int[]{0, 1, Integer.MAX_VALUE, (int) 4_000_000_000L}); + ZfpCodec uint32Codec = zfpCodec(ZfpCodec.Configuration.reversible(), DataType.UINT32, chunkShape); + ucar.ma2.Array decodedUint32 = uint32Codec.decode(uint32Codec.encode(uint32Data)); + Assertions.assertArrayEquals(new long[]{0, 1, Integer.MAX_VALUE, Integer.MAX_VALUE}, + new long[]{decodedUint32.getLong(0), decodedUint32.getLong(1), decodedUint32.getLong(2), + decodedUint32.getLong(3)}); + + ucar.ma2.Array uint64Data = ucar.ma2.Array.factory(ucar.ma2.DataType.ULONG, chunkShape, + new long[]{0, 1, Long.MAX_VALUE, Long.MIN_VALUE}); + ZfpCodec uint64Codec = zfpCodec(ZfpCodec.Configuration.reversible(), DataType.UINT64, chunkShape); + ucar.ma2.Array decodedUint64 = uint64Codec.decode(uint64Codec.encode(uint64Data)); + Assertions.assertArrayEquals(new long[]{0, 1, Long.MAX_VALUE, Long.MAX_VALUE}, + new long[]{decodedUint64.getLong(0), decodedUint64.getLong(1), decodedUint64.getLong(2), + decodedUint64.getLong(3)}); + } + + @Test + public void testZfpCodecRejectsBool() throws ZarrException { + int[] chunkShape = new int[]{4, 4}; + ZfpCodec codec = zfpCodec(ZfpCodec.Configuration.reversible(), DataType.BOOL, chunkShape); + ucar.ma2.Array testData = ucar.ma2.Array.factory(ucar.ma2.DataType.BOOLEAN, chunkShape); + assertThrows(ZarrException.class, () -> codec.encode(testData)); + } + + @Test + public void testZfpCodecRejectsMoreThanFourDimensions() throws ZarrException { + int[] chunkShape = new int[]{2, 2, 2, 2, 2}; + ZfpCodec codec = zfpCodec(ZfpCodec.Configuration.reversible(), DataType.FLOAT64, chunkShape); + ucar.ma2.Array testData = ucar.ma2.Array.factory(ucar.ma2.DataType.DOUBLE, chunkShape); + assertThrows(ZarrException.class, () -> codec.encode(testData)); + } + + @ParameterizedTest + @MethodSource("zfpConfigurationJsonProvider") + public void testZfpCodecJsonRoundTrip(String json) throws IOException { + ObjectMapper objectMapper = zfpObjectMapper(); + Codec codec = objectMapper.readValue(json, Codec.class); + Assertions.assertInstanceOf(ZfpCodec.class, codec); + Assertions.assertEquals(objectMapper.readTree(json), + objectMapper.readTree(objectMapper.writeValueAsString(codec))); + } + + @ParameterizedTest + @MethodSource("invalidZfpConfigurationJsonProvider") + public void testZfpCodecInvalidConfiguration(String json) { + ObjectMapper objectMapper = zfpObjectMapper(); + assertThrows(JsonProcessingException.class, () -> objectMapper.readValue(json, Codec.class)); + } }