diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4262a2b0..be4cfe8e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,6 +1,7 @@ name: CI on: + workflow_dispatch: push: branches: [ "main" ] pull_request: @@ -10,21 +11,33 @@ jobs: build: strategy: matrix: - os: [ ubuntu, windows, macos ] - runs-on: ${{ matrix.os }}-latest + os: [ ubuntu-latest, windows-latest, macos-latest ] + fail-fast: false + runs-on: ${{ matrix.os }} defaults: run: shell: bash steps: - uses: actions/checkout@v3 + - name: Set up JDK uses: actions/setup-java@v3 with: - java-version: '8' + java-version: '22' distribution: 'temurin' cache: maven + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install zarrita + run: | + python -m venv venv_zarrita + if [ "${{ runner.os }}" = "Windows" ]; then venv_zarrita/Scripts/pip install zarrita; else venv_zarrita/bin/pip install zarrita; fi + - name: Download blosc jar run: | mkdir -p ../blosc-java/target @@ -32,7 +45,7 @@ jobs: - name: Download testdata run: | - mkdir testdata testoutput + mkdir testoutput curl https://static.webknossos.org/data/zarr_v3/l4_sample.zip -o testdata/l4_sample.zip cd testdata unzip l4_sample.zip @@ -43,7 +56,7 @@ jobs: - name: Test env: MAVEN_OPTS: "-Xmx6g" - run: mvn test -DargLine="-Xmx6g" + run: mvn --no-transfer-progress test -DargLine="-Xmx6g" - name: Assemble JAR run: mvn package -DskipTests @@ -51,4 +64,4 @@ jobs: - uses: actions/upload-artifact@v3 with: name: jar - path: target/*.jar \ No newline at end of file + path: target/*.jar diff --git a/.gitignore b/.gitignore index bd4b34c2..49917206 100644 --- a/.gitignore +++ b/.gitignore @@ -36,5 +36,6 @@ build/ ### Custom ### -/testdata +/testdata/l4_sample /testoutput +/venv_zarrita diff --git a/README.md b/README.md index ae77c5ce..76809f73 100644 --- a/README.md +++ b/README.md @@ -38,3 +38,19 @@ array.write( ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{1, 1024, 1024, 1024}) ); ``` +## Development Start-Guide + +### Run Tests Locally +To be able to run the tests locally, make sure to have `python3.11` installed. +Also, you need to set up a venv for zarrita at the root of the project: +`python3.11 -m venv venv_zarrita`. + +Then install zarrita there with `venv_zarrita/Scripts/pip install zarrita` +for Windows and `venv_zarrita/bin/pip install zarrita` for Linux. + +Furthermore, you will need the `l4_sample` test data: + +`curl https://static.webknossos.org/data/zarr_v3/l4_sample.zip -o testdata/l4_sample.zip +&& cd testdata +&& unzip l4_sample.zip +` \ No newline at end of file diff --git a/pom.xml b/pom.xml index edd4fe74..e3f4e4b9 100644 --- a/pom.xml +++ b/pom.xml @@ -16,9 +16,31 @@ 1.12.477 5.5.3 1.5.5-5 + 5.10.2 + + + org.junit.jupiter + junit-jupiter-api + ${junit-jupiter-version} + test + + + org.junit.jupiter + junit-jupiter-engine + ${junit-jupiter-version} + test + + + org.junit.jupiter + junit-jupiter-params + ${junit-jupiter-version} + test + + + com.fasterxml.jackson.core jackson-databind @@ -54,6 +76,7 @@ okhttp 2.7.5 + junit junit @@ -70,4 +93,16 @@ - \ No newline at end of file + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.2.5 + + false + + + + + diff --git a/src/main/java/dev/zarr/zarrjava/utils/Utils.java b/src/main/java/dev/zarr/zarrjava/utils/Utils.java index 93b7cf1a..3ea4bfdf 100644 --- a/src/main/java/dev/zarr/zarrjava/utils/Utils.java +++ b/src/main/java/dev/zarr/zarrjava/utils/Utils.java @@ -77,4 +77,24 @@ public static T[] concatArrays(T[] array1, T[]... arrays) { } return result; } + + public static boolean isPermutation(int[] array) { + if (array.length==0){ + return false; + } + int[] arange = new int[array.length]; + Arrays.setAll(arange, i -> i); + int[] orderSorted = array.clone(); + Arrays.sort(orderSorted); + return Arrays.equals(orderSorted, arange); + } + + public static int[] inversePermutation(int[] origin){ + assert isPermutation(origin); + int[] inverse = new int[origin.length]; + for (int i = 0; i < origin.length; i++) { + inverse[origin[i]] = i; + } + return inverse; + } } diff --git a/src/main/java/dev/zarr/zarrjava/v3/Array.java b/src/main/java/dev/zarr/zarrjava/v3/Array.java index 631ef70f..85e04574 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/Array.java +++ b/src/main/java/dev/zarr/zarrjava/v3/Array.java @@ -27,7 +27,7 @@ protected Array(StoreHandle storeHandle, ArrayMetadata arrayMetadata) throws IOException, ZarrException { super(storeHandle); this.metadata = arrayMetadata; - this.codecPipeline = new CodecPipeline(arrayMetadata.codecs); + this.codecPipeline = new CodecPipeline(arrayMetadata.codecs, arrayMetadata.coreArrayMetadata); } /** @@ -171,8 +171,7 @@ public ucar.ma2.Array read(final long[] offset, final int[] shape) throws ZarrEx if (codecPipeline.supportsPartialDecode()) { final ucar.ma2.Array chunkArray = codecPipeline.decodePartial(chunkHandle, - Utils.toLongArray(chunkProjection.chunkOffset), chunkProjection.shape, - metadata.coreArrayMetadata); + Utils.toLongArray(chunkProjection.chunkOffset), chunkProjection.shape); MultiArrayUtils.copyRegion(chunkArray, new int[metadata.ndim()], outputArray, chunkProjection.outOffset, chunkProjection.shape ); @@ -223,7 +222,7 @@ public ucar.ma2.Array readChunk(long[] chunkCoords) return metadata.allocateFillValueChunk(); } - return codecPipeline.decode(chunkBytes, metadata.coreArrayMetadata); + return codecPipeline.decode(chunkBytes); } /** @@ -299,7 +298,7 @@ public void writeChunk(long[] chunkCoords, ucar.ma2.Array chunkArray) throws Zar if (MultiArrayUtils.allValuesEqual(chunkArray, metadata.parsedFillValue)) { chunkHandle.delete(); } else { - ByteBuffer chunkBytes = codecPipeline.encode(chunkArray, metadata.coreArrayMetadata); + ByteBuffer chunkBytes = codecPipeline.encode(chunkArray); chunkHandle.set(chunkBytes); } } diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/ArrayArrayCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/ArrayArrayCodec.java index 868183ae..a488d306 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/ArrayArrayCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/ArrayArrayCodec.java @@ -1,15 +1,14 @@ package dev.zarr.zarrjava.v3.codec; import dev.zarr.zarrjava.ZarrException; -import dev.zarr.zarrjava.v3.ArrayMetadata.CoreArrayMetadata; import ucar.ma2.Array; -public interface ArrayArrayCodec extends Codec { +public abstract class ArrayArrayCodec extends Codec { - Array encode(Array chunkArray, CoreArrayMetadata arrayMetadata) + protected abstract Array encode(Array chunkArray) throws ZarrException; - Array decode(Array chunkArray, CoreArrayMetadata arrayMetadata) + protected abstract Array decode(Array chunkArray) throws ZarrException; } diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/ArrayBytesCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/ArrayBytesCodec.java index ad042ac1..361ae61f 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/ArrayBytesCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/ArrayBytesCodec.java @@ -2,23 +2,24 @@ import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.store.StoreHandle; -import dev.zarr.zarrjava.v3.ArrayMetadata.CoreArrayMetadata; import java.nio.ByteBuffer; import ucar.ma2.Array; -public interface ArrayBytesCodec extends Codec { +public abstract class ArrayBytesCodec extends Codec { - ByteBuffer encode(Array chunkArray, CoreArrayMetadata arrayMetadata) + protected abstract ByteBuffer encode(Array chunkArray) throws ZarrException; - Array decode(ByteBuffer chunkBytes, CoreArrayMetadata arrayMetadata) + protected abstract Array decode(ByteBuffer chunkBytes) throws ZarrException; - interface WithPartialDecode extends ArrayBytesCodec { + public abstract static class WithPartialDecode extends ArrayBytesCodec { - Array decodePartial( - StoreHandle handle, long[] offset, int[] shape, - CoreArrayMetadata arrayMetadata + public abstract Array decode(ByteBuffer shardBytes) throws ZarrException; + public abstract ByteBuffer encode(Array shardArray) throws ZarrException; + + protected abstract Array decodePartial( + StoreHandle handle, long[] offset, int[] shape ) throws ZarrException; } } diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/BytesBytesCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/BytesBytesCodec.java index d8ce47ab..64354632 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/BytesBytesCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/BytesBytesCodec.java @@ -1,15 +1,13 @@ package dev.zarr.zarrjava.v3.codec; import dev.zarr.zarrjava.ZarrException; -import dev.zarr.zarrjava.v3.ArrayMetadata.CoreArrayMetadata; + import java.nio.ByteBuffer; -public interface BytesBytesCodec extends Codec { +public abstract class BytesBytesCodec extends Codec { - ByteBuffer encode(ByteBuffer chunkBytes, CoreArrayMetadata arrayMetadata) - throws ZarrException; + protected abstract ByteBuffer encode(ByteBuffer chunkBytes) throws ZarrException; - ByteBuffer decode(ByteBuffer chunkBytes, CoreArrayMetadata arrayMetadata) - throws ZarrException; + public abstract ByteBuffer decode(ByteBuffer chunkBytes) throws ZarrException; } diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/Codec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/Codec.java index 6ce7687b..988dd1dc 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/Codec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/Codec.java @@ -5,9 +5,22 @@ import dev.zarr.zarrjava.v3.ArrayMetadata; @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "name") -public interface Codec { +public abstract class Codec { - long computeEncodedSize(long inputByteLength, ArrayMetadata.CoreArrayMetadata arrayMetadata) - throws ZarrException; + protected ArrayMetadata.CoreArrayMetadata arrayMetadata; + + protected ArrayMetadata.CoreArrayMetadata resolveArrayMetadata() throws ZarrException { + if (arrayMetadata == null) { + throw new ZarrException("arrayMetadata needs to get set in for every codec"); + } + return this.arrayMetadata; + } + + protected abstract long computeEncodedSize(long inputByteLength, ArrayMetadata.CoreArrayMetadata arrayMetadata) + throws ZarrException; + + public void setCoreArrayMetadata(ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException{ + this.arrayMetadata = arrayMetadata; + } } diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java b/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java index 5e33d7c4..3776a433 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java @@ -41,6 +41,9 @@ public CodecBuilder withBlosc( } public CodecBuilder withBlosc(String cname, String shuffle, int clevel, int blockSize) { + if (shuffle.equals("shuffle")){ + shuffle = "byteshuffle"; + } return withBlosc(Blosc.Compressor.fromString(cname), Blosc.Shuffle.fromString(shuffle), clevel, dataType.getByteCount(), blockSize ); @@ -62,13 +65,9 @@ public CodecBuilder withBlosc() { return withBlosc("zstd"); } - public CodecBuilder withTranspose(String order) { - try { + public CodecBuilder withTranspose(int[] order) { codecs.add(new TransposeCodec(new TransposeCodec.Configuration(order))); - } catch (ZarrException e) { - throw new RuntimeException(e); - } - return this; + return this; } public CodecBuilder withBytes(Endian endian) { @@ -113,9 +112,10 @@ public CodecBuilder withZstd(int clevel) { public CodecBuilder withSharding(int[] chunkShape) { try { codecs.add( - new ShardingIndexedCodec(new ShardingIndexedCodec.Configuration(chunkShape, - new Codec[]{new BytesCodec(new Configuration(Endian.LITTLE))}, - new Codec[]{new BytesCodec(new Configuration(Endian.LITTLE)), new Crc32cCodec()}))); + new ShardingIndexedCodec(new ShardingIndexedCodec.Configuration(chunkShape, + new Codec[]{new BytesCodec(new Configuration(Endian.LITTLE))}, + new Codec[]{new BytesCodec(new Configuration(Endian.LITTLE)), new Crc32cCodec()}, + "end"))); } catch (ZarrException e) { throw new RuntimeException(e); } @@ -123,19 +123,29 @@ public CodecBuilder withSharding(int[] chunkShape) { } public CodecBuilder withSharding(int[] chunkShape, - Function codecBuilder) { + Function codecBuilder) { + return withSharding(chunkShape, codecBuilder, "end"); + } + + public CodecBuilder withSharding(int[] chunkShape, + Function codecBuilder, String indexLocation) { CodecBuilder nestedBuilder = new CodecBuilder(dataType); try { codecs.add(new ShardingIndexedCodec( - new ShardingIndexedCodec.Configuration(chunkShape, - codecBuilder.apply(nestedBuilder).build(), - new Codec[]{new BytesCodec(Endian.LITTLE), new Crc32cCodec()}))); + new ShardingIndexedCodec.Configuration(chunkShape, + codecBuilder.apply(nestedBuilder).build(), + new Codec[]{new BytesCodec(Endian.LITTLE), new Crc32cCodec()}, + indexLocation))); } catch (ZarrException e) { throw new RuntimeException(e); } return this; } + public CodecBuilder withCrc32c() { + codecs.add(new Crc32cCodec()); + return this; + } private void autoInsertBytesCodec() { if (codecs.stream().noneMatch(c -> c instanceof ArrayBytesCodec)) { Codec[] arrayArrayCodecs = codecs.stream().filter(c -> c instanceof ArrayArrayCodec) diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecPipeline.java b/src/main/java/dev/zarr/zarrjava/v3/codec/CodecPipeline.java index 9ece0f07..920a1f4f 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecPipeline.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/CodecPipeline.java @@ -12,8 +12,10 @@ public class CodecPipeline { @Nonnull final Codec[] codecs; + public final CoreArrayMetadata arrayMetadata; - public CodecPipeline(@Nonnull Codec[] codecs) throws ZarrException { + public CodecPipeline(@Nonnull Codec[] codecs, CoreArrayMetadata arrayMetadata) throws ZarrException { + this.arrayMetadata = arrayMetadata; long arrayBytesCodecCount = Arrays.stream(codecs).filter(c -> c instanceof ArrayBytesCodec) .count(); if (arrayBytesCodecCount != 1) { @@ -21,6 +23,7 @@ public CodecPipeline(@Nonnull Codec[] codecs) throws ZarrException { "Exactly 1 ArrayBytesCodec is required. Found " + arrayBytesCodecCount + "."); } Codec prevCodec = null; + CoreArrayMetadata codecArrayMetadata = arrayMetadata; for (Codec codec : codecs) { if (prevCodec != null) { if (codec instanceof ArrayBytesCodec && prevCodec instanceof ArrayBytesCodec) { @@ -44,6 +47,8 @@ public CodecPipeline(@Nonnull Codec[] codecs) throws ZarrException { prevCodec.getClass() + "'."); } } + codec.setCoreArrayMetadata(codecArrayMetadata); + codecArrayMetadata = codec.resolveArrayMetadata(); prevCodec = codec; } @@ -79,15 +84,14 @@ public boolean supportsPartialDecode() { @Nonnull public Array decodePartial( @Nonnull StoreHandle storeHandle, - long[] offset, int[] shape, - @Nonnull CoreArrayMetadata arrayMetadata + long[] offset, int[] shape ) throws ZarrException { if (!supportsPartialDecode()) { throw new ZarrException( "Partial decode is not supported for these codecs. " + Arrays.toString(codecs)); } Array chunkArray = ((ArrayBytesCodec.WithPartialDecode) getArrayBytesCodec()).decodePartial( - storeHandle, offset, shape, arrayMetadata); + storeHandle, offset, shape); if (chunkArray == null) { throw new ZarrException("chunkArray is null. This is likely a bug in one of the codecs."); } @@ -96,8 +100,7 @@ public Array decodePartial( @Nonnull public Array decode( - @Nonnull ByteBuffer chunkBytes, - @Nonnull CoreArrayMetadata arrayMetadata + @Nonnull ByteBuffer chunkBytes ) throws ZarrException { if (chunkBytes == null) { throw new ZarrException("chunkBytes is null. Ohh nooo."); @@ -106,7 +109,7 @@ public Array decode( BytesBytesCodec[] bytesBytesCodecs = getBytesBytesCodecs(); for (int i = bytesBytesCodecs.length - 1; i >= 0; --i) { BytesBytesCodec codec = bytesBytesCodecs[i]; - chunkBytes = codec.decode(chunkBytes, arrayMetadata); + chunkBytes = codec.decode(chunkBytes); } if (chunkBytes == null) { @@ -114,7 +117,7 @@ public Array decode( "chunkBytes is null. This is likely a bug in one of the codecs. " + Arrays.toString( getBytesBytesCodecs())); } - Array chunkArray = getArrayBytesCodec().decode(chunkBytes, arrayMetadata); + Array chunkArray = getArrayBytesCodec().decode(chunkBytes); if (chunkArray == null) { throw new ZarrException("chunkArray is null. This is likely a bug in one of the codecs."); } @@ -122,7 +125,7 @@ public Array decode( ArrayArrayCodec[] arrayArrayCodecs = getArrayArrayCodecs(); for (int i = arrayArrayCodecs.length - 1; i >= 0; --i) { ArrayArrayCodec codec = arrayArrayCodecs[i]; - chunkArray = codec.decode(chunkArray, arrayMetadata); + chunkArray = codec.decode(chunkArray); } if (chunkArray == null) { @@ -133,16 +136,16 @@ public Array decode( @Nonnull public ByteBuffer encode( - @Nonnull Array chunkArray, @Nonnull CoreArrayMetadata arrayMetadata + @Nonnull Array chunkArray ) throws ZarrException { for (ArrayArrayCodec codec : getArrayArrayCodecs()) { - chunkArray = codec.encode(chunkArray, arrayMetadata); + chunkArray = codec.encode(chunkArray); } - ByteBuffer chunkBytes = getArrayBytesCodec().encode(chunkArray, arrayMetadata); + ByteBuffer chunkBytes = getArrayBytesCodec().encode(chunkArray); for (BytesBytesCodec codec : getBytesBytesCodecs()) { - chunkBytes = codec.encode(chunkBytes, arrayMetadata); + chunkBytes = codec.encode(chunkBytes); } return chunkBytes; } diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/BloscCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/BloscCodec.java index 2c3412bb..819a610f 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/BloscCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/BloscCodec.java @@ -20,7 +20,7 @@ import java.nio.ByteBuffer; import javax.annotation.Nonnull; -public class BloscCodec implements BytesBytesCodec { +public class BloscCodec extends BytesBytesCodec { public final String name = "blosc"; @Nonnull @@ -33,7 +33,7 @@ public BloscCodec( } @Override - public ByteBuffer decode(ByteBuffer chunkBytes, ArrayMetadata.CoreArrayMetadata arrayMetadata) + public ByteBuffer decode(ByteBuffer chunkBytes) throws ZarrException { try { return ByteBuffer.wrap(Blosc.decompress(Utils.toArray(chunkBytes))); @@ -43,7 +43,7 @@ public ByteBuffer decode(ByteBuffer chunkBytes, ArrayMetadata.CoreArrayMetadata } @Override - public ByteBuffer encode(ByteBuffer chunkBytes, ArrayMetadata.CoreArrayMetadata arrayMetadata) + public ByteBuffer encode(ByteBuffer chunkBytes) throws ZarrException { try { return ByteBuffer.wrap( @@ -84,7 +84,7 @@ public void serialize(Blosc.Shuffle shuffle, JsonGenerator generator, generator.writeString("bitshuffle"); break; case BYTE_SHUFFLE: - generator.writeString("byteshuffle"); + generator.writeString("shuffle"); break; } } @@ -154,7 +154,7 @@ public Blosc.Shuffle deserialize(JsonParser jsonParser, DeserializationContext c return Blosc.Shuffle.NO_SHUFFLE; case "bitshuffle": return Blosc.Shuffle.BIT_SHUFFLE; - case "byteshuffle": + case "shuffle": return Blosc.Shuffle.BYTE_SHUFFLE; default: throw new JsonParseException( diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/BytesCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/BytesCodec.java index de9d1e79..1415da92 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/BytesCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/BytesCodec.java @@ -11,7 +11,7 @@ import javax.annotation.Nonnull; import ucar.ma2.Array; -public class BytesCodec implements ArrayBytesCodec { +public class BytesCodec extends ArrayBytesCodec { public final String name = "bytes"; @Nonnull @@ -29,14 +29,14 @@ public BytesCodec(Endian endian) { } @Override - public Array decode(ByteBuffer chunkBytes, ArrayMetadata.CoreArrayMetadata arrayMetadata) { + public Array decode(ByteBuffer chunkBytes) { chunkBytes.order(configuration.endian.getByteOrder()); return Array.factory(arrayMetadata.dataType.getMA2DataType(), arrayMetadata.chunkShape, chunkBytes); } @Override - public ByteBuffer encode(Array chunkArray, ArrayMetadata.CoreArrayMetadata arrayMetadata) { + public ByteBuffer encode(Array chunkArray) { return chunkArray.getDataAsByteBuffer(configuration.endian.getByteOrder()); } @@ -72,7 +72,7 @@ public ByteOrder getByteOrder() { } } - public static final class Configuration { + public static final class Configuration{ @Nonnull public final BytesCodec.Endian endian; diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/Crc32cCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/Crc32cCodec.java index 38be686c..a1e3cb52 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/Crc32cCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/Crc32cCodec.java @@ -9,17 +9,15 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; -public class Crc32cCodec implements BytesBytesCodec { +public class Crc32cCodec extends BytesBytesCodec { public final String name = "crc32c"; @JsonCreator - public Crc32cCodec( - ) { - } + public Crc32cCodec(){} @Override - public ByteBuffer decode(ByteBuffer chunkBytes, CoreArrayMetadata arrayMetadata) + public ByteBuffer decode(ByteBuffer chunkBytes) throws ZarrException { ByteBuffer buffer = chunkBytes.slice(); buffer.order(ByteOrder.LITTLE_ENDIAN); @@ -45,7 +43,7 @@ public ByteBuffer decode(ByteBuffer chunkBytes, CoreArrayMetadata arrayMetadata) } @Override - public ByteBuffer encode(ByteBuffer chunkBytes, CoreArrayMetadata arrayMetadata) { + public ByteBuffer encode(ByteBuffer chunkBytes) { return Utils.makeByteBuffer(chunkBytes.capacity() + 4, b -> { final CRC32C crc32c = new CRC32C(); crc32c.update(chunkBytes); diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/GzipCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/GzipCodec.java index 10545424..3ff5acd9 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/GzipCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/GzipCodec.java @@ -16,7 +16,7 @@ import java.util.zip.GZIPOutputStream; import javax.annotation.Nonnull; -public class GzipCodec implements BytesBytesCodec { +public class GzipCodec extends BytesBytesCodec { public final String name = "gzip"; @Nonnull @@ -37,7 +37,7 @@ private void copy(InputStream inputStream, OutputStream outputStream) throws IOE } @Override - public ByteBuffer decode(ByteBuffer chunkBytes, ArrayMetadata.CoreArrayMetadata arrayMetadata) + public ByteBuffer decode(ByteBuffer chunkBytes) throws ZarrException { try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); GZIPInputStream inputStream = new GZIPInputStream( new ByteArrayInputStream(Utils.toArray(chunkBytes)))) { @@ -50,7 +50,7 @@ public ByteBuffer decode(ByteBuffer chunkBytes, ArrayMetadata.CoreArrayMetadata } @Override - public ByteBuffer encode(ByteBuffer chunkBytes, ArrayMetadata.CoreArrayMetadata arrayMetadata) + public ByteBuffer encode(ByteBuffer chunkBytes) throws ZarrException { try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); GZIPOutputStream gzipStream = new GZIPOutputStream( outputStream)) { diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/ShardingIndexedCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/ShardingIndexedCodec.java index 75858a41..0dce7348 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/ShardingIndexedCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/ShardingIndexedCodec.java @@ -22,15 +22,13 @@ import ucar.ma2.InvalidRangeException; -public class ShardingIndexedCodec implements ArrayBytesCodec, ArrayBytesCodec.WithPartialDecode { +public class ShardingIndexedCodec extends ArrayBytesCodec.WithPartialDecode { public final String name = "sharding_indexed"; @Nonnull public final Configuration configuration; - @Nonnull - final CodecPipeline codecPipeline; - @Nonnull - final CodecPipeline indexCodecPipeline; + CodecPipeline codecPipeline; + CodecPipeline indexCodecPipeline; @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) public ShardingIndexedCodec( @@ -38,8 +36,18 @@ public ShardingIndexedCodec( Configuration configuration ) throws ZarrException { this.configuration = configuration; - this.codecPipeline = new CodecPipeline(configuration.codecs); - this.indexCodecPipeline = new CodecPipeline(configuration.indexCodecs); + } + + @Override + public void setCoreArrayMetadata(CoreArrayMetadata arrayMetadata) throws ZarrException { + super.setCoreArrayMetadata(arrayMetadata); + final ArrayMetadata.CoreArrayMetadata shardMetadata = + new ArrayMetadata.CoreArrayMetadata(Utils.toLongArray(arrayMetadata.chunkShape), + configuration.chunkShape, arrayMetadata.dataType, + arrayMetadata.parsedFillValue + ); + this.codecPipeline = new CodecPipeline(configuration.codecs, shardMetadata); + this.indexCodecPipeline = new CodecPipeline(configuration.indexCodecs, getShardIndexArrayMetadata(getChunksPerShard(arrayMetadata))); } ArrayMetadata.CoreArrayMetadata getShardIndexArrayMetadata(int[] chunksPerShard) { @@ -85,20 +93,15 @@ void setValueFromShardIndexArray(Array shardIndexArray, long[] chunkCoords, int } @Override - public Array decode(ByteBuffer shardBytes, ArrayMetadata.CoreArrayMetadata arrayMetadata) + public Array decode(ByteBuffer shardBytes) throws ZarrException { return decodeInternal(new ByteBufferDataProvider(shardBytes), new long[arrayMetadata.ndim()], arrayMetadata.chunkShape, arrayMetadata); } @Override - public ByteBuffer encode(final Array shardArray, - final ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException { - final ArrayMetadata.CoreArrayMetadata shardMetadata = - new ArrayMetadata.CoreArrayMetadata(Utils.toLongArray(arrayMetadata.chunkShape), - configuration.chunkShape, arrayMetadata.dataType, - arrayMetadata.parsedFillValue - ); + public ByteBuffer encode(final Array shardArray) throws ZarrException { + final ArrayMetadata.CoreArrayMetadata shardMetadata = codecPipeline.arrayMetadata; final int[] chunksPerShard = getChunksPerShard(arrayMetadata); final int chunkCount = Arrays.stream(chunksPerShard) .reduce(1, (r, a) -> r * a); @@ -127,11 +130,14 @@ public ByteBuffer encode(final Array shardArray, setValueFromShardIndexArray(shardIndexArray, chunkCoords, 0, -1); setValueFromShardIndexArray(shardIndexArray, chunkCoords, 1, -1); } else { - final ByteBuffer chunkBytes = codecPipeline.encode(chunkArray, shardMetadata); + final ByteBuffer chunkBytes = codecPipeline.encode(chunkArray); synchronized (chunkBytesList) { int chunkByteOffset = chunkBytesList.stream() - .mapToInt(ByteBuffer::capacity) - .sum(); + .mapToInt(ByteBuffer::capacity) + .sum(); + if (configuration.indexLocation.equals("start")) { + chunkByteOffset += (int) getShardIndexSize(arrayMetadata); + } setValueFromShardIndexArray(shardIndexArray, chunkCoords, 0, chunkByteOffset); setValueFromShardIndexArray(shardIndexArray, chunkCoords, 1, chunkBytes.capacity()); @@ -146,11 +152,15 @@ public ByteBuffer encode(final Array shardArray, .mapToInt(ByteBuffer::capacity) .sum() + (int) getShardIndexSize(arrayMetadata); final ByteBuffer shardBytes = ByteBuffer.allocate(shardBytesLength); + if(configuration.indexLocation.equals("start")){ + shardBytes.put(indexCodecPipeline.encode(shardIndexArray)); + } for (final ByteBuffer chunkBytes : chunkBytesList) { shardBytes.put(chunkBytes); } - shardBytes.put( - indexCodecPipeline.encode(shardIndexArray, getShardIndexArrayMetadata(chunksPerShard))); + if(configuration.indexLocation.equals("end")){ + shardBytes.put(indexCodecPipeline.encode(shardIndexArray)); + } shardBytes.rewind(); return shardBytes; } @@ -172,25 +182,22 @@ private Array decodeInternal( DataProvider dataProvider, long[] offset, int[] shape, ArrayMetadata.CoreArrayMetadata arrayMetadata ) throws ZarrException { + final ArrayMetadata.CoreArrayMetadata shardMetadata = codecPipeline.arrayMetadata; + final Array outputArray = Array.factory(arrayMetadata.dataType.getMA2DataType(), shape); - final int[] chunksPerShard = getChunksPerShard(arrayMetadata); final int shardIndexByteLength = (int) getShardIndexSize(arrayMetadata); - ByteBuffer shardIndexBytes = dataProvider.readSuffix(shardIndexByteLength); - + ByteBuffer shardIndexBytes; + if (this.configuration.indexLocation.equals("start")) { + shardIndexBytes = dataProvider.readPrefix(shardIndexByteLength); + }else if(this.configuration.indexLocation.equals("end")){ + shardIndexBytes = dataProvider.readSuffix(shardIndexByteLength); + }else{ + throw new ZarrException("Only index_location \"start\" or \"end\" are supported."); + } if (shardIndexBytes == null) { throw new ZarrException("Could not read shard index."); } - final Array shardIndexArray = indexCodecPipeline.decode( - shardIndexBytes, - getShardIndexArrayMetadata(chunksPerShard) - ); - - final ArrayMetadata.CoreArrayMetadata shardMetadata = - new ArrayMetadata.CoreArrayMetadata(Utils.toLongArray(arrayMetadata.chunkShape), - configuration.chunkShape, arrayMetadata.dataType, - arrayMetadata.parsedFillValue - ); - + final Array shardIndexArray = indexCodecPipeline.decode(shardIndexBytes); long[][] allChunkCoords = IndexingUtils.computeChunkCoords(shardMetadata.shape, shardMetadata.chunkShape, offset, shape); @@ -215,7 +222,7 @@ private Array decodeInternal( throw new ZarrException(String.format("Could not load byte data for chunk %s", Arrays.toString(chunkCoords))); } - chunkArray = codecPipeline.decode(chunkBytes, shardMetadata); + chunkArray = codecPipeline.decode(chunkBytes); } if (chunkArray == null) { chunkArray = shardMetadata.allocateFillValueChunk(); @@ -232,17 +239,13 @@ private Array decodeInternal( } @Override - public Array decodePartial( - StoreHandle chunkHandle, long[] offset, int[] shape, - ArrayMetadata.CoreArrayMetadata arrayMetadata - ) throws ZarrException { + public Array decodePartial(StoreHandle chunkHandle, long[] offset, int[] shape) throws ZarrException { if (Arrays.equals(shape, arrayMetadata.chunkShape)) { ByteBuffer chunkBytes = chunkHandle.read(); if (chunkBytes == null) { return arrayMetadata.allocateFillValueChunk(); } - return decodeInternal(new ByteBufferDataProvider(chunkHandle.read()), offset, shape, - arrayMetadata); + return decodeInternal(new ByteBufferDataProvider(chunkHandle.read()), offset, shape, arrayMetadata); } return decodeInternal(new StoreHandleDataProvider(chunkHandle), offset, shape, arrayMetadata); } @@ -253,6 +256,8 @@ interface DataProvider { ByteBuffer read(long start, long length); ByteBuffer readSuffix(long suffixLength); + + ByteBuffer readPrefix(long prefixLength); } public static final class Configuration { @@ -260,19 +265,32 @@ public static final class Configuration { @JsonProperty("chunk_shape") public final int[] chunkShape; @Nonnull + @JsonProperty("codecs") public final Codec[] codecs; @Nonnull + @JsonProperty("index_codecs") public final Codec[] indexCodecs; + @Nonnull + @JsonProperty("index_location") + public String indexLocation; @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) public Configuration( - @JsonProperty(value = "chunk_shape", required = true) int[] chunkShape, - @Nonnull @JsonProperty("codecs") Codec[] codecs, - @Nonnull @JsonProperty("index_codecs") Codec[] indexCodecs - ) { + @JsonProperty(value = "chunk_shape", required = true) int[] chunkShape, + @Nonnull @JsonProperty("codecs") Codec[] codecs, + @Nonnull @JsonProperty("index_codecs") Codec[] indexCodecs, + @JsonProperty(value = "index_location", defaultValue = "end") String indexLocation + ) throws ZarrException { + if (indexLocation == null) { + indexLocation = "end"; + } + if (!indexLocation.equals("start") && !indexLocation.equals("end")) { + throw new ZarrException("Only index_location \"start\" or \"end\" are supported."); + } this.chunkShape = chunkShape; this.codecs = codecs; this.indexCodecs = indexCodecs; + this.indexLocation = indexLocation; } } @@ -293,6 +311,12 @@ public ByteBuffer readSuffix(long suffixLength) { return bufferSlice.slice(); } + public ByteBuffer readPrefix(long prefixLength) { + ByteBuffer bufferSlice = buffer.slice(); + bufferSlice.limit((int) (prefixLength)); + return bufferSlice.slice(); + } + @Override public ByteBuffer read(long start, long length) { ByteBuffer bufferSlice = buffer.slice(); @@ -317,6 +341,11 @@ public ByteBuffer readSuffix(long suffixLength) { return storeHandle.read(-suffixLength); } + @Override + public ByteBuffer readPrefix(long prefixLength) { + return storeHandle.read(0, prefixLength); + } + @Override public ByteBuffer read(long start, long length) { return storeHandle.read(start, start + length); diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/TransposeCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/TransposeCodec.java index 69fb6fe6..4d614ae9 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/TransposeCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/TransposeCodec.java @@ -5,64 +5,87 @@ import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.v3.ArrayMetadata; import dev.zarr.zarrjava.v3.codec.ArrayArrayCodec; -import javax.annotation.Nonnull; import ucar.ma2.Array; -public class TransposeCodec implements ArrayArrayCodec { +import javax.annotation.Nonnull; +import java.util.Arrays; + +import static dev.zarr.zarrjava.utils.Utils.inversePermutation; +import static dev.zarr.zarrjava.utils.Utils.isPermutation; - @Nonnull - public final String name = "transpose"; - @Nonnull - public final Configuration configuration; +public class TransposeCodec extends ArrayArrayCodec { - @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) - public TransposeCodec( - @Nonnull @JsonProperty(value = "configuration", required = true) Configuration configuration - ) { - this.configuration = configuration; - } + @Nonnull + public final String name = "transpose"; + @Nonnull + public final Configuration configuration; - int[] reverseDims(int ndim) { - int[] dims = new int[ndim]; - for (int dimIdx = 0; dimIdx < ndim; dimIdx++) { - dims[dimIdx] = ndim - dimIdx - 1; + @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) + public TransposeCodec( + @Nonnull @JsonProperty(value = "configuration", required = true) Configuration configuration + ) { + this.configuration = configuration; } - return dims; - } - @Override - public Array decode(Array chunkArray, ArrayMetadata.CoreArrayMetadata arrayMetadata) { - if (configuration.order.equals("F")) { - chunkArray.permute(reverseDims(arrayMetadata.ndim())); + + @Override + public Array decode(Array chunkArray) throws ZarrException { + if (!isPermutation(configuration.order)){ + throw new ZarrException("Order is no permutation array"); + } + if (arrayMetadata.ndim() != configuration.order.length) { + throw new ZarrException("Array has not the same ndim as transpose codec order"); + } + chunkArray = chunkArray.permute(inversePermutation(configuration.order)); + return chunkArray; } - return chunkArray; - } - @Override - public Array encode(Array chunkArray, ArrayMetadata.CoreArrayMetadata arrayMetadata) { - if (configuration.order.equals("F")) { - chunkArray.permute(reverseDims(arrayMetadata.ndim())); + + + @Override + public Array encode(Array chunkArray) throws ZarrException { + if (!isPermutation(configuration.order)){ + throw new ZarrException("Order is no permutation array"); + } + if (arrayMetadata.ndim() != configuration.order.length) { + throw new ZarrException("Array has not the same ndim as transpose codec order"); + } + chunkArray = chunkArray.permute(configuration.order); + return chunkArray; } - return chunkArray; - } - @Override - public long computeEncodedSize(long inputByteLength, - ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException { - return inputByteLength; - } + @Override + public long computeEncodedSize(long inputByteLength, + ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException { + return inputByteLength; + } - public static final class Configuration { + public static final class Configuration { + public final int[] order; - public final String order; + @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) + public Configuration(@JsonProperty(value = "order") int[] order) { + this.order = order; + } + } - @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) - public Configuration(@JsonProperty(value = "order", defaultValue = "C") String order) - throws ZarrException { - if (!order.equals("C") && !order.equals("F")) { - throw new ZarrException("Only 'C' or 'F' are supported."); - } - this.order = order; + @Override + protected ArrayMetadata.CoreArrayMetadata resolveArrayMetadata() throws ZarrException { + super.resolveArrayMetadata(); + assert arrayMetadata.ndim() == configuration.order.length; + + int[] transposedChunkShape = new int[arrayMetadata.ndim()]; + Arrays.setAll(transposedChunkShape, i -> arrayMetadata.chunkShape[configuration.order[i]]); + + //only chunk shape gets transformed, the outer shape stays the same + long[] transposedArrayShape = new long[arrayMetadata.ndim()]; + Arrays.setAll(transposedArrayShape, i -> arrayMetadata.shape[i]/arrayMetadata.chunkShape[i]*transposedArrayShape[i]); + + return new ArrayMetadata.CoreArrayMetadata( + transposedArrayShape, + transposedChunkShape, + arrayMetadata.dataType, + arrayMetadata.parsedFillValue + ); } - } } diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZstdCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZstdCodec.java index 39a69ee3..f042f11c 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZstdCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZstdCodec.java @@ -2,89 +2,74 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.github.luben.zstd.ZstdInputStream; -import com.github.luben.zstd.ZstdOutputStream; +import com.github.luben.zstd.Zstd; +import com.github.luben.zstd.ZstdCompressCtx; import dev.zarr.zarrjava.ZarrException; -import dev.zarr.zarrjava.utils.Utils; import dev.zarr.zarrjava.v3.ArrayMetadata; import dev.zarr.zarrjava.v3.codec.BytesBytesCodec; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.ByteBuffer; -import javax.annotation.Nonnull; -public class ZstdCodec implements BytesBytesCodec { +import javax.annotation.Nonnull; +import java.nio.ByteBuffer; - public final String name = "zstd"; - @Nonnull - public final Configuration configuration; +public class ZstdCodec extends BytesBytesCodec { - @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) - public ZstdCodec( - @Nonnull @JsonProperty(value = "configuration", required = true) Configuration configuration) { - this.configuration = configuration; - } + public final String name = "zstd"; + @Nonnull + public final Configuration configuration; - private void copy(InputStream inputStream, OutputStream outputStream) throws IOException { - byte[] buffer = new byte[4096]; - int len; - while ((len = inputStream.read(buffer)) > 0) { - outputStream.write(buffer, 0, len); + @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) + public ZstdCodec( + @Nonnull @JsonProperty(value = "configuration", required = true) Configuration configuration) { + this.configuration = configuration; } - } - @Override - public ByteBuffer decode(ByteBuffer chunkBytes, ArrayMetadata.CoreArrayMetadata arrayMetadata) - throws ZarrException { - try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); ZstdInputStream inputStream = new ZstdInputStream( - new ByteArrayInputStream(Utils.toArray(chunkBytes)))) { - copy(inputStream, outputStream); - inputStream.close(); - return ByteBuffer.wrap(outputStream.toByteArray()); - } catch (IOException ex) { - throw new ZarrException("Error in decoding zstd.", ex); + @Override + public ByteBuffer decode(ByteBuffer compressedBytes) throws ZarrException { + byte[] compressedArray = compressedBytes.array(); + + long originalSize = Zstd.decompressedSize(compressedArray); + if (originalSize == 0) { + throw new ZarrException("Failed to get decompressed size"); + } + + byte[] decompressed = Zstd.decompress(compressedArray, (int) originalSize); + return ByteBuffer.wrap(decompressed); } - } - @Override - public ByteBuffer encode(ByteBuffer chunkBytes, ArrayMetadata.CoreArrayMetadata arrayMetadata) - throws ZarrException { - try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); ZstdOutputStream zstdStream = new ZstdOutputStream( - outputStream, configuration.level).setChecksum( - configuration.checksum)) { - zstdStream.write(Utils.toArray(chunkBytes)); - zstdStream.close(); - return ByteBuffer.wrap(outputStream.toByteArray()); - } catch (IOException ex) { - throw new ZarrException("Error in decoding zstd.", ex); + @Override + public ByteBuffer encode(ByteBuffer chunkBytes) throws ZarrException { + byte[] arr = chunkBytes.array(); + byte[] compressed; + try (ZstdCompressCtx ctx = new ZstdCompressCtx()) { + ctx.setLevel(configuration.level); + ctx.setChecksum(configuration.checksum); + compressed = ctx.compress(arr); + } + return ByteBuffer.wrap(compressed); } - } - @Override - public long computeEncodedSize(long inputByteLength, - ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException { - throw new ZarrException("Not implemented for Zstd codec."); - } + @Override + public long computeEncodedSize(long inputByteLength, + ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException { + throw new ZarrException("Not implemented for Zstd codec."); + } - public static final class Configuration { + public static final class Configuration { - public final int level; - public final boolean checksum; + public final int level; + public final boolean checksum; - @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) - public Configuration(@JsonProperty(value = "level", defaultValue = "5") int level, - @JsonProperty(value = "checksum", defaultValue = "true") boolean checksum) - throws ZarrException { - if (level < -131072 || level > 22) { - throw new ZarrException("'level' needs to be between -131072 and 22."); - } - this.level = level; - this.checksum = checksum; + @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) + public Configuration(@JsonProperty(value = "level", defaultValue = "5") int level, + @JsonProperty(value = "checksum", defaultValue = "true") boolean checksum) + throws ZarrException { + if (level < -131072 || level > 22) { + throw new ZarrException("'level' needs to be between -131072 and 22."); + } + this.level = level; + this.checksum = checksum; + } } - } } diff --git a/src/test/java/dev/zarr/zarrjava/TestUtils.java b/src/test/java/dev/zarr/zarrjava/TestUtils.java new file mode 100644 index 00000000..8165b4a7 --- /dev/null +++ b/src/test/java/dev/zarr/zarrjava/TestUtils.java @@ -0,0 +1,32 @@ +package dev.zarr.zarrjava; + + +import org.junit.Test; +import org.junit.jupiter.api.Assertions; + +import java.util.Arrays; + +import static dev.zarr.zarrjava.utils.Utils.inversePermutation; +import static dev.zarr.zarrjava.utils.Utils.isPermutation; +import static org.junit.Assert.assertFalse; + +public class TestUtils { + @Test + public void testIsPermutation(){ + assert isPermutation(new int[]{2, 1, 0}); + assert isPermutation(new int[]{4, 2, 1, 3, 0}); + assert !isPermutation(new int[]{0, 1, 2, 0}); + assert !isPermutation(new int[]{0, 1, 2, 3, 5}); + assert !isPermutation(new int[]{}); + } + + @Test + public void testInversePermutation(){ + Assertions.assertArrayEquals(new int[]{1, 0, 2}, inversePermutation(new int[]{1, 0, 2})); + Assertions.assertArrayEquals(new int[]{2, 0, 1}, inversePermutation(new int[]{1, 2, 0})); + Assertions.assertArrayEquals(new int[]{0, 3, 2, 4, 1}, inversePermutation(new int[]{0, 4, 2, 1, 3})); + Assertions.assertFalse(Arrays.equals(new int[]{2, 0, 1}, inversePermutation(new int[]{2, 0, 1}))); + } + +} + diff --git a/src/test/java/dev/zarr/zarrjava/ZarrTest.java b/src/test/java/dev/zarr/zarrjava/ZarrTest.java index a529bee2..b9632bed 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrTest.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrTest.java @@ -1,217 +1,449 @@ package dev.zarr.zarrjava; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; - import com.amazonaws.auth.AWSStaticCredentialsProvider; import com.amazonaws.auth.AnonymousAWSCredentials; import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.luben.zstd.Zstd; +import com.github.luben.zstd.ZstdCompressCtx; import dev.zarr.zarrjava.store.FilesystemStore; import dev.zarr.zarrjava.store.HttpStore; import dev.zarr.zarrjava.store.S3Store; +import dev.zarr.zarrjava.store.StoreHandle; import dev.zarr.zarrjava.utils.MultiArrayUtils; -import dev.zarr.zarrjava.v3.Array; -import dev.zarr.zarrjava.v3.ArrayMetadata; -import dev.zarr.zarrjava.v3.DataType; -import dev.zarr.zarrjava.v3.Group; -import dev.zarr.zarrjava.v3.GroupMetadata; -import dev.zarr.zarrjava.v3.Node; -import java.io.File; -import java.io.IOException; +import dev.zarr.zarrjava.v3.*; +import dev.zarr.zarrjava.v3.codec.CodecBuilder; +import dev.zarr.zarrjava.v3.codec.core.TransposeCodec; +import jdk.jshell.spi.ExecutionControl; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; +import ucar.ma2.MAMath; + +import java.io.*; +import java.nio.ByteBuffer; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Arrays; import java.util.Comparator; -import java.util.HashMap; +import java.util.Map; import java.util.stream.Stream; -import org.junit.Before; -import org.junit.Test; + +import static org.junit.Assert.assertThrows; public class ZarrTest { - final Path TESTDATA = Paths.get("testdata"); - final Path TESTOUTPUT = Paths.get("testoutput"); - - @Before - public void clearTestoutputFolder() throws IOException { - if (Files.exists(TESTOUTPUT)) { - try (Stream walk = Files.walk(TESTOUTPUT)) { - walk.sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete); - } - } - Files.createDirectory(TESTOUTPUT); - } - - @Test - public void testFileSystemStores() throws IOException, ZarrException { - FilesystemStore fsStore = new FilesystemStore(TESTDATA); - ObjectMapper objectMapper = Node.makeObjectMapper(); - - GroupMetadata group = objectMapper.readValue( - Files.readAllBytes(TESTDATA.resolve("l4_sample").resolve("zarr.json")), - GroupMetadata.class - ); - - System.out.println(group); - System.out.println(objectMapper.writeValueAsString(group)); - - ArrayMetadata arrayMetadata = objectMapper.readValue(Files.readAllBytes(TESTDATA.resolve( - "l4_sample").resolve("color").resolve("1").resolve("zarr.json")), - ArrayMetadata.class); - - System.out.println(arrayMetadata); - System.out.println(objectMapper.writeValueAsString(arrayMetadata)); - - System.out.println( - Array.open(fsStore.resolve("l4_sample", "color", "1"))); - System.out.println( - Arrays.toString(Group.open(fsStore.resolve("l4_sample")).list().toArray(Node[]::new))); - System.out.println( - Arrays.toString(((Group) Group.open(fsStore.resolve("l4_sample")).get("color")).list() - .toArray(Node[]::new))); - } - - @Test - public void testS3Store() throws IOException, ZarrException { - S3Store s3Store = new S3Store(AmazonS3ClientBuilder.standard() - .withRegion("eu-west-1") - .withCredentials(new AWSStaticCredentialsProvider(new AnonymousAWSCredentials())) - .build(), "static.webknossos.org", "data"); - System.out.println(Array.open(s3Store.resolve("zarr_v3", "l4_sample", "color", "1"))); - } - - @Test - public void testHttpStore() throws IOException, ZarrException { - HttpStore httpStore = new HttpStore("https://static.webknossos.org/data/"); - System.out.println( - dev.zarr.zarrjava.v2.Array.open(httpStore.resolve("l4_sample", "color", "1"))); - System.out.println(Array.open(httpStore.resolve("zarr_v3", "l4_sample", "color", "1"))); - } - - @Test - public void testV3ShardingReadCutout() throws IOException, ZarrException { - Array array = Array.open(new FilesystemStore(TESTDATA).resolve("l4_sample", "color", "1")); - - ucar.ma2.Array outArray = array.read(new long[]{0, 3073, 3073, 513}, new int[]{1, 64, 64, 64}); - assertEquals(outArray.getSize(), 64 * 64 * 64); - assertEquals(outArray.getByte(0), -98); - } - - @Test - public void testV3Access() throws IOException, ZarrException { - Array readArray = Array.open(new FilesystemStore(TESTDATA).resolve("l4_sample", "color", "1")); - - ucar.ma2.Array outArray = readArray.access().withOffset(0, 3073, 3073, 513) - .withShape(1, 64, 64, 64) - .read(); - assertEquals(outArray.getSize(), 64 * 64 * 64); - assertEquals(outArray.getByte(0), -98); - - Array writeArray = Array.create( - new FilesystemStore(TESTOUTPUT).resolve("l4_sample_2", "color", "1"), - readArray.metadata - ); - writeArray.access().withOffset(0, 3073, 3073, 513).write(outArray); - } - - @Test - public void testV3ShardingReadWrite() throws IOException, ZarrException { - Array readArray = Array.open( - new FilesystemStore(TESTDATA).resolve("l4_sample", "color", "8-8-2")); - ucar.ma2.Array readArrayContent = readArray.read(); - Array writeArray = Array.create( - new FilesystemStore(TESTOUTPUT).resolve("l4_sample_3", "color", "8-8-2"), - readArray.metadata - ); - writeArray.write(readArrayContent); - ucar.ma2.Array outArray = writeArray.read(); - - assert MultiArrayUtils.allValuesEqual(outArray, readArrayContent); - } - - @Test - public void testV3Codecs() throws IOException, ZarrException { - Array readArray = Array.open( - new FilesystemStore(TESTDATA).resolve("l4_sample", "color", "8-8-2")); - ucar.ma2.Array readArrayContent = readArray.read(); - { - Array gzipArray = Array.create( - new FilesystemStore(TESTOUTPUT).resolve("l4_sample_gzip", "color", "8-8-2"), - Array.metadataBuilder(readArray.metadata).withCodecs(c -> c.withGzip(5)).build() - ); - gzipArray.write(readArrayContent); - ucar.ma2.Array outGzipArray = gzipArray.read(); - assert MultiArrayUtils.allValuesEqual(outGzipArray, readArrayContent); - } - { - Array bloscArray = Array.create( - new FilesystemStore(TESTOUTPUT).resolve("l4_sample_blosc", "color", "8-8-2"), - Array.metadataBuilder(readArray.metadata).withCodecs(c -> c.withBlosc("zstd", 5)).build() - ); - bloscArray.write(readArrayContent); - ucar.ma2.Array outBloscArray = bloscArray.read(); - assert MultiArrayUtils.allValuesEqual(outBloscArray, readArrayContent); - } - { - Array zstdArray = Array.create( - new FilesystemStore(TESTOUTPUT).resolve("l4_sample_zstd", "color", "8-8-2"), - Array.metadataBuilder(readArray.metadata).withCodecs(c -> c.withZstd(10)).build() - ); - zstdArray.write(readArrayContent); - ucar.ma2.Array outZstdArray = zstdArray.read(); - assert MultiArrayUtils.allValuesEqual(outZstdArray, readArrayContent); - } - } - - @Test - public void testV3ArrayMetadataBuilder() throws ZarrException { - Array.metadataBuilder() - .withShape(1, 4096, 4096, 1536) - .withDataType(DataType.UINT32) - .withChunkShape(1, 1024, 1024, 1024) - .withFillValue(0) - .withCodecs( - c -> c.withSharding(new int[]{1, 32, 32, 32}, c1 -> c1.withBlosc())) - .build(); - } - - @Test - public void testV3FillValue() throws ZarrException { - assertEquals((int) ArrayMetadata.parseFillValue(0, DataType.UINT32), 0); - assertEquals((int) ArrayMetadata.parseFillValue("0x00010203", DataType.UINT32), 50462976); - assertEquals((byte) ArrayMetadata.parseFillValue("0b00000010", DataType.UINT8), 2); - assert Double.isNaN((double) ArrayMetadata.parseFillValue("NaN", DataType.FLOAT64)); - assert Double.isInfinite((double) ArrayMetadata.parseFillValue("-Infinity", DataType.FLOAT64)); - } - - @Test - public void testV3Group() throws IOException, ZarrException { - FilesystemStore fsStore = new FilesystemStore(TESTOUTPUT); - - Group group = Group.create(fsStore.resolve("testgroup")); - Group group2 = group.createGroup("test2", new HashMap() {{ - put("hello", "world"); - }}); - Array array = group2.createArray("array", b -> - b.withShape(10, 10) - .withDataType(DataType.UINT8) - .withChunkShape(5, 5) - ); - array.write(new long[]{2, 2}, ucar.ma2.Array.factory(ucar.ma2.DataType.UBYTE, new int[]{8, 8})); - - assertArrayEquals( - ((Array) ((Group) group.listAsArray()[0]).listAsArray()[0]).metadata.chunkShape(), - new int[]{5, 5}); - } - - @Test - public void testV2() throws IOException, ZarrException { - FilesystemStore fsStore = new FilesystemStore(""); - HttpStore httpStore = new HttpStore("https://static.webknossos.org/data"); - - System.out.println( - dev.zarr.zarrjava.v2.Array.open(httpStore.resolve("l4_sample", "color", "1"))); - } + final static Path TESTDATA = Paths.get("testdata"); + final static Path TESTOUTPUT = Paths.get("testoutput"); + final static Path PYTHON_TEST_PATH = Paths.get("src/test/python-scripts/"); + + public static String pythonPath() { + if (System.getProperty("os.name").startsWith("Windows")) { + return "venv_zarrita\\Scripts\\python.exe"; + } + return "venv_zarrita/bin/python"; + } + + @BeforeAll + public static void clearTestoutputFolder() throws IOException { + if (Files.exists(TESTOUTPUT)) { + try (Stream walk = Files.walk(TESTOUTPUT)) { + walk.sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete); + } + } + Files.createDirectory(TESTOUTPUT); + } + + @ParameterizedTest + @CsvSource({ + "blosc,blosclz_noshuffle_0", "blosc,lz4_shuffle_6", "blosc,lz4hc_bitshuffle_3", "blosc,zlib_shuffle_5", "blosc,zstd_bitshuffle_9", + "gzip,0", "gzip,5", + "zstd,0_true", "zstd,5_true","zstd,0_false", "zstd,5_false", + "bytes,BIG", "bytes,LITTLE", + "transpose,_", + "sharding,start", "sharding,end", + "sharding_nested,_", + "crc32c,_", + }) public void testReadFromZarrita(String codec, String codecParam) throws IOException, ZarrException, InterruptedException { + String command = pythonPath(); + ProcessBuilder pb = new ProcessBuilder(command, PYTHON_TEST_PATH.resolve("zarrita_write.py").toString(), codec, codecParam, TESTOUTPUT.toString()); + Process process = pb.start(); + + BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream())); + String line; + while ((line = reader.readLine()) != null) { + System.out.println(line); + } + + BufferedReader readerErr = new BufferedReader(new InputStreamReader(process.getErrorStream())); + while ((line = readerErr.readLine()) != null) { + System.err.println(line); + } + + int exitCode = process.waitFor(); + assert exitCode == 0; + + Array array = Array.open(new FilesystemStore(TESTOUTPUT).resolve("read_from_zarrita", codec, codecParam)); + ucar.ma2.Array result = array.read(); + + //for expected values see zarrita_write.py + Assertions.assertArrayEquals(new int[]{16, 16}, result.getShape()); + Assertions.assertEquals(DataType.INT32, array.metadata.dataType); + Assertions.assertArrayEquals(new int[]{2, 8}, array.metadata.chunkShape()); + Assertions.assertEquals(42, array.metadata.attributes.get("answer")); + + int[] expectedData = new int[16 * 16]; + Arrays.setAll(expectedData, p -> p); + Assertions.assertArrayEquals(expectedData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.INT)); + } + + @CsvSource({"0,true", "0,false", "5, true", "10, false"}) + @ParameterizedTest + public void testZstdLibrary(int clevel, boolean checksumFlag) throws IOException, InterruptedException { + //compress using ZstdCompressCtx + int number = 123456; + byte[] src = ByteBuffer.allocate(4).putInt(number).array(); + byte[] compressed; + try (ZstdCompressCtx ctx = new ZstdCompressCtx()) { + ctx.setLevel(clevel); + ctx.setChecksum(checksumFlag); + compressed = ctx.compress(src); + } + //decompress with Zstd.decompress + long originalSize = Zstd.decompressedSize(compressed); + byte[] decompressed = Zstd.decompress(compressed, (int) originalSize); + Assertions.assertEquals(number, ByteBuffer.wrap(decompressed).getInt()); + + //write compressed to file + String compressedDataPath = TESTOUTPUT.resolve("compressed" + clevel + checksumFlag + ".bin").toString(); + try (FileOutputStream fos = new FileOutputStream(compressedDataPath)) { + fos.write(compressed); + } + + //decompress in python + Process process = new ProcessBuilder( + pythonPath(), + PYTHON_TEST_PATH.resolve("zstd_decompress.py").toString(), + compressedDataPath, + Integer.toString(number) + ).start(); + int exitCode = process.waitFor(); + assert exitCode == 0; + } + + @ParameterizedTest + @CsvSource({ + "blosc,blosclz_noshuffle_0", "blosc,lz4_shuffle_6", "blosc,lz4hc_bitshuffle_3", "blosc,zlib_shuffle_5", "blosc,zstd_bitshuffle_9", + "gzip,0", "gzip,5", + "zstd,0_true", "zstd,5_true","zstd,0_false", "zstd,5_false", + "bytes,BIG", "bytes,LITTLE", + "transpose,_", + "sharding,start", "sharding,end", + "sharding_nested,_", + "crc32c,_", + }) + public void testWriteReadWithZarrita(String codec, String codecParam) throws Exception { + int[] testData = new int[16 * 16 * 16]; + Arrays.setAll(testData, p -> p); + + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("write_to_zarrita", codec, codecParam); + ArrayMetadataBuilder builder = Array.metadataBuilder() + .withShape(16, 16, 16) + .withDataType(DataType.UINT32) + .withChunkShape(2, 4, 8) + .withFillValue(0) + .withAttributes(Map.of("test_key", "test_value")); + + switch (codec) { + case "blosc": + String cname = codecParam.split("_")[0]; + String shuffle = codecParam.split("_")[1]; + int clevel_blosc = Integer.parseInt(codecParam.split("_")[2]); + builder = builder.withCodecs(c -> c.withBlosc(cname, shuffle, clevel_blosc)); + break; + case "gzip": + builder = builder.withCodecs(c -> c.withGzip(Integer.parseInt(codecParam))); + break; + case "zstd": + int clevel_zstd = Integer.parseInt(codecParam.split("_")[0]); + boolean checksum = Boolean.parseBoolean(codecParam.split("_")[1]); + builder = builder.withCodecs(c -> c.withZstd(clevel_zstd, checksum)); + break; + case "bytes": + builder = builder.withCodecs(c -> c.withBytes(codecParam)); + break; + case "transpose": + builder = builder.withCodecs(c -> c.withTranspose(new int[]{1, 0, 2})); + break; + case "sharding": + builder = builder.withCodecs(c -> c.withSharding(new int[]{2, 2, 4}, c1 -> c1.withBytes("LITTLE"), codecParam)); + break; + case "sharding_nested": + builder = builder.withCodecs(c -> c.withSharding(new int[]{2, 2, 4}, c1 -> c1.withSharding(new int[]{2, 1, 2}, c2 -> c2.withBytes("LITTLE")))); + break; + case "crc32c": + builder = builder.withCodecs(CodecBuilder::withCrc32c); + break; + default: + throw new IllegalArgumentException("Invalid Codec: " + codec); + } + + Array writeArray = Array.create(storeHandle, builder.build()); + writeArray.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{16, 16, 16}, testData)); + + //read in zarr-java + Array readArray = Array.open(storeHandle); + ucar.ma2.Array result = readArray.read(); + + Assertions.assertArrayEquals(new int[]{16, 16, 16}, result.getShape()); + Assertions.assertEquals(DataType.UINT32, readArray.metadata.dataType); + Assertions.assertArrayEquals(new int[]{2, 4, 8}, readArray.metadata.chunkShape()); + Assertions.assertEquals("test_value", readArray.metadata.attributes.get("test_key")); + + Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.INT)); + + //read in zarrita + String command = pythonPath(); + + ProcessBuilder pb = new ProcessBuilder(command, PYTHON_TEST_PATH.resolve("zarrita_read.py").toString(), codec, codecParam, TESTOUTPUT.toString()); + Process process = pb.start(); + + BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream())); + String line; + while ((line = reader.readLine()) != null) { + System.out.println(line); + } + + BufferedReader readerErr = new BufferedReader(new InputStreamReader(process.getErrorStream())); + while ((line = readerErr.readLine()) != null) { + System.err.println(line); + } + + int exitCode = process.waitFor(); + assert exitCode == 0; + } + + @ParameterizedTest + @CsvSource({"0,true", "0,false", "5, true", "5, false"}) + public void testZstdCodecReadWrite(int clevel, boolean checksum) throws ZarrException, IOException { + int[] testData = new int[16 * 16 * 16]; + Arrays.setAll(testData, p -> p); + + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testZstdCodecReadWrite", "checksum_" + checksum, "clevel_" + clevel); + ArrayMetadataBuilder builder = Array.metadataBuilder() + .withShape(16, 16, 16) + .withDataType(DataType.UINT32) + .withChunkShape(2, 4, 8) + .withFillValue(0) + .withCodecs(c -> c.withZstd(clevel, checksum)); + Array writeArray = Array.create(storeHandle, builder.build()); + writeArray.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{16, 16, 16}, testData)); + + Array readArray = Array.open(storeHandle); + ucar.ma2.Array result = readArray.read(); + + Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.INT)); + } + + @Test + public void testTransposeCodec() throws ZarrException { + ucar.ma2.Array testData = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{2, 3, 3}, new int[]{ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}); + ucar.ma2.Array testDataTransposed120 = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{3, 3, 2}, new int[]{ + 0, 9, 1, 10, 2, 11, 3, 12, 4, 13, 5, 14, 6, 15, 7, 16, 8, 17}); + + ArrayMetadata.CoreArrayMetadata metadata = new ArrayMetadata.CoreArrayMetadata( + new long[]{2, 3, 3}, + new int[]{2, 3, 3}, + DataType.UINT32, + null); + TransposeCodec transposeCodec = new TransposeCodec(new TransposeCodec.Configuration(new int[]{1, 2, 0})); + TransposeCodec transposeCodecWrongOrder1 = new TransposeCodec(new TransposeCodec.Configuration(new int[]{1, 2, 2})); + TransposeCodec transposeCodecWrongOrder2 = new TransposeCodec(new TransposeCodec.Configuration(new int[]{1, 2, 3})); + TransposeCodec transposeCodecWrongOrder3 = new TransposeCodec(new TransposeCodec.Configuration(new int[]{1, 2, 3, 0})); + transposeCodec.setCoreArrayMetadata(metadata); + transposeCodecWrongOrder1.setCoreArrayMetadata(metadata); + transposeCodecWrongOrder2.setCoreArrayMetadata(metadata); + transposeCodecWrongOrder3.setCoreArrayMetadata(metadata); + + assert MAMath.equals(testDataTransposed120, transposeCodec.encode(testData)); + assert MAMath.equals(testData, transposeCodec.decode(testDataTransposed120)); + assertThrows(ZarrException.class, () -> transposeCodecWrongOrder1.encode(testData)); + assertThrows(ZarrException.class, () -> transposeCodecWrongOrder2.encode(testData)); + assertThrows(ZarrException.class, () -> transposeCodecWrongOrder3.encode(testData)); + } + + @Test + public void testFileSystemStores() throws IOException, ZarrException { + FilesystemStore fsStore = new FilesystemStore(TESTDATA); + ObjectMapper objectMapper = Node.makeObjectMapper(); + + GroupMetadata group = objectMapper.readValue( + Files.readAllBytes(TESTDATA.resolve("l4_sample").resolve("zarr.json")), + GroupMetadata.class + ); + + System.out.println(group); + System.out.println(objectMapper.writeValueAsString(group)); + + ArrayMetadata arrayMetadata = objectMapper.readValue(Files.readAllBytes(TESTDATA.resolve( + "l4_sample").resolve("color").resolve("1").resolve("zarr.json")), + ArrayMetadata.class); + + System.out.println(arrayMetadata); + System.out.println(objectMapper.writeValueAsString(arrayMetadata)); + + System.out.println( + Array.open(fsStore.resolve("l4_sample", "color", "1"))); + System.out.println( + Arrays.toString(Group.open(fsStore.resolve("l4_sample")).list().toArray(Node[]::new))); + System.out.println( + Arrays.toString(((Group) Group.open(fsStore.resolve("l4_sample")).get("color")).list() + .toArray(Node[]::new))); + } + + @Test + public void testS3Store() throws IOException, ZarrException { + S3Store s3Store = new S3Store(AmazonS3ClientBuilder.standard() + .withRegion("eu-west-1") + .withCredentials(new AWSStaticCredentialsProvider(new AnonymousAWSCredentials())) + .build(), "static.webknossos.org", "data"); + System.out.println(Array.open(s3Store.resolve("zarr_v3", "l4_sample", "color", "1"))); + } + + @Test + public void testHttpStore() throws IOException, ZarrException { + HttpStore httpStore = new HttpStore("https://static.webknossos.org/data/"); + System.out.println( + dev.zarr.zarrjava.v2.Array.open(httpStore.resolve("l4_sample", "color", "1"))); + System.out.println(Array.open(httpStore.resolve("zarr_v3", "l4_sample", "color", "1"))); + } + + @Test + public void testV3ShardingReadCutout() throws IOException, ZarrException { + Array array = Array.open(new FilesystemStore(TESTDATA).resolve("l4_sample", "color", "1")); + + ucar.ma2.Array outArray = array.read(new long[]{0, 3073, 3073, 513}, new int[]{1, 64, 64, 64}); + Assertions.assertEquals(outArray.getSize(), 64 * 64 * 64); + Assertions.assertEquals(outArray.getByte(0), -98); + } + + @Test + public void testV3Access() throws IOException, ZarrException { + Array readArray = Array.open(new FilesystemStore(TESTDATA).resolve("l4_sample", "color", "1")); + + ucar.ma2.Array outArray = readArray.access().withOffset(0, 3073, 3073, 513) + .withShape(1, 64, 64, 64) + .read(); + Assertions.assertEquals(outArray.getSize(), 64 * 64 * 64); + Assertions.assertEquals(outArray.getByte(0), -98); + + Array writeArray = Array.create( + new FilesystemStore(TESTOUTPUT).resolve("l4_sample_2", "color", "1"), + readArray.metadata + ); + writeArray.access().withOffset(0, 3073, 3073, 513).write(outArray); + } + + @ParameterizedTest + @ValueSource(strings = {"start", "end"}) + public void testV3ShardingReadWrite(String indexLocation) throws IOException, ZarrException { + Array readArray = Array.open( + new FilesystemStore(TESTDATA).resolve("sharding_index_location", indexLocation)); + ucar.ma2.Array readArrayContent = readArray.read(); + Array writeArray = Array.create( + new FilesystemStore(TESTOUTPUT).resolve("sharding_index_location", indexLocation), + readArray.metadata + ); + writeArray.write(readArrayContent); + ucar.ma2.Array outArray = writeArray.read(); + + assert MultiArrayUtils.allValuesEqual(readArrayContent, outArray); + } + + @Test + public void testV3Codecs() throws IOException, ZarrException { + int[] readShape = new int[]{1, 1, 1024, 1024}; + Array readArray = Array.open( + new FilesystemStore(TESTDATA).resolve("l4_sample", "color", "8-8-2")); + ucar.ma2.Array readArrayContent = readArray.read(new long[4], readShape); + { + Array gzipArray = Array.create( + new FilesystemStore(TESTOUTPUT).resolve("l4_sample_gzip", "color", "8-8-2"), + Array.metadataBuilder(readArray.metadata).withCodecs(c -> c.withGzip(5)).build() + ); + gzipArray.write(readArrayContent); + ucar.ma2.Array outGzipArray = gzipArray.read(new long[4], readShape); + assert MultiArrayUtils.allValuesEqual(outGzipArray, readArrayContent); + } + { + Array bloscArray = Array.create( + new FilesystemStore(TESTOUTPUT).resolve("l4_sample_blosc", "color", "8-8-2"), + Array.metadataBuilder(readArray.metadata).withCodecs(c -> c.withBlosc("zstd", 5)).build() + ); + bloscArray.write(readArrayContent); + ucar.ma2.Array outBloscArray = bloscArray.read(new long[4], readShape); + assert MultiArrayUtils.allValuesEqual(outBloscArray, readArrayContent); + } + { + Array zstdArray = Array.create( + new FilesystemStore(TESTOUTPUT).resolve("l4_sample_zstd", "color", "8-8-2"), + Array.metadataBuilder(readArray.metadata).withCodecs(c -> c.withZstd(10)).build() + ); + zstdArray.write(readArrayContent); + ucar.ma2.Array outZstdArray = zstdArray.read(new long[4], readShape); + assert MultiArrayUtils.allValuesEqual(outZstdArray, readArrayContent); + } + } + + @Test + public void testV3ArrayMetadataBuilder() throws ZarrException { + Array.metadataBuilder() + .withShape(1, 4096, 4096, 1536) + .withDataType(DataType.UINT32) + .withChunkShape(1, 1024, 1024, 1024) + .withFillValue(0) + .withCodecs( + c -> c.withSharding(new int[]{1, 32, 32, 32}, CodecBuilder::withBlosc)) + .build(); + } + + @Test + public void testV3FillValue() throws ZarrException { + Assertions.assertEquals((int) ArrayMetadata.parseFillValue(0, DataType.UINT32), 0); + Assertions.assertEquals((int) ArrayMetadata.parseFillValue("0x00010203", DataType.UINT32), 50462976); + Assertions.assertEquals((byte) ArrayMetadata.parseFillValue("0b00000010", DataType.UINT8), 2); + assert Double.isNaN((double) ArrayMetadata.parseFillValue("NaN", DataType.FLOAT64)); + assert Double.isInfinite((double) ArrayMetadata.parseFillValue("-Infinity", DataType.FLOAT64)); + } + + @Test + public void testV3Group() throws IOException, ZarrException { + FilesystemStore fsStore = new FilesystemStore(TESTOUTPUT); + + Group group = Group.create(fsStore.resolve("testgroup")); + Group group2 = group.createGroup("test2", Map.of("hello", "world")); + Array array = group2.createArray("array", b -> + b.withShape(10, 10) + .withDataType(DataType.UINT8) + .withChunkShape(5, 5) + ); + array.write(new long[]{2, 2}, ucar.ma2.Array.factory(ucar.ma2.DataType.UBYTE, new int[]{8, 8})); + + Assertions.assertArrayEquals(((Array) ((Group) group.listAsArray()[0]).listAsArray()[0]).metadata.chunkShape(), new int[]{5, 5}); + } + + @Test + public void testV2() throws IOException { + FilesystemStore fsStore = new FilesystemStore(""); + HttpStore httpStore = new HttpStore("https://static.webknossos.org/data"); + + System.out.println(dev.zarr.zarrjava.v2.Array.open(httpStore.resolve("l4_sample", "color", "1"))); + } + + } diff --git a/src/test/python-scripts/zarrita_read.py b/src/test/python-scripts/zarrita_read.py new file mode 100644 index 00000000..f84bf9bd --- /dev/null +++ b/src/test/python-scripts/zarrita_read.py @@ -0,0 +1,53 @@ +import sys + +import numpy as np +import zarrita +from zarrita.metadata import ShardingCodecIndexLocation + +codec_string = sys.argv[1] +param_string = sys.argv[2] +codec = [] +if codec_string == "blosc": + cname, shuffle, clevel = param_string.split("_") + codec = [zarrita.codecs.bytes_codec(), + zarrita.codecs.blosc_codec(typesize=4, cname=cname, shuffle=shuffle, clevel=int(clevel))] +elif codec_string == "gzip": + codec = [zarrita.codecs.bytes_codec(), zarrita.codecs.gzip_codec(level=int(param_string))] +elif codec_string == "zstd": + level, checksum = param_string.split("_") + codec = [zarrita.codecs.bytes_codec(), zarrita.codecs.zstd_codec(checksum=checksum == 'true', level=int(level))] +elif codec_string == "bytes": + codec = [zarrita.codecs.bytes_codec(endian=param_string.lower())] +elif codec_string == "transpose": + codec = [zarrita.codecs.transpose_codec((1, 0, 2)), zarrita.codecs.bytes_codec()] +elif codec_string == "sharding": + codec = zarrita.codecs.sharding_codec(chunk_shape=(2, 2, 4), codecs=[zarrita.codecs.bytes_codec("little")], + index_location=ShardingCodecIndexLocation.start if param_string == "start" + else ShardingCodecIndexLocation.end), +elif codec_string == "sharding_nested": + codec = zarrita.codecs.sharding_codec(chunk_shape=(2, 2, 4), + codecs=[zarrita.codecs.sharding_codec(chunk_shape=(2, 1, 2), codecs=[ + zarrita.codecs.bytes_codec("little")])]), +elif codec_string == "crc32c": + codec = [zarrita.codecs.bytes_codec(), zarrita.codecs.crc32c_codec()] +else: + raise ValueError(f"Invalid {codec=}") + +store = zarrita.LocalStore(sys.argv[3]) +expected_data = np.arange(16 * 16 * 16, dtype='int32').reshape(16, 16, 16) + +a = zarrita.Array.open(store / 'write_to_zarrita' / codec_string / param_string) +read_data = a[:, :] +assert np.array_equal(read_data, expected_data), f"got:\n {read_data} \nbut expected:\n {expected_data}" + +b = zarrita.Array.create( + store / 'read_from_zarrita_expected' / codec_string / param_string, + shape=(16, 16, 16), + chunk_shape=(2, 4, 8), + dtype="uint32", + fill_value=0, + attributes={'test_key': 'test_value'}, + codecs=codec +) + +assert a.metadata == b.metadata, f"not equal: \n{a.metadata=}\n{b.metadata=}" diff --git a/src/test/python-scripts/zarrita_write.py b/src/test/python-scripts/zarrita_write.py new file mode 100644 index 00000000..2eb0fc23 --- /dev/null +++ b/src/test/python-scripts/zarrita_write.py @@ -0,0 +1,47 @@ +import sys + +import zarrita +import numpy as np +from zarrita.metadata import ShardingCodecIndexLocation + +codec_string = sys.argv[1] +param_string = sys.argv[2] +codec = [] +if codec_string == "blosc": + cname, shuffle, clevel = param_string.split("_") + codec = [zarrita.codecs.bytes_codec(), + zarrita.codecs.blosc_codec(typesize=4, cname=cname, shuffle=shuffle, clevel=int(clevel))] +elif codec_string == "gzip": + codec = [zarrita.codecs.bytes_codec(), zarrita.codecs.gzip_codec(level=int(param_string))] +elif codec_string == "zstd": + level, checksum = param_string.split("_") + codec = [zarrita.codecs.bytes_codec(), zarrita.codecs.zstd_codec(checksum=checksum == 'true', level=int(level))] +elif codec_string == "bytes": + codec = [zarrita.codecs.bytes_codec(endian=param_string.lower())] +elif codec_string == "transpose": + codec = [zarrita.codecs.transpose_codec((0, 1)), zarrita.codecs.bytes_codec()] +elif codec_string == "sharding": + codec = zarrita.codecs.sharding_codec(chunk_shape=(2, 4), codecs=[zarrita.codecs.bytes_codec("little")], + index_location=ShardingCodecIndexLocation.start if param_string == "start" + else ShardingCodecIndexLocation.end), +elif codec_string == "sharding_nested": + codec = zarrita.codecs.sharding_codec(chunk_shape=(2, 4), + codecs=[zarrita.codecs.sharding_codec(chunk_shape=(1, 2), codecs=[ + zarrita.codecs.bytes_codec("little")])]), +elif codec_string == "crc32c": + codec = [zarrita.codecs.bytes_codec(), zarrita.codecs.crc32c_codec()] +else: + raise ValueError(f"Invalid {codec_string=}") + +store = zarrita.LocalStore(sys.argv[3]) +testdata = np.arange(16 * 16, dtype='int32').reshape((16, 16)) + +a = zarrita.Array.create( + store / 'read_from_zarrita' / codec_string / param_string, + shape=(16, 16), + dtype='int32', + chunk_shape=(2, 8), + codecs=codec, + attributes={'answer': 42} +) +a[:, :] = testdata diff --git a/src/test/python-scripts/zstd_decompress.py b/src/test/python-scripts/zstd_decompress.py new file mode 100644 index 00000000..0235fdd6 --- /dev/null +++ b/src/test/python-scripts/zstd_decompress.py @@ -0,0 +1,13 @@ +import sys + +import zstandard as zstd + +data_path = sys.argv[1] +expected = sys.argv[2] + +with open(data_path, "rb") as f: + compressed = f.read() + +decompressed = zstd.ZstdDecompressor().decompress(compressed) +number = int.from_bytes(decompressed, byteorder='big') +assert number == int(expected) diff --git a/testdata/sharding_index_location/end/c/0/0/0 b/testdata/sharding_index_location/end/c/0/0/0 new file mode 100644 index 00000000..5704d5a5 Binary files /dev/null and b/testdata/sharding_index_location/end/c/0/0/0 differ diff --git a/testdata/sharding_index_location/end/c/0/0/1 b/testdata/sharding_index_location/end/c/0/0/1 new file mode 100644 index 00000000..c93d72d9 Binary files /dev/null and b/testdata/sharding_index_location/end/c/0/0/1 differ diff --git a/testdata/sharding_index_location/end/c/0/1/0 b/testdata/sharding_index_location/end/c/0/1/0 new file mode 100644 index 00000000..699a7712 Binary files /dev/null and b/testdata/sharding_index_location/end/c/0/1/0 differ diff --git a/testdata/sharding_index_location/end/c/0/1/1 b/testdata/sharding_index_location/end/c/0/1/1 new file mode 100644 index 00000000..19a595ae Binary files /dev/null and b/testdata/sharding_index_location/end/c/0/1/1 differ diff --git a/testdata/sharding_index_location/end/zarr.json b/testdata/sharding_index_location/end/zarr.json new file mode 100644 index 00000000..72e3a3f3 --- /dev/null +++ b/testdata/sharding_index_location/end/zarr.json @@ -0,0 +1,76 @@ +{ + "shape": [ + 16, + 16, + 16 + ], + "data_type": "int32", + "chunk_grid": { + "configuration": { + "chunk_shape": [ + 16, + 8, + 8 + ] + }, + "name": "regular" + }, + "chunk_key_encoding": { + "configuration": { + "separator": "/" + }, + "name": "default" + }, + "fill_value": 0, + "codecs": [ + { + "configuration": { + "chunk_shape": [ + 8, + 4, + 8 + ], + "codecs": [ + { + "configuration": { + "order": [2, 1, 0] + }, + "name": "transpose" + }, + { + "configuration": { + "endian": "little" + }, + "name": "bytes" + }, + { + "configuration": { + "typesize": 4, + "cname": "lz4", + "clevel": 5, + "shuffle": "noshuffle", + "blocksize": 0 + }, + "name": "blosc" + } + ], + "index_codecs": [ + { + "configuration": { + "endian": "little" + }, + "name": "bytes" + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + }, + "name": "sharding_indexed" + } + ], + "attributes": {}, + "zarr_format": 3, + "node_type": "array" +} \ No newline at end of file diff --git a/testdata/sharding_index_location/start/c/0/0/0 b/testdata/sharding_index_location/start/c/0/0/0 new file mode 100644 index 00000000..e8280074 Binary files /dev/null and b/testdata/sharding_index_location/start/c/0/0/0 differ diff --git a/testdata/sharding_index_location/start/c/0/0/1 b/testdata/sharding_index_location/start/c/0/0/1 new file mode 100644 index 00000000..06d4db3c Binary files /dev/null and b/testdata/sharding_index_location/start/c/0/0/1 differ diff --git a/testdata/sharding_index_location/start/c/0/1/0 b/testdata/sharding_index_location/start/c/0/1/0 new file mode 100644 index 00000000..898e31e5 Binary files /dev/null and b/testdata/sharding_index_location/start/c/0/1/0 differ diff --git a/testdata/sharding_index_location/start/c/0/1/1 b/testdata/sharding_index_location/start/c/0/1/1 new file mode 100644 index 00000000..cf8c2924 Binary files /dev/null and b/testdata/sharding_index_location/start/c/0/1/1 differ diff --git a/testdata/sharding_index_location/start/zarr.json b/testdata/sharding_index_location/start/zarr.json new file mode 100644 index 00000000..f9923e75 --- /dev/null +++ b/testdata/sharding_index_location/start/zarr.json @@ -0,0 +1,76 @@ +{ + "shape": [ + 16, + 16, + 16 + ], + "data_type": "int32", + "chunk_grid": { + "configuration": { + "chunk_shape": [ + 16, + 8, + 8 + ] + }, + "name": "regular" + }, + "chunk_key_encoding": { + "configuration": { + "separator": "/" + }, + "name": "default" + }, + "fill_value": 0, + "codecs": [ + { + "configuration": { + "chunk_shape": [ + 8, + 4, + 8 + ], + "codecs": [ + { + "configuration": { + "order": [2, 1, 0] + }, + "name": "transpose" + }, + { + "configuration": { + "endian": "little" + }, + "name": "bytes" + }, + { + "configuration": { + "typesize": 4, + "cname": "lz4", + "clevel": 5, + "shuffle": "noshuffle", + "blocksize": 0 + }, + "name": "blosc" + } + ], + "index_codecs": [ + { + "configuration": { + "endian": "little" + }, + "name": "bytes" + }, + { + "name": "crc32c" + } + ], + "index_location": "start" + }, + "name": "sharding_indexed" + } + ], + "attributes": {}, + "zarr_format": 3, + "node_type": "array" +} \ No newline at end of file