diff --git a/src/main/java/dev/zarr/zarrjava/core/ArrayMetadata.java b/src/main/java/dev/zarr/zarrjava/core/ArrayMetadata.java index aaa004e..e19d193 100644 --- a/src/main/java/dev/zarr/zarrjava/core/ArrayMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/core/ArrayMetadata.java @@ -41,6 +41,8 @@ public int ndim() { public abstract Object parsedFillValue(); + public @Nonnull abstract Attributes attributes() throws ZarrException; + public static Object parseFillValue(Object fillValue, @Nonnull DataType dataType) throws ZarrException { if (fillValue == null) { diff --git a/src/main/java/dev/zarr/zarrjava/core/Attributes.java b/src/main/java/dev/zarr/zarrjava/core/Attributes.java new file mode 100644 index 0000000..bc1cfe2 --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/core/Attributes.java @@ -0,0 +1,241 @@ +package dev.zarr.zarrjava.core; + +import dev.zarr.zarrjava.ZarrException; + +import javax.annotation.Nonnull; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; + +public class Attributes extends HashMap { + + public Attributes() { + super(); + } + + public Attributes (Function attributeMapper) { + super(); + attributeMapper.apply(this); + } + + + public Attributes(Map attributes) { + super(attributes); + } + + public Attributes set(String s, Object o){ + this.put(s, o); + return this; + } + + public Attributes delete(String s){ + this.remove(s); + return this; + } + + public boolean getBoolean(String key) { + Object value = this.get(key); + if (value instanceof Boolean) { + return (Boolean) value; + } + throw new IllegalArgumentException("Value for key " + key + " is not a Boolean"); + } + + public int getInt(String key) { + Object value = this.get(key); + if (value instanceof Number) { + return ((Number) value).intValue(); + } + throw new IllegalArgumentException("Value for key " + key + " is not an Integer"); + } + + public double getDouble(String key) { + Object value = this.get(key); + if (value instanceof Number) { + return ((Number) value).doubleValue(); + } + throw new IllegalArgumentException("Value for key " + key + " is not a Double"); + } + + public float getFloat(String key) { + Object value = this.get(key); + if (value instanceof Number) { + return ((Number) value).floatValue(); + } + throw new IllegalArgumentException("Value for key " + key + " is not a Float"); + } + + @Nonnull + public String getString(String key) { + Object value = this.get(key); + if (value instanceof String) { + return (String) value; + } + throw new IllegalArgumentException("Value for key " + key + " is not a String"); + } + + @Nonnull + public List getList(String key) { + Object value = this.get(key); + if (value instanceof List) { + return (List) value; + } + throw new IllegalArgumentException("Value for key " + key + " is not a List"); + } + + public Attributes getAttributes(String key) { + Object value = this.get(key); + if (value instanceof Attributes) { + return (Attributes) value; + } + if (value instanceof Map) { + return new Attributes((Map) value); + } + throw new IllegalArgumentException("Value for key " + key + " is not an Attributes object"); + } + + public T[] getArray(String key, Class clazz) throws ZarrException { + Object value = this.get(key); + if (value instanceof Object[] && (((Object[]) value).length == 0 || clazz.isInstance(((Object[]) value)[0]) )) { + return (T[]) value; + } + if (value instanceof List) { + List list = (List) value; + @SuppressWarnings("unchecked") + T[] array = (T[]) java.lang.reflect.Array.newInstance(clazz, list.size()); + for (int i = 0; i < list.size(); i++) { + Object elem = list.get(i); + if (clazz.isInstance(elem)) { + array[i] = clazz.cast(elem); + } else { + // Try to find a constructor that takes the element's class + java.lang.reflect.Constructor matched = null; + for (java.lang.reflect.Constructor c : clazz.getConstructors()) { + Class[] params = c.getParameterTypes(); + if (params.length == 1 && params[0].isAssignableFrom(elem.getClass())) { + matched = c; + break; + } + } + if (matched != null) { + try { + array[i] = (T) matched.newInstance(elem); + } catch (Exception e) { + throw new ZarrException("Failed to convert element at index " + i + " to type " + clazz.getName(), e); + } + } else { + throw new IllegalArgumentException("Element at index " + i + " is not of type " + clazz.getName() + " and no suitable constructor found for conversion of type " + elem.getClass().getName()); + } + } + } + return array; + } + throw new IllegalArgumentException("Value for key " + key + " is not a List or array of type " + clazz.getName()); +} + + public int[] getIntArray(String key) { + Object value = this.get(key); + if (value instanceof int[]) { + return (int[]) value; + } + if (value instanceof List) { + List list = (List) value; + int[] array = new int[list.size()]; + for (int i = 0; i < list.size(); i++) { + Object elem = list.get(i); + if (elem instanceof Number) { + array[i] = ((Number) elem).intValue(); + } else { + throw new IllegalArgumentException("Element at index " + i + " is not a Number"); + } + } + return array; + } + throw new IllegalArgumentException("Value for key " + key + " is not an int array or List"); + } + + public long[] getLongArray(String key) { + Object value = this.get(key); + if (value instanceof long[]) { + return (long[]) value; + } + if (value instanceof List) { + List list = (List) value; + long[] array = new long[list.size()]; + for (int i = 0; i < list.size(); i++) { + Object elem = list.get(i); + if (elem instanceof Number) { + array[i] = ((Number) elem).longValue(); + } else { + throw new IllegalArgumentException("Element at index " + i + " is not a Number"); + } + } + return array; + } + throw new IllegalArgumentException("Value for key " + key + " is not a long array or List"); + } + + public double[] getDoubleArray(String key) { + Object value = this.get(key); + if (value instanceof double[]) { + return (double[]) value; + } + if (value instanceof List) { + List list = (List) value; + double[] array = new double[list.size()]; + for (int i = 0; i < list.size(); i++) { + Object elem = list.get(i); + if (elem instanceof Number) { + array[i] = ((Number) elem).doubleValue(); + } else { + throw new IllegalArgumentException("Element at index " + i + " is not a Number"); + } + } + return array; + } + throw new IllegalArgumentException("Value for key " + key + " is not a double array or List"); + } + + public float[] getFloatArray(String key) { + Object value = this.get(key); + if (value instanceof float[]) { + return (float[]) value; + } + if (value instanceof List) { + List list = (List) value; + float[] array = new float[list.size()]; + for (int i = 0; i < list.size(); i++) { + Object elem = list.get(i); + if (elem instanceof Number) { + array[i] = ((Number) elem).floatValue(); + } else { + throw new IllegalArgumentException("Element at index " + i + " is not a Number"); + } + } + return array; + } + throw new IllegalArgumentException("Value for key " + key + " is not a float array or List"); + } + + public boolean[] getBooleanArray(String key) { + Object value = this.get(key); + if (value instanceof boolean[]) { + return (boolean[]) value; + } + if (value instanceof List) { + List list = (List) value; + boolean[] array = new boolean[list.size()]; + for (int i = 0; i < list.size(); i++) { + Object elem = list.get(i); + if (elem instanceof Boolean) { + array[i] = (Boolean) elem; + } else { + throw new IllegalArgumentException("Element at index " + i + " is not a Boolean"); + } + } + return array; + } + throw new IllegalArgumentException("Value for key " + key + " is not a boolean array or List"); + } +} \ No newline at end of file diff --git a/src/main/java/dev/zarr/zarrjava/core/Node.java b/src/main/java/dev/zarr/zarrjava/core/Node.java index 0b51193..fbac2ba 100644 --- a/src/main/java/dev/zarr/zarrjava/core/Node.java +++ b/src/main/java/dev/zarr/zarrjava/core/Node.java @@ -13,6 +13,7 @@ public interface Node { String ZARR_JSON = "zarr.json"; String ZARRAY = ".zarray"; + String ZATTRS = ".zattrs"; String ZGROUP = ".zgroup"; /** diff --git a/src/main/java/dev/zarr/zarrjava/v2/Array.java b/src/main/java/dev/zarr/zarrjava/v2/Array.java index 638d648..237e67b 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/Array.java +++ b/src/main/java/dev/zarr/zarrjava/v2/Array.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.core.Attributes; import dev.zarr.zarrjava.store.FilesystemStore; import dev.zarr.zarrjava.store.StoreHandle; import dev.zarr.zarrjava.utils.Utils; @@ -46,13 +47,19 @@ protected Array(StoreHandle storeHandle, ArrayMetadata arrayMetadata) throws IOE * @throws ZarrException throws ZarrException if the Zarr array cannot be opened */ public static Array open(StoreHandle storeHandle) throws IOException, ZarrException { + ObjectMapper mapper = makeObjectMapper(); + ArrayMetadata metadata = mapper.readValue( + Utils.toArray(storeHandle.resolve(ZARRAY).readNonNull()), + ArrayMetadata.class + ); + if (storeHandle.resolve(ZATTRS).exists()) + metadata.attributes = mapper.readValue( + Utils.toArray(storeHandle.resolve(ZATTRS).readNonNull()), + Attributes.class + ); return new Array( storeHandle, - makeObjectMapper() - .readValue( - Utils.toArray(storeHandle.resolve(ZARRAY).readNonNull()), - ArrayMetadata.class - ) + metadata ); } @@ -143,6 +150,12 @@ public static Array create(StoreHandle storeHandle, ArrayMetadata arrayMetadata, } ObjectMapper objectMapper = makeObjectMapper(); ByteBuffer metadataBytes = ByteBuffer.wrap(objectMapper.writeValueAsBytes(arrayMetadata)); + if (arrayMetadata.attributes != null) { + StoreHandle attrsHandle = storeHandle.resolve(ZATTRS); + ByteBuffer attrsBytes = ByteBuffer.wrap( + objectMapper.writeValueAsBytes(arrayMetadata.attributes)); + attrsHandle.set(attrsBytes); + } metadataHandle.set(metadataBytes); return new Array(storeHandle, arrayMetadata); } @@ -164,6 +177,61 @@ public static ArrayMetadataBuilder metadataBuilder(ArrayMetadata existingMetadat return ArrayMetadataBuilder.fromArrayMetadata(existingMetadata); } + private Array writeMetadata(ArrayMetadata newArrayMetadata) throws ZarrException, IOException { + return Array.create(storeHandle, newArrayMetadata, true); + } + + /** + * Sets a new shape for the Zarr array. It only changes the metadata, no array data is modified or + * deleted. This method returns a new instance of the Zarr array class and the old instance + * becomes invalid. + * + * @param newShape the new shape of the Zarr array + * @throws ZarrException if the new metadata is invalid + * @throws IOException throws IOException if the new metadata cannot be serialized + */ + public Array resize(long[] newShape) throws ZarrException, IOException { + if (newShape.length != metadata.ndim()) { + throw new IllegalArgumentException( + "'newShape' needs to have rank '" + metadata.ndim() + "'."); + } + + ArrayMetadata newArrayMetadata = ArrayMetadataBuilder.fromArrayMetadata(metadata) + .withShape(newShape) + .build(); + return writeMetadata(newArrayMetadata); + } + + /** + * Sets the attributes of the Zarr array. It overwrites and removes any existing attributes. This + * method returns a new instance of the Zarr array class and the old instance becomes invalid. + * + * @param newAttributes the new attributes of the Zarr array + * @throws ZarrException throws ZarrException if the new metadata is invalid + * @throws IOException throws IOException if the new metadata cannot be serialized + */ + public Array setAttributes(Attributes newAttributes) throws ZarrException, IOException { + ArrayMetadata newArrayMetadata = + ArrayMetadataBuilder.fromArrayMetadata(metadata, false) + .withAttributes(newAttributes) + .build(); + return writeMetadata(newArrayMetadata); + } + + /** + * Updates the attributes of the Zarr array. It provides a callback that gets the current + * attributes as input and needs to return the new set of attributes. The attributes in the + * callback may be mutated. This method overwrites and removes any existing attributes. This + * method returns a new instance of the Zarr array class and the old instance becomes invalid. + * + * @param attributeMapper the callback that is used to construct the new attributes + * @throws ZarrException throws ZarrException if the new metadata is invalid + * @throws IOException throws IOException if the new metadata cannot be serialized + */ + public Array updateAttributes(Function attributeMapper) throws ZarrException, IOException { + return setAttributes(attributeMapper.apply(metadata.attributes)); + } + @Override public String toString() { return String.format("", storeHandle, diff --git a/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java b/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java index dabd285..b53eee3 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java @@ -4,6 +4,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.core.Attributes; import dev.zarr.zarrjava.core.chunkkeyencoding.ChunkKeyEncoding; import dev.zarr.zarrjava.utils.MultiArrayUtils; import dev.zarr.zarrjava.core.chunkkeyencoding.Separator; @@ -11,6 +12,7 @@ import dev.zarr.zarrjava.v2.codec.Codec; import ucar.ma2.Array; +import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -39,6 +41,10 @@ public class ArrayMetadata extends dev.zarr.zarrjava.core.ArrayMetadata { @Nullable public final Codec compressor; + @Nullable + @JsonIgnore + public Attributes attributes; + @JsonIgnore public CoreArrayMetadata coreArrayMetadata; @@ -54,6 +60,22 @@ public ArrayMetadata( @Nullable @JsonProperty(value = "filters", required = true) Codec[] filters, @Nullable @JsonProperty(value = "compressor", required = true) Codec compressor, @Nullable @JsonProperty(value = "dimension_separator") Separator dimensionSeparator + ) throws ZarrException { + this(zarrFormat, shape, chunks, dataType, fillValue, order, filters, compressor, dimensionSeparator, null); + } + + + public ArrayMetadata( + int zarrFormat, + long[] shape, + int[] chunks, + DataType dataType, + @Nullable Object fillValue, + Order order, + @Nullable Codec[] filters, + @Nullable Codec compressor, + @Nullable Separator dimensionSeparator, + @Nullable Attributes attributes ) throws ZarrException { super(shape, fillValue, dataType); if (zarrFormat != this.zarrFormat) { @@ -78,6 +100,7 @@ public ArrayMetadata( } } this.compressor = compressor == null ? null : compressor.evolveFromCoreArrayMetadata(this.coreArrayMetadata); + this.attributes = attributes; } @@ -87,8 +110,6 @@ public int[] chunkShape() { return chunks; } - - @Override public DataType dataType() { return dataType; @@ -111,4 +132,14 @@ public ChunkKeyEncoding chunkKeyEncoding() { public Object parsedFillValue() { return parsedFillValue; } + + @Override + public @Nonnull Attributes attributes() throws ZarrException { + if (attributes == null) { + throw new ZarrException("Array attributes have not been set."); + } + return attributes; + } + + } diff --git a/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java b/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java index ea29b16..7f27567 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java +++ b/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java @@ -2,6 +2,7 @@ import com.scalableminds.bloscjava.Blosc; import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.core.Attributes; import dev.zarr.zarrjava.core.chunkkeyencoding.Separator; import dev.zarr.zarrjava.v2.codec.Codec; import dev.zarr.zarrjava.v2.codec.core.BloscCodec; @@ -16,12 +17,17 @@ public class ArrayMetadataBuilder { Object fillValue = null; Codec[] filters = null; Codec compressor = null; + Attributes attributes = new Attributes(); protected ArrayMetadataBuilder() { } protected static ArrayMetadataBuilder fromArrayMetadata(ArrayMetadata arrayMetadata) { + return fromArrayMetadata(arrayMetadata, true); + } + + protected static ArrayMetadataBuilder fromArrayMetadata(ArrayMetadata arrayMetadata, boolean withAttributes) { ArrayMetadataBuilder builder = new ArrayMetadataBuilder(); builder.shape = arrayMetadata.shape; builder.chunks = arrayMetadata.chunks; @@ -31,6 +37,9 @@ protected static ArrayMetadataBuilder fromArrayMetadata(ArrayMetadata arrayMetad builder.fillValue = arrayMetadata.parsedFillValue; builder.filters = arrayMetadata.filters; builder.compressor = arrayMetadata.compressor; + if (withAttributes) { + builder.attributes = arrayMetadata.attributes; + } return builder; } @@ -119,6 +128,19 @@ public ArrayMetadataBuilder withZlibCompressor() { return withZlibCompressor(5); } + public ArrayMetadataBuilder putAttribute(String key, Object value) { + this.attributes.put(key, value); + return this; + } + + public ArrayMetadataBuilder withAttributes(Attributes attributes) { + if (this.attributes == null) { + this.attributes = attributes; + } else { + this.attributes.putAll(attributes); + } + return this; + } public ArrayMetadata build() throws ZarrException { if (shape == null) { throw new IllegalStateException("Please call `withShape` first."); @@ -138,7 +160,8 @@ public ArrayMetadata build() throws ZarrException { order, filters, compressor, - dimensionSeparator + dimensionSeparator, + attributes ); } } \ No newline at end of file diff --git a/src/main/java/dev/zarr/zarrjava/v3/Array.java b/src/main/java/dev/zarr/zarrjava/v3/Array.java index 4d1434f..e30cd31 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/Array.java +++ b/src/main/java/dev/zarr/zarrjava/v3/Array.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.core.Attributes; import dev.zarr.zarrjava.store.FilesystemStore; import dev.zarr.zarrjava.store.StoreHandle; import dev.zarr.zarrjava.utils.Utils; @@ -11,8 +12,6 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; import java.util.function.Function; import java.util.stream.Collectors; import javax.annotation.Nonnull; @@ -182,11 +181,7 @@ public static ArrayMetadataBuilder metadataBuilder(ArrayMetadata existingMetadat } private Array writeMetadata(ArrayMetadata newArrayMetadata) throws ZarrException, IOException { - ObjectMapper objectMapper = makeObjectMapper(); - ByteBuffer metadataBytes = ByteBuffer.wrap(objectMapper.writeValueAsBytes(newArrayMetadata)); - storeHandle.resolve(ZARR_JSON) - .set(metadataBytes); - return new Array(storeHandle, newArrayMetadata); + return Array.create(storeHandle, newArrayMetadata, true); } /** @@ -218,9 +213,9 @@ public Array resize(long[] newShape) throws ZarrException, IOException { * @throws ZarrException throws ZarrException if the new metadata is invalid * @throws IOException throws IOException if the new metadata cannot be serialized */ - public Array setAttributes(Map newAttributes) throws ZarrException, IOException { + public Array setAttributes(Attributes newAttributes) throws ZarrException, IOException { ArrayMetadata newArrayMetadata = - ArrayMetadataBuilder.fromArrayMetadata(metadata) + ArrayMetadataBuilder.fromArrayMetadata(metadata, false) .withAttributes(newAttributes) .build(); return writeMetadata(newArrayMetadata); @@ -234,12 +229,10 @@ public Array setAttributes(Map newAttributes) throws ZarrExcepti * * @param attributeMapper the callback that is used to construct the new attributes * @throws ZarrException throws ZarrException if the new metadata is invalid - * @throws IOException throws IOException if the new metadata cannot be serialized + * @throws IOException throws IOException if the new metadata cannot be serialized */ - public Array updateAttributes(Function, Map> attributeMapper) - throws ZarrException, IOException { - return setAttributes(attributeMapper.apply(new HashMap(metadata.attributes) { - })); + public Array updateAttributes(Function attributeMapper) throws ZarrException, IOException { + return setAttributes(attributeMapper.apply(metadata.attributes)); } @Override diff --git a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java index 3625348..eb50032 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java @@ -4,6 +4,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.core.Attributes; import dev.zarr.zarrjava.v3.chunkgrid.ChunkGrid; import dev.zarr.zarrjava.v3.chunkgrid.RegularChunkGrid; import dev.zarr.zarrjava.v3.chunkkeyencoding.ChunkKeyEncoding; @@ -39,7 +40,7 @@ public final class ArrayMetadata extends dev.zarr.zarrjava.core.ArrayMetadata { public final Codec[] codecs; @Nullable @JsonProperty("attributes") - public final Map attributes; + public final Attributes attributes; @Nullable @JsonProperty("dimension_names") public final String[] dimensionNames; @@ -55,7 +56,7 @@ public ArrayMetadata( Object fillValue, @Nonnull Codec[] codecs, @Nullable String[] dimensionNames, - @Nullable Map attributes, + @Nullable Attributes attributes, @Nullable Map[] storageTransformers ) throws ZarrException { this(ZARR_FORMAT, NODE_TYPE, shape, dataType, chunkGrid, chunkKeyEncoding, fillValue, codecs, @@ -75,7 +76,7 @@ public ArrayMetadata( @JsonProperty(value = "fill_value", required = true) Object fillValue, @Nonnull @JsonProperty(value = "codecs") Codec[] codecs, @Nullable @JsonProperty(value = "dimension_names") String[] dimensionNames, - @Nullable @JsonProperty(value = "attributes") Map attributes, + @Nullable @JsonProperty(value = "attributes") Attributes attributes, @Nullable @JsonProperty(value = "storage_transformers") Map[] storageTransformers ) throws ZarrException { super(shape, fillValue, dataType); @@ -143,6 +144,15 @@ public Object parsedFillValue() { return parsedFillValue; } + @Nonnull + @Override + public Attributes attributes() throws ZarrException { + if (attributes == null) { + throw new ZarrException("Array attributes have not been set."); + } + return attributes; + } + public static Optional getShardingIndexedCodec(Codec[] codecs) { return Arrays.stream(codecs).filter(codec -> codec instanceof ShardingIndexedCodec).findFirst(); } diff --git a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java index 212f473..9f12466 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java +++ b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java @@ -1,6 +1,7 @@ package dev.zarr.zarrjava.v3; import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.core.Attributes; import dev.zarr.zarrjava.v3.chunkgrid.ChunkGrid; import dev.zarr.zarrjava.v3.chunkgrid.RegularChunkGrid; import dev.zarr.zarrjava.v3.chunkkeyencoding.ChunkKeyEncoding; @@ -26,7 +27,7 @@ public class ArrayMetadataBuilder { Object fillValue = 0; Codec[] codecs = new Codec[]{new BytesCodec(Endian.LITTLE)}; - Map attributes = new HashMap<>(); + Attributes attributes = new Attributes(); Map[] storageTransformers = new HashMap[]{}; String[] dimensionNames = null; @@ -34,6 +35,10 @@ protected ArrayMetadataBuilder() { } protected static ArrayMetadataBuilder fromArrayMetadata(ArrayMetadata arrayMetadata) { + return fromArrayMetadata(arrayMetadata, true); + } + + protected static ArrayMetadataBuilder fromArrayMetadata(ArrayMetadata arrayMetadata, boolean withAttributes) { ArrayMetadataBuilder builder = new ArrayMetadataBuilder(); builder.shape = arrayMetadata.shape; builder.dataType = arrayMetadata.dataType; @@ -41,9 +46,11 @@ protected static ArrayMetadataBuilder fromArrayMetadata(ArrayMetadata arrayMetad builder.chunkKeyEncoding = arrayMetadata.chunkKeyEncoding; builder.fillValue = arrayMetadata.parsedFillValue; builder.codecs = arrayMetadata.codecs; - builder.attributes = arrayMetadata.attributes; builder.dimensionNames = arrayMetadata.dimensionNames; builder.storageTransformers = arrayMetadata.storageTransformers; + if (withAttributes) { + builder.attributes = arrayMetadata.attributes; + } return builder; } @@ -121,8 +128,12 @@ public ArrayMetadataBuilder putAttribute(String key, Object value) { return this; } - public ArrayMetadataBuilder withAttributes(Map attributes) { - this.attributes = attributes; + public ArrayMetadataBuilder withAttributes(Attributes attributes) { + if (this.attributes == null) { + this.attributes = attributes; + } else { + this.attributes.putAll(attributes); + } return this; } diff --git a/src/main/java/dev/zarr/zarrjava/v3/Group.java b/src/main/java/dev/zarr/zarrjava/v3/Group.java index 9c56f25..c5e4cbc 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/Group.java +++ b/src/main/java/dev/zarr/zarrjava/v3/Group.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.core.Attributes; import dev.zarr.zarrjava.store.FilesystemStore; import dev.zarr.zarrjava.store.StoreHandle; import dev.zarr.zarrjava.utils.Utils; @@ -53,7 +54,7 @@ public static Group create( public static Group create( @Nonnull StoreHandle storeHandle, - @Nonnull Map attributes + @Nonnull Attributes attributes ) throws IOException, ZarrException { return new Group(storeHandle, new GroupMetadata(attributes)); } @@ -93,7 +94,7 @@ public Group createGroup(String key, GroupMetadata groupMetadata) return Group.create(storeHandle.resolve(key), groupMetadata); } - public Group createGroup(String key, Map attributes) + public Group createGroup(String key, Attributes attributes) throws IOException, ZarrException { return Group.create(storeHandle.resolve(key), new GroupMetadata(attributes)); } @@ -121,12 +122,12 @@ private Group writeMetadata(GroupMetadata newGroupMetadata) throws IOException { return new Group(storeHandle, newGroupMetadata); } - public Group setAttributes(Map newAttributes) throws ZarrException, IOException { + public Group setAttributes(Attributes newAttributes) throws ZarrException, IOException { GroupMetadata newGroupMetadata = new GroupMetadata(newAttributes); return writeMetadata(newGroupMetadata); } - public Group updateAttributes(Function, Map> attributeMapper) + public Group updateAttributes(Function attributeMapper) throws ZarrException, IOException { return setAttributes(attributeMapper.apply(metadata.attributes)); } diff --git a/src/main/java/dev/zarr/zarrjava/v3/GroupMetadata.java b/src/main/java/dev/zarr/zarrjava/v3/GroupMetadata.java index df32c04..5792213 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/GroupMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/v3/GroupMetadata.java @@ -3,8 +3,8 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import dev.zarr.zarrjava.ZarrException; -import java.util.HashMap; -import java.util.Map; +import dev.zarr.zarrjava.core.Attributes; + import javax.annotation.Nullable; public final class GroupMetadata extends dev.zarr.zarrjava.core.GroupMetadata { @@ -17,9 +17,9 @@ public final class GroupMetadata extends dev.zarr.zarrjava.core.GroupMetadata { public final String nodeType = "group"; @Nullable - public final Map attributes; + public final Attributes attributes; - public GroupMetadata(@Nullable Map attributes) throws ZarrException { + public GroupMetadata(@Nullable Attributes attributes) throws ZarrException { this(ZARR_FORMAT, NODE_TYPE, attributes); } @@ -27,7 +27,7 @@ public GroupMetadata(@Nullable Map attributes) throws ZarrExcept public GroupMetadata( @JsonProperty(value = "zarr_format", required = true) int zarrFormat, @JsonProperty(value = "node_type", required = true) String nodeType, - @Nullable @JsonProperty(value = "attributes") Map attributes + @Nullable @JsonProperty(value = "attributes") Attributes attributes ) throws ZarrException { if (zarrFormat != this.zarrFormat) { throw new ZarrException( @@ -41,6 +41,6 @@ public GroupMetadata( } public static GroupMetadata defaultValue() throws ZarrException { - return new GroupMetadata(ZARR_FORMAT, NODE_TYPE, new HashMap<>()); + return new GroupMetadata(ZARR_FORMAT, NODE_TYPE, new Attributes()); } } diff --git a/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java b/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java index e4e763a..4a17d74 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java @@ -2,6 +2,7 @@ import com.github.luben.zstd.Zstd; import com.github.luben.zstd.ZstdCompressCtx; +import dev.zarr.zarrjava.core.Attributes; import dev.zarr.zarrjava.store.FilesystemStore; import dev.zarr.zarrjava.store.StoreHandle; import dev.zarr.zarrjava.v2.Group; @@ -209,7 +210,7 @@ public void testReadV3(String codec, String codecParam, DataType dataType) throw @ParameterizedTest @MethodSource("compressorAndDataTypeProviderV3") public void testWriteV3(String codec, String codecParam, DataType dataType) throws Exception { - Map attributes = new HashMap<>(); + Attributes attributes = new Attributes(); attributes.put("test_key", "test_value"); StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testWriteV3", codec, codecParam, dataType.name()); @@ -314,7 +315,7 @@ public void testReadV2(String compressor, String compressorParam, dev.zarr.zarrj Assertions.assertArrayEquals(new int[]{16, 16, 16}, result.getShape()); Assertions.assertEquals(dt, array.metadata().dataType); Assertions.assertArrayEquals(new int[]{2, 4, 8}, array.metadata().chunkShape()); -// Assertions.assertEquals(42, array.metadata().attributes.get("answer")); + Assertions.assertEquals(42, array.metadata().attributes().get("answer")); assertIsTestdata(result, dt); } @@ -323,15 +324,15 @@ public void testReadV2(String compressor, String compressorParam, dev.zarr.zarrj @ParameterizedTest @MethodSource("compressorAndDataTypeProviderV2") public void testWriteV2(String compressor, String compressorParam, dev.zarr.zarrjava.v2.DataType dt) throws Exception { -// Map attributes = new HashMap<>(); -// attributes.put("test_key", "test_value"); + Attributes attributes = new Attributes(); + attributes.put("test_key", "test_value"); StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testCodecsWriteV2", compressor, compressorParam, dt.name()); dev.zarr.zarrjava.v2.ArrayMetadataBuilder builder = dev.zarr.zarrjava.v2.Array.metadataBuilder() .withShape(16, 16, 16) .withDataType(dt) .withChunks(2, 4, 8) -// .withAttributes(attributes) + .withAttributes(attributes) .withFillValue(0); switch (compressor) { @@ -358,7 +359,7 @@ public void testWriteV2(String compressor, String compressorParam, dev.zarr.zarr Assertions.assertArrayEquals(new int[]{16, 16, 16}, result.getShape()); Assertions.assertEquals(dt, readArray.metadata().dataType); Assertions.assertArrayEquals(new int[]{2, 4, 8}, readArray.metadata().chunkShape()); -// Assertions.assertEquals("test_value", readArray.metadata.attributes.get("test_key")); + Assertions.assertEquals("test_value", readArray.metadata().attributes().get("test_key")); assertIsTestdata(result, dt); //read in zarr_python diff --git a/src/test/java/dev/zarr/zarrjava/ZarrTest.java b/src/test/java/dev/zarr/zarrjava/ZarrTest.java index b40d516..8a6309d 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrTest.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrTest.java @@ -1,13 +1,16 @@ package dev.zarr.zarrjava; +import dev.zarr.zarrjava.core.Attributes; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; -import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.ArrayList; import java.util.Comparator; +import java.util.List; import java.util.stream.Stream; public class ZarrTest { @@ -30,4 +33,83 @@ public static void clearTestoutputFolder() throws IOException { } Files.createDirectory(TESTOUTPUT); } + + protected void assertListEquals(List a, List b) { + Assertions.assertEquals(a.size(), b.size()); + for (int i = 0; i < a.size(); i++) { + Object aval = a.get(i); + Object bval = b.get(i); + if (aval instanceof List && bval instanceof List) { + assertListEquals((List) aval, (List) bval); + } else { + Assertions.assertEquals(aval, bval); + } + } + } + + protected Attributes defaultTestAttributes() { + return new Attributes() {{ + put("string", "stringvalue"); + put("int", 42); + put("float", 0.5f); + put("double", 3.14); + put("boolean", true); + put("list", new ArrayList() { + { + add(1); + add(2.0d); + add("string"); + } + }); + put("int_array", new int[]{1, 2, 3}); + put("long_array", new long[]{1, 2, 3}); + put("double_array", new double[]{1.0, 2.0, 3.0}); + put("float_array", new float[]{1.0f, 2.0f, 3.0f}); + put("boolean_array", new boolean[]{true, false, true}); + put("nested", new Attributes() {{ + put("element", "value"); + }}); + put("array_of_attributes", new Attributes[]{ + new Attributes() {{ + put("a", 1); + }}, + new Attributes() {{ + put("b", 2); + }} + }); + }}; + } + + protected void assertContainsTestAttributes(Attributes attributes) throws ZarrException { + Assertions.assertEquals("stringvalue", attributes.getString("string")); + Assertions.assertEquals(42, attributes.getInt("int")); + Assertions.assertEquals(0.5, attributes.getFloat("float")); + Assertions.assertEquals(3.14, attributes.getDouble("double")); + Assertions.assertTrue(attributes.getBoolean("boolean")); + assertListEquals(new ArrayList() { + { + add(1); + add(2.0d); + add("string"); + } + }, attributes.getList("list")); + Assertions.assertArrayEquals(new int[]{1, 2, 3}, attributes.getIntArray("int_array")); + Assertions.assertArrayEquals(new long[]{1, 2, 3}, attributes.getLongArray("long_array")); + Assertions.assertArrayEquals(new double[]{1, 2, 3}, attributes.getDoubleArray("double_array")); + Assertions.assertArrayEquals(new float[]{1, 2, 3}, attributes.getFloatArray("float_array")); + Assertions.assertArrayEquals(new boolean[]{true, false, true}, attributes.getBooleanArray("boolean_array")); + Assertions.assertEquals("value", attributes.getAttributes("nested").getString("element")); + Assertions.assertArrayEquals( + new Attributes[]{ + new Attributes() {{ + put("a", 1); + }}, + new Attributes() {{ + put("b", 2); + }} + }, + attributes.getArray("array_of_attributes", Attributes.class) + ); + } + } diff --git a/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java b/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java index 765c51a..7fd1852 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java @@ -1,12 +1,9 @@ package dev.zarr.zarrjava; +import dev.zarr.zarrjava.core.Attributes; import dev.zarr.zarrjava.store.FilesystemStore; import dev.zarr.zarrjava.store.StoreHandle; -import dev.zarr.zarrjava.v2.Array; -import dev.zarr.zarrjava.v2.ArrayMetadata; -import dev.zarr.zarrjava.v2.DataType; -import dev.zarr.zarrjava.v2.Group; -import dev.zarr.zarrjava.v2.Node; +import dev.zarr.zarrjava.v2.*; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -18,6 +15,7 @@ import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.Arrays; public class ZarrV2Test extends ZarrTest { @ParameterizedTest @@ -255,4 +253,91 @@ public void testCreateGroup() throws ZarrException, IOException { Group.create(storeHandleString); Assertions.assertTrue(Files.exists(Paths.get(storeHandleString).resolve(".zgroup"))); } + + @Test + public void testAttributes() throws IOException, ZarrException { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testAttributesV2"); + + ArrayMetadata arrayMetadata = Array.metadataBuilder() + .withShape(10, 10) + .withDataType(DataType.UINT8) + .withChunks(5, 5) + .putAttribute("specific", "attribute") + .withAttributes(defaultTestAttributes()) + .withAttributes(new Attributes() {{ + put("another", "attribute"); + }}) + .build(); + + Array array = Array.create(storeHandle, arrayMetadata); + assertContainsTestAttributes(array.metadata().attributes()); + Assertions.assertEquals("attribute", array.metadata().attributes().getString("specific")); + Assertions.assertEquals("attribute", array.metadata().attributes().getString("another")); + + Array arrayOpened = Array.open(storeHandle); + assertContainsTestAttributes(array.metadata().attributes()); + Assertions.assertEquals("attribute", arrayOpened.metadata().attributes().getString("another")); + Assertions.assertEquals("attribute", arrayOpened.metadata().attributes().getString("specific")); + } + + @Test + public void testSetAndUpdateAttributes() throws IOException, ZarrException { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testSetAttributesV2"); + + ArrayMetadata arrayMetadata = Array.metadataBuilder() + .withShape(10, 10) + .withDataType(DataType.UINT8) + .withChunks(5, 5) + .withAttributes(new Attributes(b -> b.set("some", "value"))) + .build(); + + Array array = Array.create(storeHandle, arrayMetadata); + Assertions.assertEquals("value", array.metadata().attributes().getString("some")); + array.setAttributes(defaultTestAttributes()); + array = Array.open(storeHandle); + assertContainsTestAttributes(array.metadata().attributes()); + Assertions.assertNull(array.metadata().attributes().get("some")); + + // add attribute + array = array.updateAttributes(b -> b.set("new_attribute", "new_value")); + Assertions.assertEquals("new_value", array.metadata().attributes().getString("new_attribute")); + array = Array.open(storeHandle); + Assertions.assertEquals("new_value", array.metadata().attributes().getString("new_attribute")); + + // delete attribute + array = array.updateAttributes(b -> b.delete("new_value")); + Assertions.assertNull(array.metadata().attributes().get("new_value")); + array = Array.open(storeHandle); + Assertions.assertNull(array.metadata().attributes().get("new_value")); + + assertContainsTestAttributes(array.metadata().attributes()); + } + + @Test + public void testResizeArray() throws IOException, ZarrException { + int[] testData = new int[10 * 10]; + Arrays.setAll(testData, p -> p); + + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testResizeArrayV2"); + ArrayMetadata arrayMetadata = Array.metadataBuilder() + .withShape(10, 10) + .withDataType(DataType.UINT32) + .withChunks(5, 5) + .withFillValue(1) + .build(); + ucar.ma2.DataType ma2DataType = arrayMetadata.dataType.getMA2DataType(); + Array array = Array.create(storeHandle, arrayMetadata); + array.write(new long[]{0, 0}, ucar.ma2.Array.factory(ma2DataType, new int[]{10, 10}, testData)); + + array = array.resize(new long[]{20, 15}); + Assertions.assertArrayEquals(new int[]{20, 15}, array.read().getShape()); + + ucar.ma2.Array data = array.read(new long[]{0, 0}, new int[]{10, 10}); + Assertions.assertArrayEquals(testData, (int[]) data.get1DJavaArray(ma2DataType)); + + data = array.read(new long[]{10, 10}, new int[]{5, 5}); + int[] expectedData = new int[5 * 5]; + Arrays.fill(expectedData, 1); + Assertions.assertArrayEquals(expectedData, (int[]) data.get1DJavaArray(ma2DataType)); + } } \ No newline at end of file diff --git a/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java b/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java index 7f1c15c..10569e1 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java @@ -1,7 +1,6 @@ package dev.zarr.zarrjava; -import dev.zarr.zarrjava.v3.codec.core.BloscCodec; -import dev.zarr.zarrjava.v3.codec.core.ShardingIndexedCodec; +import dev.zarr.zarrjava.core.Attributes; import com.fasterxml.jackson.databind.JsonMappingException; import dev.zarr.zarrjava.store.*; @@ -9,8 +8,7 @@ import dev.zarr.zarrjava.v3.Node; import dev.zarr.zarrjava.v3.*; import dev.zarr.zarrjava.v3.codec.CodecBuilder; -import dev.zarr.zarrjava.v3.codec.core.BytesCodec; -import dev.zarr.zarrjava.v3.codec.core.TransposeCodec; +import dev.zarr.zarrjava.v3.codec.core.*; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -512,7 +510,7 @@ public void testOpenOverloads() throws ZarrException, IOException { public void testGroup() throws IOException, ZarrException { FilesystemStore fsStore = new FilesystemStore(TESTOUTPUT); - Map attributes = new HashMap<>(); + Attributes attributes = new Attributes(); attributes.put("hello", "world"); Group group = Group.create(fsStore.resolve("testgroup")); @@ -553,7 +551,7 @@ public void testCreateGroup() throws ZarrException, IOException { StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testCreateGroupV3"); Path storeHandlePath = TESTOUTPUT.resolve("testCreateGroupV3Path"); String storeHandleString = String.valueOf(TESTOUTPUT.resolve("testCreateGroupV3String")); - Map attributes = new HashMap<>(); + Attributes attributes = new Attributes(); attributes.put("hello", "world"); Group group = Group.create(storeHandle, new GroupMetadata(attributes)); @@ -568,7 +566,92 @@ public void testCreateGroup() throws ZarrException, IOException { Assertions.assertTrue(Files.exists(Paths.get(storeHandleString).resolve("zarr.json"))); Assertions.assertEquals("world", group.metadata.attributes.get("hello")); } + @Test + public void testAttributes() throws IOException, ZarrException { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testAttributesV3"); + + ArrayMetadata arrayMetadata = Array.metadataBuilder() + .withShape(10, 10) + .withDataType(DataType.UINT8) + .withChunkShape(5, 5) + .putAttribute("specific", "attribute") + .withAttributes(defaultTestAttributes()) + .withAttributes(new Attributes() {{ + put("another", "attribute"); + }}) + + .build(); + + Array array = Array.create(storeHandle, arrayMetadata); + assertContainsTestAttributes(array.metadata().attributes()); + Assertions.assertEquals("attribute", array.metadata().attributes().getString("specific")); + Assertions.assertEquals("attribute", array.metadata().attributes().getString("another")); + + Array arrayOpened = Array.open(storeHandle); + assertContainsTestAttributes(arrayOpened.metadata().attributes()); + Assertions.assertEquals("attribute", arrayOpened.metadata().attributes().getString("specific")); + Assertions.assertEquals("attribute", arrayOpened.metadata().attributes().getString("another")); + } + + + @Test + public void testSetAndUpdateAttributes() throws IOException, ZarrException { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testSetAttributesV3"); + + ArrayMetadata arrayMetadata = Array.metadataBuilder() + .withShape(10, 10) + .withDataType(DataType.UINT8) + .withChunkShape(5, 5) + .withAttributes(new Attributes(b -> b.set("some", "value"))) + .build(); + Array array = Array.create(storeHandle, arrayMetadata); + Assertions.assertEquals("value", array.metadata().attributes().getString("some")); + array.setAttributes(defaultTestAttributes()); + array = Array.open(storeHandle); + assertContainsTestAttributes(array.metadata().attributes()); + Assertions.assertNull(array.metadata().attributes().get("some")); + + // add attribute + array = array.updateAttributes(b -> b.set("new_attribute", "new_value")); + Assertions.assertEquals("new_value", array.metadata().attributes().getString("new_attribute")); + array = Array.open(storeHandle); + Assertions.assertEquals("new_value", array.metadata().attributes().getString("new_attribute")); + + // delete attribute + array = array.updateAttributes(b -> b.delete("new_value")); + Assertions.assertNull(array.metadata().attributes().get("new_value")); + array = Array.open(storeHandle); + Assertions.assertNull(array.metadata().attributes().get("new_value")); + + assertContainsTestAttributes(array.metadata().attributes()); + } + @Test + public void testResizeArray() throws IOException, ZarrException { + int[] testData = new int[10 * 10]; + Arrays.setAll(testData, p -> p); + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testResizeArrayV3"); + ArrayMetadata arrayMetadata = Array.metadataBuilder() + .withShape(10, 10) + .withDataType(DataType.UINT32) + .withChunkShape(5, 5) + .withFillValue(1) + .build(); + ucar.ma2.DataType ma2DataType = arrayMetadata.dataType.getMA2DataType(); + Array array = Array.create(storeHandle, arrayMetadata); + array.write(new long[]{0, 0}, ucar.ma2.Array.factory(ma2DataType, new int[]{10, 10}, testData)); + + array = array.resize(new long[]{20, 15}); + Assertions.assertArrayEquals(new int[]{20, 15}, array.read().getShape()); + + ucar.ma2.Array data = array.read(new long[]{0, 0}, new int[]{10, 10}); + Assertions.assertArrayEquals(testData, (int[]) data.get1DJavaArray(ma2DataType)); + + data = array.read(new long[]{10, 10}, new int[]{5, 5}); + int[] expectedData = new int[5 * 5]; + Arrays.fill(expectedData, 1); + Assertions.assertArrayEquals(expectedData, (int[]) data.get1DJavaArray(ma2DataType)); + } } diff --git a/src/test/python-scripts/zarr_python_read_v2.py b/src/test/python-scripts/zarr_python_read_v2.py index 5ff5e5a..5d0b1c6 100644 --- a/src/test/python-scripts/zarr_python_read_v2.py +++ b/src/test/python-scripts/zarr_python_read_v2.py @@ -33,7 +33,7 @@ filters=filters, serializer=serializer, compressors=compressor, -# attributes={'test_key': 'test_value'}, + attributes={'test_key': 'test_value'}, ) assert a.metadata == b.metadata, f"not equal: \n{a.metadata=}\n{b.metadata=}"