diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 18a1e44d..a60db644 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -91,6 +91,7 @@ jobs: cargo hack check --all \ --exclude numcodecs-python \ --exclude numcodecs-jpeg2000 \ + --exclude numcodecs-qpet-sperr \ --exclude numcodecs-sperr \ --exclude numcodecs-sz3 \ --exclude numcodecs-tthresh \ @@ -224,6 +225,7 @@ jobs: cargo hack clippy --all \ --exclude numcodecs-python \ --exclude numcodecs-jpeg2000 \ + --exclude numcodecs-qpet-sperr \ --exclude numcodecs-sperr \ --exclude numcodecs-sz3 \ --exclude numcodecs-tthresh \ @@ -240,6 +242,7 @@ jobs: cargo hack clippy --all \ --exclude numcodecs-python \ --exclude numcodecs-jpeg2000 \ + --exclude numcodecs-qpet-sperr \ --exclude numcodecs-sperr \ --exclude numcodecs-sz3 \ --exclude numcodecs-tthresh \ diff --git a/Cargo.toml b/Cargo.toml index 59fd0c1f..c421c396 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,7 @@ members = [ "codecs/linear-quantize", "codecs/log", "codecs/pco", + "codecs/qpet-sperr", "codecs/random-projection", "codecs/reinterpret", "codecs/round", @@ -45,14 +46,14 @@ rust-version = "1.87" [workspace.dependencies] # workspace-internal numcodecs crates numcodecs = { version = "0.3", path = "crates/numcodecs", default-features = false } -numcodecs-python = { version = "0.7", path = "crates/numcodecs-python", default-features = false } +numcodecs-python = { version = "0.7.1", path = "crates/numcodecs-python", default-features = false } numcodecs-wasm-builder = { version = "0.2", path = "crates/numcodecs-wasm-builder", default-features = false } numcodecs-wasm-guest = { version = "0.3", path = "crates/numcodecs-wasm-guest", default-features = false } numcodecs-wasm-host = { version = "0.2", path = "crates/numcodecs-wasm-host", default-features = false } numcodecs-wasm-host-reproducible = { version = "0.2.1", path = "crates/numcodecs-wasm-host-reproducible", default-features = false } numcodecs-wasm-logging = { version = "0.2", path = "crates/numcodecs-wasm-logging", default-features = false } -numcodecs-wasm = { version = "0.2.1", path = "py/numcodecs-wasm", default-features = false } +numcodecs-wasm = { version = "0.2.2", path = "py/numcodecs-wasm", default-features = false } # workspace-internal codecs crates numcodecs-asinh = { version = "0.4", path = "codecs/asinh", default-features = false } @@ -64,6 +65,7 @@ numcodecs-jpeg2000 = { version = "0.3", path = "codecs/jpeg2000", default-featur numcodecs-linear-quantize = { version = "0.5", path = "codecs/linear-quantize", default-features = false } numcodecs-log = { version = "0.5", path = "codecs/log", default-features = false } numcodecs-pco = { version = "0.3", path = "codecs/pco", default-features = false } +numcodecs-qpet-sperr = { version = "0.1", path = "codecs/qpet-sperr", default-features = false } numcodecs-random-projection = { version = "0.4", path = "codecs/random-projection", default-features = false } numcodecs-reinterpret = { version = "0.4", path = "codecs/reinterpret", default-features = false } numcodecs-round = { version = "0.5", path = "codecs/round", default-features = false } @@ -87,7 +89,6 @@ format_serde_error = { version = "0.3", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } log = { version = "0.4.27", default-features = false } -simple_logger = { version = "5.0", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy ndarray-rand = { version = "0.15", default-features = false } @@ -102,6 +103,7 @@ pyo3 = { version = "0.26", default-features = false } pyo3-error = { version = "0.6", default-features = false } pyo3-log = { version = "0.13.0", default-features = false } pythonize = { version = "0.26", default-features = false } +qpet-sperr = { version = "0.1", default-features = false } rand = { version = "0.9.1", default-features = false } schemars = { version = "1.0.3", default-features = false } scratch = { version = "1.0", default-features = false } @@ -110,6 +112,7 @@ serde = { version = "1.0.218", default-features = false } serde-transcode = { version = "1.1", default-features = false } serde_json = { version = "1.0.140", default-features = false } serde_repr = { version = "0.1.3", default-features = false } +simple_logger = { version = "5.0", default-features = false } sperr = { version = "0.2", default-features = false } sz3 = { version = "0.3", default-features = false } thiserror = { version = "2.0.12", default-features = false } diff --git a/codecs/qpet-sperr/Cargo.toml b/codecs/qpet-sperr/Cargo.toml new file mode 100644 index 00000000..10fe6422 --- /dev/null +++ b/codecs/qpet-sperr/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "numcodecs-qpet-sperr" +version = "0.1.0" +edition = { workspace = true } +authors = { workspace = true } +repository = { workspace = true } +license = { workspace = true } +rust-version = { workspace = true } + +description = "QPET-SPERR codec implementation for the numcodecs API" +readme = "README.md" +categories = ["compression", "encoding"] +keywords = ["QPET-SPERR", "numcodecs", "compression", "encoding"] + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +ndarray = { workspace = true, features = ["std"] } +numcodecs = { workspace = true } +num-traits = { workspace = true, features = ["std"] } +postcard = { workspace = true } +qpet-sperr = { workspace = true } +schemars = { workspace = true, features = ["derive", "preserve_order"] } +serde = { workspace = true, features = ["std", "derive"] } +thiserror = { workspace = true } +# Explicitly enable the `no_wasm_shim` feature for qpet-sperr-sys/zstd-sys +zstd-sys = { workspace = true, features = ["no_wasm_shim"] } + +[dev-dependencies] +serde_json = { workspace = true, features = ["std"] } + +[lints] +workspace = true diff --git a/codecs/qpet-sperr/LICENSE b/codecs/qpet-sperr/LICENSE new file mode 100644 index 00000000..4b563051 --- /dev/null +++ b/codecs/qpet-sperr/LICENSE @@ -0,0 +1,375 @@ +Copyright (c) 2024-2025, Juniper Tyree + +Mozilla Public License Version 2.0 +================================== + +1. Definitions +-------------- + +1.1. "Contributor" + means each individual or legal entity that creates, contributes to + the creation of, or owns Covered Software. + +1.2. "Contributor Version" + means the combination of the Contributions of others (if any) used + by a Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + means Source Code Form to which the initial Contributor has attached + the notice in Exhibit A, the Executable Form of such Source Code + Form, and Modifications of such Source Code Form, in each case + including portions thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + (a) that the initial Contributor has attached the notice described + in Exhibit B to the Covered Software; or + + (b) that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the + terms of a Secondary License. + +1.6. "Executable Form" + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + means a work that combines Covered Software with other material, in + a separate file or files, that is not Covered Software. + +1.8. "License" + means this document. + +1.9. "Licensable" + means having the right to grant, to the maximum extent possible, + whether at the time of the initial grant or subsequently, any and + all of the rights conveyed by this License. + +1.10. "Modifications" + means any of the following: + + (a) any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered + Software; or + + (b) any new file in Source Code Form that contains any Covered + Software. + +1.11. "Patent Claims" of a Contributor + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the + License, by the making, using, selling, offering for sale, having + made, import, or transfer of either its Contributions or its + Contributor Version. + +1.12. "Secondary License" + means either the GNU General Public License, Version 2.0, the GNU + Lesser General Public License, Version 2.1, the GNU Affero General + Public License, Version 3.0, or any later versions of those + licenses. + +1.13. "Source Code Form" + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that + controls, is controlled by, or is under common control with You. For + purposes of this definition, "control" means (a) the power, direct + or indirect, to cause the direction or management of such entity, + whether by contract or otherwise, or (b) ownership of more than + fifty percent (50%) of the outstanding shares or beneficial + ownership of such entity. + +2. License Grants and Conditions +-------------------------------- + +2.1. Grants + +Each Contributor hereby grants You a world-wide, royalty-free, +non-exclusive license: + +(a) under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + +(b) under Patent Claims of such Contributor to make, use, sell, offer + for sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + +The licenses granted in Section 2.1 with respect to any Contribution +become effective for each Contribution on the date the Contributor first +distributes such Contribution. + +2.3. Limitations on Grant Scope + +The licenses granted in this Section 2 are the only rights granted under +this License. No additional rights or licenses will be implied from the +distribution or licensing of Covered Software under this License. +Notwithstanding Section 2.1(b) above, no patent license is granted by a +Contributor: + +(a) for any code that a Contributor has removed from Covered Software; + or + +(b) for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + +(c) under Patent Claims infringed by Covered Software in the absence of + its Contributions. + +This License does not grant any rights in the trademarks, service marks, +or logos of any Contributor (except as may be necessary to comply with +the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + +No Contributor makes additional grants as a result of Your choice to +distribute the Covered Software under a subsequent version of this +License (see Section 10.2) or under the terms of a Secondary License (if +permitted under the terms of Section 3.3). + +2.5. Representation + +Each Contributor represents that the Contributor believes its +Contributions are its original creation(s) or it has sufficient rights +to grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + +This License is not intended to limit any rights You have under +applicable copyright doctrines of fair use, fair dealing, or other +equivalents. + +2.7. Conditions + +Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted +in Section 2.1. + +3. Responsibilities +------------------- + +3.1. Distribution of Source Form + +All distribution of Covered Software in Source Code Form, including any +Modifications that You create or to which You contribute, must be under +the terms of this License. You must inform recipients that the Source +Code Form of the Covered Software is governed by the terms of this +License, and how they can obtain a copy of this License. You may not +attempt to alter or restrict the recipients' rights in the Source Code +Form. + +3.2. Distribution of Executable Form + +If You distribute Covered Software in Executable Form then: + +(a) such Covered Software must also be made available in Source Code + Form, as described in Section 3.1, and You must inform recipients of + the Executable Form how they can obtain a copy of such Source Code + Form by reasonable means in a timely manner, at a charge no more + than the cost of distribution to the recipient; and + +(b) You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter + the recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + +You may create and distribute a Larger Work under terms of Your choice, +provided that You also comply with the requirements of this License for +the Covered Software. If the Larger Work is a combination of Covered +Software with a work governed by one or more Secondary Licenses, and the +Covered Software is not Incompatible With Secondary Licenses, this +License permits You to additionally distribute such Covered Software +under the terms of such Secondary License(s), so that the recipient of +the Larger Work may, at their option, further distribute the Covered +Software under the terms of either this License or such Secondary +License(s). + +3.4. Notices + +You may not remove or alter the substance of any license notices +(including copyright notices, patent notices, disclaimers of warranty, +or limitations of liability) contained within the Source Code Form of +the Covered Software, except that You may alter any license notices to +the extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + +You may choose to offer, and to charge a fee for, warranty, support, +indemnity or liability obligations to one or more recipients of Covered +Software. However, You may do so only on Your own behalf, and not on +behalf of any Contributor. You must make it absolutely clear that any +such warranty, support, indemnity, or liability obligation is offered by +You alone, and You hereby agree to indemnify every Contributor for any +liability incurred by such Contributor as a result of warranty, support, +indemnity or liability terms You offer. You may include additional +disclaimers of warranty and limitations of liability specific to any +jurisdiction. + +4. Inability to Comply Due to Statute or Regulation +--------------------------------------------------- + +If it is impossible for You to comply with any of the terms of this +License with respect to some or all of the Covered Software due to +statute, judicial order, or regulation then You must: (a) comply with +the terms of this License to the maximum extent possible; and (b) +describe the limitations and the code they affect. Such description must +be placed in a text file included with all distributions of the Covered +Software under this License. Except to the extent prohibited by statute +or regulation, such description must be sufficiently detailed for a +recipient of ordinary skill to be able to understand it. + +5. Termination +-------------- + +5.1. The rights granted under this License will terminate automatically +if You fail to comply with any of its terms. However, if You become +compliant, then the rights granted under this License from a particular +Contributor are reinstated (a) provisionally, unless and until such +Contributor explicitly and finally terminates Your grants, and (b) on an +ongoing basis, if such Contributor fails to notify You of the +non-compliance by some reasonable means prior to 60 days after You have +come back into compliance. Moreover, Your grants from a particular +Contributor are reinstated on an ongoing basis if such Contributor +notifies You of the non-compliance by some reasonable means, this is the +first time You have received notice of non-compliance with this License +from such Contributor, and You become compliant prior to 30 days after +Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent +infringement claim (excluding declaratory judgment actions, +counter-claims, and cross-claims) alleging that a Contributor Version +directly or indirectly infringes any patent, then the rights granted to +You by any and all Contributors for the Covered Software under Section +2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all +end user license agreements (excluding distributors and resellers) which +have been validly granted by You or Your distributors under this License +prior to termination shall survive termination. + +************************************************************************ +* * +* 6. Disclaimer of Warranty * +* ------------------------- * +* * +* Covered Software is provided under this License on an "as is" * +* basis, without warranty of any kind, either expressed, implied, or * +* statutory, including, without limitation, warranties that the * +* Covered Software is free of defects, merchantable, fit for a * +* particular purpose or non-infringing. The entire risk as to the * +* quality and performance of the Covered Software is with You. * +* Should any Covered Software prove defective in any respect, You * +* (not any Contributor) assume the cost of any necessary servicing, * +* repair, or correction. This disclaimer of warranty constitutes an * +* essential part of this License. No use of any Covered Software is * +* authorized under this License except under this disclaimer. * +* * +************************************************************************ + +************************************************************************ +* * +* 7. Limitation of Liability * +* -------------------------- * +* * +* Under no circumstances and under no legal theory, whether tort * +* (including negligence), contract, or otherwise, shall any * +* Contributor, or anyone who distributes Covered Software as * +* permitted above, be liable to You for any direct, indirect, * +* special, incidental, or consequential damages of any character * +* including, without limitation, damages for lost profits, loss of * +* goodwill, work stoppage, computer failure or malfunction, or any * +* and all other commercial damages or losses, even if such party * +* shall have been informed of the possibility of such damages. This * +* limitation of liability shall not apply to liability for death or * +* personal injury resulting from such party's negligence to the * +* extent applicable law prohibits such limitation. Some * +* jurisdictions do not allow the exclusion or limitation of * +* incidental or consequential damages, so this exclusion and * +* limitation may not apply to You. * +* * +************************************************************************ + +8. Litigation +------------- + +Any litigation relating to this License may be brought only in the +courts of a jurisdiction where the defendant maintains its principal +place of business and such litigation shall be governed by laws of that +jurisdiction, without reference to its conflict-of-law provisions. +Nothing in this Section shall prevent a party's ability to bring +cross-claims or counter-claims. + +9. Miscellaneous +---------------- + +This License represents the complete agreement concerning the subject +matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent +necessary to make it enforceable. Any law or regulation which provides +that the language of a contract shall be construed against the drafter +shall not be used to construe this License against a Contributor. + +10. Versions of the License +--------------------------- + +10.1. New Versions + +Mozilla Foundation is the license steward. Except as provided in Section +10.3, no one other than the license steward has the right to modify or +publish new versions of this License. Each version will be given a +distinguishing version number. + +10.2. Effect of New Versions + +You may distribute the Covered Software under the terms of the version +of the License under which You originally received the Covered Software, +or under the terms of any subsequent version published by the license +steward. + +10.3. Modified Versions + +If you create software not governed by this License, and you want to +create a new license for such software, you may create and use a +modified version of this License if you rename the license and remove +any references to the name of the license steward (except to note that +such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary +Licenses + +If You choose to distribute Source Code Form that is Incompatible With +Secondary Licenses under the terms of this version of the License, the +notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice +------------------------------------------- + + This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at https://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular +file, then You may include the notice in a location (such as a LICENSE +file in a relevant directory) where a recipient would be likely to look +for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice +--------------------------------------------------------- + + This Source Code Form is "Incompatible With Secondary Licenses", as + defined by the Mozilla Public License, v. 2.0. diff --git a/codecs/qpet-sperr/README.md b/codecs/qpet-sperr/README.md new file mode 100644 index 00000000..8a317c0d --- /dev/null +++ b/codecs/qpet-sperr/README.md @@ -0,0 +1,38 @@ +[![CI Status]][workflow] [![MSRV]][repo] [![Latest Version]][crates.io] [![PyPi Release]][pypi] [![Rust Doc Crate]][docs.rs] [![Rust Doc Main]][docs] [![Read the Docs]][rtdocs] + +[CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main +[workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain + +[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[repo]: https://github.com/juntyr/numcodecs-rs + +[Latest Version]: https://img.shields.io/crates/v/numcodecs-qpet-sperr +[crates.io]: https://crates.io/crates/numcodecs-qpet-sperr + +[PyPi Release]: https://img.shields.io/pypi/v/numcodecs-wasm-qpet-sperr.svg +[pypi]: https://pypi.python.org/pypi/numcodecs-wasm-qpet-sperr + +[Rust Doc Crate]: https://img.shields.io/docsrs/numcodecs-qpet-sperr +[docs.rs]: https://docs.rs/numcodecs-qpet-sperr/ + +[Rust Doc Main]: https://img.shields.io/badge/docs-main-blue +[docs]: https://juntyr.github.io/numcodecs-rs/numcodecs_qpet_sperr + +[Read the Docs]: https://img.shields.io/readthedocs/numcodecs-wasm?label=readthedocs +[rtdocs]: https://numcodecs-wasm.readthedocs.io/en/stable/api/numcodecs_wasm_qpet_sperr/ + +# numcodecs-qpet-sperr + +QPET-SPERR codec implementation for the [`numcodecs`] API. + +[`numcodecs`]: https://docs.rs/numcodecs/0.2/numcodecs/ + +## License + +Licensed under the Mozilla Public License, Version 2.0 ([LICENSE](LICENSE) or https://www.mozilla.org/en-US/MPL/2.0/). + +## Funding + +The `numcodecs-qpet-sperr` crate has been developed as part of [ESiWACE3](https://www.esiwace.eu), the third phase of the Centre of Excellence in Simulation of Weather and Climate in Europe. + +Funded by the European Union. This work has received funding from the European High Performance Computing Joint Undertaking (JU) under grant agreement No 101093054. diff --git a/codecs/qpet-sperr/src/lib.rs b/codecs/qpet-sperr/src/lib.rs new file mode 100644 index 00000000..dc0ed9c2 --- /dev/null +++ b/codecs/qpet-sperr/src/lib.rs @@ -0,0 +1,695 @@ +//! [![CI Status]][workflow] [![MSRV]][repo] [![Latest Version]][crates.io] [![Rust Doc Crate]][docs.rs] [![Rust Doc Main]][docs] +//! +//! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main +//! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain +//! +//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [repo]: https://github.com/juntyr/numcodecs-rs +//! +//! [Latest Version]: https://img.shields.io/crates/v/numcodecs-qpet-sperr +//! [crates.io]: https://crates.io/crates/numcodecs-qpet-sperr +//! +//! [Rust Doc Crate]: https://img.shields.io/docsrs/numcodecs-qpet-sperr +//! [docs.rs]: https://docs.rs/numcodecs-qpet-sperr/ +//! +//! [Rust Doc Main]: https://img.shields.io/badge/docs-main-blue +//! [docs]: https://juntyr.github.io/numcodecs-rs/numcodecs_qpet_sperr +//! +//! QPET-SPERR codec implementation for the [`numcodecs`] API. + +#![allow(clippy::multiple_crate_versions)] // embedded-io + +// Only included to explicitly enable the `no_wasm_shim` feature for +// qpet-sperr-sys/zstd-sys +use ::zstd_sys as _; + +#[cfg(test)] +use ::serde_json as _; + +use std::{ + borrow::Cow, + fmt, + num::{NonZeroU16, NonZeroUsize}, +}; + +use ndarray::{Array, Array1, ArrayBase, Axis, Data, Dimension, IxDyn, ShapeError}; +use num_traits::{Float, identities::Zero}; +use numcodecs::{ + AnyArray, AnyArrayAssignError, AnyArrayDType, AnyArrayView, AnyArrayViewMut, AnyCowArray, + Codec, StaticCodec, StaticCodecConfig, StaticCodecVersion, +}; +use schemars::{JsonSchema, Schema, SchemaGenerator, json_schema}; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use thiserror::Error; + +type QpetSperrCodecVersion = StaticCodecVersion<0, 1, 0>; + +#[derive(Clone, Serialize, Deserialize, JsonSchema)] +// serde cannot deny unknown fields because of the flatten +#[schemars(deny_unknown_fields)] +/// Codec providing compression using QPET-SPERR. +/// +/// Arrays that are higher-dimensional than 3D are encoded by compressing each +/// 3D slice with QPET-SPERR independently. Specifically, the array's shape is +/// interpreted as `[.., depth, height, width]`. If you want to compress 3D +/// slices along three different axes, you can swizzle the array axes +/// beforehand. +pub struct QpetSperrCodec { + /// QPET-SPERR compression mode + #[serde(flatten)] + pub mode: QpetSperrCompressionMode, + /// The codec's encoding format version. Do not provide this parameter explicitly. + #[serde(default, rename = "_version")] + pub version: QpetSperrCodecVersion, +} + +#[derive(Clone, Serialize, Deserialize, JsonSchema)] +/// QPET-SPERR compression mode +#[serde(tag = "mode")] +pub enum QpetSperrCompressionMode { + /// Symbolic Quantity of Interest + #[serde(rename = "qoi-symbolic")] + SymbolicQuantityOfInterest { + /// quantity of interest expression + qoi: String, + /// block size over which the quantity of interest errors are averaged, + /// 1 for pointwise + #[serde(default = "default_qoi_block_size")] + qoi_block_size: NonZeroU16, + /// positive (pointwise) absolute error bound over the quantity of + /// interest + qoi_pwe: Positive, + /// 3D size of the chunks (z, y, x) that SPERR uses internally + #[serde(default = "default_sperr_chunks")] + sperr_chunks: (NonZeroUsize, NonZeroUsize, NonZeroUsize), + /// optional positive pointwise absolute error bound over the data + #[serde(default)] + data_pwe: Option>, + /// positive quantity of interest k parameter (3.0 is a good default) + #[serde(default = "default_qoi_k")] + qoi_k: Positive, + /// high precision mode for SPERR, useful for small error bounds + #[serde(default)] + high_prec: bool, + }, +} + +const fn default_qoi_block_size() -> NonZeroU16 { + const NON_ZERO_ONE: NonZeroU16 = NonZeroU16::MIN; + // 1: pointwise + NON_ZERO_ONE +} + +const fn default_sperr_chunks() -> (NonZeroUsize, NonZeroUsize, NonZeroUsize) { + const NON_ZERO_256: NonZeroUsize = NonZeroUsize::MIN.saturating_add(255); + (NON_ZERO_256, NON_ZERO_256, NON_ZERO_256) +} + +const fn default_qoi_k() -> Positive { + // c=3.0, suggested default + Positive(3.0) +} + +impl Codec for QpetSperrCodec { + type Error = QpetSperrCodecError; + + fn encode(&self, data: AnyCowArray) -> Result { + match data { + AnyCowArray::F32(data) => Ok(AnyArray::U8( + Array1::from(compress(data, &self.mode)?).into_dyn(), + )), + AnyCowArray::F64(data) => Ok(AnyArray::U8( + Array1::from(compress(data, &self.mode)?).into_dyn(), + )), + encoded => Err(QpetSperrCodecError::UnsupportedDtype(encoded.dtype())), + } + } + + fn decode(&self, encoded: AnyCowArray) -> Result { + let AnyCowArray::U8(encoded) = encoded else { + return Err(QpetSperrCodecError::EncodedDataNotBytes { + dtype: encoded.dtype(), + }); + }; + + if !matches!(encoded.shape(), [_]) { + return Err(QpetSperrCodecError::EncodedDataNotOneDimensional { + shape: encoded.shape().to_vec(), + }); + } + + decompress(&AnyCowArray::U8(encoded).as_bytes()) + } + + fn decode_into( + &self, + encoded: AnyArrayView, + mut decoded: AnyArrayViewMut, + ) -> Result<(), Self::Error> { + let decoded_in = self.decode(encoded.cow())?; + + Ok(decoded.assign(&decoded_in)?) + } +} + +impl StaticCodec for QpetSperrCodec { + const CODEC_ID: &'static str = "qpet-sperr.rs"; + + type Config<'de> = Self; + + fn from_config(config: Self::Config<'_>) -> Self { + config + } + + fn get_config(&self) -> StaticCodecConfig<'_, Self> { + StaticCodecConfig::from(self) + } +} + +#[derive(Debug, Error)] +/// Errors that may occur when applying the [`QpetSperrCodec`]. +pub enum QpetSperrCodecError { + /// [`QpetSperrCodec`] does not support the dtype + #[error("QpetSperr does not support the dtype {0}")] + UnsupportedDtype(AnyArrayDType), + /// [`QpetSperrCodec`] failed to encode the header + #[error("QpetSperr failed to encode the header")] + HeaderEncodeFailed { + /// Opaque source error + source: QpetSperrHeaderError, + }, + /// [`QpetSperrCodec`] failed to encode the data + #[error("QpetSperr failed to encode the data")] + QpetSperrEncodeFailed { + /// Opaque source error + source: QpetSperrCodingError, + }, + /// [`QpetSperrCodec`] failed to encode a slice + #[error("QpetSperr failed to encode a slice")] + SliceEncodeFailed { + /// Opaque source error + source: QpetSperrSliceError, + }, + /// [`QpetSperrCodec`] can only decode one-dimensional byte arrays but + /// received an array of a different dtype + #[error( + "QpetSperr can only decode one-dimensional byte arrays but received an array of dtype {dtype}" + )] + EncodedDataNotBytes { + /// The unexpected dtype of the encoded array + dtype: AnyArrayDType, + }, + /// [`QpetSperrCodec`] can only decode one-dimensional byte arrays but + /// received an array of a different shape + #[error( + "QpetSperr can only decode one-dimensional byte arrays but received a byte array of shape {shape:?}" + )] + EncodedDataNotOneDimensional { + /// The unexpected shape of the encoded array + shape: Vec, + }, + /// [`QpetSperrCodec`] failed to decode the header + #[error("QpetSperr failed to decode the header")] + HeaderDecodeFailed { + /// Opaque source error + source: QpetSperrHeaderError, + }, + /// [`QpetSperrCodec`] failed to decode a slice + #[error("QpetSperr failed to decode a slice")] + SliceDecodeFailed { + /// Opaque source error + source: QpetSperrSliceError, + }, + /// [`QpetSperrCodec`] failed to decode from an excessive number of slices + #[error("QpetSperr failed to decode from an excessive number of slices")] + DecodeTooManySlices, + /// [`QpetSperrCodec`] failed to decode the data + #[error("QpetSperr failed to decode the data")] + SperrDecodeFailed { + /// Opaque source error + source: QpetSperrCodingError, + }, + /// [`QpetSperrCodec`] decoded into an invalid shape not matching the data size + #[error("QpetSperr decoded into an invalid shape not matching the data size")] + DecodeInvalidShape { + /// The source of the error + source: ShapeError, + }, + /// [`QpetSperrCodec`] cannot decode into the provided array + #[error("QpetSperr cannot decode into the provided array")] + MismatchedDecodeIntoArray { + /// The source of the error + #[from] + source: AnyArrayAssignError, + }, +} + +#[derive(Debug, Error)] +#[error(transparent)] +/// Opaque error for when encoding or decoding the header fails +pub struct QpetSperrHeaderError(postcard::Error); + +#[derive(Debug, Error)] +#[error(transparent)] +/// Opaque error for when encoding or decoding a slice fails +pub struct QpetSperrSliceError(postcard::Error); + +#[derive(Debug, Error)] +#[error(transparent)] +/// Opaque error for when encoding or decoding with SPERR fails +pub struct QpetSperrCodingError(qpet_sperr::Error); + +/// Compress the `data` array using QPET-SPERR with the provided `mode`. +/// +/// The compressed data can be decompressed using SPERR or QPET-SPERR. +/// +/// # Errors +/// +/// Errors with +/// - [`QpetSperrCodecError::HeaderEncodeFailed`] if encoding the header failed +/// - [`QpetSperrCodecError::QpetSperrEncodeFailed`] if encoding with +/// QPET-SPERR failed +/// - [`QpetSperrCodecError::SliceEncodeFailed`] if encoding a slice failed +#[allow(clippy::missing_panics_doc)] +pub fn compress, D: Dimension>( + data: ArrayBase, + mode: &QpetSperrCompressionMode, +) -> Result, QpetSperrCodecError> { + let mut encoded = postcard::to_extend( + &CompressionHeader { + dtype: T::DTYPE, + shape: Cow::Borrowed(data.shape()), + version: StaticCodecVersion, + }, + Vec::new(), + ) + .map_err(|err| QpetSperrCodecError::HeaderEncodeFailed { + source: QpetSperrHeaderError(err), + })?; + + // SPERR cannot handle zero-length dimensions + if data.is_empty() { + return Ok(encoded); + } + + let mut chunk_size = Vec::from(data.shape()); + let (width, height, depth) = match *chunk_size.as_mut_slice() { + [ref mut rest @ .., depth, height, width] => { + for r in rest { + *r = 1; + } + (width, height, depth) + } + [height, width] => (width, height, 1), + [width] => (width, 1, 1), + [] => (1, 1, 1), + }; + + for mut slice in data.into_dyn().exact_chunks(chunk_size.as_slice()) { + while slice.ndim() < 3 { + slice = slice.insert_axis(Axis(0)); + } + #[allow(clippy::unwrap_used)] + // slice must now have at least three axes, and all but the last three + // must be of size 1 + let slice = slice.into_shape_with_order((depth, height, width)).unwrap(); + + let QpetSperrCompressionMode::SymbolicQuantityOfInterest { + qoi, + qoi_block_size, + qoi_pwe, + sperr_chunks, + data_pwe, + qoi_k, + high_prec, + } = mode; + + let encoded_slice = qpet_sperr::compress_3d( + slice, + qpet_sperr::CompressionMode::SymbolicQuantityOfInterest { + qoi: qoi.as_str(), + qoi_block_size: *qoi_block_size, + qoi_pwe: qoi_pwe.0, + data_pwe: data_pwe.map(|data_pwe| data_pwe.0), + qoi_k: qoi_k.0, + high_prec: *high_prec, + }, + ( + sperr_chunks.0.get(), + sperr_chunks.1.get(), + sperr_chunks.2.get(), + ), + ) + .map_err(|err| QpetSperrCodecError::QpetSperrEncodeFailed { + source: QpetSperrCodingError(err), + })?; + + encoded = postcard::to_extend(encoded_slice.as_slice(), encoded).map_err(|err| { + QpetSperrCodecError::SliceEncodeFailed { + source: QpetSperrSliceError(err), + } + })?; + } + + Ok(encoded) +} + +/// Decompress the `encoded` data into an array using SPERR. +/// +/// # Errors +/// +/// Errors with +/// - [`QpetSperrCodecError::HeaderDecodeFailed`] if decoding the header failed +/// - [`QpetSperrCodecError::SliceDecodeFailed`] if decoding a slice failed +/// - [`QpetSperrCodecError::SperrDecodeFailed`] if decoding with SPERR failed +/// - [`QpetSperrCodecError::DecodeInvalidShape`] if the encoded data decodes +/// to an unexpected shape +/// - [`QpetSperrCodecError::DecodeTooManySlices`] if the encoded data contains +/// too many slices +pub fn decompress(encoded: &[u8]) -> Result { + fn decompress_typed( + mut encoded: &[u8], + shape: &[usize], + ) -> Result, QpetSperrCodecError> { + let mut decoded = Array::::zeros(shape); + + let mut chunk_size = Vec::from(shape); + let (width, height, depth) = match *chunk_size.as_mut_slice() { + [ref mut rest @ .., depth, height, width] => { + for r in rest { + *r = 1; + } + (width, height, depth) + } + [height, width] => (width, height, 1), + [width] => (width, 1, 1), + [] => (1, 1, 1), + }; + + for mut slice in decoded.exact_chunks_mut(chunk_size.as_slice()) { + let (encoded_slice, rest) = + postcard::take_from_bytes::>(encoded).map_err(|err| { + QpetSperrCodecError::SliceDecodeFailed { + source: QpetSperrSliceError(err), + } + })?; + encoded = rest; + + while slice.ndim() < 3 { + slice = slice.insert_axis(Axis(0)); + } + #[allow(clippy::unwrap_used)] + // slice must now have at least three axes, and all but the last + // three must be of size 1 + let slice = slice.into_shape_with_order((depth, height, width)).unwrap(); + + qpet_sperr::decompress_into_3d(&encoded_slice, slice).map_err(|err| { + QpetSperrCodecError::SperrDecodeFailed { + source: QpetSperrCodingError(err), + } + })?; + } + + if !encoded.is_empty() { + return Err(QpetSperrCodecError::DecodeTooManySlices); + } + + Ok(decoded) + } + + let (header, encoded) = + postcard::take_from_bytes::(encoded).map_err(|err| { + QpetSperrCodecError::HeaderDecodeFailed { + source: QpetSperrHeaderError(err), + } + })?; + + // Return empty data for zero-size arrays + if header.shape.iter().copied().product::() == 0 { + return match header.dtype { + QpetSperrDType::F32 => Ok(AnyArray::F32(Array::zeros(&*header.shape))), + QpetSperrDType::F64 => Ok(AnyArray::F64(Array::zeros(&*header.shape))), + }; + } + + match header.dtype { + QpetSperrDType::F32 => Ok(AnyArray::F32(decompress_typed(encoded, &header.shape)?)), + QpetSperrDType::F64 => Ok(AnyArray::F64(decompress_typed(encoded, &header.shape)?)), + } +} + +/// Array element types which can be compressed with QPET-SPERR. +pub trait QpetSperrElement: qpet_sperr::Element + Zero { + /// The dtype representation of the type + const DTYPE: QpetSperrDType; +} + +impl QpetSperrElement for f32 { + const DTYPE: QpetSperrDType = QpetSperrDType::F32; +} +impl QpetSperrElement for f64 { + const DTYPE: QpetSperrDType = QpetSperrDType::F64; +} + +#[expect(clippy::derive_partial_eq_without_eq)] // floats are not Eq +#[derive(Copy, Clone, PartialEq, PartialOrd, Hash)] +/// Positive floating point number +pub struct Positive(T); + +impl Positive { + #[must_use] + /// Get the positive floating point value + pub const fn get(self) -> T { + self.0 + } +} + +impl Serialize for Positive { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_f64(self.0) + } +} + +impl<'de> Deserialize<'de> for Positive { + fn deserialize>(deserializer: D) -> Result { + let x = f64::deserialize(deserializer)?; + + if x > 0.0 { + Ok(Self(x)) + } else { + Err(serde::de::Error::invalid_value( + serde::de::Unexpected::Float(x), + &"a positive value", + )) + } + } +} + +impl JsonSchema for Positive { + fn schema_name() -> Cow<'static, str> { + Cow::Borrowed("PositiveF64") + } + + fn schema_id() -> Cow<'static, str> { + Cow::Borrowed(concat!(module_path!(), "::", "Positive")) + } + + fn json_schema(_gen: &mut SchemaGenerator) -> Schema { + json_schema!({ + "type": "number", + "exclusiveMinimum": 0.0 + }) + } +} + +#[derive(Serialize, Deserialize)] +struct CompressionHeader<'a> { + dtype: QpetSperrDType, + #[serde(borrow)] + shape: Cow<'a, [usize]>, + version: QpetSperrCodecVersion, +} + +/// Dtypes that QPET-SPERR can compress and decompress +#[derive(Copy, Clone, Debug, Serialize, Deserialize)] +#[expect(missing_docs)] +pub enum QpetSperrDType { + #[serde(rename = "f32", alias = "float32")] + F32, + #[serde(rename = "f64", alias = "float64")] + F64, +} + +impl fmt::Display for QpetSperrDType { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + fmt.write_str(match self { + Self::F32 => "f32", + Self::F64 => "f64", + }) + } +} + +#[cfg(test)] +#[allow(clippy::unwrap_used)] +mod tests { + use std::f64; + + use ndarray::{Ix0, Ix1, Ix2, Ix3, Ix4}; + + use super::*; + + #[test] + fn zero_length() { + let encoded = compress( + Array::::from_shape_vec([3, 0], vec![]).unwrap(), + &QpetSperrCompressionMode::SymbolicQuantityOfInterest { + qoi: String::from("x"), + qoi_block_size: default_qoi_block_size(), + qoi_pwe: Positive(42.0), + sperr_chunks: default_sperr_chunks(), + data_pwe: None, + qoi_k: default_qoi_k(), + high_prec: false, + }, + ) + .unwrap(); + let decoded = decompress(&encoded).unwrap(); + + assert_eq!(decoded.dtype(), AnyArrayDType::F32); + assert!(decoded.is_empty()); + assert_eq!(decoded.shape(), &[3, 0]); + } + + #[test] + fn small_2d() { + let encoded = compress( + Array::::from_shape_vec([1, 1], vec![42.0]).unwrap(), + &QpetSperrCompressionMode::SymbolicQuantityOfInterest { + qoi: String::from("x"), + qoi_block_size: default_qoi_block_size(), + qoi_pwe: Positive(42.0), + sperr_chunks: default_sperr_chunks(), + data_pwe: None, + qoi_k: default_qoi_k(), + high_prec: false, + }, + ) + .unwrap(); + let decoded = decompress(&encoded).unwrap(); + + assert_eq!(decoded.dtype(), AnyArrayDType::F32); + assert_eq!(decoded.len(), 1); + assert_eq!(decoded.shape(), &[1, 1]); + } + + #[test] + fn large_3d() { + let encoded = compress( + Array::::zeros((64, 64, 64)), + &QpetSperrCompressionMode::SymbolicQuantityOfInterest { + qoi: String::from("x"), + qoi_block_size: default_qoi_block_size(), + qoi_pwe: Positive(42.0), + sperr_chunks: default_sperr_chunks(), + data_pwe: None, + qoi_k: default_qoi_k(), + high_prec: false, + }, + ) + .unwrap(); + let decoded = decompress(&encoded).unwrap(); + + assert_eq!(decoded.dtype(), AnyArrayDType::F64); + assert_eq!(decoded.len(), 64 * 64 * 64); + assert_eq!(decoded.shape(), &[64, 64, 64]); + } + + #[test] + fn all_modes() { + for mode in [QpetSperrCompressionMode::SymbolicQuantityOfInterest { + qoi: String::from("x^2"), + qoi_block_size: default_qoi_block_size(), + qoi_pwe: Positive(0.1), + sperr_chunks: default_sperr_chunks(), + data_pwe: None, + qoi_k: default_qoi_k(), + high_prec: false, + }] { + let encoded = compress(Array::::zeros((64, 64, 64)), &mode).unwrap(); + let decoded = decompress(&encoded).unwrap(); + + assert_eq!(decoded.dtype(), AnyArrayDType::F64); + assert_eq!(decoded.len(), 64 * 64 * 64); + assert_eq!(decoded.shape(), &[64, 64, 64]); + } + } + + #[test] + fn many_dimensions() { + for data in [ + Array::::from_shape_vec([], vec![42.0]) + .unwrap() + .into_dyn(), + Array::::from_shape_vec([2], vec![1.0, 2.0]) + .unwrap() + .into_dyn(), + Array::::from_shape_vec([2, 2], vec![1.0, 2.0, 3.0, 4.0]) + .unwrap() + .into_dyn(), + Array::::from_shape_vec( + [2, 2, 2], + vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0], + ) + .unwrap() + .into_dyn(), + Array::::from_shape_vec( + [2, 2, 2, 2], + vec![ + 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, + 15.0, 16.0, + ], + ) + .unwrap() + .into_dyn(), + ] { + let encoded = compress( + data.view(), + &QpetSperrCompressionMode::SymbolicQuantityOfInterest { + qoi: String::from("x"), + qoi_block_size: default_qoi_block_size(), + qoi_pwe: Positive(f64::EPSILON), + sperr_chunks: default_sperr_chunks(), + data_pwe: None, + qoi_k: default_qoi_k(), + high_prec: false, + }, + ) + .unwrap(); + let decoded = decompress(&encoded).unwrap(); + + assert_eq!(decoded, AnyArray::F32(data)); + } + } + + #[test] + fn zero_square_qoi() { + let encoded = compress( + Array::::zeros((64, 64, 1)), + &QpetSperrCompressionMode::SymbolicQuantityOfInterest { + qoi: String::from("x^2"), + qoi_block_size: default_qoi_block_size(), + qoi_pwe: Positive(0.1), + sperr_chunks: default_sperr_chunks(), + data_pwe: None, + qoi_k: default_qoi_k(), + high_prec: false, + }, + ) + .unwrap(); + let decoded = decompress(&encoded).unwrap(); + + assert_eq!(decoded.dtype(), AnyArrayDType::F64); + assert_eq!(decoded.len(), 64 * 64 * 1); + assert_eq!(decoded.shape(), &[64, 64, 1]); + } +} diff --git a/codecs/qpet-sperr/tests/config.rs b/codecs/qpet-sperr/tests/config.rs new file mode 100644 index 00000000..0ec98193 --- /dev/null +++ b/codecs/qpet-sperr/tests/config.rs @@ -0,0 +1,37 @@ +#![expect(missing_docs)] + +use ::{ + ndarray as _, num_traits as _, postcard as _, qpet_sperr as _, schemars as _, thiserror as _, + zstd_sys as _, +}; + +#[cfg(target_arch = "wasm32")] +use ::gmp_mpfr_sys as _; + +use numcodecs::StaticCodec; +use numcodecs_qpet_sperr::{QpetSperrCodec, QpetSperrCompressionMode}; +use serde::Deserialize; +use serde_json::json; + +#[test] +#[should_panic(expected = "missing field `mode`")] +fn empty_config() { + let _ = QpetSperrCodec::from_config(Deserialize::deserialize(json!({})).unwrap()); +} + +#[test] +fn symbolic_qoi_config() { + let codec = QpetSperrCodec::from_config( + Deserialize::deserialize(json!({ + "mode": "qoi-symbolic", + "qoi": "x^2", + "qoi_pwe": 0.1, + })) + .unwrap(), + ); + + assert!(matches!( + codec.mode, + QpetSperrCompressionMode::SymbolicQuantityOfInterest { qoi, .. } if qoi == "x^2" + )); +} diff --git a/codecs/qpet-sperr/tests/schema.json b/codecs/qpet-sperr/tests/schema.json new file mode 100644 index 00000000..90704764 --- /dev/null +++ b/codecs/qpet-sperr/tests/schema.json @@ -0,0 +1,97 @@ +{ + "type": "object", + "unevaluatedProperties": false, + "oneOf": [ + { + "type": "object", + "properties": { + "qoi": { + "type": "string", + "description": "quantity of interest expression" + }, + "qoi_block_size": { + "type": "integer", + "format": "uint16", + "minimum": 1, + "maximum": 65535, + "description": "block size over which the quantity of interest errors are averaged,\n1 for pointwise", + "default": 1 + }, + "qoi_pwe": { + "type": "number", + "exclusiveMinimum": 0.0, + "description": "positive (pointwise) absolute error bound over the quantity of\ninterest" + }, + "sperr_chunks": { + "type": "array", + "prefixItems": [ + { + "type": "integer", + "format": "uint", + "minimum": 1 + }, + { + "type": "integer", + "format": "uint", + "minimum": 1 + }, + { + "type": "integer", + "format": "uint", + "minimum": 1 + } + ], + "minItems": 3, + "maxItems": 3, + "description": "3D size of the chunks (z, y, x) that SPERR uses internally", + "default": [ + 256, + 256, + 256 + ] + }, + "data_pwe": { + "type": [ + "number", + "null" + ], + "exclusiveMinimum": 0.0, + "description": "optional positive pointwise absolute error bound over the data", + "default": null + }, + "qoi_k": { + "type": "number", + "exclusiveMinimum": 0.0, + "description": "positive quantity of interest k parameter (3.0 is a good default)", + "default": 3.0 + }, + "high_prec": { + "type": "boolean", + "description": "high precision mode for SPERR, useful for small error bounds", + "default": false + }, + "mode": { + "type": "string", + "const": "qoi-symbolic" + } + }, + "required": [ + "mode", + "qoi", + "qoi_pwe" + ], + "description": "Symbolic Quantity of Interest" + } + ], + "description": "Codec providing compression using QPET-SPERR.\n\nArrays that are higher-dimensional than 3D are encoded by compressing each\n3D slice with QPET-SPERR independently. Specifically, the array's shape is\ninterpreted as `[.., depth, height, width]`. If you want to compress 3D\nslices along three different axes, you can swizzle the array axes\nbeforehand.", + "properties": { + "_version": { + "type": "string", + "pattern": "^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$", + "description": "The codec's encoding format version. Do not provide this parameter explicitly.", + "default": "0.1.0" + } + }, + "title": "QpetSperrCodec", + "$schema": "https://json-schema.org/draft/2020-12/schema" +} \ No newline at end of file diff --git a/codecs/qpet-sperr/tests/schema.rs b/codecs/qpet-sperr/tests/schema.rs new file mode 100644 index 00000000..885d3d99 --- /dev/null +++ b/codecs/qpet-sperr/tests/schema.rs @@ -0,0 +1,26 @@ +#![expect(missing_docs)] + +use ::{ + ndarray as _, num_traits as _, postcard as _, qpet_sperr as _, schemars as _, serde as _, + serde_json as _, thiserror as _, zstd_sys as _, +}; + +#[cfg(target_arch = "wasm32")] +use ::gmp_mpfr_sys as _; + +use numcodecs::{DynCodecType, StaticCodecType}; +use numcodecs_qpet_sperr::QpetSperrCodec; + +#[test] +fn schema() { + let schema = format!( + "{:#}", + StaticCodecType::::of() + .codec_config_schema() + .to_value() + ); + + if schema != include_str!("schema.json") { + panic!("QPET-SPERR schema has changed\n===\n{schema}\n==="); + } +} diff --git a/codecs/sperr/tests/config.rs b/codecs/sperr/tests/config.rs index a6934b0a..aa471e55 100644 --- a/codecs/sperr/tests/config.rs +++ b/codecs/sperr/tests/config.rs @@ -1,12 +1,9 @@ #![expect(missing_docs)] -use ::{ - ndarray as _, num_traits as _, numcodecs_sperr::SperrCompressionMode, postcard as _, - schemars as _, sperr as _, thiserror as _, -}; +use ::{ndarray as _, num_traits as _, postcard as _, schemars as _, sperr as _, thiserror as _}; use numcodecs::StaticCodec; -use numcodecs_sperr::SperrCodec; +use numcodecs_sperr::{SperrCodec, SperrCompressionMode}; use serde::Deserialize; use serde_json::json; diff --git a/codecs/sz3/src/lib.rs b/codecs/sz3/src/lib.rs index 31be7c75..f95de9fc 100644 --- a/codecs/sz3/src/lib.rs +++ b/codecs/sz3/src/lib.rs @@ -31,7 +31,7 @@ use serde::{Deserialize, Serialize}; use thiserror::Error; // Only included to explicitly enable the `no_wasm_shim` feature for -// sz3-sys/Sz3-sys +// sz3-sys/zstd-sys use ::zstd_sys as _; #[cfg(test)] diff --git a/crates/numcodecs-python/Cargo.toml b/crates/numcodecs-python/Cargo.toml index d65f28e7..c8ed8daf 100644 --- a/crates/numcodecs-python/Cargo.toml +++ b/crates/numcodecs-python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "numcodecs-python" -version = "0.7.0" +version = "0.7.1" edition = { workspace = true } authors = { workspace = true } repository = { workspace = true } diff --git a/crates/numcodecs-python/src/schema.rs b/crates/numcodecs-python/src/schema.rs index 238e6859..e43ad026 100644 --- a/crates/numcodecs-python/src/schema.rs +++ b/crates/numcodecs-python/src/schema.rs @@ -1,6 +1,8 @@ use std::{ borrow::Cow, collections::{HashMap, hash_map::Entry}, + io, + num::FpCategory, }; use pyo3::{intern, prelude::*, sync::PyOnceLock}; @@ -154,10 +156,9 @@ pub fn docs_from_schema(schema: &Schema) -> Option { docs.push_str(", optional"); } - #[expect(clippy::format_push_string)] // FIXME if let Some(default) = parameter.default { docs.push_str(", default = "); - docs.push_str(&format!("{default}")); + JsonToPythonFormatter::push_to_string(&mut docs, default); } docs.push('\n'); @@ -198,10 +199,9 @@ pub fn signature_from_schema(schema: &Schema) -> String { signature.push_str(", "); signature.push_str(parameter.name); - #[expect(clippy::format_push_string)] // FIXME if let Some(default) = parameter.default { signature.push('='); - signature.push_str(&format!("{default}")); + JsonToPythonFormatter::push_to_string(&mut signature, default); } else if !parameter.required { signature.push_str("=None"); } @@ -495,10 +495,9 @@ impl<'a> VariantParameter<'a> { if let Some(tag_docs) = self.tag_docs { let mut docs = String::new(); - #[expect(clippy::format_push_string)] // FIXME for (tag, tag_docs) in tag_docs { docs.push_str(" - "); - docs.push_str(&format!("{tag}")); + JsonToPythonFormatter::push_to_string(&mut docs, tag); if let Some(tag_docs) = tag_docs { docs.push_str(": "); docs.push_str(&tag_docs.replace('\n', "\n ")); @@ -515,6 +514,62 @@ impl<'a> VariantParameter<'a> { } } +struct JsonToPythonFormatter; + +impl JsonToPythonFormatter { + fn push_to_string(buffer: &mut String, value: &Value) { + // Safety: serde_json::Serializer only produces valid UTF8 bytes + // - JsonToPythonFormatter only writes valid UTF8 bytes + // - serde_json::ser::CompactFormatter only writes valid UTF8 bytes + #[expect(unsafe_code)] + let mut ser = serde_json::Serializer::with_formatter(unsafe { buffer.as_mut_vec() }, Self); + #[allow(clippy::expect_used)] + serde::Serialize::serialize(value, &mut ser) + .expect("JSON value must not fail to serialize"); + } +} + +impl serde_json::ser::Formatter for JsonToPythonFormatter { + #[inline] + fn write_null(&mut self, writer: &mut W) -> io::Result<()> { + writer.write_all(b"None") + } + + #[inline] + fn write_bool(&mut self, writer: &mut W, value: bool) -> io::Result<()> { + let s: &[u8] = if value { b"True" } else { b"False" }; + writer.write_all(s) + } + + #[inline] + fn write_f32(&mut self, writer: &mut W, value: f32) -> io::Result<()> { + let s: &[u8] = match (value.classify(), value.is_sign_negative()) { + (FpCategory::Nan, false) => b"float('nan')", + (FpCategory::Nan, true) => b"float('-nan')", + (FpCategory::Infinite, false) => b"float('inf')", + (FpCategory::Infinite, true) => b"float('-inf')", + (FpCategory::Zero | FpCategory::Subnormal | FpCategory::Normal, false | true) => { + return serde_json::ser::CompactFormatter.write_f32(writer, value); + } + }; + writer.write_all(s) + } + + #[inline] + fn write_f64(&mut self, writer: &mut W, value: f64) -> io::Result<()> { + let s: &[u8] = match (value.classify(), value.is_sign_negative()) { + (FpCategory::Nan, false) => b"float('nan')", + (FpCategory::Nan, true) => b"float('-nan')", + (FpCategory::Infinite, false) => b"float('inf')", + (FpCategory::Infinite, true) => b"float('-inf')", + (FpCategory::Zero | FpCategory::Subnormal | FpCategory::Normal, false | true) => { + return serde_json::ser::CompactFormatter.write_f64(writer, value); + } + }; + writer.write_all(s) + } +} + #[cfg(test)] mod tests { use schemars::{JsonSchema, schema_for}; diff --git a/crates/numcodecs-python/tests/export.rs b/crates/numcodecs-python/tests/export.rs index 7759e7fa..68f0b228 100644 --- a/crates/numcodecs-python/tests/export.rs +++ b/crates/numcodecs-python/tests/export.rs @@ -178,7 +178,7 @@ impl StaticCodec for NegateCodec { config } - fn get_config(&self) -> StaticCodecConfig { + fn get_config(&self) -> StaticCodecConfig<'_, Self> { StaticCodecConfig::from(self) } } diff --git a/crates/numcodecs-wasm-builder/buildenv/flake.nix b/crates/numcodecs-wasm-builder/buildenv/flake.nix index 9a4d877c..6062a08f 100644 --- a/crates/numcodecs-wasm-builder/buildenv/flake.nix +++ b/crates/numcodecs-wasm-builder/buildenv/flake.nix @@ -34,6 +34,23 @@ phases = "installPhase"; + installPhase = '' + mkdir -p $out + tar -xf $src --strip-components=1 -C $out + ''; + }; + libclang_rt = pkgs.stdenv.mkDerivation { + pname = "libclang_rt"; + version = "27.0"; + src = pkgs.fetchurl { + url = + "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-27/libclang_rt-27.0.tar.gz"; + sha256 = + "9e0f382110a3cf9196f02432c8f2e54d151515de36f9311c8c16073f6e6b16d3"; + }; + + phases = "installPhase"; + installPhase = '' mkdir -p $out tar -xf $src --strip-components=1 -C $out @@ -45,6 +62,7 @@ (pkgs.rust-bin.fromRustupToolchainFile ./rust-toolchain) pkgs."llvmPackages_${llvmVersion}".libclang wasi-sysroot + libclang_rt pkgs.cmake pkgs.binaryen ]; @@ -55,7 +73,12 @@ MY_LIBCLANG = "${pkgs."llvmPackages_${llvmVersion}".libclang.lib}/lib"; MY_LLD = "${pkgs."llvmPackages_${llvmVersion}".lld}/bin"; MY_NM = "${pkgs."llvmPackages_${llvmVersion}".bintools}/bin/nm"; + MY_RANLIB = "${pkgs."llvmPackages_${llvmVersion}".bintools}/bin/ranlib"; + MY_STRIP = "${pkgs."llvmPackages_${llvmVersion}".bintools}/bin/strip"; + MY_OBJDUMP = "${pkgs."llvmPackages_${llvmVersion}".bintools}/bin/objdump"; + MY_DLLTOOL = "${pkgs."llvmPackages_${llvmVersion}".bintools}/bin/dlltool"; MY_WASI_SYSROOT = "${wasi-sysroot}"; + MY_LIBCLANG_RT = "${libclang_rt}"; MY_WASM_OPT = "${pkgs.binaryen}/bin/wasm-opt"; }; }; diff --git a/crates/numcodecs-wasm-builder/buildenv/include.h b/crates/numcodecs-wasm-builder/buildenv/include.h new file mode 100644 index 00000000..818aa226 --- /dev/null +++ b/crates/numcodecs-wasm-builder/buildenv/include.h @@ -0,0 +1,12 @@ +#ifndef __ASSEMBLER__ +#ifndef _CODECS_BUILD_C_INCLUDE_H +#define _CODECS_BUILD_C_INCLUDE_H + +#include + +static inline int raise(int signal) { + abort(); +} + +#endif // _CODECS_BUILD_C_INCLUDE_H +#endif // __ASSEMBLER__ diff --git a/crates/numcodecs-wasm-builder/buildenv/include.hpp b/crates/numcodecs-wasm-builder/buildenv/include.hpp index 62e8a203..037b666d 100644 --- a/crates/numcodecs-wasm-builder/buildenv/include.hpp +++ b/crates/numcodecs-wasm-builder/buildenv/include.hpp @@ -1,8 +1,10 @@ +#ifndef __ASSEMBLER__ #ifndef _CODECS_BUILD_CPP_INCLUDE_HPP #define _CODECS_BUILD_CPP_INCLUDE_HPP #include #include +#include extern "C" inline void __cxa_pure_virtual() { @@ -14,13 +16,18 @@ extern "C" inline void *__cxa_allocate_exception(size_t) throw() std::abort(); } -extern "C" inline void __cxa_throw() +extern "C" inline void *__dynamic_cast(const void* __src_ptr, const void* __src_type, const void* __dst_type, ptrdiff_t __src2dst) { std::abort(); } -extern "C" unsigned int __attribute__((weak)) __cxa_uncaught_exceptions(); -extern "C" unsigned int __cxa_uncaught_exceptions() +extern "C" inline void __cxa_throw(void *thrown_exception, std::type_info *tinfo, void *(*dest)(void *)) +{ + std::abort(); +} + +extern "C" unsigned int __attribute__((weak)) __cxa_uncaught_exceptions() noexcept; +extern "C" unsigned int __cxa_uncaught_exceptions() noexcept { return 0; } @@ -104,3 +111,4 @@ void operator delete(void *p, std::size_t sz) noexcept } #endif // _CODECS_BUILD_CPP_INCLUDE_HPP +#endif // __ASSEMBLER__ diff --git a/crates/numcodecs-wasm-builder/src/main.rs b/crates/numcodecs-wasm-builder/src/main.rs index 01fbb00e..f83f3efb 100644 --- a/crates/numcodecs-wasm-builder/src/main.rs +++ b/crates/numcodecs-wasm-builder/src/main.rs @@ -130,6 +130,10 @@ fn copy_buildenv_to_crate(crate_dir: &Path) -> io::Result<()> { include_str!("../buildenv/flake.lock"), )?; + fs::write( + crate_dir.join("include.h"), + include_str!("../buildenv/include.h"), + )?; fs::write( crate_dir.join("include.hpp"), include_str!("../buildenv/include.hpp"), @@ -150,7 +154,12 @@ struct NixEnv { libclang: PathBuf, lld: PathBuf, nm: PathBuf, + ranlib: PathBuf, + strip: PathBuf, + objdump: PathBuf, + dlltool: PathBuf, wasi_sysroot: PathBuf, + libclang_rt: PathBuf, wasm_opt: PathBuf, } @@ -210,7 +219,12 @@ impl NixEnv { libclang: try_read_env(&env, "MY_LIBCLANG")?, lld: try_read_env(&env, "MY_LLD")?, nm: try_read_env(&env, "MY_NM")?, + ranlib: try_read_env(&env, "MY_RANLIB")?, + strip: try_read_env(&env, "MY_STRIP")?, + objdump: try_read_env(&env, "MY_OBJDUMP")?, + dlltool: try_read_env(&env, "MY_DLLTOOL")?, wasi_sysroot: try_read_env(&env, "MY_WASI_SYSROOT")?, + libclang_rt: try_read_env(&env, "MY_LIBCLANG_RT")?, wasm_opt: try_read_env(&env, "MY_WASM_OPT")?, }) } @@ -225,7 +239,12 @@ fn configure_cargo_cmd(nix_env: &NixEnv, target_dir: &Path, crate_dir: &Path) -> libclang, lld, nm, + ranlib, + strip, + objdump, + dlltool, wasi_sysroot, + libclang_rt, .. } = nix_env; @@ -239,6 +258,7 @@ fn configure_cargo_cmd(nix_env: &NixEnv, target_dir: &Path, crate_dir: &Path) -> cmd.arg("path:."); cmd.arg("--command"); cmd.arg("env"); + cmd.arg("GMP_MPFR_SYS_CACHE="); cmd.arg(format!("CC={clang}", clang = clang.join("clang").display())); cmd.arg(format!( "CXX={clang}", @@ -248,6 +268,10 @@ fn configure_cargo_cmd(nix_env: &NixEnv, target_dir: &Path, crate_dir: &Path) -> cmd.arg(format!("LLD={lld}", lld = lld.join("lld").display())); cmd.arg(format!("AR={ar}", ar = ar.display())); cmd.arg(format!("NM={nm}", nm = nm.display())); + cmd.arg(format!("RANLIB={ranlib}", ranlib = ranlib.display())); + cmd.arg(format!("STRIP={strip}", strip = strip.display())); + cmd.arg(format!("OBJDUMP={objdump}", objdump = objdump.display())); + cmd.arg(format!("DLLTOOL={dlltool}", dlltool = dlltool.display())); cmd.arg(format!( "LIBCLANG_PATH={libclang}", libclang = libclang.display() @@ -255,7 +279,9 @@ fn configure_cargo_cmd(nix_env: &NixEnv, target_dir: &Path, crate_dir: &Path) -> cmd.arg(format!( "CFLAGS=--target=wasm32-wasip1 -nodefaultlibs -resource-dir {resource_dir} \ --sysroot={wasi_sysroot} -isystem {clang_include} -isystem {wasi32_wasi_include} \ - -isystem {include} -B {lld} -D_WASI_EMULATED_PROCESS_CLOCKS -O3", + -isystem {include} -B {lld} -D_WASI_EMULATED_PROCESS_CLOCKS -D_WASI_EMULATED_SIGNAL \ + -include {c_include_path} -O3 \ + -DHAVE_STRNLEN=1 -DHAVE_MEMSET=1 -DHAVE_RAISE=1", resource_dir = libclang.join("clang").join(llvm_version).display(), wasi_sysroot = wasi_sysroot.display(), clang_include = libclang @@ -266,6 +292,7 @@ fn configure_cargo_cmd(nix_env: &NixEnv, target_dir: &Path, crate_dir: &Path) -> wasi32_wasi_include = wasi_sysroot.join("include").join("wasm32-wasip1").display(), include = wasi_sysroot.join("include").display(), lld = lld.display(), + c_include_path = crate_dir.join("include.h").display(), )); cmd.arg(format!( "CXXFLAGS=--target=wasm32-wasip1 -nodefaultlibs -resource-dir {resource_dir} \ @@ -325,10 +352,17 @@ fn configure_cargo_cmd(nix_env: &NixEnv, target_dir: &Path, crate_dir: &Path) -> cmd.arg("CXXSTDLIB=c++"); // disable default flags from cc cmd.arg("CRATE_CC_NO_DEFAULTS=1"); - cmd.arg("LDFLAGS=-lc -lwasi-emulated-process-clocks"); cmd.arg(format!( - "RUSTFLAGS=-C panic=abort -C strip=symbols -C link-arg=-L{wasm32_wasi_lib}", + "LDFLAGS=-lc -lwasi-emulated-process-clocks -lwasi-emulated-signal \ + -L{libclang_rt} -lclang_rt.builtins", + libclang_rt = libclang_rt.join("wasm32-unknown-wasip1").display(), + )); + cmd.arg(format!( + "RUSTFLAGS=-C panic=abort -C strip=symbols \ + -C link-arg=-L{wasm32_wasi_lib} \ + -C link-arg=-L{libclang_rt} -C link-arg=-lclang_rt.builtins", wasm32_wasi_lib = wasi_sysroot.join("lib").join("wasm32-wasip1").display(), + libclang_rt = libclang_rt.join("wasm32-unknown-wasip1").display(), )); cmd.arg(format!( "CARGO_TARGET_DIR={target_dir}", diff --git a/py/numcodecs-wasm-template/pyproject.toml b/py/numcodecs-wasm-template/pyproject.toml index e1fea1a0..e4e06020 100644 --- a/py/numcodecs-wasm-template/pyproject.toml +++ b/py/numcodecs-wasm-template/pyproject.toml @@ -16,7 +16,7 @@ license = { file = "LICENSE" } requires-python = ">=3.10" dependencies = [ - "numcodecs-wasm~=0.2.0", # wasi 0.2.6 + "numcodecs-wasm~=0.2.2", # wasi 0.2.6 ] [project.entry-points."numcodecs.codecs"] diff --git a/py/numcodecs-wasm/Cargo.toml b/py/numcodecs-wasm/Cargo.toml index b3a44e31..c8e7d196 100644 --- a/py/numcodecs-wasm/Cargo.toml +++ b/py/numcodecs-wasm/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "numcodecs-wasm" -version = "0.2.1+wasi0.2.6" +version = "0.2.2+wasi0.2.6" edition = { workspace = true } authors = { workspace = true } repository = { workspace = true } diff --git a/py/numcodecs-wasm/pyproject.toml b/py/numcodecs-wasm/pyproject.toml index f7b8afc3..ac2b9b29 100644 --- a/py/numcodecs-wasm/pyproject.toml +++ b/py/numcodecs-wasm/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "maturin" [project] name = "numcodecs_wasm" -version = "0.2.1" # wasi 0.2.6 +version = "0.2.2" # wasi 0.2.6 description = "numcodecs compression for codecs compiled to WebAssembly" authors = [{ name = "Juniper Tyree", email = "juniper.tyree@helsinki.fi" }]