diff --git a/CHANGELOG.md b/CHANGELOG.md index 98daeccf..530f08a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). ## [Unreleased] +- +- Add DuckDB BaseConverter for efficiently transforming large datasets ## [v0.21.0] - 2026-02-16 diff --git a/fiboa_cli/conversion/duckdb.py b/fiboa_cli/conversion/duckdb.py new file mode 100644 index 00000000..9b4f9fc1 --- /dev/null +++ b/fiboa_cli/conversion/duckdb.py @@ -0,0 +1,221 @@ +import json +import os +from pathlib import Path +from tempfile import NamedTemporaryFile + +import duckdb +import numpy as np +import pyarrow as pa +import pyarrow.parquet as pq +from geopandas.array import from_wkb +from pyarrow.lib import StructArray +from vecorel_cli.encoding.geojson import VecorelJSONEncoder + +from .fiboa_converter import FiboaBaseConverter + + +# This converter is experimental, use with caution. +# Results may not be fully fiboa compliant yet. +# Use this primarily for datasets that are too large to be processed by the default converter +class FiboaDuckDBBaseConverter(FiboaBaseConverter): + def convert( + self, + output_file, + cache=None, + input_files=None, + variant=None, + compression=None, + geoparquet_version=None, + original_geometries=False, + **kwargs, + ) -> str: + if not original_geometries: + self.warning( + "original_geometries is not supported for DuckDB-based converters and will always write original geometries" + ) + + geoparquet_version = geoparquet_version or "1.1.0" + compression = compression or "brotli" + + self.variant = variant + cid = self.id.strip() + if self.bbox is not None and len(self.bbox) != 4: + raise ValueError("If provided, the bounding box must consist of 4 numbers") + + # Create output folder if it doesn't exist + directory = os.path.dirname(output_file) + if directory: + os.makedirs(directory, exist_ok=True) + + if input_files is not None and isinstance(input_files, dict) and len(input_files) > 0: + self.warning("Using user provided input file(s) instead of the pre-defined file(s)") + urls = input_files + else: + urls = self.get_urls() + if urls is None: + raise ValueError("No input files provided") + + self.info("Getting file(s) if not cached yet") + if cache: + request_args = {} + if self.avoid_range_request: + request_args["block_size"] = 0 + urls = self.download_files(urls, cache, **request_args) + elif self.avoid_range_request: + self.warning( + "avoid_range_request is set, but cache is not used, so this setting has no effect" + ) + + selections = [] + geom_column = None + for k, v in self.columns.items(): + if k in self.column_migrations: + selections.append(f'{self.column_migrations.get(k)} as "{v}"') + else: + selections.append(f'"{k}" as "{v}"') + if v == "geometry": + geom_column = k + selection = ", ".join(selections) + + filters = [] + where = "" + if self.bbox is not None: + filters.append( + f"ST_Intersects(geometry, ST_MakeEnvelope({self.bbox[0]}, {self.bbox[1]}, {self.bbox[2]}, {self.bbox[3]}))" + ) + for k, v in self.column_filters.items(): + filters.append(v) + if len(filters) > 0: + where = f"WHERE {' AND '.join(filters)}" + + if isinstance(urls, str): + sources = f'"{urls}"' + else: + paths = [] + for url in urls: + if isinstance(url, tuple): + paths.append(f'"{url[0]}"') + else: + paths.append(f'"{url}"') + sources = "[" + ",".join(paths) + "]" + + collection = self.create_collection(cid) + collection.update(self.column_additions) + collection["collection"] = self.id + + if isinstance(output_file, Path): + output_file = str(output_file) + + collection_json = json.dumps(collection, cls=VecorelJSONEncoder).encode("utf-8") + + con = duckdb.connect() + con.install_extension("spatial") + con.load_extension("spatial") + con.execute( + f""" + COPY ( + SELECT {selection} + FROM read_parquet({sources}, union_by_name=true) + {where} + ORDER BY ST_Hilbert({geom_column}) + ) TO ? ( + FORMAT parquet, + compression ?, + KV_METADATA {{ + collection: ?, + }} + ) + """, + [output_file, compression, collection_json], + ) + + # Post-process the written Parquet to proper GeoParquet v1.1 with bbox and nullability + try: + pq_file = pq.ParquetFile(output_file) + + existing_schema = pq_file.schema_arrow + col_names = existing_schema.names + assert "geometry" in col_names, "Missing geometry column in output parquet file" + + schemas = collection.merge_schemas({}) + collection_only = {k for k, v in schemas.get("collection", {}).items() if v} + required_columns = {"geometry"} | { + r + for r in schemas.get("required", []) + if r in col_names and r not in collection_only + } + if "id" in col_names: + required_columns.add("id") + + # Update for version 1.1.0 + metadata = existing_schema.metadata + if geoparquet_version > "1.0.0": + geo_meta = json.loads(existing_schema.metadata[b"geo"]) + geo_meta["version"] = geoparquet_version + metadata[b"geo"] = json.dumps(geo_meta).encode("utf-8") + + # Build a new Arrow schema with adjusted nullability + new_fields = [] + for field in existing_schema: + if field.name in required_columns and field.nullable: + new_fields.append( + pa.field(field.name, field.type, nullable=False, metadata=field.metadata) + ) + else: + new_fields.append(field) + + add_bbox = geoparquet_version > "1.0.0" and "bbox" not in col_names + if add_bbox: + new_fields.append( + pa.field( + "bbox", + pa.struct( + [ + ("xmin", pa.float64()), + ("ymin", pa.float64()), + ("xmax", pa.float64()), + ("ymax", pa.float64()), + ] + ), + ) + ) + new_schema = pa.schema(new_fields, metadata=metadata) + + # 7) Streamingly rewrite the file to a temp file and replace atomically + with NamedTemporaryFile( + "wb", delete=False, dir=os.path.dirname(output_file), suffix=".parquet" + ) as tmp: + tmp_path = tmp.name + + writer = pq.ParquetWriter( + tmp_path, + new_schema, + compression=compression, + use_dictionary=True, + write_statistics=True, + ) + try: + bbox_names = ["ymax", "xmax", "ymin", "xmin"] + for rg in range(pq_file.num_row_groups): + tbl = pq_file.read_row_group(rg) + if add_bbox: + # determine bounds, change to StructArray type + bounds = from_wkb(tbl["geometry"]).bounds + bbox_array = StructArray.from_arrays( + np.rot90(bounds), + names=bbox_names, + ) + tbl = tbl.append_column("bbox", bbox_array) + # Ensure table adheres to the new schema (mainly nullability); cast if needed + if tbl.schema != new_schema: + # Align field order/types; this does not materialize data beyond the batch + tbl = tbl.cast(new_schema, safe=False) + writer.write_table(tbl) + finally: + writer.close() + + os.replace(tmp_path, output_file) + except Exception as e: + self.warning(f"GeoParquet 1.1 post-processing failed: {e}") + + return output_file diff --git a/fiboa_cli/datasets/jp.py b/fiboa_cli/datasets/jp.py index 57c31a24..1799c7f4 100644 --- a/fiboa_cli/datasets/jp.py +++ b/fiboa_cli/datasets/jp.py @@ -1,10 +1,9 @@ -import pandas as pd +from fiboa_cli.conversion.duckdb import FiboaDuckDBBaseConverter -from ..conversion.fiboa_converter import FiboaBaseConverter - -class JPConverter(FiboaBaseConverter): +class JPConverter(FiboaDuckDBBaseConverter): variants = { + "test": "./tests/data-files/convert/jp/jp_field_polygons_2024.parquet", "2024": "https://data.source.coop/pacificspatial/field-polygon-jp/parquet/jp_field_polygons_2024.parquet", "2023": "https://data.source.coop/pacificspatial/field-polygon-jp/parquet/jp_field_polygons_2023.parquet", "2022": "https://data.source.coop/pacificspatial/field-polygon-jp/parquet/jp_field_polygons_2022.parquet", @@ -30,23 +29,11 @@ class JPConverter(FiboaBaseConverter): "polygon_uuid": "id", "land_type_en": "land_type_en", "local_government_cd": "admin_local_code", - "issue_year": "determination:datetime", - } - column_migrations = { - "issue_year": lambda col: pd.to_datetime(col, format="%Y"), } - + column_additions = {"determination:datetime": "2024-01-01T00:00:00Z"} missing_schemas = { "properties": { "land_type_en": {"type": "string"}, "admin_local_code": {"type": "string"}, } } - - def convert(self, *args, **kwargs): - # Open only these columns to limit memory usage - super().convert( - *args, - columns=["GEOM", "polygon_uuid", "land_type_en", "local_government_cd", "issue_year"], - **kwargs, - ) diff --git a/pixi.lock b/pixi.lock index 1f6b2a69..11afdf14 100644 --- a/pixi.lock +++ b/pixi.lock @@ -32,6 +32,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/cryptography-46.0.5-py314h7fe84b3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/decorator-5.2.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/distlib-0.4.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/duckdb-1.4.2-h332efcf_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/exceptiongroup-1.3.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/filelock-3.21.2-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/frozenlist-1.7.0-pyhf298e5d_0.conda @@ -110,6 +111,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-cov-6.3.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/python-3.14.3-h32b2ec7_101_cp314.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.9.0.post0-pyhe01879c_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/python-duckdb-1.4.2-py314ha160325_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.14-8_cp314.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pyu2f-0.1.5-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pyyaml-6.0.3-py314h67df5f8_1.conda @@ -201,6 +203,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-64/cryptography-46.0.5-py314h6a45124_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/decorator-5.2.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/distlib-0.4.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/duckdb-1.4.2-h332efcf_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/exceptiongroup-1.3.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/filelock-3.21.2-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/frozenlist-1.7.0-pyhf298e5d_0.conda @@ -271,6 +274,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-cov-6.3.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/python-3.14.3-h4f44bb5_101_cp314.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.9.0.post0-pyhe01879c_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/python-duckdb-1.4.2-py314h21b9a27_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.14-8_cp314.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pyu2f-0.1.5-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/pyyaml-6.0.3-py314h10d0514_1.conda @@ -362,6 +366,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/cryptography-46.0.5-py314h2cafa77_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/decorator-5.2.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/distlib-0.4.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/duckdb-1.4.2-h332efcf_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/exceptiongroup-1.3.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/filelock-3.21.2-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/frozenlist-1.7.0-pyhf298e5d_0.conda @@ -433,6 +438,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-cov-6.3.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.14.3-h4c637c5_101_cp314.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.9.0.post0-pyhe01879c_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-duckdb-1.4.2-py314h93ecee7_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.14-8_cp314.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pyu2f-0.1.5-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyyaml-6.0.3-py314h6e9b3f0_1.conda @@ -524,6 +530,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/win-64/cryptography-46.0.5-py314he884d78_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/decorator-5.2.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/distlib-0.4.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/duckdb-1.4.2-h332efcf_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/exceptiongroup-1.3.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/filelock-3.21.2-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/frozenlist-1.7.0-pyhf298e5d_0.conda @@ -588,6 +595,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-cov-6.3.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/python-3.14.3-h4b44e0e_101_cp314.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.9.0.post0-pyhe01879c_2.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/python-duckdb-1.4.2-py314h13fbf68_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.14-8_cp314.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pyu2f-0.1.5-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/win-64/pyyaml-6.0.3-py314h2359020_1.conda @@ -688,6 +696,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/click-8.3.1-pyh8f84b5b_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/cryptography-46.0.5-py314h7fe84b3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/decorator-5.2.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/duckdb-1.4.2-h332efcf_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/frozenlist-1.7.0-pyhf298e5d_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/fsspec-2025.7.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/gcsfs-2025.7.0-pyhd8ed1ab_0.conda @@ -755,6 +764,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/pysocks-1.7.1-pyha55dd90_7.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/python-3.14.3-h32b2ec7_101_cp314.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.9.0.post0-pyhe01879c_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/python-duckdb-1.4.2-py314ha160325_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.14-8_cp314.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pyu2f-0.1.5-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pyyaml-6.0.3-py314h67df5f8_1.conda @@ -832,6 +842,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/click-8.3.1-pyh8f84b5b_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/cryptography-46.0.5-py314h6a45124_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/decorator-5.2.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/duckdb-1.4.2-h332efcf_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/frozenlist-1.7.0-pyhf298e5d_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/fsspec-2025.7.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/gcsfs-2025.7.0-pyhd8ed1ab_0.conda @@ -891,6 +902,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/pysocks-1.7.1-pyha55dd90_7.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/python-3.14.3-h4f44bb5_101_cp314.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.9.0.post0-pyhe01879c_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/python-duckdb-1.4.2-py314h21b9a27_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.14-8_cp314.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pyu2f-0.1.5-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/pyyaml-6.0.3-py314h10d0514_1.conda @@ -968,6 +980,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/click-8.3.1-pyh8f84b5b_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/cryptography-46.0.5-py314h2cafa77_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/decorator-5.2.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/duckdb-1.4.2-h332efcf_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/frozenlist-1.7.0-pyhf298e5d_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/fsspec-2025.7.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/gcsfs-2025.7.0-pyhd8ed1ab_0.conda @@ -1028,6 +1041,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/pysocks-1.7.1-pyha55dd90_7.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.14.3-h4c637c5_101_cp314.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.9.0.post0-pyhe01879c_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-duckdb-1.4.2-py314h93ecee7_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.14-8_cp314.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pyu2f-0.1.5-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyyaml-6.0.3-py314h6e9b3f0_1.conda @@ -1106,6 +1120,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/win-64/cryptography-46.0.5-py314he884d78_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/decorator-5.2.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/duckdb-1.4.2-h332efcf_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/frozenlist-1.7.0-pyhf298e5d_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/fsspec-2025.7.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/gcsfs-2025.7.0-pyhd8ed1ab_0.conda @@ -1159,6 +1174,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/pysocks-1.7.1-pyh09c184e_7.conda - conda: https://conda.anaconda.org/conda-forge/win-64/python-3.14.3-h4b44e0e_101_cp314.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.9.0.post0-pyhe01879c_2.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/python-duckdb-1.4.2-py314h13fbf68_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.14-8_cp314.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pyu2f-0.1.5-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/win-64/pyyaml-6.0.3-py314h2359020_1.conda @@ -1242,6 +1258,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/certifi-2026.1.4-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/charset-normalizer-3.4.4-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/click-8.3.1-pyh8f84b5b_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/duckdb-1.4.2-h332efcf_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/frozenlist-1.7.0-pyhf298e5d_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/fsspec-2025.7.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/h2-4.3.0-pyhcf101f3_0.conda @@ -1284,6 +1301,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/pysocks-1.7.1-pyha55dd90_7.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/python-3.14.3-h32b2ec7_101_cp314.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.9.0.post0-pyhe01879c_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/python-duckdb-1.4.2-py314ha160325_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.14-8_cp314.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pyyaml-6.0.3-py314h67df5f8_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/readline-8.3-h853b02a_0.conda @@ -1348,6 +1366,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/certifi-2026.1.4-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/charset-normalizer-3.4.4-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/click-8.3.1-pyh8f84b5b_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/duckdb-1.4.2-h332efcf_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/frozenlist-1.7.0-pyhf298e5d_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/fsspec-2025.7.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/h2-4.3.0-pyhcf101f3_0.conda @@ -1383,6 +1402,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/pysocks-1.7.1-pyha55dd90_7.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/python-3.14.3-h4f44bb5_101_cp314.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.9.0.post0-pyhe01879c_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/python-duckdb-1.4.2-py314h21b9a27_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.14-8_cp314.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/pyyaml-6.0.3-py314h10d0514_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/readline-8.3-h68b038d_0.conda @@ -1447,6 +1467,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/certifi-2026.1.4-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/charset-normalizer-3.4.4-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/click-8.3.1-pyh8f84b5b_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/duckdb-1.4.2-h332efcf_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/frozenlist-1.7.0-pyhf298e5d_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/fsspec-2025.7.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/h2-4.3.0-pyhcf101f3_0.conda @@ -1483,6 +1504,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/pysocks-1.7.1-pyha55dd90_7.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.14.3-h4c637c5_101_cp314.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.9.0.post0-pyhe01879c_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-duckdb-1.4.2-py314h93ecee7_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.14-8_cp314.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyyaml-6.0.3-py314h6e9b3f0_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/readline-8.3-h46df422_0.conda @@ -1547,6 +1569,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/charset-normalizer-3.4.4-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/click-8.3.1-pyha7b4d00_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/duckdb-1.4.2-h332efcf_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/frozenlist-1.7.0-pyhf298e5d_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/fsspec-2025.7.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/h2-4.3.0-pyhcf101f3_0.conda @@ -1576,6 +1599,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/pysocks-1.7.1-pyh09c184e_7.conda - conda: https://conda.anaconda.org/conda-forge/win-64/python-3.14.3-h4b44e0e_101_cp314.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.9.0.post0-pyhe01879c_2.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/python-duckdb-1.4.2-py314h13fbf68_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.14-8_cp314.conda - conda: https://conda.anaconda.org/conda-forge/win-64/pyyaml-6.0.3-py314h2359020_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/requests-2.32.5-pyhcf101f3_1.conda @@ -1657,6 +1681,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/coverage-7.13.4-py314h67df5f8_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/distlib-0.4.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/duckdb-1.4.2-h332efcf_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/exceptiongroup-1.3.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/filelock-3.21.2-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/frozenlist-1.7.0-pyhf298e5d_0.conda @@ -1711,6 +1736,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-cov-6.3.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/python-3.14.3-h32b2ec7_101_cp314.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.9.0.post0-pyhe01879c_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/python-duckdb-1.4.2-py314ha160325_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.14-8_cp314.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pyyaml-6.0.3-py314h67df5f8_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/readline-8.3-h853b02a_0.conda @@ -1790,6 +1816,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/coverage-7.13.4-py314h10d0514_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/distlib-0.4.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/duckdb-1.4.2-h332efcf_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/exceptiongroup-1.3.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/filelock-3.21.2-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/frozenlist-1.7.0-pyhf298e5d_0.conda @@ -1837,6 +1864,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-cov-6.3.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/python-3.14.3-h4f44bb5_101_cp314.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.9.0.post0-pyhe01879c_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/python-duckdb-1.4.2-py314h21b9a27_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.14-8_cp314.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/pyyaml-6.0.3-py314h10d0514_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/readline-8.3-h68b038d_0.conda @@ -1916,6 +1944,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/coverage-7.13.4-py314h6e9b3f0_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/distlib-0.4.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/duckdb-1.4.2-h332efcf_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/exceptiongroup-1.3.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/filelock-3.21.2-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/frozenlist-1.7.0-pyhf298e5d_0.conda @@ -1964,6 +1993,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-cov-6.3.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.14.3-h4c637c5_101_cp314.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.9.0.post0-pyhe01879c_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-duckdb-1.4.2-py314h93ecee7_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.14-8_cp314.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyyaml-6.0.3-py314h6e9b3f0_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/readline-8.3-h46df422_0.conda @@ -2042,6 +2072,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/win-64/coverage-7.13.4-py314h2359020_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/distlib-0.4.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/duckdb-1.4.2-h332efcf_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/exceptiongroup-1.3.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/filelock-3.21.2-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/frozenlist-1.7.0-pyhf298e5d_0.conda @@ -2083,6 +2114,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-cov-6.3.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/python-3.14.3-h4b44e0e_101_cp314.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.9.0.post0-pyhe01879c_2.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/python-duckdb-1.4.2-py314h13fbf68_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.14-8_cp314.conda - conda: https://conda.anaconda.org/conda-forge/win-64/pyyaml-6.0.3-py314h2359020_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/requests-2.32.5-pyhcf101f3_1.conda @@ -2789,6 +2821,16 @@ packages: - pkg:pypi/distlib?source=hash-mapping size: 275642 timestamp: 1752823081585 +- conda: https://conda.anaconda.org/conda-forge/noarch/duckdb-1.4.2-h332efcf_0.conda + sha256: 2cc2cdf4a5a0ac975ed2f42b9a9f9cc5ee62213eb7eee11aabdde04127239d7c + md5: 4fff74996eacd532d60e2c544660a507 + depends: + - python-duckdb >=1.4.2,<1.4.3.0a0 + license: MIT + license_family: MIT + purls: [] + size: 7530 + timestamp: 1763377721341 - conda: https://conda.anaconda.org/conda-forge/noarch/exceptiongroup-1.3.1-pyhd8ed1ab_0.conda sha256: ee6cf346d017d954255bbcbdb424cddea4d14e4ed7e9813e429db1d795d01144 md5: 8e662bd460bda79b1ea39194e3c4c9ab @@ -2803,10 +2845,11 @@ packages: - pypi: ./ name: fiboa-cli version: 0.21.0 - sha256: 5a8a4c3d9234870ac9a54c0cc925c41f59ca033ee111dd241c1af9633591f47b + sha256: da5f2208753c0eb13ecc6e90d3d968fc4080f074550edbb6cb7bc88ab5c30d72 requires_dist: - vecorel-cli==0.2.15 - spdx-license-list==3.27.0 + - duckdb==1.4.2 requires_python: '>=3.11' editable: true - conda: https://conda.anaconda.org/conda-forge/noarch/filelock-3.21.2-pyhd8ed1ab_0.conda @@ -6297,6 +6340,65 @@ packages: - pkg:pypi/python-dateutil?source=hash-mapping size: 233310 timestamp: 1751104122689 +- conda: https://conda.anaconda.org/conda-forge/linux-64/python-duckdb-1.4.2-py314ha160325_0.conda + sha256: e5d655846320be8b2cdb52302727b86748dbe25b28460640be2c5a234a9ab506 + md5: b1463a8e885b875d931dba7dc76cb250 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + - libstdcxx >=14 + - python >=3.14,<3.15.0a0 + - python_abi 3.14.* *_cp314 + license: MIT + license_family: MIT + purls: + - pkg:pypi/duckdb?source=hash-mapping + size: 16377782 + timestamp: 1763377103187 +- conda: https://conda.anaconda.org/conda-forge/osx-64/python-duckdb-1.4.2-py314h21b9a27_0.conda + sha256: a0ee1056d10fb8d271b6ffc07030534b86ac5b7788d2907a9caf55a6e9565f08 + md5: ef180499a3ddff68c2b50f79733c9cf8 + depends: + - __osx >=10.13 + - libcxx >=19 + - python >=3.14,<3.15.0a0 + - python_abi 3.14.* *_cp314 + license: MIT + license_family: MIT + purls: + - pkg:pypi/duckdb?source=hash-mapping + size: 12398574 + timestamp: 1763377914269 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-duckdb-1.4.2-py314h93ecee7_0.conda + sha256: 71f7644bd393f2508f077b1dd68b7354329457c8ea903354d4f2341f105ef4ac + md5: 7fd8e05bc4116c3cf05513d844d496d3 + depends: + - __osx >=11.0 + - libcxx >=19 + - python >=3.14,<3.15.0a0 + - python >=3.14,<3.15.0a0 *_cp314 + - python_abi 3.14.* *_cp314 + license: MIT + license_family: MIT + purls: + - pkg:pypi/duckdb?source=hash-mapping + size: 10777580 + timestamp: 1763377988015 +- conda: https://conda.anaconda.org/conda-forge/win-64/python-duckdb-1.4.2-py314h13fbf68_0.conda + sha256: 7900099f892b9d80d4c4cce60a85dd410ac38ec73f47005250982d84c9e7eea7 + md5: 3f300010eff0b5880995e98ec5753930 + depends: + - python >=3.14,<3.15.0a0 + - python_abi 3.14.* *_cp314 + - ucrt >=10.0.20348.0 + - vc >=14.3,<15 + - vc14_runtime >=14.44.35208 + license: MIT + license_family: MIT + purls: + - pkg:pypi/duckdb?source=hash-mapping + size: 9621168 + timestamp: 1763379001464 - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.14-8_cp314.conda build_number: 8 sha256: ad6d2e9ac39751cc0529dd1566a26751a0bf2542adb0c232533d32e176e21db5 diff --git a/pyproject.toml b/pyproject.toml index 735559ce..7e56f4f4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,7 @@ requires-python = ">=3.11" dependencies = [ "vecorel-cli==0.2.15", "spdx-license-list==3.27.0", + "duckdb==1.4.2", ] [project.scripts]