From d4245a587414dd9274dcc797e4017ca767a1b3fb Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Fri, 24 Jul 2026 14:39:16 -0400 Subject: [PATCH] Add Rule: usage constraints as their own content-addressed entity required/multivalued/pattern/min/max don't live on RegistryProperty (a property required in one source's usage and optional in another's would never share a hash otherwise). They now live on a separate Rule entity that references the property it constrains via applies_to (its hash_id) - part of Rule's own content hash, so identical constraints on two different properties are different rules, and identical constraints on the same property from two sources collapse to one Rule with two ProvenanceEntry records, exactly like RegistryProperty/RegistryClass already do. Scoped to RegistryProperty only for now, as agreed - class-level rules are a possible future extension. - schemas/meta_model.yaml: Rule is_a RegistryEntity, new slots applies_to/ required/multivalued/pattern/minimum_value/maximum_value. - schema_registry_utils/: Rule Pydantic model added; hashing.py widened to accept it. - neuro_ghost/db.py: new APPLIES_TO_P (Rule -> RegistryProperty) and HAS_PROVENANCE_RULE edges (named explicitly - HAS_PROVENANCE_R was already claimed by Relation). write_registry_entities() and a new write_rule_edges() generalize the existing write-if-new/attach- provenance-if-new pattern to Rule, shared with ingest_linkml.py. - neuro_ghost/ingest_linkml.py: _slot_to_dict() now extracts minimum_value/ maximum_value from LinkML. build_registry_entities() builds a Rule for any property declaring at least one constraint - a plain property gets no Rule at all, so unconstrained properties don't accumulate noise. CLI/ --wipe/SchemaVersionSnapshot.rule_count wired through. Verified: a real schema (bbqs.yml) organically produced 10 correct Rules with correct APPLIES_TO_P edges; cross-source Rule collapsing; required affecting Rule but not RegistryProperty identity; full seed -> ingest -> align -> export pipeline end to end. All 17 tests pass. docs/ingestion.md updated with a Rule section. Added docs/comparison-with- undata.md noting this is a NeuroGhost-specific extension beyond what undata's VISION.md describes. Co-Authored-By: Claude Sonnet 5 --- docs/comparison-with-undata.md | 51 ++++++++++++++++ docs/ingestion.md | 65 +++++++++++++++++---- neuro_ghost/db.py | 52 ++++++++++++++--- neuro_ghost/ingest_linkml.py | 96 +++++++++++++++++++++++++------ schema_registry_utils/__init__.py | 2 + schema_registry_utils/hashing.py | 13 +++-- schema_registry_utils/models.py | 15 +++++ schemas/meta_model.yaml | 56 +++++++++++++++--- tests/fixtures/required_c.yml | 15 +++++ tests/test_ingest_linkml.py | 72 ++++++++++++++++++----- tests/test_ingest_registry.py | 42 +++++++++++++- 11 files changed, 416 insertions(+), 63 deletions(-) create mode 100644 docs/comparison-with-undata.md create mode 100644 tests/fixtures/required_c.yml diff --git a/docs/comparison-with-undata.md b/docs/comparison-with-undata.md new file mode 100644 index 0000000..2128d9b --- /dev/null +++ b/docs/comparison-with-undata.md @@ -0,0 +1,51 @@ +# NeuroGhost vs undata + +Both tackle the same core problem — neuroscience data standards (BIDS, NWB, +DANDI, openMINDS, AIND, ...) each redefine the same concepts under different +names, and there's no machine-readable way to know two things are "the same". +undata (`/Users/dorota/repronim_various/undata`) is a much larger, production- +platform take on this; NeuroGhost deliberately borrows its core identity idea +and leaves the rest out. This is a quick reference for what's shared and what +isn't, and why. + +## Adopted from undata + +| Idea | undata | NeuroGhost | +|---|---|---| +| Content-addressed identity | `sha256` hash from semantic content, two-mode (ontology-anchored / structural fallback) | Same principle, **single mode only** — structural content hash (`name`/`description`/`range`/`units`/etc.), no ontology-anchored mode (see below) | +| Identity ≠ provenance | `ProvenanceEntry` list per entity, accumulates across sources | Same — `ProvenanceEntry` list, same idea, thinner field set (see below) | +| LinkML as intermediate representation | Every adapter emits a `SchemaDefinition`; one standard extractor classifies | Every converter emits LinkML YAML; `parse_linkml()`/`build_registry_entities()` does the extraction+hashing | +| PROV-O–grounded provenance fields | `generated_at`, `attributed_to`, `activity`, `derived_from` | Same four fields, same PROV-O predicates (`slot_uri: prov:...`) | + +## Deliberately not adopted + +| undata has | NeuroGhost's choice | Why | +|---|---|---| +| Two-mode hashing (ontology-anchored vs structural) | Structural only | Explicit decision: no ontology store, no ontology-anchored identity, for now | +| Align **before** Commit (alignment can inform/merge the hash) | Align **after** Commit (`align.py` runs post-ingestion) | No complex alignment exists yet — pure hash equality already handles exact duplicates; align-before-commit only matters once alignment can detect and merge *near*-duplicates | +| Knowledge service (13+ ontologies, embeddings, LLM verification, multi-precision SKOS) | `SkosMapping` model field exists but isn't wired into ingestion at all | Same reason as above — no ontology grounding in scope | +| `StorageBackend` protocol (pluggable file/Postgres backends) | One embedded LadybugDB file, no abstraction | NeuroGhost is a single-file registry driven by a GitHub Action, not a multi-user service | +| Task manager, GraphQL API, OIDC auth, curator roles, staged→curated lifecycle | None of it | All production-platform machinery for a hosted, multi-user service; out of scope | +| `ProvenanceEntry.class`/`name` (which source-schema class/slot this came from) and `source_ref` (repo/commit/file/checksum) | Considered, then dropped — see `docs/ingestion.md`'s note on `source_class`/`source_slot` | Redundant with graph structure (`HAS_PROPERTY` edges) or not yet needed | + +## Where NeuroGhost has gone further + +**`Rule`** (usage constraints — `required`/`multivalued`/`pattern`/min/max — +as their own content-addressed entity, referencing the `RegistryProperty` via +`applies_to`) isn't something undata's VISION.md describes as a distinct +entity type. It falls out of the same "identity ≠ usage" principle undata +established for provenance, just applied one level further: a property's +*core meaning* is separate not only from *where it came from* but also from +*how a given source happens to use it*. See `docs/ingestion.md`'s +[Rule section](ingestion.md#rule-usage-constraints-as-their-own-entity). + +## Entity type mapping + +| undata | NeuroGhost | Status | +|---|---|---| +| `Element` | `RegistryProperty` | Built | +| `Schema` | `RegistryClass` | Built | +| — | `Rule` | Built (NeuroGhost-specific, see above) | +| `Value` / `ValueSet` | `ValueSet` | Stub only | +| `Transform` | `Transform` | Stub only | +| `OntologyAnnotation` (multi-precision SKOS) | `SkosMapping` | Modeled, not wired into ingestion | diff --git a/docs/ingestion.md b/docs/ingestion.md index afaae5d..3014efc 100644 --- a/docs/ingestion.md +++ b/docs/ingestion.md @@ -54,9 +54,10 @@ LinkML YAML intermediate dict {classes: {...}, slots: {...}} │ build_registry_entities() — compute content hashes ▼ -RegistryProperty / RegistryClass instances (Pydantic, hash_id already set) +RegistryProperty / RegistryClass / Rule instances (Pydantic, hash_id already set) │ write_registry_entities() — write-if-new, attach-provenance-if-new │ write_structural_edges() — HAS_PROPERTY, SUBCLASS_OF + │ write_rule_edges() — APPLIES_TO_P ▼ LadybugDB graph ``` @@ -92,8 +93,12 @@ Converts the dict into real, hash-identified objects: - **Every entity gets one `ProvenanceEntry`** for this ingestion — `source`, `attributed_to` (agent), `generated_at`, `activity`, `registry_version` — entirely separate from the hash computation. +- **A property that declares `required`/`multivalued`/`pattern`/min/max also + gets a `Rule`**, `applies_to` = that property's `hash_id`. A plain property + with none of these gets no `Rule` at all — see [Rule](#rule-usage-constraints-as-their-own-entity) + below. -### 3. `write_registry_entities()` + `write_structural_edges()` (`neuro_ghost/db.py`) +### 3. `write_registry_entities()` + `write_structural_edges()` + `write_rule_edges()` (`neuro_ghost/db.py`) For each entity: does a node with this `hash_id` already exist? - **No** → create it. @@ -102,9 +107,11 @@ For each entity: does a node with this `hash_id` already exist? the same file from the same source twice adds nothing the second time). Then `HAS_PROPERTY` and `SUBCLASS_OF` edges get created from the resolved -hash references. These two functions are shared between `ingest_linkml.py` +hash references, and `APPLIES_TO_P` from each `Rule` to the property it +constrains. All three writer functions are shared between `ingest_linkml.py` and `seed.py` — schema.org is ingested through the exact same path, just with -`source="schema.org"`. +`source="schema.org"` (though `seed.py` never builds any `Rule`s — schema.org's +base vocabulary doesn't declare `required`/`multivalued`/etc. on its slots). ## The data model @@ -117,16 +124,47 @@ and `seed.py` — schema.org is ingested through the exact same path, just with | `class_uri` / `slot_uri` | `RegistryClass` / `RegistryProperty` | Ontology IRI preserved from the source. **Not** part of the hash — two schemas using different IRIs (or none) for the same content still collapse to one node. | | `provenance` | both | List of `ProvenanceEntry`. Accumulates, never affects `hash_id`. | | `source`, `attributed_to`, `generated_at`, `activity`, `derived_from`, `registry_version` | `ProvenanceEntry` | PROV-O–grounded (`slot_uri: prov:wasAttributedTo` etc.). `registry_version` lives here, not on the entity — the same entity can be attested by different sources at different times, each under a different registry version, so a single scalar on the entity doesn't fit (same reasoning as dropping `source_label`). | +| `applies_to`, `required`, `multivalued`, `pattern`, `minimum_value`, `maximum_value` | `Rule` | Identity-defining — see [Rule](#rule-usage-constraints-as-their-own-entity). | Field names were deliberately aligned with LinkML's own metamodel (`description`, `range`, `class_uri`/`slot_uri`, `is_a`, `abstract`) rather than inventing parallel terminology — e.g. `parse_linkml()` already produces `is_a` straight from `SchemaView`, so there's no translation step. -**Deliberately not modeled yet:** `required`/`multivalued` used to live on -`RegistryProperty` directly, which meant a property required in one schema's -usage and optional in another's could never share a hash. They've been -removed entirely — that's a **`Rule`** concern (still a stub), not identity. +## Rule: usage constraints as their own entity + +`required`/`multivalued` used to live on `RegistryProperty` directly, which +meant a property required in one schema's usage and optional in another's +could never share a `hash_id`. They're not modeled on `RegistryProperty` at +all now — a `Rule` is a separate content-addressed entity that references +the property it constrains: + +``` +RegistryProperty "age" (hash_id = A) + ▲ + │ APPLIES_TO_P + │ +Rule { applies_to: A, required: true } (hash_id = B) + ├── ProvenanceEntry{source: "bids", ...} + └── ProvenanceEntry{source: "dandi", ...} +``` + +- `applies_to` (the constrained property's `hash_id`) **is** part of the + `Rule`'s own hash — identical constraints on two *different* properties + are different rules, by design. +- Same collapsing behavior as everything else: two sources declaring the + exact same constraints on the exact same property produce one `Rule` node + with two `ProvenanceEntry` records, not two `Rule`s + (`test_identical_rule_from_two_sources_shares_one_hash_id`). +- A property with none of `required`/`multivalued`/`pattern`/min/max gets no + `Rule` at all — `build_registry_entities()`'s `_has_constraints()` gate + keeps unconstrained properties from accumulating empty, meaningless rules. +- Scoped to `RegistryProperty` only for now (`applies_to: RegistryProperty` + in the meta-model) — class-level rules are a possible future extension, + not built. +- Written through the same `write_registry_entities()`/`entity_exists()`/ + `write_provenance()` functions as `RegistryClass`/`RegistryProperty` (now + with an optional third `rules` argument) — no separate code path. ## Alignment @@ -169,7 +207,12 @@ the other layer does with it. takes this one step further, end to end: two schemas declare the exact same `age` slot except one marks it `required: true`. Ingesting both must produce exactly one `RegistryProperty` node, not two — proving `required` doesn't -leak into identity, in the real graph, not just in an isolated object. +leak into identity, in the real graph, not just in an isolated object. The +same test also confirms where `required: true` *does* end up: one `Rule`, +`APPLIES_TO_P` the "age" property. +`test_identical_rule_from_two_sources_shares_one_hash_id` proves the +content-addressing guarantee applies to `Rule` itself, the same way it does +to `RegistryProperty`/`RegistryClass`. ## Open question: when should ProvenanceEntry.registry_version be set? @@ -201,8 +244,8 @@ registry_version` stays `None` for now, pending a decision on the above. - **`derived_from`** on `ProvenanceEntry` is never populated — nothing yet detects "this hash supersedes that one" (would need an anchor like `(name, source)` to correlate an edit against prior content). -- **`Rule`/`Transform`/`ValueSet`** are stubs (`hash_id`/`name`/`description` - only). +- **`Transform`/`ValueSet`** are still stubs (`hash_id`/`name`/`description` + only) — `Rule` is real now (see above). - **`SemanticIdentity`/`PRIOR_VERSION*`** tables in `db.py`'s DDL are dead (superseded by content-hash identity) but not yet removed. - **`pandas`** isn't in `requirements.txt`, so `align.py`'s embedding cache diff --git a/neuro_ghost/db.py b/neuro_ghost/db.py index 60b37d0..7480898 100644 --- a/neuro_ghost/db.py +++ b/neuro_ghost/db.py @@ -85,7 +85,11 @@ def bump_version(ver: str, bump: str = "patch") -> str: # .derived_from) so this module doesn't need to import schema_registry_utils. LIST_FIELDS = {"provenance", "skos_mappings", "properties", "relations", "mixins"} -HAS_PROVENANCE_REL = {"RegistryClass": "HAS_PROVENANCE", "RegistryProperty": "HAS_PROVENANCE_P"} +HAS_PROVENANCE_REL = { + "RegistryClass": "HAS_PROVENANCE", + "RegistryProperty": "HAS_PROVENANCE_P", + "Rule": "HAS_PROVENANCE_RULE", +} def scalar_fields(entity) -> dict: @@ -145,19 +149,22 @@ def write_provenance(conn, label: str, hash_id: str, prov) -> bool: def write_registry_entities(conn, properties: dict, registry_classes: dict, + rules: dict | None = None, dry_run: bool = False) -> dict: """ - Write (or reuse) each property/class node by hash_id, then attach this - ingestion's ProvenanceEntry to every one of them. Existing nodes are - never overwritten — a hash match means identical content, so there is - nothing to update; only a new ProvenanceEntry may need attaching. - - `properties`/`registry_classes` are name -> entity dicts (values just - need .hash_id and .provenance; shared by ingest_linkml.py and seed.py). + Write (or reuse) each property/class/rule node by hash_id, then attach + this ingestion's ProvenanceEntry to every one of them. Existing nodes + are never overwritten — a hash match means identical content, so there + is nothing to update; only a new ProvenanceEntry may need attaching. + + `properties`/`registry_classes`/`rules` are name -> entity dicts (values + just need .hash_id and .provenance; shared by ingest_linkml.py and + seed.py). `rules` is optional since seed.py doesn't build any yet. """ stats = { "properties_new": 0, "properties_existing": 0, "classes_new": 0, "classes_existing": 0, + "rules_new": 0, "rules_existing": 0, "provenance_added": 0, } @@ -181,9 +188,36 @@ def write_registry_entities(conn, properties: dict, registry_classes: dict, if write_provenance(conn, "RegistryClass", rc.hash_id, prov): stats["provenance_added"] += 1 + for rule in (rules or {}).values(): + is_new = not entity_exists(conn, "Rule", rule.hash_id) + if is_new and not dry_run: + create_entity_node(conn, "Rule", rule) + stats["rules_new" if is_new else "rules_existing"] += 1 + if not dry_run: + for prov in rule.provenance: + if write_provenance(conn, "Rule", rule.hash_id, prov): + stats["provenance_added"] += 1 + return stats +def write_rule_edges(conn, rules: dict) -> int: + """APPLIES_TO_P: each Rule to the RegistryProperty it constrains.""" + rels = 0 + for rule in rules.values(): + already = conn.execute(""" + MATCH (r:Rule {hash_id: $r})-[:APPLIES_TO_P]->(p:RegistryProperty {hash_id: $p}) + RETURN r.hash_id LIMIT 1 + """, {"r": rule.hash_id, "p": rule.applies_to}).has_next() + if not already: + conn.execute(""" + MATCH (r:Rule {hash_id: $r}), (p:RegistryProperty {hash_id: $p}) + CREATE (r)-[:APPLIES_TO_P]->(p) + """, {"r": rule.hash_id, "p": rule.applies_to}) + rels += 1 + return rels + + def write_structural_edges(conn, registry_classes: dict) -> int: """ HAS_PROPERTY (from each class's own `properties`) + SUBCLASS_OF (from @@ -426,6 +460,7 @@ def _build_registry_ddl(yaml_path: str | Path = SCHEMA_YAML) -> list[str]: "CREATE REL TABLE IF NOT EXISTS HAS_PROVENANCE (FROM RegistryClass TO ProvenanceEntry)", "CREATE REL TABLE IF NOT EXISTS HAS_PROVENANCE_P (FROM RegistryProperty TO ProvenanceEntry)", "CREATE REL TABLE IF NOT EXISTS HAS_PROVENANCE_R (FROM Relation TO ProvenanceEntry)", + "CREATE REL TABLE IF NOT EXISTS HAS_PROVENANCE_RULE (FROM Rule TO ProvenanceEntry)", "CREATE REL TABLE IF NOT EXISTS MIXIN (FROM RegistryClass TO RegistryClass)", "CREATE REL TABLE IF NOT EXISTS SUBCLASS_OF (FROM RegistryClass TO RegistryClass)", @@ -462,6 +497,7 @@ def _build_registry_ddl(yaml_path: str | Path = SCHEMA_YAML) -> list[str]: # --- Infrastructure edges --- "CREATE REL TABLE IF NOT EXISTS APPLIES_TO (FROM Rule TO RegistryClass)", + "CREATE REL TABLE IF NOT EXISTS APPLIES_TO_P (FROM Rule TO RegistryProperty)", "CREATE REL TABLE IF NOT EXISTS PROV_GENERATED (FROM RegistryClass TO SchemaActivity)", "CREATE REL TABLE IF NOT EXISTS PROV_GENERATED_P (FROM RegistryProperty TO SchemaActivity)", "CREATE REL TABLE IF NOT EXISTS PROV_GENERATED_R (FROM Rule TO SchemaActivity)", diff --git a/neuro_ghost/ingest_linkml.py b/neuro_ghost/ingest_linkml.py index c3879ab..588a44a 100644 --- a/neuro_ghost/ingest_linkml.py +++ b/neuro_ghost/ingest_linkml.py @@ -34,11 +34,14 @@ ------------------------------- For every class → one RegistryClass node For every slot → one RegistryProperty node +For every slot declaring required/multivalued/pattern/min/max → one Rule + node, linked to its property via APPLIES_TO_P (a plain property with none + of these gets no Rule at all) For every class→slot relationship → one HAS_PROPERTY edge For every is_a relationship → one SUBCLASS_OF edge For every schema file → one SchemaSource node + one SchemaVersionSnapshot -For every (entity, source) attestation → one ProvenanceEntry node, - linked via HAS_PROVENANCE / HAS_PROVENANCE_P +For every (entity, source) attestation → one ProvenanceEntry node, linked + via HAS_PROVENANCE / HAS_PROVENANCE_P / HAS_PROVENANCE_RULE CONTENT-ADDRESSED IDENTITY --------------------------- @@ -74,12 +77,12 @@ sys.path.insert(0, str(Path(__file__).parent.parent)) from schema_registry_utils import ( - RegistryClass, RegistryProperty, ProvenanceEntry, compute_hash_id, + RegistryClass, RegistryProperty, Rule, ProvenanceEntry, compute_hash_id, ) from db import ( get_connection, make_iri, make_uid, now_iso, REG, - write_registry_entities, write_structural_edges, + write_registry_entities, write_structural_edges, write_rule_edges, ) DB_PATH = "./registry.lbug" @@ -186,6 +189,8 @@ def _slot_to_dict(slot, prefixes: dict[str, str]) -> dict: "multivalued": bool(slot.multivalued), "required": bool(slot.required), "pattern": slot.pattern or "", + "minimum_value": slot.minimum_value, + "maximum_value": slot.maximum_value, } @@ -292,13 +297,38 @@ def _make_provenance(source_label: str, agent: str, issue: str = "", ) +def _has_constraints(slot: dict) -> bool: + """Does this slot carry any usage constraint worth recording as a Rule?""" + return bool( + slot["required"] or slot["multivalued"] or slot["pattern"] + or slot.get("minimum_value") is not None + or slot.get("maximum_value") is not None + ) + + +def _describe_constraints(slot: dict) -> str: + """Human-readable summary of a slot's usage constraints, for Rule.description.""" + parts = [] + if slot["required"]: + parts.append("required") + if slot["multivalued"]: + parts.append("multivalued") + if slot["pattern"]: + parts.append(f"pattern: {slot['pattern']}") + if slot.get("minimum_value") is not None: + parts.append(f"min: {slot['minimum_value']}") + if slot.get("maximum_value") is not None: + parts.append(f"max: {slot['maximum_value']}") + return "; ".join(parts) + + def build_registry_entities( parsed: dict, source_label: str, agent: str, issue: str = "", registry_version: str = "", -) -> tuple[dict[str, RegistryProperty], dict[str, RegistryClass]]: +) -> tuple[dict[str, RegistryProperty], dict[str, RegistryClass], dict[str, Rule]]: """ Convert parse_linkml()'s intermediate dict into content-hashed - RegistryProperty/RegistryClass instances, keyed by their original + RegistryProperty/RegistryClass/Rule instances, keyed by their original slot/class name in the source schema. Properties are built first (classes reference them by hash_id in their @@ -312,6 +342,12 @@ def build_registry_entities( parse_linkml() (via SchemaView) already requires every is_a target to resolve within the submitted schema's own import closure, so `classes` is guaranteed to contain it. + + A property that declares required/multivalued/pattern/min/max also gets + a Rule referencing it (`applies_to` = the property's hash_id) — these + usage constraints don't live on RegistryProperty itself (see + RegistryProperty's docstring), and a plain property with none of them + gets no Rule at all, so unconstrained properties don't accumulate noise. """ slots = parsed["slots"] classes = parsed["classes"] @@ -321,6 +357,7 @@ def build_registry_entities( used_slots.update(cls["slots"]) properties: dict[str, RegistryProperty] = {} + rules: dict[str, Rule] = {} for slot_name in used_slots: slot = slots.get(slot_name) if not slot: @@ -336,6 +373,21 @@ def build_registry_entities( prop.hash_id = compute_hash_id(prop) properties[slot_name] = prop + if _has_constraints(slot): + rule = Rule( + name=f"{slot_name}_rule", + description=_describe_constraints(slot), + applies_to=prop.hash_id, + required=slot["required"], + multivalued=slot["multivalued"], + pattern=slot["pattern"] or None, + minimum_value=slot.get("minimum_value"), + maximum_value=slot.get("maximum_value"), + provenance=[_make_provenance(source_label, agent, issue, registry_version)], + ) + rule.hash_id = compute_hash_id(rule) + rules[slot_name] = rule + registry_classes: dict[str, RegistryClass] = {} def resolve_class(cls_name: str) -> RegistryClass | None: @@ -369,7 +421,7 @@ def resolve_class(cls_name: str) -> RegistryClass | None: for cls_name in classes: resolve_class(cls_name) - return properties, registry_classes + return properties, registry_classes, rules # --------------------------------------------------------------------------- @@ -448,32 +500,34 @@ def insert_schema(conn, parsed: dict, source_label: str, agent: str = "anonymous """ Insert a parsed LinkML schema into the LadybugDB graph. - 1. Build content-hashed RegistryProperty/RegistryClass instances + 1. Build content-hashed RegistryProperty/RegistryClass/Rule instances 2. Write each one (skipped if its hash_id already exists) and attach this ingestion's ProvenanceEntry (skipped if this source already attested to it) - 3. Create HAS_PROPERTY and SUBCLASS_OF edges - 4. Record a SchemaVersionSnapshot — "minor" if any class/property was - newly created, "patch" if only a new ProvenanceEntry was added + 3. Create HAS_PROPERTY, SUBCLASS_OF, and APPLIES_TO_P edges + 4. Record a SchemaVersionSnapshot — "minor" if any class/property/rule + was newly created, "patch" if only a new ProvenanceEntry was added (same content, newly attested by this source), unchanged otherwise Returns a stats dict. """ meta = parsed["meta"] - properties, registry_classes = build_registry_entities( + properties, registry_classes, rules = build_registry_entities( parsed, source_label, agent, issue, registry_version, ) - stats = write_registry_entities(conn, properties, registry_classes, dry_run=dry_run) + stats = write_registry_entities(conn, properties, registry_classes, rules, dry_run=dry_run) if dry_run: return stats _ensure_schema_source(conn, source_label, meta["version"], registry_version) - stats["rels"] = write_structural_edges(conn, registry_classes) + stats["rels"] = write_structural_edges(conn, registry_classes) + write_rule_edges(conn, rules) - has_new_content = bool(stats["classes_new"] or stats["properties_new"]) + has_new_content = bool( + stats["classes_new"] or stats["properties_new"] or stats["rules_new"] + ) has_any_change = has_new_content or bool(stats["provenance_added"]) prev_ver = _prev_schema_version(conn, source_label) @@ -489,6 +543,7 @@ def insert_schema(conn, parsed: dict, source_label: str, agent: str = "anonymous changes_summary = ( f"+{stats['classes_new']} classes, +{stats['properties_new']} props, " + f"+{stats['rules_new']} rules, " f"{stats['provenance_added']} provenance entries added" ) @@ -512,7 +567,7 @@ def insert_schema(conn, parsed: dict, source_label: str, agent: str = "anonymous "yp": yml_path, "cc": len(registry_classes), "pc": len(properties), - "rc": 0, + "rc": len(rules), "cs": changes_summary, "rv": registry_version, }) @@ -586,6 +641,10 @@ def cli(file, db, dry_run, wipe, registry_version, issue, agent) -> None: MATCH (:RegistryProperty)-[:HAS_PROVENANCE_P]->(pe:ProvenanceEntry {source: $src}) DETACH DELETE pe """, {"src": source_label}) + conn.execute(""" + MATCH (:Rule)-[:HAS_PROVENANCE_RULE]->(pe:ProvenanceEntry {source: $src}) + DETACH DELETE pe + """, {"src": source_label}) stats = insert_schema( conn, parsed, source_label, agent=agent, issue=issue, @@ -601,6 +660,8 @@ def cli(file, db, dry_run, wipe, registry_version, issue, agent) -> None: f"={stats.get('classes_existing',0)} existing | " f"+{stats.get('properties_new',0)} props, " f"={stats.get('properties_existing',0)} existing | " + f"+{stats.get('rules_new',0)} rules, " + f"={stats.get('rules_existing',0)} existing | " f"+{stats.get('provenance_added',0)} provenance entries" ) if stats.get("schema_version"): @@ -611,8 +672,9 @@ def cli(file, db, dry_run, wipe, registry_version, issue, agent) -> None: if not dry_run: nc = conn.execute("MATCH (n:RegistryClass) RETURN count(n)").get_next()[0] np = conn.execute("MATCH (n:RegistryProperty) RETURN count(n)").get_next()[0] + nr = conn.execute("MATCH (n:Rule) RETURN count(n)").get_next()[0] npe = conn.execute("MATCH (n:ProvenanceEntry) RETURN count(n)").get_next()[0] - click.echo(f" Registry: {nc} classes, {np} properties, {npe} provenance entries") + click.echo(f" Registry: {nc} classes, {np} properties, {nr} rules, {npe} provenance entries") if __name__ == "__main__": diff --git a/schema_registry_utils/__init__.py b/schema_registry_utils/__init__.py index cf5e333..b8e744a 100644 --- a/schema_registry_utils/__init__.py +++ b/schema_registry_utils/__init__.py @@ -5,6 +5,7 @@ ProvenanceEntry, SkosMapping, Relation, + Rule, SkosMappingTypeEnum, ) from schema_registry_utils.hashing import compute_hash_id, assign_hash_id @@ -16,6 +17,7 @@ "ProvenanceEntry", "SkosMapping", "Relation", + "Rule", "SkosMappingTypeEnum", "compute_hash_id", "assign_hash_id", diff --git a/schema_registry_utils/hashing.py b/schema_registry_utils/hashing.py index 6154cce..0f83b09 100644 --- a/schema_registry_utils/hashing.py +++ b/schema_registry_utils/hashing.py @@ -1,7 +1,7 @@ import hashlib import json -from schema_registry_utils.models import RegistryClass, RegistryProperty +from schema_registry_utils.models import RegistryClass, RegistryProperty, Rule _EXCLUDED_FIELDS = { "hash_id", "provenance", "skos_mappings", @@ -11,11 +11,14 @@ } -def compute_hash_id(entity: RegistryClass | RegistryProperty) -> str: - """Compute a content-based hash_id for a RegistryClass or RegistryProperty. +def compute_hash_id(entity: RegistryClass | RegistryProperty | Rule) -> str: + """Compute a content-based hash_id for a RegistryClass, RegistryProperty, + or Rule. Everything but hash_id, provenance, skos_mappings, and class_uri/slot_uri - is treated as identity-defining content. + is treated as identity-defining content. Rule has neither class_uri nor + slot_uri, so all of its own fields (including applies_to) are hashed — + a rule's identity is exactly "these constraints, on this property". """ content = entity.model_dump(exclude=_EXCLUDED_FIELDS) canonical = json.dumps(_normalize(content), sort_keys=True, separators=(",", ":")) @@ -23,7 +26,7 @@ def compute_hash_id(entity: RegistryClass | RegistryProperty) -> str: return f"sha256:{digest}" -def assign_hash_id(entity: RegistryClass | RegistryProperty) -> RegistryClass | RegistryProperty: +def assign_hash_id(entity: RegistryClass | RegistryProperty | Rule) -> RegistryClass | RegistryProperty | Rule: """Compute entity's hash_id from its current content, then suffix its name with the first 4 hex characters of the digest (e.g. "age" -> "age_a1b2"). diff --git a/schema_registry_utils/models.py b/schema_registry_utils/models.py index 9d77ed1..dc1e825 100644 --- a/schema_registry_utils/models.py +++ b/schema_registry_utils/models.py @@ -64,3 +64,18 @@ class RegistryProperty(RegistryEntity): slot_uri: Optional[str] = None range: str units: Optional[str] = None + + +class Rule(RegistryEntity): + """ + A usage constraint on a specific RegistryProperty (applies_to holds its + hash_id) — required/multivalued/pattern/min/max. Not on RegistryProperty + itself: the same property can be required in one source's usage and + optional in another's without being a different concept. + """ + applies_to: str + required: Optional[bool] = None + multivalued: Optional[bool] = None + pattern: Optional[str] = None + minimum_value: Optional[float] = None + maximum_value: Optional[float] = None diff --git a/schemas/meta_model.yaml b/schemas/meta_model.yaml index 44a79f0..fd574a2 100644 --- a/schemas/meta_model.yaml +++ b/schemas/meta_model.yaml @@ -26,7 +26,8 @@ imports: # ProvenanceEntry is a standalone node (one per source attestation) — an # entity accumulates a ProvenanceEntry per source rather than being tied to # a single source, which is how identity stays separate from provenance. -# Rule, Transform, and ValueSet are stubs: present so the model shape is +# Rule is scoped and real (usage constraints on a RegistryProperty). +# Transform and ValueSet are still stubs: present so the model shape is # visible, slots intentionally left minimal until that work is scoped. classes: @@ -114,14 +115,24 @@ classes: - provenance Rule: + is_a: RegistryEntity description: >- - STUB — a validation or business rule applicable to one or more registry - entities (e.g. min/max value, pattern, required, multivalued constraints - on a RegistryProperty). Slots intentionally minimal; scope TBD. + A validation/usage constraint that applies to a specific + RegistryProperty (classes are a possible future extension) — + required, multivalued, pattern, and numeric bounds. These don't live + on RegistryProperty itself because they vary by usage: the same "age" + property can be required in one source's schema and optional in + another's, without being a different concept. A Rule is content- + addressed like any other entity — the same constraints on the same + property from two sources collapse to one Rule with two + ProvenanceEntry records, exactly like RegistryProperty does. slots: - - hash_id - - name - - description + - applies_to + - required + - multivalued + - pattern + - minimum_value + - maximum_value Transform: description: >- @@ -315,6 +326,37 @@ slots: range: uriorcurie description: The external concept this mapping points to. + # --- Rule slots ------------------------------------------------------------ + + applies_to: + range: RegistryProperty + required: true + description: >- + The RegistryProperty this rule constrains (stored as hash_id FK, same + pattern as is_a/parent_class). Part of the content hash — a rule is + defined by both its constraints and which property they apply to, so + identical constraints on two different properties are different rules. + + required: + range: boolean + description: Whether the property must be present. + + multivalued: + range: boolean + description: Whether the property can hold multiple values. + + pattern: + range: string + description: Regular expression the property's value must match. + + minimum_value: + range: float + description: Minimum permitted numeric value, if the property is numeric. + + maximum_value: + range: float + description: Maximum permitted numeric value, if the property is numeric. + # --------------------------------------------------------------------------- # Enums # --------------------------------------------------------------------------- diff --git a/tests/fixtures/required_c.yml b/tests/fixtures/required_c.yml new file mode 100644 index 0000000..fc56f10 --- /dev/null +++ b/tests/fixtures/required_c.yml @@ -0,0 +1,15 @@ +id: https://example.org/required-c +name: required_c +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types +default_range: string +classes: + Enrollee: + description: A person enrolled in the study + attributes: + age: + description: Age of the subject + range: integer + required: true diff --git a/tests/test_ingest_linkml.py b/tests/test_ingest_linkml.py index 188bf6c..a30f696 100644 --- a/tests/test_ingest_linkml.py +++ b/tests/test_ingest_linkml.py @@ -3,7 +3,7 @@ import pytest from ingest_linkml import parse_linkml, build_registry_entities -from schema_registry_utils import RegistryProperty, RegistryClass +from schema_registry_utils import RegistryProperty, RegistryClass, Rule FIXTURES = Path(__file__).parent / "fixtures" @@ -103,6 +103,8 @@ def test_parse_linkml_extracts_exactly_the_expected_dict(): "multivalued": False, "required": False, "pattern": "", + "minimum_value": None, + "maximum_value": None, }, "name": { "iri": "https://schema.org/name", @@ -112,6 +114,8 @@ def test_parse_linkml_extracts_exactly_the_expected_dict(): "multivalued": False, "required": False, "pattern": "", + "minimum_value": None, + "maximum_value": None, }, "orcid": { "iri": "https://example.org/schema#orcid", @@ -121,6 +125,8 @@ def test_parse_linkml_extracts_exactly_the_expected_dict(): "multivalued": False, "required": False, "pattern": r"^\d{4}-\d{4}-\d{4}-\d{3}[0-9X]$", + "minimum_value": None, + "maximum_value": None, }, "role": { "iri": "", @@ -130,6 +136,8 @@ def test_parse_linkml_extracts_exactly_the_expected_dict(): "multivalued": True, "required": True, "pattern": "^[A-Za-z ]+$", + "minimum_value": None, + "maximum_value": None, }, }, } @@ -138,35 +146,43 @@ def test_parse_linkml_extracts_exactly_the_expected_dict(): def test_registry_property_does_not_retain_usage_constraints(): """ parse_linkml()'s dict has multivalued/required/pattern (see above) — but - RegistryProperty deliberately doesn't model them at all (deferred to a - future Rule, since the same property can be required in one source's - usage and optional in another's without being a different concept). - Assert this at the model level, not just "the dict I built doesn't have - it" — if someone re-adds these fields to RegistryProperty, this fails. + RegistryProperty deliberately doesn't model them at all (deferred to + Rule, since the same property can be required in one source's usage + and optional in another's without being a different concept). Assert + this at the model level, not just "the dict I built doesn't have it" — + if someone re-adds these fields to RegistryProperty, this fails. Also + confirm they didn't just vanish — they moved to Rule. """ - for field in ("required", "multivalued", "pattern"): + for field in ("required", "multivalued", "pattern", "minimum_value", "maximum_value"): assert field not in RegistryProperty.model_fields + assert field in Rule.model_fields def test_build_registry_entities_produces_exactly_the_expected_objects(): """ Exact-equality check of build_registry_entities()'s output — the step that turns parse_linkml()'s dict into content-hashed RegistryProperty/ - RegistryClass instances. hash_id is a pure content hash (no randomness), - so these values are reproducible; if the hash computation, the set of - fields carried into the model, or the is_a/properties resolution ever - changes, this fails. + RegistryClass/Rule instances. hash_id is a pure content hash (no + randomness), so these values are reproducible; if the hash computation, + the set of fields carried into the model, or the is_a/properties/ + applies_to resolution ever changes, this fails. + + Only "orcid" (pattern) and "role" (required, multivalued, pattern) carry + a constraint in this fixture — "name" and "created_at" have none, so they + get no Rule at all (see test_ingest_registry.py for the "no rule created + for an unconstrained property" case, checked directly against the graph). provenance is checked separately (excluded from the equality dump) since ProvenanceEntry.uid/generated_at are non-deterministic per run. """ parsed = parse_linkml(FIXTURES / "comprehensive.yml") - properties, registry_classes = build_registry_entities(parsed, "comprehensive", "tester") + properties, registry_classes, rules = build_registry_entities(parsed, "comprehensive", "tester") assert set(properties) == {"name", "orcid", "role", "created_at"} assert set(registry_classes) == {"Timestamped", "Entity", "Person"} + assert set(rules) == {"orcid", "role"} - for entity in (*properties.values(), *registry_classes.values()): + for entity in (*properties.values(), *registry_classes.values(), *rules.values()): assert len(entity.provenance) == 1 prov = entity.provenance[0] assert prov.source == "comprehensive" @@ -262,3 +278,33 @@ def test_build_registry_entities_produces_exactly_the_expected_objects(): "mixins": [], }, } + + assert { + name: r.model_dump(exclude={"provenance"}) + for name, r in rules.items() + } == { + "orcid": { + "hash_id": "sha256:1aca93350de8d892498de3703d37a1cbea735ba7db1fa48f8a12fb03d4ae2f99", + "name": "orcid_rule", + "description": r"pattern: ^\d{4}-\d{4}-\d{4}-\d{3}[0-9X]$", + "skos_mappings": [], + "applies_to": "sha256:8bddfe8f326dab2077cd95446fa63eb7708cab8c096adc2ad53979d91b73862d", + "required": False, + "multivalued": False, + "pattern": r"^\d{4}-\d{4}-\d{4}-\d{3}[0-9X]$", + "minimum_value": None, + "maximum_value": None, + }, + "role": { + "hash_id": "sha256:a0640da17823cb046195295b7bf89c79e4dd821a26b3669aa6335a14b51933e1", + "name": "role_rule", + "description": "required; multivalued; pattern: ^[A-Za-z ]+$", + "skos_mappings": [], + "applies_to": "sha256:b401d00ccb63e9accb3a1e2360a2f5d6997c21ae205324cf58ddd72712ce1538", + "required": True, + "multivalued": True, + "pattern": "^[A-Za-z ]+$", + "minimum_value": None, + "maximum_value": None, + }, + } diff --git a/tests/test_ingest_registry.py b/tests/test_ingest_registry.py index 361544d..0f7548c 100644 --- a/tests/test_ingest_registry.py +++ b/tests/test_ingest_registry.py @@ -64,9 +64,14 @@ def test_required_does_not_affect_property_identity(tmp_path): required_a.yml and required_b.yml declare the exact same "age" slot (same name/description/range/units) except one marks it `required: true` and the other doesn't. RegistryProperty doesn't model required at all - (deferred to a future Rule — see test_registry_property_does_not_retain_ + (deferred to Rule — see test_registry_property_does_not_retain_ usage_constraints in test_ingest_linkml.py), so this must not create a second node: same hash_id, one node, provenance from both sources. + + required_a's "required: true" doesn't just vanish, though — it produces + a Rule (APPLIES_TO_P the "age" property), while required_b's plain + declaration produces no Rule at all — the constraint is real, it's just + modeled on a separate entity instead of on the property. """ conn = _conn(tmp_path) @@ -74,8 +79,10 @@ def test_required_does_not_affect_property_identity(tmp_path): stats_b = insert_schema(conn, parse_linkml(FIXTURES / "required_b.yml"), "required_b", agent="tester") assert stats_a["properties_new"] == 1 - assert stats_b["properties_new"] == 0 # not a new node — same hash as required_a's + assert stats_a["rules_new"] == 1 # required: true -> one Rule + assert stats_b["properties_new"] == 0 # not a new node — same hash as required_a's assert stats_b["properties_existing"] == 1 + assert stats_b["rules_new"] == 0 # no constraints declared -> no Rule rows = conn.execute("MATCH (p:RegistryProperty {name: 'age'}) RETURN p.hash_id").get_all() assert len(rows) == 1 # no duplicate node @@ -86,6 +93,37 @@ def test_required_does_not_affect_property_identity(tmp_path): """).get_all() assert {r[0] for r in sources} == {"required_a", "required_b"} + rule_rows = conn.execute("MATCH (r:Rule) RETURN r.required, r.applies_to").get_all() + assert len(rule_rows) == 1 # exactly one Rule total, from required_a only + assert rule_rows[0][0] is True + assert rule_rows[0][1] == rows[0][0] # applies_to the "age" property's hash_id + + +def test_identical_rule_from_two_sources_shares_one_hash_id(tmp_path): + """ + required_a.yml and required_c.yml both mark "age" required: true (under + different class names, in different files) — the resulting Rule is + content-addressed exactly like RegistryProperty/RegistryClass: one node, + provenance from both sources, not two separate Rules. + """ + conn = _conn(tmp_path) + + stats_a = insert_schema(conn, parse_linkml(FIXTURES / "required_a.yml"), "required_a", agent="tester") + stats_c = insert_schema(conn, parse_linkml(FIXTURES / "required_c.yml"), "required_c", agent="tester") + + assert stats_a["rules_new"] == 1 + assert stats_c["rules_new"] == 0 # same constraint, same property -> same hash_id + assert stats_c["rules_existing"] == 1 + + rows = conn.execute("MATCH (r:Rule) RETURN r.hash_id").get_all() + assert len(rows) == 1 # no duplicate Rule + + sources = conn.execute(""" + MATCH (:Rule)-[:HAS_PROVENANCE_RULE]->(pe:ProvenanceEntry) + RETURN pe.source + """).get_all() + assert {r[0] for r in sources} == {"required_a", "required_c"} + def test_content_change_produces_different_hash_id(tmp_path): conn = _conn(tmp_path)