refactor: decompose helpers.py into domain-focused modules

eyecite/citation_filter.py

@@ -0,0 +1,117 @@
+import logging
+
+from eyecite.models import (
+    CitationBase,
+    CitationToken,
+    FullCaseCitation,
+    ReferenceCitation,
+    ResourceCitation,
+    ShortCaseCitation,
+    SupraCitation,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def disambiguate_reporters(
+    citations: list[CitationBase],
+) -> list[CitationBase]:
+    """Filter out citations where there is more than one possible reporter."""
+    return [
+        c
+        for c in citations
+        if not isinstance(c, ResourceCitation) or c.edition_guess
+    ]
+
+
+def overlapping_citations(
+    full_span_1: tuple[int, int], full_span_2: tuple[int, int]
+) -> bool:
+    """Check if citations overlap at all"""
+    start_1, end_1 = full_span_1
+    start_2, end_2 = full_span_2
+    return max(start_1, start_2) < min(end_1, end_2)
+
+
+def filter_citations(citations: list[CitationBase]) -> list[CitationBase]:
+    """Filter and order citations, ensuring reference citations are in sequence
+
+    This function resolves rare but possible overlaps between ref. citations
+    and short citations. It also orders all citations by their `citation.span`,
+    as reference citations may be extracted out of order. The final result is a
+    properly sorted list of citations as they appear in the text
+
+    :param citations: List of citations
+    :return: Sorted and filtered citations
+    """
+    if not citations:
+        return citations
+
+    citations = list(
+        {citation.span(): citation for citation in citations}.values()
+    )
+    sorted_citations = sorted(
+        citations, key=lambda citation: citation.full_span()
+    )
+    filtered_citations: list[CitationBase] = [sorted_citations[0]]
+
+    for citation in sorted_citations[1:]:
+        last_citation = filtered_citations[-1]
+        is_overlapping = overlapping_citations(
+            citation.full_span(), last_citation.full_span()
+        )
+        if is_overlapping:
+            # In cases overlap, prefer anything to a reference citation
+            if isinstance(last_citation, ReferenceCitation):
+                filtered_citations.pop(-1)
+                filtered_citations.append(citation)
+                continue
+            if isinstance(citation, ReferenceCitation):
+                continue
+
+            if isinstance(citation, SupraCitation) and isinstance(
+                last_citation, ShortCaseCitation
+            ):
+                continue
+
+            # A citation in a paren would also overlap and should be kept.
+            paren = last_citation.metadata.parenthetical
+            if paren and citation.matched_text() in paren:
+                filtered_citations.append(citation)
+                continue
+
+            # Known overlap case are parallel full citations
+            if not (
+                isinstance(citation, FullCaseCitation)
+                and isinstance(last_citation, FullCaseCitation)
+            ):
+                logger.warning(
+                    "Unknown overlap case. Last cite: %s. Current: %s",
+                    last_citation,
+                    citation,
+                )
+
+        filtered_citations.append(citation)
+
+    return filtered_citations
+
+
+joke_cite: list[CitationBase] = [
+    FullCaseCitation(
+        CitationToken(
+            "1 FLP 1",
+            0,
+            99,
+            {
+                "volume": "1",
+                "reporter": "FLP",
+                "page": "1",
+            },
+        ),
+        0,
+        metadata={
+            "year": "2021",
+            "extra": "Eyecite is a collaborative community effort.",
+        },
+    )
+]

eyecite/citation_metadata.py

@@ -0,0 +1,177 @@
+from datetime import date
+
+import regex as re
+
+from eyecite.models import (
+    CaseCitation,
+    FullCaseCitation,
+    FullJournalCitation,
+    FullLawCitation,
+)
+from eyecite.regexes import (
+    POST_FULL_CITATION_REGEX,
+    POST_JOURNAL_CITATION_REGEX,
+    POST_LAW_CITATION_REGEX,
+    PRE_FULL_CITATION_REGEX,
+    YEAR_REGEX,
+)
+from eyecite.court_matching import get_court_by_paren
+from eyecite.token_matching import match_on_tokens
+
+
+# Highest valid year is this year + 1 because courts in December sometimes
+# cite a case to be published in January.
+_highest_valid_year = date.today().year + 1
+
+
+def get_year(word: str) -> int | None:
+    """Given a matched year string, look for a year within a reasonable
+    range."""
+    try:
+        year = int(word)
+    except ValueError:
+        return None
+
+    if year < 1600 or year > _highest_valid_year:
+        return None
+    return year
+
+
+def add_post_citation(citation: CaseCitation, words) -> None:
+    """Add to a citation object any additional information found after the base
+    citation, including court, year, and possibly page range.
+
+    See POST_CITATION_REGEX for examples.
+    """
+    m = match_on_tokens(
+        words,
+        citation.index + 1,
+        POST_FULL_CITATION_REGEX,
+    )
+    if not m:
+        return
+
+    citation.full_span_end = citation.span()[1] + m.end()
+    citation.metadata.pin_cite = clean_pin_cite(m["pin_cite"]) or None
+    if m["pin_cite"]:
+        citation.metadata.pin_cite_span_end = citation.span()[1] + len(
+            m["pin_cite"]
+        )
+
+    citation.metadata.extra = (m["extra"] or "").strip() or None
+    citation.metadata.parenthetical = process_parenthetical(m["parenthetical"])
+
+    if (
+        citation.full_span_end
+        and m["parenthetical"] is not None
+        and isinstance(citation.metadata.parenthetical, str)
+    ) and len(m["parenthetical"]) > len(citation.metadata.parenthetical):
+        offset = len(m["parenthetical"]) - len(citation.metadata.parenthetical)
+        citation.full_span_end = citation.full_span_end - offset
+    citation.metadata.year = m["year"]
+    citation.metadata.month = m["month"]
+    citation.metadata.day = m["day"]
+    if m["year"]:
+        citation.year = get_year(m["year"])
+    if m["court"]:
+        citation.metadata.court = get_court_by_paren(m["court"])
+
+
+def add_pre_citation(citation: FullCaseCitation, document) -> None:
+    """Scan backwards to find a (PartyName - Pincite) component
+
+    Do not try if plaintiff or defendant has already been found
+    """
+    if citation.metadata.plaintiff or citation.metadata.defendant:
+        return
+
+    m = match_on_tokens(
+        document.words,
+        citation.index - 1,
+        PRE_FULL_CITATION_REGEX,
+        forward=False,
+        strings_only=True,
+    )
+    if not m:
+        return
+
+    if m["pin_cite"]:
+        # if a pin cite occurs before the citation mark it down
+        start, end = m.span()
+        citation.metadata.pin_cite_span_start = citation.span()[0] - (
+            end - start
+        )
+
+    citation.metadata.pin_cite = clean_pin_cite(m["pin_cite"]) or None
+    citation.metadata.antecedent_guess = m["antecedent"]
+    match_length = m.span()[1] - m.span()[0]
+    citation.full_span_start = citation.span()[0] - match_length
+
+
+def add_law_metadata(citation: FullLawCitation, words) -> None:
+    """Annotate FullLawCitation with pin_cite, publisher, etc."""
+    m = match_on_tokens(
+        words, citation.index + 1, POST_LAW_CITATION_REGEX, strings_only=True
+    )
+    if not m:
+        return
+
+    citation.full_span_end = citation.span()[1] + m.end()
+    citation.metadata.pin_cite = clean_pin_cite(m["pin_cite"]) or None
+    citation.metadata.publisher = m["publisher"]
+    citation.metadata.day = m["day"]
+    citation.metadata.month = m["month"]
+    citation.metadata.parenthetical = process_parenthetical(m["parenthetical"])
+    citation.metadata.year = m["year"]
+    if m["year"]:
+        citation.year = get_year(m["year"])
+
+
+def add_journal_metadata(citation: FullJournalCitation, words) -> None:
+    """Annotate FullJournalCitation with pin_cite, year, etc."""
+    m = match_on_tokens(
+        words,
+        citation.index + 1,
+        POST_JOURNAL_CITATION_REGEX,
+        strings_only=True,
+    )
+    if not m:
+        return
+
+    citation.full_span_end = citation.span()[1] + m.end()
+    citation.metadata.pin_cite = clean_pin_cite(m["pin_cite"]) or None
+    citation.metadata.parenthetical = process_parenthetical(m["parenthetical"])
+    citation.metadata.year = m["year"]
+    if m["year"]:
+        citation.year = get_year(m["year"])
+
+
+def clean_pin_cite(pin_cite: str | None) -> str | None:
+    """Strip spaces and commas from pin_cite, if it is not None."""
+    if pin_cite is None:
+        return pin_cite
+    return pin_cite.strip(", ")
+
+
+def process_parenthetical(
+    matched_parenthetical: str | None,
+) -> str | None:
+    """Exclude any additional paren matched as well as year parentheticals
+
+    For example: 'something) (something else)' will be trimmed down
+    to 'something' but 'something (clarifying something) or other' will be
+    kept in full.
+    """
+    if matched_parenthetical is None:
+        return matched_parenthetical
+    paren_balance = 0
+    for i, char in enumerate(matched_parenthetical):
+        if char == "(":  # Nested parenthetical
+            paren_balance += 1
+        elif char == ")":
+            paren_balance -= 1
+        if paren_balance < 0:  # End parenthetical reached
+            return matched_parenthetical[:i] or None
+    if re.match(YEAR_REGEX, matched_parenthetical, flags=re.X):
+        return None
+    return matched_parenthetical or None

eyecite/court_matching.py

@@ -0,0 +1,33 @@
+import regex as re
+from courts_db import courts
+
+
+def get_court_by_paren(paren_string: str) -> str | None:
+    """Takes the citation string, usually something like "2d Cir", and maps
+    that back to the court code.
+
+    Does not work on SCOTUS, since that court lacks parentheticals, and
+    needs to be handled after disambiguation has been completed.
+    """
+
+    # Remove whitespace and punctuation because citation strings sometimes lack
+    # internal spaces, e.g. "Pa.Super." or "SC" (South Carolina)
+    court_str = re.sub(r"[^\w]", "", paren_string).lower()
+
+    court_code = None
+    if court_str:
+        for court in courts:
+            s = re.sub(r"[^\w]", "", court["citation_string"]).lower()
+
+            # Check for an exact match first
+            if s == court_str:
+                return str(court["id"])
+
+            # If no exact match, try to record a startswith match for possible
+            # eventual return
+            if s.startswith(court_str):
+                court_code = court["id"]
+
+        return court_code
+
+    return court_code