fix: distinguish 497K fee tables from footnotes

edgar/funds/_497k_tables.py

@@ -177,10 +177,28 @@ def _label_matches(text: str, *patterns: str) -> bool:
 # Table classification
 # ---------------------------------------------------------------------------
 
-_FEE_TABLE_LABELS = (
-    'management fee',
-    'management fees',
+_MANAGEMENT_FEE_LABEL_RE = re.compile(r'\bmanagement fees?\b')
+_ONE_YEAR_CELL_RE = re.compile(r'^(?:past\s+)?1\s*years?$')
+_FLATTENED_FEE_VALUES_RE = re.compile(
+    r'^(?:\([^)]*\)\s*)?(?:[-+]?\(?\d*\.?\d+\)?\s*%\s*){2}'
 )
+_FEE_VALUE_FOOTNOTE_RE = re.compile(r'(?:\s*\(\s*[\d*†‡§]+\s*\)|\s*[*†‡§]+)\s*$')
+_PERCENT_VALUE_SHAPE_RE = re.compile(
+    r'''
+    ^\s*\(?\s*[-+]?\s*
+    (?:\d+(?:\.\s*\d*)?|\.\s*\d+)
+    \s*\)?\s*%\s*\)?\s*
+    (?:
+        [\d*†‡§\s]+
+        | \(\s*[\d*†‡§\s]+\s*\)
+        | [A-Za-z](?:\s*,\s*[A-Za-z])*
+        | \(\s*[A-Za-z](?:\s*,\s*[A-Za-z])*\s*\)
+    )?
+    \s*$
+    ''',
+    re.VERBOSE,
+)
+_MISSING_FEE_VALUES = {'none', 'n/a', 'na', 'not applicable', '-', '--', '–', '—'}
 
 _EXPENSE_EXAMPLE_LABELS = (
     '1 year',
@@ -211,6 +229,80 @@ def _label_matches(text: str, *patterns: str) -> bool:
 )
 
 
+def _align_indented_fee_row(row: List[str]) -> List[str]:
+    """Remove empty indentation cells before a fee label, preserving headers."""
+    for index, cell in enumerate(row):
+        if cell.strip():
+            return row[index:] if index and _is_fee_label(cell) else row
+    return row
+
+
+def _align_indented_fee_table(rows: List[List[str]]) -> List[List[str]]:
+    """Remove the shared indentation offset without shifting class columns."""
+    label_columns = [
+        index
+        for row in rows
+        for index, cell in enumerate(row)
+        if _is_fee_label(cell)
+    ]
+    offset = min(label_columns, default=0)
+    return [row[offset:] for row in rows] if offset else rows
+
+
+def _is_fee_value(cell: str) -> bool:
+    """Return whether a cell has a fee value or a marked missing value."""
+    normalized = _FEE_VALUE_FOOTNOTE_RE.sub('', _normalize(cell))
+    if normalized in _MISSING_FEE_VALUES:
+        return True
+    if '%' in normalized:
+        return _PERCENT_VALUE_SHAPE_RE.fullmatch(cell.replace('\xa0', ' ')) is not None
+    return _parse_percentage(cell) is not None
+
+
+def _is_management_fee_row(row: List[str]) -> bool:
+    """Return whether a row has management-fee label/value structure."""
+    if not row:
+        return False
+
+    row = _align_indented_fee_row(row)
+    label = _normalize(row[0])
+    match = _MANAGEMENT_FEE_LABEL_RE.search(label)
+    if match is None:
+        return False
+    if len(row) > 1 and any(cell.strip() for cell in row[1:]):
+        return any(_is_fee_value(cell) for cell in row[1:])
+
+    # Some malformed nested tables flatten all fee rows into one cell. Keep
+    # those only when the label is immediately followed by two percentage values.
+    trailing = label[match.end():].lstrip()
+    return match.start() == 0 and _FLATTENED_FEE_VALUES_RE.match(trailing) is not None
+
+
+def _is_shareholder_fee_row(row: List[str]) -> bool:
+    """Return whether a row has a shareholder-fee label and a separate value."""
+    row = next((row[index:] for index, cell in enumerate(row) if cell.strip()), row)
+    return (
+        len(row) > 1
+        and any(label in _normalize(row[0]) for label in _SHAREHOLDER_FEE_LABELS)
+        and any(_is_fee_value(cell) for cell in row[1:])
+    )
+
+
+def _has_one_year_column(rows: List[List[str]]) -> bool:
+    """Return whether a table has a standalone one-year column label."""
+    return any(
+        _ONE_YEAR_CELL_RE.fullmatch(_normalize(cell)) is not None
+        for row in rows
+        for cell in row
+    )
+
+
+def _has_bar_chart_columns(rows: List[List[str]]) -> bool:
+    """Return whether a table has at least three standalone annual labels."""
+    cells = {_normalize(cell) for row in rows for cell in row}
+    return sum(year in cells for year in _BAR_CHART_LABELS) >= 3
+
+
 def _classify_table(rows: List[List[str]]) -> Optional[str]:
     """Classify a table by its content. Returns a type string or None."""
     all_text = ' '.join(' '.join(row) for row in rows)
@@ -225,10 +317,22 @@ def _classify_table(rows: List[List[str]]) -> Optional[str]:
             ('quarter' in norm or 'return' in norm)):
         return 'quarter'
 
-    # Check for operating expenses table (has "management fee")
-    if any(p in norm for p in _FEE_TABLE_LABELS):
+    # Match the same label/value structure consumed by the fee extractor so
+    # prose-only footnote tables do not change the extraction strategy.
+    if any(_is_management_fee_row(row) for row in rows):
         return 'operating_expenses'
 
+    # Management-fee prose belongs to footnotes, not another table class. Stop
+    # here so words such as "1 Year" or "sales charge" cannot displace the real
+    # expense-example or shareholder-fee table selected later.
+    if (
+        _MANAGEMENT_FEE_LABEL_RE.search(norm)
+        and not any(_is_shareholder_fee_row(row) for row in rows)
+        and not (_has_one_year_column(rows) and ('$' in all_text or '%' in all_text))
+        and not _has_bar_chart_columns(rows)
+    ):
+        return None
+
     # Tables with year-period columns (1 year, 3 years, etc.)
     # Distinguish expense example ($) from performance (%)
     # Use regex word boundary to avoid 'past 1' matching 'past 10'
@@ -278,6 +382,8 @@ def _extract_operating_expenses(rows: List[List[str]]) -> List[Dict]:
     if not rows:
         return []
 
+    rows = _align_indented_fee_table(rows)
+
     # Detect if first row is a header (class names) or data (fee labels)
     first_row = rows[0]
     has_header = not _is_fee_label(first_row[0]) if first_row else False
@@ -359,11 +465,25 @@ def _is_shareholder_fee_label(text: str) -> bool:
     return any(kw in norm for kw in keywords)
 
 
+def _align_indented_shareholder_fee_table(rows: List[List[str]]) -> List[List[str]]:
+    """Remove the shared indentation offset before shareholder-fee labels."""
+    label_columns = [
+        index
+        for row in rows
+        for index, cell in enumerate(row)
+        if _is_shareholder_fee_label(cell)
+    ]
+    offset = min(label_columns, default=0)
+    return [row[offset:] for row in rows] if offset else rows
+
+
 def _extract_shareholder_fees(rows: List[List[str]]) -> List[Dict]:
     """Extract shareholder fees (sales loads, redemption fees)."""
     if not rows:
         return []
 
+    rows = _align_indented_shareholder_fee_table(rows)
+
     # Detect if first row is a header or data
     first_row = rows[0]
     has_header = first_row and not _is_shareholder_fee_label(first_row[0])

tests/fixtures/prospectus_497k_baseline.json

@@ -37,13 +37,6 @@
         "other_expenses": null,
         "total_annual_expenses": null,
         "twelve_b1_fee": null
-      },
-      {
-        "class_name": "The Investment Adviser has agreed to waive a portion of the management fee through March\u00a031, 2013. In addition, the Investment Adviser has agreed to reimburse the\nFund for certain Fund operating expenses such that total annual Fund operating expenses (exclusive of any front-end load, deferred sales charge, 12b-1 fees, taxes, income tax expense, brokerage commissions, expenses incurred in connection with any\nmerger or reorganization, acquired fund fees and expenses, or extraordinary expenses such as litigation) will not exceed 1.40% for each of Class\u00a0A Shares, Class C Shares and Class I Shares, subject to possible recoupment from the Fund in future\nyears on a rolling three year basis (within the three years after the fees have been waived and expenses reimbursed) if such recoupment can be achieved within the foregoing expense limits. Such waiver or reimbursement may not be terminated without\nthe consent of the Board of Trustees prior to March 31, 2013 and may be modified or terminated by the Investment Adviser at any time after March 31, 2013.",
-        "expense_10yr": 2119,
-        "expense_1yr": null,
-        "expense_3yr": null,
-        "expense_5yr": null
       }
     ],
     "metadata": {
@@ -185,10 +178,7 @@
     "fee_tables": [
       {
         "class_name": "",
-        "expense_10yr": null,
         "expense_1yr": null,
-        "expense_3yr": null,
-        "expense_5yr": null,
         "fee_waiver": null,
         "management_fee": null,
         "max_deferred_sales_load": null,
@@ -197,13 +187,6 @@
         "other_expenses": null,
         "total_annual_expenses": null,
         "twelve_b1_fee": null
-      },
-      {
-        "class_name": "The Adviser has contractually agreed to waive its management fees and/or to bear expenses of the Fund through January\u00a031, 2018 to the extent necessary to prevent total Fund\noperating expenses (excluding acquired fund fees and expenses other than the advisory fees of any AB Mutual Funds in which the Fund may invest, interest expense, taxes, extraordinary expenses, and brokerage commissions and other transaction costs),\non an annualized basis, from exceeding .95%, 1.70%, .70%, 1.20%, .95%, .70% and .70% of average daily net assets, respectively, for Class\u00a0A, Class C, Advisor Class, Class R, Class K, Class I and Class Z shares (\u201cexpense limitations\u201d).\nAny fees waived and expenses borne by the Adviser prior to July\u00a015, 2015 may be reimbursed by the Fund until the end of the third fiscal year after the fiscal period in which the fee was waived or the expense was borne, provided that no\nreimbursement payment will be made that would cause the Fund\u2019s Total Annual Fund Operating Expenses to exceed the expense limitations.",
-        "expense_10yr": null,
-        "expense_1yr": null,
-        "expense_3yr": null,
-        "expense_5yr": null
       }
     ],
     "metadata": {
@@ -360,13 +343,6 @@
         "other_expenses": null,
         "total_annual_expenses": null,
         "twelve_b1_fee": null
-      },
-      {
-        "class_name": "\u201cManagement Fees\u201d and \u201cOther Expenses\u201d have been restated to reflect current fees.",
-        "expense_10yr": null,
-        "expense_1yr": null,
-        "expense_3yr": 593,
-        "expense_5yr": null
       }
     ],
     "metadata": {