diff --git a/edgar/funds/_497k_tables.py b/edgar/funds/_497k_tables.py index 1f08dbd88..a1125456c 100644 --- a/edgar/funds/_497k_tables.py +++ b/edgar/funds/_497k_tables.py @@ -177,10 +177,28 @@ def _label_matches(text: str, *patterns: str) -> bool: # Table classification # --------------------------------------------------------------------------- -_FEE_TABLE_LABELS = ( - 'management fee', - 'management fees', +_MANAGEMENT_FEE_LABEL_RE = re.compile(r'\bmanagement fees?\b') +_ONE_YEAR_CELL_RE = re.compile(r'^(?:past\s+)?1\s*years?$') +_FLATTENED_FEE_VALUES_RE = re.compile( + r'^(?:\([^)]*\)\s*)?(?:[-+]?\(?\d*\.?\d+\)?\s*%\s*){2}' ) +_FEE_VALUE_FOOTNOTE_RE = re.compile(r'(?:\s*\(\s*[\d*†‡§]+\s*\)|\s*[*†‡§]+)\s*$') +_PERCENT_VALUE_SHAPE_RE = re.compile( + r''' + ^\s*\(?\s*[-+]?\s* + (?:\d+(?:\.\s*\d*)?|\.\s*\d+) + \s*\)?\s*%\s*\)?\s* + (?: + [\d*†‡§\s]+ + | \(\s*[\d*†‡§\s]+\s*\) + | [A-Za-z](?:\s*,\s*[A-Za-z])* + | \(\s*[A-Za-z](?:\s*,\s*[A-Za-z])*\s*\) + )? + \s*$ + ''', + re.VERBOSE, +) +_MISSING_FEE_VALUES = {'none', 'n/a', 'na', 'not applicable', '-', '--', '–', '—'} _EXPENSE_EXAMPLE_LABELS = ( '1 year', @@ -211,6 +229,80 @@ def _label_matches(text: str, *patterns: str) -> bool: ) +def _align_indented_fee_row(row: List[str]) -> List[str]: + """Remove empty indentation cells before a fee label, preserving headers.""" + for index, cell in enumerate(row): + if cell.strip(): + return row[index:] if index and _is_fee_label(cell) else row + return row + + +def _align_indented_fee_table(rows: List[List[str]]) -> List[List[str]]: + """Remove the shared indentation offset without shifting class columns.""" + label_columns = [ + index + for row in rows + for index, cell in enumerate(row) + if _is_fee_label(cell) + ] + offset = min(label_columns, default=0) + return [row[offset:] for row in rows] if offset else rows + + +def _is_fee_value(cell: str) -> bool: + """Return whether a cell has a fee value or a marked missing value.""" + normalized = _FEE_VALUE_FOOTNOTE_RE.sub('', _normalize(cell)) + if normalized in _MISSING_FEE_VALUES: + return True + if '%' in normalized: + return _PERCENT_VALUE_SHAPE_RE.fullmatch(cell.replace('\xa0', ' ')) is not None + return _parse_percentage(cell) is not None + + +def _is_management_fee_row(row: List[str]) -> bool: + """Return whether a row has management-fee label/value structure.""" + if not row: + return False + + row = _align_indented_fee_row(row) + label = _normalize(row[0]) + match = _MANAGEMENT_FEE_LABEL_RE.search(label) + if match is None: + return False + if len(row) > 1 and any(cell.strip() for cell in row[1:]): + return any(_is_fee_value(cell) for cell in row[1:]) + + # Some malformed nested tables flatten all fee rows into one cell. Keep + # those only when the label is immediately followed by two percentage values. + trailing = label[match.end():].lstrip() + return match.start() == 0 and _FLATTENED_FEE_VALUES_RE.match(trailing) is not None + + +def _is_shareholder_fee_row(row: List[str]) -> bool: + """Return whether a row has a shareholder-fee label and a separate value.""" + row = next((row[index:] for index, cell in enumerate(row) if cell.strip()), row) + return ( + len(row) > 1 + and any(label in _normalize(row[0]) for label in _SHAREHOLDER_FEE_LABELS) + and any(_is_fee_value(cell) for cell in row[1:]) + ) + + +def _has_one_year_column(rows: List[List[str]]) -> bool: + """Return whether a table has a standalone one-year column label.""" + return any( + _ONE_YEAR_CELL_RE.fullmatch(_normalize(cell)) is not None + for row in rows + for cell in row + ) + + +def _has_bar_chart_columns(rows: List[List[str]]) -> bool: + """Return whether a table has at least three standalone annual labels.""" + cells = {_normalize(cell) for row in rows for cell in row} + return sum(year in cells for year in _BAR_CHART_LABELS) >= 3 + + def _classify_table(rows: List[List[str]]) -> Optional[str]: """Classify a table by its content. Returns a type string or None.""" all_text = ' '.join(' '.join(row) for row in rows) @@ -225,10 +317,22 @@ def _classify_table(rows: List[List[str]]) -> Optional[str]: ('quarter' in norm or 'return' in norm)): return 'quarter' - # Check for operating expenses table (has "management fee") - if any(p in norm for p in _FEE_TABLE_LABELS): + # Match the same label/value structure consumed by the fee extractor so + # prose-only footnote tables do not change the extraction strategy. + if any(_is_management_fee_row(row) for row in rows): return 'operating_expenses' + # Management-fee prose belongs to footnotes, not another table class. Stop + # here so words such as "1 Year" or "sales charge" cannot displace the real + # expense-example or shareholder-fee table selected later. + if ( + _MANAGEMENT_FEE_LABEL_RE.search(norm) + and not any(_is_shareholder_fee_row(row) for row in rows) + and not (_has_one_year_column(rows) and ('$' in all_text or '%' in all_text)) + and not _has_bar_chart_columns(rows) + ): + return None + # Tables with year-period columns (1 year, 3 years, etc.) # Distinguish expense example ($) from performance (%) # Use regex word boundary to avoid 'past 1' matching 'past 10' @@ -278,6 +382,8 @@ def _extract_operating_expenses(rows: List[List[str]]) -> List[Dict]: if not rows: return [] + rows = _align_indented_fee_table(rows) + # Detect if first row is a header (class names) or data (fee labels) first_row = rows[0] has_header = not _is_fee_label(first_row[0]) if first_row else False @@ -359,11 +465,25 @@ def _is_shareholder_fee_label(text: str) -> bool: return any(kw in norm for kw in keywords) +def _align_indented_shareholder_fee_table(rows: List[List[str]]) -> List[List[str]]: + """Remove the shared indentation offset before shareholder-fee labels.""" + label_columns = [ + index + for row in rows + for index, cell in enumerate(row) + if _is_shareholder_fee_label(cell) + ] + offset = min(label_columns, default=0) + return [row[offset:] for row in rows] if offset else rows + + def _extract_shareholder_fees(rows: List[List[str]]) -> List[Dict]: """Extract shareholder fees (sales loads, redemption fees).""" if not rows: return [] + rows = _align_indented_shareholder_fee_table(rows) + # Detect if first row is a header or data first_row = rows[0] has_header = first_row and not _is_shareholder_fee_label(first_row[0]) diff --git a/tests/fixtures/prospectus_497k_baseline.json b/tests/fixtures/prospectus_497k_baseline.json index c43229168..76761662b 100644 --- a/tests/fixtures/prospectus_497k_baseline.json +++ b/tests/fixtures/prospectus_497k_baseline.json @@ -37,13 +37,6 @@ "other_expenses": null, "total_annual_expenses": null, "twelve_b1_fee": null - }, - { - "class_name": "The Investment Adviser has agreed to waive a portion of the management fee through March\u00a031, 2013. In addition, the Investment Adviser has agreed to reimburse the\nFund for certain Fund operating expenses such that total annual Fund operating expenses (exclusive of any front-end load, deferred sales charge, 12b-1 fees, taxes, income tax expense, brokerage commissions, expenses incurred in connection with any\nmerger or reorganization, acquired fund fees and expenses, or extraordinary expenses such as litigation) will not exceed 1.40% for each of Class\u00a0A Shares, Class C Shares and Class I Shares, subject to possible recoupment from the Fund in future\nyears on a rolling three year basis (within the three years after the fees have been waived and expenses reimbursed) if such recoupment can be achieved within the foregoing expense limits. Such waiver or reimbursement may not be terminated without\nthe consent of the Board of Trustees prior to March 31, 2013 and may be modified or terminated by the Investment Adviser at any time after March 31, 2013.", - "expense_10yr": 2119, - "expense_1yr": null, - "expense_3yr": null, - "expense_5yr": null } ], "metadata": { @@ -185,10 +178,7 @@ "fee_tables": [ { "class_name": "", - "expense_10yr": null, "expense_1yr": null, - "expense_3yr": null, - "expense_5yr": null, "fee_waiver": null, "management_fee": null, "max_deferred_sales_load": null, @@ -197,13 +187,6 @@ "other_expenses": null, "total_annual_expenses": null, "twelve_b1_fee": null - }, - { - "class_name": "The Adviser has contractually agreed to waive its management fees and/or to bear expenses of the Fund through January\u00a031, 2018 to the extent necessary to prevent total Fund\noperating expenses (excluding acquired fund fees and expenses other than the advisory fees of any AB Mutual Funds in which the Fund may invest, interest expense, taxes, extraordinary expenses, and brokerage commissions and other transaction costs),\non an annualized basis, from exceeding .95%, 1.70%, .70%, 1.20%, .95%, .70% and .70% of average daily net assets, respectively, for Class\u00a0A, Class C, Advisor Class, Class R, Class K, Class I and Class Z shares (\u201cexpense limitations\u201d).\nAny fees waived and expenses borne by the Adviser prior to July\u00a015, 2015 may be reimbursed by the Fund until the end of the third fiscal year after the fiscal period in which the fee was waived or the expense was borne, provided that no\nreimbursement payment will be made that would cause the Fund\u2019s Total Annual Fund Operating Expenses to exceed the expense limitations.", - "expense_10yr": null, - "expense_1yr": null, - "expense_3yr": null, - "expense_5yr": null } ], "metadata": { @@ -360,13 +343,6 @@ "other_expenses": null, "total_annual_expenses": null, "twelve_b1_fee": null - }, - { - "class_name": "\u201cManagement Fees\u201d and \u201cOther Expenses\u201d have been restated to reflect current fees.", - "expense_10yr": null, - "expense_1yr": null, - "expense_3yr": 593, - "expense_5yr": null } ], "metadata": { diff --git a/tests/issues/regression/test_issue_912_497k_fee_waiver.py b/tests/issues/regression/test_issue_912_497k_fee_waiver.py index ab2cb3c88..147a50767 100644 --- a/tests/issues/regression/test_issue_912_497k_fee_waiver.py +++ b/tests/issues/regression/test_issue_912_497k_fee_waiver.py @@ -9,6 +9,11 @@ matched the waiver branch, overwrote the real waiver value, and made the net_expenses branch unreachable. +The same filing also rendered its fee-waiver footnote as a one-row table. +Because the prose mentions "management fees", `_classify_table` counted it as +a second operating-expenses table. That invented a phantom share class and +changed the expense-example parser, producing $3 instead of $109 and $381. + Ground truth is hand-verified from the fee table of the filing named in the issue: https://www.sec.gov/Archives/edgar/data/1314414/000158064224004234/ @@ -25,13 +30,38 @@ import pytest from edgar import find -from edgar.funds._497k_tables import _extract_operating_expenses, _parse_percentage +from edgar.funds._497k_tables import ( + _classify_table, + _extract_operating_expenses, + _parse_percentage, + extract_fee_tables, +) from edgar.funds.prospectus497k import Prospectus497K from tests._offline_filings import offline_filing # The 497K named in GH #912: Ocean Park High Income ETF, series S000085658. OCEAN_PARK_ACCESSION = "0001580642-24-004234" +OCEAN_PARK_FEE_EXCERPT = """ + + + + + + + + + +
Annual Fund Operating Expenses
Management Fees0.65%
Distribution and Service (12b-1) Fees0.00%
Other Expenses (1)0.32%
Acquired Fund Fees and Expenses (1)(2)0.29%
Total Annual Fund Operating Expenses1.26%
Fee Waiver and Reimbursement (3)(0.19)%
Total Annual Fund Operating Expenses after Fee Waiver and Reimbursement1.07%
+ + +
(3)The Adviser has contractually agreed to waive its management fees.
+ + + +
1 Year3 Years
$109$381
+""" + def _ocean_park_class(prospectus: Prospectus497K): """The real share class, ignoring any phantom classes (see edgartools-5owe).""" @@ -41,6 +71,258 @@ def _ocean_park_class(prospectus: Prospectus497K): raise AssertionError("no share class carried operating-expense data") +class TestFeeWaiverFootnoteIsNotAFeeTable: + def test_footnote_does_not_change_fee_table_layout(self): + assert extract_fee_tables(OCEAN_PARK_FEE_EXCERPT) == [ + { + "class_name": "", + "management_fee": Decimal("0.65"), + "twelve_b1_fee": Decimal("0.00"), + "other_expenses": Decimal("0.32"), + "acquired_fund_fees": Decimal("0.29"), + "total_annual_expenses": Decimal("1.26"), + "fee_waiver": Decimal("-0.19"), + "net_expenses": Decimal("1.07"), + "expense_1yr": 109, + "expense_3yr": 381, + } + ] + + def test_structured_management_fee_row_controls_classification(self): + fee_table = """ + + + +
Annual Management Fees (1)0.65%
Total Annual Fund Operating Expenses0.65%
+ """ + + assert extract_fee_tables(fee_table) == [ + { + "class_name": "", + "management_fee": Decimal("0.65"), + "total_annual_expenses": Decimal("0.65"), + } + ] + + def test_en_dash_preserves_a_missing_management_fee(self): + fee_table = """ + + + +
Management Fees–
Total Annual Fund Operating Expenses0.50%
+ """ + + assert extract_fee_tables(fee_table) == [ + { + "class_name": "", + "management_fee": None, + "total_annual_expenses": Decimal("0.50"), + } + ] + + @pytest.mark.parametrize( + "prose_footnote", + [ + "
Management fees have been restated.
", + "
Management fees have been restated.
", + "
Management fees 0.65% may be waived until expenses reach 1.00%.
", + ], + ) + def test_prose_management_fee_text_is_not_a_fee_table(self, prose_footnote): + assert extract_fee_tables(prose_footnote) == [] + + @pytest.mark.parametrize( + "rows", + [ + [["Management fees are waived for 1 Year at a cost of $25."]], + [["The sales charge does not affect waived management fees."]], + [ + ["Management fees are waived for 1 Year."], + ["(2)", "0.50%"], + ], + ], + ) + def test_prose_management_fee_tables_are_terminally_unclassified(self, rows): + assert _classify_table(rows) is None + + def test_split_cell_sales_charge_prose_is_terminally_unclassified(self): + rows = [ + [ + "The sales charge does not affect waived management fees.", + "See footnote (2).", + ] + ] + + assert _classify_table(rows) is None + + def test_percent_bearing_management_fee_prose_is_not_a_fee_table(self): + prose_footnote = """ + + + + + +
Management Feesmay be waived up to 0.50% through 2027
+ """ + + assert extract_fee_tables(prose_footnote) == [] + + def test_percent_leading_management_fee_prose_is_not_a_fee_table(self): + prose_footnote = """ + + + + + +
Management Fees0.50% may be waived through 2027
+ """ + + assert extract_fee_tables(prose_footnote) == [] + + @pytest.mark.parametrize( + ("management_fee", "expected"), + [ + ("None (1)", None), + ("— (1)", None), + ("N/A*", None), + ("1", Decimal("1")), + ], + ) + def test_marked_missing_and_integer_values_keep_fee_table_classified( + self, management_fee, expected + ): + fee_table = f""" + + + +
Management Fees{management_fee}
Total Annual Fund Operating Expenses0.60%
+ """ + + assert extract_fee_tables(fee_table) == [ + { + "class_name": "", + "management_fee": expected, + "total_annual_expenses": Decimal("0.60"), + } + ] + + def test_leading_spacer_cells_do_not_hide_fee_labels(self): + fee_table = """ + + + +
Management Fees0.65%
Total Annual Fund Operating Expenses0.75%
+ """ + + assert extract_fee_tables(fee_table) == [ + { + "class_name": "", + "management_fee": Decimal("0.65"), + "total_annual_expenses": Decimal("0.75"), + } + ] + + def test_leading_spacer_preserves_multi_class_value_alignment(self): + fee_table = """ + + + + + + + +
Class AClass C
Management Fees0.50%0.60%
Total Annual Fund Operating Expenses0.75%0.85%
+ """ + + assert extract_fee_tables(fee_table) == [ + { + "class_name": "Class A", + "management_fee": Decimal("0.50"), + "total_annual_expenses": Decimal("0.75"), + }, + { + "class_name": "Class C", + "management_fee": Decimal("0.60"), + "total_annual_expenses": Decimal("0.85"), + }, + ] + + def test_empty_trailing_cell_preserves_flattened_fee_row(self): + rows = [["Management Fees 0.65% 0.70%", ""]] + + assert _classify_table(rows) == "operating_expenses" + + def test_management_fee_footnote_does_not_hide_shareholder_fee_table(self): + rows = [ + ["Maximum Sales Charge (Load) Imposed on Purchases", "5.75%"], + ["The adviser may waive management fees for some shareholders."], + ] + + assert _classify_table(rows) == "shareholder_fees" + + def test_indented_shareholder_row_stays_visible_with_management_fee_footnote(self): + rows = [ + ["", "Maximum Sales Charge (Load) Imposed on Purchases", "5.75%"], + ["The adviser may waive management fees for some shareholders."], + ] + + assert _classify_table(rows) == "shareholder_fees" + + def test_indented_shareholder_row_preserves_extracted_fee(self): + filing_tables = """ + + + +
Management Fees0.50%
Total Annual Fund Operating Expenses0.75%
+ + + + + + + + + +
Maximum Sales Charge (Load) Imposed on Purchases5.75%
The adviser may waive management fees for some shareholders.
+ """ + + assert extract_fee_tables(filing_tables) == [ + { + "class_name": "", + "management_fee": Decimal("0.50"), + "total_annual_expenses": Decimal("0.75"), + "max_sales_load": Decimal("5.75"), + } + ] + + def test_management_fee_footnote_does_not_hide_expense_example(self): + rows = [ + ["", "1 Year", "3 Years"], + ["Class A", "$109", "$381"], + ["Management fees may be waived under the expense limitation agreement."], + ] + + assert _classify_table(rows) == "expense_example" + + def test_management_fee_footnote_does_not_hide_performance_table(self): + rows = [ + ["", "1 Year", "5 Years"], + ["Return", "7.50%", "8.25%"], + ["Management fees may be waived under the expense limitation agreement."], + ] + + assert _classify_table(rows) == "performance" + + def test_management_fee_footnote_does_not_hide_bar_chart(self): + rows = [ + ["2020", "2021", "2022", "2023"], + ["4.25%", "6.10%", "-3.50%", "8.75%"], + ["Management fees may be waived under the expense limitation agreement."], + ] + + assert _classify_table(rows) == "bar_chart" + + class TestFeeWaiverIsNotTheNetRatio: """The waiver and the net ratio land in their own fields."""