diff --git a/evals/data/dev/solar/manifest.json5 b/evals/data/dev/solar/manifest.json5 index 1833f3367..767e3c859 100644 --- a/evals/data/dev/solar/manifest.json5 +++ b/evals/data/dev/solar/manifest.json5 @@ -6,6 +6,7 @@ "jurisdiction_type": "county", "file": "Bartow_County_Georgia.pdf", "source": "https://cms2.revize.com/revize/bartowga/ordinance%20updated%20thru%2011-04-20.pdf", + "document_satus": "Final", "expected": { "year": 2020 } @@ -17,6 +18,7 @@ "jurisdiction_type": "county", "file": "Benton_County_Washington.pdf", "source": "https://bentoncountywa.municipalone.com/docview.aspx?docid=29828", + "document_satus": "Final", "expected": { "year": null } @@ -28,6 +30,7 @@ "jurisdiction_type": "county", "file": "Boone_County_Missouri.pdf", "source": "https://www.showmeboone.com/resource-management/Solar/03-18-24%20Final%20Drafts/SECTION_30_SOLAR_ENERGY_OVERLAY_DISTRICT_FINAL_DRAFT_20240318.pdf", + "document_satus": "Draft", "expected": { "year": 2024 } @@ -39,6 +42,7 @@ "jurisdiction_type": "parish", "file": "Bossier_Parish_Louisiana.pdf", "source": "https://www.bossierparishla.gov/docs/default-source/police-jury/recent-ordinances/4910.pdf?sfvrsn=aa884719_3#:~:text=BE%20IT%20ORDAINED%20that%20the,Police%20Jury%2C%20excluding%20state%2Downed", + "document_satus": "Final", "expected": { "year": 2022 } @@ -50,6 +54,7 @@ "jurisdiction_type": "county", "file": "Bowman_County_North_Dakota.pdf", "source": "https://bowmannd.com/wp-content/uploads/2024/12/2024-ZONING-ORDINACE.pdf", + "document_satus": "Draft", "expected": { "year": 2024 } @@ -61,6 +66,7 @@ "jurisdiction_type": "county", "file": "Bremer_County_Iowa.pdf", "source": "https://cms7files.revize.com/bremercountyia/document_center/Building%20Zoning/Ordinances%20and%20codes/Recorded%20Ord%2022-07%20(Zoning%20Ord).pdf", + "document_satus": "Final", "expected": { "year": 2022 } @@ -72,6 +78,7 @@ "jurisdiction_type": "county", "file": "Carlton_County_Minnesota.pdf", "source": "https://www.carltoncountymn.gov/DocumentCenter/View/931/Renewable-Energy-Ordinance-PDF", + "document_satus": "Final", "expected": { "year": 2017 } @@ -83,6 +90,7 @@ "jurisdiction_type": "county", "file": "Carroll_County_Illinois.pdf", "source": "https://cms9files.revize.com/carrollil/Chapter%20725%20Solar%20Amended%2005.18.2023.pdf", + "document_satus": "Final", "expected": { "year": 2023 } @@ -94,6 +102,7 @@ "jurisdiction_type": "county", "file": "Chisago_County_Minnesota.pdf", "source": "https://www.chisagocountymn.gov/DocumentCenter/View/7211/Solar-Energy-Systems-Ordinance-PDF", + "document_satus": "Final", "expected": { "year": 2020 } @@ -105,6 +114,7 @@ "jurisdiction_type": "county", "file": "Clay_County_Minnesota.pdf", "source": "https://claycountymn.gov/DocumentCenter/View/16228", + "document_satus": "Final", "expected": { "year": null } @@ -116,6 +126,7 @@ "jurisdiction_type": "county", "file": "Crook_County_Wyoming.pdf", "source": "https://cms2.revize.com/revize/crookscountynew/elected_officials/commissioners/docs/Proposed_Rules_&_Regs/Wind%20and%20Solar%20Regulation%20after%20August%205%202025.pdf?t=202508192041490&t=202508192041490", + "document_satus": "Final", "expected": { "year": 2025 } @@ -127,6 +138,7 @@ "jurisdiction_type": "county", "file": "Cumberland_County_Tennessee.pdf", "source": "https://www.cumberlandcountync.gov/vd-Planning/downloads/ord_amendments/county/Solar%20Farm%20__Approved__051815.pdf", + "document_satus": "Final", "expected": { "year": 2015 } @@ -138,6 +150,7 @@ "jurisdiction_type": "county", "file": "Dubuque_County_Iowa.pdf", "source": "https://www.dubuquecountyiowa.gov/DocumentCenter/View/7375/Ch-05---Renewable-Energy-Ordinance-DRAFT-2024-07-29-PDF", + "document_satus": "Draft", "expected": { "year": 2024 } @@ -149,6 +162,7 @@ "jurisdiction_type": "county", "file": "Garrard_County_Kentucky.pdf", "source": "https://garrardcountyky.gov/wp-content/uploads/2024/01/Solar-Ordinance-Second-Reading-Signed.pdf", + "document_satus": "Final", "expected": { "year": 2024 } @@ -160,6 +174,7 @@ "jurisdiction_type": "county", "file": "Greene_County_Tennessee.pdf", "source": "https://www.greenecountytngov.com/wp-content/uploads/2017/09/Greene%20County,%20TN.Zoning%20Res.Dec%202015.pdf", + "document_satus": "Final", "expected": { "year": 2015 } @@ -171,6 +186,7 @@ "jurisdiction_type": "county", "file": "Hamilton_County_Illinois.pdf", "source": "https://www.hamiltoncountyil.gov/app/uploads/2022/06/Hamilton-County-IL-Commercial-Solar-Energy-Ordinance-2024-05-02.pdf", + "document_satus": "Final", "expected": { "year": 2024 } @@ -182,6 +198,7 @@ "jurisdiction_type": "county", "file": "Medina_County_Ohio.txt", "source": "https://codelibrary.amlegal.com/codes/medina/latest/medina_oh/0-0-0-29637", + "document_satus": "Final", "expected": { "year": 2014 } @@ -193,6 +210,7 @@ "jurisdiction_type": "county", "file": "Okanogan_County_Washington.txt", "source": "https://www.codepublishing.com/WA/OkanoganCounty/html/OkanoganCounty17/OkanoganCounty17255.html", + "document_satus": "Final", "expected": { "year": 2024 } @@ -204,6 +222,7 @@ "jurisdiction_type": "county", "file": "Pottawattamie_County_Iowa.pdf", "source": "https://www.pottcounty-ia.gov/files/meetings/2023-11-14_packet_board_of_supervisors_50640.pdf", + "document_satus": "Draft", "expected": { "year": 2023 } @@ -215,6 +234,7 @@ "jurisdiction_type": "county", "file": "Queen_Anne'S_County_Maryland.pdf", "source": "https://dnr.maryland.gov/pprp/Documents/QueenAnneZoningDocs.pdf", + "document_satus": "Final", "expected": { "year": 2020 } @@ -226,6 +246,7 @@ "jurisdiction_type": "county", "file": "Saline_County_Illinois.pdf", "source": "https://energyzoning.org/sites/default/files/PDF/17165_Saline%20_20230118.pdf", + "document_satus": "Final", "expected": { "year": 2020 } @@ -237,6 +258,7 @@ "jurisdiction_type": "county", "file": "Southampton_County_Virginia.pdf", "source": "https://www.southamptoncounty.org/BOS%20Agenda%202022/March%2022,%202022/No.%2015%20-%20Draft%20Ordinance%20-%20Utility%20Scale%20Solar%20Projects.pdf", + "document_satus": "Draft", "expected": { "year": 2022 } @@ -248,6 +270,7 @@ "jurisdiction_type": "county", "file": "St._Lucie_County_Florida.pdf", "source": "https://mcclibraryfunctions.azurewebsites.us/api/ordinanceDownload/14641/485031/pdf", + "document_satus": "Final", "expected": { "year": 2010 } @@ -259,6 +282,7 @@ "jurisdiction_type": "town", "file": "Town_of_Berne_Albany_County_New_York.pdf", "source": "https://berneny.org/wp-content/uploads/Proposed-Local-Law-3-of-2022.pdf", + "document_satus": "Proposal", "expected": { "year": 2022 } @@ -270,6 +294,7 @@ "jurisdiction_type": "town", "file": "Town_of_Binghamton_Broome_County_New_York.pdf", "source": "https://townofbinghamton.com/wp-content/uploads/2020/10/TOB-Draft-Solar-Law-REV.-8-31-20-.pdf", + "document_satus": "Draft", "expected": { "year": 2020 } @@ -281,6 +306,7 @@ "jurisdiction_type": "town", "file": "Town_of_Boston_Erie_County_New_York.pdf", "source": "https://ecode360.com/BO0557/laws/LF1193731.pdf", + "document_satus": "Final", "expected": { "year": 2019 } @@ -292,6 +318,7 @@ "jurisdiction_type": "town", "file": "Town_of_Bourne_Barnstable_County_Massachusetts.pdf", "source": "https://www.townofbourne.com/sites/g/files/vyhlif12841/f/uploads/zoning_bylaw_2022_for_website_1.pdf", + "document_satus": "Final", "expected": { "year": 2022 } @@ -303,6 +330,7 @@ "jurisdiction_type": "town", "file": "Town_of_Chautauqua_Chautauqua_County_New_York.pdf", "source": "https://municipality-management.s3.us-east-2.amazonaws.com/48/agendas/gCGk4Sv6Jidg9xpX_11%208%2023%20Work%20Session%20Agenda-%20Solar%20Review.pdf", + "document_satus": "Draft", "expected": { "year": 2023 } @@ -314,6 +342,7 @@ "jurisdiction_type": "town", "file": "Town_of_Denmark_Oxford_County_Maine.pdf", "source": "https://denmarkmaine.org/uploads/zoning_ordinance_amended_02.27.2021_approved_with_5.16__8.8_additions-1-.pdf", + "document_satus": "Final", "expected": { "year": 2021 } @@ -325,6 +354,7 @@ "jurisdiction_type": "town", "file": "Town_of_Dover_Dutchess_County_New_York.pdf", "source": "https://www.doverny.us/sites/g/files/vyhlif4341/f/news/draft_new_dover_solar_code_updated_09_13_23__0.pdf", + "document_satus": "Draft", "expected": { "year": 2023 } @@ -336,6 +366,7 @@ "jurisdiction_type": "town", "file": "Town_of_Glenville_Schenectady_County_New_York.pdf", "source": "https://www.townofglenville.org/home/files/gecc-solar-attachments", + "document_satus": "Final", "expected": { "year": 2024 } @@ -347,6 +378,7 @@ "jurisdiction_type": "town", "file": "Town_of_Hartland_Niagara_County_New_York.pdf", "source": "https://cms5.revize.com/revize/hartlandny/Businesses/Town%20of%20Hartland%20Code/Zoning%20&%20All%20Codes/HA0923-144.pdf", + "document_satus": "Final", "expected": { "year": 2017 } @@ -358,6 +390,7 @@ "jurisdiction_type": "town", "file": "Town_of_Hunter_Greene_County_New_York.pdf", "source": "https://townofhunterny.gov/wp-content/uploads/2021/06/solar-law-2021.pdf", + "document_satus": "Final", "expected": { "year": 2021 } @@ -369,6 +402,7 @@ "jurisdiction_type": "town", "file": "Town_of_Livingston_Columbia_County_New_York.pdf", "source": "https://townoflivingston.org/wp-content/uploads/2022/06/Local-Law-2-of-2021.pdf", + "document_satus": "Final", "expected": { "year": 2022 } @@ -380,6 +414,7 @@ "jurisdiction_type": "town", "file": "Town_of_Livonia_Livingston_County_New_York.txt", "source": "https://ecode360.com/38054462", + "document_satus": "Final", "expected": { "year": 2022 } @@ -391,6 +426,7 @@ "jurisdiction_type": "town", "file": "Town_of_Middleton_Strafford_County_New_Hampshire.pdf", "source": "https://middletonnh.gov/uploads/proposed-zoning-ordinances.pdf", + "document_satus": "Final", "expected": { "year": 2023 } @@ -402,6 +438,7 @@ "jurisdiction_type": "town", "file": "Town_of_Otis_Berkshire_County_Massachusetts.pdf", "source": "https://ecode360.com/43529091#43529410", + "document_satus": "Proposal", "expected": { "year": 2025 } @@ -413,6 +450,7 @@ "jurisdiction_type": "town", "file": "Town_of_Pendleton_Niagara_County_New_York.txt", "source": "https://ecode360.com/5125460", + "document_satus": "Final", "expected": { "year": 2022 } @@ -424,6 +462,7 @@ "jurisdiction_type": "town", "file": "Town_of_Sugar_Hill_Grafton_County_New_Hampshire.pdf", "source": "https://www.sugarhillnh.org/wp-content/uploads/2024/04/SH-Zoning-Ordinance-Amended-03122024.pdf", + "document_satus": "Final", "expected": { "year": 2024 } @@ -435,6 +474,7 @@ "jurisdiction_type": "town", "file": "Town_of_Ware_Hampshire_County_Massachusetts.pdf", "source": "https://cms1files.revize.com/warema/Ware%20Bylaw%20-%20Utility%20Scale%20Solar%20Revision.pdf", + "document_satus": "Final", "expected": { "year": null } @@ -446,6 +486,7 @@ "jurisdiction_type": "town", "file": "Town_of_West_Greenwich_Kent_County_Rhode_Island.pdf", "source": "https://ecode360.com/33936687#33936687", + "document_satus": "Draft", "expected": { "year": 2025 } @@ -457,6 +498,7 @@ "jurisdiction_type": "town", "file": "Town_of_Westerlo_Albany_County_New_York.pdf", "source": "https://townofwesterlony.com/wp-content/uploads/2021/12/LL-No-4-of-2021-Solar-Energy-Systems-FINAL-2.pdf", + "document_satus": "Final", "expected": { "year": 2021 } @@ -468,6 +510,7 @@ "jurisdiction_type": "town", "file": "Town_of_Whitehall_Washington_County_New_York.pdf", "source": "https://www.washingtoncountyny.gov/DocumentCenter/View/25318/Town-Whitehall-Moratorium-with-SEQRAFS", + "document_satus": "Draft", "expected": { "year": 2024 } @@ -479,6 +522,7 @@ "jurisdiction_type": "town", "file": "Town_of_Windham_Greene_County_New_York.pdf", "source": "https://www.townofwindhamny.com/wp-content/uploads/2022/06/Local-Law-1-of-2022-Regulating-Solar-Energy-Systems.pdf", + "document_satus": "Final", "expected": { "year": 2022 } @@ -490,6 +534,7 @@ "jurisdiction_type": "county", "file": "Wayne_County_Georgia.pdf", "source": "https://mcclibraryfunctions.azurewebsites.us/api/ordinanceDownload/12423/1246775/pdf?forceDownload=true#:~:text=(j)%20All%20solar%20farms%20shall,of%20the%20International%20Fire%20Code.", + "document_satus": "Final", "expected": { "year": 2023 } @@ -501,6 +546,7 @@ "jurisdiction_type": "county", "file": "Winnebago_County_Illinois.pdf", "source": "https://energyzoning.org/sites/default/files/PDF/17201_Winnebago%20_20220929.pdf", + "document_satus": "Final", "expected": { "year": 2015 } @@ -512,6 +558,7 @@ "jurisdiction_type": "county", "file": "Woodford_County_Illinois.pdf", "source": "https://woodford-county.org/AgendaCenter/ViewFile/Agenda/_02152023-1370", + "document_satus": "Draft", "expected": { "year": 2018 } @@ -523,6 +570,7 @@ "jurisdiction_type": "county", "file": "Yavapai_County_Arizona.pdf", "source": "https://www.yavapaiaz.gov/files/sharedassets/public/v/1/development-and-permits/development-services/documents/news/sec-608-solar-facilities-approved-11-6-24-in-effect-12-4-24-watermarked.pdf", + "document_satus": "Final", "expected": { "year": 2024 } diff --git a/evals/data/held-out/solar/Carroll_County_Indiana.pdf b/evals/data/held-out/solar/Carroll_County_Indiana.pdf index 4c793bb77..6c277bf14 100644 Binary files a/evals/data/held-out/solar/Carroll_County_Indiana.pdf and b/evals/data/held-out/solar/Carroll_County_Indiana.pdf differ diff --git a/evals/data/held-out/solar/manifest.json5 b/evals/data/held-out/solar/manifest.json5 index 62979da21..070d3c3c2 100644 --- a/evals/data/held-out/solar/manifest.json5 +++ b/evals/data/held-out/solar/manifest.json5 @@ -6,6 +6,7 @@ "jurisdiction_type": "county", "file": "Carroll_County_Indiana.pdf", "source": "https://www.in.gov/counties/carroll/files/ORDINANCE-2024-01.pdf", + "document_satus": "Final", "expected": { "year": 2024 } @@ -17,6 +18,7 @@ "jurisdiction_type": "city", "file": "City_of_Agawam_Town_Hampden_County_Massachusetts.txt", "source": "https://ecode360.com/34134372", + "document_satus": "Final", "expected": { "year": 2025 } @@ -28,6 +30,7 @@ "jurisdiction_type": "county", "file": "Jasper_County_Indiana.pdf", "source": "https://jasper.county.codes/JCC/20.50.950", + "document_satus": "Final", "expected": { "year": 2025 } @@ -39,6 +42,7 @@ "jurisdiction_type": "county", "file": "Montgomery_County_Iowa.pdf", "source": "https://www.montgomerycountyia.gov/docview.aspx?docid=50714", + "document_satus": "Final", "expected": { "year": 2022 } @@ -50,6 +54,7 @@ "jurisdiction_type": "county", "file": "Mountrail_County_North_Dakota.pdf", "source": "http://www.co.mountrail.nd.us/Documents/Uploads/Mountrail-Co-LDC-Recorded-Document-455088.pdf", + "document_satus": "Final", "expected": { "year": 2022 } @@ -61,6 +66,7 @@ "jurisdiction_type": "county", "file": "Murray_County_Minnesota.pdf", "source": "https://cms9files.revize.com/murraycounty/Renewable_Energy_eff_01-01-20.pdf", + "document_satus": "Final", "expected": { "year": 2020 } @@ -72,6 +78,7 @@ "jurisdiction_type": "county", "file": "Otter_Tail_County_Minnesota.pdf", "source": "https://ottertailcounty.gov/wp-content/uploads/2021/07/Renewable-Energy-Ordinance-11-23-2021.pdf", + "document_satus": "Final", "expected": { "year": 2021 } @@ -83,6 +90,7 @@ "jurisdiction_type": "county", "file": "Pope_County_Minnesota.pdf", "source": "https://www.popecountymn.gov/wp-content/uploads/2024/09/Resolution-202424-Approving-Amendments-to-the-Pope-County-LUCO-Sec2.2-10.4-10.12-10.24-and-11.pdf", + "document_satus": "Final", "expected": { "year": 2024 } @@ -94,6 +102,7 @@ "jurisdiction_type": "town", "file": "Town_of_Ancram_Columbia_County_New_York.pdf", "source": "https://www.ancramny.org/ws/wp-content/uploads/2021/04/Abstract-of-Local-Law-1-of-2021-Solar-Law.pdf", + "document_satus": "Final", "expected": { "year": 2022 } @@ -105,6 +114,7 @@ "jurisdiction_type": "town", "file": "Town_of_Arcade_Wyoming_County_New_York.pdf", "source": "https://villageofarcade.org/wp-content/uploads/VOA-files/Zoning%20Board%20Documents/Town%20of%20Arcade%20Zoning%20Law%20December%202020.pdf", + "document_satus": "Final", "expected": { "year": 2020 } @@ -116,6 +126,7 @@ "jurisdiction_type": "town", "file": "Town_of_Caneadea_Allegany_County_New_York.pdf", "source": "https://cms2.revize.com/revize/caneadea/Article%2015.pdf", + "document_satus": "Final", "expected": { "year": null } @@ -127,6 +138,7 @@ "jurisdiction_type": "town", "file": "Town_of_Crystal_Aroostook_County_Maine.pdf", "source": "https://www.islandfallsme.us/vertical/sites/%7B22C70384-7073-450B-957A-33378347F000%7D/uploads/Solar_Ordinance_Crystal(1).pdf", + "document_satus": "Final", "expected": { "year": 2023 } @@ -138,6 +150,7 @@ "jurisdiction_type": "town", "file": "Town_of_Enfield_Penobscot_County_Maine.pdf", "source": "https://www.enfieldmaine.org/Commercial%20Solar%20Ordinance.pdf", + "document_satus": "Final", "expected": { "year": 2024 } @@ -149,6 +162,7 @@ "jurisdiction_type": "town", "file": "Town_of_Greenville_Orange_County_New_York.pdf", "source": "https://ecode360.com/GR1575/laws/LF2129396.pdf", + "document_satus": "Final", "expected": { "year": 2024 } @@ -160,6 +174,7 @@ "jurisdiction_type": "town", "file": "Town_of_Hampden_Penobscot_County_Maine.pdf", "source": "https://www.hampdenmaine.gov/vertical/sites/%7B1FCAF0C4-5C5E-476D-A92E-1BED5B1F9E05%7D/uploads/SOLAR_ENERGY_ORDINANCE.pdf", + "document_satus": "Final", "expected": { "year": 2021 } @@ -171,6 +186,7 @@ "jurisdiction_type": "town", "file": "Town_of_Niagara_Niagara_County_New_York.txt", "source": "https://ecode360.com/45924925", + "document_satus": "Final", "expected": { "year": 2024 } @@ -182,6 +198,7 @@ "jurisdiction_type": "town", "file": "Town_of_Little_Falls_Herkimer_County_New_York.pdf", "source": "https://townoflittlefallsny.com/wp-content/uploads/2023/12/1-2020-1-Solar-Energy-Regulations-Law-AMENDMENT-TO-LAW-2017-2-2.pdf", + "document_satus": "Final", "expected": { "year": 2020 } @@ -193,6 +210,7 @@ "jurisdiction_type": "town", "file": "Town_of_Moscow_Somerset_County_Maine.pdf", "source": "https://moscowmaine.org/s/Solar-Farm-Ordinance.pdf", + "document_satus": "Final", "expected": { "year": null } @@ -204,6 +222,7 @@ "jurisdiction_type": "town", "file": "Town_of_Otisco_Onondaga_County_New_York.pdf", "source": "https://townofotisco.com/wp-content/uploads/2024/04/Local-Law-2024-05.pdf", + "document_satus": "Final", "expected": { "year": 2024 } @@ -215,6 +234,7 @@ "jurisdiction_type": "town", "file": "Town_of_Summerhill_Cayuga_County_New_York.pdf", "source": "https://www.cayugacounty.us/DocumentCenter/View/19649/Town-of-Summerhill-Proposed-Solar-Land-Use-Regs-PDF", + "document_satus": "Final", "expected": { "year": 2022 } @@ -226,6 +246,7 @@ "jurisdiction_type": "town", "file": "Town_of_Walton_Delaware_County_New_York.txt", "source": "https://ecode360.com/34256634", + "document_satus": "Final", "expected": { "year": 2019 } @@ -237,6 +258,7 @@ "jurisdiction_type": "county", "file": "Wabasha_County_Minnesota.pdf", "source": "https://cms9files1.revize.com/wabasha/document_center/Wabasha%20County%20Ordinances/ordinance_89.pdf", + "document_satus": "Final", "expected": { "year": 2024 } diff --git a/evals/results/dev/date_extraction_evals.json b/evals/results/dev/date_extraction_evals.json index 62900b97a..1d83ed10c 100644 --- a/evals/results/dev/date_extraction_evals.json +++ b/evals/results/dev/date_extraction_evals.json @@ -1,22 +1,22 @@ [ { "feature": "year", - "n_cases": 48, - "true_positive": 23, + "n_cases": 35, + "true_positive": 15, "true_negative": 3, - "false_positive": 2, - "false_negative": 22, - "failing_cases": 22, - "accuracy": 0.5417, - "precision": 0.92, - "recall": 0.5111, - "f1": 0.6571, - "accuracy_95_percent_confidence_interval": "0.4029 - 0.6742", - "precision_95_percent_confidence_interval": "0.7503 - 0.9778", - "recall_95_percent_confidence_interval": "0.3700 - 0.6504", - "total_prompt_tokens": 35325, - "total_response_tokens": 4667, - "total_time_taken_s": 796.13, - "total_cost_usd": 0.0792 + "false_positive": 0, + "false_negative": 17, + "failing_cases": 17, + "accuracy": 0.5143, + "precision": 1.0, + "recall": 0.4688, + "f1": 0.6383, + "accuracy_95_percent_confidence_interval": "0.3557 - 0.6701", + "precision_95_percent_confidence_interval": "0.7961 - 1.0000", + "recall_95_percent_confidence_interval": "0.3087 - 0.6355", + "total_prompt_tokens": 32023, + "total_response_tokens": 3856, + "total_time_taken_s": 702.59, + "total_cost_usd": 0.0689 } ] diff --git a/evals/results/dev/date_extraction_evals_breakdown.csv b/evals/results/dev/date_extraction_evals_breakdown.csv index bedd25ee1..4ba6ef1da 100644 --- a/evals/results/dev/date_extraction_evals_breakdown.csv +++ b/evals/results/dev/date_extraction_evals_breakdown.csv @@ -1,49 +1,36 @@ -state,county,subdivision,jurisdiction_type,file,source,feature,expected,extracted,comparison_result,prompt_tokens,response_tokens,time_taken_s,cost -Arizona,Yavapai,,county,Yavapai_County_Arizona.pdf,https://www.yavapaiaz.gov/files/sharedassets/public/v/1/development-and-permits/development-services/documents/news/sec-608-solar-facilities-approved-11-6-24-in-effect-12-4-24-watermarked.pdf,year,2024,2024,Success,274,65,8.657,0.00083 -Florida,St. Lucie,,county,St._Lucie_County_Florida.pdf,https://mcclibraryfunctions.azurewebsites.us/api/ordinanceDownload/14641/485031/pdf,year,2010,,Failure,244,48,8.351,0.000665 -Georgia,Bartow,,county,Bartow_County_Georgia.pdf,https://cms2.revize.com/revize/bartowga/ordinance%20updated%20thru%2011-04-20.pdf,year,2020,2020,Success,254,53,9.923,0.0007149999999999999 -Georgia,Wayne,,county,Wayne_County_Georgia.pdf,"https://mcclibraryfunctions.azurewebsites.us/api/ordinanceDownload/12423/1246775/pdf?forceDownload=true#:~:text=(j)%20All%20solar%20farms%20shall,of%20the%20International%20Fire%20Code.",year,2023,,Failure,283,48,23.956,0.00071375 -Illinois,Carroll,,county,Carroll_County_Illinois.pdf,https://cms9files.revize.com/carrollil/Chapter%20725%20Solar%20Amended%2005.18.2023.pdf,year,2023,2023,Success,254,51,8.301,0.0006999999999999999 -Illinois,Hamilton,,county,Hamilton_County_Illinois.pdf,https://www.hamiltoncountyil.gov/app/uploads/2022/06/Hamilton-County-IL-Commercial-Solar-Energy-Ordinance-2024-05-02.pdf,year,2024,2024,Success,260,63,36.133,0.0007975 -Illinois,Saline,,county,Saline_County_Illinois.pdf,https://energyzoning.org/sites/default/files/PDF/17165_Saline%20_20230118.pdf,year,2020,,Failure,3140,261,29.774,0.0058825 -Illinois,Winnebago,,county,Winnebago_County_Illinois.pdf,https://energyzoning.org/sites/default/files/PDF/17201_Winnebago%20_20220929.pdf,year,2015,,Failure,20491,1586,49.033,0.037508749999999993 -Illinois,Woodford,,county,Woodford_County_Illinois.pdf,https://woodford-county.org/AgendaCenter/ViewFile/Agenda/_02152023-1370,year,2018,2023,Failure,243,55,8.553,0.00071625 -Iowa,Bremer,,county,Bremer_County_Iowa.pdf,https://cms7files.revize.com/bremercountyia/document_center/Building%20Zoning/Ordinances%20and%20codes/Recorded%20Ord%2022-07%20(Zoning%20Ord).pdf,year,2022,,Failure,272,64,102.427,0.00082 -Iowa,Dubuque,,county,Dubuque_County_Iowa.pdf,https://www.dubuquecountyiowa.gov/DocumentCenter/View/7375/Ch-05---Renewable-Energy-Ordinance-DRAFT-2024-07-29-PDF,year,2024,2024,Success,262,62,9.838,0.0007925 -Iowa,Pottawattamie,,county,Pottawattamie_County_Iowa.pdf,https://www.pottcounty-ia.gov/files/meetings/2023-11-14_packet_board_of_supervisors_50640.pdf,year,2023,2023,Success,252,77,9.21,0.0008925000000000001 -Kentucky,Garrard,,county,Garrard_County_Kentucky.pdf,https://garrardcountyky.gov/wp-content/uploads/2024/01/Solar-Ordinance-Second-Reading-Signed.pdf,year,2024,2024,Success,250,69,26.787,0.00083 -Louisiana,Bossier,,parish,Bossier_Parish_Louisiana.pdf,"https://www.bossierparishla.gov/docs/default-source/police-jury/recent-ordinances/4910.pdf?sfvrsn=aa884719_3#:~:text=BE%20IT%20ORDAINED%20that%20the,Police%20Jury%2C%20excluding%20state%2Downed",year,2022,,Failure,299,62,8.983,0.0008387500000000001 -Maine,Oxford,Denmark,town,Town_of_Denmark_Oxford_County_Maine.pdf,https://denmarkmaine.org/uploads/zoning_ordinance_amended_02.27.2021_approved_with_5.16__8.8_additions-1-.pdf,year,2021,2021,Success,260,64,10.536,0.0008049999999999999 -Maryland,Queen Anne's,,county,Queen_Anne'S_County_Maryland.pdf,https://dnr.maryland.gov/pprp/Documents/QueenAnneZoningDocs.pdf,year,2020,,Failure,241,57,9.722,0.00072875 -Massachusetts,Barnstable,Bourne,town,Town_of_Bourne_Barnstable_County_Massachusetts.pdf,https://www.townofbourne.com/sites/g/files/vyhlif12841/f/uploads/zoning_bylaw_2022_for_website_1.pdf,year,2022,2022,Success,255,69,9.28,0.00083625 -Massachusetts,Berkshire,Otis,town,Town_of_Otis_Berkshire_County_Massachusetts.pdf,https://ecode360.com/43529091#43529410,year,2025,,Failure,235,45,38.699,0.0006312500000000001 -Massachusetts,Hampshire,Ware,town,Town_of_Ware_Hampshire_County_Massachusetts.pdf,https://cms1files.revize.com/warema/Ware%20Bylaw%20-%20Utility%20Scale%20Solar%20Revision.pdf,year,,,Success,254,62,8.728,0.0007825 -Minnesota,Carlton,,county,Carlton_County_Minnesota.pdf,https://www.carltoncountymn.gov/DocumentCenter/View/931/Renewable-Energy-Ordinance-PDF,year,2017,,Failure,249,56,10.4,0.00073125 -Minnesota,Chisago,,county,Chisago_County_Minnesota.pdf,https://www.chisagocountymn.gov/DocumentCenter/View/7211/Solar-Energy-Systems-Ordinance-PDF,year,2020,,Failure,251,48,28.366,0.00067375 -Minnesota,Clay,,county,Clay_County_Minnesota.pdf,https://claycountymn.gov/DocumentCenter/View/16228,year,,,Success,237,51,9.795,0.0006787499999999999 -Missouri,Boone,,county,Boone_County_Missouri.pdf,https://www.showmeboone.com/resource-management/Solar/03-18-24%20Final%20Drafts/SECTION_30_SOLAR_ENERGY_OVERLAY_DISTRICT_FINAL_DRAFT_20240318.pdf,year,2024,2024,Success,267,62,9.42,0.00079875 -New Hampshire,Grafton,Sugar Hill,town,Town_of_Sugar_Hill_Grafton_County_New_Hampshire.pdf,https://www.sugarhillnh.org/wp-content/uploads/2024/04/SH-Zoning-Ordinance-Amended-03122024.pdf,year,2024,2024,Success,252,70,9.189,0.00084 -New Hampshire,Strafford,Middleton,town,Town_of_Middleton_Strafford_County_New_Hampshire.pdf,https://middletonnh.gov/uploads/proposed-zoning-ordinances.pdf,year,2023,,Failure,237,60,44.5,0.00074625 -New York,Albany,Berne,town,Town_of_Berne_Albany_County_New_York.pdf,https://berneny.org/wp-content/uploads/Proposed-Local-Law-3-of-2022.pdf,year,2022,2022,Success,245,68,8.515,0.00081625 -New York,Albany,Westerlo,town,Town_of_Westerlo_Albany_County_New_York.pdf,https://townofwesterlony.com/wp-content/uploads/2021/12/LL-No-4-of-2021-Solar-Energy-Systems-FINAL-2.pdf,year,2021,2021,Success,259,68,8.637,0.0008337500000000001 -New York,Broome,Binghamton,town,Town_of_Binghamton_Broome_County_New_York.pdf,https://townofbinghamton.com/wp-content/uploads/2020/10/TOB-Draft-Solar-Law-REV.-8-31-20-.pdf,year,2020,2020,Success,257,64,8.888,0.00080125 -New York,Chautauqua,Chautauqua,town,Town_of_Chautauqua_Chautauqua_County_New_York.pdf,https://municipality-management.s3.us-east-2.amazonaws.com/48/agendas/gCGk4Sv6Jidg9xpX_11%208%2023%20Work%20Session%20Agenda-%20Solar%20Review.pdf,year,2023,2023,Success,274,70,34.105,0.0008675 -New York,Columbia,Livingston,town,Town_of_Livingston_Columbia_County_New_York.pdf,https://townoflivingston.org/wp-content/uploads/2022/06/Local-Law-2-of-2021.pdf,year,2022,2022,Success,248,105,24.332,0.0010975 -New York,Dutchess,Dover,town,Town_of_Dover_Dutchess_County_New_York.pdf,https://www.doverny.us/sites/g/files/vyhlif4341/f/news/draft_new_dover_solar_code_updated_09_13_23__0.pdf,year,2023,2023,Success,258,48,9.681,0.0006825 -New York,Erie,Boston,town,Town_of_Boston_Erie_County_New_York.pdf,https://ecode360.com/BO0557/laws/LF1193731.pdf,year,2019,,Failure,239,51,8.663,0.00068125 -New York,Greene,Hunter,town,Town_of_Hunter_Greene_County_New_York.pdf,https://townofhunterny.gov/wp-content/uploads/2021/06/solar-law-2021.pdf,year,2021,2021,Success,245,84,49.952,0.0009362499999999999 -New York,Greene,Windham,town,Town_of_Windham_Greene_County_New_York.pdf,https://www.townofwindhamny.com/wp-content/uploads/2022/06/Local-Law-1-of-2022-Regulating-Solar-Energy-Systems.pdf,year,2022,2022,Success,259,67,10.164,0.00082625 -New York,Livingston,Livonia,town,Town_of_Livonia_Livingston_County_New_York.txt,https://ecode360.com/38054462,year,2022,,Failure,231,53,2.142,0.00068625 -New York,Niagara,Hartland,town,Town_of_Hartland_Niagara_County_New_York.pdf,https://cms5.revize.com/revize/hartlandny/Businesses/Town%20of%20Hartland%20Code/Zoning%20&%20All%20Codes/HA0923-144.pdf,year,2017,,Failure,268,63,10.443,0.0008075000000000001 -New York,Niagara,Pendleton,town,Town_of_Pendleton_Niagara_County_New_York.txt,https://ecode360.com/5125460,year,2022,,Failure,231,58,2.567,0.00072375 -New York,Schenectady,Glenville,town,Town_of_Glenville_Schenectady_County_New_York.pdf,https://www.townofglenville.org/home/files/gecc-solar-attachments,year,2024,,Failure,240,55,9.804,0.0007125 -New York,Washington,Whitehall,town,Town_of_Whitehall_Washington_County_New_York.pdf,https://www.washingtoncountyny.gov/DocumentCenter/View/25318/Town-Whitehall-Moratorium-with-SEQRAFS,year,2024,,Failure,251,70,8.879,0.0008387499999999999 -North Dakota,Bowman,,county,Bowman_County_North_Dakota.pdf,https://bowmannd.com/wp-content/uploads/2024/12/2024-ZONING-ORDINACE.pdf,year,2024,2024,Success,247,73,10.01,0.00085625 -Ohio,Medina,,county,Medina_County_Ohio.txt,https://codelibrary.amlegal.com/codes/medina/latest/medina_oh/0-0-0-29637,year,2014,,Failure,250,51,2.362,0.000695 -Rhode Island,Kent,West Greenwich,town,Town_of_West_Greenwich_Kent_County_Rhode_Island.pdf,https://ecode360.com/33936687#33936687,year,2025,,Failure,235,47,8.547,0.00064625 -Tennessee,Cumberland,,county,Cumberland_County_Tennessee.pdf,https://www.cumberlandcountync.gov/vd-Planning/downloads/ord_amendments/county/Solar%20Farm%20__Approved__051815.pdf,year,2015,2015,Success,256,54,9.448,0.000725 -Tennessee,Greene,,county,Greene_County_Tennessee.pdf,"https://www.greenecountytngov.com/wp-content/uploads/2017/09/Greene%20County,%20TN.Zoning%20Res.Dec%202015.pdf",year,2015,2017,Failure,259,68,8.737,0.0008337500000000001 -Virginia,Southampton,,county,Southampton_County_Virginia.pdf,"https://www.southamptoncounty.org/BOS%20Agenda%202022/March%2022,%202022/No.%2015%20-%20Draft%20Ordinance%20-%20Utility%20Scale%20Solar%20Projects.pdf",year,2022,2022,Success,276,46,10.222,0.00069 -Washington,Benton,,county,Benton_County_Washington.pdf,https://bentoncountywa.municipalone.com/docview.aspx?docid=29828,year,,,Success,241,39,9.111,0.00059375 -Washington,Okanogan,,county,Okanogan_County_Washington.txt,https://www.codepublishing.com/WA/OkanoganCounty/html/OkanoganCounty17/OkanoganCounty17255.html,year,2024,,Failure,248,62,2.557,0.0007750000000000001 -Wyoming,Crook,,county,Crook_County_Wyoming.pdf,https://cms2.revize.com/revize/crookscountynew/elected_officials/commissioners/docs/Proposed_Rules_&_Regs/Wind%20and%20Solar%20Regulation%20after%20August%205%202025.pdf?t=202508192041490&t=202508192041490,year,2025,2025,Success,292,95,9.807,0.0010775 +state,county,subdivision,jurisdiction_type,file,source,feature,expected,extracted,comparison_result,prompt_tokens,response_tokens,time_taken_s,cost +Arizona,Yavapai,,county,Yavapai_County_Arizona.pdf,https://www.yavapaiaz.gov/files/sharedassets/public/v/1/development-and-permits/development-services/documents/news/sec-608-solar-facilities-approved-11-6-24-in-effect-12-4-24-watermarked.pdf,year,2024.0,2024.0,Success,274,73,11.569,0.00089 +Florida,St. Lucie,,county,St._Lucie_County_Florida.pdf,https://mcclibraryfunctions.azurewebsites.us/api/ordinanceDownload/14641/485031/pdf,year,2010.0,,Failure,244,48,11.083,0.000665 +Georgia,Bartow,,county,Bartow_County_Georgia.pdf,https://cms2.revize.com/revize/bartowga/ordinance%20updated%20thru%2011-04-20.pdf,year,2020.0,2020.0,Success,254,53,13.618,0.0007149999999999999 +Georgia,Wayne,,county,Wayne_County_Georgia.pdf,"https://mcclibraryfunctions.azurewebsites.us/api/ordinanceDownload/12423/1246775/pdf?forceDownload=true#:~:text=(j)%20All%20solar%20farms%20shall,of%20the%20International%20Fire%20Code.",year,2023.0,,Failure,283,55,24.315,0.00076625 +Illinois,Carroll,,county,Carroll_County_Illinois.pdf,https://cms9files.revize.com/carrollil/Chapter%20725%20Solar%20Amended%2005.18.2023.pdf,year,2023.0,2023.0,Success,254,51,14.486,0.0006999999999999999 +Illinois,Hamilton,,county,Hamilton_County_Illinois.pdf,https://www.hamiltoncountyil.gov/app/uploads/2022/06/Hamilton-County-IL-Commercial-Solar-Energy-Ordinance-2024-05-02.pdf,year,2024.0,2024.0,Success,260,63,41.114,0.0007975 +Illinois,Saline,,county,Saline_County_Illinois.pdf,https://energyzoning.org/sites/default/files/PDF/17165_Saline%20_20230118.pdf,year,2020.0,,Failure,3140,253,27.504,0.0058224999999999996 +Illinois,Winnebago,,county,Winnebago_County_Illinois.pdf,https://energyzoning.org/sites/default/files/PDF/17201_Winnebago%20_20220929.pdf,year,2015.0,,Failure,20491,1591,13.091,0.037546249999999996 +Iowa,Bremer,,county,Bremer_County_Iowa.pdf,https://cms7files.revize.com/bremercountyia/document_center/Building%20Zoning/Ordinances%20and%20codes/Recorded%20Ord%2022-07%20(Zoning%20Ord).pdf,year,2022.0,,Failure,272,65,113.212,0.0008275 +Kentucky,Garrard,,county,Garrard_County_Kentucky.pdf,https://garrardcountyky.gov/wp-content/uploads/2024/01/Solar-Ordinance-Second-Reading-Signed.pdf,year,2024.0,2024.0,Success,250,66,31.845,0.0008075000000000001 +Louisiana,Bossier,,parish,Bossier_Parish_Louisiana.pdf,"https://www.bossierparishla.gov/docs/default-source/police-jury/recent-ordinances/4910.pdf?sfvrsn=aa884719_3#:~:text=BE%20IT%20ORDAINED%20that%20the,Police%20Jury%2C%20excluding%20state%2Downed",year,2022.0,,Failure,299,65,13.12,0.00086125 +Maine,Oxford,Denmark,town,Town_of_Denmark_Oxford_County_Maine.pdf,https://denmarkmaine.org/uploads/zoning_ordinance_amended_02.27.2021_approved_with_5.16__8.8_additions-1-.pdf,year,2021.0,2021.0,Success,260,86,14.752,0.00097 +Maryland,Queen Anne's,,county,Queen_Anne'S_County_Maryland.pdf,https://dnr.maryland.gov/pprp/Documents/QueenAnneZoningDocs.pdf,year,2020.0,,Failure,241,52,12.58,0.00069125 +Massachusetts,Barnstable,Bourne,town,Town_of_Bourne_Barnstable_County_Massachusetts.pdf,https://www.townofbourne.com/sites/g/files/vyhlif12841/f/uploads/zoning_bylaw_2022_for_website_1.pdf,year,2022.0,2022.0,Success,255,67,12.105,0.00082125 +Massachusetts,Hampshire,Ware,town,Town_of_Ware_Hampshire_County_Massachusetts.pdf,https://cms1files.revize.com/warema/Ware%20Bylaw%20-%20Utility%20Scale%20Solar%20Revision.pdf,year,,,Success,254,53,11.115,0.0007149999999999999 +Minnesota,Carlton,,county,Carlton_County_Minnesota.pdf,https://www.carltoncountymn.gov/DocumentCenter/View/931/Renewable-Energy-Ordinance-PDF,year,2017.0,,Failure,249,46,13.536,0.0006562499999999999 +Minnesota,Chisago,,county,Chisago_County_Minnesota.pdf,https://www.chisagocountymn.gov/DocumentCenter/View/7211/Solar-Energy-Systems-Ordinance-PDF,year,2020.0,,Failure,251,56,32.907,0.00073375 +Minnesota,Clay,,county,Clay_County_Minnesota.pdf,https://claycountymn.gov/DocumentCenter/View/16228,year,,,Success,237,47,12.777,0.00064875 +New Hampshire,Grafton,Sugar Hill,town,Town_of_Sugar_Hill_Grafton_County_New_Hampshire.pdf,https://www.sugarhillnh.org/wp-content/uploads/2024/04/SH-Zoning-Ordinance-Amended-03122024.pdf,year,2024.0,2024.0,Success,252,60,12.748,0.0007650000000000001 +New Hampshire,Strafford,Middleton,town,Town_of_Middleton_Strafford_County_New_Hampshire.pdf,https://middletonnh.gov/uploads/proposed-zoning-ordinances.pdf,year,2023.0,,Failure,237,59,48.541,0.0007387499999999999 +New York,Albany,Westerlo,town,Town_of_Westerlo_Albany_County_New_York.pdf,https://townofwesterlony.com/wp-content/uploads/2021/12/LL-No-4-of-2021-Solar-Energy-Systems-FINAL-2.pdf,year,2021.0,2021.0,Success,259,73,14.447,0.0008712500000000001 +New York,Columbia,Livingston,town,Town_of_Livingston_Columbia_County_New_York.pdf,https://townoflivingston.org/wp-content/uploads/2022/06/Local-Law-2-of-2021.pdf,year,2022.0,2022.0,Success,248,104,27.194,0.00109 +New York,Erie,Boston,town,Town_of_Boston_Erie_County_New_York.pdf,https://ecode360.com/BO0557/laws/LF1193731.pdf,year,2019.0,,Failure,239,59,14.733,0.0007412499999999999 +New York,Greene,Hunter,town,Town_of_Hunter_Greene_County_New_York.pdf,https://townofhunterny.gov/wp-content/uploads/2021/06/solar-law-2021.pdf,year,2021.0,2021.0,Success,245,67,50.041,0.00080875 +New York,Greene,Windham,town,Town_of_Windham_Greene_County_New_York.pdf,https://www.townofwindhamny.com/wp-content/uploads/2022/06/Local-Law-1-of-2022-Regulating-Solar-Energy-Systems.pdf,year,2022.0,2022.0,Success,259,67,11.539,0.00082625 +New York,Livingston,Livonia,town,Town_of_Livonia_Livingston_County_New_York.txt,https://ecode360.com/38054462,year,2022.0,,Failure,231,52,5.141,0.00067875 +New York,Niagara,Hartland,town,Town_of_Hartland_Niagara_County_New_York.pdf,https://cms5.revize.com/revize/hartlandny/Businesses/Town%20of%20Hartland%20Code/Zoning%20&%20All%20Codes/HA0923-144.pdf,year,2017.0,,Failure,268,66,11.675,0.00083 +New York,Niagara,Pendleton,town,Town_of_Pendleton_Niagara_County_New_York.txt,https://ecode360.com/5125460,year,2022.0,,Failure,231,46,8.132,0.00063375 +New York,Schenectady,Glenville,town,Town_of_Glenville_Schenectady_County_New_York.pdf,https://www.townofglenville.org/home/files/gecc-solar-attachments,year,2024.0,,Failure,240,53,11.773,0.0006975 +Ohio,Medina,,county,Medina_County_Ohio.txt,https://codelibrary.amlegal.com/codes/medina/latest/medina_oh/0-0-0-29637,year,2014.0,,Failure,250,52,5.275,0.0007025 +Tennessee,Cumberland,,county,Cumberland_County_Tennessee.pdf,https://www.cumberlandcountync.gov/vd-Planning/downloads/ord_amendments/county/Solar%20Farm%20__Approved__051815.pdf,year,2015.0,2015.0,Success,256,54,12.183,0.000725 +Tennessee,Greene,,county,Greene_County_Tennessee.pdf,"https://www.greenecountytngov.com/wp-content/uploads/2017/09/Greene%20County,%20TN.Zoning%20Res.Dec%202015.pdf",year,2015.0,2015.0,Success,259,83,13.879,0.00094625 +Washington,Benton,,county,Benton_County_Washington.pdf,https://bentoncountywa.municipalone.com/docview.aspx?docid=29828,year,,,Success,241,42,12.656,0.0006162499999999999 +Washington,Okanogan,,county,Okanogan_County_Washington.txt,https://www.codepublishing.com/WA/OkanoganCounty/html/OkanoganCounty17/OkanoganCounty17255.html,year,2024.0,,Failure,248,50,5.2,0.0006850000000000001 +Wyoming,Crook,,county,Crook_County_Wyoming.pdf,https://cms2.revize.com/revize/crookscountynew/elected_officials/commissioners/docs/Proposed_Rules_&_Regs/Wind%20and%20Solar%20Regulation%20after%20August%205%202025.pdf?t=202508192041490&t=202508192041490,year,2025.0,2025.0,Success,292,79,12.703,0.0009574999999999999 diff --git a/evals/results/held_out/date_extraction_evals.json b/evals/results/held_out/date_extraction_evals.json index 3a197b690..2446c6d6d 100644 --- a/evals/results/held_out/date_extraction_evals.json +++ b/evals/results/held_out/date_extraction_evals.json @@ -15,8 +15,8 @@ "precision_95_percent_confidence_interval": "0.4093 - 0.9285", "recall_95_percent_confidence_interval": "0.1455 - 0.5190", "total_prompt_tokens": 5496, - "total_response_tokens": 1315, - "total_time_taken_s": 309.15, - "total_cost_usd": 0.0167 + "total_response_tokens": 1323, + "total_time_taken_s": 398.24, + "total_cost_usd": 0.0168 } ] diff --git a/evals/test_run_date_extraction_evals.py b/evals/test_run_date_extraction_evals.py index 36c188085..b59047298 100644 --- a/evals/test_run_date_extraction_evals.py +++ b/evals/test_run_date_extraction_evals.py @@ -100,6 +100,9 @@ def pytest_generate_tests(metafunc): else _DEV_DATASET_DIR ) cases = load_config(dataset_dir / "manifest.json5") + # Date extraction is only meaningful for enacted (Final) ordinances + # since drafts/proposals have no "adoption date" + cases = [c for c in cases if c["document_satus"].strip().lower() == "final"] metafunc.parametrize( "case", [(case, dataset_dir) for case in cases],