Skip to content

Commit 342c57c

Browse files
rajeswari1301rajeswari1301
committed
Add nightly codes sanity checker
1 parent d5a8dc9 commit 342c57c

10 files changed

Lines changed: 448 additions & 5 deletions

File tree

‎proxyserver/src/main/java/edu/suffolk/litlab/efsp/server/setup/tyler/TylerModuleSetup.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -241,6 +241,19 @@ public void preSetup() {
241241
log.info("Scheduling immediate Tyler EFM code update job.");
242242
scheduler.scheduleJob(
243243
buildJob("job-immediate-" + jurisdiction.getName()), immediateTrigger);
244+
245+
// Runs after the refresh job's immediate trigger above, so there's
246+
// something in the codes db for it to check by the time it fires.
247+
Trigger immediateCheckTrigger =
248+
TriggerBuilder.newTrigger()
249+
.withIdentity(
250+
"check-trigger-immediate-" + jurisdiction.getName(), "codes-check-group")
251+
.startNow()
252+
.withSchedule(SimpleScheduleBuilder.simpleSchedule().withIntervalInSeconds(40))
253+
.build();
254+
log.info("Scheduling immediate codes sanity check job.");
255+
scheduler.scheduleJob(
256+
buildCheckJob("check-job-immediate-" + jurisdiction.getName()), immediateCheckTrigger);
244257
}
245258
} catch (SchedulerException se) {
246259
log.error("Scheduler Exception: ", se);

‎proxyserver/src/main/java/edu/suffolk/litlab/efsp/server/utils/CodesSanityCheckJob.java‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
package edu.suffolk.litlab.efsp.server.utils;
2+
3+
import edu.suffolk.litlab.efsp.Jurisdiction;
4+
import edu.suffolk.litlab.efsp.db.DatabaseCreator;
5+
import edu.suffolk.litlab.efsp.server.logging.MDCWrappers;
6+
import edu.suffolk.litlab.efsp.tyler.ecfcodes.CodeDatabase;
7+
import edu.suffolk.litlab.efsp.tyler.ecfcodes.CodesSanityChecker;
8+
import java.sql.Connection;
9+
import java.sql.SQLException;
10+
import org.quartz.Job;
11+
import org.quartz.JobDataMap;
12+
import org.quartz.JobExecutionContext;
13+
import org.quartz.JobExecutionException;
14+
import org.slf4j.Logger;
15+
import org.slf4j.LoggerFactory;
16+
import org.slf4j.MDC;
17+
18+
public class CodesSanityCheckJob implements Job {
19+
private static Logger log = LoggerFactory.getLogger(CodesSanityCheckJob.class);
20+
21+
// Runs as a separate task so if this check goes wrong, it won't break the actual nightly database
22+
// update.
23+
public void execute(JobExecutionContext context) throws JobExecutionException {
24+
JobDataMap dataMap = context.getJobDetail().getJobDataMap();
25+
var jurisdiction = Jurisdiction.parse(dataMap.getString("TYLER_JURISDICTION"));
26+
MDC.put(MDCWrappers.OPERATION, "CodesSanityCheckJob.execute");
27+
MDC.put(MDCWrappers.USER_ID, jurisdiction.getName());
28+
29+
String pgFullUrl = dataMap.getString("POSTGRES_URL");
30+
String pgDb = dataMap.getString("POSTGRES_DB");
31+
String pgUser = dataMap.getString("POSTGRES_USERNAME");
32+
String pgPassword = dataMap.getString("POSTGRES_PASSWORD");
33+
34+
try (Connection conn =
35+
DatabaseCreator.makeSingleConnection(pgDb, pgFullUrl, pgUser, pgPassword);
36+
CodeDatabase cd = new CodeDatabase(jurisdiction, conn)) {
37+
new CodesSanityChecker(cd, jurisdiction.getName()).runAll();
38+
} catch (SQLException e) {
39+
log.error("Couldn't connect to Codes db from CodesSanityCheckJob: ", e);
40+
}
41+
MDCWrappers.removeAllMDCs();
42+
}
43+
}

‎proxyserver/src/main/java/edu/suffolk/litlab/efsp/tyler/ecfcodes/CodeSearchMatcher.java‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
package edu.suffolk.litlab.efsp.tyler.ecfcodes;
2+
3+
import edu.suffolk.litlab.efsp.ecfcodes.NameAndCode;
4+
import java.util.ArrayList;
5+
import java.util.List;
6+
import java.util.function.Predicate;
7+
8+
// Java port of filter_codes() from docassemble-EFSPIntegration's interview_logic.py.
9+
public class CodeSearchMatcher {
10+
11+
public enum ResultType {
12+
OK,
13+
NO_MATCH,
14+
AMBIGUOUS
15+
}
16+
17+
public record MatchResult(ResultType type, List<NameAndCode> matches) {}
18+
19+
private CodeSearchMatcher() {}
20+
21+
public static MatchResult filterCodes(List<NameAndCode> options, List<String> filters) {
22+
List<Predicate<NameAndCode>> attempts = new ArrayList<>();
23+
for (String filter : filters) {
24+
attempts.add(exactMatch(filter));
25+
}
26+
for (String filter : filters) {
27+
attempts.add(substringMatch(filter));
28+
}
29+
30+
for (Predicate<NameAndCode> attempt : attempts) {
31+
List<NameAndCode> matched = options.stream().filter(attempt).toList();
32+
if (!matched.isEmpty()) {
33+
return new MatchResult(matched.size() == 1 ? ResultType.OK : ResultType.AMBIGUOUS, matched);
34+
}
35+
}
36+
return new MatchResult(ResultType.NO_MATCH, List.of());
37+
}
38+
39+
private static Predicate<NameAndCode> exactMatch(String filter) {
40+
String needle = filter.toLowerCase().strip();
41+
return opt -> opt.getName() != null && opt.getName().toLowerCase().strip().equals(needle);
42+
}
43+
44+
private static Predicate<NameAndCode> substringMatch(String filter) {
45+
String needle = filter.toLowerCase();
46+
return opt -> opt.getName() != null && opt.getName().toLowerCase().contains(needle);
47+
}
48+
}

‎proxyserver/src/main/java/edu/suffolk/litlab/efsp/tyler/ecfcodes/CodesSanityChecker.java‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
package edu.suffolk.litlab.efsp.tyler.ecfcodes;
2+
3+
import edu.suffolk.litlab.efsp.ecfcodes.NameAndCode;
4+
import java.util.List;
5+
import java.util.Optional;
6+
import java.util.function.Function;
7+
import org.slf4j.Logger;
8+
import org.slf4j.LoggerFactory;
9+
10+
// Runs health checks for a single state/jurisdiction. It verifies configuration rules for every
11+
// court,
12+
// and makes sure our hardcoded code lookups actually match what is currently in the database.
13+
public class CodesSanityChecker {
14+
15+
private static final Logger log = LoggerFactory.getLogger(CodesSanityChecker.class);
16+
17+
private final CodeDatabase cd;
18+
private final String jurisdiction;
19+
20+
public CodesSanityChecker(CodeDatabase cd, String jurisdiction) {
21+
this.cd = cd;
22+
this.jurisdiction = jurisdiction;
23+
}
24+
25+
public void runAll() {
26+
checkNeverRequiredFieldsForAllCourts();
27+
checkKnownCodeLookups();
28+
}
29+
30+
private void checkNeverRequiredFieldsForAllCourts() {
31+
for (String courtCode : cd.getAllLocations()) {
32+
if (courtCode.equals("1")) {
33+
continue;
34+
}
35+
Optional<CourtLocationInfo> court = cd.getFullLocationInfo(courtCode);
36+
if (court.isEmpty()) {
37+
continue;
38+
}
39+
TylerCodesParser parser = new TylerCodesParser(cd, null, court.get(), false);
40+
parser.checkNeverRequiredFields();
41+
}
42+
}
43+
44+
private void checkKnownCodeLookups() {
45+
for (MaintainedCodeCheck check : KnownCodeLookups.ALL) {
46+
if (!check.jurisdiction().equals(jurisdiction)) {
47+
continue;
48+
}
49+
List<NameAndCode> options = fetchOptions(check);
50+
CodeSearchMatcher.MatchResult result =
51+
CodeSearchMatcher.filterCodes(options, check.filters());
52+
report(check, result);
53+
}
54+
}
55+
56+
private List<NameAndCode> fetchOptions(MaintainedCodeCheck check) {
57+
Function<String, List<NameAndCode>> fetch =
58+
courtCode ->
59+
switch (check.table()) {
60+
case CASE_CATEGORY -> cd.getCaseCategoryNames(courtCode);
61+
case CASE_TYPE ->
62+
cd.getCaseTypeNamesFor(courtCode, check.caseCategoryCode(), Optional.empty());
63+
default ->
64+
throw new UnsupportedOperationException(
65+
"No lookup wired up yet for "
66+
+ check.table()
67+
+ " - add one before seeding an entry for it");
68+
};
69+
70+
for (String court : cd.getParentList(check.courtCode())) {
71+
List<NameAndCode> options = fetch.apply(court);
72+
if (!options.isEmpty()) {
73+
return options;
74+
}
75+
}
76+
return List.of();
77+
}
78+
79+
private void report(MaintainedCodeCheck check, CodeSearchMatcher.MatchResult result) {
80+
switch (result.type()) {
81+
case OK -> {
82+
String actualCode = result.matches().get(0).getCode();
83+
if (!actualCode.equals(check.expectedDefaultCode())) {
84+
log.error(
85+
"Court {} {}: filters {} now resolve to {}, but the interview at {} expects {}",
86+
check.courtCode(),
87+
check.table(),
88+
check.filters(),
89+
actualCode,
90+
check.sourceInterviewUrl(),
91+
check.expectedDefaultCode());
92+
}
93+
}
94+
case NO_MATCH ->
95+
log.error(
96+
"Court {} {}: filters {} matched nothing, but the interview at {} expects {}",
97+
check.courtCode(),
98+
check.table(),
99+
check.filters(),
100+
check.sourceInterviewUrl(),
101+
check.expectedDefaultCode());
102+
case AMBIGUOUS ->
103+
log.error(
104+
"Court {} {}: filters {} matched more than one code ({}), but the interview at {}"
105+
+ " expects {}",
106+
check.courtCode(),
107+
check.table(),
108+
check.filters(),
109+
result.matches(),
110+
check.sourceInterviewUrl(),
111+
check.expectedDefaultCode());
112+
}
113+
}
114+
}

‎proxyserver/src/main/java/edu/suffolk/litlab/efsp/tyler/ecfcodes/KnownCodeLookups.java‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
package edu.suffolk.litlab.efsp.tyler.ecfcodes;
2+
3+
import java.util.List;
4+
5+
// Seed list, pulled from docassemble-MotionToStayEviction's efiling.yml. Add more as they come up.
6+
public class KnownCodeLookups {
7+
8+
public static final List<MaintainedCodeCheck> ALL =
9+
List.of(
10+
new MaintainedCodeCheck(
11+
"massachusetts",
12+
"appeals:acsj",
13+
MaintainedCodeCheck.CodeTable.CASE_CATEGORY,
14+
List.of("Appeals Court Single Justice - Civil", "Civil"),
15+
"8151",
16+
"https://github.com/SuffolkLITLab/docassemble-MotionToStayEviction/blob/"
17+
+ "213bd40790525bb45fbc5b012c5649325c7e0bae/docassemble/MotionToStayEviction/"
18+
+ "data/questions/efiling.yml#L33-L34",
19+
null),
20+
new MaintainedCodeCheck(
21+
"massachusetts",
22+
"appeals:acsj",
23+
MaintainedCodeCheck.CodeTable.CASE_TYPE,
24+
List.of("MAC Rule 6.0"),
25+
"12644",
26+
"https://github.com/SuffolkLITLab/docassemble-MotionToStayEviction/blob/"
27+
+ "213bd40790525bb45fbc5b012c5649325c7e0bae/docassemble/MotionToStayEviction/"
28+
+ "data/questions/efiling.yml#L36-L38",
29+
"8151"));
30+
31+
private KnownCodeLookups() {}
32+
}

‎proxyserver/src/main/java/edu/suffolk/litlab/efsp/tyler/ecfcodes/MaintainedCodeCheck.java‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
package edu.suffolk.litlab.efsp.tyler.ecfcodes;
2+
3+
import java.util.List;
4+
5+
// One entry in the hand-maintained list of interview code lookups.
6+
// Each entry mirrors a _filters/_default pair from a real interview's efiling.yml.
7+
public record MaintainedCodeCheck(
8+
String jurisdiction,
9+
String courtCode,
10+
CodeTable table,
11+
List<String> filters,
12+
String expectedDefaultCode,
13+
String sourceInterviewUrl,
14+
String caseCategoryCode) {
15+
16+
public enum CodeTable {
17+
CASE_CATEGORY,
18+
CASE_TYPE,
19+
FILING_TYPE,
20+
DOCUMENT_TYPE,
21+
MOTION_TYPE,
22+
FILING_COMPONENT
23+
}
24+
}

‎proxyserver/src/main/java/edu/suffolk/litlab/efsp/tyler/ecfcodes/TylerCodesParser.java‎

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -186,15 +186,28 @@ public Result<FilingCode, CodeError> vetFilingType(
186186
});
187187
}
188188

189+
private static final List<String> neverRequiredFields =
190+
List.of("PartyNameSuffix", "PartyMiddleName");
191+
192+
public List<String> checkNeverRequiredFields() {
193+
List<String> offendingFields = new ArrayList<>();
194+
for (String fieldCode : neverRequiredFields) {
195+
DataFieldRow row = allDataFields.getFieldRow(fieldCode);
196+
if (row.isrequired) {
197+
log.error(
198+
"DEV WARNING: Court {}: {} shouldn't ever be required, but this court has it set that way",
199+
this.court.code,
200+
fieldCode);
201+
offendingFields.add(fieldCode);
202+
}
203+
}
204+
return offendingFields;
205+
}
206+
189207
public Result<String, CodeError> vetSuffix(Optional<String> maybeSuffix) {
190208
DataFieldRow suffixRow = allDataFields.getFieldRow("PartyNameSuffix");
191209
if (suffixRow.isvisible) {
192210
List<NameAndCode> suffixes = cd.getNameSuffixes(this.court.code);
193-
if ((maybeSuffix.isEmpty() || maybeSuffix.get().isBlank()) && suffixRow.isrequired) {
194-
log.error(
195-
"DEV WARNING: Court {}: WHY would you ever require a suffix? There aren't empty suffix codes at all.",
196-
this.court.code);
197-
}
198211
String suffix = maybeSuffix.orElse("");
199212
Optional<NameAndCode> suffixMatch =
200213
suffixes.stream().filter(s -> s.getName().equalsIgnoreCase(suffix)).findFirst();

‎proxyserver/src/test/java/edu/suffolk/litlab/efsp/server/ecf4/tyler/TylerCodesParserTest.java‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -470,6 +470,29 @@ public void testFilingComponents() {
470470
assertThat(resAgain).containsErr(new NoMatchingCode("333", List.of()));
471471
}
472472

473+
@Test
474+
public void testCheckNeverRequiredFieldsNoneRequired() {
475+
when(dataFields.getFieldRow("PartyNameSuffix"))
476+
.thenReturn(
477+
new DataFieldRow("1", "Suffix", true, false, "", "", "", "", "", "", false, "01"));
478+
when(dataFields.getFieldRow("PartyMiddleName"))
479+
.thenReturn(
480+
new DataFieldRow("2", "Middle Name", true, false, "", "", "", "", "", "", false, "01"));
481+
assertThat(((TylerCodesParser) parser).checkNeverRequiredFields()).isEmpty();
482+
}
483+
484+
@Test
485+
public void testCheckNeverRequiredFieldsSuffixRequired() {
486+
when(dataFields.getFieldRow("PartyNameSuffix"))
487+
.thenReturn(
488+
new DataFieldRow("1", "Suffix", true, true, "", "", "", "", "", "", false, "01"));
489+
when(dataFields.getFieldRow("PartyMiddleName"))
490+
.thenReturn(
491+
new DataFieldRow("2", "Middle Name", true, false, "", "", "", "", "", "", false, "01"));
492+
assertThat(((TylerCodesParser) parser).checkNeverRequiredFields())
493+
.containsExactly("PartyNameSuffix");
494+
}
495+
473496
DocumentTypeTableRow docType =
474497
new DocumentTypeTableRow("4444", null, filingCode.code, "false", null, null, null);
475498

0 commit comments

Comments
 (0)