Repository navigation
docs: update quickstart for v0.3.1 - self-healing, CI integration, ta… #75
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| branches: [main] | |
| jobs: | |
| test: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| python-version: ["3.11", "3.12"] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install llm-failure-atlas | |
| run: | | |
| cd .. | |
| git clone https://github.com/kiyoshisasano/llm-failure-atlas.git | |
| cd llm-failure-atlas | |
| pip install -e . | |
| - name: Install agent-failure-debugger | |
| run: pip install -e . | |
| - name: Regression test (10 examples) | |
| run: | | |
| PASS=0 | |
| FAIL=0 | |
| for dir in ../llm-failure-atlas/examples/*/; do | |
| name=$(basename "$dir") | |
| mf="$dir/matcher_output.json" | |
| ef="$dir/expected_debugger_output.json" | |
| [ -f "$mf" ] && [ -f "$ef" ] || continue | |
| python -m agent_failure_debugger.main "$mf" > /tmp/actual.json 2>&1 | |
| python -c " | |
| import json, sys | |
| with open('/tmp/actual.json') as f: actual = json.load(f) | |
| with open('$ef') as f: expected = json.load(f) | |
| got = [r.get('id') for r in actual.get('root_ranking', [])] | |
| exp = [r.get('id') for r in expected.get('root_ranking', [])] | |
| if got == exp: | |
| print('PASS $name') | |
| else: | |
| print(f'FAIL $name: expected {exp}, got {got}') | |
| sys.exit(1) | |
| " | |
| if [ $? -eq 0 ]; then | |
| PASS=$((PASS+1)) | |
| else | |
| FAIL=$((FAIL+1)) | |
| fi | |
| done | |
| echo "Results: $PASS passed, $FAIL failed" | |
| [ "$FAIL" -eq 0 ] || exit 1 | |
| - name: Pipeline test | |
| run: | | |
| python -c " | |
| import json | |
| from agent_failure_debugger.pipeline import run_pipeline | |
| with open('../llm-failure-atlas/examples/simple/matcher_output.json') as f: | |
| matcher_output = json.load(f) | |
| if isinstance(matcher_output, dict): | |
| matcher_output = [matcher_output] | |
| result = run_pipeline(matcher_output, use_learning=True) | |
| s = result['summary'] | |
| print(f'Root cause: {s[\"root_cause\"]}') | |
| print(f'Fixes: {s[\"fix_count\"]}') | |
| assert s.get('root_cause'), 'Pipeline did not produce root cause' | |
| print('OK: pipeline test passed') | |
| " | |
| - name: Labels coverage check | |
| run: | | |
| python -c " | |
| import yaml | |
| from pathlib import Path | |
| from agent_failure_debugger.labels import FAILURE_MAP, SIGNAL_MAP | |
| from llm_failure_atlas.resource_loader import get_graph_path, get_patterns_dir | |
| # Check FAILURE_MAP covers all graph nodes | |
| g = yaml.safe_load(open(str(get_graph_path()), encoding='utf-8')) | |
| missing_failures = [n['id'] for n in g['nodes'] if n['id'] not in FAILURE_MAP] | |
| if missing_failures: | |
| print(f'FAIL: missing from FAILURE_MAP: {missing_failures}') | |
| exit(1) | |
| print(f'OK: FAILURE_MAP covers all {len(g[\"nodes\"])} nodes') | |
| # Check SIGNAL_MAP covers all signals | |
| signals_in_yaml = set() | |
| for f in sorted(Path(get_patterns_dir()).glob('*.yaml')): | |
| p = yaml.safe_load(f.read_text(encoding='utf-8')) | |
| for rule in p.get('signal_extraction', {}).get('rules', []): | |
| signals_in_yaml.add(rule['signal']) | |
| missing_signals = signals_in_yaml - set(SIGNAL_MAP.keys()) | |
| if missing_signals: | |
| print(f'FAIL: missing from SIGNAL_MAP: {sorted(missing_signals)}') | |
| exit(1) | |
| print(f'OK: SIGNAL_MAP covers all {len(signals_in_yaml)} signals') | |
| " |