Skip to content

Commit abe490d

Browse files
Copilotjonathangoeke
Copilot
and
jonathangoeke
authored
Apply remaining changes
Co-authored-by: jonathangoeke <10560051+jonathangoeke@users.noreply.github.com>
1 parent bcc12a5 commit abe490d

9 files changed

Lines changed: 27 additions & 669 deletions

‎tests/conftest.py‎

Lines changed: 3 additions & 108 deletions
Original file line numberDiff line numberDiff line change
@@ -1,108 +1,3 @@
1-
"""Shared helpers and fixtures for the xpore test suite.
2-
3-
The functions here build the small synthetic inputs the tests need:
4-
5-
* ``make_eventalign_line`` / ``make_eventalign_str`` -- eventalign text, the input to
6-
``dataprep.combine``.
7-
* ``make_events_array`` -- a single read's event array (what ``combine`` returns and what
8-
``preprocess_tx`` consumes), built with fixed string widths so several reads can be
9-
``np.concatenate``-d together.
10-
* ``read_json_output`` -- parse the ``data.json`` that ``preprocess_tx`` writes.
11-
12-
The ``locks`` and ``out_paths`` fixtures provide the two arguments ``preprocess_tx`` writes
13-
through, so the tests never touch multiprocessing.
14-
"""
15-
import json
16-
import threading
17-
18-
import numpy as np
19-
import pytest
20-
21-
22-
# Column order of a nanopolish/f5c eventalign.txt, exactly as combine() parses it.
23-
EVENTALIGN_COLUMNS = [
24-
'contig', 'position', 'reference_kmer', 'read_index', 'strand',
25-
'event_index', 'event_level_mean', 'event_stdv', 'event_length',
26-
'model_kmer', 'model_mean', 'model_stdv', 'standardized_level',
27-
'start_idx', 'end_idx',
28-
]
29-
30-
# Sensible defaults for one eventalign row; override any field per call. With start_idx=0,
31-
# end_idx=5 the per-row length is 5, so a lone row's norm_mean == its event_level_mean.
32-
_EVENTALIGN_DEFAULTS = {
33-
'contig': 'ENST1',
34-
'position': 100,
35-
'reference_kmer': 'GGACT',
36-
'read_index': 0,
37-
'strand': 't',
38-
'event_index': 1,
39-
'event_level_mean': 120.0,
40-
'event_stdv': 2.0,
41-
'event_length': 0.005,
42-
'model_kmer': 'GGACT',
43-
'model_mean': 120.0,
44-
'model_stdv': 1.5,
45-
'standardized_level': 0.5,
46-
'start_idx': 0,
47-
'end_idx': 5,
48-
}
49-
50-
51-
52-
#helper functions for the combine() tests (in test_combine.py)
53-
def make_eventalign_line(**overrides):
54-
"""Return one tab-separated eventalign row (no trailing newline)."""
55-
row = dict(_EVENTALIGN_DEFAULTS, **overrides)
56-
return '\t'.join(str(row[col]) for col in EVENTALIGN_COLUMNS)
57-
58-
def make_eventalign_str(rows):
59-
"""Join a list of row-override dicts into one eventalign string (what combine expects)."""
60-
return '\n'.join(make_eventalign_line(**row) for row in rows)
61-
62-
63-
64-
#helper functions for the preprocess_tx() tests (in test_preprocess_tx.py)
65-
def _events_dtype(kmer_col):
66-
# Matches combine()'s output fields. Fixed string widths let arrays for different reads
67-
# be np.concatenate-d inside preprocess_tx.
68-
return np.dtype([
69-
('transcript_id', '<U15'),
70-
('transcriptomic_position', '<i8'),
71-
(kmer_col, '<U5'),
72-
('norm_mean', '<f8'),
73-
])
74-
75-
def make_events_array(rows, kmer_col='reference_kmer', transcript_id='tx1'):
76-
"""Build one read's event array.
77-
78-
``rows`` is a list of ``(transcriptomic_position, kmer, norm_mean)`` tuples.
79-
"""
80-
records = [(transcript_id, pos, kmer, norm_mean) for pos, kmer, norm_mean in rows]
81-
return np.array(records, dtype=_events_dtype(kmer_col))
82-
83-
def read_json_output(json_path):
84-
"""Parse preprocess_tx's data.json into ``{tx_id: {pos: {kmer: [values]}}}``.
85-
86-
NOTE: JSON object keys are always strings, so genomic positions come back as ``str``
87-
(e.g. ``'102'``), not ``int``.
88-
"""
89-
result = {}
90-
with open(json_path) as f:
91-
for line in f:
92-
line = line.strip()
93-
if line:
94-
result.update(json.loads(line))
95-
return result
96-
97-
#Fixtures for the preprocess_tx() tests (test_preprocess_tx.py, test_xpore_v2_1_back_compatibility.py).
98-
@pytest.fixture
99-
def locks():
100-
"""The four locks preprocess_tx uses as context managers."""
101-
return {name: threading.Lock() for name in ('json', 'index', 'readcount', 'log')}
102-
103-
104-
@pytest.fixture
105-
def out_paths(tmp_path):
106-
"""The four output files preprocess_tx appends to (created lazily in append mode)."""
107-
return {name: str(tmp_path / f'data.{name}')
108-
for name in ('json', 'index', 'readcount', 'log')}
1+
version https://git-lfs.github.com/spec/v1
2+
oid sha256:a8b3a55b23dfcc6590319dfe4c825e3a2d8522fada28e0449c137d54be886d06
3+
size 3869

‎tests/fixtures/_regenerate_combine_v2_1.py‎

Lines changed: 3 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -1,67 +1,3 @@
1-
"""Regenerate the xPore v2.1 reference fixture for combine() on the transcriptome path.
2-
3-
`combine_transcriptome_v2_1.json` freezes combine()'s output on a fixed transcriptome
4-
eventalign input, captured from xPore v2.1 (the pre-PR upstream version). The test
5-
(../test_xpore_v2_1_back_compatibility.py) asserts the current (v2.2) code reproduces it
6-
byte-for-byte -- a regression guard that the v2.2 changes did not alter transcriptome
7-
(reference_kmer) behaviour.
8-
9-
The committed fixture was captured from upstream/master (GoekeLab/xpore, xPore v2.1) via a
10-
git worktree:
11-
12-
git worktree add /tmp/xpore-v2.1 upstream/master
13-
PYTHONPATH=/tmp/xpore-v2.1 python tests/fixtures/_regenerate_combine_v2_1.py
14-
git worktree remove /tmp/xpore-v2.1
15-
16-
Point PYTHONPATH at whichever checkout you want as the reference; the script prints the actual
17-
xpore it imported so you can confirm the source. (Underscore prefix keeps pytest from
18-
collecting this as a test module.)
19-
"""
20-
import json
21-
import os
22-
23-
import xpore
24-
from xpore.scripts.dataprep import combine
25-
26-
# A small but non-trivial transcriptome eventalign snippet for ONE read (read_index 0):
27-
# reference_kmer == model_kmer throughout (forward transcriptome alignment), with position
28-
# 100 carrying two events so the length-weighted merge is exercised.
29-
INPUT_EVENTALIGN = "\n".join([
30-
"ENST1\t100\tGGACT\t0\tt\t1\t120.5\t2.0\t0.01\tGGACT\t120.0\t1.5\t0.5\t1000\t1005",
31-
"ENST1\t100\tGGACT\t0\tt\t2\t118.0\t2.1\t0.01\tGGACT\t120.0\t1.5\t0.4\t1005\t1008",
32-
"ENST1\t101\tGACTA\t0\tt\t3\t95.0\t1.8\t0.01\tGACTA\t95.5\t1.4\t0.3\t1008\t1013",
33-
"ENST1\t102\tACTAG\t0\tt\t4\t104.5\t1.9\t0.01\tACTAG\t104.0\t1.6\t0.2\t1013\t1018",
34-
])
35-
36-
37-
def main():
38-
result = combine(INPUT_EVENTALIGN)
39-
# xPore v2.1 combine returns just the array; v2.2 returns (array, kmer_col).
40-
np_events = result[0] if isinstance(result, tuple) else result
41-
kmer_field = np_events.dtype.names[2] # 'reference_kmer' on the transcriptome path
42-
43-
expected = [
44-
[str(row["transcript_id"]), int(row["transcriptomic_position"]),
45-
str(row[kmer_field]), float(row["norm_mean"])]
46-
for row in np_events
47-
]
48-
49-
fixture = {
50-
"_comment": "xPore v2.1 reference output captured from upstream/master; regenerate via _regenerate_combine_v2_1.py.",
51-
"input_eventalign": INPUT_EVENTALIGN,
52-
"kmer_col": kmer_field,
53-
"expected": expected,
54-
}
55-
56-
out_path = os.path.join(os.path.dirname(__file__), "combine_transcriptome_v2_1.json")
57-
with open(out_path, "w") as f:
58-
json.dump(fixture, f, indent=2)
59-
f.write("\n")
60-
61-
print("imported xpore from:", xpore.__file__)
62-
print("wrote:", out_path)
63-
print(json.dumps(expected, indent=2))
64-
65-
66-
if __name__ == "__main__":
67-
main()
1+
version https://git-lfs.github.com/spec/v1
2+
oid sha256:d4578e0dc0e3fc473d772c45c81bb51022e8607d555e403647e1927fd16719c1
3+
size 2754

‎tests/fixtures/_regenerate_data_json_v2_1.py‎

Lines changed: 3 additions & 112 deletions
Original file line numberDiff line numberDiff line change
@@ -1,112 +1,3 @@
1-
"""Regenerate the xPore v2.1 data.json reference fixture (transcriptome path).
2-
3-
`data_json_transcriptome_v2_1.json` freezes the data.json that combine() + preprocess_tx()
4-
produce on a fixed multi-read transcriptome input, captured from xPore v2.1 (the pre-PR
5-
upstream version). The test
6-
(../test_xpore_v2_1_back_compatibility.py::test_preprocess_tx_data_json_matches_v2_1)
7-
asserts the current (v2.2) code reproduces it.
8-
9-
The committed fixture was captured from upstream/master (GoekeLab/xpore, xPore v2.1) via a
10-
git worktree:
11-
12-
git worktree add /tmp/xpore-v2.1 upstream/master
13-
PYTHONPATH=/tmp/xpore-v2.1 python tests/fixtures/_regenerate_data_json_v2_1.py
14-
git worktree remove /tmp/xpore-v2.1
15-
16-
Handles both the v2.1 and v2.2 combine()/preprocess_tx() signatures, so it can be pointed at
17-
either version; it prints the xpore it imported so you can confirm the source. (Underscore
18-
prefix keeps pytest from collecting this as a test module.)
19-
"""
20-
import inspect
21-
import json
22-
import os
23-
import tempfile
24-
import threading
25-
26-
import xpore
27-
from xpore.scripts.dataprep import combine, preprocess_tx
28-
29-
TX_ID = "ENST1"
30-
31-
32-
def _line(pos, kmer, read_index, event_index, mean, start, end):
33-
# One eventalign row (reference_kmer == model_kmer, i.e. forward transcriptome alignment).
34-
return "\t".join(str(x) for x in [
35-
"ENST1", pos, kmer, read_index, "t", event_index, mean, 2.0, 0.01,
36-
kmer, mean, 1.5, 0.5, start, end,
37-
])
38-
39-
40-
# Three reads with overlapping positions, so multiple reads land on the same site and
41-
# preprocess_tx's per-position grouping is exercised. Each read spans >1 position (size > 1)
42-
# so the caller's inclusion rule keeps it.
43-
READS = [
44-
{"read_index": 0, "events": "\n".join([
45-
_line(100, "GGACT", 0, 1, 120.0, 1000, 1005),
46-
_line(101, "GACTA", 0, 2, 95.0, 1005, 1010),
47-
_line(102, "ACTAG", 0, 3, 104.0, 1010, 1015),
48-
])},
49-
{"read_index": 1, "events": "\n".join([
50-
_line(100, "GGACT", 1, 1, 121.0, 2000, 2005),
51-
_line(101, "GACTA", 1, 2, 96.0, 2005, 2010),
52-
_line(102, "ACTAG", 1, 3, 105.0, 2010, 2015),
53-
])},
54-
{"read_index": 2, "events": "\n".join([
55-
_line(101, "GACTA", 2, 1, 94.0, 3000, 3005),
56-
_line(102, "ACTAG", 2, 2, 106.0, 3005, 3010),
57-
_line(103, "CTAGT", 2, 3, 110.0, 3010, 3015),
58-
])},
59-
]
60-
61-
62-
def _run_combine(events):
63-
result = combine(events) # v2.1 returns array; v2.2 returns (array, kmer_col)
64-
return result[0] if isinstance(result, tuple) else result
65-
66-
67-
def _call_preprocess_tx(tx_id, data_dict, kmer_col, out_paths, locks):
68-
# v2.2 added kmer_col + readcount_max params; v2.1 has neither. readcount_max=None on v2.2
69-
# matches v2.1 (no per-site cap), so both reshape the same reads.
70-
if "kmer_col" in inspect.signature(preprocess_tx).parameters:
71-
preprocess_tx(tx_id, data_dict, kmer_col, None, out_paths, locks)
72-
else:
73-
preprocess_tx(tx_id, data_dict, out_paths, locks)
74-
75-
76-
def main():
77-
data_dict, kmer_col = {}, "reference_kmer"
78-
for read in READS:
79-
np_events = _run_combine(read["events"])
80-
kmer_col = np_events.dtype.names[2]
81-
if np_events.size > 1: # mirror the caller's inclusion rule
82-
data_dict[read["read_index"]] = np_events
83-
84-
with tempfile.TemporaryDirectory() as tmp:
85-
out_paths = {name: os.path.join(tmp, "data.%s" % name)
86-
for name in ("json", "index", "readcount", "log")}
87-
locks = {name: threading.Lock() for name in out_paths}
88-
_call_preprocess_tx(TX_ID, data_dict, kmer_col, out_paths, locks)
89-
expected = {}
90-
with open(out_paths["json"]) as f:
91-
for line in f:
92-
if line.strip():
93-
expected.update(json.loads(line))
94-
95-
fixture = {
96-
"_comment": "xPore v2.1 data.json reference captured from upstream/master; regenerate via _regenerate_data_json_v2_1.py.",
97-
"tx_id": TX_ID,
98-
"reads": READS,
99-
"expected": expected,
100-
}
101-
out_path = os.path.join(os.path.dirname(__file__), "data_json_transcriptome_v2_1.json")
102-
with open(out_path, "w") as f:
103-
json.dump(fixture, f, indent=2)
104-
f.write("\n")
105-
106-
print("imported xpore from:", xpore.__file__)
107-
print("wrote:", out_path)
108-
print(json.dumps(expected, indent=2))
109-
110-
111-
if __name__ == "__main__":
112-
main()
1+
version https://git-lfs.github.com/spec/v1
2+
oid sha256:d56135ef5b4215f68e275eaec48ca0873a9a4393bb7c3f7cfa3759fff9218aa3
3+
size 4340

‎tests/fixtures/combine_transcriptome_v2_1.json‎

Lines changed: 3 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,3 @@
1-
{
2-
"_comment": "xPore v2.1 reference output captured from upstream/master; regenerate via _regenerate_combine_v2_1.py.",
3-
"input_eventalign": "ENST1\t100\tGGACT\t0\tt\t1\t120.5\t2.0\t0.01\tGGACT\t120.0\t1.5\t0.5\t1000\t1005\nENST1\t100\tGGACT\t0\tt\t2\t118.0\t2.1\t0.01\tGGACT\t120.0\t1.5\t0.4\t1005\t1008\nENST1\t101\tGACTA\t0\tt\t3\t95.0\t1.8\t0.01\tGACTA\t95.5\t1.4\t0.3\t1008\t1013\nENST1\t102\tACTAG\t0\tt\t4\t104.5\t1.9\t0.01\tACTAG\t104.0\t1.6\t0.2\t1013\t1018",
4-
"kmer_col": "reference_kmer",
5-
"expected": [
6-
[
7-
"ENST1",
8-
102,
9-
"GGACT",
10-
119.6
11-
],
12-
[
13-
"ENST1",
14-
103,
15-
"GACTA",
16-
95.0
17-
],
18-
[
19-
"ENST1",
20-
104,
21-
"ACTAG",
22-
104.5
23-
]
24-
]
25-
}
1+
version https://git-lfs.github.com/spec/v1
2+
oid sha256:35d97fbab80c9ed355eb790348cfc72eb38c6f4b9369f5de3cabaeaf5bd1f566
3+
size 722

‎tests/fixtures/data_json_transcriptome_v2_1.json‎

Lines changed: 3 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -1,47 +1,3 @@
1-
{
2-
"_comment": "xPore v2.1 data.json reference captured from upstream/master; regenerate via _regenerate_data_json_v2_1.py.",
3-
"tx_id": "ENST1",
4-
"reads": [
5-
{
6-
"read_index": 0,
7-
"events": "ENST1\t100\tGGACT\t0\tt\t1\t120.0\t2.0\t0.01\tGGACT\t120.0\t1.5\t0.5\t1000\t1005\nENST1\t101\tGACTA\t0\tt\t2\t95.0\t2.0\t0.01\tGACTA\t95.0\t1.5\t0.5\t1005\t1010\nENST1\t102\tACTAG\t0\tt\t3\t104.0\t2.0\t0.01\tACTAG\t104.0\t1.5\t0.5\t1010\t1015"
8-
},
9-
{
10-
"read_index": 1,
11-
"events": "ENST1\t100\tGGACT\t1\tt\t1\t121.0\t2.0\t0.01\tGGACT\t121.0\t1.5\t0.5\t2000\t2005\nENST1\t101\tGACTA\t1\tt\t2\t96.0\t2.0\t0.01\tGACTA\t96.0\t1.5\t0.5\t2005\t2010\nENST1\t102\tACTAG\t1\tt\t3\t105.0\t2.0\t0.01\tACTAG\t105.0\t1.5\t0.5\t2010\t2015"
12-
},
13-
{
14-
"read_index": 2,
15-
"events": "ENST1\t101\tGACTA\t2\tt\t1\t94.0\t2.0\t0.01\tGACTA\t94.0\t1.5\t0.5\t3000\t3005\nENST1\t102\tACTAG\t2\tt\t2\t106.0\t2.0\t0.01\tACTAG\t106.0\t1.5\t0.5\t3005\t3010\nENST1\t103\tCTAGT\t2\tt\t3\t110.0\t2.0\t0.01\tCTAGT\t110.0\t1.5\t0.5\t3010\t3015"
16-
}
17-
],
18-
"expected": {
19-
"ENST1": {
20-
"102": {
21-
"GGACT": [
22-
120.0,
23-
121.0
24-
]
25-
},
26-
"103": {
27-
"GACTA": [
28-
95.0,
29-
96.0,
30-
94.0
31-
]
32-
},
33-
"104": {
34-
"ACTAG": [
35-
104.0,
36-
105.0,
37-
106.0
38-
]
39-
},
40-
"105": {
41-
"CTAGT": [
42-
110.0
43-
]
44-
}
45-
}
46-
}
47-
}
1+
version https://git-lfs.github.com/spec/v1
2+
oid sha256:5874541bad968372859b64d28a546062951c1c471c0ac6fb8434e629de50d105
3+
size 1455

0 commit comments

Comments
 (0)