From 718e67ef9a233a8d3bbf77d019a981fce3f3f4d6 Mon Sep 17 00:00:00 2001 From: kaeldai Date: Mon, 2 Jun 2025 12:15:02 -0700 Subject: [PATCH 1/6] adding initial modification for nwb output of spikes and lfp --- bmtk/simulator/bionet/modules/ecp.py | 94 ++++++++++++++++++- .../simulator/bionet/modules/record_spikes.py | 2 +- bmtk/simulator/bionet/pointprocesscell.py | 2 +- .../spike_trains/spikes_file_writers.py | 24 +++-- .../bio_14cells/config.simulation_ecp.json | 63 +++++++++++++ 5 files changed, 172 insertions(+), 13 deletions(-) create mode 100644 examples/bio_14cells/config.simulation_ecp.json diff --git a/bmtk/simulator/bionet/modules/ecp.py b/bmtk/simulator/bionet/modules/ecp.py index 3bce7a55b..3fb8118bd 100644 --- a/bmtk/simulator/bionet/modules/ecp.py +++ b/bmtk/simulator/bionet/modules/ecp.py @@ -26,6 +26,9 @@ import pandas as pd from neuron import h import numpy as np +from datetime import datetime +from dateutil.tz import tzlocal +from uuid import uuid4 from bmtk.simulator.bionet.modules.sim_module import SimulatorMod from bmtk.utils.sonata.utils import add_hdf5_magic, add_hdf5_version @@ -37,13 +40,12 @@ class EcpMod(SimulatorMod): - def __init__(self, tmp_dir, file_name, electrode_positions, contributions_dir=None, cells=None, variable_name='v', + def __init__(self, tmp_dir, file_name, electrode_positions, file_name_nwb=None, contributions_dir=None, cells=None, variable_name='v', electrode_channels=None, cell_bounds=None, minimum_distance=None): self._ecp_output = file_name if os.path.isabs(file_name) else os.path.join(tmp_dir, file_name) self._positions_file = electrode_positions self._tmp_outputdir = tmp_dir - if contributions_dir is not None: self._save_individ_cells = True self._contributions_dir = contributions_dir if os.path.isabs(contributions_dir) else os.path.join(tmp_dir, contributions_dir) @@ -79,6 +81,11 @@ def __init__(self, tmp_dir, file_name, electrode_positions, contributions_dir=No self._tmp_ecp_handle = None # self._tmp_ecp_dataset = None + self._nwb_path = None + if file_name_nwb: + self._nwb_path = file_name_nwb if os.path.isabs(file_name_nwb) else os.path.join(tmp_dir, file_name_nwb) + + self._local_gids = [] def _get_tmp_fname(self, rank): @@ -239,6 +246,89 @@ def finalize(self, sim): self._delete_tmp_files() pc.barrier() + if self._nwb_path: + convert2nwb(self._nwb_path, self._ecp_output, self._positions_file) + + + +def convert2nwb(nwb_path, orig_hdf5_lfp, electrodes_file): + import pynwb + + if os.path.exists(nwb_path): + io = pynwb.NWBHDF5IO(nwb_path, 'a') + nwbfile = io.read() + print('appending lfp') + else: + io = pynwb.NWBHDF5IO(nwb_path, "w") + nwbfile = pynwb.NWBFile( + session_start_time=datetime.now(tzlocal()), + session_description="test time", + identifier=str(uuid4()), + ) + print('new nwb file') + + electrodes_metdata_df = pd.read_csv(electrodes_file, sep=' ').set_index('channel') + + device = nwbfile.create_device( + name="ecp electrode matrix" + ) + + electrode_group = nwbfile.create_electrode_group( + name="array 1", + device=device, + location="Brain", + description="desc" + ) + + # electrodes_file + + nwbfile.add_electrode_column(name="channel_id", description="label of electrode") + with h5py.File(orig_hdf5_lfp, 'r') as orig_h5: + channels = orig_h5['ecp/channel_id'][()] + for chan in channels: + chan_data = electrodes_metdata_df.loc[chan] + # print(chan_data.get('location', 'None')) + # print(chan_data.get('x_pos', 'Not Avail')) + nwbfile.add_electrode( + group=electrode_group, + channel_id=chan, + # label="1", + location=chan_data.get('location', 'None'), + x=chan_data.get('x_pos', 'Not Avail'), + y=chan_data.get('y_pos', 'Not Avail'), + z=chan_data.get('z_pos', 'Not Avail'), + ) + electrode_region_table = nwbfile.create_electrode_table_region( + region=list(channels), + description="all electrodes", + ) + + time = orig_h5['ecp/time'][()] + rate = (time[1] - time[0])/time[2] + lfp_electrical_series = pynwb.ecephys.ElectricalSeries( + name="ElectricalSeries", + description="LFP data", + data=orig_h5['ecp/data'][()], + # filtering='Low-pass filter at 300 Hz', + electrodes=electrode_region_table, + starting_time=time[0], + rate=rate, + ) + + nwbfile.add_acquisition(lfp_electrical_series) + + lfp = pynwb.ecephys.LFP(lfp_electrical_series) + + ecephys_module = nwbfile.create_processing_module( + name="ecephys", + description="processed extracellular electrophysiology data", + ) + ecephys_module.add(lfp) + + io.write(nwbfile) + + # print(nwb_path) + class RecXElectrode(object): """Extracellular electrode diff --git a/bmtk/simulator/bionet/modules/record_spikes.py b/bmtk/simulator/bionet/modules/record_spikes.py index 61a7da894..1b0f6683a 100644 --- a/bmtk/simulator/bionet/modules/record_spikes.py +++ b/bmtk/simulator/bionet/modules/record_spikes.py @@ -107,7 +107,7 @@ def finalize(self, sim): pc.barrier() if self._save_nwb: - self._spike_writer.to_nwb(self._nwb_fname, sort_order=self._sort_order) + self._spike_writer.to_nwb(self._nwb_fname, mode='a', sort_order=self._sort_order) pc.barrier() self._spike_writer.close() diff --git a/bmtk/simulator/bionet/pointprocesscell.py b/bmtk/simulator/bionet/pointprocesscell.py index 4cf38b254..22a0c3b07 100644 --- a/bmtk/simulator/bionet/pointprocesscell.py +++ b/bmtk/simulator/bionet/pointprocesscell.py @@ -73,7 +73,7 @@ def set_im_ptr(self): def set_syn_connection(self, edge_prop, src_node, stim=None): syn_params = edge_prop.dynamics_params - nsyns = edge_prop.nsyns + nsyns = int(edge_prop.nsyns) delay = edge_prop.delay syn_weight = edge_prop.syn_weight(src_node, self._node) diff --git a/bmtk/utils/reports/spike_trains/spikes_file_writers.py b/bmtk/utils/reports/spike_trains/spikes_file_writers.py index 96dd7b9ac..76dcf0cbb 100644 --- a/bmtk/utils/reports/spike_trains/spikes_file_writers.py +++ b/bmtk/utils/reports/spike_trains/spikes_file_writers.py @@ -149,7 +149,7 @@ def write_csv_itr(path, spiketrain_reader, mode='w', sort_order=SortOrder.none, comm_barrier() -def write_nwb(path, spiketrain_reader, mode='w', include_population=True, units='ms', **kwargs): +def write_nwb(path, spiketrain_reader, mode='a', include_population=True, units='ms', **kwargs): import pynwb path_dir = os.path.dirname(path) @@ -159,12 +159,18 @@ def write_nwb(path, spiketrain_reader, mode='w', include_population=True, units= if MPI_rank == 0: # Last checked pynwb doesn't support writing on multiple cores, must let first core do all the # writing to NWB. - nwbfile = pynwb.NWBFile( - session_description='BMTK {} generated NWB spikes file'.format(bmtk.__version__), - identifier='Generated in-silico, no session id', # TODO: No idea what to put here? - session_start_time=datetime.now().astimezone(), - # experiment_description=str(session.experiment_metadata['experiment_id']) - ) + + if os.path.exists(path) and mode != 'w': + io = pynwb.NWBHDF5IO(path, 'a') + nwbfile = io.read() + else: + io = pynwb.NWBHDF5IO(path, mode) + nwbfile = pynwb.NWBFile( + session_description='BMTK {} generated NWB spikes file'.format(bmtk.__version__), + identifier='Generated in-silico, no session id', # TODO: No idea what to put here? + session_start_time=datetime.now().astimezone(), + # experiment_description=str(session.experiment_metadata['experiment_id']) + ) if include_population: nwbfile.add_unit_column(name="population", description="node population identifier") @@ -187,7 +193,7 @@ def write_nwb(path, spiketrain_reader, mode='w', include_population=True, units= node_id = int(node_id) add_unit(node_id, population, spikes_times) - with pynwb.NWBHDF5IO(path, mode) as io: - io.write(nwbfile) + # with pynwb.NWBHDF5IO(path, mode) as io: + io.write(nwbfile) comm_barrier() diff --git a/examples/bio_14cells/config.simulation_ecp.json b/examples/bio_14cells/config.simulation_ecp.json new file mode 100644 index 000000000..b8a56313d --- /dev/null +++ b/examples/bio_14cells/config.simulation_ecp.json @@ -0,0 +1,63 @@ +{ + "manifest": { + "$BASE_DIR": ".", + "$OUTPUT_DIR": "$BASE_DIR/output", + "$INPUT_DIR": "$BASE_DIR/inputs", + "$NETWORK_DIR": "$BASE_DIR/network", + "$COMPONENTS_DIR": "$BASE_DIR/../bio_components" + }, + + "run": { + "tstop": 1000.0, + "dt": 0.1, + "dL": 20.0, + "spike_threshold": -15, + "nsteps_block": 5000 + }, + + "target_simulator":"NEURON", + + "conditions": { + "celsius": 34.0, + "v_init": -80 + }, + + "inputs": { + "LGN_spikes": { + "input_type": "spikes", + "module": "sonata", + "input_file": "$INPUT_DIR/lgn_spikes.h5", + "node_set": "lgn" + }, + "TW_spikes": { + "input_type": "spikes", + "module": "sonata", + "input_file": "$INPUT_DIR/tw_spikes.h5", + "node_set": "tw" + } + }, + + "output":{ + "log_file": "log.txt", + "output_dir": "$OUTPUT_DIR", + "spikes_file": "spikes.h5", + "spikes_file_csv": "spikes.csv", + "spikes_file_nwb": "session.nwb", + "spikes_sort_order": "time", + "overwrite_output_dir": true + }, + + "reports": { + "ecp": { + "cells": "all", + "variable_name": "v", + "module": "extracellular", + "electrode_positions": "$COMPONENTS_DIR/recXelectrodes/linear_electrode.csv", + "file_name": "ecp.h5", + "file_name_nwb": "session.nwb", + "electrode_channels": "all" + } + }, + + "network": "config.circuit.json" +} From 79689ec231bbd6ecd16e1e501265e689d6dc0978 Mon Sep 17 00:00:00 2001 From: kaeldai Date: Mon, 9 Jun 2025 14:59:02 -0700 Subject: [PATCH 2/6] updating ubuntu vm machine --- .github/workflows/validate-pull-request.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/validate-pull-request.yaml b/.github/workflows/validate-pull-request.yaml index a665434fe..e0e27881f 100644 --- a/.github/workflows/validate-pull-request.yaml +++ b/.github/workflows/validate-pull-request.yaml @@ -13,8 +13,8 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-20.04] - python-version: ["3.6", "3.7", "3.8", "3.9"] + os: [ubuntu-24.04] + python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"] steps: - uses: actions/checkout@v2 From 03e6eea9ff9ae7d34be6870d99a96c468b5f23a2 Mon Sep 17 00:00:00 2001 From: kaeldai Date: Mon, 9 Jun 2025 15:02:49 -0700 Subject: [PATCH 3/6] updating gh workflow --- .github/workflows/validate-pull-request.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/validate-pull-request.yaml b/.github/workflows/validate-pull-request.yaml index e0e27881f..adad2dc4f 100644 --- a/.github/workflows/validate-pull-request.yaml +++ b/.github/workflows/validate-pull-request.yaml @@ -13,8 +13,8 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-24.04] - python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"] + os: [ubuntu-22.04] + python-version: ["3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v2 From ea5641b80aef18ec50968f7d26cff3106bada555 Mon Sep 17 00:00:00 2001 From: kaeldai Date: Mon, 9 Jun 2025 15:04:29 -0700 Subject: [PATCH 4/6] need to update neuron version --- .github/workflows/validate-pull-request.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate-pull-request.yaml b/.github/workflows/validate-pull-request.yaml index adad2dc4f..f72a74b90 100644 --- a/.github/workflows/validate-pull-request.yaml +++ b/.github/workflows/validate-pull-request.yaml @@ -27,7 +27,7 @@ jobs: - name: Install Requirements run: | pip install -r test_requirements.txt - pip install neuron==8.0.0 + pip install neuron pip install . - name: Check NRN mechanisms directory exists From 615bcc8b40923efb4c1fa7247fc5d3b1b37d8390 Mon Sep 17 00:00:00 2001 From: kaeldai Date: Mon, 9 Jun 2025 15:19:00 -0700 Subject: [PATCH 5/6] allowing BioNet simulations to run even when intervals are not exactly consistent --- bmtk/simulator/bionet/modules/iclamp.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/bmtk/simulator/bionet/modules/iclamp.py b/bmtk/simulator/bionet/modules/iclamp.py index bc9861561..aabe793b4 100644 --- a/bmtk/simulator/bionet/modules/iclamp.py +++ b/bmtk/simulator/bionet/modules/iclamp.py @@ -26,13 +26,11 @@ def __init__(self, **args): # NRN Vector.play function requires consistent dts = np.unique(np.diff(self._inputs_df[self._ts_col].values)) if len(dts) > 1: - io.log_exception('{}: csv timestamps column ({}) must have a consistent intervals.'.format( + io.log_warning('{}: csv timestamps column ({}) may not have consistent intervals; please check that timestamp are consistent to within the same significance as simulation dt.'.format( IClampMod.__name__, self._ts_col) ) self._idt = dts[0] - self._istart = self.delays[0] # self._inputs_df[self._ts_col].values[0] - # self._amps_vals = self._inputs_df[self._amps_col].values - # self._ts_vals = self._inputs_df[self._ts_col].values + self._istart = self.delays[0] # The way NEURON's Vector.play([amps], dt) works is that it will access the [amps] at every dt interval (0.0, # dt, 2dt, 3dt, ...) in ms regardless of when the IClamp starts. Thus if the initial onset stimuli is > 0.0 ms, From 584985abba1cf03b5d7fce0ce68b7b68b6e2e0f6 Mon Sep 17 00:00:00 2001 From: kaeldai Date: Mon, 9 Jun 2025 15:43:09 -0700 Subject: [PATCH 6/6] updatin iclamp test --- tests/simulator/bionet/test_iclamp.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/simulator/bionet/test_iclamp.py b/tests/simulator/bionet/test_iclamp.py index 77ee852b5..08aa4fbf9 100644 --- a/tests/simulator/bionet/test_iclamp.py +++ b/tests/simulator/bionet/test_iclamp.py @@ -174,6 +174,7 @@ def test_sim_csv(): assert(np.max(list(sim.vm)) > -60) +@pytest.mark.skip def test_invalid_csv(): # Timestamps are not evenly spaced tmpfile = create_csv(