From 2afe9f8386d0004d0cfa2a4bce019a8fe0eea268 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Mon, 15 Jul 2019 17:00:39 +0100 Subject: [PATCH 01/24] Implement 1D animations --- xbout/boutdataarray.py | 3 +- xbout/plotting/animate.py | 83 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 81 insertions(+), 5 deletions(-) diff --git a/xbout/boutdataarray.py b/xbout/boutdataarray.py index 1bb41242..95a6eed0 100644 --- a/xbout/boutdataarray.py +++ b/xbout/boutdataarray.py @@ -100,8 +100,7 @@ def animate1D(self, animate_over='t', x='x', y='y', animate=True, print("{} data passed has {} dimensions - will use " "animatplot.blocks.Line()".format(variable, str(n_dims))) line_block = animate_line(data=data, animate_over=animate_over, - x=x, y=y, sep_pos=sep_pos, - animate=animate, fps=fps, + sep_pos=sep_pos, animate=animate, fps=fps, save_as=save_as, ax=ax, **kwargs) return line_block diff --git a/xbout/plotting/animate.py b/xbout/plotting/animate.py index 5a0a4cb9..020e1353 100644 --- a/xbout/plotting/animate.py +++ b/xbout/plotting/animate.py @@ -100,9 +100,86 @@ def animate_imshow(data, animate_over='t', x='x', y='y', animate=True, return imshow_block -def animate_line(data, animate_over='t', x='x', y='y', animate=True, - fps=10, save_as=None, sep_pos=None, ax=None, **kwargs): +def animate_line(data, animate_over='t', animate=True, + vmin='min', vmax='max', fps=10, save_as=None, sep_pos=None, ax=None, + **kwargs): + """ + Plots a line plot which is animated with time. + + Currently only supports 1D+1 data, which it plots with xarray's + wrapping of matplotlib's plot. + + Parameters + ---------- + data : xarray.DataArray + animate_over : str, optional + Dimension over which to animate + vmin : float, optional + Minimum value to use for colorbar. Default is to use minimum value of + data across whole timeseries. + vmax : float, optional + Maximum value to use for colorbar. Default is to use maximum value of + data across whole timeseries. + sep_pos : int, optional + Radial position at which to plot the separatrix + save_as: str, optional + Filename to give to the resulting gif + fps : int, optional + Frames per second of resulting gif + kwargs : dict, optional + Additional keyword arguments are passed on to the plotting function + (e.g. imshow for 2D plots). + """ + variable = data.name + # Check plot is the right orientation t_read, x_read = data.dims - raise NotImplementedError + if (t_read is animate_over): + pass + else: + data = data.transpose(x_read, animate_over) + + # Load values eagerly otherwise for some reason the plotting takes + # 100's of times longer - for some reason animatplot does not deal + # well with dask arrays! + image_data = data.values + + # If not specified, determine max and min values across entire data series + if vmax is 'max': + vmax = np.max(image_data) + if vmin is 'min': + vmin = np.min(image_data) + + if not ax: + fig, ax = plt.subplots() + + # set range of plot + ax.set_ylim([vmin, vmax]) + + line_block = amp.blocks.Line(image_data, ax=ax, **kwargs) + + timeline = amp.Timeline(np.arange(data.sizes[animate_over]), fps=fps) + + if animate: + anim = amp.Animation([line_block], timeline) + + # Add title and axis labels + ax.set_title("{} variation over {}".format(variable, animate_over)) + ax.set_xlabel(x_read) + ax.set_ylabel(variable) + + # Plot separatrix + if sep_pos: + ax.plot_vline(sep_pos, '--') + + if animate: + anim.controls(timeline_slider_args={'text': animate_over}) + + if not save_as: + save_as = "{}_over_{}".format(variable, animate_over) + # TODO save using PillowWriter instead once matplotlib 3.1 comes out + # see https://github.com/t-makaro/animatplot/issues/24 + anim.save(save_as + '.gif', writer='imagemagick') + + return line_block From 317420e4fb9c1f076137b87d8a2b020a9373a423 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Tue, 16 Jul 2019 09:42:01 +0100 Subject: [PATCH 02/24] Use None as default argument for vmin/vmax in animate functions --- xbout/plotting/animate.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/xbout/plotting/animate.py b/xbout/plotting/animate.py index 020e1353..64731215 100644 --- a/xbout/plotting/animate.py +++ b/xbout/plotting/animate.py @@ -7,7 +7,7 @@ def animate_imshow(data, animate_over='t', x='x', y='y', animate=True, - vmin='min', vmax='max', fps=10, save_as=None, + vmin=None, vmax=None, fps=10, save_as=None, sep_pos=None, ax=None, **kwargs): """ Plots a color plot which is animated with time over the specified @@ -60,9 +60,9 @@ def animate_imshow(data, animate_over='t', x='x', y='y', animate=True, image_data = data.values # If not specified, determine max and min values across entire data series - if vmax is 'max': + if vmax is None: vmax = np.max(image_data) - if vmin is 'min': + if vmin is None: vmin = np.min(image_data) if not ax: @@ -101,7 +101,7 @@ def animate_imshow(data, animate_over='t', x='x', y='y', animate=True, def animate_line(data, animate_over='t', animate=True, - vmin='min', vmax='max', fps=10, save_as=None, sep_pos=None, ax=None, + vmin=None, vmax=None, fps=10, save_as=None, sep_pos=None, ax=None, **kwargs): """ Plots a line plot which is animated with time. @@ -146,9 +146,9 @@ def animate_line(data, animate_over='t', animate=True, image_data = data.values # If not specified, determine max and min values across entire data series - if vmax is 'max': + if vmax is None: vmax = np.max(image_data) - if vmin is 'min': + if vmin is None: vmin = np.min(image_data) if not ax: From 48ab3578dc00f422d91ae87419ae270073cb7615 Mon Sep 17 00:00:00 2001 From: Rhys Doyle Date: Tue, 10 Sep 2019 22:07:11 +0100 Subject: [PATCH 03/24] Animation Writer changed to PillowWriter in animate1D and animate2D --- xbout/plotting/animate.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/xbout/plotting/animate.py b/xbout/plotting/animate.py index 64731215..298419d1 100644 --- a/xbout/plotting/animate.py +++ b/xbout/plotting/animate.py @@ -4,7 +4,7 @@ import animatplot as amp from .utils import plot_separatrix - +from matplotlib.animation import PillowWriter def animate_imshow(data, animate_over='t', x='x', y='y', animate=True, vmin=None, vmax=None, fps=10, save_as=None, @@ -93,9 +93,8 @@ def animate_imshow(data, animate_over='t', x='x', y='y', animate=True, if not save_as: save_as = "{}_over_{}".format(variable, animate_over) - # TODO save using PillowWriter instead once matplotlib 3.1 comes out - # see https://github.com/t-makaro/animatplot/issues/24 - anim.save(save_as + '.gif', writer='imagemagick') + + anim.save(save_as + '.gif', writer=PillowWriter(fps=fps)) return imshow_block @@ -178,8 +177,7 @@ def animate_line(data, animate_over='t', animate=True, if not save_as: save_as = "{}_over_{}".format(variable, animate_over) - # TODO save using PillowWriter instead once matplotlib 3.1 comes out - # see https://github.com/t-makaro/animatplot/issues/24 - anim.save(save_as + '.gif', writer='imagemagick') + + anim.save(save_as + '.gif', writer=PillowWriter(fps=fps)) return line_block From 283d098b8b20fe65f0a8ca0abec72e0bd92007d7 Mon Sep 17 00:00:00 2001 From: Rhys Doyle <48983334+rdoyle45@users.noreply.github.com> Date: Wed, 11 Sep 2019 11:00:33 +0100 Subject: [PATCH 04/24] Update xbout/plotting/animate.py --- xbout/plotting/animate.py | 1 - 1 file changed, 1 deletion(-) diff --git a/xbout/plotting/animate.py b/xbout/plotting/animate.py index 298419d1..55e68d06 100644 --- a/xbout/plotting/animate.py +++ b/xbout/plotting/animate.py @@ -93,7 +93,6 @@ def animate_imshow(data, animate_over='t', x='x', y='y', animate=True, if not save_as: save_as = "{}_over_{}".format(variable, animate_over) - anim.save(save_as + '.gif', writer=PillowWriter(fps=fps)) return imshow_block From 6e72dd8189a9c773d53ac393ebc9260f3a371f08 Mon Sep 17 00:00:00 2001 From: Rhys Doyle <48983334+rdoyle45@users.noreply.github.com> Date: Thu, 12 Sep 2019 12:11:56 +0100 Subject: [PATCH 05/24] Update xbout/plotting/animate.py --- xbout/plotting/animate.py | 1 - 1 file changed, 1 deletion(-) diff --git a/xbout/plotting/animate.py b/xbout/plotting/animate.py index 55e68d06..5124bbf9 100644 --- a/xbout/plotting/animate.py +++ b/xbout/plotting/animate.py @@ -176,7 +176,6 @@ def animate_line(data, animate_over='t', animate=True, if not save_as: save_as = "{}_over_{}".format(variable, animate_over) - anim.save(save_as + '.gif', writer=PillowWriter(fps=fps)) return line_block From ef8f2111cff6cf3b48d2c0ff2bb8af6f70b3a7c7 Mon Sep 17 00:00:00 2001 From: Rhys Doyle <48983334+rdoyle45@users.noreply.github.com> Date: Thu, 12 Sep 2019 17:29:29 +0100 Subject: [PATCH 06/24] Update to required Matplotlib version To allow for the use of PillowWriter in plotting/animate.py --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2df45747..af512084 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ xarray >= 0.12.2 dask[array] >= 1.0.0 natsort >= 5.5.0 -matplotlib >= 2.2 +matplotlib >= 3.1 animatplot >= 0.3 netcdf4 >= 1.4.0 From 75ebd07f83ef7f380235a21843faf384f9cb9378 Mon Sep 17 00:00:00 2001 From: Rhys Doyle <48983334+rdoyle45@users.noreply.github.com> Date: Thu, 12 Sep 2019 18:10:07 +0100 Subject: [PATCH 07/24] Python 3.5 doesn't support Matplotlib 3.1.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index af512084..377a1347 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ xarray >= 0.12.2 dask[array] >= 1.0.0 natsort >= 5.5.0 -matplotlib >= 3.1 +matplotlib >= 3.0.3 animatplot >= 0.3 netcdf4 >= 1.4.0 From f9848d0a0756a8676ef87bd2497fa9fb69b39815 Mon Sep 17 00:00:00 2001 From: Rhys Doyle Date: Thu, 12 Sep 2019 20:00:33 +0100 Subject: [PATCH 08/24] Test file for animate1D and 2D --- xbout/tests/test_animate.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 xbout/tests/test_animate.py diff --git a/xbout/tests/test_animate.py b/xbout/tests/test_animate.py new file mode 100644 index 00000000..f8ba8357 --- /dev/null +++ b/xbout/tests/test_animate.py @@ -0,0 +1,34 @@ +import pytest +from os import remove + +from xbout import open_boutdataset +from xbout.boutdataarray import BoutDataArrayAccessor + +DATA_PATH = './data/dump_files/along_x/BOUT.dmp.*.nc' + +class TestAnimate: + """ + Set of tests to check whether animate1D() and animate2D() are running properly + and PillowWriter is saving each animation correctly + """ + def test_animate2D(self): + + bd = open_boutdataset(DATA_PATH).squeeze(drop=True) + + anim_creator = str(bd['T'].bout.animate2D(y='z')) + checker = ' Date: Thu, 12 Sep 2019 20:06:40 +0100 Subject: [PATCH 09/24] PEP8 issues fixed --- xbout/tests/test_animate.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/xbout/tests/test_animate.py b/xbout/tests/test_animate.py index f8ba8357..98f8572a 100644 --- a/xbout/tests/test_animate.py +++ b/xbout/tests/test_animate.py @@ -16,10 +16,10 @@ def test_animate2D(self): bd = open_boutdataset(DATA_PATH).squeeze(drop=True) anim_creator = str(bd['T'].bout.animate2D(y='z')) - checker = ' Date: Thu, 12 Sep 2019 20:08:14 +0100 Subject: [PATCH 10/24] PEP8 issues fixed --- xbout/tests/test_animate.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/xbout/tests/test_animate.py b/xbout/tests/test_animate.py index 98f8572a..9e0e60c2 100644 --- a/xbout/tests/test_animate.py +++ b/xbout/tests/test_animate.py @@ -6,6 +6,7 @@ DATA_PATH = './data/dump_files/along_x/BOUT.dmp.*.nc' + class TestAnimate: """ Set of tests to check whether animate1D() and animate2D() are running properly @@ -25,7 +26,7 @@ def test_animate1D(self): bd = open_boutdataset(DATA_PATH).squeeze(drop=True) - anim_creator = str(bd['T'][:,:,0].bout.animate1D()) + anim_creator = str(bd['T'][:, :, 0].bout.animate1D()) print(anim_creator) checker = ' Date: Thu, 12 Sep 2019 20:10:37 +0100 Subject: [PATCH 11/24] Path to data incorrect --- xbout/tests/test_animate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xbout/tests/test_animate.py b/xbout/tests/test_animate.py index 9e0e60c2..9e80a9a5 100644 --- a/xbout/tests/test_animate.py +++ b/xbout/tests/test_animate.py @@ -4,7 +4,7 @@ from xbout import open_boutdataset from xbout.boutdataarray import BoutDataArrayAccessor -DATA_PATH = './data/dump_files/along_x/BOUT.dmp.*.nc' +DATA_PATH = './xbout/tests/data/dump_files/along_x/BOUT.dmp.*.nc' class TestAnimate: From 33d85a8e926dd9f59e3f87d0d7c1839daaa492ed Mon Sep 17 00:00:00 2001 From: Rhys Doyle Date: Thu, 12 Sep 2019 20:38:24 +0100 Subject: [PATCH 12/24] Removal of unwanted line --- xbout/tests/test_animate.py | 1 - 1 file changed, 1 deletion(-) diff --git a/xbout/tests/test_animate.py b/xbout/tests/test_animate.py index 9e80a9a5..b857bd49 100644 --- a/xbout/tests/test_animate.py +++ b/xbout/tests/test_animate.py @@ -27,7 +27,6 @@ def test_animate1D(self): bd = open_boutdataset(DATA_PATH).squeeze(drop=True) anim_creator = str(bd['T'][:, :, 0].bout.animate1D()) - print(anim_creator) checker = ' Date: Thu, 12 Sep 2019 20:48:06 +0100 Subject: [PATCH 13/24] Pillow required for PillowWriter --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 377a1347..cd273591 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,4 @@ natsort >= 5.5.0 matplotlib >= 3.0.3 animatplot >= 0.3 netcdf4 >= 1.4.0 +Pillow >= 6.1.0 From 45245c29d724dc28a0bf03400fcf74786e51403f Mon Sep 17 00:00:00 2001 From: Rhys Doyle <48983334+rdoyle45@users.noreply.github.com> Date: Sun, 15 Sep 2019 01:04:49 +0100 Subject: [PATCH 14/24] Dependency update --- setup.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index a9925e1f..a805e44a 100644 --- a/setup.py +++ b/setup.py @@ -24,8 +24,9 @@ 'xarray>=v0.10.0', 'dask[array]>=1.0.0', 'natsort>=5.5.0', - 'matplotlib>=2.2', - 'animatplot>=0.3' + 'matplotlib>=3.0.3', + 'animatplot>=0.3', + 'Pillow>=6.1.0' ], extras_require={ 'tests': ['pytest >= 3.3.0'], From 88868d0c14c53a9bb46cca21064140b49b37e3f5 Mon Sep 17 00:00:00 2001 From: Rhys Doyle Date: Sun, 13 Oct 2019 11:50:55 +0100 Subject: [PATCH 15/24] Improved use of python functions for pytest and added use of pytest fixture to create temporary directory to deal with the output of subsequent animate function --- xbout/tests/test_animate.py | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/xbout/tests/test_animate.py b/xbout/tests/test_animate.py index b857bd49..5b5aecb5 100644 --- a/xbout/tests/test_animate.py +++ b/xbout/tests/test_animate.py @@ -1,33 +1,36 @@ import pytest -from os import remove from xbout import open_boutdataset from xbout.boutdataarray import BoutDataArrayAccessor -DATA_PATH = './xbout/tests/data/dump_files/along_x/BOUT.dmp.*.nc' +from animatplot.blocks import Imshow, Line +DATA_PATH = './xbout/tests/data/dump_files/along_x/BOUT.dmp.*.nc' # Path to test dmp files + +@pytest.fixture +def create_test_file(tmpdir_factory): + + save_dir = tmpdir_factory.mktemp("test_data") # Create temp dir for output of animate1D/2D + ds = open_boutdataset(DATA_PATH).squeeze(drop=True) # Open test data + + return save_dir, ds class TestAnimate: """ Set of tests to check whether animate1D() and animate2D() are running properly and PillowWriter is saving each animation correctly """ - def test_animate2D(self): - - bd = open_boutdataset(DATA_PATH).squeeze(drop=True) + def test_animate2D(self, create_test_file): - anim_creator = str(bd['T'].bout.animate2D(y='z')) - checker = ' Date: Sun, 13 Oct 2019 14:21:16 +0100 Subject: [PATCH 16/24] PEP8 fixes --- xbout/tests/test_animate.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/xbout/tests/test_animate.py b/xbout/tests/test_animate.py index 5b5aecb5..1dd27ceb 100644 --- a/xbout/tests/test_animate.py +++ b/xbout/tests/test_animate.py @@ -5,16 +5,20 @@ from animatplot.blocks import Imshow, Line -DATA_PATH = './xbout/tests/data/dump_files/along_x/BOUT.dmp.*.nc' # Path to test dmp files +# Path to test dmp files +DATA_PATH = './xbout/tests/data/dump_files/along_x/BOUT.dmp.*.nc' + @pytest.fixture def create_test_file(tmpdir_factory): - save_dir = tmpdir_factory.mktemp("test_data") # Create temp dir for output of animate1D/2D - ds = open_boutdataset(DATA_PATH).squeeze(drop=True) # Open test data + # Create temp dir for output of animate1D/2D + save_dir = tmpdir_factory.mktemp("test_data") + ds = open_boutdataset(DATA_PATH).squeeze(drop=True) # Open test data return save_dir, ds + class TestAnimate: """ Set of tests to check whether animate1D() and animate2D() are running properly @@ -34,3 +38,4 @@ def test_animate1D(self, create_test_file): assert isinstance(animation, Line) + From 6447865e189676af11bff6dd4eb676d33f797e8a Mon Sep 17 00:00:00 2001 From: Rhys Doyle Date: Sun, 13 Oct 2019 14:23:09 +0100 Subject: [PATCH 17/24] PEP8 fixes --- xbout/tests/test_animate.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/xbout/tests/test_animate.py b/xbout/tests/test_animate.py index 1dd27ceb..e161cbf9 100644 --- a/xbout/tests/test_animate.py +++ b/xbout/tests/test_animate.py @@ -37,5 +37,3 @@ def test_animate1D(self, create_test_file): animation = ds['T'][:, :, 0].bout.animate1D(save_as="%s/test.gif" % save_dir) assert isinstance(animation, Line) - - From bfad80e7126a268a7956affa8ab79ae51843d1e6 Mon Sep 17 00:00:00 2001 From: Rhys Doyle <48983334+rdoyle45@users.noreply.github.com> Date: Mon, 14 Oct 2019 11:56:08 +0100 Subject: [PATCH 18/24] Remove .gif suffix from test --- xbout/tests/test_animate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xbout/tests/test_animate.py b/xbout/tests/test_animate.py index e161cbf9..45ffbca6 100644 --- a/xbout/tests/test_animate.py +++ b/xbout/tests/test_animate.py @@ -27,13 +27,13 @@ class TestAnimate: def test_animate2D(self, create_test_file): save_dir, ds = create_test_file - animation = ds['T'].bout.animate2D(y='z', save_as="%s/test.gif" % save_dir) + animation = ds['T'].bout.animate2D(y='z', save_as="%s/test" % save_dir) assert isinstance(animation, Imshow) def test_animate1D(self, create_test_file): save_dir, ds = create_test_file - animation = ds['T'][:, :, 0].bout.animate1D(save_as="%s/test.gif" % save_dir) + animation = ds['T'][:, :, 0].bout.animate1D(save_as="%s/test" % save_dir) assert isinstance(animation, Line) From 6a29502d7fc116fae7644ed16fabb5067e75be9d Mon Sep 17 00:00:00 2001 From: John Omotani Date: Wed, 23 Oct 2019 16:21:15 +0100 Subject: [PATCH 19/24] Add more fake grid data in test_load.create_bout_ds Makes test files more similar to BOUT++ dump files. --- xbout/tests/test_load.py | 115 +++++++++++++++++++++++++++------------ 1 file changed, 81 insertions(+), 34 deletions(-) diff --git a/xbout/tests/test_load.py b/xbout/tests/test_load.py index 59482f70..943f0324 100644 --- a/xbout/tests/test_load.py +++ b/xbout/tests/test_load.py @@ -217,17 +217,8 @@ def create_bout_ds_list(prefix, lengths=(2, 4, 7, 6), nxpe=4, nype=2, nt=1, guar upper_bndry_cells = {dim: guards.get(dim) for dim in guards.keys()} lower_bndry_cells = {dim: guards.get(dim) for dim in guards.keys()} - # Include boundary cells - for dim in ['x', 'y']: - if dim in guards.keys(): - if i == 0: - lower_bndry_cells[dim] = guards[dim] - if i == nxpe-1: - upper_bndry_cells[dim] = guards[dim] - ds = create_bout_ds(syn_data_type=syn_data_type, num=num, lengths=lengths, nxpe=nxpe, nype=nype, - upper_bndry_cells=upper_bndry_cells, lower_bndry_cells=lower_bndry_cells, - guards=guards) + xproc=i, yproc=j, guards=guards) ds_list.append(ds) # Sort this in order of num to remove any BOUT-specific structure @@ -238,16 +229,21 @@ def create_bout_ds_list(prefix, lengths=(2, 4, 7, 6), nxpe=4, nype=2, nt=1, guar def create_bout_ds(syn_data_type='random', lengths=(2,4,7,6), num=0, nxpe=1, nype=1, - upper_bndry_cells={}, lower_bndry_cells={}, guards={}): + xproc=0, yproc=0, guards={}): # Set the shape of the data in this dataset x_length, y_length, z_length, t_length = lengths - x_length += upper_bndry_cells.get('x', 0) + lower_bndry_cells.get('x', 0) - y_length += upper_bndry_cells.get('y', 0) + lower_bndry_cells.get('y', 0) - z_length += upper_bndry_cells.get('z', 0) + lower_bndry_cells.get('z', 0) - t_length += upper_bndry_cells.get('t', 0) + lower_bndry_cells.get('t', 0) + mxg = guards.get('x', 0) + myg = guards.get('y', 0) + x_length += 2*mxg + y_length += 2*myg shape = (x_length, y_length, z_length, t_length) + # calculate global nx, ny and nz + nx = nxpe*lengths[1] + 2*mxg + ny = nype*lengths[2] + nz = 1*lengths[3] + # Fill with some kind of synthetic data if syn_data_type is 'random': # Each dataset contains the same random noise @@ -268,25 +264,70 @@ def create_bout_ds(syn_data_type='random', lengths=(2,4,7,6), num=0, nxpe=1, nyp n = DataArray(data, dims=['x', 'y', 'z', 't']) ds = Dataset({'n': n, 'T': T}) - # Include metadata + # Include grid data ds['NXPE'] = nxpe ds['NYPE'] = nype - ds['MXG'] = guards.get('x', 0) - ds['MYG'] = guards.get('y', 0) - ds['nx'] = x_length - ds['MXSUB'] = guards.get('x', 0) - ds['MYSUB'] = guards.get('y', 0) - ds['MZ'] = z_length - ds['jyseps1_1'] = -1 - ds['jyseps1_2'] = -1 - ds['jyseps2_1'] = -1 - ds['jyseps2_2'] = -1 + ds['NZPE'] = 1 + ds['PE_XIND'] = xproc + ds['PE_YIND'] = yproc + ds['MYPE'] = num + + ds['MXG'] = mxg + ds['MYG'] = myg + ds['nx'] = nx + ds['ny'] = ny + ds['nz'] = nz + ds['MZ'] = 1*lengths[3] + ds['MXSUB'] = lengths[1] + ds['MYSUB'] = lengths[2] + ds['MZSUB'] = lengths[3] + ds['ixseps1'] = nx + ds['ixseps2'] = nx + ds['jyseps1_1'] = 0 + ds['jyseps1_2'] = ny + ds['jyseps2_1'] = ny//2 - 1 + ds['jyseps2_2'] = ny//2 - 1 + ds['ny_inner'] = ny//2 + + one = DataArray(np.ones((x_length, y_length)), dims=['x', 'y']) + zero = DataArray(np.zeros((x_length, y_length)), dims=['x', 'y']) + + ds['zperiod'] = 1 + ds['ZMIN'] = 0. + ds['ZMAX'] = 2.*np.pi + ds['g11'] = one + ds['g22'] = one + ds['g33'] = one + ds['g12'] = zero + ds['g13'] = zero + ds['g23'] = zero + ds['g_11'] = one + ds['g_22'] = one + ds['g_33'] = one + ds['g_12'] = zero + ds['g_13'] = zero + ds['g_23'] = zero + ds['G1'] = zero + ds['G2'] = zero + ds['G3'] = zero + ds['J'] = one + ds['Bxy'] = one + ds['zShift'] = zero + + ds['dx'] = 0.5*one + ds['dy'] = 2.*one + ds['dz'] = 0.7 + + ds['iteration'] = t_length + ds['t_array'] = DataArray(np.arange(t_length, dtype=float)*10., dims='t') return ds -METADATA_VARS = ['NXPE', 'NYPE', 'MXG', 'MYG', 'nx', 'MXSUB', 'MYSUB', 'MZ', - 'jyseps1_1', 'jyseps1_2', 'jyseps2_1', 'jyseps2_2'] +METADATA_VARS = ['NXPE', 'NYPE', 'NZPE', 'PE_XIND', 'PE_YIND', 'MYPE', 'MXG', 'MYG', 'nx', + 'ny', 'nz', 'MZ', 'MXSUB', 'MYSUB', 'MZSUB', 'ixseps1', 'ixseps2', + 'jyseps1_1', 'jyseps1_2', 'jyseps2_1', 'jyseps2_2', 'ny_inner', + 'zperiod', 'ZMIN', 'ZMAX', 'dz', 'iteration'] class TestStripMetadata(): @@ -315,7 +356,8 @@ def test_combine_along_x(self, tmpdir_factory, bout_xyt_example_files): actual = open_boutdataset(datapath=path, keep_xboundaries=False) bout_ds = create_bout_ds - expected = concat([bout_ds(0), bout_ds(1), bout_ds(2), bout_ds(3)], dim='x') + expected = concat([bout_ds(0), bout_ds(1), bout_ds(2), bout_ds(3)], dim='x', + data_vars='minimal') xrt.assert_equal(actual.load(), expected.drop(METADATA_VARS)) def test_combine_along_y(self, tmpdir_factory, bout_xyt_example_files): @@ -324,7 +366,8 @@ def test_combine_along_y(self, tmpdir_factory, bout_xyt_example_files): actual = open_boutdataset(datapath=path, keep_xboundaries=False) bout_ds = create_bout_ds - expected = concat([bout_ds(0), bout_ds(1), bout_ds(2)], dim='y') + expected = concat([bout_ds(0), bout_ds(1), bout_ds(2)], dim='y', + data_vars='minimal') xrt.assert_equal(actual.load(), expected.drop(METADATA_VARS)) @pytest.mark.skip @@ -337,10 +380,14 @@ def test_combine_along_xy(self, tmpdir_factory, bout_xyt_example_files): actual = open_boutdataset(datapath=path, keep_xboundaries=False) bout_ds = create_bout_ds - line1 = concat([bout_ds(0), bout_ds(1), bout_ds(2), bout_ds(3)], dim='x') - line2 = concat([bout_ds(4), bout_ds(5), bout_ds(6), bout_ds(7)], dim='x') - line3 = concat([bout_ds(8), bout_ds(9), bout_ds(10), bout_ds(11)], dim='x') - expected = concat([line1, line2, line3], dim='y') + line1 = concat([bout_ds(0), bout_ds(1), bout_ds(2), bout_ds(3)], dim='x', + data_vars='minimal') + line2 = concat([bout_ds(4), bout_ds(5), bout_ds(6), bout_ds(7)], dim='x', + data_vars='minimal') + line3 = concat([bout_ds(8), bout_ds(9), bout_ds(10), bout_ds(11)], dim='x', + data_vars='minimal') + expected = concat([line1, line2, line3], dim='y', + data_vars='minimal') xrt.assert_equal(actual.load(), expected.drop(METADATA_VARS)) @pytest.mark.skip From 021dcd2999ac60305bd28325a4e45d2cac277e40 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Wed, 23 Oct 2019 16:22:22 +0100 Subject: [PATCH 20/24] Unique random data in files created by create_bout_ds_list Use the file number as a seed for the random number generation so that all files are different. --- xbout/tests/test_load.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xbout/tests/test_load.py b/xbout/tests/test_load.py index 943f0324..59ffbccf 100644 --- a/xbout/tests/test_load.py +++ b/xbout/tests/test_load.py @@ -246,8 +246,8 @@ def create_bout_ds(syn_data_type='random', lengths=(2,4,7,6), num=0, nxpe=1, nyp # Fill with some kind of synthetic data if syn_data_type is 'random': - # Each dataset contains the same random noise - np.random.seed(seed=0) + # Each dataset contains unique random noise + np.random.seed(seed = num) data = np.random.randn(*shape) elif syn_data_type is 'linear': # Variables increase linearly across entire domain From 14262109089be857730c7420b022452b918a5fc4 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Wed, 23 Oct 2019 16:55:05 +0100 Subject: [PATCH 21/24] Reorder dimensions in create_bout_ds to ('t','x','y','z') Previously was ('x','y','z','t'). The new order matches the way BOUT++ stores data. --- xbout/tests/test_load.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/xbout/tests/test_load.py b/xbout/tests/test_load.py index 59ffbccf..0fca3e2b 100644 --- a/xbout/tests/test_load.py +++ b/xbout/tests/test_load.py @@ -168,7 +168,7 @@ def bout_xyt_example_files(tmpdir_factory): return _bout_xyt_example_files -def _bout_xyt_example_files(tmpdir_factory, prefix='BOUT.dmp', lengths=(2,4,7,6), +def _bout_xyt_example_files(tmpdir_factory, prefix='BOUT.dmp', lengths=(6,2,4,7), nxpe=4, nype=2, nt=1, guards={}, syn_data_type='random'): """ Mocks up a set of BOUT-like netCDF files, and return the temporary test directory containing them. @@ -197,7 +197,7 @@ def _bout_xyt_example_files(tmpdir_factory, prefix='BOUT.dmp', lengths=(2,4,7,6) return glob_pattern -def create_bout_ds_list(prefix, lengths=(2, 4, 7, 6), nxpe=4, nype=2, nt=1, guards={}, +def create_bout_ds_list(prefix, lengths=(6,2,4,7), nxpe=4, nype=2, nt=1, guards={}, syn_data_type='random'): """ Mocks up a set of BOUT-like datasets. @@ -228,16 +228,16 @@ def create_bout_ds_list(prefix, lengths=(2, 4, 7, 6), nxpe=4, nype=2, nt=1, guar return ds_list_sorted, file_list_sorted -def create_bout_ds(syn_data_type='random', lengths=(2,4,7,6), num=0, nxpe=1, nype=1, +def create_bout_ds(syn_data_type='random', lengths=(6,2,4,7), num=0, nxpe=1, nype=1, xproc=0, yproc=0, guards={}): # Set the shape of the data in this dataset - x_length, y_length, z_length, t_length = lengths + t_length, x_length, y_length, z_length = lengths mxg = guards.get('x', 0) myg = guards.get('y', 0) x_length += 2*mxg y_length += 2*myg - shape = (x_length, y_length, z_length, t_length) + shape = (t_length, x_length, y_length, z_length) # calculate global nx, ny and nz nx = nxpe*lengths[1] + 2*mxg @@ -260,8 +260,8 @@ def create_bout_ds(syn_data_type='random', lengths=(2,4,7,6), num=0, nxpe=1, nyp else: raise ValueError('Not a recognised choice of type of synthetic bout data.') - T = DataArray(data, dims=['x', 'y', 'z', 't']) - n = DataArray(data, dims=['x', 'y', 'z', 't']) + T = DataArray(data, dims=['t', 'x', 'y', 'z']) + n = DataArray(data, dims=['t', 'x', 'y', 'z']) ds = Dataset({'n': n, 'T': T}) # Include grid data From 3ca6274023bc10b1f7bdbb2ba94ad61cf30a89d7 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Wed, 23 Oct 2019 16:55:49 +0100 Subject: [PATCH 22/24] Implement 'linear' option for create_bout_ds --- xbout/tests/test_load.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/xbout/tests/test_load.py b/xbout/tests/test_load.py index 0fca3e2b..ad8b2d48 100644 --- a/xbout/tests/test_load.py +++ b/xbout/tests/test_load.py @@ -251,7 +251,21 @@ def create_bout_ds(syn_data_type='random', lengths=(6,2,4,7), num=0, nxpe=1, nyp data = np.random.randn(*shape) elif syn_data_type is 'linear': # Variables increase linearly across entire domain - raise NotImplementedError + data = DataArray(-np.ones(shape), dims=('t', 'x', 'y', 'z')) + + t_array = DataArray((nx - 2*mxg)*ny*nz*np.arange(t_length, dtype=float), + dims='t') + x_array = DataArray(ny*nz*(xproc*lengths[1] + mxg + + np.arange(lengths[1], dtype=float)), + dims='x') + y_array = DataArray(nz*(yproc*lengths[2] + myg + + np.arange(lengths[2], dtype=float)), + dims='y') + z_array = DataArray(np.arange(z_length, dtype=float), dims='z') + + data[:, mxg:x_length-mxg, myg:y_length-myg, :] = ( + t_array + x_array + y_array + z_array + ) elif syn_data_type is 'stepped': # Each dataset contains a different number depending on the filename data = np.ones(shape) * num From 452e69d742dc8c859a7d16b2cffae5a180ab56d3 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Mon, 21 Oct 2019 17:50:12 +0100 Subject: [PATCH 23/24] Remove dependence of test_animate on saved binary files Instead generate the input files in a temporary directory using test_load.create_bout_ds_list(). --- xbout/tests/test_animate.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/xbout/tests/test_animate.py b/xbout/tests/test_animate.py index 45ffbca6..fb67e4be 100644 --- a/xbout/tests/test_animate.py +++ b/xbout/tests/test_animate.py @@ -2,19 +2,24 @@ from xbout import open_boutdataset from xbout.boutdataarray import BoutDataArrayAccessor +from .test_load import create_bout_ds_list from animatplot.blocks import Imshow, Line -# Path to test dmp files -DATA_PATH = './xbout/tests/data/dump_files/along_x/BOUT.dmp.*.nc' - @pytest.fixture def create_test_file(tmpdir_factory): # Create temp dir for output of animate1D/2D save_dir = tmpdir_factory.mktemp("test_data") - ds = open_boutdataset(DATA_PATH).squeeze(drop=True) # Open test data + + # Generate some test data + ds_list, file_list = create_bout_ds_list("BOUT.dmp", nxpe=3, nype=3, + syn_data_type="linear") + for ds, file_name in zip(ds_list, file_list): + ds.to_netcdf(str(save_dir.join(str(file_name)))) + + ds = open_boutdataset(save_dir.join("BOUT.dmp.*.nc")) # Open test data return save_dir, ds @@ -27,13 +32,13 @@ class TestAnimate: def test_animate2D(self, create_test_file): save_dir, ds = create_test_file - animation = ds['T'].bout.animate2D(y='z', save_as="%s/test" % save_dir) + animation = ds['n'].isel(y=1).bout.animate2D(y='z', save_as="%s/test" % save_dir) assert isinstance(animation, Imshow) def test_animate1D(self, create_test_file): save_dir, ds = create_test_file - animation = ds['T'][:, :, 0].bout.animate1D(save_as="%s/test" % save_dir) + animation = ds['n'].isel(y=2, z=0).bout.animate1D(save_as="%s/test" % save_dir) assert isinstance(animation, Line) From f09f7a7ebf811050bba985baa041162abb650328 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Wed, 23 Oct 2019 14:01:40 +0100 Subject: [PATCH 24/24] Bump required Python version to 3.6, test 3.6 and 3.7 with Travis In Python-3.5 there is an incompatibility between pathlib.Path and pytest.LocalPath, which is fixed in Python-3.6. --- .travis.yml | 2 +- setup.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index a0758ee1..641e3c31 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ language: python python: - - "3.5" - "3.6" + - "3.7" install: - pip install --upgrade setuptools pip pytest pytest-cov coverage codecov - pip install -r requirements.txt diff --git a/setup.py b/setup.py index f11eb043..fed11c39 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ author_email="thomas.nicholas@york.ac.uk", description='Collect data from BOUT++ runs in python using xarray', license="Apache", - python_requires='>=3.5', + python_requires='>=3.6', install_requires=[ 'xarray>=v0.12.2', 'dask[array]>=1.0.0', @@ -43,7 +43,6 @@ "License :: OSI Approved :: Apache License", "Natural Language :: English", "Operating System :: POSIX :: Linux", - "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Topic :: Scientific/Engineering :: Visualization"