diff --git a/src/CSET/cset_workflow/app/metplus_ascii2nc/file/ASCII2NC_python_embedding.conf b/src/CSET/cset_workflow/app/metplus_ascii2nc/file/ASCII2NC_python_embedding.conf deleted file mode 100644 index ddc13381c..000000000 --- a/src/CSET/cset_workflow/app/metplus_ascii2nc/file/ASCII2NC_python_embedding.conf +++ /dev/null @@ -1,101 +0,0 @@ -[dir] -INPUT_BASE = {ENV[METPLUS_OBS_DIR]} -OUTPUT_BASE = {ENV[CYLC_WORKFLOW_SHARE_DIR]} -# INPUT_READ_SCRIPT = {ENV[INPUT_READ_SCRIPT]} -MET_INSTALL_DIR = {ENV[MET_INSTALL_DIR]} -TIME_START = {ENV[TIME_START]} - -[config] - -# Documentation for this use case can be found at -# https://metplus.readthedocs.io/en/latest/generated/met_tool_wrapper/ASCII2NC/ASCII2NC_python_embedding.html - -# For additional information, please see the METplus Users Guide. -# https://metplus.readthedocs.io/en/latest/Users_Guide - -### -# Processes to run -# https://metplus.readthedocs.io/en/latest/Users_Guide/systemconfiguration.html#process-list -### - -PROCESS_LIST = ASCII2NC - - -### -# Time Info -# LOOP_BY options are INIT, VALID, RETRO, and REALTIME -# If set to INIT or RETRO: -# INIT_TIME_FMT, INIT_BEG, INIT_END, and INIT_INCREMENT must also be seti# If set to VALID or REALTIME: -# VALID_TIME_FMT, VALID_BEG, VALID_END, and VALID_INCREMENT must also be set -# LEAD_SEQ is the list of forecast leads to process -# https://metplus.readthedocs.io/en/latest/Users_Guide/systemconfiguration.html#timing-control -### - -LOOP_BY = INIT -INIT_TIME_FMT = %Y%m%dT%H -INIT_BEG = {ENV[TASK_TIME]} -INIT_END = {ENV[TASK_ENDTIME]} -INIT_INCREMENT = 6H - -LEAD_SEQ = begin_end_incr(0,5,1) - - -### -# File I/O -# https://metplus.readthedocs.io/en/latest/Users_Guide/systemconfiguration.html#directory-and-filename-template-info -### - -ASCII2NC_INPUT_DIR = -ASCII2NC_INPUT_TEMPLATE = "python/{ENV[INPUT_READ_SCRIPT]} {INPUT_BASE}" - -ASCII2NC_OUTPUT_DIR = -ASCII2NC_OUTPUT_TEMPLATE = {OUTPUT_BASE}/obs_{TIME_START}.nc - -ASCII2NC_SKIP_IF_OUTPUT_EXISTS = False - -ASCII2NC_FILE_WINDOW_BEGIN = 0 -ASCII2NC_FILE_WINDOW_END = 0 - - -########################################### -ASCII2NC_FILE_WINDOW_BEGIN = -1800 -ASCII2NC_FILE_WINDOW_END = 1800 - -ASCII2NC_INPUT_DIR = -ASCII2NC_INPUT_TEMPLATE = "python/{ENV[INPUT_READ_SCRIPT]} {INPUT_BASE}/Surface.NiwaList.{init?fmt=%Y%m%dT%2H} {valid?fmt=%Y%m%dT%2H?shift=-10800}00 {ASCII2NC_FILE_WINDOW_BEGIN} {ASCII2NC_FILE_WINDOW_END}" - -ASCII2NC_OUTPUT_DIR = -ASCII2NC_OUTPUT_TEMPLATE = {OUTPUT_BASE}/obs_nc/ascii2nc_python_niwa_{valid?fmt=%Y%m%dT%2H?shift=-10800}00.nc - -ASCII2NC_SKIP_IF_OUTPUT_EXISTS = False - - - -### -# ASCII2NC Settings -# https://metplus.readthedocs.io/en/latest/Users_Guide/wrappers.html#ascii2nc -### - -#LOG_ASCII2NC_VERBOSITY = 1 -#ASCII2NC_CONFIG_FILE = - -ASCII2NC_WINDOW_BEGIN = 0 -ASCII2NC_WINDOW_END = 0 - -ASCII2NC_INPUT_FORMAT = python - -ASCII2NC_MASK_GRID = -ASCII2NC_MASK_POLY = -ASCII2NC_MASK_SID = - -ASCII2NC_TIME_SUMMARY_FLAG = False -ASCII2NC_TIME_SUMMARY_RAW_DATA = False -ASCII2NC_TIME_SUMMARY_BEG = 000000 -ASCII2NC_TIME_SUMMARY_END = 235959 -ASCII2NC_TIME_SUMMARY_STEP = 300 -ASCII2NC_TIME_SUMMARY_WIDTH = 600 -ASCII2NC_TIME_SUMMARY_GRIB_CODES = 11, 204, 211 -ASCII2NC_TIME_SUMMARY_VAR_NAMES = -ASCII2NC_TIME_SUMMARY_TYPES = min, max, range, mean, stdev, median, p80 -ASCII2NC_TIME_SUMMARY_VALID_FREQ = 0 -ASCII2NC_TIME_SUMMARY_VALID_THRESH = 0.0 diff --git a/src/CSET/cset_workflow/app/metplus_ascii2nc/file/python/read_ascii_point.py b/src/CSET/cset_workflow/app/metplus_ascii2nc/file/python/read_ascii_point.py deleted file mode 100755 index f22251b13..000000000 --- a/src/CSET/cset_workflow/app/metplus_ascii2nc/file/python/read_ascii_point.py +++ /dev/null @@ -1,68 +0,0 @@ -"""Read ASCII point data for METplus.""" - -import os -import sys - -import pandas as pd -from met_point_obs import convert_point_data - -######################################################################## - -print(f"Python Script:\t{sys.argv[0]}") - -## -# input file specified on the command line -# load the data into the numpy array -## - -if len(sys.argv) != 2: - print("ERROR: read_ascii_point.py -> Must specify exactly one input file.") - sys.exit(1) - -# Read the input file as the first argument -input_file = os.path.expandvars(sys.argv[1]) -try: - print("Input File:\t" + repr(input_file)) - - # Read and format the input 11-column observations: - # (1) string: Message_Type - # (2) string: Station_ID - # (3) string: Valid_Time(YYYYMMDD_HHMMSS) - # (4) numeric: Lat(Deg North) - # (5) numeric: Lon(Deg East) - # (6) numeric: Elevation(msl) - # (7) string: Var_Name(or GRIB_Code) - # (8) numeric: Level - # (9) numeric: Height(msl or agl) - # (10) string: QC_String - # (11) numeric: Observation_Value - - point_data = pd.read_csv( - input_file, - header=None, - delim_whitespace=True, - keep_default_na=False, - names=[ - "typ", - "sid", - "vld", - "lat", - "lon", - "elv", - "var", - "lvl", - "hgt", - "qc", - "obs", - ], - dtype={"typ": "str", "sid": "str", "vld": "str", "var": "str", "qc": "str"}, - ).values.tolist() - print(f" point_data: Data Length:\t{len(point_data)}") - print(f" point_data: Data Type:\t{type(point_data)}") - met_point_data = convert_point_data(point_data) - print(f" met_point_data: Data Type:\t{type(met_point_data)}") -except NameError: - print("Can't find the input file") - sys.exit(1) - -######################################################################## diff --git a/src/CSET/cset_workflow/app/metplus_ascii2nc/opt/rose-app-niwa.conf b/src/CSET/cset_workflow/app/metplus_ascii2nc/opt/rose-app-niwa.conf index 63b665273..e5e0b7d11 100644 --- a/src/CSET/cset_workflow/app/metplus_ascii2nc/opt/rose-app-niwa.conf +++ b/src/CSET/cset_workflow/app/metplus_ascii2nc/opt/rose-app-niwa.conf @@ -1,9 +1,9 @@ [command] -default=app_env_wrapper run_metplus.py ASCII2NC_python_embedding.conf +default=app_env_wrapper run_metplus.py niwa/restricted_ASCII2NC_python_embedding_niwa.conf [env] -CONDA_VENV_LOCATION = ${CONDA_METPLUS_VENV_LOCATION} +CONDA_VENV_LOCATION=${CONDA_METPLUS_VENV_LOCATION} METPLUS_OBS_DIR=${METPLUS_OBS_DIR} INPUT_READ_SCRIPT=restricted_read_ascii_point_niwa.py -TASK_TIME=20230705T00 -TASK_ENDTIME=20230706T18 +START_TIME=${TASK_START_TIME} +END_TIME=${TASK_END_TIME} diff --git a/src/CSET/cset_workflow/app/metplus_point_stat/opt/rose-app-niwa.conf b/src/CSET/cset_workflow/app/metplus_point_stat/opt/rose-app-niwa.conf index c0fd2c647..284c8a64d 100644 --- a/src/CSET/cset_workflow/app/metplus_point_stat/opt/rose-app-niwa.conf +++ b/src/CSET/cset_workflow/app/metplus_point_stat/opt/rose-app-niwa.conf @@ -1,7 +1,8 @@ [command] -default=app_env_wrapper run_metplus.py restricted_PointStat_cycling_niwa.conf restricted_PointStat_science_niwa.conf +default=app_env_wrapper run_metplus.py niwa/restricted_PointStat_cycling_niwa.conf niwa/restricted_PointStat_science_niwa.conf [env] CONDA_VENV_LOCATION = ${CONDA_METPLUS_VENV_LOCATION} METPLUS_FCST_DIR = ${METPLUS_FCST_DIR} -TIME_START=20230705T00 +START_TIME=${TASK_START_TIME} +FORECAST_LENTGH = ${FORECAST_LENGTH} diff --git a/src/CSET/cset_workflow/flow.cylc b/src/CSET/cset_workflow/flow.cylc index bc2febf58..07aebcc31 100644 --- a/src/CSET/cset_workflow/flow.cylc +++ b/src/CSET/cset_workflow/flow.cylc @@ -114,15 +114,6 @@ final cycle point = {{CSET_TRIAL_END_DATE}} start_baking='start baking' skip_baking='skip baking' - [[METPLUS]] - [[[environment]]] - {% if METPLUS_GRID_STAT|default(False) %} - METPLUS_ANA_DIR = {{METPLUS_ANA_DIR}} - METPLUS_FCST_DIR = {{METPLUS_FCST_DIR}} - METPLUS_OBS_DIR = {{METPLUS_OBS_DIR}} - ROSE_APP_OPT_CONF_KEYS = {{METPLUS_OPT_CONFIG_KEYS}} - {% endif %} - # Noop tasks to ensure a complete/efficient workflow graph. [[DUMMY_TASK]] script = true @@ -217,5 +208,10 @@ final cycle point = {{CSET_TRIAL_END_DATE}} WEB_ADDR = {{WEB_ADDR}} WEB_DIR = {{WEB_DIR}} +# Include files bring their own graph and runtime sections. +{% include 'includes/metplus_common.cylc' %} +{% include 'includes/metplus_point_stat.cylc' %} +{% include 'includes/metplus_grid_stat.cylc' %} + # Site-specific details that add to or override the core workflow definition. {% include 'site/' ~ SITE ~ '.cylc' %} diff --git a/src/CSET/cset_workflow/includes/metplus_common.cylc b/src/CSET/cset_workflow/includes/metplus_common.cylc new file mode 100644 index 000000000..93beedf8d --- /dev/null +++ b/src/CSET/cset_workflow/includes/metplus_common.cylc @@ -0,0 +1,11 @@ +{% if METPLUS_GRID_STAT|default(False) or METPLUS_POINT_STAT|default(False) %} + [[METPLUS]] + [[[environment]]] + METPLUS_ANA_DIR = {{METPLUS_ANA_DIR}} + METPLUS_FCST_DIR = {{METPLUS_FCST_DIR}} + METPLUS_OBS_DIR = {{METPLUS_OBS_DIR}} + ROSE_APP_OPT_CONF_KEYS = {{METPLUS_OPT_CONFIG_KEYS}} + TASK_START_TIME = $(cylc cyclepoint --template CCYYMMDDThh) + FORECAST_LENGTH = $(isodatetime --as-total H -- {{ANALYSIS_LENGTH}}) + TASK_END_TIME = $(cylc cyclepoint --offset {{ANALYSIS_LENGTH}} --template CCYYMMDDThh) +{% endif %} diff --git a/src/CSET/cset_workflow/includes/metplus_point_stat.cylc b/src/CSET/cset_workflow/includes/metplus_point_stat.cylc new file mode 100644 index 000000000..eff3c71a0 --- /dev/null +++ b/src/CSET/cset_workflow/includes/metplus_point_stat.cylc @@ -0,0 +1,24 @@ +{% if METPLUS_POINT_STAT|default(False) %} +[scheduling] + [[graph]] + {% if CSET_CYCLING_MODE == "case_study" %} + {% for date in CSET_CASE_DATES %} + R1/{{date}} = """ + metplus_ascii2nc => metplus_point_stat + """ + {% endfor %} + {% elif CSET_CYCLING_MODE == "trial" %} + {{CSET_TRIAL_CYCLE_PERIOD}} = """ + metplus_ascii2nc => metplus_point_stat + """ + {% endif %} + +[runtime] + [[metplus_ascii2nc]] + # Runs METplus wrappers for ASCII to nc ingestion of obs. + inherit = METPLUS + + [[metplus_point_stat]] + # Runs METplus wrappers for point stat calculations. + inherit = METPLUS +{% endif %} diff --git a/src/CSET/cset_workflow/includes/point_stat.cylc b/src/CSET/cset_workflow/includes/point_stat.cylc deleted file mode 100644 index 19e03a034..000000000 --- a/src/CSET/cset_workflow/includes/point_stat.cylc +++ /dev/null @@ -1,16 +0,0 @@ -{% if METPLUS_POINT_STAT|default(False) %} -[scheduling] - [[graph]] - {{CSET_CYCLE_PERIOD}} = """ - metplus_ascii2nc => metplus_point_stat => housekeeping_full - """ - -[runtime] - [[metplus_ascii2nc]] - # Runs METplus wrappers for ASCII to nc ingestion of obs. - inherit = METPLUS - - [[metplus_point_stat]] - # Runs METplus wrappers for point stat calculations. - inherit = METPLUS -{% endif %}