From 888bb5ba5d8aca8813e47cbcd1f2cc0fd0859e38 Mon Sep 17 00:00:00 2001 From: Corey Godine Date: Wed, 1 Jul 2026 21:04:08 +0000 Subject: [PATCH 1/3] ADD: reader for mfrsr7nchaod files --- act/io/__init__.py | 1 + act/io/arm.py | 85 +++++++++++++++++++++++++++++++++++++++ act/tests/__init__.py | 1 + act/tests/sample_files.py | 7 ++++ tests/io/test_arm.py | 31 ++++++++++++++ 5 files changed, 125 insertions(+) diff --git a/act/io/__init__.py b/act/io/__init__.py index 2889b7edad..6af27c96c2 100644 --- a/act/io/__init__.py +++ b/act/io/__init__.py @@ -27,6 +27,7 @@ 'read_arm_netcdf', 'check_if_tar_gz_file', 'read_arm_mmcr', + 'read_arm_mfrsr7nchaod', ], 'ameriflux': ['convert_to_ameriflux', 'read_ameriflux'], 'text': ['read_csv'], diff --git a/act/io/arm.py b/act/io/arm.py index 7dbe5a7607..6b98331beb 100644 --- a/act/io/arm.py +++ b/act/io/arm.py @@ -908,3 +908,88 @@ def read_arm_mmcr(filenames): ds[new_var_name] = da return ds + + +def read_arm_mfrsr7nchaod(filenames): + """ + + Reads in mfrsr7nchaod files and merges along time dimension. MFRSR7NCHAOD files have + coordinate dimensions Io_interquartile_time and Io_gauss_time that cannot be concatenated + in xarray.open_mfdataset() due to overlapping values within adjacent files. + + Parameters + ---------- + filenames : str, pathlib.PosixPath, or list of str + Name(s) of fullpath(s) to read. + + Returns + ------- + ds : xarray.Dataset or None + ACT Xarray Dataset. If no files found will return None. + """ + + # Sort files in correct order for merging + if not isinstance(filenames, list): + filenames = [filenames] + filenames.sort() + + multi_ds = [] + + # Open Datasets of files and add to list + for f in filenames: + try: + obj = xr.open_dataset(f) + if obj is not None: + multi_ds.append(obj) + except FileNotFoundError: + continue + + # Merge objects + if len(multi_ds) > 1: + ds = xr.merge(multi_ds, compat="minimal") + elif len(multi_ds) == 1: + ds = multi_ds[0] + else: + ds = None + + # Adding support for wildcards + if isinstance(filenames, str): + filenames = glob.glob(filenames) + elif isinstance(filenames, PosixPath): + filenames = [filenames] + + # Adding attributes for file dates, file times, datastream, and flag for arm standards + # Get file dates and times that were read in to the dataset + file_dates = [] + file_times = [] + for f in filenames: + f = Path(f).name + pts = re.match(r'(^[a-zA-Z0-9]+)\.([0-9a-z]{2})\.([\d]{8})\.([\d]{6})\.([a-z]{2,3}$)', f) + # If Not ARM format, read in first time for info + if pts is not None: + pts = pts.groups() + file_dates.append(pts[2]) + file_times.append(pts[3]) + else: + if len(ds['time'].shape) > 0: + dummy = ds['time'].values[0] + else: + dummy = ds['time'].values + file_dates.append(utils.numpy_to_arm_date(dummy)) + file_times.append(utils.numpy_to_arm_date(dummy, returnTime=True)) + + # Add attributes + ds.attrs['_file_dates'] = file_dates + ds.attrs['_file_times'] = file_times + is_arm_file_flag = check_arm_standards(ds) + + # Ensure that we have _datastream set whether or no there's + # a datastream attribute already. + if is_arm_file_flag == 0: + ds.attrs['_datastream'] = DEFAULT_DATASTREAM_NAME + else: + ds.attrs['_datastream'] = ds.attrs['datastream'] + + ds.attrs['_arm_standards_flag'] = is_arm_file_flag + + return ds diff --git a/act/tests/__init__.py b/act/tests/__init__.py index dae86f3f34..114df67f54 100644 --- a/act/tests/__init__.py +++ b/act/tests/__init__.py @@ -67,6 +67,7 @@ 'EXAMPLE_AMERIFLUX_BASE', 'EXAMPLE_AMERIFLUX_META', 'EXAMPLE_SMPS', + 'EXAMPLE_MFRSR7NCHAOD', ] }, ) diff --git a/act/tests/sample_files.py b/act/tests/sample_files.py index df46a1ba1d..6150c0e8af 100644 --- a/act/tests/sample_files.py +++ b/act/tests/sample_files.py @@ -160,3 +160,10 @@ ) EXAMPLE_SMPS = DATASETS.fetch('houmergedsmpsapsmlM1.c1.20220801.000000.nc') + +mfrsr7nchaod_list = [ + 'sgpmfrsr7nchaod1michC1.c1.20260603.000000.nc', + 'sgpmfrsr7nchaod1michC1.c1.20260602.000000.nc', + 'sgpmfrsr7nchaod1michC1.c1.20260601.000000.nc', +] +EXAMPLE_MFRSR7NCHAOD = [DATASETS.fetch(file) for file in mfrsr7nchaod_list] diff --git a/tests/io/test_arm.py b/tests/io/test_arm.py index 1fc9c85dfa..cd3d1f61fd 100644 --- a/tests/io/test_arm.py +++ b/tests/io/test_arm.py @@ -293,3 +293,34 @@ def test_read_mmcr(): assert 'SpectralWidth_BL' in ds np.testing.assert_almost_equal(ds['Reflectivity_GE'].mean(), -34.62, decimal=2) np.testing.assert_almost_equal(ds['MeanDopplerVelocity_Receiver1'].max(), 9.98, decimal=2) + + +def test_read_mfrsr7nchaod(): + results = act.tests.EXAMPLE_MFRSR7NCHAOD + ds = act.io.arm.read_arm_mfrsr7nchaod(results) + ds.load() + + assert ds is not None + assert 'Io_gauss_time' in ds.coords + assert 'Io_interquartile_time' in ds.coords + assert isinstance(ds.attrs['_file_dates'], list) + assert len(ds.attrs['_file_dates']) == 3 + assert isinstance(ds.attrs['_file_times'], list) + assert len(ds.attrs['_file_times']) == 3 + assert ds.attrs['_datastream'] == 'sgpmfrsr7nchaod1michC1.c1' + assert ds.attrs['_arm_standards_flag'] == 1 + + ds.close() + del ds + + ds = act.io.arm.read_arm_mfrsr7nchaod(results[0]) + ds.load() + + assert ds is not None + assert isinstance(ds.attrs['_file_dates'], list) + assert len(ds.attrs['_file_dates']) == 1 + assert isinstance(ds.attrs['_file_times'], list) + assert len(ds.attrs['_file_times']) == 1 + + ds.close() + del ds From 8842fca771a7073d1920ca911a5a4a760af277ca Mon Sep 17 00:00:00 2001 From: Corey Godine Date: Mon, 6 Jul 2026 19:06:50 +0000 Subject: [PATCH 2/3] FIX: add exception handling for when CAPE and CIN cannot be calculated --- act/retrievals/sonde.py | 18 +++++++++++++++--- tests/retrievals/test_sonde.py | 2 +- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/act/retrievals/sonde.py b/act/retrievals/sonde.py index e739ac159e..56ca972415 100644 --- a/act/retrievals/sonde.py +++ b/act/retrievals/sonde.py @@ -172,16 +172,28 @@ def calculate_stability_indicies( ds['parcel_temperature'] = t_profile.magnitude ds['parcel_temperature'].attrs['units'] = t_profile.units - # Calculate CAPE, CIN, LCL - sbcape, sbcin = mpcalc.surface_based_cape_cin(p_sorted, t_sorted, td_sorted) + # Calculate SBCAPE, SBCIN + try: + sbcape, sbcin = mpcalc.surface_based_cape_cin(p_sorted, t_sorted, td_sorted) + except IndexError: + sbcape = units.Quantity(np.nan, 'J/kg') + sbcin = units.Quantity(np.nan, 'J/kg') + # Calculate lifting condensation level lcl = mpcalc.lcl(p_sorted[0], t_sorted[0], td_sorted[0]) + + # Calculate level of free convection try: lfc = mpcalc.lfc(p_sorted[0], t_sorted[0], td_sorted[0]) except IndexError: lfc = np.nan * p_sorted.units - mucape, mucin = mpcalc.most_unstable_cape_cin(p_sorted, t_sorted, td_sorted) + # Calculate MUCAPE, MUCIN + try: + mucape, mucin = mpcalc.most_unstable_cape_cin(p_sorted, t_sorted, td_sorted) + except IndexError: + mucape = units.Quantity(np.nan, 'J/kg') + mucin = units.Quantity(np.nan, 'J/kg') where_500 = np.argmin(np.abs(p_sorted - 500 * units.hPa)) li = t_sorted[where_500] - t_profile[where_500] diff --git a/tests/retrievals/test_sonde.py b/tests/retrievals/test_sonde.py index 629430b387..9f92531151 100644 --- a/tests/retrievals/test_sonde.py +++ b/tests/retrievals/test_sonde.py @@ -15,7 +15,7 @@ def test_get_stability_indices(): rtol=1e-5, ) assert sonde_ds['parcel_temperature'].attrs['units'] == 'kelvin' - np.testing.assert_almost_equal(sonde_ds['surface_based_cape'], 0.98, decimal=2) + np.testing.assert_almost_equal(sonde_ds['surface_based_cape'], 0.96, decimal=2) assert sonde_ds['surface_based_cape'].attrs['units'] == 'J/kg' assert sonde_ds['surface_based_cape'].attrs['long_name'] == 'Surface-based CAPE' np.testing.assert_almost_equal(sonde_ds['surface_based_cin'], 0.000, decimal=3) From ec762c7d9361926d4b330b662690c4bbdcf5d633 Mon Sep 17 00:00:00 2001 From: Corey Godine Date: Wed, 8 Jul 2026 20:42:01 +0000 Subject: [PATCH 3/3] FIX: revert changes rolled over from separate pull request --- act/io/__init__.py | 1 - act/io/arm.py | 85 --------------------------------------- act/tests/__init__.py | 1 - act/tests/sample_files.py | 7 ---- tests/io/test_arm.py | 31 -------------- 5 files changed, 125 deletions(-) diff --git a/act/io/__init__.py b/act/io/__init__.py index 6af27c96c2..2889b7edad 100644 --- a/act/io/__init__.py +++ b/act/io/__init__.py @@ -27,7 +27,6 @@ 'read_arm_netcdf', 'check_if_tar_gz_file', 'read_arm_mmcr', - 'read_arm_mfrsr7nchaod', ], 'ameriflux': ['convert_to_ameriflux', 'read_ameriflux'], 'text': ['read_csv'], diff --git a/act/io/arm.py b/act/io/arm.py index 6b98331beb..7dbe5a7607 100644 --- a/act/io/arm.py +++ b/act/io/arm.py @@ -908,88 +908,3 @@ def read_arm_mmcr(filenames): ds[new_var_name] = da return ds - - -def read_arm_mfrsr7nchaod(filenames): - """ - - Reads in mfrsr7nchaod files and merges along time dimension. MFRSR7NCHAOD files have - coordinate dimensions Io_interquartile_time and Io_gauss_time that cannot be concatenated - in xarray.open_mfdataset() due to overlapping values within adjacent files. - - Parameters - ---------- - filenames : str, pathlib.PosixPath, or list of str - Name(s) of fullpath(s) to read. - - Returns - ------- - ds : xarray.Dataset or None - ACT Xarray Dataset. If no files found will return None. - """ - - # Sort files in correct order for merging - if not isinstance(filenames, list): - filenames = [filenames] - filenames.sort() - - multi_ds = [] - - # Open Datasets of files and add to list - for f in filenames: - try: - obj = xr.open_dataset(f) - if obj is not None: - multi_ds.append(obj) - except FileNotFoundError: - continue - - # Merge objects - if len(multi_ds) > 1: - ds = xr.merge(multi_ds, compat="minimal") - elif len(multi_ds) == 1: - ds = multi_ds[0] - else: - ds = None - - # Adding support for wildcards - if isinstance(filenames, str): - filenames = glob.glob(filenames) - elif isinstance(filenames, PosixPath): - filenames = [filenames] - - # Adding attributes for file dates, file times, datastream, and flag for arm standards - # Get file dates and times that were read in to the dataset - file_dates = [] - file_times = [] - for f in filenames: - f = Path(f).name - pts = re.match(r'(^[a-zA-Z0-9]+)\.([0-9a-z]{2})\.([\d]{8})\.([\d]{6})\.([a-z]{2,3}$)', f) - # If Not ARM format, read in first time for info - if pts is not None: - pts = pts.groups() - file_dates.append(pts[2]) - file_times.append(pts[3]) - else: - if len(ds['time'].shape) > 0: - dummy = ds['time'].values[0] - else: - dummy = ds['time'].values - file_dates.append(utils.numpy_to_arm_date(dummy)) - file_times.append(utils.numpy_to_arm_date(dummy, returnTime=True)) - - # Add attributes - ds.attrs['_file_dates'] = file_dates - ds.attrs['_file_times'] = file_times - is_arm_file_flag = check_arm_standards(ds) - - # Ensure that we have _datastream set whether or no there's - # a datastream attribute already. - if is_arm_file_flag == 0: - ds.attrs['_datastream'] = DEFAULT_DATASTREAM_NAME - else: - ds.attrs['_datastream'] = ds.attrs['datastream'] - - ds.attrs['_arm_standards_flag'] = is_arm_file_flag - - return ds diff --git a/act/tests/__init__.py b/act/tests/__init__.py index 114df67f54..dae86f3f34 100644 --- a/act/tests/__init__.py +++ b/act/tests/__init__.py @@ -67,7 +67,6 @@ 'EXAMPLE_AMERIFLUX_BASE', 'EXAMPLE_AMERIFLUX_META', 'EXAMPLE_SMPS', - 'EXAMPLE_MFRSR7NCHAOD', ] }, ) diff --git a/act/tests/sample_files.py b/act/tests/sample_files.py index 6150c0e8af..df46a1ba1d 100644 --- a/act/tests/sample_files.py +++ b/act/tests/sample_files.py @@ -160,10 +160,3 @@ ) EXAMPLE_SMPS = DATASETS.fetch('houmergedsmpsapsmlM1.c1.20220801.000000.nc') - -mfrsr7nchaod_list = [ - 'sgpmfrsr7nchaod1michC1.c1.20260603.000000.nc', - 'sgpmfrsr7nchaod1michC1.c1.20260602.000000.nc', - 'sgpmfrsr7nchaod1michC1.c1.20260601.000000.nc', -] -EXAMPLE_MFRSR7NCHAOD = [DATASETS.fetch(file) for file in mfrsr7nchaod_list] diff --git a/tests/io/test_arm.py b/tests/io/test_arm.py index cd3d1f61fd..1fc9c85dfa 100644 --- a/tests/io/test_arm.py +++ b/tests/io/test_arm.py @@ -293,34 +293,3 @@ def test_read_mmcr(): assert 'SpectralWidth_BL' in ds np.testing.assert_almost_equal(ds['Reflectivity_GE'].mean(), -34.62, decimal=2) np.testing.assert_almost_equal(ds['MeanDopplerVelocity_Receiver1'].max(), 9.98, decimal=2) - - -def test_read_mfrsr7nchaod(): - results = act.tests.EXAMPLE_MFRSR7NCHAOD - ds = act.io.arm.read_arm_mfrsr7nchaod(results) - ds.load() - - assert ds is not None - assert 'Io_gauss_time' in ds.coords - assert 'Io_interquartile_time' in ds.coords - assert isinstance(ds.attrs['_file_dates'], list) - assert len(ds.attrs['_file_dates']) == 3 - assert isinstance(ds.attrs['_file_times'], list) - assert len(ds.attrs['_file_times']) == 3 - assert ds.attrs['_datastream'] == 'sgpmfrsr7nchaod1michC1.c1' - assert ds.attrs['_arm_standards_flag'] == 1 - - ds.close() - del ds - - ds = act.io.arm.read_arm_mfrsr7nchaod(results[0]) - ds.load() - - assert ds is not None - assert isinstance(ds.attrs['_file_dates'], list) - assert len(ds.attrs['_file_dates']) == 1 - assert isinstance(ds.attrs['_file_times'], list) - assert len(ds.attrs['_file_times']) == 1 - - ds.close() - del ds