From 48b4a3f75b3abb92bb1350981ed95f9b5a94b49f Mon Sep 17 00:00:00 2001 From: Ron McLaren Date: Tue, 8 Oct 2024 12:27:24 -0400 Subject: [PATCH 01/72] added dump diur --- dump/mapping/make_files_please.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 dump/mapping/make_files_please.txt diff --git a/dump/mapping/make_files_please.txt b/dump/mapping/make_files_please.txt new file mode 100644 index 0000000..e69de29 From df491148b23e2056102991f1e1a61f26015973ff Mon Sep 17 00:00:00 2001 From: Nicholas Esposito Date: Thu, 31 Oct 2024 14:42:40 -0400 Subject: [PATCH 02/72] ADPSFC mapping and encoder file --- .../iodatest_prepbufr_adpsfc_encoder.py | 60 ++++++ .../iodatest_prepbufr_adpsfc_mapping.yaml | 174 ++++++++++++++++++ 2 files changed, 234 insertions(+) create mode 100755 dump/mapping/iodatest_prepbufr_adpsfc_encoder.py create mode 100755 dump/mapping/iodatest_prepbufr_adpsfc_mapping.yaml diff --git a/dump/mapping/iodatest_prepbufr_adpsfc_encoder.py b/dump/mapping/iodatest_prepbufr_adpsfc_encoder.py new file mode 100755 index 0000000..37f4abb --- /dev/null +++ b/dump/mapping/iodatest_prepbufr_adpsfc_encoder.py @@ -0,0 +1,60 @@ +import os +import sys +sys.path.append('/work2/noaa/da/nesposito/backend_20240701/bufr_query/build/lib/python3.10') +sys.path.append('/work2/noaa/da/nesposito/backend_20240701/ioda-bundle/build/lib/python3.10') +sys.path.append('/work2/noaa/da/nesposito/backend_20240701/ioda-bundle/build/lib/python3.10/pyioda') +sys.path.append('/work2/noaa/da/nesposito/backend_20240701/ioda-bundle/build/lib/python3.10/pyiodaconv') +import bufr +from pyioda.ioda.Engines.Bufr import Encoder +import copy +import numpy as np +import numpy.ma as ma +import math +import calendar +import time +from datetime import datetime + + +def Compute_dateTime(cycleTimeSinceEpoch, dhr): + + int64_fill_value = np.int64(0) + + dateTime = np.zeros(dhr.shape, dtype=np.int64) + for i in range(len(dateTime)): + if ma.is_masked(dhr[i]): + continue + else: + dateTime[i] = np.int64(dhr[i]*3600) + cycleTimeSinceEpoch + + dateTime = ma.array(dateTime) + dateTime = ma.masked_values(dateTime, int64_fill_value) + + return dateTime + +def create_obs_group(cycle_time,input_mapping,input_path): + CYCLE_TIME = cycle_time + YAML_PATH = input_mapping #"./iodatest_prepbufr_adpsfc_mapping.yaml" + INPUT_PATH = input_path + print(" CYCLE_TIME: ", CYCLE_TIME) + + container = bufr.Parser(INPUT_PATH, YAML_PATH).parse() + + print(" Do DateTime calculation") + otmct = container.get('variables/obsTimeMinusCycleTime') + otmct_paths = container.get_paths('variables/obsTimeMinusCycleTime') + otmct2 = np.array(otmct) + cycleTimeSinceEpoch = np.int64(calendar.timegm(time.strptime(str(int(CYCLE_TIME)), '%Y%m%d%H'))) + dateTime = Compute_dateTime(cycleTimeSinceEpoch, otmct2) + + print("Make an array of 0s for MetaData/sequenceNumber") + sequenceNum = np.zeros(otmct.shape, dtype=np.int32) + + print(" Add variables to container.") + container.add('variables/dateTime', dateTime, otmct_paths) + container.add('variables/sequenceNumber', sequenceNum, otmct_paths) + description = bufr.encoders.Description(YAML_PATH) + + print(" Add container and descriptions to dataset.") + dataset = next(iter(Encoder(description).encode(container).values())) + return dataset + diff --git a/dump/mapping/iodatest_prepbufr_adpsfc_mapping.yaml b/dump/mapping/iodatest_prepbufr_adpsfc_mapping.yaml new file mode 100755 index 0000000..e99a958 --- /dev/null +++ b/dump/mapping/iodatest_prepbufr_adpsfc_mapping.yaml @@ -0,0 +1,174 @@ +# (C) Copyright 2023 NOAA/NWS/NCEP/EMC +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +bufr: + variables: + # ObsType + observationType: + query: "*/TYP" + + # MetaData + obsTimeMinusCycleTime: + query: "*/DHR" + longitude: + query: "*/XOB" + latitude: + query: "*/YOB" + stationIdentification: + query: "*/SID" + pressure: + query: "*/P___INFO/P__EVENT{1}/POB" + transforms: + - scale: 100 + height: + query: "*/Z___INFO/Z__EVENT{1}/ZOB" + type: float + stationElevation: + query: "*/ELV" + type: float + + # ObsValue + stationPressureObsValue: + query: "*/P___INFO/P__EVENT{1}/POB" + transforms: + - scale: 100 + heightObsValue: + query: "*/Z___INFO/Z__EVENT{1}/ZOB" + type: float + stationElevationObsValue: + query: "*/ELV" + type: float + + # QualityMarker + stationPressureQualityMarker: + query: "*/P___INFO/P__EVENT{1}/PQM" + heightQualityMarker: + query: "*/Z___INFO/Z__EVENT{1}/ZQM" + stationElevationQualityMarker: + query: "*/Z___INFO/Z__EVENT{1}/ZQM" + + # ObsError + stationPressureObsError: + query: "*/P___INFO/P__BACKG/POE" + transforms: + - scale: 100 + +encoder: + + globals: + - name: "data_format" + type: string + value: "prepbufr" + + - name: "subsets" + type: string + value: "ADPSFC" + + - name: "source" + type: string + value: "prepBUFR" + + - name: "data_type" + type: string + value: "ADPSFC" + + - name: "data_description" + type: string + value: "ADPSFC_prepbufr" + + - name: "data_provider" + type: string + value: "U.S. NOAA" + + variables: + + # ObsType + - name: "ObsType/stationPressure" + source: variables/observationType + longName: "Station Pressure ObsType" + + - name: "ObsType/stationElevation" + source: variables/observationType + longName: "Station Elevation ObsType" + + - name: "ObsType/height" + source: variables/observationType + longName: "Height ObsType" + + # MetaData + - name: "MetaData/dateTime" + source: variables/dateTime + units: 'seconds since 1970-01-01T00:00:00Z' + longName: 'dateTime' + + - name: "MetaData/latitude" + source: variables/latitude + longName: "Latitude" + units: "degree_north" + range: [-90, 90] + + - name: "MetaData/longitude" + source: variables/longitude + longName: "Longitude" + units: "degree_east" + range: [0, 360] + + - name: "MetaData/sequenceNumber" + source: variables/sequenceNumber + longName: "Sequence Number (Obs Subtype)" + + - name: "MetaData/stationIdentification" + source: variables/stationIdentification + longName: "Station Identification" + + - name: "MetaData/pressure" + source: variables/pressure + longName: "Pressure" + + - name: "MetaData/height" + source: variables/height + longName: "Height" + units: "m" + + - name: "MetaData/stationElevation" + source: variables/stationElevation + longName: "Station Elevation" + units: "m" + + # ObsValue + - name: "ObsValue/stationPressure" + source: variables/stationPressureObsValue + longName: "Station Pressure" + units: "Pa" + + - name: "ObsValue/stationElevation" + source: variables/stationElevationObsValue + longName: "Station Pressure" + units: "m" + + - name: "ObsValue/height" + source: variables/heightObsValue + longName: "height" + units: "m" + + # QualityMarker + - name: "QualityMarker/stationPressure" + source: variables/stationPressureQualityMarker + longName: "Station Pressure Quality Marker" + + - name: "QualityMarker/stationElevation" + source: variables/heightQualityMarker + longName: "StationElevation Quality Marker" + + - name: "QualityMarker/height" + source: variables/heightQualityMarker + longName: "Height Quality Marker" + + # ObsError + - name: "ObsError/stationPressure" + source: variables/stationPressureObsError + longName: "Station Pressure Error" + units: "Pa" + From f13cc3ddad73c27f85cebcefeee3baec3e278b9d Mon Sep 17 00:00:00 2001 From: Nicholas Esposito Date: Thu, 31 Oct 2024 14:50:31 -0400 Subject: [PATCH 03/72] remove path appends --- dump/mapping/iodatest_prepbufr_adpsfc_encoder.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/dump/mapping/iodatest_prepbufr_adpsfc_encoder.py b/dump/mapping/iodatest_prepbufr_adpsfc_encoder.py index 37f4abb..d4f88ca 100755 --- a/dump/mapping/iodatest_prepbufr_adpsfc_encoder.py +++ b/dump/mapping/iodatest_prepbufr_adpsfc_encoder.py @@ -1,9 +1,5 @@ import os import sys -sys.path.append('/work2/noaa/da/nesposito/backend_20240701/bufr_query/build/lib/python3.10') -sys.path.append('/work2/noaa/da/nesposito/backend_20240701/ioda-bundle/build/lib/python3.10') -sys.path.append('/work2/noaa/da/nesposito/backend_20240701/ioda-bundle/build/lib/python3.10/pyioda') -sys.path.append('/work2/noaa/da/nesposito/backend_20240701/ioda-bundle/build/lib/python3.10/pyiodaconv') import bufr from pyioda.ioda.Engines.Bufr import Encoder import copy From 51928b5e7f7973763e3ed364de91fbbdfe26ac08 Mon Sep 17 00:00:00 2001 From: Nicholas Esposito Date: Thu, 31 Oct 2024 15:50:57 -0400 Subject: [PATCH 04/72] CAT mnemonic --- dump/mapping/iodatest_prepbufr_adpsfc_mapping.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/dump/mapping/iodatest_prepbufr_adpsfc_mapping.yaml b/dump/mapping/iodatest_prepbufr_adpsfc_mapping.yaml index e99a958..b423444 100755 --- a/dump/mapping/iodatest_prepbufr_adpsfc_mapping.yaml +++ b/dump/mapping/iodatest_prepbufr_adpsfc_mapping.yaml @@ -10,6 +10,8 @@ bufr: query: "*/TYP" # MetaData + prepbufrDataLevelCategory: + query: "*/CAT" obsTimeMinusCycleTime: query: "*/DHR" longitude: @@ -98,6 +100,11 @@ encoder: longName: "Height ObsType" # MetaData + - name: "MetaData/prepbufrDataLevelCategory" + source: variables/prepbufrDataLevelCategory + units: '1' + longName: "Prepbufr Data Level Category" + - name: "MetaData/dateTime" source: variables/dateTime units: 'seconds since 1970-01-01T00:00:00Z' From d3478fda38c876004396185f60bacb9d147bf792 Mon Sep 17 00:00:00 2001 From: Emily Liu Date: Wed, 13 Nov 2024 19:48:45 +0000 Subject: [PATCH 05/72] Add mapping file, python scripts, and shell scripts for satwind goes --- .../bufr2ioda_script_backend_satwnd.yaml | 61 ++++++ dump/mapping/bufr2ioda_satwnd_amv_goes.py | 183 +++++++++++++++++ .../bufr2ioda_satwnd_amv_goes_mapping.yaml | 189 +++++++++++++++++ dump/mapping/bufr2netcdf_satwnd_amv_goes.py | 190 ++++++++++++++++++ ush/test/process_script2netcdf | 48 +++++ ush/test/process_script_backend | 44 ++++ 6 files changed, 715 insertions(+) create mode 100644 dump/config/bufr2ioda_script_backend_satwnd.yaml create mode 100644 dump/mapping/bufr2ioda_satwnd_amv_goes.py create mode 100755 dump/mapping/bufr2ioda_satwnd_amv_goes_mapping.yaml create mode 100755 dump/mapping/bufr2netcdf_satwnd_amv_goes.py create mode 100755 ush/test/process_script2netcdf create mode 100755 ush/test/process_script_backend diff --git a/dump/config/bufr2ioda_script_backend_satwnd.yaml b/dump/config/bufr2ioda_script_backend_satwnd.yaml new file mode 100644 index 0000000..c7d965e --- /dev/null +++ b/dump/config/bufr2ioda_script_backend_satwnd.yaml @@ -0,0 +1,61 @@ +time window: + begin: "2018-04-14T21:00:00Z" + end: "2023-12-15T03:00:00Z" + +observations: +- obs space: + name: "satwind_goes-16" + observed variables: [windSpeed, windDirection] + derived variables: [windEastward, windNorthward] + simulated variables: [windEastward, windNorthward] + obsdatain: + engine: + type: script + script file: "bufr2ioda_satwind_amv_goes.py" + args: + input_path: "./testinput/gdas.t00z.satwnd.tm00.bufr_d" + mapping_path: "./bufr2ioda_satwind_amv_goes_mapping.yaml" + category: "goes-16" + log_level: "DEBUG" + obsdataout: + engine: + type: H5File + obsfile: "./testoutput/gdas.t00z.satwnd.abi_goes-16.tm00.nc" + +- obs space: + name: "satwind_goes-17" + observed variables: [windSpeed, windDirection] + derived variables: [windEastward, windNorthward] + simulated variables: [windEastward, windNorthward] + obsdatain: + engine: + type: script + script file: "bufr2ioda_satwind_amv_goes.py" + args: + input_path: "./testinput/gdas.t00z.satwnd.tm00.bufr_d" + mapping_path: "./bufr2ioda_satwind_amv_goes_mapping.yaml" + category: "goes-17" + log_level: "DEBUG" + obsdataout: + engine: + type: H5File + obsfile: "./testoutput/gdas.t00z.satwnd.abi_goes-17.tm00.nc" + +- obs space: + name: "satwind_goes-18" + observed variables: [windSpeed, windDirection] + derived variables: [windEastward, windNorthward] + simulated variables: [windEastward, windNorthward] + obsdatain: + engine: + type: script + script file: "bufr2ioda_satwind_amv_goes.py" + args: + input_path: "./testinput/gdas.t00z.satwnd.tm00.bufr_d" + mapping_path: "./bufr2ioda_satwind_amv_goes_mapping.yaml" + category: "goes-18" + log_level: "DEBUG" + obsdataout: + engine: + type: H5File + obsfile: "./testoutput/gdas.t00z.satwnd.abi_goes-18.tm00.nc" diff --git a/dump/mapping/bufr2ioda_satwnd_amv_goes.py b/dump/mapping/bufr2ioda_satwnd_amv_goes.py new file mode 100644 index 0000000..3522776 --- /dev/null +++ b/dump/mapping/bufr2ioda_satwnd_amv_goes.py @@ -0,0 +1,183 @@ +import bufr +from pyioda.ioda.Engines.Bufr import Encoder +import numpy as np +import warnings +from wxflow import Logger + +def get_description(mapping_path, update=False): + + description = bufr.encoders.Description(mapping_path) + + if update: + + description.add_variable(name='ObsType/windEastward', + source='variables/obstype_uwind', + units='1', + longName='Observation Type based on Satellite-derived Wind Computation Method and Spectral Band') + + description.add_variable(name='ObsType/windNorthward', + source='variables/obstype_vwind', + units='1', + longName='Observation Type based on Satellite-derived Wind Computation Method and Spectral Band') + + description.add_variable(name='ObsValue/windEastward', + source='variables/windEastward', + units='m/s', + longName='Eastward Wind Component') + + description.add_variable(name='ObsValue/windNorthward', + source='variables/windNorthward', + units='m/s', + longName='Northward Wind Component') + + return description + +def get_all_keys(nested_dict): + + keys = [] + for key, value in nested_dict.items(): + keys.append(key) + if isinstance(value, dict): + keys.extend(get_all_keys(value)) + return keys + +def Compute_WindComponents_from_WindDirection_and_WindSpeed(wdir, wspd): + + uob = (-wspd * np.sin(np.radians(wdir))).astype(np.float32) + vob = (-wspd * np.cos(np.radians(wdir))).astype(np.float32) + + return uob, vob + +def Get_ObsType(swcm, chanfreq): + + obstype = swcm.copy() + + # Use numpy vectorized operations + obstype = np.where(swcm == 5, 247, obstype) # WVCA/DL + obstype = np.where(swcm == 3, 246, obstype) # WVCT + obstype = np.where(swcm == 2, 251, obstype) # VIS + obstype = np.where(swcm == 1, 245, obstype) # IRLW + + condition = np.logical_and(swcm == 1, chanfreq >= 50000000000000.0) # IRSW + obstype = np.where(condition, 240, obstype) + + if not np.any(np.isin(obstype, [247, 246, 251, 245, 240])): + raise ValueError("Error: Unassigned ObsType found ... ") + + return obstype + +def create_obs_group(input_path, mapping_path, category, env, log_level=None ): + + # Set MPI parameters + comm = bufr.mpi.Comm(env["comm_name"]) + rank = comm.rank() + + # Set logger level + if log_level is None: + log_level = 'INFO' + elif log_level not in ['DEBUG', 'INFO']: + raise ValueError("Invalid log level! log_level must be 'DEBUG' or 'INFO'.") + + # Initialize logger + logger = Logger('BUFR2IODA_satwnd_amv_goes.py', level=log_level, colored_log=False) + + # Generate keys for cache + inputKey = input_path + mappingKey = mapping_path + + if rank == 0: logger.info(f'Create obs group for category: {category}') + + # Check the cache for the data and return it if it exists + if rank == 0: logger.debug(f'Check if bufr.DataCache exists? {bufr.DataCache.has(inputKey, mappingKey)}') + if bufr.DataCache.has(inputKey, mappingKey): + container = bufr.DataCache.get(inputKey, mappingKey) + if rank == 0: logger.info(f'Encode {category} from cache') + data = Encoder(get_description(mapping_path, update=True)).encode(container)[(category,)] + if rank == 0: logger.info(f'Mark {category} as finished in the cache') + bufr.DataCache.mark_finished(inputKey, mappingKey, [category]) + if rank == 0: logger.info(f'Return the encoded data for {category}') + return data + + if rank == 0: logger.info(f'Get all cagegories info cache') + # If cache does not exist, get data into cache + # Get container from mapping file first + if rank == 0: logger.info(f'Get container from bufr parser') + container = bufr.Parser(input_path, mapping_path).parse(comm) + if rank == 0: logger.debug('container list (original):\n' + '\n'.join(str(item) for item in container.list())) + if rank == 0: logger.debug(f'all_sub_categories = {container.all_sub_categories()}') + if rank == 0: logger.debug(f'category map = {container.get_category_map()}') + + categories = container.all_sub_categories() + for cat in categories: + + if rank == 0: logger.debug(f'category = {cat}') + description = get_description(mapping_path, update=True) + + satid = container.get('variables/satelliteId', cat) + if satid.size == 0: + if rank == 0: logger.info(f'category {cat[0]} does not exist in input file') + paths = container.get_paths('variables/windComputationMethod', cat) + obstype = container.get('variables/windComputationMethod', cat) + container.add('variables/obstype_uwind', obstype, paths, cat) + container.add('variables/obstype_vwind', obstype, paths, cat) + + paths = container.get_paths('variables/windSpeed', cat) + wob = container.get('variables/windSpeed', cat) + container.add('variables/windEastward', wob, paths, cat) + container.add('variables/windNorthward', wob, paths, cat) + + else: + # Add new variables: ObsType/windEastward & ObsType/windNorthward + swcm = container.get('variables/windComputationMethod', cat) + chanfreq = container.get('variables/sensorCentralFrequency', cat) + + if rank == 0: logger.debug(f'swcm min/max = {swcm.min()} {swcm.max()}') + if rank == 0: logger.debug(f'chanfreq min/max = {chanfreq.min()} {chanfreq.max()}') + + obstype = Get_ObsType(swcm, chanfreq) + + if rank == 0: logger.debug(f'obstype = {obstype}') + if rank == 0: logger.debug(f'obstype min/max = {obstype.min()} {obstype.max()}') + + paths = container.get_paths('variables/windComputationMethod', cat) + container.add('variables/obstype_uwind', obstype, paths, cat) + container.add('variables/obstype_vwind', obstype, paths, cat) + + # Add new variables: ObsValue/windEastward & ObsValue/windNorthward + wdir = container.get('variables/windDirection', cat) + wspd = container.get('variables/windSpeed', cat) + + if rank == 0: logger.debug(f'wdir min/max = {wdir.min()} {wdir.max()}') + if rank == 0: logger.debug(f'wspd min/max = {wspd.min()} {wspd.max()}') + + uob, vob = Compute_WindComponents_from_WindDirection_and_WindSpeed(wdir, wspd) + + if rank == 0: logger.debug(f'uob min/max = {uob.min()} {uob.max()}') + if rank == 0: logger.debug(f'vob min/max = {vob.min()} {vob.max()}') + + paths = container.get_paths('variables/windSpeed', cat) + container.add('variables/windEastward', uob, paths, cat) + container.add('variables/windNorthward', vob, paths, cat) + + # Check + if rank == 0: logger.debug('container list (updated):\n' + '\n'.join(str(item) for item in container.list())) + if rank == 0: logger.debug(f'all_sub_categories {container.all_sub_categories()}') + + # Gather data from all tasks into all tasks. Each task will have the complete record + if rank == 0: logger.info(f'Gather data from all tasks into all tasks') + container.all_gather(comm) + + if rank == 0: logger.info(f'Add container to cache') + # Add the container to the cache + bufr.DataCache.add(inputKey, mappingKey, container.all_sub_categories(), container) + + # Encode the data + if rank == 0: logger.info(f'Encode {category}') + data = Encoder(description).encode(container)[(category,)] + + if rank == 0: logger.info(f'Mark {category} as finished in the cache') + # Mark the data as finished in the cache + bufr.DataCache.mark_finished(inputKey, mappingKey, [category]) + + if rank == 0: logger.info(f'Return the encoded data for {category}') + return data diff --git a/dump/mapping/bufr2ioda_satwnd_amv_goes_mapping.yaml b/dump/mapping/bufr2ioda_satwnd_amv_goes_mapping.yaml new file mode 100755 index 0000000..7815b81 --- /dev/null +++ b/dump/mapping/bufr2ioda_satwnd_amv_goes_mapping.yaml @@ -0,0 +1,189 @@ +bufr: + subsets: + - NC005030 + - NC005031 + - NC005032 + - NC005034 + - NC005039 + + variables: + # MetaData + timestamp: + datetime: + year: "*/YEAR" + month: "*/MNTH" + day: "*/DAYS" + hour: "*/HOUR" + minute: "*/MINU" + second: "*/SECO" + + latitude: + query: "*/CLATH" + + longitude: + query: "*/CLONH" + + satelliteId: + query: "*/SAID" + + satelliteZenithAngle: + query: "*/SAZA" + + sensorCentralFrequency: + query: "*/SCCF" + + pressure: + query: "*/PRLC[1]" + + # Processing Center + dataProviderOrigin: + query: '*/OGCE[1]' + + # Quality Information - Quality Indicator + qualityInformationWithoutForecast: + query: '*/AMVQIC{2}/PCCF' + + # Quality Information - Expected Error + expectedError: + query: '*/AMVQIC{4}/PCCF' + + # Derived Motion Wind (DMW) Intermediate Vectors - Coefficient of Variation + coefficientOfVariation: + query: '*/AMVIVR{1}/CVWD' + + # Wind Retrieval Method Information - Computation + windComputationMethod: + query: '*/SWCM' + + # Wind Retrieval Method Information - Hight Assignment + windHeightAssignMethod: + query: '*/EHAM' + + # ObsValue - Wind Direction + windDirection: + query: '*/WDIR' + + # ObsValue - Wind Speed + windSpeed: + query: '*/WSPD' + + splits: + satId: + category: + variable: satelliteId + map: + _270: goes-16 + _271: goes-17 + _272: goes-18 + +encoder: + type: netcdf + +# dimensions: + + globals: + - name: "platformCommonName" + type: string + value: "Meteorological Operational Satellite" + + - name: "platformLongDescription" + type: string + value: "EUMETSAT Polar System in sunsynchronous orbit" + + - name: "source" + type: string + value: "MTYP 021-241 IASI 1C RADIANCES (VARIABLE CHNS) (METOP)" + + - name: "sourceFiles" + type: string + value: "gdas.t00z.mtiasi.tm00.bufr_d" + + - name: "datetimeReference" + type: string + value: "2021-08-01T00:00:00Z" + + - name: "sensor" + type: string + value: "IASI" + + - name: "processingLevel" + type: string + value: "Level-1C" + + - name: "converter" + type: string + value: "BUFR" + + variables: + # MetaData + - name: "MetaData/dateTime" + source: variables/timestamp + longName: "Datetime" + units: "seconds since 1970-01-01T00:00:00Z" + + - name: "MetaData/latitude" + source: variables/latitude + longName: "Latitude" + units: "degrees_north" + range: [ -90, 90 ] + + - name: "MetaData/longitude" + source: variables/longitude + longName: "Longitude" + units: "degrees_east" + range: [ -180, 180 ] + + - name: "MetaData/satelliteIdentifier" + source: variables/satelliteId + longName: "Satellite Identifier" + + - name: "MetaData/satelliteZenithAngle" + source: variables/satelliteZenithAngle + longName: "Satellite Zenith Angle" + units: "degree" + range: [ 0, 90 ] + + - name: "MetaData/sensorCentralFrequency" + source: variables/sensorCentralFrequency + longName: "Satellite Channel Center Frequency" + units: "Hz" + + - name: "MetaData/dataProviderOrigin" + source: variables/dataProviderOrigin + longName: "Identification of Originating/Generating Center" + + - name: "MetaData/qualityInformationWithoutForecast" + source: variables/qualityInformationWithoutForecast + longName: "Quality Information Without Forecast" + + - name: "MetaData/expectedError" + source: variables/expectedError + longName: "Expected Error" + units: "m/s" + + - name: "MetaData/coefficientOfVariation" + source: variables/coefficientOfVariation + longName: "Coefficient of Variation" + + - name: "MetaData/windComputationMethod" + source: variables/windComputationMethod + longName: "Satellite-derived Wind Computation Method" + + - name: "MetaData/windHeightAssignMethod" + source: variables/windHeightAssignMethod + longName: "Wind Height Assignment Method" + + - name: "MetaData/pressure" + source: variables/pressure + longName: "Pressure" + units: "pa" + + - name: "ObsValue/windDirection" + source: variables/windDirection + longName: "Wind Direction" + units: "degree" + + - name: "ObsValue/windSpeed" + source: variables/windSpeed + longName: "Wind Direction" + units: "m/s" diff --git a/dump/mapping/bufr2netcdf_satwnd_amv_goes.py b/dump/mapping/bufr2netcdf_satwnd_amv_goes.py new file mode 100755 index 0000000..e6f6455 --- /dev/null +++ b/dump/mapping/bufr2netcdf_satwnd_amv_goes.py @@ -0,0 +1,190 @@ +import sys +import os +import argparse +import numpy as np +import bufr +import time +import warnings +from bufr.encoders import netcdf +from wxflow import Logger + +def get_description(yaml_path, update=False): + + description = bufr.encoders.Description(yaml_path) + + if update: + + description.add_variable(name='ObsType/windEastward', + source='variables/obstype_uwind', + units='1', + longName='Observation Type based on Satellite-derived Wind Computation Method and Spectral Band') + + description.add_variable(name='ObsType/windNorthward', + source='variables/obstype_vwind', + units='1', + longName='Observation Type based on Satellite-derived Wind Computation Method and Spectral Band') + + description.add_variable(name='ObsValue/windEastward', + source='variables/windEastward', + units='m/s', + longName='Eastward Wind Component') + + description.add_variable(name='ObsValue/windNorthward', + source='variables/windNorthward', + units='m/s', + longName='Northward Wind Component') + + return description + +def get_all_keys(nested_dict): + + keys = [] + for key, value in nested_dict.items(): + keys.append(key) + if isinstance(value, dict): + keys.extend(get_all_keys(value)) + return keys + +def Compute_WindComponents_from_WindDirection_and_WindSpeed(wdir, wspd): + + uob = (-wspd * np.sin(np.radians(wdir))).astype(np.float32) + vob = (-wspd * np.cos(np.radians(wdir))).astype(np.float32) + + return uob, vob + +def Get_ObsType(swcm, chanfreq): + + obstype = swcm.copy() + + # Use numpy vectorized operations + obstype = np.where(swcm == 5, 247, obstype) # WVCA/DL + obstype = np.where(swcm == 3, 246, obstype) # WVCT + obstype = np.where(swcm == 2, 251, obstype) # VIS + obstype = np.where(swcm == 1, 245, obstype) # IRLW + + condition = np.logical_and(swcm == 1, chanfreq >= 50000000000000.0) # IRSW + obstype = np.where(condition, 240, obstype) + + if not np.any(np.isin(obstype, [247, 246, 251, 245, 240])): + raise ValueError("Error: Unassigned ObsType found ... ") + + return obstype + +def create_obs_group(yaml_path, input_path, output_path, logger): + + # Get MPI communicator + comm = bufr.mpi.Comm("world") + rank = comm.rank() + + # Generate keys for cache + inputKey = input_path + mappingKey = yaml_path + + if rank == 0: logger.info('Input parameters') + if rank == 0: logger.info(f' ... input file: {input_path}') + if rank == 0: logger.info(f' ... output file: {output_path}') + if rank == 0: logger.info(f' ... mapping file: {yaml_path}') + + # Get container from mapping file first + if rank == 0: logger.info('Create data container based on mapping file') + container = bufr.Parser(input_path, yaml_path).parse(comm) + if rank == 0: logger.info(f' ... all_sub_categories = {container.all_sub_categories()}') + if rank == 0: logger.debug(' ... container list (original):\n' + '\n'.join(str(item) for item in container.list())) + + categories = container.all_sub_categories() + if rank == 0: logger.info('Loop through data categories to add new/derived variables to data container') + for cat in categories: + + description = get_description(yaml_path, update=True) + if rank == 0: logger.info(f' ... category = {cat[0]}') + + satid = container.get('variables/satelliteId', cat) + if satid.size == 0: + if rank == 0: logger.info(f' ... category {cat[0]} does not exist in input file') + + paths = container.get_paths('variables/windComputationMethod', cat) + obstype = container.get('variables/windComputationMethod', cat) + container.add('variables/obstype_uwind', obstype, paths, cat) + container.add('variables/obstype_vwind', obstype, paths, cat) + + paths = container.get_paths('variables/windSpeed', cat) + wob = container.get('variables/windSpeed', cat) + container.add('variables/windEastward', wob, paths, cat) + container.add('variables/windNorthward', wob, paths, cat) + + else: + # Add new variables: ObsType/windEastward & ObsType/windNorthward + swcm = container.get('variables/windComputationMethod', cat) + chanfreq = container.get('variables/sensorCentralFrequency', cat) + + if rank == 0: logger.debug(f' ... swcm min/max = {swcm.min()} {swcm.max()}') + if rank == 0: logger.debug(f' ... chanfreq min/max = {chanfreq.min()} {chanfreq.max()}') + + obstype = Get_ObsType(swcm, chanfreq) + + if rank == 0: logger.info(f' ... adding obstype to data container') + if rank == 0: logger.debug(f' ... obstype = {obstype}') + if rank == 0: logger.debug(f' ... obstype min/max = {obstype.min()} {obstype.max()}') + + paths = container.get_paths('variables/windComputationMethod', cat) + container.add('variables/obstype_uwind', obstype, paths, cat) + container.add('variables/obstype_vwind', obstype, paths, cat) + + # Add new variables: ObsValue/windEastward & ObsValue/windNorthward + wdir = container.get('variables/windDirection', cat) + wspd = container.get('variables/windSpeed', cat) + + if rank == 0: logger.debug(f' ... wdir min/max = {wdir.min()} {wdir.max()}') + if rank == 0: logger.debug(f' ... wspd min/max = {wspd.min()} {wspd.max()}') + + uob, vob = Compute_WindComponents_from_WindDirection_and_WindSpeed(wdir, wspd) + + if rank == 0: logger.info(f' ... adding windEastward and windNorthward to data container') + if rank == 0: logger.debug(f' ... uob min/max = {uob.min()} {uob.max()}') + if rank == 0: logger.debug(f' ... vob min/max = {vob.min()} {vob.max()}') + + paths = container.get_paths('variables/windSpeed', cat) + container.add('variables/windEastward', uob, paths, cat) + container.add('variables/windNorthward', vob, paths, cat) + + # Check + if rank == 0: logger.debug(' container list (updated):\n' + '\n'.join(str(item) for item in container.list())) + + # Gather the DataContainer data from all MPI ranks [Optional] + if rank == 0: logger.info('Gather data container from all MPI tasks rank 0 task') + container.gather(comm) + + # Encode all categories + if rank == 0: + logger.info('Encoding the data at MPI rank 0') + netcdf.Encoder(description).encode(container, output_path) + + +if __name__ == '__main__': + + start_time = time.time() + + bufr.mpi.App(sys.argv) + comm = bufr.mpi.Comm("world") + rank = comm.rank() + + # Required input arguments + parser = argparse.ArgumentParser() + parser.add_argument('-i', '--input', type=str, help='Input NetCDF', required=True) + parser.add_argument('-m', '--mapping', type=str, help='Mapping File', required=True) + parser.add_argument('-o', '--output', type=str, help='Output NetCDF', required=True) + parser.add_argument('-v', '--verbose', help='print debug logging info', action='store_true') + + args = parser.parse_args() + mapping = args.mapping + infile = args.input + output = args.output + + log_level = 'DEBUG' if args.verbose else 'INFO' + logger = Logger('BUFR2IODA_satwind_amv_goes.py', level=log_level, colored_log=True) + + create_obs_group(mapping, infile, output, logger) + + end_time = time.time() + running_time = end_time - start_time + if rank == 0: logger.info(f'Total running time: {running_time}') diff --git a/ush/test/process_script2netcdf b/ush/test/process_script2netcdf new file mode 100755 index 0000000..902e243 --- /dev/null +++ b/ush/test/process_script2netcdf @@ -0,0 +1,48 @@ +#!/bin/bash -x + +# Enable debugging mode +set -x + +# Set unlimited stack size +ulimit -s unlimited +ulimit -a + +# Define directories and set environment variables +src_dir="/work/noaa/da/eliu/HERCULES/EMC-bufr-query/bufr-query" +export LD_LIBRARY_PATH="${src_dir}/build/lib:${LD_LIBRARY_PATH}" +export PYTHONPATH="${PYTHONPATH}:${src_dir}/build/lib/python3.10/site-packages" +export PYTHONPATH="${PYTHONPATH}:/work/noaa/da/eliu/HERCULES/EMC-wxflow/wxflow/src" + +# Date and processor count +cdate="2021080100" +nproc="$1" + +# Extract year, month, day, and hour +y4="${cdate:0:4}" +m2="${cdate:4:2}" +d2="${cdate:6:2}" +h2="${cdate:8:2}" + +# Define working, input, and output directories +work_dir="$PWD" +out_dir="${work_dir}/testoutput/${cdate}/script_backend_mpi${nproc}" +in_dir="${work_dir}/testinput/${cdate}" + +# File paths +mapping_file="${work_dir}/bufr2ioda_satwind_amv_goes_mapping.yaml" +input_file="${in_dir}/gdas.t${h2}z.satwnd.tm00.bufr_d" +output_file="${out_dir}/gdas.t${h2}z.satwind_abi_{splits/satId}.tm00.nc" + +# Create output directory with specific permissions +mkdir -p -m770 "$out_dir" + +# Run the Python script +if [[ "$nproc" == "0" ]]; then + echo Run Python script without MPI ... + python bufr2ioda_satwind_amv_goes.py -m "$mapping_file" -o "$output_file" -i "$input_file" +else + echo Run Python script with MPI ${nproc} ... + srun -n "$nproc" --time 00:30:00 --mem 96G python bufr2ioda_satwind_amv_goes.py -m "$mapping_file" -o "$output_file" -i "$input_file" +fi + + diff --git a/ush/test/process_script_backend b/ush/test/process_script_backend new file mode 100755 index 0000000..7961ee3 --- /dev/null +++ b/ush/test/process_script_backend @@ -0,0 +1,44 @@ +#!/bin/bash -x + +# Enable debugging mode +set -x + +# Set unlimited stack size +ulimit -s unlimited +ulimit -a + +export OOPS_TRACE=1 +export OOPS_DEBUG=1 + +export PYTHONPATH="$PYTHONPATH:/work/noaa/da/eliu/HERCULES/JEDI-ioda/ioda-bundle/build/lib/python3.10" +export PYTHONPATH="$PYTHONPATH:/work/noaa/da/eliu/HERCULES/EMC-bufr-query/bufr-query/build/lib/python3.10/site-packages" +export PYTHONPATH="${PYTHONPATH}:/work/noaa/da/eliu/HERCULES/EMC-wxflow/wxflow/src" + +# Date and processor count +cdate="2021080100" +nproc="$1" + +# Extract year, month, day, and hour +y4="${cdate:0:4}" +m2="${cdate:4:2}" +d2="${cdate:6:2}" +h2="${cdate:8:2}" + +work_dir="$PWD" +src_dir="/work/noaa/da/eliu/HERCULES/JEDI-ioda/ioda-bundle" +out_dir="${work_dir}/testoutput/$cdate/script_backend" +in_dir="${work_dir}/testinput/$cdate" + +mkdir -p -m770 $out_dir + +process_yaml="${work_dir}/bufr2ioda_script_backend_satwind.yaml" + +# Run +if [[ "$nproc" == "0" ]]; then + echo Run time_IodaIO.x without MPI ... + ${src_dir}/build/bin/time_IodaIO.x ${process_yaml} +else + echo Run time_IodaIO.x with MPI ${nproc} ... + srun -n $nproc --mem 96G --time 00:30:00 ${src_dir}/build/bin/time_IodaIO.x ${process_yaml} +fi + From 7277cc56caf509cd022c9891b8ee1da8f899369e Mon Sep 17 00:00:00 2001 From: Emily Liu Date: Mon, 18 Nov 2024 20:00:38 +0000 Subject: [PATCH 06/72] remove test --- ush/test/process_script2netcdf | 48 --------------------------------- ush/test/process_script_backend | 44 ------------------------------ 2 files changed, 92 deletions(-) delete mode 100755 ush/test/process_script2netcdf delete mode 100755 ush/test/process_script_backend diff --git a/ush/test/process_script2netcdf b/ush/test/process_script2netcdf deleted file mode 100755 index 902e243..0000000 --- a/ush/test/process_script2netcdf +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash -x - -# Enable debugging mode -set -x - -# Set unlimited stack size -ulimit -s unlimited -ulimit -a - -# Define directories and set environment variables -src_dir="/work/noaa/da/eliu/HERCULES/EMC-bufr-query/bufr-query" -export LD_LIBRARY_PATH="${src_dir}/build/lib:${LD_LIBRARY_PATH}" -export PYTHONPATH="${PYTHONPATH}:${src_dir}/build/lib/python3.10/site-packages" -export PYTHONPATH="${PYTHONPATH}:/work/noaa/da/eliu/HERCULES/EMC-wxflow/wxflow/src" - -# Date and processor count -cdate="2021080100" -nproc="$1" - -# Extract year, month, day, and hour -y4="${cdate:0:4}" -m2="${cdate:4:2}" -d2="${cdate:6:2}" -h2="${cdate:8:2}" - -# Define working, input, and output directories -work_dir="$PWD" -out_dir="${work_dir}/testoutput/${cdate}/script_backend_mpi${nproc}" -in_dir="${work_dir}/testinput/${cdate}" - -# File paths -mapping_file="${work_dir}/bufr2ioda_satwind_amv_goes_mapping.yaml" -input_file="${in_dir}/gdas.t${h2}z.satwnd.tm00.bufr_d" -output_file="${out_dir}/gdas.t${h2}z.satwind_abi_{splits/satId}.tm00.nc" - -# Create output directory with specific permissions -mkdir -p -m770 "$out_dir" - -# Run the Python script -if [[ "$nproc" == "0" ]]; then - echo Run Python script without MPI ... - python bufr2ioda_satwind_amv_goes.py -m "$mapping_file" -o "$output_file" -i "$input_file" -else - echo Run Python script with MPI ${nproc} ... - srun -n "$nproc" --time 00:30:00 --mem 96G python bufr2ioda_satwind_amv_goes.py -m "$mapping_file" -o "$output_file" -i "$input_file" -fi - - diff --git a/ush/test/process_script_backend b/ush/test/process_script_backend deleted file mode 100755 index 7961ee3..0000000 --- a/ush/test/process_script_backend +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash -x - -# Enable debugging mode -set -x - -# Set unlimited stack size -ulimit -s unlimited -ulimit -a - -export OOPS_TRACE=1 -export OOPS_DEBUG=1 - -export PYTHONPATH="$PYTHONPATH:/work/noaa/da/eliu/HERCULES/JEDI-ioda/ioda-bundle/build/lib/python3.10" -export PYTHONPATH="$PYTHONPATH:/work/noaa/da/eliu/HERCULES/EMC-bufr-query/bufr-query/build/lib/python3.10/site-packages" -export PYTHONPATH="${PYTHONPATH}:/work/noaa/da/eliu/HERCULES/EMC-wxflow/wxflow/src" - -# Date and processor count -cdate="2021080100" -nproc="$1" - -# Extract year, month, day, and hour -y4="${cdate:0:4}" -m2="${cdate:4:2}" -d2="${cdate:6:2}" -h2="${cdate:8:2}" - -work_dir="$PWD" -src_dir="/work/noaa/da/eliu/HERCULES/JEDI-ioda/ioda-bundle" -out_dir="${work_dir}/testoutput/$cdate/script_backend" -in_dir="${work_dir}/testinput/$cdate" - -mkdir -p -m770 $out_dir - -process_yaml="${work_dir}/bufr2ioda_script_backend_satwind.yaml" - -# Run -if [[ "$nproc" == "0" ]]; then - echo Run time_IodaIO.x without MPI ... - ${src_dir}/build/bin/time_IodaIO.x ${process_yaml} -else - echo Run time_IodaIO.x with MPI ${nproc} ... - srun -n $nproc --mem 96G --time 00:30:00 ${src_dir}/build/bin/time_IodaIO.x ${process_yaml} -fi - From a4efbb2d656f8133261dd1b8ab8be1c51c7e64c5 Mon Sep 17 00:00:00 2001 From: Emily Liu Date: Mon, 18 Nov 2024 20:09:20 +0000 Subject: [PATCH 07/72] Update components (mapping, Python script, configuration) for satwind goes --- .../bufr2ioda_script_backend_satwnd.yaml | 9 +- dump/mapping/bufr2ioda_satwnd_amv_goes.py | 284 ++++++++++++------ .../bufr2ioda_satwnd_amv_goes_mapping.yaml | 4 +- dump/mapping/bufr2netcdf_satwnd_amv_goes.py | 190 ------------ 4 files changed, 191 insertions(+), 296 deletions(-) delete mode 100755 dump/mapping/bufr2netcdf_satwnd_amv_goes.py diff --git a/dump/config/bufr2ioda_script_backend_satwnd.yaml b/dump/config/bufr2ioda_script_backend_satwnd.yaml index c7d965e..ecbdf14 100644 --- a/dump/config/bufr2ioda_script_backend_satwnd.yaml +++ b/dump/config/bufr2ioda_script_backend_satwnd.yaml @@ -16,11 +16,10 @@ observations: input_path: "./testinput/gdas.t00z.satwnd.tm00.bufr_d" mapping_path: "./bufr2ioda_satwind_amv_goes_mapping.yaml" category: "goes-16" - log_level: "DEBUG" obsdataout: engine: type: H5File - obsfile: "./testoutput/gdas.t00z.satwnd.abi_goes-16.tm00.nc" + obsfile: "./testoutput/bufr_backend/gdas.t00z.satwnd.abi_goes-16.tm00.nc" - obs space: name: "satwind_goes-17" @@ -35,11 +34,10 @@ observations: input_path: "./testinput/gdas.t00z.satwnd.tm00.bufr_d" mapping_path: "./bufr2ioda_satwind_amv_goes_mapping.yaml" category: "goes-17" - log_level: "DEBUG" obsdataout: engine: type: H5File - obsfile: "./testoutput/gdas.t00z.satwnd.abi_goes-17.tm00.nc" + obsfile: "./testoutput/bufr_backend/gdas.t00z.satwnd.abi_goes-17.tm00.nc" - obs space: name: "satwind_goes-18" @@ -54,8 +52,7 @@ observations: input_path: "./testinput/gdas.t00z.satwnd.tm00.bufr_d" mapping_path: "./bufr2ioda_satwind_amv_goes_mapping.yaml" category: "goes-18" - log_level: "DEBUG" obsdataout: engine: type: H5File - obsfile: "./testoutput/gdas.t00z.satwnd.abi_goes-18.tm00.nc" + obsfile: "./testoutput/bufr_backend/gdas.t00z.satwnd.abi_goes-18.tm00.nc" diff --git a/dump/mapping/bufr2ioda_satwnd_amv_goes.py b/dump/mapping/bufr2ioda_satwnd_amv_goes.py index 3522776..0299a29 100644 --- a/dump/mapping/bufr2ioda_satwnd_amv_goes.py +++ b/dump/mapping/bufr2ioda_satwnd_amv_goes.py @@ -1,54 +1,115 @@ -import bufr -from pyioda.ioda.Engines.Bufr import Encoder +#!/usr/bin/env python3 +import sys +import os +import argparse +import time import numpy as np -import warnings +import bufr +from pyioda.ioda.Engines.Bufr import Encoder as iodaEncoder +from bufr.encoders.netcdf import Encoder as netcdfEncoder from wxflow import Logger -def get_description(mapping_path, update=False): +# Initialize Logger +# Get log level from the environment variable, default to 'INFO it not set +log_level = os.getenv('LOG_LEVEL', 'INFO') +logger = Logger('BUFR2IODA_satwnd_amv_goes.py', level=log_level, colored_log=False) - description = bufr.encoders.Description(mapping_path) +def logging(comm, level, message): - if update: + if comm.rank() == 0: + # Define a dictionary to map levels to logger methods + log_methods = { + 'DEBUG': logger.debug, + 'INFO': logger.info, + 'WARNING': logger.warning, + 'ERROR': logger.error, + 'CRITICAL': logger.critical, + } - description.add_variable(name='ObsType/windEastward', - source='variables/obstype_uwind', - units='1', - longName='Observation Type based on Satellite-derived Wind Computation Method and Spectral Band') + # Get the appropriate logging method, default to 'INFO' + log_method = log_methods.get(level.upper(), logger.info) - description.add_variable(name='ObsType/windNorthward', - source='variables/obstype_vwind', - units='1', - longName='Observation Type based on Satellite-derived Wind Computation Method and Spectral Band') + if log_method == logger.info and level.upper() not in log_methods: + # Log a warning if the level is invalid + logger.warning(f'log level = {level}: not a valid level --> set to INFO') - description.add_variable(name='ObsValue/windEastward', - source='variables/windEastward', - units='m/s', - longName='Eastward Wind Component') + # Call the logging method + log_method(message) - description.add_variable(name='ObsValue/windNorthward', - source='variables/windNorthward', - units='m/s', - longName='Northward Wind Component') +def _make_description(mapping_path, update=False): + description = bufr.encoders.Description(mapping_path) + + if update: + # Define the variables to be added in a list of dictionaries + variables = [ + { + 'name': 'ObsType/windEastward', + 'source': 'variables/obstype_uwind', + 'units': '1', + 'longName': 'Observation Type based on Satellite-derived Wind Computation Method and Spectral Band', + }, + { + 'name': 'ObsType/windNorthward', + 'source': 'variables/obstype_vwind', + 'units': '1', + 'longName': 'Observation Type based on Satellite-derived Wind Computation Method and Spectral Band', + }, + { + 'name': 'ObsValue/windEastward', + 'source': 'variables/windEastward', + 'units': 'm/s', + 'longName': 'Eastward Wind Component', + }, + { + 'name': 'ObsValue/windNorthward', + 'source': 'variables/windNorthward', + 'units': 'm/s', + 'longName': 'Northward Wind Component', + }, + ] + + # Loop through each variable and add it to the description + for var in variables: + description.add_variable( + name=var['name'], + source=var['source'], + units=var['units'], + longName=var['longName'] + ) return description -def get_all_keys(nested_dict): +def compute_wind_components(wdir, wspd): + """ + Compute the U and V wind components from wind direction and wind speed. + + Parameters: + wdir (array-like): Wind direction in degrees (meteorological convention: 0° = North, 90° = East). + wspd (array-like): Wind speed. - keys = [] - for key, value in nested_dict.items(): - keys.append(key) - if isinstance(value, dict): - keys.extend(get_all_keys(value)) - return keys + Returns: + tuple: U and V wind components as numpy arrays with dtype float32. + """ + wdir_rad = np.radians(wdir) # Convert degrees to radians + u = -wspd * np.sin(wdir_rad) + v = -wspd * np.cos(wdir_rad) + + return u.astype(np.float32), v.astype(np.float32) -def Compute_WindComponents_from_WindDirection_and_WindSpeed(wdir, wspd): +def _get_obs_type(swcm, chanfreq): + """ + Determine the observation type based on `swcm` and `chanfreq`. - uob = (-wspd * np.sin(np.radians(wdir))).astype(np.float32) - vob = (-wspd * np.cos(np.radians(wdir))).astype(np.float32) + Parameters: + swcm (array-like): Switch mode values. + chanfreq (array-like): Channel frequency values (Hz). - return uob, vob + Returns: + numpy.ndarray: Observation type array. -def Get_ObsType(swcm, chanfreq): + Raises: + ValueError: If any `obstype` is unassigned. + """ obstype = swcm.copy() @@ -58,7 +119,7 @@ def Get_ObsType(swcm, chanfreq): obstype = np.where(swcm == 2, 251, obstype) # VIS obstype = np.where(swcm == 1, 245, obstype) # IRLW - condition = np.logical_and(swcm == 1, chanfreq >= 50000000000000.0) # IRSW + condition = np.logical_and(swcm == 1, chanfreq >= 5e13) # IRSW obstype = np.where(condition, 240, obstype) if not np.any(np.isin(obstype, [247, 246, 251, 245, 240])): @@ -66,56 +127,24 @@ def Get_ObsType(swcm, chanfreq): return obstype -def create_obs_group(input_path, mapping_path, category, env, log_level=None ): - - # Set MPI parameters - comm = bufr.mpi.Comm(env["comm_name"]) - rank = comm.rank() - - # Set logger level - if log_level is None: - log_level = 'INFO' - elif log_level not in ['DEBUG', 'INFO']: - raise ValueError("Invalid log level! log_level must be 'DEBUG' or 'INFO'.") +def _make_obs(comm, input_path, mapping_path): - # Initialize logger - logger = Logger('BUFR2IODA_satwnd_amv_goes.py', level=log_level, colored_log=False) - - # Generate keys for cache - inputKey = input_path - mappingKey = mapping_path - - if rank == 0: logger.info(f'Create obs group for category: {category}') - - # Check the cache for the data and return it if it exists - if rank == 0: logger.debug(f'Check if bufr.DataCache exists? {bufr.DataCache.has(inputKey, mappingKey)}') - if bufr.DataCache.has(inputKey, mappingKey): - container = bufr.DataCache.get(inputKey, mappingKey) - if rank == 0: logger.info(f'Encode {category} from cache') - data = Encoder(get_description(mapping_path, update=True)).encode(container)[(category,)] - if rank == 0: logger.info(f'Mark {category} as finished in the cache') - bufr.DataCache.mark_finished(inputKey, mappingKey, [category]) - if rank == 0: logger.info(f'Return the encoded data for {category}') - return data - - if rank == 0: logger.info(f'Get all cagegories info cache') - # If cache does not exist, get data into cache # Get container from mapping file first - if rank == 0: logger.info(f'Get container from bufr parser') + logging(comm, 'INFO', 'Get container from bufr') container = bufr.Parser(input_path, mapping_path).parse(comm) - if rank == 0: logger.debug('container list (original):\n' + '\n'.join(str(item) for item in container.list())) - if rank == 0: logger.debug(f'all_sub_categories = {container.all_sub_categories()}') - if rank == 0: logger.debug(f'category map = {container.get_category_map()}') - categories = container.all_sub_categories() - for cat in categories: + logging(comm, 'DEBUG', f'container list (original): {container.list()}') + logging(comm, 'DEBUG', f'all_sub_categories = {container.all_sub_categories()}') + logging(comm, 'DEBUG', f'category map = {container.get_category_map()}') + + # Add new/derived data into container + for cat in container.all_sub_categories(): - if rank == 0: logger.debug(f'category = {cat}') - description = get_description(mapping_path, update=True) + logging(comm, 'DEBUG', f'category = {cat}') satid = container.get('variables/satelliteId', cat) if satid.size == 0: - if rank == 0: logger.info(f'category {cat[0]} does not exist in input file') + logging(comm, 'WARNING', f'category {cat[0]} does not exist in input file') paths = container.get_paths('variables/windComputationMethod', cat) obstype = container.get('variables/windComputationMethod', cat) container.add('variables/obstype_uwind', obstype, paths, cat) @@ -131,13 +160,13 @@ def create_obs_group(input_path, mapping_path, category, env, log_level=None ): swcm = container.get('variables/windComputationMethod', cat) chanfreq = container.get('variables/sensorCentralFrequency', cat) - if rank == 0: logger.debug(f'swcm min/max = {swcm.min()} {swcm.max()}') - if rank == 0: logger.debug(f'chanfreq min/max = {chanfreq.min()} {chanfreq.max()}') + logging(comm, 'DEBUG', f'swcm min/max = {swcm.min()} {swcm.max()}') + logging(comm, 'DEBUG', f'chanfreq min/max = {chanfreq.min()} {chanfreq.max()}') - obstype = Get_ObsType(swcm, chanfreq) + obstype = _get_obs_type(swcm, chanfreq) - if rank == 0: logger.debug(f'obstype = {obstype}') - if rank == 0: logger.debug(f'obstype min/max = {obstype.min()} {obstype.max()}') + logging(comm, 'DEBUG', f'obstype = {obstype}') + logging(comm, 'DEBUG', f'obstype min/max = {obstype.min()} {obstype.max()}') paths = container.get_paths('variables/windComputationMethod', cat) container.add('variables/obstype_uwind', obstype, paths, cat) @@ -147,37 +176,96 @@ def create_obs_group(input_path, mapping_path, category, env, log_level=None ): wdir = container.get('variables/windDirection', cat) wspd = container.get('variables/windSpeed', cat) - if rank == 0: logger.debug(f'wdir min/max = {wdir.min()} {wdir.max()}') - if rank == 0: logger.debug(f'wspd min/max = {wspd.min()} {wspd.max()}') + logging(comm, 'DEBUG', f'wdir min/max = {wdir.min()} {wdir.max()}') + logging(comm, 'DEBUG', f'wspd min/max = {wspd.min()} {wspd.max()}') - uob, vob = Compute_WindComponents_from_WindDirection_and_WindSpeed(wdir, wspd) + uob, vob = compute_wind_components(wdir, wspd) - if rank == 0: logger.debug(f'uob min/max = {uob.min()} {uob.max()}') - if rank == 0: logger.debug(f'vob min/max = {vob.min()} {vob.max()}') + logging(comm, 'DEBUG', f'uob min/max = {uob.min()} {uob.max()}') + logging(comm, 'DEBUG', f'vob min/max = {vob.min()} {vob.max()}') paths = container.get_paths('variables/windSpeed', cat) container.add('variables/windEastward', uob, paths, cat) container.add('variables/windNorthward', vob, paths, cat) # Check - if rank == 0: logger.debug('container list (updated):\n' + '\n'.join(str(item) for item in container.list())) - if rank == 0: logger.debug(f'all_sub_categories {container.all_sub_categories()}') - + logging(comm, 'DEBUG', f'container list (updated): {container.list()}') + logging(comm, 'DEBUG', f'all_sub_categories {container.all_sub_categories()}') + + return container + +def create_obs_group(input_path, mapping_path, category, env): + + comm = bufr.mpi.Comm(env["comm_name"]) + + description = _make_description(mapping_path, update=True) + + # Check the cache for the data and return it if it exists + logging(comm, 'DEBUG', f'Check if bufr.DataCache exists? {bufr.DataCache.has(input_path, mapping_path)}') + if bufr.DataCache.has(input_path, mapping_path): + container = bufr.DataCache.get(input_path, mapping_path) + logging(comm, 'INFO', f'Encode {category} from cache') + data = iodaEncoder(description).encode(container)[(category,)] + logging(comm, 'INFO', f'Mark {category} as finished in the cache') + bufr.DataCache.mark_finished(input_path, mapping_path, [category]) + logging(comm, 'INFO', f'Return the encoded data for {category}') + return data + + container = _make_obs(comm, input_path, mapping_path) + # Gather data from all tasks into all tasks. Each task will have the complete record - if rank == 0: logger.info(f'Gather data from all tasks into all tasks') + logging(comm, 'INFO', f'Gather data from all tasks into all tasks') container.all_gather(comm) - if rank == 0: logger.info(f'Add container to cache') + logging(comm, 'INFO', f'Add container to cache') # Add the container to the cache - bufr.DataCache.add(inputKey, mappingKey, container.all_sub_categories(), container) + bufr.DataCache.add(input_path, mapping_path, container.all_sub_categories(), container) # Encode the data - if rank == 0: logger.info(f'Encode {category}') - data = Encoder(description).encode(container)[(category,)] + logging(comm, 'INFO', f'Encode {category}') + data = iodaEncoder(description).encode(container)[(category,)] - if rank == 0: logger.info(f'Mark {category} as finished in the cache') + logging(comm, 'INFO', f'Mark {category} as finished in the cache') # Mark the data as finished in the cache - bufr.DataCache.mark_finished(inputKey, mappingKey, [category]) + bufr.DataCache.mark_finished(input_path, mapping_path, [category]) - if rank == 0: logger.info(f'Return the encoded data for {category}') + logging(comm, 'INFO', f'Return the encoded data for {category}') return data + +def create_obs_file(input_path, mapping_path, output_path): + + comm = bufr.mpi.Comm("world") + container = _make_obs(comm, input_path, mapping_path) + container.gather(comm) + + description = _make_description(mapping_path, update=True) + + # Encode the data + if comm.rank() == 0: + netcdfEncoder(description).encode(container, output_path) + + logging(comm, 'INFO', f'Return the encoded data') + +if __name__ == '__main__': + + start_time = time.time() + + bufr.mpi.App(sys.argv) + comm = bufr.mpi.Comm("world") + + # Required input arguments + parser = argparse.ArgumentParser() + parser.add_argument('-i', '--input', type=str, help='Input BUFR', required=True) + parser.add_argument('-m', '--mapping', type=str, help='BUFR2IODA Mapping File', required=True) + parser.add_argument('-o', '--output', type=str, help='Output NetCDF', required=True) + + args = parser.parse_args() + mapping = args.mapping + infile = args.input + output = args.output + + create_obs_file(infile, mapping, output) + + end_time = time.time() + running_time = end_time - start_time + logging(comm, 'INFO', f'Total running time: {running_time}') diff --git a/dump/mapping/bufr2ioda_satwnd_amv_goes_mapping.yaml b/dump/mapping/bufr2ioda_satwnd_amv_goes_mapping.yaml index 7815b81..371cf8c 100755 --- a/dump/mapping/bufr2ioda_satwnd_amv_goes_mapping.yaml +++ b/dump/mapping/bufr2ioda_satwnd_amv_goes_mapping.yaml @@ -77,9 +77,9 @@ bufr: _272: goes-18 encoder: - type: netcdf +# type: netcdf -# dimensions: +# dimensions: globals: - name: "platformCommonName" diff --git a/dump/mapping/bufr2netcdf_satwnd_amv_goes.py b/dump/mapping/bufr2netcdf_satwnd_amv_goes.py deleted file mode 100755 index e6f6455..0000000 --- a/dump/mapping/bufr2netcdf_satwnd_amv_goes.py +++ /dev/null @@ -1,190 +0,0 @@ -import sys -import os -import argparse -import numpy as np -import bufr -import time -import warnings -from bufr.encoders import netcdf -from wxflow import Logger - -def get_description(yaml_path, update=False): - - description = bufr.encoders.Description(yaml_path) - - if update: - - description.add_variable(name='ObsType/windEastward', - source='variables/obstype_uwind', - units='1', - longName='Observation Type based on Satellite-derived Wind Computation Method and Spectral Band') - - description.add_variable(name='ObsType/windNorthward', - source='variables/obstype_vwind', - units='1', - longName='Observation Type based on Satellite-derived Wind Computation Method and Spectral Band') - - description.add_variable(name='ObsValue/windEastward', - source='variables/windEastward', - units='m/s', - longName='Eastward Wind Component') - - description.add_variable(name='ObsValue/windNorthward', - source='variables/windNorthward', - units='m/s', - longName='Northward Wind Component') - - return description - -def get_all_keys(nested_dict): - - keys = [] - for key, value in nested_dict.items(): - keys.append(key) - if isinstance(value, dict): - keys.extend(get_all_keys(value)) - return keys - -def Compute_WindComponents_from_WindDirection_and_WindSpeed(wdir, wspd): - - uob = (-wspd * np.sin(np.radians(wdir))).astype(np.float32) - vob = (-wspd * np.cos(np.radians(wdir))).astype(np.float32) - - return uob, vob - -def Get_ObsType(swcm, chanfreq): - - obstype = swcm.copy() - - # Use numpy vectorized operations - obstype = np.where(swcm == 5, 247, obstype) # WVCA/DL - obstype = np.where(swcm == 3, 246, obstype) # WVCT - obstype = np.where(swcm == 2, 251, obstype) # VIS - obstype = np.where(swcm == 1, 245, obstype) # IRLW - - condition = np.logical_and(swcm == 1, chanfreq >= 50000000000000.0) # IRSW - obstype = np.where(condition, 240, obstype) - - if not np.any(np.isin(obstype, [247, 246, 251, 245, 240])): - raise ValueError("Error: Unassigned ObsType found ... ") - - return obstype - -def create_obs_group(yaml_path, input_path, output_path, logger): - - # Get MPI communicator - comm = bufr.mpi.Comm("world") - rank = comm.rank() - - # Generate keys for cache - inputKey = input_path - mappingKey = yaml_path - - if rank == 0: logger.info('Input parameters') - if rank == 0: logger.info(f' ... input file: {input_path}') - if rank == 0: logger.info(f' ... output file: {output_path}') - if rank == 0: logger.info(f' ... mapping file: {yaml_path}') - - # Get container from mapping file first - if rank == 0: logger.info('Create data container based on mapping file') - container = bufr.Parser(input_path, yaml_path).parse(comm) - if rank == 0: logger.info(f' ... all_sub_categories = {container.all_sub_categories()}') - if rank == 0: logger.debug(' ... container list (original):\n' + '\n'.join(str(item) for item in container.list())) - - categories = container.all_sub_categories() - if rank == 0: logger.info('Loop through data categories to add new/derived variables to data container') - for cat in categories: - - description = get_description(yaml_path, update=True) - if rank == 0: logger.info(f' ... category = {cat[0]}') - - satid = container.get('variables/satelliteId', cat) - if satid.size == 0: - if rank == 0: logger.info(f' ... category {cat[0]} does not exist in input file') - - paths = container.get_paths('variables/windComputationMethod', cat) - obstype = container.get('variables/windComputationMethod', cat) - container.add('variables/obstype_uwind', obstype, paths, cat) - container.add('variables/obstype_vwind', obstype, paths, cat) - - paths = container.get_paths('variables/windSpeed', cat) - wob = container.get('variables/windSpeed', cat) - container.add('variables/windEastward', wob, paths, cat) - container.add('variables/windNorthward', wob, paths, cat) - - else: - # Add new variables: ObsType/windEastward & ObsType/windNorthward - swcm = container.get('variables/windComputationMethod', cat) - chanfreq = container.get('variables/sensorCentralFrequency', cat) - - if rank == 0: logger.debug(f' ... swcm min/max = {swcm.min()} {swcm.max()}') - if rank == 0: logger.debug(f' ... chanfreq min/max = {chanfreq.min()} {chanfreq.max()}') - - obstype = Get_ObsType(swcm, chanfreq) - - if rank == 0: logger.info(f' ... adding obstype to data container') - if rank == 0: logger.debug(f' ... obstype = {obstype}') - if rank == 0: logger.debug(f' ... obstype min/max = {obstype.min()} {obstype.max()}') - - paths = container.get_paths('variables/windComputationMethod', cat) - container.add('variables/obstype_uwind', obstype, paths, cat) - container.add('variables/obstype_vwind', obstype, paths, cat) - - # Add new variables: ObsValue/windEastward & ObsValue/windNorthward - wdir = container.get('variables/windDirection', cat) - wspd = container.get('variables/windSpeed', cat) - - if rank == 0: logger.debug(f' ... wdir min/max = {wdir.min()} {wdir.max()}') - if rank == 0: logger.debug(f' ... wspd min/max = {wspd.min()} {wspd.max()}') - - uob, vob = Compute_WindComponents_from_WindDirection_and_WindSpeed(wdir, wspd) - - if rank == 0: logger.info(f' ... adding windEastward and windNorthward to data container') - if rank == 0: logger.debug(f' ... uob min/max = {uob.min()} {uob.max()}') - if rank == 0: logger.debug(f' ... vob min/max = {vob.min()} {vob.max()}') - - paths = container.get_paths('variables/windSpeed', cat) - container.add('variables/windEastward', uob, paths, cat) - container.add('variables/windNorthward', vob, paths, cat) - - # Check - if rank == 0: logger.debug(' container list (updated):\n' + '\n'.join(str(item) for item in container.list())) - - # Gather the DataContainer data from all MPI ranks [Optional] - if rank == 0: logger.info('Gather data container from all MPI tasks rank 0 task') - container.gather(comm) - - # Encode all categories - if rank == 0: - logger.info('Encoding the data at MPI rank 0') - netcdf.Encoder(description).encode(container, output_path) - - -if __name__ == '__main__': - - start_time = time.time() - - bufr.mpi.App(sys.argv) - comm = bufr.mpi.Comm("world") - rank = comm.rank() - - # Required input arguments - parser = argparse.ArgumentParser() - parser.add_argument('-i', '--input', type=str, help='Input NetCDF', required=True) - parser.add_argument('-m', '--mapping', type=str, help='Mapping File', required=True) - parser.add_argument('-o', '--output', type=str, help='Output NetCDF', required=True) - parser.add_argument('-v', '--verbose', help='print debug logging info', action='store_true') - - args = parser.parse_args() - mapping = args.mapping - infile = args.input - output = args.output - - log_level = 'DEBUG' if args.verbose else 'INFO' - logger = Logger('BUFR2IODA_satwind_amv_goes.py', level=log_level, colored_log=True) - - create_obs_group(mapping, infile, output, logger) - - end_time = time.time() - running_time = end_time - start_time - if rank == 0: logger.info(f'Total running time: {running_time}') From 568933207ae7f57d461d4f114fd095dace3a295f Mon Sep 17 00:00:00 2001 From: Emily Liu Date: Tue, 19 Nov 2024 00:33:23 +0000 Subject: [PATCH 08/72] remove old yaml --- .../bufr2ioda_script_backend_satwnd.yaml | 58 ------------------- 1 file changed, 58 deletions(-) delete mode 100644 dump/config/bufr2ioda_script_backend_satwnd.yaml diff --git a/dump/config/bufr2ioda_script_backend_satwnd.yaml b/dump/config/bufr2ioda_script_backend_satwnd.yaml deleted file mode 100644 index ecbdf14..0000000 --- a/dump/config/bufr2ioda_script_backend_satwnd.yaml +++ /dev/null @@ -1,58 +0,0 @@ -time window: - begin: "2018-04-14T21:00:00Z" - end: "2023-12-15T03:00:00Z" - -observations: -- obs space: - name: "satwind_goes-16" - observed variables: [windSpeed, windDirection] - derived variables: [windEastward, windNorthward] - simulated variables: [windEastward, windNorthward] - obsdatain: - engine: - type: script - script file: "bufr2ioda_satwind_amv_goes.py" - args: - input_path: "./testinput/gdas.t00z.satwnd.tm00.bufr_d" - mapping_path: "./bufr2ioda_satwind_amv_goes_mapping.yaml" - category: "goes-16" - obsdataout: - engine: - type: H5File - obsfile: "./testoutput/bufr_backend/gdas.t00z.satwnd.abi_goes-16.tm00.nc" - -- obs space: - name: "satwind_goes-17" - observed variables: [windSpeed, windDirection] - derived variables: [windEastward, windNorthward] - simulated variables: [windEastward, windNorthward] - obsdatain: - engine: - type: script - script file: "bufr2ioda_satwind_amv_goes.py" - args: - input_path: "./testinput/gdas.t00z.satwnd.tm00.bufr_d" - mapping_path: "./bufr2ioda_satwind_amv_goes_mapping.yaml" - category: "goes-17" - obsdataout: - engine: - type: H5File - obsfile: "./testoutput/bufr_backend/gdas.t00z.satwnd.abi_goes-17.tm00.nc" - -- obs space: - name: "satwind_goes-18" - observed variables: [windSpeed, windDirection] - derived variables: [windEastward, windNorthward] - simulated variables: [windEastward, windNorthward] - obsdatain: - engine: - type: script - script file: "bufr2ioda_satwind_amv_goes.py" - args: - input_path: "./testinput/gdas.t00z.satwnd.tm00.bufr_d" - mapping_path: "./bufr2ioda_satwind_amv_goes_mapping.yaml" - category: "goes-18" - obsdataout: - engine: - type: H5File - obsfile: "./testoutput/bufr_backend/gdas.t00z.satwnd.abi_goes-18.tm00.nc" From 16db06fe04a215e90a41404d987a2050f59fc2e6 Mon Sep 17 00:00:00 2001 From: Emily Liu Date: Tue, 19 Nov 2024 00:35:58 +0000 Subject: [PATCH 09/72] Add configuration files and test shell script --- .../bufr2ioda_bufr_backend_satwnd_goes.yaml | 58 ++++++++ .../bufr2ioda_script_backend_satwnd_goes.yaml | 58 ++++++++ ush/test/process_bufr2ioda | 124 ++++++++++++++++++ 3 files changed, 240 insertions(+) create mode 100644 dump/config/bufr2ioda_bufr_backend_satwnd_goes.yaml create mode 100644 dump/config/bufr2ioda_script_backend_satwnd_goes.yaml create mode 100755 ush/test/process_bufr2ioda diff --git a/dump/config/bufr2ioda_bufr_backend_satwnd_goes.yaml b/dump/config/bufr2ioda_bufr_backend_satwnd_goes.yaml new file mode 100644 index 0000000..2dd61eb --- /dev/null +++ b/dump/config/bufr2ioda_bufr_backend_satwnd_goes.yaml @@ -0,0 +1,58 @@ +time window: + begin: "2018-04-14T21:00:00Z" + end: "2023-12-15T03:00:00Z" + +observations: +- obs space: + name: "satwind_goes-16" + simulated variables: [windDirection, windSpeed] + obsdatain: + engine: + type: bufr + obsfile: "./testinput/2021080100/gdas.t00z.satwnd.tm00.bufr_d" + mapping file: "./bufr2ioda_satwnd_amv_goes_mapping.yaml" + category: ["goes-16"] + cache categories: # optional + - ["goes-16"] + - ["goes-17"] + - ["goes-18"] + obsdataout: + engine: + type: H5File + obsfile: "./testoutput/2021080100/bufr_backend/gdas.t00z.satwnd.abi_goes-16.tm00.nc" + +- obs space: + name: "satwind_goes-17" + simulated variables: [windDirection, windSpeed] + obsdatain: + engine: + type: bufr + obsfile: "./testinput/2021080100/gdas.t00z.satwnd.tm00.bufr_d" + mapping file: "./bufr2ioda_satwnd_amv_goes_mapping.yaml" + category: ["goes-17"] + cache categories: # optional + - ["goes-16"] + - ["goes-17"] + - ["goes-18"] + obsdataout: + engine: + type: H5File + obsfile: "./testoutput/2021080100/bufr_backend/gdas.t00z.satwnd.abi_goes-17.tm00.nc" + +- obs space: + name: "satwind_goes-18" + simulated variables: [windDirection, windSpeed] + obsdatain: + engine: + type: bufr + obsfile: "./testinput/2021080100/gdas.t00z.satwnd.tm00.bufr_d" + mapping file: "./bufr2ioda_satwnd_amv_goes_mapping.yaml" + category: ["goes-18"] + cache categories: # optional + - ["goes-16"] + - ["goes-17"] + - ["goes-18"] + obsdataout: + engine: + type: H5File + obsfile: "./testoutput/2021080100/bufr_backend/gdas.t00z.satwnd.abi_goes-18.tm00.nc" diff --git a/dump/config/bufr2ioda_script_backend_satwnd_goes.yaml b/dump/config/bufr2ioda_script_backend_satwnd_goes.yaml new file mode 100644 index 0000000..7172e6a --- /dev/null +++ b/dump/config/bufr2ioda_script_backend_satwnd_goes.yaml @@ -0,0 +1,58 @@ +time window: + begin: "2018-04-14T21:00:00Z" + end: "2023-12-15T03:00:00Z" + +observations: +- obs space: + name: "satwind_goes-16" + observed variables: [windSpeed, windDirection] + derived variables: [windEastward, windNorthward] + simulated variables: [windEastward, windNorthward] + obsdatain: + engine: + type: script + script file: "bufr2ioda_satwnd_amv_goes.py" + args: + input_path: "./testinput/gdas.t00z.satwnd.tm00.bufr_d" + mapping_path: "./bufr2ioda_satwnd_amv_goes_mapping.yaml" + category: "goes-16" + obsdataout: + engine: + type: H5File + obsfile: "./testoutput/2021080100/script_backend/gdas.t00z.satwnd.abi_goes-16.tm00.nc" + +- obs space: + name: "satwind_goes-17" + observed variables: [windSpeed, windDirection] + derived variables: [windEastward, windNorthward] + simulated variables: [windEastward, windNorthward] + obsdatain: + engine: + type: script + script file: "bufr2ioda_satwnd_amv_goes.py" + args: + input_path: "./testinput/gdas.t00z.satwnd.tm00.bufr_d" + mapping_path: "./bufr2ioda_satwnd_amv_goes_mapping.yaml" + category: "goes-17" + obsdataout: + engine: + type: H5File + obsfile: "./testoutput/2021080100/script_backend/gdas.t00z.satwnd.abi_goes-17.tm00.nc" + +- obs space: + name: "satwind_goes-18" + observed variables: [windSpeed, windDirection] + derived variables: [windEastward, windNorthward] + simulated variables: [windEastward, windNorthward] + obsdatain: + engine: + type: script + script file: "bufr2ioda_satwnd_amv_goes.py" + args: + input_path: "./testinput/gdas.t00z.satwnd.tm00.bufr_d" + mapping_path: "./bufr2ioda_satwnd_amv_goes_mapping.yaml" + category: "goes-18" + obsdataout: + engine: + type: H5File + obsfile: "./testoutput/2021080100/script_backend/gdas.t00z.satwnd.abi_goes-18.tm00.nc" diff --git a/ush/test/process_bufr2ioda b/ush/test/process_bufr2ioda new file mode 100755 index 0000000..5d51133 --- /dev/null +++ b/ush/test/process_bufr2ioda @@ -0,0 +1,124 @@ +#!/bin/bash -x + +# Enable debugging mode +set -x + +# Set unlimited stack size +ulimit -s unlimited +ulimit -a + +export OOPS_TRACE=1 +export OOPS_DEBUG=1 +export LOG_LEVEL=DEBUG + +# Load JEDI spack-stack-1.7.0 +module use /work/noaa/epic/role-epic/spack-stack/hercules/modulefiles +module load git-lfs/3.1.2 + +module use /work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.7.0/envs/ue-intel/install/modulefiles/Core +module load stack-intel/2021.9.0 +module load stack-intel-oneapi-mpi/2021.9.0 +module load stack-python/3.10.13 + +module load ewok-env + +# Configure SLUM environment +export SLURM_ACCOUNT=da-cpu +export SALLOC_ACCOUNT=$SLURM_ACCOUNT +export SBATCH_ACCOUNT=$SLURM_ACCOUNT +export SLURM_QOS=debug + +# bufr-query +bufr_query_dir="/work/noaa/da/eliu/HERCULES/EMC-bufr-query/bufr-query" +export LD_LIBRARY_PATH="${bufr_query_dir}/build/lib:${LD_LIBRARY_PATH}" +export PYTHONPATH="${PYTHONPATH}:${bufr_query_dir}/build/lib/python3.10/site-packages" + +# ioda +ioda_dir="/work/noaa/da/eliu/HERCULES/JEDI-ioda/ioda-bundle" +export PYTHONPATH="${PYTHONPATH}:${ioda_dir}/build/lib/python3.10" + +# wxflow +wxflow_dir="/work/noaa/da/eliu/HERCULES/EMC-wxflow/wxflow" +export PYTHONPATH="${PYTHONPATH}:${wxflow_dir}/src" + +# Date and processor count +cdate="2021080100" +mode="$1" +nproc="$2" + +# Get input arguments +mode="$1" +nproc="$2" + +# Function to display usage +usage() { + echo "Usage: $0 " + echo " : Mode of operation (e.g., bufr_backend, script_backend, bufr2netcdf)" + echo " : Number of processors (must be a positive integer)" + exit 1 +} + +# Check if both arguments are provided +if [[ -z "$mode" || -z "$nproc" ]]; then + echo "Error: Missing arguments." + usage +fi + +# Validate mode +if [[ "$mode" != "bufr_backend" && "$mode" != "script_backend" && "$mode" != "bufr2netcdf" ]]; then + echo "Error: Invalid mode '$mode'. Expected 'bufr_backend' or 'script_backend' or 'bufr2netcdf'." + usage +fi + +# Validate nproc is a positive integer +if ! [[ "$nproc" =~ ^[0-9]+$ && "$nproc" -gt 0 ]]; then + echo "Error: nproc must be a positive integer." + usage +fi + +# If all checks pass +echo "Mode: $mode" +echo "Number of processors: $nproc" + +# Extract year, month, day, and hour +y4="${cdate:0:4}" +m2="${cdate:4:2}" +d2="${cdate:6:2}" +h2="${cdate:8:2}" + +# Directory paths +work_dir="$PWD" +out_dir="${work_dir}/testoutput/$cdate/${mode}" +in_dir="${work_dir}/testinput/" +mkdir -p -m770 ${out_dir} + +# File paths +mapping_file="${work_dir}/bufr2ioda_satwnd_amv_goes_mapping.yaml" +input_file="${in_dir}/gdas.t${h2}z.satwnd.tm00.bufr_d" +output_file="${out_dir}/gdas.t${h2}z.satwnd_abi_{splits/satId}.tm00.nc" + +config_yaml="${work_dir}/bufr2ioda_${mode}_satwnd_goes.yaml" + +# Run IODA bufr/script backend +if [[ "$mode" == "bufr_backend" || "$mode" == "script_backend" ]]; then + if [[ "$nproc" == "0" ]]; then + echo Run time_IodaIO.x without MPI ... + ${ioda_dir}/build/bin/time_IodaIO.x ${config_yaml} + else + echo Run time_IodaIO.x with MPI ${nproc} ... + srun -n $nproc --mem 96G --time 00:30:00 ${ioda_dir}/build/bin/time_IodaIO.x ${config_yaml} + fi +# Run bufr_query netcdf +elif [[ "$mode" == "bufr2netcdf" ]]; then + if [[ "$nproc" == "0" ]]; then + echo Run Python script without MPI ... + python bufr2ioda_satwnd_amv_goes.py -m "$mapping_file" -o "$output_file" -i "$input_file" + else + echo Run Python script with MPI ${nproc} ... + srun -n "$nproc" --time 00:30:00 --mem 96G python bufr2ioda_satwnd_amv_goes.py -m "$mapping_file" -o "$output_file" -i "$input_file" + fi +else + echo Incorrect running mode ${mode} ... Valid modes are: bufr_backend, script_back, or bufr2netcdf +fi + + From 927bae2b6042b0cfd35c04960453c1360c2b9b08 Mon Sep 17 00:00:00 2001 From: Emily Liu Date: Tue, 19 Nov 2024 03:13:06 +0000 Subject: [PATCH 10/72] rename configuration yaml --- ...twnd_goes.yaml => bufr2ioda_bufr_backend_satwnd_amv_goes.yaml} | 0 ...nd_goes.yaml => bufr2ioda_script_backend_satwnd_amv_goes.yaml} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename dump/config/{bufr2ioda_bufr_backend_satwnd_goes.yaml => bufr2ioda_bufr_backend_satwnd_amv_goes.yaml} (100%) rename dump/config/{bufr2ioda_script_backend_satwnd_goes.yaml => bufr2ioda_script_backend_satwnd_amv_goes.yaml} (100%) diff --git a/dump/config/bufr2ioda_bufr_backend_satwnd_goes.yaml b/dump/config/bufr2ioda_bufr_backend_satwnd_amv_goes.yaml similarity index 100% rename from dump/config/bufr2ioda_bufr_backend_satwnd_goes.yaml rename to dump/config/bufr2ioda_bufr_backend_satwnd_amv_goes.yaml diff --git a/dump/config/bufr2ioda_script_backend_satwnd_goes.yaml b/dump/config/bufr2ioda_script_backend_satwnd_amv_goes.yaml similarity index 100% rename from dump/config/bufr2ioda_script_backend_satwnd_goes.yaml rename to dump/config/bufr2ioda_script_backend_satwnd_amv_goes.yaml From 9dfd16dc379d19d6ec1dc6db4550d26ee3df61e0 Mon Sep 17 00:00:00 2001 From: Emily Liu Date: Tue, 19 Nov 2024 03:22:51 +0000 Subject: [PATCH 11/72] Add temporary testing script --- ush/test/process_bufr2ioda | 145 ++++++++++++++++++++----------------- 1 file changed, 78 insertions(+), 67 deletions(-) diff --git a/ush/test/process_bufr2ioda b/ush/test/process_bufr2ioda index 5d51133..4e3035f 100755 --- a/ush/test/process_bufr2ioda +++ b/ush/test/process_bufr2ioda @@ -1,65 +1,34 @@ -#!/bin/bash -x - -# Enable debugging mode -set -x - -# Set unlimited stack size -ulimit -s unlimited -ulimit -a - -export OOPS_TRACE=1 -export OOPS_DEBUG=1 -export LOG_LEVEL=DEBUG - -# Load JEDI spack-stack-1.7.0 -module use /work/noaa/epic/role-epic/spack-stack/hercules/modulefiles -module load git-lfs/3.1.2 - -module use /work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.7.0/envs/ue-intel/install/modulefiles/Core -module load stack-intel/2021.9.0 -module load stack-intel-oneapi-mpi/2021.9.0 -module load stack-python/3.10.13 - -module load ewok-env - -# Configure SLUM environment -export SLURM_ACCOUNT=da-cpu -export SALLOC_ACCOUNT=$SLURM_ACCOUNT -export SBATCH_ACCOUNT=$SLURM_ACCOUNT -export SLURM_QOS=debug - -# bufr-query -bufr_query_dir="/work/noaa/da/eliu/HERCULES/EMC-bufr-query/bufr-query" -export LD_LIBRARY_PATH="${bufr_query_dir}/build/lib:${LD_LIBRARY_PATH}" -export PYTHONPATH="${PYTHONPATH}:${bufr_query_dir}/build/lib/python3.10/site-packages" - -# ioda -ioda_dir="/work/noaa/da/eliu/HERCULES/JEDI-ioda/ioda-bundle" -export PYTHONPATH="${PYTHONPATH}:${ioda_dir}/build/lib/python3.10" - -# wxflow -wxflow_dir="/work/noaa/da/eliu/HERCULES/EMC-wxflow/wxflow" -export PYTHONPATH="${PYTHONPATH}:${wxflow_dir}/src" - -# Date and processor count -cdate="2021080100" -mode="$1" -nproc="$2" +#!/bin/bash # Get input arguments -mode="$1" -nproc="$2" +obsforge_dir="${1:-/scratch1/NCEPDEV/da/Emily.Liu/EMC-obsForge/obsForge}" +wxflow_dir="${2:-/scratch1/NCEPDEV/da/Emily.Liu/EMC-wxflow/wxflow}" +cycle="${3:-2021080100}" +obstype="${4:-satwnd_amv_goes}" +sensor="${5:-abi}" +mode="${6:-script_backend}" +nproc="${7:-4}" # Function to display usage usage() { - echo "Usage: $0 " - echo " : Mode of operation (e.g., bufr_backend, script_backend, bufr2netcdf)" - echo " : Number of processors (must be a positive integer)" + echo "Usage: $0 " + echo " : root directory of obsForge build" + echo " : root directory of wxflow build" + echo " : cycle time (e.g., 2021080100)" + echo " : observation type to create (e.g., satwnd_amv_goes)" + echo " : sensor (e.g., abi)" + echo " : mode of operation (three valid modes: bufr_backend, script_backend, and bufr2netcdf)" + echo " : number of processors (must be a positive integer)" exit 1 } +# Check for --help or -h flags +if [[ "$1" == "--help" || "$1" == "-h" ]]; then + usage +fi + # Check if both arguments are provided -if [[ -z "$mode" || -z "$nproc" ]]; then +if [[ -z "$mode" || -z "$nproc" || -z "$sensor" || -z "$obstype" || -z "$cycle" || -z "${obsforge_dir}" || -z "${wxflow_dir}" ]]; then echo "Error: Missing arguments." usage fi @@ -77,45 +46,87 @@ if ! [[ "$nproc" =~ ^[0-9]+$ && "$nproc" -gt 0 ]]; then fi # If all checks pass -echo "Mode: $mode" -echo "Number of processors: $nproc" +echo "root directory of obsForge: $obsforge_dir" +echo "root directory of wxflow: $wxflow_dir" +echo "cycle: $cycle" +echo "obstype: $obstype" +echo "sensor: $sensor" +echo "mode: $mode" +echo "number of processors: $nproc" + +# Enable debugging mode +set -x + +# Set unlimited stack size +ulimit -s unlimited +ulimit -a + +# Set OOPS run time output parameters +export OOPS_TRACE=1 +export OOPS_DEBUG=1 + +# Set WXFLOW log level +export LOG_LEVEL=DEBUG + +# Load obsForge required modules +#obsforge_dir="/scratch1/NCEPDEV/da/Emily.Liu/EMC-obsForge/obsForge" +module use ${obsforge_dir}/modulefiles +module load obsforge/hera.intel +module list + +# Set bufr-query python library +export LD_LIBRARY_PATH="${obsforge_dir}/build/lib:${LD_LIBRARY_PATH}" +export PYTHONPATH="${PYTHONPATH}:${obsforge_dir}/build/lib/python3.10/site-packages" + +# Set ioda python library +export PYTHONPATH="${PYTHONPATH}:${obsforge_dir}/build/lib/python3.10" + +# Set wxfloww +#wxflow_dir="/scratch1/NCEPDEV/da/Emily.Liu/EMC-wxflow/wxflow" +export PYTHONPATH="${PYTHONPATH}:${wxflow_dir}/src" + +# Configure SLUM environment +export SLURM_ACCOUNT=da-cpu +export SALLOC_ACCOUNT=$SLURM_ACCOUNT +export SBATCH_ACCOUNT=$SLURM_ACCOUNT +export SLURM_QOS=debug # Extract year, month, day, and hour -y4="${cdate:0:4}" -m2="${cdate:4:2}" -d2="${cdate:6:2}" -h2="${cdate:8:2}" +y4="${cycle:0:4}" +m2="${cycle:4:2}" +d2="${cycle:6:2}" +h2="${cycle:8:2}" # Directory paths work_dir="$PWD" -out_dir="${work_dir}/testoutput/$cdate/${mode}" +out_dir="${work_dir}/testoutput/$cycle/${mode}" in_dir="${work_dir}/testinput/" mkdir -p -m770 ${out_dir} # File paths -mapping_file="${work_dir}/bufr2ioda_satwnd_amv_goes_mapping.yaml" +mapping_file="${work_dir}/bufr2ioda_${obstype}_mapping.yaml" input_file="${in_dir}/gdas.t${h2}z.satwnd.tm00.bufr_d" -output_file="${out_dir}/gdas.t${h2}z.satwnd_abi_{splits/satId}.tm00.nc" +output_file="${out_dir}/gdas.t${h2}z.satwnd_${sensor}_{splits/satId}.tm00.nc" -config_yaml="${work_dir}/bufr2ioda_${mode}_satwnd_goes.yaml" +config_yaml="${work_dir}/bufr2ioda_${mode}_${obstype}.yaml" # Run IODA bufr/script backend if [[ "$mode" == "bufr_backend" || "$mode" == "script_backend" ]]; then if [[ "$nproc" == "0" ]]; then echo Run time_IodaIO.x without MPI ... - ${ioda_dir}/build/bin/time_IodaIO.x ${config_yaml} + ${obsforge_dir}/build/bin/time_IodaIO.x ${config_yaml} else echo Run time_IodaIO.x with MPI ${nproc} ... - srun -n $nproc --mem 96G --time 00:30:00 ${ioda_dir}/build/bin/time_IodaIO.x ${config_yaml} + srun -n $nproc --mem 96G --time 00:30:00 ${obsforge_dir}/build/bin/time_IodaIO.x ${config_yaml} fi # Run bufr_query netcdf elif [[ "$mode" == "bufr2netcdf" ]]; then if [[ "$nproc" == "0" ]]; then echo Run Python script without MPI ... - python bufr2ioda_satwnd_amv_goes.py -m "$mapping_file" -o "$output_file" -i "$input_file" + python bufr2ioda_${obstype}.py -m "$mapping_file" -o "$output_file" -i "$input_file" else echo Run Python script with MPI ${nproc} ... - srun -n "$nproc" --time 00:30:00 --mem 96G python bufr2ioda_satwnd_amv_goes.py -m "$mapping_file" -o "$output_file" -i "$input_file" + srun -n "$nproc" --time 00:30:00 --mem 96G python bufr2ioda_${obstype}.py -m "$mapping_file" -o "$output_file" -i "$input_file" fi else echo Incorrect running mode ${mode} ... Valid modes are: bufr_backend, script_back, or bufr2netcdf From 54305c988c3883a872813c0aa574a6c03e739626 Mon Sep 17 00:00:00 2001 From: Emily Liu Date: Tue, 19 Nov 2024 04:01:47 +0000 Subject: [PATCH 12/72] Add README file --- ush/test/README | 58 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 ush/test/README diff --git a/ush/test/README b/ush/test/README new file mode 100644 index 0000000..382b013 --- /dev/null +++ b/ush/test/README @@ -0,0 +1,58 @@ +# ================================================================= +# Prerequisite +# ================================================================= + 1. Clone and build obsForge + git clone --recursive https://github.com/NOAA-EMC/obsForge + ./build.sh + + 2. Clone wxflow (no need to build) + git clone https://github.com/NOAA-EMC/wxflow + + + e.g. Emily's obsForge and wxflow builds on HERA + obsForge: /scratch1/NCEPDEV/da/Emily.Liu/EMC-obsForge/obsForge + wxflow: /scratch1/NCEPDEV/da/Emily.Liu/EMC-wxflow/wxflow + + +# ================================================================= +# Elements should be in the working directory (e.g. run_satwnd) +# ================================================================= + 1. Required input files: + (1) Can be obtained from SPOC: + - bufr2ioda_bufr_backend_satwnd_amv_goes.yaml + - bufr2ioda_satwnd_amv_goes_mapping.yaml + - bufr2ioda_satwnd_amv_goes.py + - bufr2ioda_script_backend_satwnd_amv_goes.yaml + (2) Can be obtained from global dump: + - testinput/2021080100/gdas.t00z.satwnd.tm00.bufr_d + + 2. Processing shell script: + (1) Can be obtained from SPOC: + - process_bufr2ioda + +# ================================================================= +# How to run the test shell script +# ================================================================= + 1. Get help page for usage + ./process_bufrioda -h + + : root directory of obsForge build + : root directory of wxflow build + : cycle time (e.g., 2021080100) + : observation type to create (e.g., satwnd_amv_goes) + : sensor (e.g., abi) + : mode of operation (three valid modes: bufr_backend, script_backend, and bufr2netcdf) + : number of processors (must be a positive integer) + + + 2. Run with default input parameters + ./process_bufrioda + + 3. Run with user-defined input parameters + ./process_bufr2ioda /scratch1/NCEPDEV/da/Emily.Liu/EMC-obsForge/obsForge /scratch1/NCEPDEV/da/Emily.Liu/EMC-wxflow/wxflow 2021080100 satwnd_amv_goes abi bufr2netcdf 12 + + 4. Run with user-defined mode + ./process_bufr2ioda "" "" "" "" "" bufr2netcdf" 4 + ./process_bufr2ioda "" "" "" "" "" bufr_backend" 4 + ./process_bufr2ioda "" "" "" "" "" script_backend" 4 + From 89f429aa8edd2061728bbae6f983fc2d806d3a8b Mon Sep 17 00:00:00 2001 From: Emily Liu Date: Tue, 19 Nov 2024 04:11:43 +0000 Subject: [PATCH 13/72] rename README to README.md --- ush/test/{README => README.md} | 35 +++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) rename ush/test/{README => README.md} (71%) diff --git a/ush/test/README b/ush/test/README.md similarity index 71% rename from ush/test/README rename to ush/test/README.md index 382b013..21a5dcf 100644 --- a/ush/test/README +++ b/ush/test/README.md @@ -1,22 +1,23 @@ -# ================================================================= -# Prerequisite -# ================================================================= +## Prerequisite 1. Clone and build obsForge + + ``` git clone --recursive https://github.com/NOAA-EMC/obsForge ./build.sh + ``` 2. Clone wxflow (no need to build) - git clone https://github.com/NOAA-EMC/wxflow + ``` + git clone https://github.com/NOAA-EMC/wxflow + ``` e.g. Emily's obsForge and wxflow builds on HERA obsForge: /scratch1/NCEPDEV/da/Emily.Liu/EMC-obsForge/obsForge wxflow: /scratch1/NCEPDEV/da/Emily.Liu/EMC-wxflow/wxflow -# ================================================================= -# Elements should be in the working directory (e.g. run_satwnd) -# ================================================================= +## Elements should be in the working directory (e.g. run_satwnd) 1. Required input files: (1) Can be obtained from SPOC: - bufr2ioda_bufr_backend_satwnd_amv_goes.yaml @@ -30,11 +31,12 @@ (1) Can be obtained from SPOC: - process_bufr2ioda -# ================================================================= -# How to run the test shell script -# ================================================================= +## How to run the test shell script 1. Get help page for usage + + ``` ./process_bufrioda -h + ``` : root directory of obsForge build : root directory of wxflow build @@ -46,13 +48,24 @@ 2. Run with default input parameters + + ``` ./process_bufrioda + ``` 3. Run with user-defined input parameters - ./process_bufr2ioda /scratch1/NCEPDEV/da/Emily.Liu/EMC-obsForge/obsForge /scratch1/NCEPDEV/da/Emily.Liu/EMC-wxflow/wxflow 2021080100 satwnd_amv_goes abi bufr2netcdf 12 + + ``` + obsforge_dir="/scratch1/NCEPDEV/da/Emily.Liu/EMC-obsForge/obsForge" + wxflow_dir="/scratch1/NCEPDEV/da/Emily.Liu/EMC-wxflow/wxflow" + ./process_bufr2ioda ${obsforge_dir} ${wxflow_dir} 2021080100 satwnd_amv_goes abi bufr2netcdf 12 + ``` 4. Run with user-defined mode + + ``` ./process_bufr2ioda "" "" "" "" "" bufr2netcdf" 4 ./process_bufr2ioda "" "" "" "" "" bufr_backend" 4 ./process_bufr2ioda "" "" "" "" "" script_backend" 4 + ``` From b11513059a3b761d029d6eda75ae048be7830362 Mon Sep 17 00:00:00 2001 From: emilyhcliu <36091766+emilyhcliu@users.noreply.github.com> Date: Mon, 18 Nov 2024 23:21:36 -0500 Subject: [PATCH 14/72] Update README.md --- ush/test/README.md | 52 +++++++++++++++++++++------------------------- 1 file changed, 24 insertions(+), 28 deletions(-) diff --git a/ush/test/README.md b/ush/test/README.md index 21a5dcf..da9afc8 100644 --- a/ush/test/README.md +++ b/ush/test/README.md @@ -12,31 +12,27 @@ git clone https://github.com/NOAA-EMC/wxflow ``` - e.g. Emily's obsForge and wxflow builds on HERA - obsForge: /scratch1/NCEPDEV/da/Emily.Liu/EMC-obsForge/obsForge - wxflow: /scratch1/NCEPDEV/da/Emily.Liu/EMC-wxflow/wxflow + Example: obsForge and wxflow builds on HERA + - obsForge: /scratch1/NCEPDEV/da/Emily.Liu/EMC-obsForge/obsForge + - wxflow: /scratch1/NCEPDEV/da/Emily.Liu/EMC-wxflow/wxflow ## Elements should be in the working directory (e.g. run_satwnd) - 1. Required input files: - (1) Can be obtained from SPOC: - - bufr2ioda_bufr_backend_satwnd_amv_goes.yaml - - bufr2ioda_satwnd_amv_goes_mapping.yaml - - bufr2ioda_satwnd_amv_goes.py - - bufr2ioda_script_backend_satwnd_amv_goes.yaml - (2) Can be obtained from global dump: - - testinput/2021080100/gdas.t00z.satwnd.tm00.bufr_d - - 2. Processing shell script: - (1) Can be obtained from SPOC: - - process_bufr2ioda +- Required input files: + - bufr2ioda_bufr_backend_satwnd_amv_goes.yaml + - bufr2ioda_satwnd_amv_goes_mapping.yaml + - bufr2ioda_satwnd_amv_goes.py + - bufr2ioda_script_backend_satwnd_amv_goes.yaml + - testinput/2021080100/gdas.t00z.satwnd.tm00.bufr_d + +- Processing shell script: + - process_bufr2ioda ## How to run the test shell script - 1. Get help page for usage +- Get help page for usage - ``` +``` ./process_bufrioda -h - ``` : root directory of obsForge build : root directory of wxflow build @@ -45,27 +41,27 @@ : sensor (e.g., abi) : mode of operation (three valid modes: bufr_backend, script_backend, and bufr2netcdf) : number of processors (must be a positive integer) +``` +- Run with default input parameters - 2. Run with default input parameters - - ``` +``` ./process_bufrioda - ``` +``` - 3. Run with user-defined input parameters +- Run with user-defined input parameters - ``` +``` obsforge_dir="/scratch1/NCEPDEV/da/Emily.Liu/EMC-obsForge/obsForge" wxflow_dir="/scratch1/NCEPDEV/da/Emily.Liu/EMC-wxflow/wxflow" ./process_bufr2ioda ${obsforge_dir} ${wxflow_dir} 2021080100 satwnd_amv_goes abi bufr2netcdf 12 - ``` +``` - 4. Run with user-defined mode +- Run with user-defined mode - ``` +``` ./process_bufr2ioda "" "" "" "" "" bufr2netcdf" 4 ./process_bufr2ioda "" "" "" "" "" bufr_backend" 4 ./process_bufr2ioda "" "" "" "" "" script_backend" 4 - ``` +``` From f0ba20f3521e9ab5c67f7a954a80735e683e2851 Mon Sep 17 00:00:00 2001 From: Emily Liu Date: Tue, 19 Nov 2024 04:34:35 +0000 Subject: [PATCH 15/72] Update readme file --- ush/test/README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ush/test/README.md b/ush/test/README.md index da9afc8..55761b0 100644 --- a/ush/test/README.md +++ b/ush/test/README.md @@ -2,7 +2,12 @@ 1. Clone and build obsForge ``` - git clone --recursive https://github.com/NOAA-EMC/obsForge + git clone --recursive https://github.com/noaa-emc/obsForge -b feature/initial + cd ./obsForge/sorc/ioda + git remote -v + git remote set-url origin https://github.com/jcsda-internal/ioda.git + git pull + git checkout feature/bufr_in_parallel_emily ./build.sh ``` From 8d324d042c82d5a3e251c794197cbfe30bd9b0c1 Mon Sep 17 00:00:00 2001 From: emilyhcliu <36091766+emilyhcliu@users.noreply.github.com> Date: Mon, 18 Nov 2024 23:44:18 -0500 Subject: [PATCH 16/72] Update README.md --- ush/test/README.md | 47 +++++++++++++++++++++++++++++++--------------- 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/ush/test/README.md b/ush/test/README.md index 55761b0..a8cab11 100644 --- a/ush/test/README.md +++ b/ush/test/README.md @@ -1,33 +1,46 @@ ## Prerequisite - 1. Clone and build obsForge +- Clone and build obsForge - ``` - git clone --recursive https://github.com/noaa-emc/obsForge -b feature/initial + ``` + git clone --recursive https://github.com/noaa-emc/obsForge -b feature/initial + cd ./obsForge/sorc/ioda + git remote -v + git remote set-url origin https://github.com/jcsda-internal/ioda.git + git pull + git checkout feature/bufr_in_parallel_emily + ./build.sh - ``` + ``` - 2. Clone wxflow (no need to build) +- Clone wxflow (no need to build) - ``` + ``` git clone https://github.com/NOAA-EMC/wxflow - ``` + ``` - Example: obsForge and wxflow builds on HERA - - obsForge: /scratch1/NCEPDEV/da/Emily.Liu/EMC-obsForge/obsForge - - wxflow: /scratch1/NCEPDEV/da/Emily.Liu/EMC-wxflow/wxflow - +- Example: obsForge and wxflow builds on HERA + ``` + obsForge /scratch1/NCEPDEV/da/Emily.Liu/EMC-obsForge/obsForge + + wxflow /scratch1/NCEPDEV/da/Emily.Liu/EMC-wxflow/wxflow + ``` ## Elements should be in the working directory (e.g. run_satwnd) - Required input files: - - bufr2ioda_bufr_backend_satwnd_amv_goes.yaml - - bufr2ioda_satwnd_amv_goes_mapping.yaml + - bufr2ioda_satwnd_amv_goes.py + + - bufr2ioda_satwnd_amv_goes_mapping.yaml + + - bufr2ioda_bufr_backend_satwnd_amv_goes.yaml + - bufr2ioda_script_backend_satwnd_amv_goes.yaml + - testinput/2021080100/gdas.t00z.satwnd.tm00.bufr_d - Processing shell script: @@ -58,15 +71,19 @@ ``` obsforge_dir="/scratch1/NCEPDEV/da/Emily.Liu/EMC-obsForge/obsForge" + wxflow_dir="/scratch1/NCEPDEV/da/Emily.Liu/EMC-wxflow/wxflow" + ./process_bufr2ioda ${obsforge_dir} ${wxflow_dir} 2021080100 satwnd_amv_goes abi bufr2netcdf 12 ``` - Run with user-defined mode ``` - ./process_bufr2ioda "" "" "" "" "" bufr2netcdf" 4 - ./process_bufr2ioda "" "" "" "" "" bufr_backend" 4 + ./process_bufr2ioda "" "" "" "" "" bufr2netcdf" 4 + + ./process_bufr2ioda "" "" "" "" "" bufr_backend" 4 + ./process_bufr2ioda "" "" "" "" "" script_backend" 4 ``` From d70867f24ba73dd29e1d22dedc74684ddc62be6e Mon Sep 17 00:00:00 2001 From: Emily Liu Date: Tue, 19 Nov 2024 17:14:28 +0000 Subject: [PATCH 17/72] Update README --- ush/test/README.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/ush/test/README.md b/ush/test/README.md index a8cab11..0d8f841 100644 --- a/ush/test/README.md +++ b/ush/test/README.md @@ -44,27 +44,27 @@ - testinput/2021080100/gdas.t00z.satwnd.tm00.bufr_d - Processing shell script: - - process_bufr2ioda + - bufr2ioda.sh ## How to run the test shell script - Get help page for usage ``` - ./process_bufrioda -h + ./bufrioda.sh -h : root directory of obsForge build : root directory of wxflow build : cycle time (e.g., 2021080100) : observation type to create (e.g., satwnd_amv_goes) : sensor (e.g., abi) - : mode of operation (three valid modes: bufr_backend, script_backend, and bufr2netcdf) + : mode of operation (three valid modes: bufr_backend, script_backend, bufr2netcdf, script2netcdf) : number of processors (must be a positive integer) ``` - Run with default input parameters ``` - ./process_bufrioda + ./bufrioda.sh ``` - Run with user-defined input parameters @@ -74,16 +74,16 @@ wxflow_dir="/scratch1/NCEPDEV/da/Emily.Liu/EMC-wxflow/wxflow" - ./process_bufr2ioda ${obsforge_dir} ${wxflow_dir} 2021080100 satwnd_amv_goes abi bufr2netcdf 12 + ./bufr2ioda.sh ${obsforge_dir} ${wxflow_dir} 2021080100 satwnd_amv_goes abi script_backend 4 ``` -- Run with user-defined mode +- Run with user-defined mode and number of processes ``` - ./process_bufr2ioda "" "" "" "" "" bufr2netcdf" 4 + ./bufr2ioda.sh "" "" "" "" "" bufr2netcdf" 8 - ./process_bufr2ioda "" "" "" "" "" bufr_backend" 4 + ./bufr2ioda.sh "" "" "" "" "" script2netcdf" 0 - ./process_bufr2ioda "" "" "" "" "" script_backend" 4 + ./bufr2ioda.sh "" "" "" "" "" bufr_backend" 12 ``` From e5a95e7dae5d142b01d2a2739d542193bbe79fe3 Mon Sep 17 00:00:00 2001 From: Emily Liu Date: Tue, 19 Nov 2024 17:15:00 +0000 Subject: [PATCH 18/72] rename process_bufr2ioda to bufr2ioda.sh --- ush/test/{process_bufr2ioda => bufr2ioda.sh} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename ush/test/{process_bufr2ioda => bufr2ioda.sh} (100%) diff --git a/ush/test/process_bufr2ioda b/ush/test/bufr2ioda.sh similarity index 100% rename from ush/test/process_bufr2ioda rename to ush/test/bufr2ioda.sh From 7dbc826b15573af6ed47508ba90bd41c2a17c280 Mon Sep 17 00:00:00 2001 From: emilyhcliu <36091766+emilyhcliu@users.noreply.github.com> Date: Tue, 19 Nov 2024 12:24:37 -0500 Subject: [PATCH 19/72] Update README.md --- ush/test/README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ush/test/README.md b/ush/test/README.md index 0d8f841..60f301d 100644 --- a/ush/test/README.md +++ b/ush/test/README.md @@ -1,3 +1,6 @@ +## Test bufr_query mapping, Python converter script, and ioda configuration YAML in obsForge +This is a prototype of testing BUFR to IODA conversion and is still evolving. + ## Prerequisite - Clone and build obsForge From 634684427e5c0a1f629adef8bfc89e72ee5bd9e3 Mon Sep 17 00:00:00 2001 From: emilyhcliu <36091766+emilyhcliu@users.noreply.github.com> Date: Tue, 19 Nov 2024 12:26:46 -0500 Subject: [PATCH 20/72] Update README.md --- ush/test/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/test/README.md b/ush/test/README.md index 60f301d..5580788 100644 --- a/ush/test/README.md +++ b/ush/test/README.md @@ -33,7 +33,7 @@ This is a prototype of testing BUFR to IODA conversion and is still evolving. wxflow /scratch1/NCEPDEV/da/Emily.Liu/EMC-wxflow/wxflow ``` -## Elements should be in the working directory (e.g. run_satwnd) +## Elements should be in the working directory from SPOC - Required input files: - bufr2ioda_satwnd_amv_goes.py @@ -50,7 +50,7 @@ This is a prototype of testing BUFR to IODA conversion and is still evolving. - bufr2ioda.sh ## How to run the test shell script -- Get help page for usage +- Get the help page for usage ``` ./bufrioda.sh -h From f91d1d15fe6d7a1d31a4fe79d45c85f082a485dc Mon Sep 17 00:00:00 2001 From: Emily Liu Date: Tue, 19 Nov 2024 17:27:58 +0000 Subject: [PATCH 21/72] Update README.md --- ush/test/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/test/README.md b/ush/test/README.md index 5580788..1a4852e 100644 --- a/ush/test/README.md +++ b/ush/test/README.md @@ -1,5 +1,5 @@ ## Test bufr_query mapping, Python converter script, and ioda configuration YAML in obsForge -This is a prototype of testing BUFR to IODA conversion and is still evolving. +This is a prototype for testing BUFR to IODA conversion and is still evolving. ## Prerequisite - Clone and build obsForge From 8ba4d7b16da619a598dbcbc3c15a987793dbab35 Mon Sep 17 00:00:00 2001 From: emilyhcliu <36091766+emilyhcliu@users.noreply.github.com> Date: Tue, 19 Nov 2024 12:29:48 -0500 Subject: [PATCH 22/72] Update README.md --- ush/test/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/test/README.md b/ush/test/README.md index 1a4852e..5a1f41d 100644 --- a/ush/test/README.md +++ b/ush/test/README.md @@ -44,7 +44,7 @@ This is a prototype for testing BUFR to IODA conversion and is still evolving. - bufr2ioda_script_backend_satwnd_amv_goes.yaml - - testinput/2021080100/gdas.t00z.satwnd.tm00.bufr_d + - testinput/2021080100/gdas.t00z.satwnd.tm00.bufr_d (copied from the global dump) - Processing shell script: - bufr2ioda.sh From 87096eb55c0f7ccb74a1138df086666e3f3e633f Mon Sep 17 00:00:00 2001 From: Emily Liu Date: Tue, 19 Nov 2024 19:44:32 +0000 Subject: [PATCH 23/72] Update readme --- ush/test/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/test/README.md b/ush/test/README.md index 5a1f41d..388a85f 100644 --- a/ush/test/README.md +++ b/ush/test/README.md @@ -60,7 +60,7 @@ This is a prototype for testing BUFR to IODA conversion and is still evolving. : cycle time (e.g., 2021080100) : observation type to create (e.g., satwnd_amv_goes) : sensor (e.g., abi) - : mode of operation (three valid modes: bufr_backend, script_backend, bufr2netcdf, script2netcdf) + : mode of operation (four valid modes: bufr_backend, script_backend, bufr2netcdf, script2netcdf) : number of processors (must be a positive integer) ``` From 3e7789cc0978d63137994a84f4f64c4866b71d86 Mon Sep 17 00:00:00 2001 From: Emily Liu Date: Tue, 19 Nov 2024 20:00:43 +0000 Subject: [PATCH 24/72] Add comments --- ush/test/bufr2ioda.sh | 97 +++++++++++++++++++++++++++++++++---------- 1 file changed, 74 insertions(+), 23 deletions(-) diff --git a/ush/test/bufr2ioda.sh b/ush/test/bufr2ioda.sh index 4e3035f..2ace608 100755 --- a/ush/test/bufr2ioda.sh +++ b/ush/test/bufr2ioda.sh @@ -1,7 +1,10 @@ #!/bin/bash +# ==================== # Get input arguments -obsforge_dir="${1:-/scratch1/NCEPDEV/da/Emily.Liu/EMC-obsForge/obsForge}" +# ==================== +#obsforge_dir="${1:-/scratch1/NCEPDEV/da/Emily.Liu/EMC-obsForge/obsForge}" +obsforge_dir="${1:-/scratch1/NCEPDEV/da/Emily.Liu/obsforge}" wxflow_dir="${2:-/scratch1/NCEPDEV/da/Emily.Liu/EMC-wxflow/wxflow}" cycle="${3:-2021080100}" obstype="${4:-satwnd_amv_goes}" @@ -9,7 +12,9 @@ sensor="${5:-abi}" mode="${6:-script_backend}" nproc="${7:-4}" +# ========================== # Function to display usage +# ========================== usage() { echo "Usage: $0 " echo " : root directory of obsForge build" @@ -17,35 +22,45 @@ usage() { echo " : cycle time (e.g., 2021080100)" echo " : observation type to create (e.g., satwnd_amv_goes)" echo " : sensor (e.g., abi)" - echo " : mode of operation (three valid modes: bufr_backend, script_backend, and bufr2netcdf)" - echo " : number of processors (must be a positive integer)" + echo " : mode of operation (e.g., bufr_backend, script_backend, bufr2netcdf, script2netcdf)" + echo " : number of processors (must be a positive integer to run with MPI or zero to run without MPI)" exit 1 } +# ============================= # Check for --help or -h flags +# ============================= if [[ "$1" == "--help" || "$1" == "-h" ]]; then usage fi -# Check if both arguments are provided +# ============================================= +# Check if all required arguments are provided +# ============================================= if [[ -z "$mode" || -z "$nproc" || -z "$sensor" || -z "$obstype" || -z "$cycle" || -z "${obsforge_dir}" || -z "${wxflow_dir}" ]]; then echo "Error: Missing arguments." usage fi +# ============== # Validate mode -if [[ "$mode" != "bufr_backend" && "$mode" != "script_backend" && "$mode" != "bufr2netcdf" ]]; then - echo "Error: Invalid mode '$mode'. Expected 'bufr_backend' or 'script_backend' or 'bufr2netcdf'." +# ============== +if [[ "$mode" != "bufr_backend" && "$mode" != "script_backend" && "$mode" != "bufr2netcdf" && "$mode" != "script2netcdf" ]]; then + echo "Error: Invalid mode '$mode'. Expected 'bufr_backend' or 'script_backend' or 'bufr2netcdf' or 'script2netcdf'." usage fi -# Validate nproc is a positive integer -if ! [[ "$nproc" =~ ^[0-9]+$ && "$nproc" -gt 0 ]]; then - echo "Error: nproc must be a positive integer." +# ============================================= +# Validate nproc is a positive integer or zero +# ============================================= +if ! [[ "$nproc" =~ ^[0-9]+$ && "$nproc" -ge 0 ]]; then + echo "Error: nproc must be a positive integer or zero." usage fi -# If all checks pass +# ========================= +# Check if all checks pass +# ========================= echo "root directory of obsForge: $obsforge_dir" echo "root directory of wxflow: $wxflow_dir" echo "cycle: $cycle" @@ -54,82 +69,118 @@ echo "sensor: $sensor" echo "mode: $mode" echo "number of processors: $nproc" +# ====================== # Enable debugging mode +# ====================== set -x +# ========================= # Set unlimited stack size +# ========================= ulimit -s unlimited ulimit -a +# ==================================== # Set OOPS run time output parameters +# ==================================== export OOPS_TRACE=1 export OOPS_DEBUG=1 +# ===================== # Set WXFLOW log level +# ===================== export LOG_LEVEL=DEBUG +# =============================== # Load obsForge required modules -#obsforge_dir="/scratch1/NCEPDEV/da/Emily.Liu/EMC-obsForge/obsForge" +# =============================== module use ${obsforge_dir}/modulefiles module load obsforge/hera.intel module list +# ============================== # Set bufr-query python library +# ============================== export LD_LIBRARY_PATH="${obsforge_dir}/build/lib:${LD_LIBRARY_PATH}" export PYTHONPATH="${PYTHONPATH}:${obsforge_dir}/build/lib/python3.10/site-packages" +# ======================== # Set ioda python library +# ========================= export PYTHONPATH="${PYTHONPATH}:${obsforge_dir}/build/lib/python3.10" +# ============ # Set wxfloww -#wxflow_dir="/scratch1/NCEPDEV/da/Emily.Liu/EMC-wxflow/wxflow" +# ============ export PYTHONPATH="${PYTHONPATH}:${wxflow_dir}/src" +# =========================== # Configure SLUM environment +# =========================== export SLURM_ACCOUNT=da-cpu export SALLOC_ACCOUNT=$SLURM_ACCOUNT export SBATCH_ACCOUNT=$SLURM_ACCOUNT export SLURM_QOS=debug +# =================================== # Extract year, month, day, and hour +# =================================== y4="${cycle:0:4}" m2="${cycle:4:2}" d2="${cycle:6:2}" h2="${cycle:8:2}" -# Directory paths +# ==================== +# Set directory paths +# ==================== work_dir="$PWD" out_dir="${work_dir}/testoutput/$cycle/${mode}" in_dir="${work_dir}/testinput/" mkdir -p -m770 ${out_dir} -# File paths +# =============== +# Set file paths +# =============== mapping_file="${work_dir}/bufr2ioda_${obstype}_mapping.yaml" input_file="${in_dir}/gdas.t${h2}z.satwnd.tm00.bufr_d" output_file="${out_dir}/gdas.t${h2}z.satwnd_${sensor}_{splits/satId}.tm00.nc" +ioda_config_yaml="${work_dir}/bufr2ioda_${mode}_${obstype}.yaml" -config_yaml="${work_dir}/bufr2ioda_${mode}_${obstype}.yaml" - -# Run IODA bufr/script backend +# ============================= +# Run ioda bufr/script backend +# ============================= if [[ "$mode" == "bufr_backend" || "$mode" == "script_backend" ]]; then if [[ "$nproc" == "0" ]]; then echo Run time_IodaIO.x without MPI ... - ${obsforge_dir}/build/bin/time_IodaIO.x ${config_yaml} + ${obsforge_dir}/build/bin/time_IodaIO.x ${ioda_config_yaml} else echo Run time_IodaIO.x with MPI ${nproc} ... - srun -n $nproc --mem 96G --time 00:30:00 ${obsforge_dir}/build/bin/time_IodaIO.x ${config_yaml} + srun -n $nproc --mem 96G --time 00:30:00 ${obsforge_dir}/build/bin/time_IodaIO.x ${ioda_config_yaml} fi -# Run bufr_query netcdf +# ================ +# Run bufr2netcdf +# ================ elif [[ "$mode" == "bufr2netcdf" ]]; then if [[ "$nproc" == "0" ]]; then - echo Run Python script without MPI ... + echo Run bufr2netcdf without MPI ... + ${obsforge_dir}/build/bin/bufr2netcdf.x "$input_file" "${mapping_file}" "$output_file" + else + echo Run bufr2netcdf with MPI ${nproc} ... + srun -n "$nproc" --time 00:30:00 --mem 96G ${obsforge_dir}/build/bin/bufr2netcdf.x "$input_file" "${mapping_file}" "$output_file" + fi +# ================== +# Run script2netcdf +# ================== +elif [[ "$mode" == "script2netcdf" ]]; then + if [[ "$nproc" == "0" ]]; then + echo Run script2netcdf without MPI ... python bufr2ioda_${obstype}.py -m "$mapping_file" -o "$output_file" -i "$input_file" else - echo Run Python script with MPI ${nproc} ... + echo Run script2netcdf with MPI ${nproc} ... srun -n "$nproc" --time 00:30:00 --mem 96G python bufr2ioda_${obstype}.py -m "$mapping_file" -o "$output_file" -i "$input_file" fi else - echo Incorrect running mode ${mode} ... Valid modes are: bufr_backend, script_back, or bufr2netcdf + echo Incorrect running mode ${mode} ... Valid modes are: bufr_backend, script_back, bufr2netcdf, or script2netcdf fi From a645370ea03456ae685bf89cec35c70d0a2c70f6 Mon Sep 17 00:00:00 2001 From: Emily Liu Date: Tue, 19 Nov 2024 20:02:08 +0000 Subject: [PATCH 25/72] update comments --- ush/test/bufr2ioda.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ush/test/bufr2ioda.sh b/ush/test/bufr2ioda.sh index 2ace608..15af35d 100755 --- a/ush/test/bufr2ioda.sh +++ b/ush/test/bufr2ioda.sh @@ -3,8 +3,7 @@ # ==================== # Get input arguments # ==================== -#obsforge_dir="${1:-/scratch1/NCEPDEV/da/Emily.Liu/EMC-obsForge/obsForge}" -obsforge_dir="${1:-/scratch1/NCEPDEV/da/Emily.Liu/obsforge}" +obsforge_dir="${1:-/scratch1/NCEPDEV/da/Emily.Liu/EMC-obsForge/obsForge}" wxflow_dir="${2:-/scratch1/NCEPDEV/da/Emily.Liu/EMC-wxflow/wxflow}" cycle="${3:-2021080100}" obstype="${4:-satwnd_amv_goes}" From c0fdb952efa24ecce0cf5445e8656e41b1ecf1f3 Mon Sep 17 00:00:00 2001 From: Emily Liu Date: Tue, 19 Nov 2024 20:45:51 +0000 Subject: [PATCH 26/72] update bufr2ioda.sh --- ush/test/bufr2ioda.sh | 38 ++++++++++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/ush/test/bufr2ioda.sh b/ush/test/bufr2ioda.sh index 15af35d..a174992 100755 --- a/ush/test/bufr2ioda.sh +++ b/ush/test/bufr2ioda.sh @@ -22,7 +22,7 @@ usage() { echo " : observation type to create (e.g., satwnd_amv_goes)" echo " : sensor (e.g., abi)" echo " : mode of operation (e.g., bufr_backend, script_backend, bufr2netcdf, script2netcdf)" - echo " : number of processors (must be a positive integer to run with MPI or zero to run without MPI)" + echo " : number of processors (positive integer to run with MPI, or zero for serial execution)" exit 1 } @@ -37,7 +37,12 @@ fi # Check if all required arguments are provided # ============================================= if [[ -z "$mode" || -z "$nproc" || -z "$sensor" || -z "$obstype" || -z "$cycle" || -z "${obsforge_dir}" || -z "${wxflow_dir}" ]]; then - echo "Error: Missing arguments." + echo "Error: Missing one or more required arguments." + usage +fi + +if ! [[ "$cycle" =~ ^[0-9]{10}$ ]]; then + echo "Error: Invalid cycle format. Expected YYYYMMDDHH." usage fi @@ -94,7 +99,7 @@ export LOG_LEVEL=DEBUG # Load obsForge required modules # =============================== module use ${obsforge_dir}/modulefiles -module load obsforge/hera.intel +module load obsforge/hera.intel || { echo "Error loading obsforge module"; exit 1; } module list # ============================== @@ -135,7 +140,7 @@ h2="${cycle:8:2}" work_dir="$PWD" out_dir="${work_dir}/testoutput/$cycle/${mode}" in_dir="${work_dir}/testinput/" -mkdir -p -m770 ${out_dir} +mkdir -p -m770 ${out_dir} || { echo "Error creating output directory: ${out_dir}"; exit 1; } # =============== # Set file paths @@ -145,16 +150,29 @@ input_file="${in_dir}/gdas.t${h2}z.satwnd.tm00.bufr_d" output_file="${out_dir}/gdas.t${h2}z.satwnd_${sensor}_{splits/satId}.tm00.nc" ioda_config_yaml="${work_dir}/bufr2ioda_${mode}_${obstype}.yaml" +if [[ ! -f "$input_file" ]]; then + echo "Error: Input file not found: $input_file" + exit 1 +fi +if [[ ! -f "$mapping_file" ]]; then + echo "Error: mapping file not found: $mapping" + exit 1 +fi + # ============================= # Run ioda bufr/script backend # ============================= if [[ "$mode" == "bufr_backend" || "$mode" == "script_backend" ]]; then + if [[ ! -f "$ioda_config_yaml" ]]; then + echo "Error: ioda configuration file not found: $ioda_config_yaml" + exit 1 + fi if [[ "$nproc" == "0" ]]; then echo Run time_IodaIO.x without MPI ... - ${obsforge_dir}/build/bin/time_IodaIO.x ${ioda_config_yaml} + ${obsforge_dir}/build/bin/time_IodaIO.x ${ioda_config_yaml} || { echo "Error: time_IodaIO.x failed"; exit 1; } else echo Run time_IodaIO.x with MPI ${nproc} ... - srun -n $nproc --mem 96G --time 00:30:00 ${obsforge_dir}/build/bin/time_IodaIO.x ${ioda_config_yaml} + srun -n $nproc --mem 96G --time 00:30:00 ${obsforge_dir}/build/bin/time_IodaIO.x ${ioda_config_yaml} || { echo "Error: MPI time_IodaIO.x failed"; exit 1; } fi # ================ # Run bufr2netcdf @@ -162,10 +180,10 @@ if [[ "$mode" == "bufr_backend" || "$mode" == "script_backend" ]]; then elif [[ "$mode" == "bufr2netcdf" ]]; then if [[ "$nproc" == "0" ]]; then echo Run bufr2netcdf without MPI ... - ${obsforge_dir}/build/bin/bufr2netcdf.x "$input_file" "${mapping_file}" "$output_file" + ${obsforge_dir}/build/bin/bufr2netcdf.x "$input_file" "${mapping_file}" "$output_file" || { echo "Error: bufr2netcdf.x failed"; exit 1; } else echo Run bufr2netcdf with MPI ${nproc} ... - srun -n "$nproc" --time 00:30:00 --mem 96G ${obsforge_dir}/build/bin/bufr2netcdf.x "$input_file" "${mapping_file}" "$output_file" + srun -n "$nproc" --time 00:30:00 --mem 96G ${obsforge_dir}/build/bin/bufr2netcdf.x "$input_file" "${mapping_file}" "$output_file" || { echo "Error: MPI bufr2netcdf.x failed"; exit 1; } fi # ================== # Run script2netcdf @@ -173,10 +191,10 @@ elif [[ "$mode" == "bufr2netcdf" ]]; then elif [[ "$mode" == "script2netcdf" ]]; then if [[ "$nproc" == "0" ]]; then echo Run script2netcdf without MPI ... - python bufr2ioda_${obstype}.py -m "$mapping_file" -o "$output_file" -i "$input_file" + python bufr2ioda_${obstype}.py -m "$mapping_file" -o "$output_file" -i "$input_file" || { echo "Error: Python script2netcdf failed"; exit 1; } else echo Run script2netcdf with MPI ${nproc} ... - srun -n "$nproc" --time 00:30:00 --mem 96G python bufr2ioda_${obstype}.py -m "$mapping_file" -o "$output_file" -i "$input_file" + srun -n "$nproc" --time 00:30:00 --mem 96G python bufr2ioda_${obstype}.py -m "$mapping_file" -o "$output_file" -i "$input_file" || { echo "Error: MPI Python script2netcdf failed"; exit 1; } fi else echo Incorrect running mode ${mode} ... Valid modes are: bufr_backend, script_back, bufr2netcdf, or script2netcdf From c262e3d753586999d8767c1df6eca7090bb30c1f Mon Sep 17 00:00:00 2001 From: Emily Liu Date: Tue, 19 Nov 2024 20:47:16 +0000 Subject: [PATCH 27/72] update usage --- ush/test/bufr2ioda.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/test/bufr2ioda.sh b/ush/test/bufr2ioda.sh index a174992..8655aee 100755 --- a/ush/test/bufr2ioda.sh +++ b/ush/test/bufr2ioda.sh @@ -15,7 +15,7 @@ nproc="${7:-4}" # Function to display usage # ========================== usage() { - echo "Usage: $0 " + echo "Usage: $0 " echo " : root directory of obsForge build" echo " : root directory of wxflow build" echo " : cycle time (e.g., 2021080100)" From dde4d2780b8e429f1229e002d2b6d31a4564dd0d Mon Sep 17 00:00:00 2001 From: Emily Liu Date: Tue, 19 Nov 2024 20:49:14 +0000 Subject: [PATCH 28/72] Add comments --- ush/test/bufr2ioda.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ush/test/bufr2ioda.sh b/ush/test/bufr2ioda.sh index 8655aee..d189c0f 100755 --- a/ush/test/bufr2ioda.sh +++ b/ush/test/bufr2ioda.sh @@ -41,6 +41,9 @@ if [[ -z "$mode" || -z "$nproc" || -z "$sensor" || -z "$obstype" || -z "$cycle" usage fi +# =============== +# Validate cycle +# =============== if ! [[ "$cycle" =~ ^[0-9]{10}$ ]]; then echo "Error: Invalid cycle format. Expected YYYYMMDDHH." usage From d080c2398865cd9e6cdd9a68ba4a9ff72cd3d76a Mon Sep 17 00:00:00 2001 From: Emily Liu Date: Wed, 20 Nov 2024 17:05:55 +0000 Subject: [PATCH 29/72] add cycle in input path --- dump/config/bufr2ioda_script_backend_satwnd_amv_goes.yaml | 6 +++--- ush/test/bufr2ioda.sh | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dump/config/bufr2ioda_script_backend_satwnd_amv_goes.yaml b/dump/config/bufr2ioda_script_backend_satwnd_amv_goes.yaml index 7172e6a..b9f19ce 100644 --- a/dump/config/bufr2ioda_script_backend_satwnd_amv_goes.yaml +++ b/dump/config/bufr2ioda_script_backend_satwnd_amv_goes.yaml @@ -13,7 +13,7 @@ observations: type: script script file: "bufr2ioda_satwnd_amv_goes.py" args: - input_path: "./testinput/gdas.t00z.satwnd.tm00.bufr_d" + input_path: "./testinput/2021080100/gdas.t00z.satwnd.tm00.bufr_d" mapping_path: "./bufr2ioda_satwnd_amv_goes_mapping.yaml" category: "goes-16" obsdataout: @@ -31,7 +31,7 @@ observations: type: script script file: "bufr2ioda_satwnd_amv_goes.py" args: - input_path: "./testinput/gdas.t00z.satwnd.tm00.bufr_d" + input_path: "./testinput/2021080100/gdas.t00z.satwnd.tm00.bufr_d" mapping_path: "./bufr2ioda_satwnd_amv_goes_mapping.yaml" category: "goes-17" obsdataout: @@ -49,7 +49,7 @@ observations: type: script script file: "bufr2ioda_satwnd_amv_goes.py" args: - input_path: "./testinput/gdas.t00z.satwnd.tm00.bufr_d" + input_path: "./testinput/2021080100/gdas.t00z.satwnd.tm00.bufr_d" mapping_path: "./bufr2ioda_satwnd_amv_goes_mapping.yaml" category: "goes-18" obsdataout: diff --git a/ush/test/bufr2ioda.sh b/ush/test/bufr2ioda.sh index d189c0f..d3fb905 100755 --- a/ush/test/bufr2ioda.sh +++ b/ush/test/bufr2ioda.sh @@ -142,7 +142,7 @@ h2="${cycle:8:2}" # ==================== work_dir="$PWD" out_dir="${work_dir}/testoutput/$cycle/${mode}" -in_dir="${work_dir}/testinput/" +in_dir="${work_dir}/testinput/$cycle" mkdir -p -m770 ${out_dir} || { echo "Error creating output directory: ${out_dir}"; exit 1; } # =============== From 09d00616cf589bba6d71cc28022360dea0f63b67 Mon Sep 17 00:00:00 2001 From: Emily Liu Date: Wed, 20 Nov 2024 17:53:41 +0000 Subject: [PATCH 30/72] Modify global attribute in the mapping file --- .../bufr2ioda_satwnd_amv_goes_mapping.yaml | 26 ++++++++----------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/dump/mapping/bufr2ioda_satwnd_amv_goes_mapping.yaml b/dump/mapping/bufr2ioda_satwnd_amv_goes_mapping.yaml index 371cf8c..aecba66 100755 --- a/dump/mapping/bufr2ioda_satwnd_amv_goes_mapping.yaml +++ b/dump/mapping/bufr2ioda_satwnd_amv_goes_mapping.yaml @@ -77,38 +77,34 @@ bufr: _272: goes-18 encoder: -# type: netcdf +# type: netcdf -# dimensions: +# dimensions: globals: - name: "platformCommonName" type: string - value: "Meteorological Operational Satellite" + value: "GOES" - name: "platformLongDescription" type: string - value: "EUMETSAT Polar System in sunsynchronous orbit" + value: "Geostationary Operational Satellite" - - name: "source" - type: string - value: "MTYP 021-241 IASI 1C RADIANCES (VARIABLE CHNS) (METOP)" - - - name: "sourceFiles" + - name: "sensor" type: string - value: "gdas.t00z.mtiasi.tm00.bufr_d" + value: "Advanced Baseline Imager" - - name: "datetimeReference" + - name: "source" type: string - value: "2021-08-01T00:00:00Z" + value: "NCEP BUFR Dump for satellite derived atmospheric motion vectors (satwnd)" - - name: "sensor" + - name: "providerFullName" type: string - value: "IASI" + value: "National Environmental Satellite, Data, and Information Service" - name: "processingLevel" type: string - value: "Level-1C" + value: "Level-2" - name: "converter" type: string From cd74eeed877bbb75316bdebaa1e89c66b03bc24b Mon Sep 17 00:00:00 2001 From: Emily Liu Date: Thu, 21 Nov 2024 02:49:34 +0000 Subject: [PATCH 31/72] remove . before bufr2ioda.sh --- ush/test/README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ush/test/README.md b/ush/test/README.md index 388a85f..632dee9 100644 --- a/ush/test/README.md +++ b/ush/test/README.md @@ -53,7 +53,7 @@ This is a prototype for testing BUFR to IODA conversion and is still evolving. - Get the help page for usage ``` - ./bufrioda.sh -h + bufrioda.sh -h : root directory of obsForge build : root directory of wxflow build @@ -67,7 +67,7 @@ This is a prototype for testing BUFR to IODA conversion and is still evolving. - Run with default input parameters ``` - ./bufrioda.sh + bufrioda.sh ``` - Run with user-defined input parameters @@ -77,16 +77,16 @@ This is a prototype for testing BUFR to IODA conversion and is still evolving. wxflow_dir="/scratch1/NCEPDEV/da/Emily.Liu/EMC-wxflow/wxflow" - ./bufr2ioda.sh ${obsforge_dir} ${wxflow_dir} 2021080100 satwnd_amv_goes abi script_backend 4 + bufr2ioda.sh ${obsforge_dir} ${wxflow_dir} 2021080100 satwnd_amv_goes abi script_backend 4 ``` - Run with user-defined mode and number of processes ``` - ./bufr2ioda.sh "" "" "" "" "" bufr2netcdf" 8 + bufr2ioda.sh "" "" "" "" "" bufr2netcdf" 8 - ./bufr2ioda.sh "" "" "" "" "" script2netcdf" 0 + bufr2ioda.sh "" "" "" "" "" script2netcdf" 0 - ./bufr2ioda.sh "" "" "" "" "" bufr_backend" 12 + bufr2ioda.sh "" "" "" "" "" bufr_backend" 12 ``` From 6351d3494045b07b24a5b4b8215dd45208939c62 Mon Sep 17 00:00:00 2001 From: emilyhcliu <36091766+emilyhcliu@users.noreply.github.com> Date: Thu, 21 Nov 2024 12:19:22 -0500 Subject: [PATCH 32/72] Update README.md --- ush/test/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/test/README.md b/ush/test/README.md index 632dee9..a36d95a 100644 --- a/ush/test/README.md +++ b/ush/test/README.md @@ -5,7 +5,7 @@ This is a prototype for testing BUFR to IODA conversion and is still evolving. - Clone and build obsForge ``` - git clone --recursive https://github.com/noaa-emc/obsForge -b feature/initial + git clone --recursive https://github.com/noaa-emc/obsForge cd ./obsForge/sorc/ioda From b2ea3aef14ba4de61bad38b0bb1b8cebc8943686 Mon Sep 17 00:00:00 2001 From: emilyhcliu <36091766+emilyhcliu@users.noreply.github.com> Date: Fri, 22 Nov 2024 14:29:17 -0500 Subject: [PATCH 33/72] Update README.md --- ush/test/README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ush/test/README.md b/ush/test/README.md index a36d95a..ef612bc 100644 --- a/ush/test/README.md +++ b/ush/test/README.md @@ -16,6 +16,12 @@ This is a prototype for testing BUFR to IODA conversion and is still evolving. git pull git checkout feature/bufr_in_parallel_emily + + cd ../spoc + + git checkout feature/dump_satwind_goes + + cd ../../ ./build.sh ``` From d6781901101d8dfcb5b9dc6ea0a36ebfbb8f7ae9 Mon Sep 17 00:00:00 2001 From: emilyhcliu <36091766+emilyhcliu@users.noreply.github.com> Date: Fri, 22 Nov 2024 14:33:00 -0500 Subject: [PATCH 34/72] Update README.md --- ush/test/README.md | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/ush/test/README.md b/ush/test/README.md index ef612bc..0a0ec51 100644 --- a/ush/test/README.md +++ b/ush/test/README.md @@ -62,7 +62,6 @@ This is a prototype for testing BUFR to IODA conversion and is still evolving. bufrioda.sh -h : root directory of obsForge build - : root directory of wxflow build : cycle time (e.g., 2021080100) : observation type to create (e.g., satwnd_amv_goes) : sensor (e.g., abi) @@ -81,18 +80,16 @@ This is a prototype for testing BUFR to IODA conversion and is still evolving. ``` obsforge_dir="/scratch1/NCEPDEV/da/Emily.Liu/EMC-obsForge/obsForge" - wxflow_dir="/scratch1/NCEPDEV/da/Emily.Liu/EMC-wxflow/wxflow" - - bufr2ioda.sh ${obsforge_dir} ${wxflow_dir} 2021080100 satwnd_amv_goes abi script_backend 4 + bufr2ioda.sh ${obsforge_dir} 2021080100 satwnd_amv_goes abi script_backend 4 ``` - Run with user-defined mode and number of processes ``` - bufr2ioda.sh "" "" "" "" "" bufr2netcdf" 8 + bufr2ioda.sh "" "" "" "" bufr2netcdf" 8 - bufr2ioda.sh "" "" "" "" "" script2netcdf" 0 + bufr2ioda.sh "" "" "" "" script2netcdf" 0 - bufr2ioda.sh "" "" "" "" "" bufr_backend" 12 + bufr2ioda.sh "" "" "" "" bufr_backend" 12 ``` From 67311d6642008da9c89a5495495ddc39a97a167d Mon Sep 17 00:00:00 2001 From: Emily Liu Date: Fri, 22 Nov 2024 19:34:32 +0000 Subject: [PATCH 35/72] remove wxflow from the test script --- ush/test/bufr2ioda.sh | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/ush/test/bufr2ioda.sh b/ush/test/bufr2ioda.sh index d3fb905..5fb85b0 100755 --- a/ush/test/bufr2ioda.sh +++ b/ush/test/bufr2ioda.sh @@ -4,20 +4,18 @@ # Get input arguments # ==================== obsforge_dir="${1:-/scratch1/NCEPDEV/da/Emily.Liu/EMC-obsForge/obsForge}" -wxflow_dir="${2:-/scratch1/NCEPDEV/da/Emily.Liu/EMC-wxflow/wxflow}" -cycle="${3:-2021080100}" -obstype="${4:-satwnd_amv_goes}" -sensor="${5:-abi}" -mode="${6:-script_backend}" -nproc="${7:-4}" +cycle="${2:-2021080100}" +obstype="${3:-satwnd_amv_ahi}" +sensor="${4:-ahi}" +mode="${5:-script_backend}" +nproc="${6:-4}" # ========================== # Function to display usage # ========================== usage() { - echo "Usage: $0 " + echo "Usage: $0 " echo " : root directory of obsForge build" - echo " : root directory of wxflow build" echo " : cycle time (e.g., 2021080100)" echo " : observation type to create (e.g., satwnd_amv_goes)" echo " : sensor (e.g., abi)" @@ -36,7 +34,7 @@ fi # ============================================= # Check if all required arguments are provided # ============================================= -if [[ -z "$mode" || -z "$nproc" || -z "$sensor" || -z "$obstype" || -z "$cycle" || -z "${obsforge_dir}" || -z "${wxflow_dir}" ]]; then +if [[ -z "$mode" || -z "$nproc" || -z "$sensor" || -z "$obstype" || -z "$cycle" || -z "${obsforge_dir}" ]]; then echo "Error: Missing one or more required arguments." usage fi @@ -69,7 +67,6 @@ fi # Check if all checks pass # ========================= echo "root directory of obsForge: $obsforge_dir" -echo "root directory of wxflow: $wxflow_dir" echo "cycle: $cycle" echo "obstype: $obstype" echo "sensor: $sensor" @@ -93,9 +90,10 @@ ulimit -a export OOPS_TRACE=1 export OOPS_DEBUG=1 -# ===================== -# Set WXFLOW log level -# ===================== +# ======================================================= +# Set WXFLOW log level +# Valid Log_LEVEL: DEBUG, INFO, WARNING, ERROR, CRITICAL +# ======================================================= export LOG_LEVEL=DEBUG # =============================== @@ -119,7 +117,7 @@ export PYTHONPATH="${PYTHONPATH}:${obsforge_dir}/build/lib/python3.10" # ============ # Set wxfloww # ============ -export PYTHONPATH="${PYTHONPATH}:${wxflow_dir}/src" +export PYTHONPATH="${PYTHONPATH}:${obsforge_dir}/sorc/wxflow/src" # =========================== # Configure SLUM environment @@ -175,7 +173,8 @@ if [[ "$mode" == "bufr_backend" || "$mode" == "script_backend" ]]; then ${obsforge_dir}/build/bin/time_IodaIO.x ${ioda_config_yaml} || { echo "Error: time_IodaIO.x failed"; exit 1; } else echo Run time_IodaIO.x with MPI ${nproc} ... - srun -n $nproc --mem 96G --time 00:30:00 ${obsforge_dir}/build/bin/time_IodaIO.x ${ioda_config_yaml} || { echo "Error: MPI time_IodaIO.x failed"; exit 1; } +# srun -n $nproc --mem 40G --time 00:30:00 ${obsforge_dir}/build/bin/time_IodaIO.x ${ioda_config_yaml} || { echo "Error: MPI time_IodaIO.x failed"; exit 1; } + srun -n $nproc --time 00:30:00 ${obsforge_dir}/build/bin/time_IodaIO.x ${ioda_config_yaml} || { echo "Error: MPI time_IodaIO.x failed"; exit 1; } fi # ================ # Run bufr2netcdf @@ -186,7 +185,8 @@ elif [[ "$mode" == "bufr2netcdf" ]]; then ${obsforge_dir}/build/bin/bufr2netcdf.x "$input_file" "${mapping_file}" "$output_file" || { echo "Error: bufr2netcdf.x failed"; exit 1; } else echo Run bufr2netcdf with MPI ${nproc} ... - srun -n "$nproc" --time 00:30:00 --mem 96G ${obsforge_dir}/build/bin/bufr2netcdf.x "$input_file" "${mapping_file}" "$output_file" || { echo "Error: MPI bufr2netcdf.x failed"; exit 1; } + srun -n "$nproc" --time 00:30:00 ${obsforge_dir}/build/bin/bufr2netcdf.x "$input_file" "${mapping_file}" "$output_file" || { echo "Error: MPI bufr2netcdf.x failed"; exit 1; } +# srun -n "$nproc" --time 00:30:00 --mem 40G ${obsforge_dir}/build/bin/bufr2netcdf.x "$input_file" "${mapping_file}" "$output_file" || { echo "Error: MPI bufr2netcdf.x failed"; exit 1; } fi # ================== # Run script2netcdf @@ -197,7 +197,8 @@ elif [[ "$mode" == "script2netcdf" ]]; then python bufr2ioda_${obstype}.py -m "$mapping_file" -o "$output_file" -i "$input_file" || { echo "Error: Python script2netcdf failed"; exit 1; } else echo Run script2netcdf with MPI ${nproc} ... - srun -n "$nproc" --time 00:30:00 --mem 96G python bufr2ioda_${obstype}.py -m "$mapping_file" -o "$output_file" -i "$input_file" || { echo "Error: MPI Python script2netcdf failed"; exit 1; } + srun -n "$nproc" --time 00:30:00 python bufr2ioda_${obstype}.py -m "$mapping_file" -o "$output_file" -i "$input_file" || { echo "Error: MPI Python script2netcdf failed"; exit 1; } +# srun -n "$nproc" --time 00:30:00 --mem 40G python bufr2ioda_${obstype}.py -m "$mapping_file" -o "$output_file" -i "$input_file" || { echo "Error: MPI Python script2netcdf failed"; exit 1; } fi else echo Incorrect running mode ${mode} ... Valid modes are: bufr_backend, script_back, bufr2netcdf, or script2netcdf From 428e6004515e71c0dfc372041a6b1257e06bec83 Mon Sep 17 00:00:00 2001 From: Emily Liu Date: Fri, 22 Nov 2024 19:48:03 +0000 Subject: [PATCH 36/72] remove wxflow from input --- ush/test/bufr2ioda.sh | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/ush/test/bufr2ioda.sh b/ush/test/bufr2ioda.sh index 5fb85b0..8f8da8d 100755 --- a/ush/test/bufr2ioda.sh +++ b/ush/test/bufr2ioda.sh @@ -5,8 +5,8 @@ # ==================== obsforge_dir="${1:-/scratch1/NCEPDEV/da/Emily.Liu/EMC-obsForge/obsForge}" cycle="${2:-2021080100}" -obstype="${3:-satwnd_amv_ahi}" -sensor="${4:-ahi}" +obstype="${3:-satwnd_amv_goes}" +sensor="${4:-abi}" mode="${5:-script_backend}" nproc="${6:-4}" @@ -173,8 +173,7 @@ if [[ "$mode" == "bufr_backend" || "$mode" == "script_backend" ]]; then ${obsforge_dir}/build/bin/time_IodaIO.x ${ioda_config_yaml} || { echo "Error: time_IodaIO.x failed"; exit 1; } else echo Run time_IodaIO.x with MPI ${nproc} ... -# srun -n $nproc --mem 40G --time 00:30:00 ${obsforge_dir}/build/bin/time_IodaIO.x ${ioda_config_yaml} || { echo "Error: MPI time_IodaIO.x failed"; exit 1; } - srun -n $nproc --time 00:30:00 ${obsforge_dir}/build/bin/time_IodaIO.x ${ioda_config_yaml} || { echo "Error: MPI time_IodaIO.x failed"; exit 1; } + srun -n $nproc --mem 96G --time 00:30:00 ${obsforge_dir}/build/bin/time_IodaIO.x ${ioda_config_yaml} || { echo "Error: MPI time_IodaIO.x failed"; exit 1; } fi # ================ # Run bufr2netcdf @@ -185,8 +184,7 @@ elif [[ "$mode" == "bufr2netcdf" ]]; then ${obsforge_dir}/build/bin/bufr2netcdf.x "$input_file" "${mapping_file}" "$output_file" || { echo "Error: bufr2netcdf.x failed"; exit 1; } else echo Run bufr2netcdf with MPI ${nproc} ... - srun -n "$nproc" --time 00:30:00 ${obsforge_dir}/build/bin/bufr2netcdf.x "$input_file" "${mapping_file}" "$output_file" || { echo "Error: MPI bufr2netcdf.x failed"; exit 1; } -# srun -n "$nproc" --time 00:30:00 --mem 40G ${obsforge_dir}/build/bin/bufr2netcdf.x "$input_file" "${mapping_file}" "$output_file" || { echo "Error: MPI bufr2netcdf.x failed"; exit 1; } + srun -n "$nproc" --mem 96G --time 00:30:00 ${obsforge_dir}/build/bin/bufr2netcdf.x "$input_file" "${mapping_file}" "$output_file" || { echo "Error: MPI bufr2netcdf.x failed"; exit 1; } fi # ================== # Run script2netcdf @@ -197,8 +195,7 @@ elif [[ "$mode" == "script2netcdf" ]]; then python bufr2ioda_${obstype}.py -m "$mapping_file" -o "$output_file" -i "$input_file" || { echo "Error: Python script2netcdf failed"; exit 1; } else echo Run script2netcdf with MPI ${nproc} ... - srun -n "$nproc" --time 00:30:00 python bufr2ioda_${obstype}.py -m "$mapping_file" -o "$output_file" -i "$input_file" || { echo "Error: MPI Python script2netcdf failed"; exit 1; } -# srun -n "$nproc" --time 00:30:00 --mem 40G python bufr2ioda_${obstype}.py -m "$mapping_file" -o "$output_file" -i "$input_file" || { echo "Error: MPI Python script2netcdf failed"; exit 1; } + srun -n "$nproc" --mem 96G --time 00:30:00 python bufr2ioda_${obstype}.py -m "$mapping_file" -o "$output_file" -i "$input_file" || { echo "Error: MPI Python script2netcdf failed"; exit 1; } fi else echo Incorrect running mode ${mode} ... Valid modes are: bufr_backend, script_back, bufr2netcdf, or script2netcdf From 6564a18a813e93d0d1f31b4e922b245bf1472f8a Mon Sep 17 00:00:00 2001 From: emilyhcliu <36091766+emilyhcliu@users.noreply.github.com> Date: Fri, 22 Nov 2024 17:44:28 -0500 Subject: [PATCH 37/72] Update README.md --- ush/test/README.md | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/ush/test/README.md b/ush/test/README.md index 0a0ec51..68eb673 100644 --- a/ush/test/README.md +++ b/ush/test/README.md @@ -26,17 +26,10 @@ This is a prototype for testing BUFR to IODA conversion and is still evolving. ./build.sh ``` -- Clone wxflow (no need to build) - - ``` - git clone https://github.com/NOAA-EMC/wxflow - ``` - - Example: obsForge and wxflow builds on HERA + ``` obsForge /scratch1/NCEPDEV/da/Emily.Liu/EMC-obsForge/obsForge - - wxflow /scratch1/NCEPDEV/da/Emily.Liu/EMC-wxflow/wxflow ``` ## Elements should be in the working directory from SPOC From f757393327b5912a9835af33b49483ee30cc843a Mon Sep 17 00:00:00 2001 From: Emily Liu Date: Sun, 24 Nov 2024 02:56:42 +0000 Subject: [PATCH 38/72] Add comment block for logger --- dump/mapping/bufr2ioda_satwnd_amv_goes.py | 37 +++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/dump/mapping/bufr2ioda_satwnd_amv_goes.py b/dump/mapping/bufr2ioda_satwnd_amv_goes.py index 0299a29..a3cbf9e 100644 --- a/dump/mapping/bufr2ioda_satwnd_amv_goes.py +++ b/dump/mapping/bufr2ioda_satwnd_amv_goes.py @@ -15,6 +15,43 @@ logger = Logger('BUFR2IODA_satwnd_amv_goes.py', level=log_level, colored_log=False) def logging(comm, level, message): + """ + Logs a message to the console or log file, based on the specified logging level. + + This function ensures that logging is only performed by the root process (`rank 0`) + in a distributed computing environment. The function maps the logging level to + appropriate logger methods and defaults to the 'INFO' level if an invalid level is provided. + + Parameters: + comm: object + The communicator object, typically from a distributed computing framework + (e.g., MPI). It must have a `rank()` method to determine the process rank. + level: str + The logging level as a string. Supported levels are: + - 'DEBUG' + - 'INFO' + - 'WARNING' + - 'ERROR' + - 'CRITICAL' + If an invalid level is provided, a warning will be logged, and the level + will default to 'INFO'. + message: str + The message to be logged. + + Behavior: + - Logs messages only on the root process (`comm.rank() == 0`). + - Maps the provided logging level to a method of the logger object. + - Defaults to 'INFO' and logs a warning if an invalid logging level is given. + - Supports standard logging levels for granular control over log verbosity. + + Example: + >>> logging(comm, 'DEBUG', 'This is a debug message.') + >>> logging(comm, 'ERROR', 'An error occurred!') + + Notes: + - Ensure that a global `logger` object is configured before using this function. + - The `comm` object should conform to MPI-like conventions (e.g., `rank()` method). + """ if comm.rank() == 0: # Define a dictionary to map levels to logger methods From 77f2e92c355b4d3b3768513582bf307afaa18079 Mon Sep 17 00:00:00 2001 From: Emily Liu Date: Sun, 24 Nov 2024 04:55:19 +0000 Subject: [PATCH 39/72] add bufrtype (this is bufr dump list) --- dump/mapping/bufr2ioda_satwnd_amv_goes.py | 5 +++-- .../bufr2ioda_satwnd_amv_goes_mapping.yaml | 2 +- ush/test/bufr2ioda.sh | 19 +++++++++++-------- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/dump/mapping/bufr2ioda_satwnd_amv_goes.py b/dump/mapping/bufr2ioda_satwnd_amv_goes.py index a3cbf9e..1225cf5 100644 --- a/dump/mapping/bufr2ioda_satwnd_amv_goes.py +++ b/dump/mapping/bufr2ioda_satwnd_amv_goes.py @@ -122,7 +122,7 @@ def compute_wind_components(wdir, wspd): Parameters: wdir (array-like): Wind direction in degrees (meteorological convention: 0° = North, 90° = East). - wspd (array-like): Wind speed. + wspd (array-like): Wind speed (m/s). Returns: tuple: U and V wind components as numpy arrays with dtype float32. @@ -139,7 +139,7 @@ def _get_obs_type(swcm, chanfreq): Parameters: swcm (array-like): Switch mode values. - chanfreq (array-like): Channel frequency values (Hz). + chanfreq (array-like): Satellite channel center frequency (Hz). Returns: numpy.ndarray: Observation type array. @@ -283,6 +283,7 @@ def create_obs_file(input_path, mapping_path, output_path): logging(comm, 'INFO', f'Return the encoded data') + if __name__ == '__main__': start_time = time.time() diff --git a/dump/mapping/bufr2ioda_satwnd_amv_goes_mapping.yaml b/dump/mapping/bufr2ioda_satwnd_amv_goes_mapping.yaml index aecba66..26a75f5 100755 --- a/dump/mapping/bufr2ioda_satwnd_amv_goes_mapping.yaml +++ b/dump/mapping/bufr2ioda_satwnd_amv_goes_mapping.yaml @@ -92,7 +92,7 @@ encoder: - name: "sensor" type: string - value: "Advanced Baseline Imager" + value: "Advanced Baseline Imager - ABI" - name: "source" type: string diff --git a/ush/test/bufr2ioda.sh b/ush/test/bufr2ioda.sh index 8f8da8d..a1e213b 100755 --- a/ush/test/bufr2ioda.sh +++ b/ush/test/bufr2ioda.sh @@ -5,10 +5,11 @@ # ==================== obsforge_dir="${1:-/scratch1/NCEPDEV/da/Emily.Liu/EMC-obsForge/obsForge}" cycle="${2:-2021080100}" -obstype="${3:-satwnd_amv_goes}" -sensor="${4:-abi}" -mode="${5:-script_backend}" -nproc="${6:-4}" +bufrtype="${3:-satwnd}" +obstype="${4:-satwnd_amv_goes}" +sensor="${5:-abi}" +mode="${6:-script_backend}" +nproc="${7:-4}" # ========================== # Function to display usage @@ -17,8 +18,9 @@ usage() { echo "Usage: $0 " echo " : root directory of obsForge build" echo " : cycle time (e.g., 2021080100)" - echo " : observation type to create (e.g., satwnd_amv_goes)" - echo " : sensor (e.g., abi)" + echo " : bufr type to process (e.g., satwnd, atms)" + echo " : observation type to create (e.g., satwnd_amv_goes, atms)" + echo " : sensor (e.g., abi, atms)" echo " : mode of operation (e.g., bufr_backend, script_backend, bufr2netcdf, script2netcdf)" echo " : number of processors (positive integer to run with MPI, or zero for serial execution)" exit 1 @@ -68,6 +70,7 @@ fi # ========================= echo "root directory of obsForge: $obsforge_dir" echo "cycle: $cycle" +echo "bufrtype: $bufrtype" echo "obstype: $obstype" echo "sensor: $sensor" echo "mode: $mode" @@ -147,8 +150,8 @@ mkdir -p -m770 ${out_dir} || { echo "Error creating output directory: ${out_dir} # Set file paths # =============== mapping_file="${work_dir}/bufr2ioda_${obstype}_mapping.yaml" -input_file="${in_dir}/gdas.t${h2}z.satwnd.tm00.bufr_d" -output_file="${out_dir}/gdas.t${h2}z.satwnd_${sensor}_{splits/satId}.tm00.nc" +input_file="${in_dir}/gdas.t${h2}z.${bufrtype}.tm00.bufr_d" +output_file="${out_dir}/gdas.t${h2}z.${bufrtype}_${sensor}_{splits/satId}.tm00.nc" ioda_config_yaml="${work_dir}/bufr2ioda_${mode}_${obstype}.yaml" if [[ ! -f "$input_file" ]]; then From 00cd504f19002bc1b9a08cf2758c7e8c0aa8d463 Mon Sep 17 00:00:00 2001 From: Emily Liu Date: Sun, 24 Nov 2024 04:58:28 +0000 Subject: [PATCH 40/72] Update documentation --- dump/mapping/bufr2ioda_satwnd_amv_goes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dump/mapping/bufr2ioda_satwnd_amv_goes.py b/dump/mapping/bufr2ioda_satwnd_amv_goes.py index 1225cf5..ca560a9 100644 --- a/dump/mapping/bufr2ioda_satwnd_amv_goes.py +++ b/dump/mapping/bufr2ioda_satwnd_amv_goes.py @@ -138,7 +138,7 @@ def _get_obs_type(swcm, chanfreq): Determine the observation type based on `swcm` and `chanfreq`. Parameters: - swcm (array-like): Switch mode values. + swcm (array-like): Satellite derived wind calculation method. chanfreq (array-like): Satellite channel center frequency (Hz). Returns: From 9d9e821f2a1c3a2692063b05e9fb363c048fa9f7 Mon Sep 17 00:00:00 2001 From: Emily Liu Date: Sun, 24 Nov 2024 05:15:21 +0000 Subject: [PATCH 41/72] Add bufrtype and update README --- ush/test/README.md | 14 ++++++++------ ush/test/bufr2ioda.sh | 2 +- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/ush/test/README.md b/ush/test/README.md index 68eb673..a2437b7 100644 --- a/ush/test/README.md +++ b/ush/test/README.md @@ -26,7 +26,7 @@ This is a prototype for testing BUFR to IODA conversion and is still evolving. ./build.sh ``` -- Example: obsForge and wxflow builds on HERA +- Example: obsForge builds on HERA ``` obsForge /scratch1/NCEPDEV/da/Emily.Liu/EMC-obsForge/obsForge @@ -56,9 +56,10 @@ This is a prototype for testing BUFR to IODA conversion and is still evolving. : root directory of obsForge build : cycle time (e.g., 2021080100) + : BUFR dump type to process (e.g., satwnd) : observation type to create (e.g., satwnd_amv_goes) : sensor (e.g., abi) - : mode of operation (four valid modes: bufr_backend, script_backend, bufr2netcdf, script2netcdf) + : mode of operation (four valid modes: bufr2netcdf, script2netcdf, bufr_backend, script_backend) : number of processors (must be a positive integer) ``` @@ -73,16 +74,17 @@ This is a prototype for testing BUFR to IODA conversion and is still evolving. ``` obsforge_dir="/scratch1/NCEPDEV/da/Emily.Liu/EMC-obsForge/obsForge" - bufr2ioda.sh ${obsforge_dir} 2021080100 satwnd_amv_goes abi script_backend 4 + bufr2ioda.sh ${obsforge_dir} 2021080100 satwnd satwnd_amv_goes abi script_backend 4 ``` - Run with user-defined mode and number of processes ``` - bufr2ioda.sh "" "" "" "" bufr2netcdf" 8 + bufr2ioda.sh "" "" "" "" "" bufr2netcdf" 8 - bufr2ioda.sh "" "" "" "" script2netcdf" 0 + bufr2ioda.sh "" "" "" "" "" script2netcdf" 0 - bufr2ioda.sh "" "" "" "" bufr_backend" 12 + bufr2ioda.sh "" "" "" "" "" bufr_backend" 12 + bufr2ioda.sh "" "" "" "" "" script_backend" 4 ``` diff --git a/ush/test/bufr2ioda.sh b/ush/test/bufr2ioda.sh index a1e213b..a4df9d8 100755 --- a/ush/test/bufr2ioda.sh +++ b/ush/test/bufr2ioda.sh @@ -18,7 +18,7 @@ usage() { echo "Usage: $0 " echo " : root directory of obsForge build" echo " : cycle time (e.g., 2021080100)" - echo " : bufr type to process (e.g., satwnd, atms)" + echo " : BUFR dump type to process (e.g., satwnd, atms)" echo " : observation type to create (e.g., satwnd_amv_goes, atms)" echo " : sensor (e.g., abi, atms)" echo " : mode of operation (e.g., bufr_backend, script_backend, bufr2netcdf, script2netcdf)" From e9b8833d09f77822ce8657b07bb0ed6fc1fe8ee4 Mon Sep 17 00:00:00 2001 From: Emily Liu Date: Mon, 25 Nov 2024 03:13:27 +0000 Subject: [PATCH 42/72] Add split_by_category input --- ush/test/bufr2ioda.sh | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/ush/test/bufr2ioda.sh b/ush/test/bufr2ioda.sh index a4df9d8..e4e48aa 100755 --- a/ush/test/bufr2ioda.sh +++ b/ush/test/bufr2ioda.sh @@ -8,21 +8,23 @@ cycle="${2:-2021080100}" bufrtype="${3:-satwnd}" obstype="${4:-satwnd_amv_goes}" sensor="${5:-abi}" -mode="${6:-script_backend}" -nproc="${7:-4}" +split_by_category="${6:-true}" +mode="${7:-script_backend}" +nproc="${8:-4}" # ========================== # Function to display usage # ========================== usage() { echo "Usage: $0 " - echo " : root directory of obsForge build" - echo " : cycle time (e.g., 2021080100)" - echo " : BUFR dump type to process (e.g., satwnd, atms)" - echo " : observation type to create (e.g., satwnd_amv_goes, atms)" - echo " : sensor (e.g., abi, atms)" - echo " : mode of operation (e.g., bufr_backend, script_backend, bufr2netcdf, script2netcdf)" - echo " : number of processors (positive integer to run with MPI, or zero for serial execution)" + echo " : root directory of obsForge build" + echo " : cycle time (e.g., 2021080100)" + echo " : BUFR dump type to process (e.g., satwnd, atms, cris, sfcsno)" + echo " : observation type to create (e.g., satwnd_amv_goes, atms, cris, sfcsno)" + echo " : sensor (e.g., abi, atms); for non-satellite dta, sensor is usually obstype (e.g., sfcsno)" + echo " : split the output file into multiple files based on category (false or true)" + echo " : mode of operation (e.g., bufr_backend, script_backend, bufr2netcdf, script2netcdf)" + echo " : number of processors (positive integer to run with MPI, or zero for serial execution)" exit 1 } @@ -36,7 +38,7 @@ fi # ============================================= # Check if all required arguments are provided # ============================================= -if [[ -z "$mode" || -z "$nproc" || -z "$sensor" || -z "$obstype" || -z "$cycle" || -z "${obsforge_dir}" ]]; then +if [[ -z "$mode" || -z "$nproc" || -z "$sensor" || -z "${split_by_category}" || -z "$obstype" || -z "$cycle" || -z "${obsforge_dir}" ]]; then echo "Error: Missing one or more required arguments." usage fi @@ -73,6 +75,7 @@ echo "cycle: $cycle" echo "bufrtype: $bufrtype" echo "obstype: $obstype" echo "sensor: $sensor" +echo "split_by_category: ${split_by_category}" echo "mode: $mode" echo "number of processors: $nproc" @@ -149,15 +152,20 @@ mkdir -p -m770 ${out_dir} || { echo "Error creating output directory: ${out_dir} # =============== # Set file paths # =============== +ioda_config_yaml="${work_dir}/bufr2ioda_${mode}_${obstype}.yaml" mapping_file="${work_dir}/bufr2ioda_${obstype}_mapping.yaml" input_file="${in_dir}/gdas.t${h2}z.${bufrtype}.tm00.bufr_d" -output_file="${out_dir}/gdas.t${h2}z.${bufrtype}_${sensor}_{splits/satId}.tm00.nc" -ioda_config_yaml="${work_dir}/bufr2ioda_${mode}_${obstype}.yaml" +if [[ "${split_by_category}" = "true" ]]; then + output_file="${out_dir}/gdas.t${h2}z.${bufrtype}_${sensor}_{splits/satId}.tm00.nc" +else + output_file="${out_dir}/gdas.t${h2}z.${bufrtype}.tm00.nc" +fi if [[ ! -f "$input_file" ]]; then echo "Error: Input file not found: $input_file" exit 1 fi + if [[ ! -f "$mapping_file" ]]; then echo "Error: mapping file not found: $mapping" exit 1 From aec8e56fd0c9f59907dc11c4ef15eeff916eb35d Mon Sep 17 00:00:00 2001 From: Emily Liu Date: Mon, 25 Nov 2024 03:14:11 +0000 Subject: [PATCH 43/72] Update README --- ush/test/README.md | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/ush/test/README.md b/ush/test/README.md index a2437b7..6da077e 100644 --- a/ush/test/README.md +++ b/ush/test/README.md @@ -54,13 +54,14 @@ This is a prototype for testing BUFR to IODA conversion and is still evolving. ``` bufrioda.sh -h - : root directory of obsForge build - : cycle time (e.g., 2021080100) - : BUFR dump type to process (e.g., satwnd) - : observation type to create (e.g., satwnd_amv_goes) - : sensor (e.g., abi) - : mode of operation (four valid modes: bufr2netcdf, script2netcdf, bufr_backend, script_backend) - : number of processors (must be a positive integer) + : root directory of obsForge build + : cycle time (e.g., 2021080100) + : BUFR dump type to process (e.g., satwnd, atms, cris, sfcsno) + : observation type to create (e.g., satwnd_amv_goes, atms, cris, sfcsno) + : sensor (e.g., abi, atms); for non-satellite dta, sensor is usually obstype (e.g., sfcsno) + : split the data into multiple files based on category (false or true) + : mode of operation (e.g., bufr_backend, script_backend, bufr2netcdf, script2netcdf) + : number of processors (positive integer to run with MPI, or zero for serial execution) ``` - Run with default input parameters @@ -74,17 +75,21 @@ This is a prototype for testing BUFR to IODA conversion and is still evolving. ``` obsforge_dir="/scratch1/NCEPDEV/da/Emily.Liu/EMC-obsForge/obsForge" - bufr2ioda.sh ${obsforge_dir} 2021080100 satwnd satwnd_amv_goes abi script_backend 4 + bufr2ioda.sh ${obsforge_dir} 2021080100 satwnd satwnd_amv_goes abi true script_backend 4 + + bufr2ioda.sh ${obsforge_dir} 2021080100 sfcsno sfcsno sfcsno false script_backend 4 + + bufr2ioda.sh ${obsforge_dir} 2021080100 atms atms atms true script_backend 4 ``` - Run with user-defined mode and number of processes ``` - bufr2ioda.sh "" "" "" "" "" bufr2netcdf" 8 + bufr2ioda.sh "" "" "" "" "" "" bufr2netcdf" 8 - bufr2ioda.sh "" "" "" "" "" script2netcdf" 0 + bufr2ioda.sh "" "" "" "" "" "" script2netcdf" 0 - bufr2ioda.sh "" "" "" "" "" bufr_backend" 12 + bufr2ioda.sh "" "" "" "" "" "" bufr_backend" 12 - bufr2ioda.sh "" "" "" "" "" script_backend" 4 + bufr2ioda.sh "" "" "" "" "" "" script_backend" 4 ``` From 5679a321572e513c25ae07fadbafc342da4a362f Mon Sep 17 00:00:00 2001 From: emilyhcliu <36091766+emilyhcliu@users.noreply.github.com> Date: Sun, 24 Nov 2024 23:07:33 -0500 Subject: [PATCH 44/72] Update README.md --- ush/test/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/test/README.md b/ush/test/README.md index 6da077e..fddffe1 100644 --- a/ush/test/README.md +++ b/ush/test/README.md @@ -56,8 +56,8 @@ This is a prototype for testing BUFR to IODA conversion and is still evolving. : root directory of obsForge build : cycle time (e.g., 2021080100) - : BUFR dump type to process (e.g., satwnd, atms, cris, sfcsno) - : observation type to create (e.g., satwnd_amv_goes, atms, cris, sfcsno) + : BUFR dump type to process (e.g., satwnd, atms, sfcsno) + : observation type to create (e.g., satwnd_amv_goes, atms, sfcsno) : sensor (e.g., abi, atms); for non-satellite dta, sensor is usually obstype (e.g., sfcsno) : split the data into multiple files based on category (false or true) : mode of operation (e.g., bufr_backend, script_backend, bufr2netcdf, script2netcdf) From 70f90d9465f73783bfe14e148306b6ce4b095325 Mon Sep 17 00:00:00 2001 From: Emily Liu Date: Mon, 25 Nov 2024 17:44:53 +0000 Subject: [PATCH 45/72] Rename bufr2ioda to encodeBufr and update readme --- ush/test/README.md | 18 +++++++++--------- ush/test/{bufr2ioda.sh => encodeBufr.sh} | 0 2 files changed, 9 insertions(+), 9 deletions(-) rename ush/test/{bufr2ioda.sh => encodeBufr.sh} (100%) diff --git a/ush/test/README.md b/ush/test/README.md index fddffe1..a66830d 100644 --- a/ush/test/README.md +++ b/ush/test/README.md @@ -46,7 +46,7 @@ This is a prototype for testing BUFR to IODA conversion and is still evolving. - testinput/2021080100/gdas.t00z.satwnd.tm00.bufr_d (copied from the global dump) - Processing shell script: - - bufr2ioda.sh + - encodeBufr.sh ## How to run the test shell script - Get the help page for usage @@ -67,7 +67,7 @@ This is a prototype for testing BUFR to IODA conversion and is still evolving. - Run with default input parameters ``` - bufrioda.sh + encodeBufr.sh ``` - Run with user-defined input parameters @@ -75,21 +75,21 @@ This is a prototype for testing BUFR to IODA conversion and is still evolving. ``` obsforge_dir="/scratch1/NCEPDEV/da/Emily.Liu/EMC-obsForge/obsForge" - bufr2ioda.sh ${obsforge_dir} 2021080100 satwnd satwnd_amv_goes abi true script_backend 4 + encodeBufr.sh ${obsforge_dir} 2021080100 satwnd satwnd_amv_goes abi true script_backend 4 - bufr2ioda.sh ${obsforge_dir} 2021080100 sfcsno sfcsno sfcsno false script_backend 4 + encodeBufr.sh ${obsforge_dir} 2021080100 sfcsno sfcsno sfcsno false script_backend 4 - bufr2ioda.sh ${obsforge_dir} 2021080100 atms atms atms true script_backend 4 + encodeBufr.sh ${obsforge_dir} 2021080100 atms atms atms true script_backend 4 ``` - Run with user-defined mode and number of processes ``` - bufr2ioda.sh "" "" "" "" "" "" bufr2netcdf" 8 + encodeBufr.sh "" "" "" "" "" "" bufr2netcdf" 8 - bufr2ioda.sh "" "" "" "" "" "" script2netcdf" 0 + encodeBufr.sh "" "" "" "" "" "" script2netcdf" 0 - bufr2ioda.sh "" "" "" "" "" "" bufr_backend" 12 + encodeBufr.sh "" "" "" "" "" "" bufr_backend" 12 - bufr2ioda.sh "" "" "" "" "" "" script_backend" 4 + encodeBufr.sh "" "" "" "" "" "" script_backend" 4 ``` diff --git a/ush/test/bufr2ioda.sh b/ush/test/encodeBufr.sh similarity index 100% rename from ush/test/bufr2ioda.sh rename to ush/test/encodeBufr.sh From 1365772ca6042bf693927bbdcd557483dd3c2473 Mon Sep 17 00:00:00 2001 From: emilyhcliu <36091766+emilyhcliu@users.noreply.github.com> Date: Tue, 26 Nov 2024 12:39:09 -0500 Subject: [PATCH 46/72] Update README.md --- ush/test/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/test/README.md b/ush/test/README.md index a66830d..83eb41a 100644 --- a/ush/test/README.md +++ b/ush/test/README.md @@ -15,7 +15,7 @@ This is a prototype for testing BUFR to IODA conversion and is still evolving. git pull - git checkout feature/bufr_in_parallel_emily + git checkout feature/bufr_in_parallel cd ../spoc From 0f5dce2adb4938be3c3e94ba3f4009eb3529daab Mon Sep 17 00:00:00 2001 From: Nicholas Esposito Date: Thu, 5 Dec 2024 14:45:48 +0000 Subject: [PATCH 47/72] adpsfc changes. works --- .../bufr2ioda_prepbufr_backend_adpsfc.yaml | 18 ++ ...ncoder.py => bufr2ioda_adpsfc_prepbufr.py} | 0 ush/test/bufr2ioda_adpsfc_prepbufr.py | 232 ++++++++++++++++++ ...ufr2ioda_bufr_backend_prepbufr_adpsfc.yaml | 19 ++ .../bufr2ioda_prepbufr_adpsfc_mapping.yaml | 187 ++++++++++++++ ...r2ioda_script_backend_prepbufr_adpsfc.yaml | 23 ++ ush/test/encodeBufr.sh | 11 +- 7 files changed, 485 insertions(+), 5 deletions(-) create mode 100644 dump/config/bufr2ioda_prepbufr_backend_adpsfc.yaml rename dump/mapping/{iodatest_prepbufr_adpsfc_encoder.py => bufr2ioda_adpsfc_prepbufr.py} (100%) create mode 100755 ush/test/bufr2ioda_adpsfc_prepbufr.py create mode 100644 ush/test/bufr2ioda_bufr_backend_prepbufr_adpsfc.yaml create mode 100755 ush/test/bufr2ioda_prepbufr_adpsfc_mapping.yaml create mode 100644 ush/test/bufr2ioda_script_backend_prepbufr_adpsfc.yaml diff --git a/dump/config/bufr2ioda_prepbufr_backend_adpsfc.yaml b/dump/config/bufr2ioda_prepbufr_backend_adpsfc.yaml new file mode 100644 index 0000000..0347947 --- /dev/null +++ b/dump/config/bufr2ioda_prepbufr_backend_adpsfc.yaml @@ -0,0 +1,18 @@ +time window: + begin: "2021-07-31T21:00:00Z" + end: "2021-08-01T03:00:00Z" + +observations: +- obs space: + name: "ADPSFC" + simulated variables: ['stationElevation','stationPressure'] + obsdatain: + engine: + type: bufr + obsfile: "./testinput/gdas.t00z.adpsfc.prepbufr" + mapping file: "./iodatest_prepbufr_adpsfc_mapping.yaml" + # category: ["goes-16"] + obsdataout: + engine: + type: H5File + obsfile: "./testoutput/2021080100/bufr_backend/gdas.t00z.adpsfc.prepbufr.nc" diff --git a/dump/mapping/iodatest_prepbufr_adpsfc_encoder.py b/dump/mapping/bufr2ioda_adpsfc_prepbufr.py similarity index 100% rename from dump/mapping/iodatest_prepbufr_adpsfc_encoder.py rename to dump/mapping/bufr2ioda_adpsfc_prepbufr.py diff --git a/ush/test/bufr2ioda_adpsfc_prepbufr.py b/ush/test/bufr2ioda_adpsfc_prepbufr.py new file mode 100755 index 0000000..eb706ef --- /dev/null +++ b/ush/test/bufr2ioda_adpsfc_prepbufr.py @@ -0,0 +1,232 @@ +#!/usr/bin/env python3 +import os +import sys +import bufr +import argparse +import copy +import numpy as np +import numpy.ma as ma +import math +import calendar +import time +from datetime import datetime +from pyioda.ioda.Engines.Bufr import Encoder as iodaEncoder +from bufr.encoders.netcdf import Encoder as netcdfEncoder +from wxflow import Logger + +# Initialize Logger +# Get log level from the environment variable, default to 'INFO it not set +log_level = os.getenv('LOG_LEVEL', 'INFO') +logger = Logger('BUFR2IODA_adpsfc_prepbufr.py', level=log_level, colored_log=False) + +def logging(comm, level, message): + """ + Logs a message to the console or log file, based on the specified logging level. + + This function ensures that logging is only performed by the root process (`rank 0`) + in a distributed computing environment. The function maps the logging level to + appropriate logger methods and defaults to the 'INFO' level if an invalid level is provided. + + Parameters: + comm: object + The communicator object, typically from a distributed computing framework + (e.g., MPI). It must have a `rank()` method to determine the process rank. + level: str + The logging level as a string. Supported levels are: + - 'DEBUG' + - 'INFO' + - 'WARNING' + - 'ERROR' + - 'CRITICAL' + If an invalid level is provided, a warning will be logged, and the level + will default to 'INFO'. + message: str + The message to be logged. + + Behavior: + - Logs messages only on the root process (`comm.rank() == 0`). + - Maps the provided logging level to a method of the logger object. + - Defaults to 'INFO' and logs a warning if an invalid logging level is given. + - Supports standard logging levels for granular control over log verbosity. + + Example: + >>> logging(comm, 'DEBUG', 'This is a debug message.') + >>> logging(comm, 'ERROR', 'An error occurred!') + + Notes: + - Ensure that a global `logger` object is configured before using this function. + - The `comm` object should conform to MPI-like conventions (e.g., `rank()` method). + """ + + if comm.rank() == 0: + # Define a dictionary to map levels to logger methods + log_methods = { + 'DEBUG': logger.debug, + 'INFO': logger.info, + 'WARNING': logger.warning, + 'ERROR': logger.error, + 'CRITICAL': logger.critical, + } + + # Get the appropriate logging method, default to 'INFO' + log_method = log_methods.get(level.upper(), logger.info) + + if log_method == logger.info and level.upper() not in log_methods: + # Log a warning if the level is invalid + logger.warning(f'log level = {level}: not a valid level --> set to INFO') + + # Call the logging method + log_method(message) + +def _make_description(mapping_path, update=False): + description = bufr.encoders.Description(mapping_path) + + if update: + # Define the variables to be added in a list of dictionaries + variables = [ + { + 'name': 'MetaData/sequenceNumber', + 'source': 'variables/sequenceNumber', + 'units': '1', + 'longName': 'Sequence Number (Obs Subtype)', + } + ] + + # Loop through each variable and add it to the description + for var in variables: + description.add_variable( + name=var['name'], + source=var['source'], + units=var['units'], + longName=var['longName'] + ) + + return description + + +#def Compute_dateTime(cycleTimeSinceEpoch, dhr): +# """ +# Compute dateTime using the cycleTimeSinceEpoch and Cycle Time +# minus Cycle Time +# +# Parameters: +# cycleTimeSinceEpoch: Time of cycle in Epoch Time +# dhr: Observation Time Minus Cycle Time +# +# Returns: +# Masked array of dateTime values +# """ +# +# int64_fill_value = np.int64(0) +# +# dateTime = np.zeros(dhr.shape, dtype=np.int64) +# for i in range(len(dateTime)): +# if ma.is_masked(dhr[i]): +# continue +# else: +# dateTime[i] = np.int64(dhr[i]*3600) + cycleTimeSinceEpoch +# +# dateTime = ma.array(dateTime) +# dateTime = ma.masked_values(dateTime, int64_fill_value) +# +# return dateTime + +def _make_obs(comm, input_path, mapping_path): + """ + Create the ioda adpsfc prepbufr observations: + - reads values + - adds sequenceNum + + Parameters + ---------- + comm: object + The communicator object (e.g., MPI) + input_path: str + The input bufr file + mapping_path: str + The input bufr2ioda mapping file + """ + + # Get container from mapping file first + logging(comm, 'INFO', 'Get container from bufr') + container = bufr.Parser(input_path, mapping_path).parse(comm) + + logging(comm, 'DEBUG', f'container list (original): {container.list()}') + logging(comm, 'DEBUG', f'Change longitude range from [0,360] to [-180,180]') + lon = container.get('variables/longitude') + lon_paths = container.get_paths('variables/longitude') + lon[lon>180] -= 360 + lon = ma.round(lon, decimals=2) + + print("Make an array of 0s for MetaData/sequenceNumber") + sequenceNum = np.zeros(lon.shape, dtype=np.int32) + logging(comm, 'DEBUG', f' sequenceNummin/max = {sequenceNum.min()} {sequenceNum.max()}') + + logging(comm, 'DEBUG', f'Update variables in container') + container.replace('variables/longitude', lon) + + logging(comm, 'DEBUG', f'Add variables to container') + container.add('variables/sequenceNumber', sequenceNum, lon_paths) + + # Check + logging(comm, 'DEBUG', f'container list (updated): {container.list()}') + + return container + + +def create_obs_group(input_path, mapping_path, env): + + comm = bufr.mpi.Comm(env["comm_name"]) + + logging(comm, 'INFO', f'Make description and make obs') + description = _make_description(mapping_path, update=True) + container = _make_obs(comm, input_path, mapping_path) + + # Gather data from all tasks into all tasks. Each task will have the complete record + logging(comm, 'INFO', f'Gather data from all tasks into all tasks') + container.all_gather(comm) + + logging(comm, 'INFO', f'Encode the data') + data = next(iter(iodaEncoder(description).encode(container).values())) + + logging(comm, 'INFO', f'Return the encoded data.') + return data + + +def create_obs_file(input_path, mapping_path, output_path): + + comm = bufr.mpi.Comm("world") + container = _make_obs(comm, input_path, mapping_path) + container.gather(comm) + + description = _make_description(mapping_path, update=True) + + # Encode the data + if comm.rank() == 0: + netcdfEncoder(description).encode(container, output_path) + + logging(comm, 'INFO', f'Return the encoded data') + + +if __name__ == '__main__': + start_time = time.time() + + bufr.mpi.App(sys.argv) + comm = bufr.mpi.Comm("world") + + # Required input arguments + parser = argparse.ArgumentParser() + parser.add_argument('-i', '--input', type=str, help='Input BUFR', required=True) + parser.add_argument('-m', '--mapping', type=str, help='BUFR2IODA Mapping File', required=True) + parser.add_argument('-o', '--output', type=str, help='Output NetCDF', required=True) + + args = parser.parse_args() + mapping = args.mapping + infile = args.input + output = args.output + + create_obs_file(infile, mapping, output) + + end_time = time.time() + running_time = end_time - start_time + logging(comm, 'INFO', f'Total running time: {running_time}') diff --git a/ush/test/bufr2ioda_bufr_backend_prepbufr_adpsfc.yaml b/ush/test/bufr2ioda_bufr_backend_prepbufr_adpsfc.yaml new file mode 100644 index 0000000..ef78f26 --- /dev/null +++ b/ush/test/bufr2ioda_bufr_backend_prepbufr_adpsfc.yaml @@ -0,0 +1,19 @@ +time window: + begin: "2021-07-31T21:00:00Z" + end: "2021-08-01T03:00:00Z" + bound to include: begin + +observations: +- obs space: + name: "ADPSFC" + simulated variables: ['stationElevation','stationPressure'] + obsdatain: + engine: + type: bufr + obsfile: "./testinput/2021080100/gdas.t00z.adpsfc.prepbufr" + mapping file: "./bufr2ioda_prepbufr_adpsfc_mapping.yaml" + # category: ["goes-16"] + obsdataout: + engine: + type: H5File + obsfile: "./testoutput/2021080100/bufr_backend/gdas.t00z.adpsfc.prepbufr.nc" diff --git a/ush/test/bufr2ioda_prepbufr_adpsfc_mapping.yaml b/ush/test/bufr2ioda_prepbufr_adpsfc_mapping.yaml new file mode 100755 index 0000000..6f97459 --- /dev/null +++ b/ush/test/bufr2ioda_prepbufr_adpsfc_mapping.yaml @@ -0,0 +1,187 @@ +# (C) Copyright 2023 NOAA/NWS/NCEP/EMC +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +bufr: + variables: + # ObsType + observationType: + query: "*/TYP" + + # MetaData + timestamp: + timeoffset: + timeOffset: "*/DHR" + transforms: + - scale: 3600 + referenceTime: "2021-08-01T00:00:00Z" + prepbufrDataLevelCategory: + query: "*/CAT" + obsTimeMinusCycleTime: + query: "*/DHR" + longitude: + query: "*/XOB" + latitude: + query: "*/YOB" + stationIdentification: + query: "*/SID" + pressure: + query: "*/P___INFO/P__EVENT{1}/POB" + transforms: + - scale: 100 + height: + query: "*/Z___INFO/Z__EVENT{1}/ZOB" + type: float + stationElevation: + query: "*/ELV" + type: float + + # ObsValue + stationPressureObsValue: + query: "*/P___INFO/P__EVENT{1}/POB" + transforms: + - scale: 100 + heightObsValue: + query: "*/Z___INFO/Z__EVENT{1}/ZOB" + type: float + stationElevationObsValue: + query: "*/ELV" + type: float + + # QualityMarker + stationPressureQualityMarker: + query: "*/P___INFO/P__EVENT{1}/PQM" + heightQualityMarker: + query: "*/Z___INFO/Z__EVENT{1}/ZQM" + stationElevationQualityMarker: + query: "*/Z___INFO/Z__EVENT{1}/ZQM" + + # ObsError + stationPressureObsError: + query: "*/P___INFO/P__BACKG/POE" + transforms: + - scale: 100 + +encoder: + + globals: + - name: "data_format" + type: string + value: "prepbufr" + + - name: "subsets" + type: string + value: "ADPSFC" + + - name: "source" + type: string + value: "prepBUFR" + + - name: "data_type" + type: string + value: "ADPSFC" + + - name: "data_description" + type: string + value: "ADPSFC_prepbufr" + + - name: "data_provider" + type: string + value: "U.S. NOAA" + + variables: + + # ObsType + - name: "ObsType/stationPressure" + source: variables/observationType + longName: "Station Pressure ObsType" + + - name: "ObsType/stationElevation" + source: variables/observationType + longName: "Station Elevation ObsType" + + - name: "ObsType/height" + source: variables/observationType + longName: "Height ObsType" + + # MetaData + - name: "MetaData/prepbufrDataLevelCategory" + source: variables/prepbufrDataLevelCategory + units: '1' + longName: "Prepbufr Data Level Category" + + - name: "MetaData/dateTime" + source: variables/timestamp + units: 'seconds since 1970-01-01T00:00:00Z' + longName: 'dateTime' + + - name: "MetaData/latitude" + source: variables/latitude + longName: "Latitude" + units: "degree_north" + range: [-90, 90] + + - name: "MetaData/longitude" + source: variables/longitude + longName: "Longitude" + units: "degree_east" + range: [0, 360] + + # - name: "MetaData/sequenceNumber" + #source: variables/sequenceNumber + #longName: "Sequence Number (Obs Subtype)" + + - name: "MetaData/stationIdentification" + source: variables/stationIdentification + longName: "Station Identification" + + - name: "MetaData/pressure" + source: variables/pressure + longName: "Pressure" + + - name: "MetaData/height" + source: variables/height + longName: "Height" + units: "m" + + - name: "MetaData/stationElevation" + source: variables/stationElevation + longName: "Station Elevation" + units: "m" + + # ObsValue + - name: "ObsValue/stationPressure" + source: variables/stationPressureObsValue + longName: "Station Pressure" + units: "Pa" + + - name: "ObsValue/stationElevation" + source: variables/stationElevationObsValue + longName: "Station Pressure" + units: "m" + + - name: "ObsValue/height" + source: variables/heightObsValue + longName: "height" + units: "m" + + # QualityMarker + - name: "QualityMarker/stationPressure" + source: variables/stationPressureQualityMarker + longName: "Station Pressure Quality Marker" + + - name: "QualityMarker/stationElevation" + source: variables/heightQualityMarker + longName: "StationElevation Quality Marker" + + - name: "QualityMarker/height" + source: variables/heightQualityMarker + longName: "Height Quality Marker" + + # ObsError + - name: "ObsError/stationPressure" + source: variables/stationPressureObsError + longName: "Station Pressure Error" + units: "Pa" + diff --git a/ush/test/bufr2ioda_script_backend_prepbufr_adpsfc.yaml b/ush/test/bufr2ioda_script_backend_prepbufr_adpsfc.yaml new file mode 100644 index 0000000..2b8e097 --- /dev/null +++ b/ush/test/bufr2ioda_script_backend_prepbufr_adpsfc.yaml @@ -0,0 +1,23 @@ +time window: + begin: "2021-07-31T21:00:00Z" + end: "2021-08-01T03:00:00Z" + bound to include: begin + +observations: +- obs space: + name: "ADPSFC" + observed variables: ['stationElevation','stationPressure'] + derived variables: ['stationElevation','stationPressure'] + simulated variables: ['stationElevation','stationPressure'] + obsdatain: + engine: + type: script + script file: "bufr2ioda_adpsfc_prepbufr.py" + args: + input_path: "/scratch1/NCEPDEV/da/Nicholas.Esposito/JEDI/obsForge_sfcshp_20241126/sorc/spoc/ush/test/testinput/2021080100/gdas.t00z.adpsfc.prepbufr" + mapping_path: "/scratch1/NCEPDEV/da/Nicholas.Esposito/JEDI/obsForge_sfcshp_20241126/sorc/spoc/ush/test/bufr2ioda_prepbufr_adpsfc_mapping.yaml" + #category: "goes-16" + obsdataout: + engine: + type: H5File + obsfile: "./testoutput/2021080100/script_backend/gdas.t00z.adpsfc.prepbufr.nc" diff --git a/ush/test/encodeBufr.sh b/ush/test/encodeBufr.sh index e4e48aa..7fa01e4 100755 --- a/ush/test/encodeBufr.sh +++ b/ush/test/encodeBufr.sh @@ -152,9 +152,10 @@ mkdir -p -m770 ${out_dir} || { echo "Error creating output directory: ${out_dir} # =============== # Set file paths # =============== -ioda_config_yaml="${work_dir}/bufr2ioda_${mode}_${obstype}.yaml" -mapping_file="${work_dir}/bufr2ioda_${obstype}_mapping.yaml" -input_file="${in_dir}/gdas.t${h2}z.${bufrtype}.tm00.bufr_d" +ioda_config_yaml="${work_dir}/bufr2ioda_${mode}_prepbufr_${obstype}.yaml" +mapping_file="${work_dir}/bufr2ioda_prepbufr_${obstype}_mapping.yaml" +#input_file="${in_dir}/gdas.t${h2}z.${bufrtype}.tm00.bufr_d" +input_file="${in_dir}/gdas.t${h2}z.${bufrtype}.prepbufr" if [[ "${split_by_category}" = "true" ]]; then output_file="${out_dir}/gdas.t${h2}z.${bufrtype}_${sensor}_{splits/satId}.tm00.nc" else @@ -203,10 +204,10 @@ elif [[ "$mode" == "bufr2netcdf" ]]; then elif [[ "$mode" == "script2netcdf" ]]; then if [[ "$nproc" == "0" ]]; then echo Run script2netcdf without MPI ... - python bufr2ioda_${obstype}.py -m "$mapping_file" -o "$output_file" -i "$input_file" || { echo "Error: Python script2netcdf failed"; exit 1; } + python bufr2ioda_${obstype}_prepbufr.py -m "$mapping_file" -o "$output_file" -i "$input_file" || { echo "Error: Python script2netcdf failed"; exit 1; } else echo Run script2netcdf with MPI ${nproc} ... - srun -n "$nproc" --mem 96G --time 00:30:00 python bufr2ioda_${obstype}.py -m "$mapping_file" -o "$output_file" -i "$input_file" || { echo "Error: MPI Python script2netcdf failed"; exit 1; } + srun -n "$nproc" --mem 96G --time 00:30:00 python bufr2ioda_${obstype}_prepbufr.py -m "$mapping_file" -o "$output_file" -i "$input_file" || { echo "Error: MPI Python script2netcdf failed"; exit 1; } fi else echo Incorrect running mode ${mode} ... Valid modes are: bufr_backend, script_back, bufr2netcdf, or script2netcdf From e523fdbdd0097053f629880824bec002e9d401a6 Mon Sep 17 00:00:00 2001 From: Nicholas Esposito Date: Thu, 5 Dec 2024 14:48:28 +0000 Subject: [PATCH 48/72] update 1 --- dump/mapping/bufr2ioda_adpsfc_prepbufr.py | 294 ++++++++++++++++++++++ 1 file changed, 294 insertions(+) diff --git a/dump/mapping/bufr2ioda_adpsfc_prepbufr.py b/dump/mapping/bufr2ioda_adpsfc_prepbufr.py index d4f88ca..dcf6dfa 100755 --- a/dump/mapping/bufr2ioda_adpsfc_prepbufr.py +++ b/dump/mapping/bufr2ioda_adpsfc_prepbufr.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 import os import sys import bufr @@ -9,9 +10,128 @@ import calendar import time from datetime import datetime +from wxflow import Logger + +# Initialize Logger +# Get log level from the environment variable, default to 'INFO it not set +log_level = os.getenv('LOG_LEVEL', 'INFO') +logger = Logger('BUFR2IODA_adpsfc_prepbufr.py', level=log_level, colored_log=False) + +def logging(comm, level, message): + """ + Logs a message to the console or log file, based on the specified logging level. + + This function ensures that logging is only performed by the root process (`rank 0`) + in a distributed computing environment. The function maps the logging level to + appropriate logger methods and defaults to the 'INFO' level if an invalid level is provided. + + Parameters: + comm: object + The communicator object, typically from a distributed computing framework + (e.g., MPI). It must have a `rank()` method to determine the process rank. + level: str + The logging level as a string. Supported levels are: + - 'DEBUG' + - 'INFO' + - 'WARNING' + - 'ERROR' + - 'CRITICAL' + If an invalid level is provided, a warning will be logged, and the level + will default to 'INFO'. + message: str + The message to be logged. + + Behavior: + - Logs messages only on the root process (`comm.rank() == 0`). + - Maps the provided logging level to a method of the logger object. + - Defaults to 'INFO' and logs a warning if an invalid logging level is given. + - Supports standard logging levels for granular control over log verbosity. + + Example: + >>> logging(comm, 'DEBUG', 'This is a debug message.') + >>> logging(comm, 'ERROR', 'An error occurred!') + + Notes: + - Ensure that a global `logger` object is configured before using this function. + - The `comm` object should conform to MPI-like conventions (e.g., `rank()` method). + """ + + if comm.rank() == 0: + # Define a dictionary to map levels to logger methods + log_methods = { + 'DEBUG': logger.debug, + 'INFO': logger.info, + 'WARNING': logger.warning, + 'ERROR': logger.error, + 'CRITICAL': logger.critical, + } + + # Get the appropriate logging method, default to 'INFO' + log_method = log_methods.get(level.upper(), logger.info) + + if log_method == logger.info and level.upper() not in log_methods: + # Log a warning if the level is invalid + logger.warning(f'log level = {level}: not a valid level --> set to INFO') + + # Call the logging method + log_method(message) + +def _make_description(mapping_path, update=False): + description = bufr.encoders.Description(mapping_path) + + if update: + # Define the variables to be added in a list of dictionaries + variables = [ + { + 'name': 'MetaData/dateTime' + 'source': 'variables/dateTime', + 'units': 'seconds since 1970-01-01T00:00:00Z', + 'longName': 'dateTime', + }, + { + 'name': 'MetaData/sequenceNumber', + 'source': 'variables/sequenceNumber' + 'units': '1', + 'longName': 'Sequence Number (Obs Subtype)', + }, + { + 'name': 'ObsValue/windEastward', + 'source': 'variables/windEastward', + 'units': 'm/s', + 'longName': 'Eastward Wind Component', + }, + { + 'name': 'ObsValue/windNorthward', + 'source': 'variables/windNorthward', + 'units': 'm/s', + 'longName': 'Northward Wind Component', + }, + ] + + # Loop through each variable and add it to the description + for var in variables: + description.add_variable( + name=var['name'], + source=var['source'], + units=var['units'], + longName=var['longName'] + ) + + return description def Compute_dateTime(cycleTimeSinceEpoch, dhr): + """ + Compute dateTime using the cycleTimeSinceEpoch and Cycle Time + minus Cycle Time + + Parameters: + cycleTimeSinceEpoch: Time of cycle in Epoch Time + dhr: Observation Time Minus Cycle Time + + Returns: + Masked array of dateTime values + """ int64_fill_value = np.int64(0) @@ -27,6 +147,94 @@ def Compute_dateTime(cycleTimeSinceEpoch, dhr): return dateTime +def _make_obs(comm, input_path, mapping_path): + + # Get container from mapping file first + logging(comm, 'INFO', 'Get container from bufr') + container = bufr.Parser(input_path, mapping_path).parse(comm) + + logging(comm, 'DEBUG', f'container list (original): {container.list()}') + #logging(comm, 'DEBUG', f'all_sub_categories = {container.all_sub_categories()}') + #logging(comm, 'DEBUG', f'category map = {container.get_category_map()}') + + # Add new/derived data into container + #for cat in container.all_sub_categories(): + + # logging(comm, 'DEBUG', f'category = {cat}') + + # satid = container.get('variables/obsTimeMinusCycleTime', cat) + # if satid.size == 0: + # logging(comm, 'WARNING', f'category {cat[0]} does not exist in input file') + #paths = container.get_paths('variables/windComputationMethod', cat) + #obstype = container.get('variables/windComputationMethod', cat) + #container.add('variables/obstype_uwind', obstype, paths, cat) + #container.add('variables/obstype_vwind', obstype, paths, cat) + + #paths = container.get_paths('variables/windSpeed', cat) + #wob = container.get('variables/windSpeed', cat) + #container.add('variables/windEastward', wob, paths, cat) + #container.add('variables/windNorthward', wob, paths, cat) + + # else: + print(" Do DateTime calculation") + otmct = container.get('variables/obsTimeMinusCycleTime') + otmct_paths = container.get_paths('variables/obsTimeMinusCycleTime') + otmct2 = np.array(otmct) + cycleTimeSinceEpoch = np.int64(calendar.timegm(time.strptime(str(int(CYCLE_TIME)), '%Y%m%d%H'))) + dateTime = Compute_dateTime(cycleTimeSinceEpoch, otmct2) + logging(comm, 'DEBUG', f'dateTime min/max = {dateTime.min()} {dateTime.max()}') + + print("Make an array of 0s for MetaData/sequenceNumber") + sequenceNum = np.zeros(otmct.shape, dtype=np.int32) + logging(comm, 'DEBUG', f' sequenceNummin/max = {sequenceNum.min()} {sequenceNum.max()}') + + print(" Add variables to container.") + container.add('variables/dateTime', dateTime, otmct_paths) + container.add('variables/sequenceNumber', sequenceNum, otmct_paths) + #description = bufr.encoders.Description(YAML_PATH) + + #print(" Add container and descriptions to dataset.") + #dataset = next(iter(Encoder(description).encode(container).values())) + + + # Add new variables: ObsType/windEastward & ObsType/windNorthward +# swcm = container.get('variables/windComputationMethod', cat) +# chanfreq = container.get('variables/sensorCentralFrequency', cat) +# +# logging(comm, 'DEBUG', f'swcm min/max = {swcm.min()} {swcm.max()}') +# logging(comm, 'DEBUG', f'chanfreq min/max = {chanfreq.min()} {chanfreq.max()}') +# +# obstype = _get_obs_type(swcm, chanfreq) +# +# logging(comm, 'DEBUG', f'obstype = {obstype}') +# logging(comm, 'DEBUG', f'obstype min/max = {obstype.min()} {obstype.max()}') +# +# paths = container.get_paths('variables/windComputationMethod', cat) +# container.add('variables/obstype_uwind', obstype, paths, cat) +# container.add('variables/obstype_vwind', obstype, paths, cat) +# +# # Add new variables: ObsValue/windEastward & ObsValue/windNorthward +# wdir = container.get('variables/windDirection', cat) +# wspd = container.get('variables/windSpeed', cat) +# +# logging(comm, 'DEBUG', f'wdir min/max = {wdir.min()} {wdir.max()}') +# logging(comm, 'DEBUG', f'wspd min/max = {wspd.min()} {wspd.max()}') +# +# uob, vob = compute_wind_components(wdir, wspd) +# +# logging(comm, 'DEBUG', f'uob min/max = {uob.min()} {uob.max()}') +# logging(comm, 'DEBUG', f'vob min/max = {vob.min()} {vob.max()}') +# +# paths = container.get_paths('variables/windSpeed', cat) +# container.add('variables/windEastward', uob, paths, cat) +# container.add('variables/windNorthward', vob, paths, cat) + + # Check + logging(comm, 'DEBUG', f'container list (updated): {container.list()}') + #logging(comm, 'DEBUG', f'all_sub_categories {container.all_sub_categories()}') + + return container + def create_obs_group(cycle_time,input_mapping,input_path): CYCLE_TIME = cycle_time YAML_PATH = input_mapping #"./iodatest_prepbufr_adpsfc_mapping.yaml" @@ -54,3 +262,89 @@ def create_obs_group(cycle_time,input_mapping,input_path): dataset = next(iter(Encoder(description).encode(container).values())) return dataset +#def create_obs_group(input_path, mapping_path, category, env): +def create_obs_group(input_path, mapping_path, env): + + comm = bufr.mpi.Comm(env["comm_name"]) + + description = _make_description(mapping_path, update=True) + + # Check the cache for the data and return it if it exists + logging(comm, 'DEBUG', f'Check if bufr.DataCache exists? {bufr.DataCache.has(input_path, mapping_path)}') + if bufr.DataCache.has(input_path, mapping_path): + container = bufr.DataCache.get(input_path, mapping_path) + #logging(comm, 'INFO', f'Encode {category} from cache') + logging(comm, 'INFO', f'Encode from cache') + #data = iodaEncoder(description).encode(container)[(category,)] + data = iodaEncoder(description).encode(container) + #logging(comm, 'INFO', f'Mark {category} as finished in the cache') + logging(comm, 'INFO', f'Mark as finished in the cache') + #bufr.DataCache.mark_finished(input_path, mapping_path, [category]) + bufr.DataCache.mark_finished(input_path, mapping_path) + #logging(comm, 'INFO', f'Return the encoded data for {category}') + logging(comm, 'INFO', f'Return the encoded data.') + return data + + container = _make_obs(comm, input_path, mapping_path) + + # Gather data from all tasks into all tasks. Each task will have the complete record + logging(comm, 'INFO', f'Gather data from all tasks into all tasks') + container.all_gather(comm) + + logging(comm, 'INFO', f'Add container to cache') + # Add the container to the cache + #bufr.DataCache.add(input_path, mapping_path, container.all_sub_categories(), container) + bufr.DataCache.add(input_path, mapping_path, container) + + # Encode the data + logging(comm, 'INFO', f'Encode {category}') + #data = iodaEncoder(description).encode(container)[(category,)] + data = iodaEncoder(description).encode(container) + + logging(comm, 'INFO', f'Mark {category} as finished in the cache') + # Mark the data as finished in the cache + #bufr.DataCache.mark_finished(input_path, mapping_path, [category]) + bufr.DataCache.mark_finished(input_path, mapping_path) + + logging(comm, 'INFO', f'Return the encoded data.') + return data + + +def create_obs_file(input_path, mapping_path, output_path): + + comm = bufr.mpi.Comm("world") + container = _make_obs(comm, input_path, mapping_path) + container.gather(comm) + + description = _make_description(mapping_path, update=True) + + # Encode the data + if comm.rank() == 0: + netcdfEncoder(description).encode(container, output_path) + + logging(comm, 'INFO', f'Return the encoded data') + + +if __name__ == '__main__': + + start_time = time.time() + + bufr.mpi.App(sys.argv) + comm = bufr.mpi.Comm("world") + + # Required input arguments + parser = argparse.ArgumentParser() + parser.add_argument('-i', '--input', type=str, help='Input BUFR', required=True) + parser.add_argument('-m', '--mapping', type=str, help='BUFR2IODA Mapping File', required=True) + parser.add_argument('-o', '--output', type=str, help='Output NetCDF', required=True) + + args = parser.parse_args() + mapping = args.mapping + infile = args.input + output = args.output + + create_obs_file(infile, mapping, output) + + end_time = time.time() + running_time = end_time - start_time + logging(comm, 'INFO', f'Total running time: {running_time}') From eba41c2f822d307bd4bca27ca409b7871254c506 Mon Sep 17 00:00:00 2001 From: Nicholas Esposito Date: Thu, 5 Dec 2024 15:11:45 +0000 Subject: [PATCH 49/72] name changes --- ...apping.yaml => bufr2ioda_adpsfc_prepbufr_mapping.yaml} | 0 ...c.yaml => bufr2ioda_bufr_backend_adpsfc_prepbufr.yaml} | 2 +- ...yaml => bufr2ioda_script_backend_adpsfc_prepbufr.yaml} | 2 +- ush/test/encodeBufr.sh | 8 ++++---- 4 files changed, 6 insertions(+), 6 deletions(-) rename ush/test/{bufr2ioda_prepbufr_adpsfc_mapping.yaml => bufr2ioda_adpsfc_prepbufr_mapping.yaml} (100%) rename ush/test/{bufr2ioda_bufr_backend_prepbufr_adpsfc.yaml => bufr2ioda_bufr_backend_adpsfc_prepbufr.yaml} (88%) rename ush/test/{bufr2ioda_script_backend_prepbufr_adpsfc.yaml => bufr2ioda_script_backend_adpsfc_prepbufr.yaml} (90%) diff --git a/ush/test/bufr2ioda_prepbufr_adpsfc_mapping.yaml b/ush/test/bufr2ioda_adpsfc_prepbufr_mapping.yaml similarity index 100% rename from ush/test/bufr2ioda_prepbufr_adpsfc_mapping.yaml rename to ush/test/bufr2ioda_adpsfc_prepbufr_mapping.yaml diff --git a/ush/test/bufr2ioda_bufr_backend_prepbufr_adpsfc.yaml b/ush/test/bufr2ioda_bufr_backend_adpsfc_prepbufr.yaml similarity index 88% rename from ush/test/bufr2ioda_bufr_backend_prepbufr_adpsfc.yaml rename to ush/test/bufr2ioda_bufr_backend_adpsfc_prepbufr.yaml index ef78f26..0655875 100644 --- a/ush/test/bufr2ioda_bufr_backend_prepbufr_adpsfc.yaml +++ b/ush/test/bufr2ioda_bufr_backend_adpsfc_prepbufr.yaml @@ -11,7 +11,7 @@ observations: engine: type: bufr obsfile: "./testinput/2021080100/gdas.t00z.adpsfc.prepbufr" - mapping file: "./bufr2ioda_prepbufr_adpsfc_mapping.yaml" + mapping file: "./bufr2ioda_adpsfc_prepbufr_mapping.yaml" # category: ["goes-16"] obsdataout: engine: diff --git a/ush/test/bufr2ioda_script_backend_prepbufr_adpsfc.yaml b/ush/test/bufr2ioda_script_backend_adpsfc_prepbufr.yaml similarity index 90% rename from ush/test/bufr2ioda_script_backend_prepbufr_adpsfc.yaml rename to ush/test/bufr2ioda_script_backend_adpsfc_prepbufr.yaml index 2b8e097..42df1e0 100644 --- a/ush/test/bufr2ioda_script_backend_prepbufr_adpsfc.yaml +++ b/ush/test/bufr2ioda_script_backend_adpsfc_prepbufr.yaml @@ -15,7 +15,7 @@ observations: script file: "bufr2ioda_adpsfc_prepbufr.py" args: input_path: "/scratch1/NCEPDEV/da/Nicholas.Esposito/JEDI/obsForge_sfcshp_20241126/sorc/spoc/ush/test/testinput/2021080100/gdas.t00z.adpsfc.prepbufr" - mapping_path: "/scratch1/NCEPDEV/da/Nicholas.Esposito/JEDI/obsForge_sfcshp_20241126/sorc/spoc/ush/test/bufr2ioda_prepbufr_adpsfc_mapping.yaml" + mapping_path: "/scratch1/NCEPDEV/da/Nicholas.Esposito/JEDI/obsForge_sfcshp_20241126/sorc/spoc/ush/test/bufr2ioda_adpsfc_prepbufr_mapping.yaml" #category: "goes-16" obsdataout: engine: diff --git a/ush/test/encodeBufr.sh b/ush/test/encodeBufr.sh index 7fa01e4..327c550 100755 --- a/ush/test/encodeBufr.sh +++ b/ush/test/encodeBufr.sh @@ -152,8 +152,8 @@ mkdir -p -m770 ${out_dir} || { echo "Error creating output directory: ${out_dir} # =============== # Set file paths # =============== -ioda_config_yaml="${work_dir}/bufr2ioda_${mode}_prepbufr_${obstype}.yaml" -mapping_file="${work_dir}/bufr2ioda_prepbufr_${obstype}_mapping.yaml" +ioda_config_yaml="${work_dir}/bufr2ioda_${mode}_${obstype}.yaml" +mapping_file="${work_dir}/bufr2ioda_${obstype}_mapping.yaml" #input_file="${in_dir}/gdas.t${h2}z.${bufrtype}.tm00.bufr_d" input_file="${in_dir}/gdas.t${h2}z.${bufrtype}.prepbufr" if [[ "${split_by_category}" = "true" ]]; then @@ -204,10 +204,10 @@ elif [[ "$mode" == "bufr2netcdf" ]]; then elif [[ "$mode" == "script2netcdf" ]]; then if [[ "$nproc" == "0" ]]; then echo Run script2netcdf without MPI ... - python bufr2ioda_${obstype}_prepbufr.py -m "$mapping_file" -o "$output_file" -i "$input_file" || { echo "Error: Python script2netcdf failed"; exit 1; } + python bufr2ioda_${obstype}.py -m "$mapping_file" -o "$output_file" -i "$input_file" || { echo "Error: Python script2netcdf failed"; exit 1; } else echo Run script2netcdf with MPI ${nproc} ... - srun -n "$nproc" --mem 96G --time 00:30:00 python bufr2ioda_${obstype}_prepbufr.py -m "$mapping_file" -o "$output_file" -i "$input_file" || { echo "Error: MPI Python script2netcdf failed"; exit 1; } + srun -n "$nproc" --mem 96G --time 00:30:00 python bufr2ioda_${obstype}.py -m "$mapping_file" -o "$output_file" -i "$input_file" || { echo "Error: MPI Python script2netcdf failed"; exit 1; } fi else echo Incorrect running mode ${mode} ... Valid modes are: bufr_backend, script_back, bufr2netcdf, or script2netcdf From 4f92d5c544d25b546863c8b4804a114ca9b4fcb2 Mon Sep 17 00:00:00 2001 From: Nicholas Esposito Date: Tue, 10 Dec 2024 18:17:12 +0000 Subject: [PATCH 50/72] all works now --- ush/test/bufr_bufr_backend_adpsfc_prepbufr.yaml | 2 +- ush/test/bufr_script_backend_adpsfc_prepbufr.yaml | 4 ++-- ush/test/encodeBufr_Nick.sh | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/ush/test/bufr_bufr_backend_adpsfc_prepbufr.yaml b/ush/test/bufr_bufr_backend_adpsfc_prepbufr.yaml index 0655875..1a143d5 100644 --- a/ush/test/bufr_bufr_backend_adpsfc_prepbufr.yaml +++ b/ush/test/bufr_bufr_backend_adpsfc_prepbufr.yaml @@ -11,7 +11,7 @@ observations: engine: type: bufr obsfile: "./testinput/2021080100/gdas.t00z.adpsfc.prepbufr" - mapping file: "./bufr2ioda_adpsfc_prepbufr_mapping.yaml" + mapping file: "./bufr_adpsfc_prepbufr_mapping.yaml" # category: ["goes-16"] obsdataout: engine: diff --git a/ush/test/bufr_script_backend_adpsfc_prepbufr.yaml b/ush/test/bufr_script_backend_adpsfc_prepbufr.yaml index 42df1e0..9e048e1 100644 --- a/ush/test/bufr_script_backend_adpsfc_prepbufr.yaml +++ b/ush/test/bufr_script_backend_adpsfc_prepbufr.yaml @@ -12,10 +12,10 @@ observations: obsdatain: engine: type: script - script file: "bufr2ioda_adpsfc_prepbufr.py" + script file: "bufr_adpsfc_prepbufr.py" args: input_path: "/scratch1/NCEPDEV/da/Nicholas.Esposito/JEDI/obsForge_sfcshp_20241126/sorc/spoc/ush/test/testinput/2021080100/gdas.t00z.adpsfc.prepbufr" - mapping_path: "/scratch1/NCEPDEV/da/Nicholas.Esposito/JEDI/obsForge_sfcshp_20241126/sorc/spoc/ush/test/bufr2ioda_adpsfc_prepbufr_mapping.yaml" + mapping_path: "/scratch1/NCEPDEV/da/Nicholas.Esposito/JEDI/obsForge_sfcshp_20241126/sorc/spoc/ush/test/bufr_adpsfc_prepbufr_mapping.yaml" #category: "goes-16" obsdataout: engine: diff --git a/ush/test/encodeBufr_Nick.sh b/ush/test/encodeBufr_Nick.sh index 0d97e0b..a4d1a1d 100755 --- a/ush/test/encodeBufr_Nick.sh +++ b/ush/test/encodeBufr_Nick.sh @@ -152,8 +152,8 @@ mkdir -p -m770 ${out_dir} || { echo "Error creating output directory: ${out_dir} # =============== # Set file paths # =============== -ioda_config_yaml="${work_dir}/bufr2ioda_${mode}_${obstype}.yaml" -mapping_file="${work_dir}/bufr2ioda_${obstype}_mapping.yaml" +ioda_config_yaml="${work_dir}/bufr_${mode}_${obstype}.yaml" +mapping_file="${work_dir}/bufr_${obstype}_mapping.yaml" #input_file="${in_dir}/gdas.t${h2}z.${bufrtype}.tm00.bufr_d" input_file="${in_dir}/gdas.t${h2}z.${bufrtype}.prepbufr" if [[ "${split_by_category}" = "true" ]]; then @@ -204,10 +204,10 @@ elif [[ "$mode" == "bufr2netcdf" ]]; then elif [[ "$mode" == "script2netcdf" ]]; then if [[ "$nproc" == "0" ]]; then echo Run script2netcdf without MPI ... - python bufr2ioda_${obstype}.py -m "$mapping_file" -o "$output_file" -i "$input_file" || { echo "Error: Python script2netcdf failed"; exit 1; } + python bufr_${obstype}.py -m "$mapping_file" -o "$output_file" -i "$input_file" || { echo "Error: Python script2netcdf failed"; exit 1; } else echo Run script2netcdf with MPI ${nproc} ... - srun -n "$nproc" --mem 96G --time 00:30:00 python bufr2ioda_${obstype}.py -m "$mapping_file" -o "$output_file" -i "$input_file" || { echo "Error: MPI Python script2netcdf failed"; exit 1; } + srun -n "$nproc" --mem 96G --time 00:30:00 python bufr_${obstype}.py -m "$mapping_file" -o "$output_file" -i "$input_file" || { echo "Error: MPI Python script2netcdf failed"; exit 1; } fi else echo Incorrect running mode ${mode} ... Valid modes are: bufr_backend, script_back, bufr2netcdf, or script2netcdf From 282e177f12116be5bea57d4d8a24bf9ac2fde2c9 Mon Sep 17 00:00:00 2001 From: Nicholas Esposito Date: Tue, 10 Dec 2024 18:24:45 +0000 Subject: [PATCH 51/72] small changes for script --- ush/test/bufr_adpsfc_prepbufr.py | 40 ++++++-------------------------- ush/test/encodeBufr_Nick.sh | 6 +++-- 2 files changed, 11 insertions(+), 35 deletions(-) diff --git a/ush/test/bufr_adpsfc_prepbufr.py b/ush/test/bufr_adpsfc_prepbufr.py index eb706ef..d69f954 100755 --- a/ush/test/bufr_adpsfc_prepbufr.py +++ b/ush/test/bufr_adpsfc_prepbufr.py @@ -104,33 +104,6 @@ def _make_description(mapping_path, update=False): return description -#def Compute_dateTime(cycleTimeSinceEpoch, dhr): -# """ -# Compute dateTime using the cycleTimeSinceEpoch and Cycle Time -# minus Cycle Time -# -# Parameters: -# cycleTimeSinceEpoch: Time of cycle in Epoch Time -# dhr: Observation Time Minus Cycle Time -# -# Returns: -# Masked array of dateTime values -# """ -# -# int64_fill_value = np.int64(0) -# -# dateTime = np.zeros(dhr.shape, dtype=np.int64) -# for i in range(len(dateTime)): -# if ma.is_masked(dhr[i]): -# continue -# else: -# dateTime[i] = np.int64(dhr[i]*3600) + cycleTimeSinceEpoch -# -# dateTime = ma.array(dateTime) -# dateTime = ma.masked_values(dateTime, int64_fill_value) -# -# return dateTime - def _make_obs(comm, input_path, mapping_path): """ Create the ioda adpsfc prepbufr observations: @@ -214,17 +187,18 @@ def create_obs_file(input_path, mapping_path, output_path): bufr.mpi.App(sys.argv) comm = bufr.mpi.Comm("world") - # Required input arguments - parser = argparse.ArgumentParser() - parser.add_argument('-i', '--input', type=str, help='Input BUFR', required=True) - parser.add_argument('-m', '--mapping', type=str, help='BUFR2IODA Mapping File', required=True) - parser.add_argument('-o', '--output', type=str, help='Output NetCDF', required=True) + # Required input arguments as positional arguments + parser = argparse.ArgumentParser(description="Convert BUFR to NetCDF using a mapping file.") + parser.add_argument('input', type=str, help='Input BUFR file') + parser.add_argument('mapping', type=str, help='BUFR2IODA Mapping File') + parser.add_argument('output', type=str, help='Output NetCDF file') args = parser.parse_args() - mapping = args.mapping infile = args.input + mapping = args.mapping output = args.output + create_obs_file(infile, mapping, output) end_time = time.time() diff --git a/ush/test/encodeBufr_Nick.sh b/ush/test/encodeBufr_Nick.sh index a4d1a1d..3ea26b7 100755 --- a/ush/test/encodeBufr_Nick.sh +++ b/ush/test/encodeBufr_Nick.sh @@ -204,10 +204,12 @@ elif [[ "$mode" == "bufr2netcdf" ]]; then elif [[ "$mode" == "script2netcdf" ]]; then if [[ "$nproc" == "0" ]]; then echo Run script2netcdf without MPI ... - python bufr_${obstype}.py -m "$mapping_file" -o "$output_file" -i "$input_file" || { echo "Error: Python script2netcdf failed"; exit 1; } + #python bufr_${obstype}.py -m "$mapping_file" -o "$output_file" -i "$input_file" || { echo "Error: Python script2netcdf failed"; exit 1; } + python bufr_${obstype}.py "$input_file" "$mapping_file" "$output_file" || { echo "Error: Python script2netcdf failed"; python bufr_${obstype}.py --help; exit 1; } else echo Run script2netcdf with MPI ${nproc} ... - srun -n "$nproc" --mem 96G --time 00:30:00 python bufr_${obstype}.py -m "$mapping_file" -o "$output_file" -i "$input_file" || { echo "Error: MPI Python script2netcdf failed"; exit 1; } + #srun -n "$nproc" --mem 96G --time 00:30:00 python bufr_${obstype}.py -m "$mapping_file" -o "$output_file" -i "$input_file" || { echo "Error: MPI Python script2netcdf failed"; exit 1; } + srun -n "$nproc" --mem 96G --time 00:30:00 python bufr_${obstype}.py "$input_file" "$mapping_file" "$output_file" || { echo "Error: MPI Python script2netcdf failed"; python bufr_${obstype}.py --help; exit 1; } fi else echo Incorrect running mode ${mode} ... Valid modes are: bufr_backend, script_back, bufr2netcdf, or script2netcdf From 16d858640bc79b6d950b9f581519a3bf29b619a3 Mon Sep 17 00:00:00 2001 From: Nicholas Esposito Date: Thu, 12 Dec 2024 20:22:02 +0000 Subject: [PATCH 52/72] update encode_Nick.sh from sfcshp --- ush/test/encodeBufr_Nick.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ush/test/encodeBufr_Nick.sh b/ush/test/encodeBufr_Nick.sh index 3ea26b7..d258e66 100755 --- a/ush/test/encodeBufr_Nick.sh +++ b/ush/test/encodeBufr_Nick.sh @@ -140,7 +140,7 @@ y4="${cycle:0:4}" m2="${cycle:4:2}" d2="${cycle:6:2}" h2="${cycle:8:2}" - +cycle_time="${cycle}" # ==================== # Set directory paths # ==================== @@ -193,6 +193,7 @@ if [[ "$mode" == "bufr_backend" || "$mode" == "script_backend" ]]; then elif [[ "$mode" == "bufr2netcdf" ]]; then if [[ "$nproc" == "0" ]]; then echo Run bufr2netcdf without MPI ... + echo NICK ${obsforge_dir}/build/bin/bufr2netcdf.x "$input_file" "${mapping_file}" "$output_file" ${obsforge_dir}/build/bin/bufr2netcdf.x "$input_file" "${mapping_file}" "$output_file" || { echo "Error: bufr2netcdf.x failed"; exit 1; } else echo Run bufr2netcdf with MPI ${nproc} ... @@ -205,11 +206,11 @@ elif [[ "$mode" == "script2netcdf" ]]; then if [[ "$nproc" == "0" ]]; then echo Run script2netcdf without MPI ... #python bufr_${obstype}.py -m "$mapping_file" -o "$output_file" -i "$input_file" || { echo "Error: Python script2netcdf failed"; exit 1; } - python bufr_${obstype}.py "$input_file" "$mapping_file" "$output_file" || { echo "Error: Python script2netcdf failed"; python bufr_${obstype}.py --help; exit 1; } + python bufr_${obstype}.py "$input_file" "$mapping_file" "$output_file" "$cycle_time" || { echo "Error: Python script2netcdf failed"; python bufr_${obstype}.py --help; exit 1; } else echo Run script2netcdf with MPI ${nproc} ... #srun -n "$nproc" --mem 96G --time 00:30:00 python bufr_${obstype}.py -m "$mapping_file" -o "$output_file" -i "$input_file" || { echo "Error: MPI Python script2netcdf failed"; exit 1; } - srun -n "$nproc" --mem 96G --time 00:30:00 python bufr_${obstype}.py "$input_file" "$mapping_file" "$output_file" || { echo "Error: MPI Python script2netcdf failed"; python bufr_${obstype}.py --help; exit 1; } + srun -n "$nproc" --mem 96G --time 00:30:00 python bufr_${obstype}.py "$input_file" "$mapping_file" "$output_file" "$cycle_time" || { echo "Error: MPI Python script2netcdf failed"; python bufr_${obstype}.py --help; exit 1; } fi else echo Incorrect running mode ${mode} ... Valid modes are: bufr_backend, script_back, bufr2netcdf, or script2netcdf From a907f3de47fff99322d31a13469817cfac0cbea9 Mon Sep 17 00:00:00 2001 From: Nicholas Esposito Date: Thu, 12 Dec 2024 20:45:04 +0000 Subject: [PATCH 53/72] add datetime, cycle_time --- ush/test/bufr_adpsfc_prepbufr.py | 54 ++++++++++++++++--- .../bufr_script_backend_adpsfc_prepbufr.yaml | 2 +- 2 files changed, 48 insertions(+), 8 deletions(-) diff --git a/ush/test/bufr_adpsfc_prepbufr.py b/ush/test/bufr_adpsfc_prepbufr.py index d69f954..0f99108 100755 --- a/ush/test/bufr_adpsfc_prepbufr.py +++ b/ush/test/bufr_adpsfc_prepbufr.py @@ -19,6 +19,34 @@ log_level = os.getenv('LOG_LEVEL', 'INFO') logger = Logger('BUFR2IODA_adpsfc_prepbufr.py', level=log_level, colored_log=False) +def Compute_dateTime(cycleTimeSinceEpoch, dhr): + """ + Compute dateTime using the cycleTimeSinceEpoch and Cycle Time + minus Cycle Time + + Parameters: + cycleTimeSinceEpoch: Time of cycle in Epoch Time + dhr: Observation Time Minus Cycle Time + + Returns: + Masked array of dateTime values + """ + + int64_fill_value = np.int64(0) + + dateTime = np.zeros(dhr.shape, dtype=np.int64) + for i in range(len(dateTime)): + if ma.is_masked(dhr[i]): + continue + else: + dateTime[i] = np.int64(dhr[i]*3600) + cycleTimeSinceEpoch + + dateTime = ma.array(dateTime) + dateTime = ma.masked_values(dateTime, int64_fill_value) + + return dateTime + + def logging(comm, level, message): """ Logs a message to the console or log file, based on the specified logging level. @@ -104,7 +132,7 @@ def _make_description(mapping_path, update=False): return description -def _make_obs(comm, input_path, mapping_path): +def _make_obs(comm, input_path, mapping_path, cycle_time): """ Create the ioda adpsfc prepbufr observations: - reads values @@ -118,6 +146,8 @@ def _make_obs(comm, input_path, mapping_path): The input bufr file mapping_path: str The input bufr2ioda mapping file + cycle_time: str + The cycle in YYYYMMDDHH format """ # Get container from mapping file first @@ -131,12 +161,21 @@ def _make_obs(comm, input_path, mapping_path): lon[lon>180] -= 360 lon = ma.round(lon, decimals=2) + logging(comm, 'DEBUG', f'Do DateTime calculation') + otmct = container.get('variables/obsTimeMinusCycleTime') + otmct_paths = container.get_paths('variables/obsTimeMinusCycleTime') + otmct2 = np.array(otmct) + cycleTimeSinceEpoch = np.int64(calendar.timegm(time.strptime(str(int(cycle_time)), '%Y%m%d%H'))) + dateTime = Compute_dateTime(cycleTimeSinceEpoch, otmct2) + logging(comm, 'DEBUG', f'dateTime min/max = {dateTime.min()} {dateTime.max()}') + print("Make an array of 0s for MetaData/sequenceNumber") sequenceNum = np.zeros(lon.shape, dtype=np.int32) logging(comm, 'DEBUG', f' sequenceNummin/max = {sequenceNum.min()} {sequenceNum.max()}') logging(comm, 'DEBUG', f'Update variables in container') container.replace('variables/longitude', lon) + container.replace('variables/timestamp', dateTime) logging(comm, 'DEBUG', f'Add variables to container') container.add('variables/sequenceNumber', sequenceNum, lon_paths) @@ -147,13 +186,13 @@ def _make_obs(comm, input_path, mapping_path): return container -def create_obs_group(input_path, mapping_path, env): +def create_obs_group(input_path, mapping_path, cycle_time, env): comm = bufr.mpi.Comm(env["comm_name"]) logging(comm, 'INFO', f'Make description and make obs') description = _make_description(mapping_path, update=True) - container = _make_obs(comm, input_path, mapping_path) + container = _make_obs(comm, input_path, mapping_path, cycle_time) # Gather data from all tasks into all tasks. Each task will have the complete record logging(comm, 'INFO', f'Gather data from all tasks into all tasks') @@ -166,10 +205,10 @@ def create_obs_group(input_path, mapping_path, env): return data -def create_obs_file(input_path, mapping_path, output_path): +def create_obs_file(input_path, mapping_path, output_path, cycle_time): comm = bufr.mpi.Comm("world") - container = _make_obs(comm, input_path, mapping_path) + container = _make_obs(comm, input_path, mapping_path, cycle_time) container.gather(comm) description = _make_description(mapping_path, update=True) @@ -192,14 +231,15 @@ def create_obs_file(input_path, mapping_path, output_path): parser.add_argument('input', type=str, help='Input BUFR file') parser.add_argument('mapping', type=str, help='BUFR2IODA Mapping File') parser.add_argument('output', type=str, help='Output NetCDF file') + parser.add_argument('cycle_time', type=str, help='cycle time in YYYYMMDDHH format') args = parser.parse_args() infile = args.input mapping = args.mapping output = args.output + cycle_time = args.cycle_time - - create_obs_file(infile, mapping, output) + create_obs_file(infile, mapping, output, cycle_time) end_time = time.time() running_time = end_time - start_time diff --git a/ush/test/bufr_script_backend_adpsfc_prepbufr.yaml b/ush/test/bufr_script_backend_adpsfc_prepbufr.yaml index 9e048e1..ac764f2 100644 --- a/ush/test/bufr_script_backend_adpsfc_prepbufr.yaml +++ b/ush/test/bufr_script_backend_adpsfc_prepbufr.yaml @@ -16,7 +16,7 @@ observations: args: input_path: "/scratch1/NCEPDEV/da/Nicholas.Esposito/JEDI/obsForge_sfcshp_20241126/sorc/spoc/ush/test/testinput/2021080100/gdas.t00z.adpsfc.prepbufr" mapping_path: "/scratch1/NCEPDEV/da/Nicholas.Esposito/JEDI/obsForge_sfcshp_20241126/sorc/spoc/ush/test/bufr_adpsfc_prepbufr_mapping.yaml" - #category: "goes-16" + cycle_time: "2021080100" obsdataout: engine: type: H5File From 3770d7e55e710cab91b9662906caf594736e8ecf Mon Sep 17 00:00:00 2001 From: Nicholas Esposito Date: Fri, 20 Dec 2024 11:18:42 -0500 Subject: [PATCH 54/72] remove longitude range for now, encodeBufr_Nick python 3.10/3.7 --- ush/test/bufr_adpsfc_prepbufr_mapping.yaml | 2 +- ush/test/encodeBufr_Nick.sh | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/ush/test/bufr_adpsfc_prepbufr_mapping.yaml b/ush/test/bufr_adpsfc_prepbufr_mapping.yaml index 6f97459..dd99089 100755 --- a/ush/test/bufr_adpsfc_prepbufr_mapping.yaml +++ b/ush/test/bufr_adpsfc_prepbufr_mapping.yaml @@ -126,7 +126,7 @@ encoder: source: variables/longitude longName: "Longitude" units: "degree_east" - range: [0, 360] + #range: [0, 360] # - name: "MetaData/sequenceNumber" #source: variables/sequenceNumber diff --git a/ush/test/encodeBufr_Nick.sh b/ush/test/encodeBufr_Nick.sh index d258e66..df2eb61 100755 --- a/ush/test/encodeBufr_Nick.sh +++ b/ush/test/encodeBufr_Nick.sh @@ -106,19 +106,21 @@ export LOG_LEVEL=DEBUG # Load obsForge required modules # =============================== module use ${obsforge_dir}/modulefiles -module load obsforge/hera.intel || { echo "Error loading obsforge module"; exit 1; } +module load obsforge/hercules.intel || { echo "Error loading obsforge module"; exit 1; } module list # ============================== # Set bufr-query python library # ============================== export LD_LIBRARY_PATH="${obsforge_dir}/build/lib:${LD_LIBRARY_PATH}" -export PYTHONPATH="${PYTHONPATH}:${obsforge_dir}/build/lib/python3.10/site-packages" +#export PYTHONPATH="${PYTHONPATH}:${obsforge_dir}/build/lib/python3.10/site-packages" +export PYTHONPATH="${PYTHONPATH}:${obsforge_dir}/build/lib/python3.7/site-packages" # ======================== # Set ioda python library # ========================= -export PYTHONPATH="${PYTHONPATH}:${obsforge_dir}/build/lib/python3.10" +#export PYTHONPATH="${PYTHONPATH}:${obsforge_dir}/build/lib/python3.10" +export PYTHONPATH="${PYTHONPATH}:${obsforge_dir}/build/lib/python3.7" # ============ # Set wxfloww From 8b52af41ceeca8d37a795f0a0ce70cc2c579aa93 Mon Sep 17 00:00:00 2001 From: Nicholas Esposito Date: Thu, 26 Dec 2024 11:12:49 -0500 Subject: [PATCH 55/72] filename changes, mpi additions --- ush/test/bufr_adpsfc_prepbufr.py | 2 +- .../bufr_bufr_backend_adpsfc_prepbufr.yaml | 4 ++-- ...ufr_bufr_backend_adpsfc_prepbufr_mpi0.yaml | 19 +++++++++++++++ ...ufr_bufr_backend_adpsfc_prepbufr_mpi4.yaml | 19 +++++++++++++++ .../bufr_script_backend_adpsfc_prepbufr.yaml | 6 ++--- ...r_script_backend_adpsfc_prepbufr_mpi0.yaml | 23 +++++++++++++++++++ ...r_script_backend_adpsfc_prepbufr_mpi4.yaml | 23 +++++++++++++++++++ ush/test/encodeBufr_Nick.sh | 6 ++--- 8 files changed, 93 insertions(+), 9 deletions(-) create mode 100644 ush/test/bufr_bufr_backend_adpsfc_prepbufr_mpi0.yaml create mode 100644 ush/test/bufr_bufr_backend_adpsfc_prepbufr_mpi4.yaml create mode 100644 ush/test/bufr_script_backend_adpsfc_prepbufr_mpi0.yaml create mode 100644 ush/test/bufr_script_backend_adpsfc_prepbufr_mpi4.yaml diff --git a/ush/test/bufr_adpsfc_prepbufr.py b/ush/test/bufr_adpsfc_prepbufr.py index 0f99108..1d797e9 100755 --- a/ush/test/bufr_adpsfc_prepbufr.py +++ b/ush/test/bufr_adpsfc_prepbufr.py @@ -169,7 +169,7 @@ def _make_obs(comm, input_path, mapping_path, cycle_time): dateTime = Compute_dateTime(cycleTimeSinceEpoch, otmct2) logging(comm, 'DEBUG', f'dateTime min/max = {dateTime.min()} {dateTime.max()}') - print("Make an array of 0s for MetaData/sequenceNumber") + logging(comm, 'DEBUG', f'Make an array of 0s for MetaData/sequenceNumber') sequenceNum = np.zeros(lon.shape, dtype=np.int32) logging(comm, 'DEBUG', f' sequenceNummin/max = {sequenceNum.min()} {sequenceNum.max()}') diff --git a/ush/test/bufr_bufr_backend_adpsfc_prepbufr.yaml b/ush/test/bufr_bufr_backend_adpsfc_prepbufr.yaml index 1a143d5..84b640e 100644 --- a/ush/test/bufr_bufr_backend_adpsfc_prepbufr.yaml +++ b/ush/test/bufr_bufr_backend_adpsfc_prepbufr.yaml @@ -10,10 +10,10 @@ observations: obsdatain: engine: type: bufr - obsfile: "./testinput/2021080100/gdas.t00z.adpsfc.prepbufr" + obsfile: "./testinput/2021080100/gdas.t00z.adpsfc.tm00.prepbufr" mapping file: "./bufr_adpsfc_prepbufr_mapping.yaml" # category: ["goes-16"] obsdataout: engine: type: H5File - obsfile: "./testoutput/2021080100/bufr_backend/gdas.t00z.adpsfc.prepbufr.nc" + obsfile: "./testoutput/2021080100/bufr_backend/gdas.t00z.adpsfc_prepbufr.tm00.nc" diff --git a/ush/test/bufr_bufr_backend_adpsfc_prepbufr_mpi0.yaml b/ush/test/bufr_bufr_backend_adpsfc_prepbufr_mpi0.yaml new file mode 100644 index 0000000..3b390e3 --- /dev/null +++ b/ush/test/bufr_bufr_backend_adpsfc_prepbufr_mpi0.yaml @@ -0,0 +1,19 @@ +time window: + begin: "2021-07-31T21:00:00Z" + end: "2021-08-01T03:00:00Z" + bound to include: begin + +observations: +- obs space: + name: "ADPSFC" + simulated variables: ['stationElevation','stationPressure'] + obsdatain: + engine: + type: bufr + obsfile: "./testinput/2021080100/gdas.t00z.adpsfc.tm00.prepbufr" + mapping file: "./bufr_adpsfc_prepbufr_mapping.yaml" + # category: ["goes-16"] + obsdataout: + engine: + type: H5File + obsfile: "./testoutput/2021080100/bufr_backend/gdas.t00z.adpsfc_prepbufr.tm00_mpi0.nc" diff --git a/ush/test/bufr_bufr_backend_adpsfc_prepbufr_mpi4.yaml b/ush/test/bufr_bufr_backend_adpsfc_prepbufr_mpi4.yaml new file mode 100644 index 0000000..98be1ea --- /dev/null +++ b/ush/test/bufr_bufr_backend_adpsfc_prepbufr_mpi4.yaml @@ -0,0 +1,19 @@ +time window: + begin: "2021-07-31T21:00:00Z" + end: "2021-08-01T03:00:00Z" + bound to include: begin + +observations: +- obs space: + name: "ADPSFC" + simulated variables: ['stationElevation','stationPressure'] + obsdatain: + engine: + type: bufr + obsfile: "./testinput/2021080100/gdas.t00z.adpsfc.tm00.prepbufr" + mapping file: "./bufr_adpsfc_prepbufr_mapping.yaml" + # category: ["goes-16"] + obsdataout: + engine: + type: H5File + obsfile: "./testoutput/2021080100/bufr_backend/gdas.t00z.adpsfc_prepbufr.tm00_mpi4.nc" diff --git a/ush/test/bufr_script_backend_adpsfc_prepbufr.yaml b/ush/test/bufr_script_backend_adpsfc_prepbufr.yaml index ac764f2..3e318c1 100644 --- a/ush/test/bufr_script_backend_adpsfc_prepbufr.yaml +++ b/ush/test/bufr_script_backend_adpsfc_prepbufr.yaml @@ -14,10 +14,10 @@ observations: type: script script file: "bufr_adpsfc_prepbufr.py" args: - input_path: "/scratch1/NCEPDEV/da/Nicholas.Esposito/JEDI/obsForge_sfcshp_20241126/sorc/spoc/ush/test/testinput/2021080100/gdas.t00z.adpsfc.prepbufr" - mapping_path: "/scratch1/NCEPDEV/da/Nicholas.Esposito/JEDI/obsForge_sfcshp_20241126/sorc/spoc/ush/test/bufr_adpsfc_prepbufr_mapping.yaml" + input_path: "./testinput/2021080100/gdas.t00z.adpsfc.tm00.prepbufr" + mapping_path: "./bufr_adpsfc_prepbufr_mapping.yaml" cycle_time: "2021080100" obsdataout: engine: type: H5File - obsfile: "./testoutput/2021080100/script_backend/gdas.t00z.adpsfc.prepbufr.nc" + obsfile: "./testoutput/2021080100/script_backend/gdas.t00z.adpsfc_prepbufr.tm00.nc" diff --git a/ush/test/bufr_script_backend_adpsfc_prepbufr_mpi0.yaml b/ush/test/bufr_script_backend_adpsfc_prepbufr_mpi0.yaml new file mode 100644 index 0000000..bfd521f --- /dev/null +++ b/ush/test/bufr_script_backend_adpsfc_prepbufr_mpi0.yaml @@ -0,0 +1,23 @@ +time window: + begin: "2021-07-31T21:00:00Z" + end: "2021-08-01T03:00:00Z" + bound to include: begin + +observations: +- obs space: + name: "ADPSFC" + observed variables: ['stationElevation','stationPressure'] + derived variables: ['stationElevation','stationPressure'] + simulated variables: ['stationElevation','stationPressure'] + obsdatain: + engine: + type: script + script file: "bufr_adpsfc_prepbufr.py" + args: + input_path: "testinput/2021080100/gdas.t00z.adpsfc.tm00.prepbufr" + mapping_path: "./bufr_adpsfc_prepbufr_mapping.yaml" + cycle_time: "2021080100" + obsdataout: + engine: + type: H5File + obsfile: "./testoutput/2021080100/script_backend/gdas.t00z.adpsfc_prepbufr.tm00_mpi0.nc" diff --git a/ush/test/bufr_script_backend_adpsfc_prepbufr_mpi4.yaml b/ush/test/bufr_script_backend_adpsfc_prepbufr_mpi4.yaml new file mode 100644 index 0000000..19d6120 --- /dev/null +++ b/ush/test/bufr_script_backend_adpsfc_prepbufr_mpi4.yaml @@ -0,0 +1,23 @@ +time window: + begin: "2021-07-31T21:00:00Z" + end: "2021-08-01T03:00:00Z" + bound to include: begin + +observations: +- obs space: + name: "ADPSFC" + observed variables: ['stationElevation','stationPressure'] + derived variables: ['stationElevation','stationPressure'] + simulated variables: ['stationElevation','stationPressure'] + obsdatain: + engine: + type: script + script file: "bufr_adpsfc_prepbufr.py" + args: + input_path: "testinput/2021080100/gdas.t00z.adpsfc.tm00.prepbufr" + mapping_path: "./bufr_adpsfc_prepbufr_mapping.yaml" + cycle_time: "2021080100" + obsdataout: + engine: + type: H5File + obsfile: "./testoutput/2021080100/script_backend/gdas.t00z.adpsfc_prepbufr.tm00_mpi4.nc" diff --git a/ush/test/encodeBufr_Nick.sh b/ush/test/encodeBufr_Nick.sh index df2eb61..73e00e0 100755 --- a/ush/test/encodeBufr_Nick.sh +++ b/ush/test/encodeBufr_Nick.sh @@ -154,14 +154,14 @@ mkdir -p -m770 ${out_dir} || { echo "Error creating output directory: ${out_dir} # =============== # Set file paths # =============== -ioda_config_yaml="${work_dir}/bufr_${mode}_${obstype}.yaml" +ioda_config_yaml="${work_dir}/bufr_${mode}_${obstype}_mpi${nproc}.yaml" mapping_file="${work_dir}/bufr_${obstype}_mapping.yaml" #input_file="${in_dir}/gdas.t${h2}z.${bufrtype}.tm00.bufr_d" -input_file="${in_dir}/gdas.t${h2}z.${bufrtype}.prepbufr" +input_file="${in_dir}/gdas.t${h2}z.${bufrtype}.tm00.prepbufr" if [[ "${split_by_category}" = "true" ]]; then output_file="${out_dir}/gdas.t${h2}z.${bufrtype}_${sensor}_{splits/satId}.tm00.nc" else - output_file="${out_dir}/gdas.t${h2}z.${bufrtype}.tm00.nc" + output_file="${out_dir}/gdas.t${h2}z.${bufrtype}_prepbufr.tm00_mpi${nproc}.nc" fi if [[ ! -f "$input_file" ]]; then From 3cc6f791f23dd243aeb75bf32be4786d3df2797e Mon Sep 17 00:00:00 2001 From: Nicholas Esposito Date: Thu, 26 Dec 2024 13:12:33 -0500 Subject: [PATCH 56/72] bufr_backend script_backend -> bufr4backend, script4backend --- ...dpsfc_prepbufr.yaml => bufr_bufr4backend_adpsfc_prepbufr.yaml} | 0 ...bufr_mpi0.yaml => bufr_bufr4backend_adpsfc_prepbufr_mpi0.yaml} | 0 ...bufr_mpi4.yaml => bufr_bufr4backend_adpsfc_prepbufr_mpi4.yaml} | 0 ...sfc_prepbufr.yaml => bufr_script4backend_adpsfc_prepbufr.yaml} | 0 ...fr_mpi0.yaml => bufr_script4backend_adpsfc_prepbufr_mpi0.yaml} | 0 ...fr_mpi4.yaml => bufr_script4backend_adpsfc_prepbufr_mpi4.yaml} | 0 6 files changed, 0 insertions(+), 0 deletions(-) rename ush/test/{bufr_bufr_backend_adpsfc_prepbufr.yaml => bufr_bufr4backend_adpsfc_prepbufr.yaml} (100%) rename ush/test/{bufr_bufr_backend_adpsfc_prepbufr_mpi0.yaml => bufr_bufr4backend_adpsfc_prepbufr_mpi0.yaml} (100%) rename ush/test/{bufr_bufr_backend_adpsfc_prepbufr_mpi4.yaml => bufr_bufr4backend_adpsfc_prepbufr_mpi4.yaml} (100%) rename ush/test/{bufr_script_backend_adpsfc_prepbufr.yaml => bufr_script4backend_adpsfc_prepbufr.yaml} (100%) rename ush/test/{bufr_script_backend_adpsfc_prepbufr_mpi0.yaml => bufr_script4backend_adpsfc_prepbufr_mpi0.yaml} (100%) rename ush/test/{bufr_script_backend_adpsfc_prepbufr_mpi4.yaml => bufr_script4backend_adpsfc_prepbufr_mpi4.yaml} (100%) diff --git a/ush/test/bufr_bufr_backend_adpsfc_prepbufr.yaml b/ush/test/bufr_bufr4backend_adpsfc_prepbufr.yaml similarity index 100% rename from ush/test/bufr_bufr_backend_adpsfc_prepbufr.yaml rename to ush/test/bufr_bufr4backend_adpsfc_prepbufr.yaml diff --git a/ush/test/bufr_bufr_backend_adpsfc_prepbufr_mpi0.yaml b/ush/test/bufr_bufr4backend_adpsfc_prepbufr_mpi0.yaml similarity index 100% rename from ush/test/bufr_bufr_backend_adpsfc_prepbufr_mpi0.yaml rename to ush/test/bufr_bufr4backend_adpsfc_prepbufr_mpi0.yaml diff --git a/ush/test/bufr_bufr_backend_adpsfc_prepbufr_mpi4.yaml b/ush/test/bufr_bufr4backend_adpsfc_prepbufr_mpi4.yaml similarity index 100% rename from ush/test/bufr_bufr_backend_adpsfc_prepbufr_mpi4.yaml rename to ush/test/bufr_bufr4backend_adpsfc_prepbufr_mpi4.yaml diff --git a/ush/test/bufr_script_backend_adpsfc_prepbufr.yaml b/ush/test/bufr_script4backend_adpsfc_prepbufr.yaml similarity index 100% rename from ush/test/bufr_script_backend_adpsfc_prepbufr.yaml rename to ush/test/bufr_script4backend_adpsfc_prepbufr.yaml diff --git a/ush/test/bufr_script_backend_adpsfc_prepbufr_mpi0.yaml b/ush/test/bufr_script4backend_adpsfc_prepbufr_mpi0.yaml similarity index 100% rename from ush/test/bufr_script_backend_adpsfc_prepbufr_mpi0.yaml rename to ush/test/bufr_script4backend_adpsfc_prepbufr_mpi0.yaml diff --git a/ush/test/bufr_script_backend_adpsfc_prepbufr_mpi4.yaml b/ush/test/bufr_script4backend_adpsfc_prepbufr_mpi4.yaml similarity index 100% rename from ush/test/bufr_script_backend_adpsfc_prepbufr_mpi4.yaml rename to ush/test/bufr_script4backend_adpsfc_prepbufr_mpi4.yaml From b2165796e15d2417607ea363595f76564e6d8dc1 Mon Sep 17 00:00:00 2001 From: Nicholas Esposito Date: Thu, 26 Dec 2024 13:58:50 -0500 Subject: [PATCH 57/72] encodeBufr_Nick.sh _backend -> 4backend --- ush/test/encodeBufr_Nick.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ush/test/encodeBufr_Nick.sh b/ush/test/encodeBufr_Nick.sh index 73e00e0..7abbc4c 100755 --- a/ush/test/encodeBufr_Nick.sh +++ b/ush/test/encodeBufr_Nick.sh @@ -9,7 +9,7 @@ bufrtype="${3:-satwnd}" obstype="${4:-satwnd_amv_goes}" sensor="${5:-abi}" split_by_category="${6:-true}" -mode="${7:-script_backend}" +mode="${7:-script4backend}" nproc="${8:-4}" # ========================== @@ -23,7 +23,7 @@ usage() { echo " : observation type to create (e.g., satwnd_amv_goes, atms, cris, sfcsno )" echo " : sensor (e.g., abi, atms); for non-satellite dta, sensor is usually obstype (e.g., sfcsno)" echo " : split the output file into multiple files based on category (false or true)" - echo " : mode of operation (e.g., bufr_backend, script_backend, bufr2netcdf, script2netcdf)" + echo " : mode of operation (e.g., bufr4backend, script4backend, bufr2netcdf, script2netcdf)" echo " : number of processors (positive integer to run with MPI, or zero for serial execution)" exit 1 } @@ -54,8 +54,8 @@ fi # ============== # Validate mode # ============== -if [[ "$mode" != "bufr_backend" && "$mode" != "script_backend" && "$mode" != "bufr2netcdf" && "$mode" != "script2netcdf" ]]; then - echo "Error: Invalid mode '$mode'. Expected 'bufr_backend' or 'script_backend' or 'bufr2netcdf' or 'script2netcdf'." +if [[ "$mode" != "bufr4backend" && "$mode" != "script4backend" && "$mode" != "bufr2netcdf" && "$mode" != "script2netcdf" ]]; then + echo "Error: Invalid mode '$mode'. Expected 'bufr4backend' or 'script4backend' or 'bufr2netcdf' or 'script2netcdf'." usage fi @@ -177,7 +177,7 @@ fi # ============================= # Run ioda bufr/script backend # ============================= -if [[ "$mode" == "bufr_backend" || "$mode" == "script_backend" ]]; then +if [[ "$mode" == "bufr4backend" || "$mode" == "script4backend" ]]; then if [[ ! -f "$ioda_config_yaml" ]]; then echo "Error: ioda configuration file not found: $ioda_config_yaml" exit 1 @@ -215,7 +215,7 @@ elif [[ "$mode" == "script2netcdf" ]]; then srun -n "$nproc" --mem 96G --time 00:30:00 python bufr_${obstype}.py "$input_file" "$mapping_file" "$output_file" "$cycle_time" || { echo "Error: MPI Python script2netcdf failed"; python bufr_${obstype}.py --help; exit 1; } fi else - echo Incorrect running mode ${mode} ... Valid modes are: bufr_backend, script_back, bufr2netcdf, or script2netcdf + echo Incorrect running mode ${mode} ... Valid modes are: bufr4backend, script4backend, bufr2netcdf, or script2netcdf fi From 109c39fcc59a0aa43150218482d2d9e4ae8ad55a Mon Sep 17 00:00:00 2001 From: Nicholas Esposito Date: Thu, 26 Dec 2024 14:06:21 -0500 Subject: [PATCH 58/72] _backend -> 4backend --- ush/test/README.md | 16 ++++++++-------- ush/test/bufr_bufr4backend_adpsfc_prepbufr.yaml | 2 +- .../bufr_bufr4backend_adpsfc_prepbufr_mpi0.yaml | 2 +- .../bufr_bufr4backend_adpsfc_prepbufr_mpi4.yaml | 2 +- .../bufr_script4backend_adpsfc_prepbufr.yaml | 2 +- ...bufr_script4backend_adpsfc_prepbufr_mpi0.yaml | 2 +- ...bufr_script4backend_adpsfc_prepbufr_mpi4.yaml | 2 +- ush/test/encodeBufr.sh | 12 ++++++------ 8 files changed, 20 insertions(+), 20 deletions(-) diff --git a/ush/test/README.md b/ush/test/README.md index 9254b93..9bf62cd 100644 --- a/ush/test/README.md +++ b/ush/test/README.md @@ -39,9 +39,9 @@ This is a prototype for testing BUFR to IODA conversion and is still evolving. - bufr_satwnd_amv_goes_mapping.yaml - - bufr_bufr_backend_satwnd_amv_goes.yaml + - bufr_bufr4backend_satwnd_amv_goes.yaml - - bufr_script_backend_satwnd_amv_goes.yaml + - bufr_script4backend_satwnd_amv_goes.yaml - testinput/2021080100/gdas.t00z.satwnd.tm00.bufr_d (copied from the global dump) @@ -60,7 +60,7 @@ This is a prototype for testing BUFR to IODA conversion and is still evolving. : observation type to create (e.g., satwnd_amv_goes, atms, sfcsno) : sensor (e.g., abi, atms); for non-satellite dta, sensor is usually obstype (e.g., sfcsno) : split the data into multiple files based on category (false or true) - : mode of operation (e.g., bufr_backend, script_backend, bufr2netcdf, script2netcdf) + : mode of operation (e.g., bufr4backend, script4backend, bufr2netcdf, script2netcdf) : number of processors (positive integer to run with MPI, or zero for serial execution) ``` @@ -75,11 +75,11 @@ This is a prototype for testing BUFR to IODA conversion and is still evolving. ``` obsforge_dir="/scratch1/NCEPDEV/da/Emily.Liu/EMC-obsForge/obsForge" - encodeBufr.sh ${obsforge_dir} 2021080100 satwnd satwnd_amv_goes abi true script_backend 4 + encodeBufr.sh ${obsforge_dir} 2021080100 satwnd satwnd_amv_goes abi true script4backend 4 - encodeBufr.sh ${obsforge_dir} 2021080100 sfcsno sfcsno sfcsno false script_backend 4 + encodeBufr.sh ${obsforge_dir} 2021080100 sfcsno sfcsno sfcsno false script4backend 4 - encodeBufr.sh ${obsforge_dir} 2021080100 atms atms atms true script_backend 4 + encodeBufr.sh ${obsforge_dir} 2021080100 atms atms atms true script4backend 4 ``` - Run with user-defined mode and number of processes @@ -89,7 +89,7 @@ This is a prototype for testing BUFR to IODA conversion and is still evolving. encodeBufr.sh "" "" "" "" "" "" script2netcdf" 0 - encodeBufr.sh "" "" "" "" "" "" bufr_backend" 12 + encodeBufr.sh "" "" "" "" "" "" bufr4backend" 12 - encodeBufr.sh "" "" "" "" "" "" script_backend" 4 + encodeBufr.sh "" "" "" "" "" "" script4backend" 4 ``` diff --git a/ush/test/bufr_bufr4backend_adpsfc_prepbufr.yaml b/ush/test/bufr_bufr4backend_adpsfc_prepbufr.yaml index 84b640e..d00d47d 100644 --- a/ush/test/bufr_bufr4backend_adpsfc_prepbufr.yaml +++ b/ush/test/bufr_bufr4backend_adpsfc_prepbufr.yaml @@ -16,4 +16,4 @@ observations: obsdataout: engine: type: H5File - obsfile: "./testoutput/2021080100/bufr_backend/gdas.t00z.adpsfc_prepbufr.tm00.nc" + obsfile: "./testoutput/2021080100/bufr4backend/gdas.t00z.adpsfc_prepbufr.tm00.nc" diff --git a/ush/test/bufr_bufr4backend_adpsfc_prepbufr_mpi0.yaml b/ush/test/bufr_bufr4backend_adpsfc_prepbufr_mpi0.yaml index 3b390e3..29936e8 100644 --- a/ush/test/bufr_bufr4backend_adpsfc_prepbufr_mpi0.yaml +++ b/ush/test/bufr_bufr4backend_adpsfc_prepbufr_mpi0.yaml @@ -16,4 +16,4 @@ observations: obsdataout: engine: type: H5File - obsfile: "./testoutput/2021080100/bufr_backend/gdas.t00z.adpsfc_prepbufr.tm00_mpi0.nc" + obsfile: "./testoutput/2021080100/bufr4backend/gdas.t00z.adpsfc_prepbufr.tm00_mpi0.nc" diff --git a/ush/test/bufr_bufr4backend_adpsfc_prepbufr_mpi4.yaml b/ush/test/bufr_bufr4backend_adpsfc_prepbufr_mpi4.yaml index 98be1ea..ca03b28 100644 --- a/ush/test/bufr_bufr4backend_adpsfc_prepbufr_mpi4.yaml +++ b/ush/test/bufr_bufr4backend_adpsfc_prepbufr_mpi4.yaml @@ -16,4 +16,4 @@ observations: obsdataout: engine: type: H5File - obsfile: "./testoutput/2021080100/bufr_backend/gdas.t00z.adpsfc_prepbufr.tm00_mpi4.nc" + obsfile: "./testoutput/2021080100/bufr4backend/gdas.t00z.adpsfc_prepbufr.tm00_mpi4.nc" diff --git a/ush/test/bufr_script4backend_adpsfc_prepbufr.yaml b/ush/test/bufr_script4backend_adpsfc_prepbufr.yaml index 3e318c1..a0035cc 100644 --- a/ush/test/bufr_script4backend_adpsfc_prepbufr.yaml +++ b/ush/test/bufr_script4backend_adpsfc_prepbufr.yaml @@ -20,4 +20,4 @@ observations: obsdataout: engine: type: H5File - obsfile: "./testoutput/2021080100/script_backend/gdas.t00z.adpsfc_prepbufr.tm00.nc" + obsfile: "./testoutput/2021080100/script4backend/gdas.t00z.adpsfc_prepbufr.tm00.nc" diff --git a/ush/test/bufr_script4backend_adpsfc_prepbufr_mpi0.yaml b/ush/test/bufr_script4backend_adpsfc_prepbufr_mpi0.yaml index bfd521f..b5741f8 100644 --- a/ush/test/bufr_script4backend_adpsfc_prepbufr_mpi0.yaml +++ b/ush/test/bufr_script4backend_adpsfc_prepbufr_mpi0.yaml @@ -20,4 +20,4 @@ observations: obsdataout: engine: type: H5File - obsfile: "./testoutput/2021080100/script_backend/gdas.t00z.adpsfc_prepbufr.tm00_mpi0.nc" + obsfile: "./testoutput/2021080100/script4backend/gdas.t00z.adpsfc_prepbufr.tm00_mpi0.nc" diff --git a/ush/test/bufr_script4backend_adpsfc_prepbufr_mpi4.yaml b/ush/test/bufr_script4backend_adpsfc_prepbufr_mpi4.yaml index 19d6120..1ddd60c 100644 --- a/ush/test/bufr_script4backend_adpsfc_prepbufr_mpi4.yaml +++ b/ush/test/bufr_script4backend_adpsfc_prepbufr_mpi4.yaml @@ -20,4 +20,4 @@ observations: obsdataout: engine: type: H5File - obsfile: "./testoutput/2021080100/script_backend/gdas.t00z.adpsfc_prepbufr.tm00_mpi4.nc" + obsfile: "./testoutput/2021080100/script4backend/gdas.t00z.adpsfc_prepbufr.tm00_mpi4.nc" diff --git a/ush/test/encodeBufr.sh b/ush/test/encodeBufr.sh index 970f4b3..49c2abf 100644 --- a/ush/test/encodeBufr.sh +++ b/ush/test/encodeBufr.sh @@ -9,7 +9,7 @@ bufrtype="${3:-satwnd}" obstype="${4:-satwnd_amv_goes}" sensor="${5:-abi}" split_by_category="${6:-true}" -mode="${7:-script_backend}" +mode="${7:-script4backend}" nproc="${8:-4}" # ========================== @@ -23,7 +23,7 @@ usage() { echo " : observation type to create (e.g., satwnd_amv_goes, atms, cris, sfcsno )" echo " : sensor (e.g., abi, atms); for non-satellite dta, sensor is usually obstype (e.g., sfcsno)" echo " : split the output file into multiple files based on category (false or true)" - echo " : mode of operation (e.g., bufr_backend, script_backend, bufr2netcdf, script2netcdf)" + echo " : mode of operation (e.g., bufr4backend, script4backend, bufr2netcdf, script2netcdf)" echo " : number of processors (positive integer to run with MPI, or zero for serial execution)" exit 1 } @@ -54,8 +54,8 @@ fi # ============== # Validate mode # ============== -if [[ "$mode" != "bufr_backend" && "$mode" != "script_backend" && "$mode" != "bufr2netcdf" && "$mode" != "script2netcdf" ]]; then - echo "Error: Invalid mode '$mode'. Expected 'bufr_backend' or 'script_backend' or 'bufr2netcdf' or 'script2netcdf'." +if [[ "$mode" != "bufr4backend" && "$mode" != "script4backend" && "$mode" != "bufr2netcdf" && "$mode" != "script2netcdf" ]]; then + echo "Error: Invalid mode '$mode'. Expected 'bufr4backend' or 'script4backend' or 'bufr2netcdf' or 'script2netcdf'." usage fi @@ -174,7 +174,7 @@ fi # ============================= # Run ioda bufr/script backend # ============================= -if [[ "$mode" == "bufr_backend" || "$mode" == "script_backend" ]]; then +if [[ "$mode" == "bufr4backend" || "$mode" == "script4backend" ]]; then if [[ ! -f "$ioda_config_yaml" ]]; then echo "Error: ioda configuration file not found: $ioda_config_yaml" exit 1 @@ -209,5 +209,5 @@ elif [[ "$mode" == "script2netcdf" ]]; then srun -n "$nproc" --mem 96G --time 00:30:00 python bufr_${obstype}.py "$input_file" "$mapping_file" "$output_file" || { echo "Error: MPI Python script2netcdf failed"; python bufr_${obstype}.py --help; exit 1; } fi else - echo Incorrect running mode ${mode} ... Valid modes are: bufr_backend, script_back, bufr2netcdf, or script2netcdf + echo Incorrect running mode ${mode} ... Valid modes are: bufr4backend, script_back, bufr2netcdf, or script2netcdf fi From b39382b27523ec7d7998b0eec50d31f519306297 Mon Sep 17 00:00:00 2001 From: Nicholas Esposito Date: Thu, 26 Dec 2024 17:07:45 -0500 Subject: [PATCH 59/72] cp to mapping config --- ...=> bufr_bufr4backend_adpsfc_prepbufr.yaml} | 7 +- ...ufr_bufr4backend_adpsfc_prepbufr_mpi0.yaml | 19 + ...ufr_bufr4backend_adpsfc_prepbufr_mpi4.yaml | 19 + .../bufr_script4backend_adpsfc_prepbufr.yaml | 23 ++ ...r_script4backend_adpsfc_prepbufr_mpi0.yaml | 23 ++ ...r_script4backend_adpsfc_prepbufr_mpi4.yaml | 23 ++ dump/mapping/bufr2ioda_adpsfc_prepbufr.py | 350 ------------------ ...yaml => bufr_adpsfc_prepbufr_mapping.yaml} | 16 +- 8 files changed, 122 insertions(+), 358 deletions(-) rename dump/config/{bufr2ioda_prepbufr_backend_adpsfc.yaml => bufr_bufr4backend_adpsfc_prepbufr.yaml} (56%) create mode 100644 dump/config/bufr_bufr4backend_adpsfc_prepbufr_mpi0.yaml create mode 100644 dump/config/bufr_bufr4backend_adpsfc_prepbufr_mpi4.yaml create mode 100644 dump/config/bufr_script4backend_adpsfc_prepbufr.yaml create mode 100644 dump/config/bufr_script4backend_adpsfc_prepbufr_mpi0.yaml create mode 100644 dump/config/bufr_script4backend_adpsfc_prepbufr_mpi4.yaml delete mode 100755 dump/mapping/bufr2ioda_adpsfc_prepbufr.py rename dump/mapping/{iodatest_prepbufr_adpsfc_mapping.yaml => bufr_adpsfc_prepbufr_mapping.yaml} (92%) diff --git a/dump/config/bufr2ioda_prepbufr_backend_adpsfc.yaml b/dump/config/bufr_bufr4backend_adpsfc_prepbufr.yaml similarity index 56% rename from dump/config/bufr2ioda_prepbufr_backend_adpsfc.yaml rename to dump/config/bufr_bufr4backend_adpsfc_prepbufr.yaml index 0347947..d00d47d 100644 --- a/dump/config/bufr2ioda_prepbufr_backend_adpsfc.yaml +++ b/dump/config/bufr_bufr4backend_adpsfc_prepbufr.yaml @@ -1,6 +1,7 @@ time window: begin: "2021-07-31T21:00:00Z" end: "2021-08-01T03:00:00Z" + bound to include: begin observations: - obs space: @@ -9,10 +10,10 @@ observations: obsdatain: engine: type: bufr - obsfile: "./testinput/gdas.t00z.adpsfc.prepbufr" - mapping file: "./iodatest_prepbufr_adpsfc_mapping.yaml" + obsfile: "./testinput/2021080100/gdas.t00z.adpsfc.tm00.prepbufr" + mapping file: "./bufr_adpsfc_prepbufr_mapping.yaml" # category: ["goes-16"] obsdataout: engine: type: H5File - obsfile: "./testoutput/2021080100/bufr_backend/gdas.t00z.adpsfc.prepbufr.nc" + obsfile: "./testoutput/2021080100/bufr4backend/gdas.t00z.adpsfc_prepbufr.tm00.nc" diff --git a/dump/config/bufr_bufr4backend_adpsfc_prepbufr_mpi0.yaml b/dump/config/bufr_bufr4backend_adpsfc_prepbufr_mpi0.yaml new file mode 100644 index 0000000..29936e8 --- /dev/null +++ b/dump/config/bufr_bufr4backend_adpsfc_prepbufr_mpi0.yaml @@ -0,0 +1,19 @@ +time window: + begin: "2021-07-31T21:00:00Z" + end: "2021-08-01T03:00:00Z" + bound to include: begin + +observations: +- obs space: + name: "ADPSFC" + simulated variables: ['stationElevation','stationPressure'] + obsdatain: + engine: + type: bufr + obsfile: "./testinput/2021080100/gdas.t00z.adpsfc.tm00.prepbufr" + mapping file: "./bufr_adpsfc_prepbufr_mapping.yaml" + # category: ["goes-16"] + obsdataout: + engine: + type: H5File + obsfile: "./testoutput/2021080100/bufr4backend/gdas.t00z.adpsfc_prepbufr.tm00_mpi0.nc" diff --git a/dump/config/bufr_bufr4backend_adpsfc_prepbufr_mpi4.yaml b/dump/config/bufr_bufr4backend_adpsfc_prepbufr_mpi4.yaml new file mode 100644 index 0000000..ca03b28 --- /dev/null +++ b/dump/config/bufr_bufr4backend_adpsfc_prepbufr_mpi4.yaml @@ -0,0 +1,19 @@ +time window: + begin: "2021-07-31T21:00:00Z" + end: "2021-08-01T03:00:00Z" + bound to include: begin + +observations: +- obs space: + name: "ADPSFC" + simulated variables: ['stationElevation','stationPressure'] + obsdatain: + engine: + type: bufr + obsfile: "./testinput/2021080100/gdas.t00z.adpsfc.tm00.prepbufr" + mapping file: "./bufr_adpsfc_prepbufr_mapping.yaml" + # category: ["goes-16"] + obsdataout: + engine: + type: H5File + obsfile: "./testoutput/2021080100/bufr4backend/gdas.t00z.adpsfc_prepbufr.tm00_mpi4.nc" diff --git a/dump/config/bufr_script4backend_adpsfc_prepbufr.yaml b/dump/config/bufr_script4backend_adpsfc_prepbufr.yaml new file mode 100644 index 0000000..a0035cc --- /dev/null +++ b/dump/config/bufr_script4backend_adpsfc_prepbufr.yaml @@ -0,0 +1,23 @@ +time window: + begin: "2021-07-31T21:00:00Z" + end: "2021-08-01T03:00:00Z" + bound to include: begin + +observations: +- obs space: + name: "ADPSFC" + observed variables: ['stationElevation','stationPressure'] + derived variables: ['stationElevation','stationPressure'] + simulated variables: ['stationElevation','stationPressure'] + obsdatain: + engine: + type: script + script file: "bufr_adpsfc_prepbufr.py" + args: + input_path: "./testinput/2021080100/gdas.t00z.adpsfc.tm00.prepbufr" + mapping_path: "./bufr_adpsfc_prepbufr_mapping.yaml" + cycle_time: "2021080100" + obsdataout: + engine: + type: H5File + obsfile: "./testoutput/2021080100/script4backend/gdas.t00z.adpsfc_prepbufr.tm00.nc" diff --git a/dump/config/bufr_script4backend_adpsfc_prepbufr_mpi0.yaml b/dump/config/bufr_script4backend_adpsfc_prepbufr_mpi0.yaml new file mode 100644 index 0000000..b5741f8 --- /dev/null +++ b/dump/config/bufr_script4backend_adpsfc_prepbufr_mpi0.yaml @@ -0,0 +1,23 @@ +time window: + begin: "2021-07-31T21:00:00Z" + end: "2021-08-01T03:00:00Z" + bound to include: begin + +observations: +- obs space: + name: "ADPSFC" + observed variables: ['stationElevation','stationPressure'] + derived variables: ['stationElevation','stationPressure'] + simulated variables: ['stationElevation','stationPressure'] + obsdatain: + engine: + type: script + script file: "bufr_adpsfc_prepbufr.py" + args: + input_path: "testinput/2021080100/gdas.t00z.adpsfc.tm00.prepbufr" + mapping_path: "./bufr_adpsfc_prepbufr_mapping.yaml" + cycle_time: "2021080100" + obsdataout: + engine: + type: H5File + obsfile: "./testoutput/2021080100/script4backend/gdas.t00z.adpsfc_prepbufr.tm00_mpi0.nc" diff --git a/dump/config/bufr_script4backend_adpsfc_prepbufr_mpi4.yaml b/dump/config/bufr_script4backend_adpsfc_prepbufr_mpi4.yaml new file mode 100644 index 0000000..1ddd60c --- /dev/null +++ b/dump/config/bufr_script4backend_adpsfc_prepbufr_mpi4.yaml @@ -0,0 +1,23 @@ +time window: + begin: "2021-07-31T21:00:00Z" + end: "2021-08-01T03:00:00Z" + bound to include: begin + +observations: +- obs space: + name: "ADPSFC" + observed variables: ['stationElevation','stationPressure'] + derived variables: ['stationElevation','stationPressure'] + simulated variables: ['stationElevation','stationPressure'] + obsdatain: + engine: + type: script + script file: "bufr_adpsfc_prepbufr.py" + args: + input_path: "testinput/2021080100/gdas.t00z.adpsfc.tm00.prepbufr" + mapping_path: "./bufr_adpsfc_prepbufr_mapping.yaml" + cycle_time: "2021080100" + obsdataout: + engine: + type: H5File + obsfile: "./testoutput/2021080100/script4backend/gdas.t00z.adpsfc_prepbufr.tm00_mpi4.nc" diff --git a/dump/mapping/bufr2ioda_adpsfc_prepbufr.py b/dump/mapping/bufr2ioda_adpsfc_prepbufr.py deleted file mode 100755 index dcf6dfa..0000000 --- a/dump/mapping/bufr2ioda_adpsfc_prepbufr.py +++ /dev/null @@ -1,350 +0,0 @@ -#!/usr/bin/env python3 -import os -import sys -import bufr -from pyioda.ioda.Engines.Bufr import Encoder -import copy -import numpy as np -import numpy.ma as ma -import math -import calendar -import time -from datetime import datetime -from wxflow import Logger - -# Initialize Logger -# Get log level from the environment variable, default to 'INFO it not set -log_level = os.getenv('LOG_LEVEL', 'INFO') -logger = Logger('BUFR2IODA_adpsfc_prepbufr.py', level=log_level, colored_log=False) - -def logging(comm, level, message): - """ - Logs a message to the console or log file, based on the specified logging level. - - This function ensures that logging is only performed by the root process (`rank 0`) - in a distributed computing environment. The function maps the logging level to - appropriate logger methods and defaults to the 'INFO' level if an invalid level is provided. - - Parameters: - comm: object - The communicator object, typically from a distributed computing framework - (e.g., MPI). It must have a `rank()` method to determine the process rank. - level: str - The logging level as a string. Supported levels are: - - 'DEBUG' - - 'INFO' - - 'WARNING' - - 'ERROR' - - 'CRITICAL' - If an invalid level is provided, a warning will be logged, and the level - will default to 'INFO'. - message: str - The message to be logged. - - Behavior: - - Logs messages only on the root process (`comm.rank() == 0`). - - Maps the provided logging level to a method of the logger object. - - Defaults to 'INFO' and logs a warning if an invalid logging level is given. - - Supports standard logging levels for granular control over log verbosity. - - Example: - >>> logging(comm, 'DEBUG', 'This is a debug message.') - >>> logging(comm, 'ERROR', 'An error occurred!') - - Notes: - - Ensure that a global `logger` object is configured before using this function. - - The `comm` object should conform to MPI-like conventions (e.g., `rank()` method). - """ - - if comm.rank() == 0: - # Define a dictionary to map levels to logger methods - log_methods = { - 'DEBUG': logger.debug, - 'INFO': logger.info, - 'WARNING': logger.warning, - 'ERROR': logger.error, - 'CRITICAL': logger.critical, - } - - # Get the appropriate logging method, default to 'INFO' - log_method = log_methods.get(level.upper(), logger.info) - - if log_method == logger.info and level.upper() not in log_methods: - # Log a warning if the level is invalid - logger.warning(f'log level = {level}: not a valid level --> set to INFO') - - # Call the logging method - log_method(message) - -def _make_description(mapping_path, update=False): - description = bufr.encoders.Description(mapping_path) - - if update: - # Define the variables to be added in a list of dictionaries - variables = [ - { - 'name': 'MetaData/dateTime' - 'source': 'variables/dateTime', - 'units': 'seconds since 1970-01-01T00:00:00Z', - 'longName': 'dateTime', - }, - { - 'name': 'MetaData/sequenceNumber', - 'source': 'variables/sequenceNumber' - 'units': '1', - 'longName': 'Sequence Number (Obs Subtype)', - }, - { - 'name': 'ObsValue/windEastward', - 'source': 'variables/windEastward', - 'units': 'm/s', - 'longName': 'Eastward Wind Component', - }, - { - 'name': 'ObsValue/windNorthward', - 'source': 'variables/windNorthward', - 'units': 'm/s', - 'longName': 'Northward Wind Component', - }, - ] - - # Loop through each variable and add it to the description - for var in variables: - description.add_variable( - name=var['name'], - source=var['source'], - units=var['units'], - longName=var['longName'] - ) - - return description - - -def Compute_dateTime(cycleTimeSinceEpoch, dhr): - """ - Compute dateTime using the cycleTimeSinceEpoch and Cycle Time - minus Cycle Time - - Parameters: - cycleTimeSinceEpoch: Time of cycle in Epoch Time - dhr: Observation Time Minus Cycle Time - - Returns: - Masked array of dateTime values - """ - - int64_fill_value = np.int64(0) - - dateTime = np.zeros(dhr.shape, dtype=np.int64) - for i in range(len(dateTime)): - if ma.is_masked(dhr[i]): - continue - else: - dateTime[i] = np.int64(dhr[i]*3600) + cycleTimeSinceEpoch - - dateTime = ma.array(dateTime) - dateTime = ma.masked_values(dateTime, int64_fill_value) - - return dateTime - -def _make_obs(comm, input_path, mapping_path): - - # Get container from mapping file first - logging(comm, 'INFO', 'Get container from bufr') - container = bufr.Parser(input_path, mapping_path).parse(comm) - - logging(comm, 'DEBUG', f'container list (original): {container.list()}') - #logging(comm, 'DEBUG', f'all_sub_categories = {container.all_sub_categories()}') - #logging(comm, 'DEBUG', f'category map = {container.get_category_map()}') - - # Add new/derived data into container - #for cat in container.all_sub_categories(): - - # logging(comm, 'DEBUG', f'category = {cat}') - - # satid = container.get('variables/obsTimeMinusCycleTime', cat) - # if satid.size == 0: - # logging(comm, 'WARNING', f'category {cat[0]} does not exist in input file') - #paths = container.get_paths('variables/windComputationMethod', cat) - #obstype = container.get('variables/windComputationMethod', cat) - #container.add('variables/obstype_uwind', obstype, paths, cat) - #container.add('variables/obstype_vwind', obstype, paths, cat) - - #paths = container.get_paths('variables/windSpeed', cat) - #wob = container.get('variables/windSpeed', cat) - #container.add('variables/windEastward', wob, paths, cat) - #container.add('variables/windNorthward', wob, paths, cat) - - # else: - print(" Do DateTime calculation") - otmct = container.get('variables/obsTimeMinusCycleTime') - otmct_paths = container.get_paths('variables/obsTimeMinusCycleTime') - otmct2 = np.array(otmct) - cycleTimeSinceEpoch = np.int64(calendar.timegm(time.strptime(str(int(CYCLE_TIME)), '%Y%m%d%H'))) - dateTime = Compute_dateTime(cycleTimeSinceEpoch, otmct2) - logging(comm, 'DEBUG', f'dateTime min/max = {dateTime.min()} {dateTime.max()}') - - print("Make an array of 0s for MetaData/sequenceNumber") - sequenceNum = np.zeros(otmct.shape, dtype=np.int32) - logging(comm, 'DEBUG', f' sequenceNummin/max = {sequenceNum.min()} {sequenceNum.max()}') - - print(" Add variables to container.") - container.add('variables/dateTime', dateTime, otmct_paths) - container.add('variables/sequenceNumber', sequenceNum, otmct_paths) - #description = bufr.encoders.Description(YAML_PATH) - - #print(" Add container and descriptions to dataset.") - #dataset = next(iter(Encoder(description).encode(container).values())) - - - # Add new variables: ObsType/windEastward & ObsType/windNorthward -# swcm = container.get('variables/windComputationMethod', cat) -# chanfreq = container.get('variables/sensorCentralFrequency', cat) -# -# logging(comm, 'DEBUG', f'swcm min/max = {swcm.min()} {swcm.max()}') -# logging(comm, 'DEBUG', f'chanfreq min/max = {chanfreq.min()} {chanfreq.max()}') -# -# obstype = _get_obs_type(swcm, chanfreq) -# -# logging(comm, 'DEBUG', f'obstype = {obstype}') -# logging(comm, 'DEBUG', f'obstype min/max = {obstype.min()} {obstype.max()}') -# -# paths = container.get_paths('variables/windComputationMethod', cat) -# container.add('variables/obstype_uwind', obstype, paths, cat) -# container.add('variables/obstype_vwind', obstype, paths, cat) -# -# # Add new variables: ObsValue/windEastward & ObsValue/windNorthward -# wdir = container.get('variables/windDirection', cat) -# wspd = container.get('variables/windSpeed', cat) -# -# logging(comm, 'DEBUG', f'wdir min/max = {wdir.min()} {wdir.max()}') -# logging(comm, 'DEBUG', f'wspd min/max = {wspd.min()} {wspd.max()}') -# -# uob, vob = compute_wind_components(wdir, wspd) -# -# logging(comm, 'DEBUG', f'uob min/max = {uob.min()} {uob.max()}') -# logging(comm, 'DEBUG', f'vob min/max = {vob.min()} {vob.max()}') -# -# paths = container.get_paths('variables/windSpeed', cat) -# container.add('variables/windEastward', uob, paths, cat) -# container.add('variables/windNorthward', vob, paths, cat) - - # Check - logging(comm, 'DEBUG', f'container list (updated): {container.list()}') - #logging(comm, 'DEBUG', f'all_sub_categories {container.all_sub_categories()}') - - return container - -def create_obs_group(cycle_time,input_mapping,input_path): - CYCLE_TIME = cycle_time - YAML_PATH = input_mapping #"./iodatest_prepbufr_adpsfc_mapping.yaml" - INPUT_PATH = input_path - print(" CYCLE_TIME: ", CYCLE_TIME) - - container = bufr.Parser(INPUT_PATH, YAML_PATH).parse() - - print(" Do DateTime calculation") - otmct = container.get('variables/obsTimeMinusCycleTime') - otmct_paths = container.get_paths('variables/obsTimeMinusCycleTime') - otmct2 = np.array(otmct) - cycleTimeSinceEpoch = np.int64(calendar.timegm(time.strptime(str(int(CYCLE_TIME)), '%Y%m%d%H'))) - dateTime = Compute_dateTime(cycleTimeSinceEpoch, otmct2) - - print("Make an array of 0s for MetaData/sequenceNumber") - sequenceNum = np.zeros(otmct.shape, dtype=np.int32) - - print(" Add variables to container.") - container.add('variables/dateTime', dateTime, otmct_paths) - container.add('variables/sequenceNumber', sequenceNum, otmct_paths) - description = bufr.encoders.Description(YAML_PATH) - - print(" Add container and descriptions to dataset.") - dataset = next(iter(Encoder(description).encode(container).values())) - return dataset - -#def create_obs_group(input_path, mapping_path, category, env): -def create_obs_group(input_path, mapping_path, env): - - comm = bufr.mpi.Comm(env["comm_name"]) - - description = _make_description(mapping_path, update=True) - - # Check the cache for the data and return it if it exists - logging(comm, 'DEBUG', f'Check if bufr.DataCache exists? {bufr.DataCache.has(input_path, mapping_path)}') - if bufr.DataCache.has(input_path, mapping_path): - container = bufr.DataCache.get(input_path, mapping_path) - #logging(comm, 'INFO', f'Encode {category} from cache') - logging(comm, 'INFO', f'Encode from cache') - #data = iodaEncoder(description).encode(container)[(category,)] - data = iodaEncoder(description).encode(container) - #logging(comm, 'INFO', f'Mark {category} as finished in the cache') - logging(comm, 'INFO', f'Mark as finished in the cache') - #bufr.DataCache.mark_finished(input_path, mapping_path, [category]) - bufr.DataCache.mark_finished(input_path, mapping_path) - #logging(comm, 'INFO', f'Return the encoded data for {category}') - logging(comm, 'INFO', f'Return the encoded data.') - return data - - container = _make_obs(comm, input_path, mapping_path) - - # Gather data from all tasks into all tasks. Each task will have the complete record - logging(comm, 'INFO', f'Gather data from all tasks into all tasks') - container.all_gather(comm) - - logging(comm, 'INFO', f'Add container to cache') - # Add the container to the cache - #bufr.DataCache.add(input_path, mapping_path, container.all_sub_categories(), container) - bufr.DataCache.add(input_path, mapping_path, container) - - # Encode the data - logging(comm, 'INFO', f'Encode {category}') - #data = iodaEncoder(description).encode(container)[(category,)] - data = iodaEncoder(description).encode(container) - - logging(comm, 'INFO', f'Mark {category} as finished in the cache') - # Mark the data as finished in the cache - #bufr.DataCache.mark_finished(input_path, mapping_path, [category]) - bufr.DataCache.mark_finished(input_path, mapping_path) - - logging(comm, 'INFO', f'Return the encoded data.') - return data - - -def create_obs_file(input_path, mapping_path, output_path): - - comm = bufr.mpi.Comm("world") - container = _make_obs(comm, input_path, mapping_path) - container.gather(comm) - - description = _make_description(mapping_path, update=True) - - # Encode the data - if comm.rank() == 0: - netcdfEncoder(description).encode(container, output_path) - - logging(comm, 'INFO', f'Return the encoded data') - - -if __name__ == '__main__': - - start_time = time.time() - - bufr.mpi.App(sys.argv) - comm = bufr.mpi.Comm("world") - - # Required input arguments - parser = argparse.ArgumentParser() - parser.add_argument('-i', '--input', type=str, help='Input BUFR', required=True) - parser.add_argument('-m', '--mapping', type=str, help='BUFR2IODA Mapping File', required=True) - parser.add_argument('-o', '--output', type=str, help='Output NetCDF', required=True) - - args = parser.parse_args() - mapping = args.mapping - infile = args.input - output = args.output - - create_obs_file(infile, mapping, output) - - end_time = time.time() - running_time = end_time - start_time - logging(comm, 'INFO', f'Total running time: {running_time}') diff --git a/dump/mapping/iodatest_prepbufr_adpsfc_mapping.yaml b/dump/mapping/bufr_adpsfc_prepbufr_mapping.yaml similarity index 92% rename from dump/mapping/iodatest_prepbufr_adpsfc_mapping.yaml rename to dump/mapping/bufr_adpsfc_prepbufr_mapping.yaml index b423444..dd99089 100755 --- a/dump/mapping/iodatest_prepbufr_adpsfc_mapping.yaml +++ b/dump/mapping/bufr_adpsfc_prepbufr_mapping.yaml @@ -10,6 +10,12 @@ bufr: query: "*/TYP" # MetaData + timestamp: + timeoffset: + timeOffset: "*/DHR" + transforms: + - scale: 3600 + referenceTime: "2021-08-01T00:00:00Z" prepbufrDataLevelCategory: query: "*/CAT" obsTimeMinusCycleTime: @@ -106,7 +112,7 @@ encoder: longName: "Prepbufr Data Level Category" - name: "MetaData/dateTime" - source: variables/dateTime + source: variables/timestamp units: 'seconds since 1970-01-01T00:00:00Z' longName: 'dateTime' @@ -120,11 +126,11 @@ encoder: source: variables/longitude longName: "Longitude" units: "degree_east" - range: [0, 360] + #range: [0, 360] - - name: "MetaData/sequenceNumber" - source: variables/sequenceNumber - longName: "Sequence Number (Obs Subtype)" + # - name: "MetaData/sequenceNumber" + #source: variables/sequenceNumber + #longName: "Sequence Number (Obs Subtype)" - name: "MetaData/stationIdentification" source: variables/stationIdentification From 9a83e0d9fb518a8a04ae1121343fed083df7392c Mon Sep 17 00:00:00 2001 From: Nicholas Esposito Date: Thu, 26 Dec 2024 17:12:15 -0500 Subject: [PATCH 60/72] py to mapping --- dump/mapping/bufr_adpsfc_prepbufr.py | 246 +++++++++++++++++++++++++++ 1 file changed, 246 insertions(+) create mode 100755 dump/mapping/bufr_adpsfc_prepbufr.py diff --git a/dump/mapping/bufr_adpsfc_prepbufr.py b/dump/mapping/bufr_adpsfc_prepbufr.py new file mode 100755 index 0000000..1d797e9 --- /dev/null +++ b/dump/mapping/bufr_adpsfc_prepbufr.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python3 +import os +import sys +import bufr +import argparse +import copy +import numpy as np +import numpy.ma as ma +import math +import calendar +import time +from datetime import datetime +from pyioda.ioda.Engines.Bufr import Encoder as iodaEncoder +from bufr.encoders.netcdf import Encoder as netcdfEncoder +from wxflow import Logger + +# Initialize Logger +# Get log level from the environment variable, default to 'INFO it not set +log_level = os.getenv('LOG_LEVEL', 'INFO') +logger = Logger('BUFR2IODA_adpsfc_prepbufr.py', level=log_level, colored_log=False) + +def Compute_dateTime(cycleTimeSinceEpoch, dhr): + """ + Compute dateTime using the cycleTimeSinceEpoch and Cycle Time + minus Cycle Time + + Parameters: + cycleTimeSinceEpoch: Time of cycle in Epoch Time + dhr: Observation Time Minus Cycle Time + + Returns: + Masked array of dateTime values + """ + + int64_fill_value = np.int64(0) + + dateTime = np.zeros(dhr.shape, dtype=np.int64) + for i in range(len(dateTime)): + if ma.is_masked(dhr[i]): + continue + else: + dateTime[i] = np.int64(dhr[i]*3600) + cycleTimeSinceEpoch + + dateTime = ma.array(dateTime) + dateTime = ma.masked_values(dateTime, int64_fill_value) + + return dateTime + + +def logging(comm, level, message): + """ + Logs a message to the console or log file, based on the specified logging level. + + This function ensures that logging is only performed by the root process (`rank 0`) + in a distributed computing environment. The function maps the logging level to + appropriate logger methods and defaults to the 'INFO' level if an invalid level is provided. + + Parameters: + comm: object + The communicator object, typically from a distributed computing framework + (e.g., MPI). It must have a `rank()` method to determine the process rank. + level: str + The logging level as a string. Supported levels are: + - 'DEBUG' + - 'INFO' + - 'WARNING' + - 'ERROR' + - 'CRITICAL' + If an invalid level is provided, a warning will be logged, and the level + will default to 'INFO'. + message: str + The message to be logged. + + Behavior: + - Logs messages only on the root process (`comm.rank() == 0`). + - Maps the provided logging level to a method of the logger object. + - Defaults to 'INFO' and logs a warning if an invalid logging level is given. + - Supports standard logging levels for granular control over log verbosity. + + Example: + >>> logging(comm, 'DEBUG', 'This is a debug message.') + >>> logging(comm, 'ERROR', 'An error occurred!') + + Notes: + - Ensure that a global `logger` object is configured before using this function. + - The `comm` object should conform to MPI-like conventions (e.g., `rank()` method). + """ + + if comm.rank() == 0: + # Define a dictionary to map levels to logger methods + log_methods = { + 'DEBUG': logger.debug, + 'INFO': logger.info, + 'WARNING': logger.warning, + 'ERROR': logger.error, + 'CRITICAL': logger.critical, + } + + # Get the appropriate logging method, default to 'INFO' + log_method = log_methods.get(level.upper(), logger.info) + + if log_method == logger.info and level.upper() not in log_methods: + # Log a warning if the level is invalid + logger.warning(f'log level = {level}: not a valid level --> set to INFO') + + # Call the logging method + log_method(message) + +def _make_description(mapping_path, update=False): + description = bufr.encoders.Description(mapping_path) + + if update: + # Define the variables to be added in a list of dictionaries + variables = [ + { + 'name': 'MetaData/sequenceNumber', + 'source': 'variables/sequenceNumber', + 'units': '1', + 'longName': 'Sequence Number (Obs Subtype)', + } + ] + + # Loop through each variable and add it to the description + for var in variables: + description.add_variable( + name=var['name'], + source=var['source'], + units=var['units'], + longName=var['longName'] + ) + + return description + + +def _make_obs(comm, input_path, mapping_path, cycle_time): + """ + Create the ioda adpsfc prepbufr observations: + - reads values + - adds sequenceNum + + Parameters + ---------- + comm: object + The communicator object (e.g., MPI) + input_path: str + The input bufr file + mapping_path: str + The input bufr2ioda mapping file + cycle_time: str + The cycle in YYYYMMDDHH format + """ + + # Get container from mapping file first + logging(comm, 'INFO', 'Get container from bufr') + container = bufr.Parser(input_path, mapping_path).parse(comm) + + logging(comm, 'DEBUG', f'container list (original): {container.list()}') + logging(comm, 'DEBUG', f'Change longitude range from [0,360] to [-180,180]') + lon = container.get('variables/longitude') + lon_paths = container.get_paths('variables/longitude') + lon[lon>180] -= 360 + lon = ma.round(lon, decimals=2) + + logging(comm, 'DEBUG', f'Do DateTime calculation') + otmct = container.get('variables/obsTimeMinusCycleTime') + otmct_paths = container.get_paths('variables/obsTimeMinusCycleTime') + otmct2 = np.array(otmct) + cycleTimeSinceEpoch = np.int64(calendar.timegm(time.strptime(str(int(cycle_time)), '%Y%m%d%H'))) + dateTime = Compute_dateTime(cycleTimeSinceEpoch, otmct2) + logging(comm, 'DEBUG', f'dateTime min/max = {dateTime.min()} {dateTime.max()}') + + logging(comm, 'DEBUG', f'Make an array of 0s for MetaData/sequenceNumber') + sequenceNum = np.zeros(lon.shape, dtype=np.int32) + logging(comm, 'DEBUG', f' sequenceNummin/max = {sequenceNum.min()} {sequenceNum.max()}') + + logging(comm, 'DEBUG', f'Update variables in container') + container.replace('variables/longitude', lon) + container.replace('variables/timestamp', dateTime) + + logging(comm, 'DEBUG', f'Add variables to container') + container.add('variables/sequenceNumber', sequenceNum, lon_paths) + + # Check + logging(comm, 'DEBUG', f'container list (updated): {container.list()}') + + return container + + +def create_obs_group(input_path, mapping_path, cycle_time, env): + + comm = bufr.mpi.Comm(env["comm_name"]) + + logging(comm, 'INFO', f'Make description and make obs') + description = _make_description(mapping_path, update=True) + container = _make_obs(comm, input_path, mapping_path, cycle_time) + + # Gather data from all tasks into all tasks. Each task will have the complete record + logging(comm, 'INFO', f'Gather data from all tasks into all tasks') + container.all_gather(comm) + + logging(comm, 'INFO', f'Encode the data') + data = next(iter(iodaEncoder(description).encode(container).values())) + + logging(comm, 'INFO', f'Return the encoded data.') + return data + + +def create_obs_file(input_path, mapping_path, output_path, cycle_time): + + comm = bufr.mpi.Comm("world") + container = _make_obs(comm, input_path, mapping_path, cycle_time) + container.gather(comm) + + description = _make_description(mapping_path, update=True) + + # Encode the data + if comm.rank() == 0: + netcdfEncoder(description).encode(container, output_path) + + logging(comm, 'INFO', f'Return the encoded data') + + +if __name__ == '__main__': + start_time = time.time() + + bufr.mpi.App(sys.argv) + comm = bufr.mpi.Comm("world") + + # Required input arguments as positional arguments + parser = argparse.ArgumentParser(description="Convert BUFR to NetCDF using a mapping file.") + parser.add_argument('input', type=str, help='Input BUFR file') + parser.add_argument('mapping', type=str, help='BUFR2IODA Mapping File') + parser.add_argument('output', type=str, help='Output NetCDF file') + parser.add_argument('cycle_time', type=str, help='cycle time in YYYYMMDDHH format') + + args = parser.parse_args() + infile = args.input + mapping = args.mapping + output = args.output + cycle_time = args.cycle_time + + create_obs_file(infile, mapping, output, cycle_time) + + end_time = time.time() + running_time = end_time - start_time + logging(comm, 'INFO', f'Total running time: {running_time}') From 68bf722be84ecf21ea33e3e5d349a68fc3d2fb4e Mon Sep 17 00:00:00 2001 From: Nicholas Esposito Date: Fri, 27 Dec 2024 12:48:53 -0500 Subject: [PATCH 61/72] add reference time --- ush/test/bufr_adpsfc_prepbufr.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/ush/test/bufr_adpsfc_prepbufr.py b/ush/test/bufr_adpsfc_prepbufr.py index 1d797e9..a415215 100755 --- a/ush/test/bufr_adpsfc_prepbufr.py +++ b/ush/test/bufr_adpsfc_prepbufr.py @@ -106,9 +106,11 @@ def logging(comm, level, message): # Call the logging method log_method(message) -def _make_description(mapping_path, update=False): +def _make_description(mapping_path, cycle_time, update=False): description = bufr.encoders.Description(mapping_path) + ReferenceTime = np.int64(calendar.timegm(time.strptime(str(int(cycle_time)), '%Y%m%d%H'))) + if update: # Define the variables to be added in a list of dictionaries variables = [ @@ -129,6 +131,8 @@ def _make_description(mapping_path, update=False): longName=var['longName'] ) + description.add_global(name='Reference_time', value=str(ReferenceTime)) + return description @@ -191,8 +195,10 @@ def create_obs_group(input_path, mapping_path, cycle_time, env): comm = bufr.mpi.Comm(env["comm_name"]) logging(comm, 'INFO', f'Make description and make obs') - description = _make_description(mapping_path, update=True) + container = _make_obs(comm, input_path, mapping_path, cycle_time) + description = _make_description(mapping_path, cycle_time, update=True) + # Gather data from all tasks into all tasks. Each task will have the complete record logging(comm, 'INFO', f'Gather data from all tasks into all tasks') @@ -211,7 +217,7 @@ def create_obs_file(input_path, mapping_path, output_path, cycle_time): container = _make_obs(comm, input_path, mapping_path, cycle_time) container.gather(comm) - description = _make_description(mapping_path, update=True) + description = _make_description(mapping_path, cycle_time, update=True) # Encode the data if comm.rank() == 0: From 9d0e653f325c60f4f845223fc44f5549f901311e Mon Sep 17 00:00:00 2001 From: Nicholas Esposito Date: Thu, 2 Jan 2025 13:36:38 -0500 Subject: [PATCH 62/72] subsets, input filename, global attributes, dateTime min --- ush/test/bufr_adpsfc_prepbufr.py | 5 ++-- ush/test/bufr_adpsfc_prepbufr_mapping.yaml | 15 ++++++------ .../bufr_bufr4backend_adpsfc_prepbufr.yaml | 2 +- ...ufr_bufr4backend_adpsfc_prepbufr_mpi0.yaml | 19 --------------- ...ufr_bufr4backend_adpsfc_prepbufr_mpi4.yaml | 2 +- .../bufr_script4backend_adpsfc_prepbufr.yaml | 2 +- ...r_script4backend_adpsfc_prepbufr_mpi0.yaml | 23 ------------------- ...r_script4backend_adpsfc_prepbufr_mpi4.yaml | 2 +- ush/test/encodeBufr_Nick.sh | 13 ++++++++--- 9 files changed, 24 insertions(+), 59 deletions(-) delete mode 100644 ush/test/bufr_bufr4backend_adpsfc_prepbufr_mpi0.yaml delete mode 100644 ush/test/bufr_script4backend_adpsfc_prepbufr_mpi0.yaml diff --git a/ush/test/bufr_adpsfc_prepbufr.py b/ush/test/bufr_adpsfc_prepbufr.py index a415215..0ea9088 100755 --- a/ush/test/bufr_adpsfc_prepbufr.py +++ b/ush/test/bufr_adpsfc_prepbufr.py @@ -131,7 +131,7 @@ def _make_description(mapping_path, cycle_time, update=False): longName=var['longName'] ) - description.add_global(name='Reference_time', value=str(ReferenceTime)) + description.add_global(name='datetimeReference', value=str(ReferenceTime)) return description @@ -171,7 +171,8 @@ def _make_obs(comm, input_path, mapping_path, cycle_time): otmct2 = np.array(otmct) cycleTimeSinceEpoch = np.int64(calendar.timegm(time.strptime(str(int(cycle_time)), '%Y%m%d%H'))) dateTime = Compute_dateTime(cycleTimeSinceEpoch, otmct2) - logging(comm, 'DEBUG', f'dateTime min/max = {dateTime.min()} {dateTime.max()}') + min_dateTime_ge_zero = min(x for x in dateTime if x > -1) + logging(comm, 'DEBUG', f'dateTime min/max = {min_dateTime_ge_zero} {dateTime.max()}') logging(comm, 'DEBUG', f'Make an array of 0s for MetaData/sequenceNumber') sequenceNum = np.zeros(lon.shape, dtype=np.int32) diff --git a/ush/test/bufr_adpsfc_prepbufr_mapping.yaml b/ush/test/bufr_adpsfc_prepbufr_mapping.yaml index dd99089..2307e5c 100755 --- a/ush/test/bufr_adpsfc_prepbufr_mapping.yaml +++ b/ush/test/bufr_adpsfc_prepbufr_mapping.yaml @@ -4,6 +4,9 @@ # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. bufr: + subsets: + - ADPSFC + variables: # ObsType observationType: @@ -66,11 +69,11 @@ bufr: encoder: globals: - - name: "data_format" + - name: "dataOriginalFormatSpec" type: string value: "prepbufr" - - name: "subsets" + - name: "platforms" type: string value: "ADPSFC" @@ -78,15 +81,11 @@ encoder: type: string value: "prepBUFR" - - name: "data_type" - type: string - value: "ADPSFC" - - - name: "data_description" + - name: "description" type: string value: "ADPSFC_prepbufr" - - name: "data_provider" + - name: "dataProviderOrigin" type: string value: "U.S. NOAA" diff --git a/ush/test/bufr_bufr4backend_adpsfc_prepbufr.yaml b/ush/test/bufr_bufr4backend_adpsfc_prepbufr.yaml index d00d47d..668b7d7 100644 --- a/ush/test/bufr_bufr4backend_adpsfc_prepbufr.yaml +++ b/ush/test/bufr_bufr4backend_adpsfc_prepbufr.yaml @@ -10,7 +10,7 @@ observations: obsdatain: engine: type: bufr - obsfile: "./testinput/2021080100/gdas.t00z.adpsfc.tm00.prepbufr" + obsfile: "./testinput/2021080100/gdas.t00z.prepbufr" mapping file: "./bufr_adpsfc_prepbufr_mapping.yaml" # category: ["goes-16"] obsdataout: diff --git a/ush/test/bufr_bufr4backend_adpsfc_prepbufr_mpi0.yaml b/ush/test/bufr_bufr4backend_adpsfc_prepbufr_mpi0.yaml deleted file mode 100644 index 29936e8..0000000 --- a/ush/test/bufr_bufr4backend_adpsfc_prepbufr_mpi0.yaml +++ /dev/null @@ -1,19 +0,0 @@ -time window: - begin: "2021-07-31T21:00:00Z" - end: "2021-08-01T03:00:00Z" - bound to include: begin - -observations: -- obs space: - name: "ADPSFC" - simulated variables: ['stationElevation','stationPressure'] - obsdatain: - engine: - type: bufr - obsfile: "./testinput/2021080100/gdas.t00z.adpsfc.tm00.prepbufr" - mapping file: "./bufr_adpsfc_prepbufr_mapping.yaml" - # category: ["goes-16"] - obsdataout: - engine: - type: H5File - obsfile: "./testoutput/2021080100/bufr4backend/gdas.t00z.adpsfc_prepbufr.tm00_mpi0.nc" diff --git a/ush/test/bufr_bufr4backend_adpsfc_prepbufr_mpi4.yaml b/ush/test/bufr_bufr4backend_adpsfc_prepbufr_mpi4.yaml index ca03b28..db82329 100644 --- a/ush/test/bufr_bufr4backend_adpsfc_prepbufr_mpi4.yaml +++ b/ush/test/bufr_bufr4backend_adpsfc_prepbufr_mpi4.yaml @@ -10,7 +10,7 @@ observations: obsdatain: engine: type: bufr - obsfile: "./testinput/2021080100/gdas.t00z.adpsfc.tm00.prepbufr" + obsfile: "./testinput/2021080100/gdas.t00z.prepbufr" mapping file: "./bufr_adpsfc_prepbufr_mapping.yaml" # category: ["goes-16"] obsdataout: diff --git a/ush/test/bufr_script4backend_adpsfc_prepbufr.yaml b/ush/test/bufr_script4backend_adpsfc_prepbufr.yaml index a0035cc..88253fc 100644 --- a/ush/test/bufr_script4backend_adpsfc_prepbufr.yaml +++ b/ush/test/bufr_script4backend_adpsfc_prepbufr.yaml @@ -14,7 +14,7 @@ observations: type: script script file: "bufr_adpsfc_prepbufr.py" args: - input_path: "./testinput/2021080100/gdas.t00z.adpsfc.tm00.prepbufr" + input_path: "./testinput/2021080100/gdas.t00z.prepbufr" mapping_path: "./bufr_adpsfc_prepbufr_mapping.yaml" cycle_time: "2021080100" obsdataout: diff --git a/ush/test/bufr_script4backend_adpsfc_prepbufr_mpi0.yaml b/ush/test/bufr_script4backend_adpsfc_prepbufr_mpi0.yaml deleted file mode 100644 index b5741f8..0000000 --- a/ush/test/bufr_script4backend_adpsfc_prepbufr_mpi0.yaml +++ /dev/null @@ -1,23 +0,0 @@ -time window: - begin: "2021-07-31T21:00:00Z" - end: "2021-08-01T03:00:00Z" - bound to include: begin - -observations: -- obs space: - name: "ADPSFC" - observed variables: ['stationElevation','stationPressure'] - derived variables: ['stationElevation','stationPressure'] - simulated variables: ['stationElevation','stationPressure'] - obsdatain: - engine: - type: script - script file: "bufr_adpsfc_prepbufr.py" - args: - input_path: "testinput/2021080100/gdas.t00z.adpsfc.tm00.prepbufr" - mapping_path: "./bufr_adpsfc_prepbufr_mapping.yaml" - cycle_time: "2021080100" - obsdataout: - engine: - type: H5File - obsfile: "./testoutput/2021080100/script4backend/gdas.t00z.adpsfc_prepbufr.tm00_mpi0.nc" diff --git a/ush/test/bufr_script4backend_adpsfc_prepbufr_mpi4.yaml b/ush/test/bufr_script4backend_adpsfc_prepbufr_mpi4.yaml index 1ddd60c..dad81cf 100644 --- a/ush/test/bufr_script4backend_adpsfc_prepbufr_mpi4.yaml +++ b/ush/test/bufr_script4backend_adpsfc_prepbufr_mpi4.yaml @@ -14,7 +14,7 @@ observations: type: script script file: "bufr_adpsfc_prepbufr.py" args: - input_path: "testinput/2021080100/gdas.t00z.adpsfc.tm00.prepbufr" + input_path: "testinput/2021080100/gdas.t00z.prepbufr" mapping_path: "./bufr_adpsfc_prepbufr_mapping.yaml" cycle_time: "2021080100" obsdataout: diff --git a/ush/test/encodeBufr_Nick.sh b/ush/test/encodeBufr_Nick.sh index 7abbc4c..c1cab1a 100755 --- a/ush/test/encodeBufr_Nick.sh +++ b/ush/test/encodeBufr_Nick.sh @@ -154,14 +154,21 @@ mkdir -p -m770 ${out_dir} || { echo "Error creating output directory: ${out_dir} # =============== # Set file paths # =============== -ioda_config_yaml="${work_dir}/bufr_${mode}_${obstype}_mpi${nproc}.yaml" +#ioda_config_yaml="${work_dir}/bufr_${mode}_${obstype}_mpi${nproc}.yaml" mapping_file="${work_dir}/bufr_${obstype}_mapping.yaml" #input_file="${in_dir}/gdas.t${h2}z.${bufrtype}.tm00.bufr_d" -input_file="${in_dir}/gdas.t${h2}z.${bufrtype}.tm00.prepbufr" +#input_file="${in_dir}/gdas.t${h2}z.${bufrtype}.tm00.prepbufr" +input_file="${in_dir}/gdas.t${h2}z.prepbufr" if [[ "${split_by_category}" = "true" ]]; then output_file="${out_dir}/gdas.t${h2}z.${bufrtype}_${sensor}_{splits/satId}.tm00.nc" else - output_file="${out_dir}/gdas.t${h2}z.${bufrtype}_prepbufr.tm00_mpi${nproc}.nc" + if [[ ${nproc} -ge 2 ]] ; then + ioda_config_yaml="${work_dir}/bufr_${mode}_${obstype}_mpi${nproc}.yaml" + output_file="${out_dir}/gdas.t${h2}z.${bufrtype}_prepbufr.tm00_mpi${nproc}.nc" + else + ioda_config_yaml="${work_dir}/bufr_${mode}_${obstype}.yaml" + output_file="${out_dir}/gdas.t${h2}z.${bufrtype}_prepbufr.tm00.nc" + fi fi if [[ ! -f "$input_file" ]]; then From bdeab474bea8d0f9605f36947503d065eb62cae3 Mon Sep 17 00:00:00 2001 From: Nicholas Esposito Date: Thu, 2 Jan 2025 13:41:08 -0500 Subject: [PATCH 63/72] rm some comments --- ush/test/bufr_bufr4backend_adpsfc_prepbufr.yaml | 1 - ush/test/bufr_bufr4backend_adpsfc_prepbufr_mpi4.yaml | 1 - 2 files changed, 2 deletions(-) diff --git a/ush/test/bufr_bufr4backend_adpsfc_prepbufr.yaml b/ush/test/bufr_bufr4backend_adpsfc_prepbufr.yaml index 668b7d7..5d4fea9 100644 --- a/ush/test/bufr_bufr4backend_adpsfc_prepbufr.yaml +++ b/ush/test/bufr_bufr4backend_adpsfc_prepbufr.yaml @@ -12,7 +12,6 @@ observations: type: bufr obsfile: "./testinput/2021080100/gdas.t00z.prepbufr" mapping file: "./bufr_adpsfc_prepbufr_mapping.yaml" - # category: ["goes-16"] obsdataout: engine: type: H5File diff --git a/ush/test/bufr_bufr4backend_adpsfc_prepbufr_mpi4.yaml b/ush/test/bufr_bufr4backend_adpsfc_prepbufr_mpi4.yaml index db82329..9eac61e 100644 --- a/ush/test/bufr_bufr4backend_adpsfc_prepbufr_mpi4.yaml +++ b/ush/test/bufr_bufr4backend_adpsfc_prepbufr_mpi4.yaml @@ -12,7 +12,6 @@ observations: type: bufr obsfile: "./testinput/2021080100/gdas.t00z.prepbufr" mapping file: "./bufr_adpsfc_prepbufr_mapping.yaml" - # category: ["goes-16"] obsdataout: engine: type: H5File From b3e6a7aebad562ff7cd7e4aa447061a496e20767 Mon Sep 17 00:00:00 2001 From: Nicholas Esposito Date: Wed, 15 Jan 2025 16:28:12 -0500 Subject: [PATCH 64/72] update files in config --- .../bufr_bufr4backend_adpsfc_prepbufr.yaml | 3 +-- ...ufr_bufr4backend_adpsfc_prepbufr_mpi0.yaml | 19 --------------- ...ufr_bufr4backend_adpsfc_prepbufr_mpi4.yaml | 3 +-- .../bufr_script4backend_adpsfc_prepbufr.yaml | 2 +- ...r_script4backend_adpsfc_prepbufr_mpi0.yaml | 23 ------------------- ...r_script4backend_adpsfc_prepbufr_mpi4.yaml | 2 +- dump/mapping/bufr_adpsfc_prepbufr.py | 15 ++++++++---- .../mapping/bufr_adpsfc_prepbufr_mapping.yaml | 15 ++++++------ 8 files changed, 22 insertions(+), 60 deletions(-) delete mode 100644 dump/config/bufr_bufr4backend_adpsfc_prepbufr_mpi0.yaml delete mode 100644 dump/config/bufr_script4backend_adpsfc_prepbufr_mpi0.yaml diff --git a/dump/config/bufr_bufr4backend_adpsfc_prepbufr.yaml b/dump/config/bufr_bufr4backend_adpsfc_prepbufr.yaml index d00d47d..5d4fea9 100644 --- a/dump/config/bufr_bufr4backend_adpsfc_prepbufr.yaml +++ b/dump/config/bufr_bufr4backend_adpsfc_prepbufr.yaml @@ -10,9 +10,8 @@ observations: obsdatain: engine: type: bufr - obsfile: "./testinput/2021080100/gdas.t00z.adpsfc.tm00.prepbufr" + obsfile: "./testinput/2021080100/gdas.t00z.prepbufr" mapping file: "./bufr_adpsfc_prepbufr_mapping.yaml" - # category: ["goes-16"] obsdataout: engine: type: H5File diff --git a/dump/config/bufr_bufr4backend_adpsfc_prepbufr_mpi0.yaml b/dump/config/bufr_bufr4backend_adpsfc_prepbufr_mpi0.yaml deleted file mode 100644 index 29936e8..0000000 --- a/dump/config/bufr_bufr4backend_adpsfc_prepbufr_mpi0.yaml +++ /dev/null @@ -1,19 +0,0 @@ -time window: - begin: "2021-07-31T21:00:00Z" - end: "2021-08-01T03:00:00Z" - bound to include: begin - -observations: -- obs space: - name: "ADPSFC" - simulated variables: ['stationElevation','stationPressure'] - obsdatain: - engine: - type: bufr - obsfile: "./testinput/2021080100/gdas.t00z.adpsfc.tm00.prepbufr" - mapping file: "./bufr_adpsfc_prepbufr_mapping.yaml" - # category: ["goes-16"] - obsdataout: - engine: - type: H5File - obsfile: "./testoutput/2021080100/bufr4backend/gdas.t00z.adpsfc_prepbufr.tm00_mpi0.nc" diff --git a/dump/config/bufr_bufr4backend_adpsfc_prepbufr_mpi4.yaml b/dump/config/bufr_bufr4backend_adpsfc_prepbufr_mpi4.yaml index ca03b28..9eac61e 100644 --- a/dump/config/bufr_bufr4backend_adpsfc_prepbufr_mpi4.yaml +++ b/dump/config/bufr_bufr4backend_adpsfc_prepbufr_mpi4.yaml @@ -10,9 +10,8 @@ observations: obsdatain: engine: type: bufr - obsfile: "./testinput/2021080100/gdas.t00z.adpsfc.tm00.prepbufr" + obsfile: "./testinput/2021080100/gdas.t00z.prepbufr" mapping file: "./bufr_adpsfc_prepbufr_mapping.yaml" - # category: ["goes-16"] obsdataout: engine: type: H5File diff --git a/dump/config/bufr_script4backend_adpsfc_prepbufr.yaml b/dump/config/bufr_script4backend_adpsfc_prepbufr.yaml index a0035cc..88253fc 100644 --- a/dump/config/bufr_script4backend_adpsfc_prepbufr.yaml +++ b/dump/config/bufr_script4backend_adpsfc_prepbufr.yaml @@ -14,7 +14,7 @@ observations: type: script script file: "bufr_adpsfc_prepbufr.py" args: - input_path: "./testinput/2021080100/gdas.t00z.adpsfc.tm00.prepbufr" + input_path: "./testinput/2021080100/gdas.t00z.prepbufr" mapping_path: "./bufr_adpsfc_prepbufr_mapping.yaml" cycle_time: "2021080100" obsdataout: diff --git a/dump/config/bufr_script4backend_adpsfc_prepbufr_mpi0.yaml b/dump/config/bufr_script4backend_adpsfc_prepbufr_mpi0.yaml deleted file mode 100644 index b5741f8..0000000 --- a/dump/config/bufr_script4backend_adpsfc_prepbufr_mpi0.yaml +++ /dev/null @@ -1,23 +0,0 @@ -time window: - begin: "2021-07-31T21:00:00Z" - end: "2021-08-01T03:00:00Z" - bound to include: begin - -observations: -- obs space: - name: "ADPSFC" - observed variables: ['stationElevation','stationPressure'] - derived variables: ['stationElevation','stationPressure'] - simulated variables: ['stationElevation','stationPressure'] - obsdatain: - engine: - type: script - script file: "bufr_adpsfc_prepbufr.py" - args: - input_path: "testinput/2021080100/gdas.t00z.adpsfc.tm00.prepbufr" - mapping_path: "./bufr_adpsfc_prepbufr_mapping.yaml" - cycle_time: "2021080100" - obsdataout: - engine: - type: H5File - obsfile: "./testoutput/2021080100/script4backend/gdas.t00z.adpsfc_prepbufr.tm00_mpi0.nc" diff --git a/dump/config/bufr_script4backend_adpsfc_prepbufr_mpi4.yaml b/dump/config/bufr_script4backend_adpsfc_prepbufr_mpi4.yaml index 1ddd60c..dad81cf 100644 --- a/dump/config/bufr_script4backend_adpsfc_prepbufr_mpi4.yaml +++ b/dump/config/bufr_script4backend_adpsfc_prepbufr_mpi4.yaml @@ -14,7 +14,7 @@ observations: type: script script file: "bufr_adpsfc_prepbufr.py" args: - input_path: "testinput/2021080100/gdas.t00z.adpsfc.tm00.prepbufr" + input_path: "testinput/2021080100/gdas.t00z.prepbufr" mapping_path: "./bufr_adpsfc_prepbufr_mapping.yaml" cycle_time: "2021080100" obsdataout: diff --git a/dump/mapping/bufr_adpsfc_prepbufr.py b/dump/mapping/bufr_adpsfc_prepbufr.py index 1d797e9..0ea9088 100755 --- a/dump/mapping/bufr_adpsfc_prepbufr.py +++ b/dump/mapping/bufr_adpsfc_prepbufr.py @@ -106,9 +106,11 @@ def logging(comm, level, message): # Call the logging method log_method(message) -def _make_description(mapping_path, update=False): +def _make_description(mapping_path, cycle_time, update=False): description = bufr.encoders.Description(mapping_path) + ReferenceTime = np.int64(calendar.timegm(time.strptime(str(int(cycle_time)), '%Y%m%d%H'))) + if update: # Define the variables to be added in a list of dictionaries variables = [ @@ -129,6 +131,8 @@ def _make_description(mapping_path, update=False): longName=var['longName'] ) + description.add_global(name='datetimeReference', value=str(ReferenceTime)) + return description @@ -167,7 +171,8 @@ def _make_obs(comm, input_path, mapping_path, cycle_time): otmct2 = np.array(otmct) cycleTimeSinceEpoch = np.int64(calendar.timegm(time.strptime(str(int(cycle_time)), '%Y%m%d%H'))) dateTime = Compute_dateTime(cycleTimeSinceEpoch, otmct2) - logging(comm, 'DEBUG', f'dateTime min/max = {dateTime.min()} {dateTime.max()}') + min_dateTime_ge_zero = min(x for x in dateTime if x > -1) + logging(comm, 'DEBUG', f'dateTime min/max = {min_dateTime_ge_zero} {dateTime.max()}') logging(comm, 'DEBUG', f'Make an array of 0s for MetaData/sequenceNumber') sequenceNum = np.zeros(lon.shape, dtype=np.int32) @@ -191,8 +196,10 @@ def create_obs_group(input_path, mapping_path, cycle_time, env): comm = bufr.mpi.Comm(env["comm_name"]) logging(comm, 'INFO', f'Make description and make obs') - description = _make_description(mapping_path, update=True) + container = _make_obs(comm, input_path, mapping_path, cycle_time) + description = _make_description(mapping_path, cycle_time, update=True) + # Gather data from all tasks into all tasks. Each task will have the complete record logging(comm, 'INFO', f'Gather data from all tasks into all tasks') @@ -211,7 +218,7 @@ def create_obs_file(input_path, mapping_path, output_path, cycle_time): container = _make_obs(comm, input_path, mapping_path, cycle_time) container.gather(comm) - description = _make_description(mapping_path, update=True) + description = _make_description(mapping_path, cycle_time, update=True) # Encode the data if comm.rank() == 0: diff --git a/dump/mapping/bufr_adpsfc_prepbufr_mapping.yaml b/dump/mapping/bufr_adpsfc_prepbufr_mapping.yaml index dd99089..2307e5c 100755 --- a/dump/mapping/bufr_adpsfc_prepbufr_mapping.yaml +++ b/dump/mapping/bufr_adpsfc_prepbufr_mapping.yaml @@ -4,6 +4,9 @@ # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. bufr: + subsets: + - ADPSFC + variables: # ObsType observationType: @@ -66,11 +69,11 @@ bufr: encoder: globals: - - name: "data_format" + - name: "dataOriginalFormatSpec" type: string value: "prepbufr" - - name: "subsets" + - name: "platforms" type: string value: "ADPSFC" @@ -78,15 +81,11 @@ encoder: type: string value: "prepBUFR" - - name: "data_type" - type: string - value: "ADPSFC" - - - name: "data_description" + - name: "description" type: string value: "ADPSFC_prepbufr" - - name: "data_provider" + - name: "dataProviderOrigin" type: string value: "U.S. NOAA" From c0aec918e9a0bbadac68ca500f7255e84d63210d Mon Sep 17 00:00:00 2001 From: Nicholas Esposito Date: Wed, 15 Jan 2025 16:30:20 -0500 Subject: [PATCH 65/72] rm testdir files --- ush/test/bufr_adpsfc_prepbufr.py | 253 ------------------ ush/test/bufr_adpsfc_prepbufr_mapping.yaml | 186 ------------- .../bufr_bufr4backend_adpsfc_prepbufr.yaml | 18 -- ...ufr_bufr4backend_adpsfc_prepbufr_mpi4.yaml | 18 -- .../bufr_script4backend_adpsfc_prepbufr.yaml | 23 -- ...r_script4backend_adpsfc_prepbufr_mpi4.yaml | 23 -- 6 files changed, 521 deletions(-) delete mode 100755 ush/test/bufr_adpsfc_prepbufr.py delete mode 100755 ush/test/bufr_adpsfc_prepbufr_mapping.yaml delete mode 100644 ush/test/bufr_bufr4backend_adpsfc_prepbufr.yaml delete mode 100644 ush/test/bufr_bufr4backend_adpsfc_prepbufr_mpi4.yaml delete mode 100644 ush/test/bufr_script4backend_adpsfc_prepbufr.yaml delete mode 100644 ush/test/bufr_script4backend_adpsfc_prepbufr_mpi4.yaml diff --git a/ush/test/bufr_adpsfc_prepbufr.py b/ush/test/bufr_adpsfc_prepbufr.py deleted file mode 100755 index 0ea9088..0000000 --- a/ush/test/bufr_adpsfc_prepbufr.py +++ /dev/null @@ -1,253 +0,0 @@ -#!/usr/bin/env python3 -import os -import sys -import bufr -import argparse -import copy -import numpy as np -import numpy.ma as ma -import math -import calendar -import time -from datetime import datetime -from pyioda.ioda.Engines.Bufr import Encoder as iodaEncoder -from bufr.encoders.netcdf import Encoder as netcdfEncoder -from wxflow import Logger - -# Initialize Logger -# Get log level from the environment variable, default to 'INFO it not set -log_level = os.getenv('LOG_LEVEL', 'INFO') -logger = Logger('BUFR2IODA_adpsfc_prepbufr.py', level=log_level, colored_log=False) - -def Compute_dateTime(cycleTimeSinceEpoch, dhr): - """ - Compute dateTime using the cycleTimeSinceEpoch and Cycle Time - minus Cycle Time - - Parameters: - cycleTimeSinceEpoch: Time of cycle in Epoch Time - dhr: Observation Time Minus Cycle Time - - Returns: - Masked array of dateTime values - """ - - int64_fill_value = np.int64(0) - - dateTime = np.zeros(dhr.shape, dtype=np.int64) - for i in range(len(dateTime)): - if ma.is_masked(dhr[i]): - continue - else: - dateTime[i] = np.int64(dhr[i]*3600) + cycleTimeSinceEpoch - - dateTime = ma.array(dateTime) - dateTime = ma.masked_values(dateTime, int64_fill_value) - - return dateTime - - -def logging(comm, level, message): - """ - Logs a message to the console or log file, based on the specified logging level. - - This function ensures that logging is only performed by the root process (`rank 0`) - in a distributed computing environment. The function maps the logging level to - appropriate logger methods and defaults to the 'INFO' level if an invalid level is provided. - - Parameters: - comm: object - The communicator object, typically from a distributed computing framework - (e.g., MPI). It must have a `rank()` method to determine the process rank. - level: str - The logging level as a string. Supported levels are: - - 'DEBUG' - - 'INFO' - - 'WARNING' - - 'ERROR' - - 'CRITICAL' - If an invalid level is provided, a warning will be logged, and the level - will default to 'INFO'. - message: str - The message to be logged. - - Behavior: - - Logs messages only on the root process (`comm.rank() == 0`). - - Maps the provided logging level to a method of the logger object. - - Defaults to 'INFO' and logs a warning if an invalid logging level is given. - - Supports standard logging levels for granular control over log verbosity. - - Example: - >>> logging(comm, 'DEBUG', 'This is a debug message.') - >>> logging(comm, 'ERROR', 'An error occurred!') - - Notes: - - Ensure that a global `logger` object is configured before using this function. - - The `comm` object should conform to MPI-like conventions (e.g., `rank()` method). - """ - - if comm.rank() == 0: - # Define a dictionary to map levels to logger methods - log_methods = { - 'DEBUG': logger.debug, - 'INFO': logger.info, - 'WARNING': logger.warning, - 'ERROR': logger.error, - 'CRITICAL': logger.critical, - } - - # Get the appropriate logging method, default to 'INFO' - log_method = log_methods.get(level.upper(), logger.info) - - if log_method == logger.info and level.upper() not in log_methods: - # Log a warning if the level is invalid - logger.warning(f'log level = {level}: not a valid level --> set to INFO') - - # Call the logging method - log_method(message) - -def _make_description(mapping_path, cycle_time, update=False): - description = bufr.encoders.Description(mapping_path) - - ReferenceTime = np.int64(calendar.timegm(time.strptime(str(int(cycle_time)), '%Y%m%d%H'))) - - if update: - # Define the variables to be added in a list of dictionaries - variables = [ - { - 'name': 'MetaData/sequenceNumber', - 'source': 'variables/sequenceNumber', - 'units': '1', - 'longName': 'Sequence Number (Obs Subtype)', - } - ] - - # Loop through each variable and add it to the description - for var in variables: - description.add_variable( - name=var['name'], - source=var['source'], - units=var['units'], - longName=var['longName'] - ) - - description.add_global(name='datetimeReference', value=str(ReferenceTime)) - - return description - - -def _make_obs(comm, input_path, mapping_path, cycle_time): - """ - Create the ioda adpsfc prepbufr observations: - - reads values - - adds sequenceNum - - Parameters - ---------- - comm: object - The communicator object (e.g., MPI) - input_path: str - The input bufr file - mapping_path: str - The input bufr2ioda mapping file - cycle_time: str - The cycle in YYYYMMDDHH format - """ - - # Get container from mapping file first - logging(comm, 'INFO', 'Get container from bufr') - container = bufr.Parser(input_path, mapping_path).parse(comm) - - logging(comm, 'DEBUG', f'container list (original): {container.list()}') - logging(comm, 'DEBUG', f'Change longitude range from [0,360] to [-180,180]') - lon = container.get('variables/longitude') - lon_paths = container.get_paths('variables/longitude') - lon[lon>180] -= 360 - lon = ma.round(lon, decimals=2) - - logging(comm, 'DEBUG', f'Do DateTime calculation') - otmct = container.get('variables/obsTimeMinusCycleTime') - otmct_paths = container.get_paths('variables/obsTimeMinusCycleTime') - otmct2 = np.array(otmct) - cycleTimeSinceEpoch = np.int64(calendar.timegm(time.strptime(str(int(cycle_time)), '%Y%m%d%H'))) - dateTime = Compute_dateTime(cycleTimeSinceEpoch, otmct2) - min_dateTime_ge_zero = min(x for x in dateTime if x > -1) - logging(comm, 'DEBUG', f'dateTime min/max = {min_dateTime_ge_zero} {dateTime.max()}') - - logging(comm, 'DEBUG', f'Make an array of 0s for MetaData/sequenceNumber') - sequenceNum = np.zeros(lon.shape, dtype=np.int32) - logging(comm, 'DEBUG', f' sequenceNummin/max = {sequenceNum.min()} {sequenceNum.max()}') - - logging(comm, 'DEBUG', f'Update variables in container') - container.replace('variables/longitude', lon) - container.replace('variables/timestamp', dateTime) - - logging(comm, 'DEBUG', f'Add variables to container') - container.add('variables/sequenceNumber', sequenceNum, lon_paths) - - # Check - logging(comm, 'DEBUG', f'container list (updated): {container.list()}') - - return container - - -def create_obs_group(input_path, mapping_path, cycle_time, env): - - comm = bufr.mpi.Comm(env["comm_name"]) - - logging(comm, 'INFO', f'Make description and make obs') - - container = _make_obs(comm, input_path, mapping_path, cycle_time) - description = _make_description(mapping_path, cycle_time, update=True) - - - # Gather data from all tasks into all tasks. Each task will have the complete record - logging(comm, 'INFO', f'Gather data from all tasks into all tasks') - container.all_gather(comm) - - logging(comm, 'INFO', f'Encode the data') - data = next(iter(iodaEncoder(description).encode(container).values())) - - logging(comm, 'INFO', f'Return the encoded data.') - return data - - -def create_obs_file(input_path, mapping_path, output_path, cycle_time): - - comm = bufr.mpi.Comm("world") - container = _make_obs(comm, input_path, mapping_path, cycle_time) - container.gather(comm) - - description = _make_description(mapping_path, cycle_time, update=True) - - # Encode the data - if comm.rank() == 0: - netcdfEncoder(description).encode(container, output_path) - - logging(comm, 'INFO', f'Return the encoded data') - - -if __name__ == '__main__': - start_time = time.time() - - bufr.mpi.App(sys.argv) - comm = bufr.mpi.Comm("world") - - # Required input arguments as positional arguments - parser = argparse.ArgumentParser(description="Convert BUFR to NetCDF using a mapping file.") - parser.add_argument('input', type=str, help='Input BUFR file') - parser.add_argument('mapping', type=str, help='BUFR2IODA Mapping File') - parser.add_argument('output', type=str, help='Output NetCDF file') - parser.add_argument('cycle_time', type=str, help='cycle time in YYYYMMDDHH format') - - args = parser.parse_args() - infile = args.input - mapping = args.mapping - output = args.output - cycle_time = args.cycle_time - - create_obs_file(infile, mapping, output, cycle_time) - - end_time = time.time() - running_time = end_time - start_time - logging(comm, 'INFO', f'Total running time: {running_time}') diff --git a/ush/test/bufr_adpsfc_prepbufr_mapping.yaml b/ush/test/bufr_adpsfc_prepbufr_mapping.yaml deleted file mode 100755 index 2307e5c..0000000 --- a/ush/test/bufr_adpsfc_prepbufr_mapping.yaml +++ /dev/null @@ -1,186 +0,0 @@ -# (C) Copyright 2023 NOAA/NWS/NCEP/EMC -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - -bufr: - subsets: - - ADPSFC - - variables: - # ObsType - observationType: - query: "*/TYP" - - # MetaData - timestamp: - timeoffset: - timeOffset: "*/DHR" - transforms: - - scale: 3600 - referenceTime: "2021-08-01T00:00:00Z" - prepbufrDataLevelCategory: - query: "*/CAT" - obsTimeMinusCycleTime: - query: "*/DHR" - longitude: - query: "*/XOB" - latitude: - query: "*/YOB" - stationIdentification: - query: "*/SID" - pressure: - query: "*/P___INFO/P__EVENT{1}/POB" - transforms: - - scale: 100 - height: - query: "*/Z___INFO/Z__EVENT{1}/ZOB" - type: float - stationElevation: - query: "*/ELV" - type: float - - # ObsValue - stationPressureObsValue: - query: "*/P___INFO/P__EVENT{1}/POB" - transforms: - - scale: 100 - heightObsValue: - query: "*/Z___INFO/Z__EVENT{1}/ZOB" - type: float - stationElevationObsValue: - query: "*/ELV" - type: float - - # QualityMarker - stationPressureQualityMarker: - query: "*/P___INFO/P__EVENT{1}/PQM" - heightQualityMarker: - query: "*/Z___INFO/Z__EVENT{1}/ZQM" - stationElevationQualityMarker: - query: "*/Z___INFO/Z__EVENT{1}/ZQM" - - # ObsError - stationPressureObsError: - query: "*/P___INFO/P__BACKG/POE" - transforms: - - scale: 100 - -encoder: - - globals: - - name: "dataOriginalFormatSpec" - type: string - value: "prepbufr" - - - name: "platforms" - type: string - value: "ADPSFC" - - - name: "source" - type: string - value: "prepBUFR" - - - name: "description" - type: string - value: "ADPSFC_prepbufr" - - - name: "dataProviderOrigin" - type: string - value: "U.S. NOAA" - - variables: - - # ObsType - - name: "ObsType/stationPressure" - source: variables/observationType - longName: "Station Pressure ObsType" - - - name: "ObsType/stationElevation" - source: variables/observationType - longName: "Station Elevation ObsType" - - - name: "ObsType/height" - source: variables/observationType - longName: "Height ObsType" - - # MetaData - - name: "MetaData/prepbufrDataLevelCategory" - source: variables/prepbufrDataLevelCategory - units: '1' - longName: "Prepbufr Data Level Category" - - - name: "MetaData/dateTime" - source: variables/timestamp - units: 'seconds since 1970-01-01T00:00:00Z' - longName: 'dateTime' - - - name: "MetaData/latitude" - source: variables/latitude - longName: "Latitude" - units: "degree_north" - range: [-90, 90] - - - name: "MetaData/longitude" - source: variables/longitude - longName: "Longitude" - units: "degree_east" - #range: [0, 360] - - # - name: "MetaData/sequenceNumber" - #source: variables/sequenceNumber - #longName: "Sequence Number (Obs Subtype)" - - - name: "MetaData/stationIdentification" - source: variables/stationIdentification - longName: "Station Identification" - - - name: "MetaData/pressure" - source: variables/pressure - longName: "Pressure" - - - name: "MetaData/height" - source: variables/height - longName: "Height" - units: "m" - - - name: "MetaData/stationElevation" - source: variables/stationElevation - longName: "Station Elevation" - units: "m" - - # ObsValue - - name: "ObsValue/stationPressure" - source: variables/stationPressureObsValue - longName: "Station Pressure" - units: "Pa" - - - name: "ObsValue/stationElevation" - source: variables/stationElevationObsValue - longName: "Station Pressure" - units: "m" - - - name: "ObsValue/height" - source: variables/heightObsValue - longName: "height" - units: "m" - - # QualityMarker - - name: "QualityMarker/stationPressure" - source: variables/stationPressureQualityMarker - longName: "Station Pressure Quality Marker" - - - name: "QualityMarker/stationElevation" - source: variables/heightQualityMarker - longName: "StationElevation Quality Marker" - - - name: "QualityMarker/height" - source: variables/heightQualityMarker - longName: "Height Quality Marker" - - # ObsError - - name: "ObsError/stationPressure" - source: variables/stationPressureObsError - longName: "Station Pressure Error" - units: "Pa" - diff --git a/ush/test/bufr_bufr4backend_adpsfc_prepbufr.yaml b/ush/test/bufr_bufr4backend_adpsfc_prepbufr.yaml deleted file mode 100644 index 5d4fea9..0000000 --- a/ush/test/bufr_bufr4backend_adpsfc_prepbufr.yaml +++ /dev/null @@ -1,18 +0,0 @@ -time window: - begin: "2021-07-31T21:00:00Z" - end: "2021-08-01T03:00:00Z" - bound to include: begin - -observations: -- obs space: - name: "ADPSFC" - simulated variables: ['stationElevation','stationPressure'] - obsdatain: - engine: - type: bufr - obsfile: "./testinput/2021080100/gdas.t00z.prepbufr" - mapping file: "./bufr_adpsfc_prepbufr_mapping.yaml" - obsdataout: - engine: - type: H5File - obsfile: "./testoutput/2021080100/bufr4backend/gdas.t00z.adpsfc_prepbufr.tm00.nc" diff --git a/ush/test/bufr_bufr4backend_adpsfc_prepbufr_mpi4.yaml b/ush/test/bufr_bufr4backend_adpsfc_prepbufr_mpi4.yaml deleted file mode 100644 index 9eac61e..0000000 --- a/ush/test/bufr_bufr4backend_adpsfc_prepbufr_mpi4.yaml +++ /dev/null @@ -1,18 +0,0 @@ -time window: - begin: "2021-07-31T21:00:00Z" - end: "2021-08-01T03:00:00Z" - bound to include: begin - -observations: -- obs space: - name: "ADPSFC" - simulated variables: ['stationElevation','stationPressure'] - obsdatain: - engine: - type: bufr - obsfile: "./testinput/2021080100/gdas.t00z.prepbufr" - mapping file: "./bufr_adpsfc_prepbufr_mapping.yaml" - obsdataout: - engine: - type: H5File - obsfile: "./testoutput/2021080100/bufr4backend/gdas.t00z.adpsfc_prepbufr.tm00_mpi4.nc" diff --git a/ush/test/bufr_script4backend_adpsfc_prepbufr.yaml b/ush/test/bufr_script4backend_adpsfc_prepbufr.yaml deleted file mode 100644 index 88253fc..0000000 --- a/ush/test/bufr_script4backend_adpsfc_prepbufr.yaml +++ /dev/null @@ -1,23 +0,0 @@ -time window: - begin: "2021-07-31T21:00:00Z" - end: "2021-08-01T03:00:00Z" - bound to include: begin - -observations: -- obs space: - name: "ADPSFC" - observed variables: ['stationElevation','stationPressure'] - derived variables: ['stationElevation','stationPressure'] - simulated variables: ['stationElevation','stationPressure'] - obsdatain: - engine: - type: script - script file: "bufr_adpsfc_prepbufr.py" - args: - input_path: "./testinput/2021080100/gdas.t00z.prepbufr" - mapping_path: "./bufr_adpsfc_prepbufr_mapping.yaml" - cycle_time: "2021080100" - obsdataout: - engine: - type: H5File - obsfile: "./testoutput/2021080100/script4backend/gdas.t00z.adpsfc_prepbufr.tm00.nc" diff --git a/ush/test/bufr_script4backend_adpsfc_prepbufr_mpi4.yaml b/ush/test/bufr_script4backend_adpsfc_prepbufr_mpi4.yaml deleted file mode 100644 index dad81cf..0000000 --- a/ush/test/bufr_script4backend_adpsfc_prepbufr_mpi4.yaml +++ /dev/null @@ -1,23 +0,0 @@ -time window: - begin: "2021-07-31T21:00:00Z" - end: "2021-08-01T03:00:00Z" - bound to include: begin - -observations: -- obs space: - name: "ADPSFC" - observed variables: ['stationElevation','stationPressure'] - derived variables: ['stationElevation','stationPressure'] - simulated variables: ['stationElevation','stationPressure'] - obsdatain: - engine: - type: script - script file: "bufr_adpsfc_prepbufr.py" - args: - input_path: "testinput/2021080100/gdas.t00z.prepbufr" - mapping_path: "./bufr_adpsfc_prepbufr_mapping.yaml" - cycle_time: "2021080100" - obsdataout: - engine: - type: H5File - obsfile: "./testoutput/2021080100/script4backend/gdas.t00z.adpsfc_prepbufr.tm00_mpi4.nc" From 0110977410de63723a7d93dff7745d86b46db086 Mon Sep 17 00:00:00 2001 From: Nicholas Esposito Date: Wed, 15 Jan 2025 16:32:01 -0500 Subject: [PATCH 66/72] rm encodeBufr_Nick.sh --- ush/test/encodeBufr_Nick.sh | 228 ------------------------------------ 1 file changed, 228 deletions(-) delete mode 100755 ush/test/encodeBufr_Nick.sh diff --git a/ush/test/encodeBufr_Nick.sh b/ush/test/encodeBufr_Nick.sh deleted file mode 100755 index c1cab1a..0000000 --- a/ush/test/encodeBufr_Nick.sh +++ /dev/null @@ -1,228 +0,0 @@ -#!/bin/bash - -# ==================== -# Get input arguments -# ==================== -obsforge_dir="${1:-/scratch1/NCEPDEV/da/Emily.Liu/EMC-obsForge/obsForge}" -cycle="${2:-2021080100}" -bufrtype="${3:-satwnd}" -obstype="${4:-satwnd_amv_goes}" -sensor="${5:-abi}" -split_by_category="${6:-true}" -mode="${7:-script4backend}" -nproc="${8:-4}" - -# ========================== -# Function to display usage -# ========================== -usage() { - echo "Usage: $0 " - echo " : root directory of obsForge build" - echo " : cycle time (e.g., 2021080100)" - echo " : BUFR dump type to process (e.g., satwnd, atms, cris, sfcsno)" - echo " : observation type to create (e.g., satwnd_amv_goes, atms, cris, sfcsno )" - echo " : sensor (e.g., abi, atms); for non-satellite dta, sensor is usually obstype (e.g., sfcsno)" - echo " : split the output file into multiple files based on category (false or true)" - echo " : mode of operation (e.g., bufr4backend, script4backend, bufr2netcdf, script2netcdf)" - echo " : number of processors (positive integer to run with MPI, or zero for serial execution)" - exit 1 -} - -# ============================= -# Check for --help or -h flags -# ============================= -if [[ "$1" == "--help" || "$1" == "-h" ]]; then - usage -fi - -# ============================================= -# Check if all required arguments are provided -# ============================================= -if [[ -z "$mode" || -z "$nproc" || -z "$sensor" || -z "${split_by_category}" || -z "$obstype" || -z "$cycle" || -z "${obsforge_dir}" ]]; then - echo "Error: Missing one or more required arguments." - usage -fi - -# =============== -# Validate cycle -# =============== -if ! [[ "$cycle" =~ ^[0-9]{10}$ ]]; then - echo "Error: Invalid cycle format. Expected YYYYMMDDHH." - usage -fi - -# ============== -# Validate mode -# ============== -if [[ "$mode" != "bufr4backend" && "$mode" != "script4backend" && "$mode" != "bufr2netcdf" && "$mode" != "script2netcdf" ]]; then - echo "Error: Invalid mode '$mode'. Expected 'bufr4backend' or 'script4backend' or 'bufr2netcdf' or 'script2netcdf'." - usage -fi - -# ============================================= -# Validate nproc is a positive integer or zero -# ============================================= -if ! [[ "$nproc" =~ ^[0-9]+$ && "$nproc" -ge 0 ]]; then - echo "Error: nproc must be a positive integer or zero." - usage -fi - -# ========================= -# Check if all checks pass -# ========================= -echo "root directory of obsForge: $obsforge_dir" -echo "cycle: $cycle" -echo "bufrtype: $bufrtype" -echo "obstype: $obstype" -echo "sensor: $sensor" -echo "split_by_category: ${split_by_category}" -echo "mode: $mode" -echo "number of processors: $nproc" - -# ====================== -# Enable debugging mode -# ====================== -set -x - -# ========================= -# Set unlimited stack size -# ========================= -ulimit -s unlimited -ulimit -a - -# ==================================== -# Set OOPS run time output parameters -# ==================================== -export OOPS_TRACE=1 -export OOPS_DEBUG=1 - -# ======================================================= -# Set WXFLOW log level -# Valid Log_LEVEL: DEBUG, INFO, WARNING, ERROR, CRITICAL -# ======================================================= -export LOG_LEVEL=DEBUG - -# =============================== -# Load obsForge required modules -# =============================== -module use ${obsforge_dir}/modulefiles -module load obsforge/hercules.intel || { echo "Error loading obsforge module"; exit 1; } -module list - -# ============================== -# Set bufr-query python library -# ============================== -export LD_LIBRARY_PATH="${obsforge_dir}/build/lib:${LD_LIBRARY_PATH}" -#export PYTHONPATH="${PYTHONPATH}:${obsforge_dir}/build/lib/python3.10/site-packages" -export PYTHONPATH="${PYTHONPATH}:${obsforge_dir}/build/lib/python3.7/site-packages" - -# ======================== -# Set ioda python library -# ========================= -#export PYTHONPATH="${PYTHONPATH}:${obsforge_dir}/build/lib/python3.10" -export PYTHONPATH="${PYTHONPATH}:${obsforge_dir}/build/lib/python3.7" - -# ============ -# Set wxfloww -# ============ -export PYTHONPATH="${PYTHONPATH}:${obsforge_dir}/sorc/wxflow/src" - -# =========================== -# Configure SLUM environment -# =========================== -export SLURM_ACCOUNT=da-cpu -export SALLOC_ACCOUNT=$SLURM_ACCOUNT -export SBATCH_ACCOUNT=$SLURM_ACCOUNT -export SLURM_QOS=debug - -# =================================== -# Extract year, month, day, and hour -# =================================== -y4="${cycle:0:4}" -m2="${cycle:4:2}" -d2="${cycle:6:2}" -h2="${cycle:8:2}" -cycle_time="${cycle}" -# ==================== -# Set directory paths -# ==================== -work_dir="$PWD" -out_dir="${work_dir}/testoutput/$cycle/${mode}" -in_dir="${work_dir}/testinput/$cycle" -mkdir -p -m770 ${out_dir} || { echo "Error creating output directory: ${out_dir}"; exit 1; } - -# =============== -# Set file paths -# =============== -#ioda_config_yaml="${work_dir}/bufr_${mode}_${obstype}_mpi${nproc}.yaml" -mapping_file="${work_dir}/bufr_${obstype}_mapping.yaml" -#input_file="${in_dir}/gdas.t${h2}z.${bufrtype}.tm00.bufr_d" -#input_file="${in_dir}/gdas.t${h2}z.${bufrtype}.tm00.prepbufr" -input_file="${in_dir}/gdas.t${h2}z.prepbufr" -if [[ "${split_by_category}" = "true" ]]; then - output_file="${out_dir}/gdas.t${h2}z.${bufrtype}_${sensor}_{splits/satId}.tm00.nc" -else - if [[ ${nproc} -ge 2 ]] ; then - ioda_config_yaml="${work_dir}/bufr_${mode}_${obstype}_mpi${nproc}.yaml" - output_file="${out_dir}/gdas.t${h2}z.${bufrtype}_prepbufr.tm00_mpi${nproc}.nc" - else - ioda_config_yaml="${work_dir}/bufr_${mode}_${obstype}.yaml" - output_file="${out_dir}/gdas.t${h2}z.${bufrtype}_prepbufr.tm00.nc" - fi -fi - -if [[ ! -f "$input_file" ]]; then - echo "Error: Input file not found: $input_file" - exit 1 -fi - -if [[ ! -f "$mapping_file" ]]; then - echo "Error: mapping file not found: $mapping" - exit 1 -fi - -# ============================= -# Run ioda bufr/script backend -# ============================= -if [[ "$mode" == "bufr4backend" || "$mode" == "script4backend" ]]; then - if [[ ! -f "$ioda_config_yaml" ]]; then - echo "Error: ioda configuration file not found: $ioda_config_yaml" - exit 1 - fi - if [[ "$nproc" == "0" ]]; then - echo Run time_IodaIO.x without MPI ... - ${obsforge_dir}/build/bin/time_IodaIO.x ${ioda_config_yaml} || { echo "Error: time_IodaIO.x failed"; exit 1; } - else - echo Run time_IodaIO.x with MPI ${nproc} ... - srun -n $nproc --mem 96G --time 00:30:00 ${obsforge_dir}/build/bin/time_IodaIO.x ${ioda_config_yaml} || { echo "Error: MPI time_IodaIO.x failed"; exit 1; } - fi -# ================ -# Run bufr2netcdf -# ================ -elif [[ "$mode" == "bufr2netcdf" ]]; then - if [[ "$nproc" == "0" ]]; then - echo Run bufr2netcdf without MPI ... - echo NICK ${obsforge_dir}/build/bin/bufr2netcdf.x "$input_file" "${mapping_file}" "$output_file" - ${obsforge_dir}/build/bin/bufr2netcdf.x "$input_file" "${mapping_file}" "$output_file" || { echo "Error: bufr2netcdf.x failed"; exit 1; } - else - echo Run bufr2netcdf with MPI ${nproc} ... - srun -n "$nproc" --mem 96G --time 00:30:00 ${obsforge_dir}/build/bin/bufr2netcdf.x "$input_file" "${mapping_file}" "$output_file" || { echo "Error: MPI bufr2netcdf.x failed"; exit 1; } - fi -# ================== -# Run script2netcdf -# ================== -elif [[ "$mode" == "script2netcdf" ]]; then - if [[ "$nproc" == "0" ]]; then - echo Run script2netcdf without MPI ... - #python bufr_${obstype}.py -m "$mapping_file" -o "$output_file" -i "$input_file" || { echo "Error: Python script2netcdf failed"; exit 1; } - python bufr_${obstype}.py "$input_file" "$mapping_file" "$output_file" "$cycle_time" || { echo "Error: Python script2netcdf failed"; python bufr_${obstype}.py --help; exit 1; } - else - echo Run script2netcdf with MPI ${nproc} ... - #srun -n "$nproc" --mem 96G --time 00:30:00 python bufr_${obstype}.py -m "$mapping_file" -o "$output_file" -i "$input_file" || { echo "Error: MPI Python script2netcdf failed"; exit 1; } - srun -n "$nproc" --mem 96G --time 00:30:00 python bufr_${obstype}.py "$input_file" "$mapping_file" "$output_file" "$cycle_time" || { echo "Error: MPI Python script2netcdf failed"; python bufr_${obstype}.py --help; exit 1; } - fi -else - echo Incorrect running mode ${mode} ... Valid modes are: bufr4backend, script4backend, bufr2netcdf, or script2netcdf -fi - - From 1e738907de0afd012c406d8f0be6d7cd74bbf9b6 Mon Sep 17 00:00:00 2001 From: Nicholas Esposito Date: Wed, 15 Jan 2025 16:53:54 -0500 Subject: [PATCH 67/72] move configs --- .../test}/config/bufr2ioda_bufr_backend_satwnd_amv_goes.yaml | 0 {dump => ush/test}/config/bufr_bufr4backend_adpsfc_prepbufr.yaml | 0 .../test}/config/bufr_bufr4backend_adpsfc_prepbufr_mpi4.yaml | 0 .../test}/config/bufr_script4backend_adpsfc_prepbufr.yaml | 0 .../test}/config/bufr_script4backend_adpsfc_prepbufr_mpi4.yaml | 0 5 files changed, 0 insertions(+), 0 deletions(-) rename {dump => ush/test}/config/bufr2ioda_bufr_backend_satwnd_amv_goes.yaml (100%) rename {dump => ush/test}/config/bufr_bufr4backend_adpsfc_prepbufr.yaml (100%) rename {dump => ush/test}/config/bufr_bufr4backend_adpsfc_prepbufr_mpi4.yaml (100%) rename {dump => ush/test}/config/bufr_script4backend_adpsfc_prepbufr.yaml (100%) rename {dump => ush/test}/config/bufr_script4backend_adpsfc_prepbufr_mpi4.yaml (100%) diff --git a/dump/config/bufr2ioda_bufr_backend_satwnd_amv_goes.yaml b/ush/test/config/bufr2ioda_bufr_backend_satwnd_amv_goes.yaml similarity index 100% rename from dump/config/bufr2ioda_bufr_backend_satwnd_amv_goes.yaml rename to ush/test/config/bufr2ioda_bufr_backend_satwnd_amv_goes.yaml diff --git a/dump/config/bufr_bufr4backend_adpsfc_prepbufr.yaml b/ush/test/config/bufr_bufr4backend_adpsfc_prepbufr.yaml similarity index 100% rename from dump/config/bufr_bufr4backend_adpsfc_prepbufr.yaml rename to ush/test/config/bufr_bufr4backend_adpsfc_prepbufr.yaml diff --git a/dump/config/bufr_bufr4backend_adpsfc_prepbufr_mpi4.yaml b/ush/test/config/bufr_bufr4backend_adpsfc_prepbufr_mpi4.yaml similarity index 100% rename from dump/config/bufr_bufr4backend_adpsfc_prepbufr_mpi4.yaml rename to ush/test/config/bufr_bufr4backend_adpsfc_prepbufr_mpi4.yaml diff --git a/dump/config/bufr_script4backend_adpsfc_prepbufr.yaml b/ush/test/config/bufr_script4backend_adpsfc_prepbufr.yaml similarity index 100% rename from dump/config/bufr_script4backend_adpsfc_prepbufr.yaml rename to ush/test/config/bufr_script4backend_adpsfc_prepbufr.yaml diff --git a/dump/config/bufr_script4backend_adpsfc_prepbufr_mpi4.yaml b/ush/test/config/bufr_script4backend_adpsfc_prepbufr_mpi4.yaml similarity index 100% rename from dump/config/bufr_script4backend_adpsfc_prepbufr_mpi4.yaml rename to ush/test/config/bufr_script4backend_adpsfc_prepbufr_mpi4.yaml From e2e31e2ae42075e88939a434156d886f875c016c Mon Sep 17 00:00:00 2001 From: Nicholas Esposito Date: Wed, 15 Jan 2025 17:17:15 -0500 Subject: [PATCH 68/72] cleanup --- ...ufr2ioda_bufr_backend_satwnd_amv_goes.yaml | 58 ------------------- ush/test/encodeBufr.sh | 2 + 2 files changed, 2 insertions(+), 58 deletions(-) delete mode 100644 ush/test/config/bufr2ioda_bufr_backend_satwnd_amv_goes.yaml diff --git a/ush/test/config/bufr2ioda_bufr_backend_satwnd_amv_goes.yaml b/ush/test/config/bufr2ioda_bufr_backend_satwnd_amv_goes.yaml deleted file mode 100644 index 2dd61eb..0000000 --- a/ush/test/config/bufr2ioda_bufr_backend_satwnd_amv_goes.yaml +++ /dev/null @@ -1,58 +0,0 @@ -time window: - begin: "2018-04-14T21:00:00Z" - end: "2023-12-15T03:00:00Z" - -observations: -- obs space: - name: "satwind_goes-16" - simulated variables: [windDirection, windSpeed] - obsdatain: - engine: - type: bufr - obsfile: "./testinput/2021080100/gdas.t00z.satwnd.tm00.bufr_d" - mapping file: "./bufr2ioda_satwnd_amv_goes_mapping.yaml" - category: ["goes-16"] - cache categories: # optional - - ["goes-16"] - - ["goes-17"] - - ["goes-18"] - obsdataout: - engine: - type: H5File - obsfile: "./testoutput/2021080100/bufr_backend/gdas.t00z.satwnd.abi_goes-16.tm00.nc" - -- obs space: - name: "satwind_goes-17" - simulated variables: [windDirection, windSpeed] - obsdatain: - engine: - type: bufr - obsfile: "./testinput/2021080100/gdas.t00z.satwnd.tm00.bufr_d" - mapping file: "./bufr2ioda_satwnd_amv_goes_mapping.yaml" - category: ["goes-17"] - cache categories: # optional - - ["goes-16"] - - ["goes-17"] - - ["goes-18"] - obsdataout: - engine: - type: H5File - obsfile: "./testoutput/2021080100/bufr_backend/gdas.t00z.satwnd.abi_goes-17.tm00.nc" - -- obs space: - name: "satwind_goes-18" - simulated variables: [windDirection, windSpeed] - obsdatain: - engine: - type: bufr - obsfile: "./testinput/2021080100/gdas.t00z.satwnd.tm00.bufr_d" - mapping file: "./bufr2ioda_satwnd_amv_goes_mapping.yaml" - category: ["goes-18"] - cache categories: # optional - - ["goes-16"] - - ["goes-17"] - - ["goes-18"] - obsdataout: - engine: - type: H5File - obsfile: "./testoutput/2021080100/bufr_backend/gdas.t00z.satwnd.abi_goes-18.tm00.nc" diff --git a/ush/test/encodeBufr.sh b/ush/test/encodeBufr.sh index 50fbf99..ff0d621 100644 --- a/ush/test/encodeBufr.sh +++ b/ush/test/encodeBufr.sh @@ -211,3 +211,5 @@ elif [[ "$mode" == "script2netcdf" ]]; then else echo Incorrect running mode ${mode} ... Valid modes are: bufr4backend, script4backend, bufr2netcdf, or script2netcdf fi + + From b2356f7a96fac98818309a3f336633f8887b11bd Mon Sep 17 00:00:00 2001 From: Nicholas Esposito Date: Wed, 15 Jan 2025 17:44:39 -0500 Subject: [PATCH 69/72] chmod encodeBufr.sh, remove bufr2ioda*satwnd* --- dump/mapping/bufr2ioda_satwnd_amv_goes.py | 309 ------------------ .../bufr2ioda_satwnd_amv_goes_mapping.yaml | 185 ----------- ush/test/encodeBufr.sh | 0 3 files changed, 494 deletions(-) delete mode 100644 dump/mapping/bufr2ioda_satwnd_amv_goes.py delete mode 100755 dump/mapping/bufr2ioda_satwnd_amv_goes_mapping.yaml mode change 100644 => 100755 ush/test/encodeBufr.sh diff --git a/dump/mapping/bufr2ioda_satwnd_amv_goes.py b/dump/mapping/bufr2ioda_satwnd_amv_goes.py deleted file mode 100644 index ca560a9..0000000 --- a/dump/mapping/bufr2ioda_satwnd_amv_goes.py +++ /dev/null @@ -1,309 +0,0 @@ -#!/usr/bin/env python3 -import sys -import os -import argparse -import time -import numpy as np -import bufr -from pyioda.ioda.Engines.Bufr import Encoder as iodaEncoder -from bufr.encoders.netcdf import Encoder as netcdfEncoder -from wxflow import Logger - -# Initialize Logger -# Get log level from the environment variable, default to 'INFO it not set -log_level = os.getenv('LOG_LEVEL', 'INFO') -logger = Logger('BUFR2IODA_satwnd_amv_goes.py', level=log_level, colored_log=False) - -def logging(comm, level, message): - """ - Logs a message to the console or log file, based on the specified logging level. - - This function ensures that logging is only performed by the root process (`rank 0`) - in a distributed computing environment. The function maps the logging level to - appropriate logger methods and defaults to the 'INFO' level if an invalid level is provided. - - Parameters: - comm: object - The communicator object, typically from a distributed computing framework - (e.g., MPI). It must have a `rank()` method to determine the process rank. - level: str - The logging level as a string. Supported levels are: - - 'DEBUG' - - 'INFO' - - 'WARNING' - - 'ERROR' - - 'CRITICAL' - If an invalid level is provided, a warning will be logged, and the level - will default to 'INFO'. - message: str - The message to be logged. - - Behavior: - - Logs messages only on the root process (`comm.rank() == 0`). - - Maps the provided logging level to a method of the logger object. - - Defaults to 'INFO' and logs a warning if an invalid logging level is given. - - Supports standard logging levels for granular control over log verbosity. - - Example: - >>> logging(comm, 'DEBUG', 'This is a debug message.') - >>> logging(comm, 'ERROR', 'An error occurred!') - - Notes: - - Ensure that a global `logger` object is configured before using this function. - - The `comm` object should conform to MPI-like conventions (e.g., `rank()` method). - """ - - if comm.rank() == 0: - # Define a dictionary to map levels to logger methods - log_methods = { - 'DEBUG': logger.debug, - 'INFO': logger.info, - 'WARNING': logger.warning, - 'ERROR': logger.error, - 'CRITICAL': logger.critical, - } - - # Get the appropriate logging method, default to 'INFO' - log_method = log_methods.get(level.upper(), logger.info) - - if log_method == logger.info and level.upper() not in log_methods: - # Log a warning if the level is invalid - logger.warning(f'log level = {level}: not a valid level --> set to INFO') - - # Call the logging method - log_method(message) - -def _make_description(mapping_path, update=False): - description = bufr.encoders.Description(mapping_path) - - if update: - # Define the variables to be added in a list of dictionaries - variables = [ - { - 'name': 'ObsType/windEastward', - 'source': 'variables/obstype_uwind', - 'units': '1', - 'longName': 'Observation Type based on Satellite-derived Wind Computation Method and Spectral Band', - }, - { - 'name': 'ObsType/windNorthward', - 'source': 'variables/obstype_vwind', - 'units': '1', - 'longName': 'Observation Type based on Satellite-derived Wind Computation Method and Spectral Band', - }, - { - 'name': 'ObsValue/windEastward', - 'source': 'variables/windEastward', - 'units': 'm/s', - 'longName': 'Eastward Wind Component', - }, - { - 'name': 'ObsValue/windNorthward', - 'source': 'variables/windNorthward', - 'units': 'm/s', - 'longName': 'Northward Wind Component', - }, - ] - - # Loop through each variable and add it to the description - for var in variables: - description.add_variable( - name=var['name'], - source=var['source'], - units=var['units'], - longName=var['longName'] - ) - - return description - -def compute_wind_components(wdir, wspd): - """ - Compute the U and V wind components from wind direction and wind speed. - - Parameters: - wdir (array-like): Wind direction in degrees (meteorological convention: 0° = North, 90° = East). - wspd (array-like): Wind speed (m/s). - - Returns: - tuple: U and V wind components as numpy arrays with dtype float32. - """ - wdir_rad = np.radians(wdir) # Convert degrees to radians - u = -wspd * np.sin(wdir_rad) - v = -wspd * np.cos(wdir_rad) - - return u.astype(np.float32), v.astype(np.float32) - -def _get_obs_type(swcm, chanfreq): - """ - Determine the observation type based on `swcm` and `chanfreq`. - - Parameters: - swcm (array-like): Satellite derived wind calculation method. - chanfreq (array-like): Satellite channel center frequency (Hz). - - Returns: - numpy.ndarray: Observation type array. - - Raises: - ValueError: If any `obstype` is unassigned. - """ - - obstype = swcm.copy() - - # Use numpy vectorized operations - obstype = np.where(swcm == 5, 247, obstype) # WVCA/DL - obstype = np.where(swcm == 3, 246, obstype) # WVCT - obstype = np.where(swcm == 2, 251, obstype) # VIS - obstype = np.where(swcm == 1, 245, obstype) # IRLW - - condition = np.logical_and(swcm == 1, chanfreq >= 5e13) # IRSW - obstype = np.where(condition, 240, obstype) - - if not np.any(np.isin(obstype, [247, 246, 251, 245, 240])): - raise ValueError("Error: Unassigned ObsType found ... ") - - return obstype - -def _make_obs(comm, input_path, mapping_path): - - # Get container from mapping file first - logging(comm, 'INFO', 'Get container from bufr') - container = bufr.Parser(input_path, mapping_path).parse(comm) - - logging(comm, 'DEBUG', f'container list (original): {container.list()}') - logging(comm, 'DEBUG', f'all_sub_categories = {container.all_sub_categories()}') - logging(comm, 'DEBUG', f'category map = {container.get_category_map()}') - - # Add new/derived data into container - for cat in container.all_sub_categories(): - - logging(comm, 'DEBUG', f'category = {cat}') - - satid = container.get('variables/satelliteId', cat) - if satid.size == 0: - logging(comm, 'WARNING', f'category {cat[0]} does not exist in input file') - paths = container.get_paths('variables/windComputationMethod', cat) - obstype = container.get('variables/windComputationMethod', cat) - container.add('variables/obstype_uwind', obstype, paths, cat) - container.add('variables/obstype_vwind', obstype, paths, cat) - - paths = container.get_paths('variables/windSpeed', cat) - wob = container.get('variables/windSpeed', cat) - container.add('variables/windEastward', wob, paths, cat) - container.add('variables/windNorthward', wob, paths, cat) - - else: - # Add new variables: ObsType/windEastward & ObsType/windNorthward - swcm = container.get('variables/windComputationMethod', cat) - chanfreq = container.get('variables/sensorCentralFrequency', cat) - - logging(comm, 'DEBUG', f'swcm min/max = {swcm.min()} {swcm.max()}') - logging(comm, 'DEBUG', f'chanfreq min/max = {chanfreq.min()} {chanfreq.max()}') - - obstype = _get_obs_type(swcm, chanfreq) - - logging(comm, 'DEBUG', f'obstype = {obstype}') - logging(comm, 'DEBUG', f'obstype min/max = {obstype.min()} {obstype.max()}') - - paths = container.get_paths('variables/windComputationMethod', cat) - container.add('variables/obstype_uwind', obstype, paths, cat) - container.add('variables/obstype_vwind', obstype, paths, cat) - - # Add new variables: ObsValue/windEastward & ObsValue/windNorthward - wdir = container.get('variables/windDirection', cat) - wspd = container.get('variables/windSpeed', cat) - - logging(comm, 'DEBUG', f'wdir min/max = {wdir.min()} {wdir.max()}') - logging(comm, 'DEBUG', f'wspd min/max = {wspd.min()} {wspd.max()}') - - uob, vob = compute_wind_components(wdir, wspd) - - logging(comm, 'DEBUG', f'uob min/max = {uob.min()} {uob.max()}') - logging(comm, 'DEBUG', f'vob min/max = {vob.min()} {vob.max()}') - - paths = container.get_paths('variables/windSpeed', cat) - container.add('variables/windEastward', uob, paths, cat) - container.add('variables/windNorthward', vob, paths, cat) - - # Check - logging(comm, 'DEBUG', f'container list (updated): {container.list()}') - logging(comm, 'DEBUG', f'all_sub_categories {container.all_sub_categories()}') - - return container - -def create_obs_group(input_path, mapping_path, category, env): - - comm = bufr.mpi.Comm(env["comm_name"]) - - description = _make_description(mapping_path, update=True) - - # Check the cache for the data and return it if it exists - logging(comm, 'DEBUG', f'Check if bufr.DataCache exists? {bufr.DataCache.has(input_path, mapping_path)}') - if bufr.DataCache.has(input_path, mapping_path): - container = bufr.DataCache.get(input_path, mapping_path) - logging(comm, 'INFO', f'Encode {category} from cache') - data = iodaEncoder(description).encode(container)[(category,)] - logging(comm, 'INFO', f'Mark {category} as finished in the cache') - bufr.DataCache.mark_finished(input_path, mapping_path, [category]) - logging(comm, 'INFO', f'Return the encoded data for {category}') - return data - - container = _make_obs(comm, input_path, mapping_path) - - # Gather data from all tasks into all tasks. Each task will have the complete record - logging(comm, 'INFO', f'Gather data from all tasks into all tasks') - container.all_gather(comm) - - logging(comm, 'INFO', f'Add container to cache') - # Add the container to the cache - bufr.DataCache.add(input_path, mapping_path, container.all_sub_categories(), container) - - # Encode the data - logging(comm, 'INFO', f'Encode {category}') - data = iodaEncoder(description).encode(container)[(category,)] - - logging(comm, 'INFO', f'Mark {category} as finished in the cache') - # Mark the data as finished in the cache - bufr.DataCache.mark_finished(input_path, mapping_path, [category]) - - logging(comm, 'INFO', f'Return the encoded data for {category}') - return data - -def create_obs_file(input_path, mapping_path, output_path): - - comm = bufr.mpi.Comm("world") - container = _make_obs(comm, input_path, mapping_path) - container.gather(comm) - - description = _make_description(mapping_path, update=True) - - # Encode the data - if comm.rank() == 0: - netcdfEncoder(description).encode(container, output_path) - - logging(comm, 'INFO', f'Return the encoded data') - - -if __name__ == '__main__': - - start_time = time.time() - - bufr.mpi.App(sys.argv) - comm = bufr.mpi.Comm("world") - - # Required input arguments - parser = argparse.ArgumentParser() - parser.add_argument('-i', '--input', type=str, help='Input BUFR', required=True) - parser.add_argument('-m', '--mapping', type=str, help='BUFR2IODA Mapping File', required=True) - parser.add_argument('-o', '--output', type=str, help='Output NetCDF', required=True) - - args = parser.parse_args() - mapping = args.mapping - infile = args.input - output = args.output - - create_obs_file(infile, mapping, output) - - end_time = time.time() - running_time = end_time - start_time - logging(comm, 'INFO', f'Total running time: {running_time}') diff --git a/dump/mapping/bufr2ioda_satwnd_amv_goes_mapping.yaml b/dump/mapping/bufr2ioda_satwnd_amv_goes_mapping.yaml deleted file mode 100755 index 26a75f5..0000000 --- a/dump/mapping/bufr2ioda_satwnd_amv_goes_mapping.yaml +++ /dev/null @@ -1,185 +0,0 @@ -bufr: - subsets: - - NC005030 - - NC005031 - - NC005032 - - NC005034 - - NC005039 - - variables: - # MetaData - timestamp: - datetime: - year: "*/YEAR" - month: "*/MNTH" - day: "*/DAYS" - hour: "*/HOUR" - minute: "*/MINU" - second: "*/SECO" - - latitude: - query: "*/CLATH" - - longitude: - query: "*/CLONH" - - satelliteId: - query: "*/SAID" - - satelliteZenithAngle: - query: "*/SAZA" - - sensorCentralFrequency: - query: "*/SCCF" - - pressure: - query: "*/PRLC[1]" - - # Processing Center - dataProviderOrigin: - query: '*/OGCE[1]' - - # Quality Information - Quality Indicator - qualityInformationWithoutForecast: - query: '*/AMVQIC{2}/PCCF' - - # Quality Information - Expected Error - expectedError: - query: '*/AMVQIC{4}/PCCF' - - # Derived Motion Wind (DMW) Intermediate Vectors - Coefficient of Variation - coefficientOfVariation: - query: '*/AMVIVR{1}/CVWD' - - # Wind Retrieval Method Information - Computation - windComputationMethod: - query: '*/SWCM' - - # Wind Retrieval Method Information - Hight Assignment - windHeightAssignMethod: - query: '*/EHAM' - - # ObsValue - Wind Direction - windDirection: - query: '*/WDIR' - - # ObsValue - Wind Speed - windSpeed: - query: '*/WSPD' - - splits: - satId: - category: - variable: satelliteId - map: - _270: goes-16 - _271: goes-17 - _272: goes-18 - -encoder: -# type: netcdf - -# dimensions: - - globals: - - name: "platformCommonName" - type: string - value: "GOES" - - - name: "platformLongDescription" - type: string - value: "Geostationary Operational Satellite" - - - name: "sensor" - type: string - value: "Advanced Baseline Imager - ABI" - - - name: "source" - type: string - value: "NCEP BUFR Dump for satellite derived atmospheric motion vectors (satwnd)" - - - name: "providerFullName" - type: string - value: "National Environmental Satellite, Data, and Information Service" - - - name: "processingLevel" - type: string - value: "Level-2" - - - name: "converter" - type: string - value: "BUFR" - - variables: - # MetaData - - name: "MetaData/dateTime" - source: variables/timestamp - longName: "Datetime" - units: "seconds since 1970-01-01T00:00:00Z" - - - name: "MetaData/latitude" - source: variables/latitude - longName: "Latitude" - units: "degrees_north" - range: [ -90, 90 ] - - - name: "MetaData/longitude" - source: variables/longitude - longName: "Longitude" - units: "degrees_east" - range: [ -180, 180 ] - - - name: "MetaData/satelliteIdentifier" - source: variables/satelliteId - longName: "Satellite Identifier" - - - name: "MetaData/satelliteZenithAngle" - source: variables/satelliteZenithAngle - longName: "Satellite Zenith Angle" - units: "degree" - range: [ 0, 90 ] - - - name: "MetaData/sensorCentralFrequency" - source: variables/sensorCentralFrequency - longName: "Satellite Channel Center Frequency" - units: "Hz" - - - name: "MetaData/dataProviderOrigin" - source: variables/dataProviderOrigin - longName: "Identification of Originating/Generating Center" - - - name: "MetaData/qualityInformationWithoutForecast" - source: variables/qualityInformationWithoutForecast - longName: "Quality Information Without Forecast" - - - name: "MetaData/expectedError" - source: variables/expectedError - longName: "Expected Error" - units: "m/s" - - - name: "MetaData/coefficientOfVariation" - source: variables/coefficientOfVariation - longName: "Coefficient of Variation" - - - name: "MetaData/windComputationMethod" - source: variables/windComputationMethod - longName: "Satellite-derived Wind Computation Method" - - - name: "MetaData/windHeightAssignMethod" - source: variables/windHeightAssignMethod - longName: "Wind Height Assignment Method" - - - name: "MetaData/pressure" - source: variables/pressure - longName: "Pressure" - units: "pa" - - - name: "ObsValue/windDirection" - source: variables/windDirection - longName: "Wind Direction" - units: "degree" - - - name: "ObsValue/windSpeed" - source: variables/windSpeed - longName: "Wind Direction" - units: "m/s" diff --git a/ush/test/encodeBufr.sh b/ush/test/encodeBufr.sh old mode 100644 new mode 100755 From 33fa954ecddda17f730dc64c544f78f2f8fd6b8c Mon Sep 17 00:00:00 2001 From: Nicholas Esposito Date: Thu, 23 Jan 2025 11:27:55 -0500 Subject: [PATCH 70/72] function names, some coding norms --- dump/mapping/bufr_adpsfc_prepbufr.py | 68 +++++++++---------- .../mapping/bufr_adpsfc_prepbufr_mapping.yaml | 4 +- 2 files changed, 36 insertions(+), 36 deletions(-) diff --git a/dump/mapping/bufr_adpsfc_prepbufr.py b/dump/mapping/bufr_adpsfc_prepbufr.py index 0ea9088..9047deb 100755 --- a/dump/mapping/bufr_adpsfc_prepbufr.py +++ b/dump/mapping/bufr_adpsfc_prepbufr.py @@ -19,34 +19,6 @@ log_level = os.getenv('LOG_LEVEL', 'INFO') logger = Logger('BUFR2IODA_adpsfc_prepbufr.py', level=log_level, colored_log=False) -def Compute_dateTime(cycleTimeSinceEpoch, dhr): - """ - Compute dateTime using the cycleTimeSinceEpoch and Cycle Time - minus Cycle Time - - Parameters: - cycleTimeSinceEpoch: Time of cycle in Epoch Time - dhr: Observation Time Minus Cycle Time - - Returns: - Masked array of dateTime values - """ - - int64_fill_value = np.int64(0) - - dateTime = np.zeros(dhr.shape, dtype=np.int64) - for i in range(len(dateTime)): - if ma.is_masked(dhr[i]): - continue - else: - dateTime[i] = np.int64(dhr[i]*3600) + cycleTimeSinceEpoch - - dateTime = ma.array(dateTime) - dateTime = ma.masked_values(dateTime, int64_fill_value) - - return dateTime - - def logging(comm, level, message): """ Logs a message to the console or log file, based on the specified logging level. @@ -106,6 +78,35 @@ def logging(comm, level, message): # Call the logging method log_method(message) + +def _compute_datetime(cycleTimeSinceEpoch, dhr): + """ + Compute dateTime using the cycleTimeSinceEpoch and Cycle Time + minus Cycle Time + + Parameters: + cycleTimeSinceEpoch: Time of cycle in Epoch Time + dhr: Observation Time Minus Cycle Time + + Returns: + Masked array of dateTime values + """ + + int64_fill_value = np.int64(0) + + dateTime = np.zeros(dhr.shape, dtype=np.int64) + for i in range(len(dateTime)): + if ma.is_masked(dhr[i]): + continue + else: + dateTime[i] = np.int64(dhr[i]*3600) + cycleTimeSinceEpoch + + dateTime = ma.array(dateTime) + dateTime = ma.masked_values(dateTime, int64_fill_value) + + return dateTime + + def _make_description(mapping_path, cycle_time, update=False): description = bufr.encoders.Description(mapping_path) @@ -139,13 +140,13 @@ def _make_description(mapping_path, cycle_time, update=False): def _make_obs(comm, input_path, mapping_path, cycle_time): """ Create the ioda adpsfc prepbufr observations: - - reads values - - adds sequenceNum + - reads values + - adds sequenceNum Parameters ---------- comm: object - The communicator object (e.g., MPI) + The communicator object (e.g., MPI) input_path: str The input bufr file mapping_path: str @@ -170,7 +171,7 @@ def _make_obs(comm, input_path, mapping_path, cycle_time): otmct_paths = container.get_paths('variables/obsTimeMinusCycleTime') otmct2 = np.array(otmct) cycleTimeSinceEpoch = np.int64(calendar.timegm(time.strptime(str(int(cycle_time)), '%Y%m%d%H'))) - dateTime = Compute_dateTime(cycleTimeSinceEpoch, otmct2) + dateTime = _compute_datetime(cycleTimeSinceEpoch, otmct2) min_dateTime_ge_zero = min(x for x in dateTime if x > -1) logging(comm, 'DEBUG', f'dateTime min/max = {min_dateTime_ge_zero} {dateTime.max()}') @@ -200,8 +201,7 @@ def create_obs_group(input_path, mapping_path, cycle_time, env): container = _make_obs(comm, input_path, mapping_path, cycle_time) description = _make_description(mapping_path, cycle_time, update=True) - - # Gather data from all tasks into all tasks. Each task will have the complete record + # Gather data from all tasks into all tasks. Each task will have the complete record logging(comm, 'INFO', f'Gather data from all tasks into all tasks') container.all_gather(comm) diff --git a/dump/mapping/bufr_adpsfc_prepbufr_mapping.yaml b/dump/mapping/bufr_adpsfc_prepbufr_mapping.yaml index 2307e5c..6d02794 100755 --- a/dump/mapping/bufr_adpsfc_prepbufr_mapping.yaml +++ b/dump/mapping/bufr_adpsfc_prepbufr_mapping.yaml @@ -75,7 +75,7 @@ encoder: - name: "platforms" type: string - value: "ADPSFC" + value: "ADPSFC" - name: "source" type: string @@ -111,7 +111,7 @@ encoder: longName: "Prepbufr Data Level Category" - name: "MetaData/dateTime" - source: variables/timestamp + source: variables/timestamp units: 'seconds since 1970-01-01T00:00:00Z' longName: 'dateTime' From e6222b99f4f9dfd3a60643a93e4e9b6e83e9934e Mon Sep 17 00:00:00 2001 From: Nicholas Esposito Date: Thu, 23 Jan 2025 11:32:53 -0500 Subject: [PATCH 71/72] function name --- dump/mapping/bufr_adpsfc_prepbufr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dump/mapping/bufr_adpsfc_prepbufr.py b/dump/mapping/bufr_adpsfc_prepbufr.py index 9047deb..a6b6ff7 100755 --- a/dump/mapping/bufr_adpsfc_prepbufr.py +++ b/dump/mapping/bufr_adpsfc_prepbufr.py @@ -17,7 +17,7 @@ # Initialize Logger # Get log level from the environment variable, default to 'INFO it not set log_level = os.getenv('LOG_LEVEL', 'INFO') -logger = Logger('BUFR2IODA_adpsfc_prepbufr.py', level=log_level, colored_log=False) +logger = Logger('bufr_adpsfc_prepbufr.py', level=log_level, colored_log=False) def logging(comm, level, message): """ From a18ab32b0fe2a632049c3a20c9d635086fad1bc7 Mon Sep 17 00:00:00 2001 From: Nicholas Esposito Date: Thu, 30 Jan 2025 16:43:30 +0000 Subject: [PATCH 72/72] rm add_global referenceDateTime fornow --- dump/mapping/bufr_adpsfc_prepbufr.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dump/mapping/bufr_adpsfc_prepbufr.py b/dump/mapping/bufr_adpsfc_prepbufr.py index a6b6ff7..c6a9be6 100755 --- a/dump/mapping/bufr_adpsfc_prepbufr.py +++ b/dump/mapping/bufr_adpsfc_prepbufr.py @@ -132,7 +132,7 @@ def _make_description(mapping_path, cycle_time, update=False): longName=var['longName'] ) - description.add_global(name='datetimeReference', value=str(ReferenceTime)) + #description.add_global(name='datetimeReference', value=str(ReferenceTime)) return description @@ -165,6 +165,7 @@ def _make_obs(comm, input_path, mapping_path, cycle_time): lon_paths = container.get_paths('variables/longitude') lon[lon>180] -= 360 lon = ma.round(lon, decimals=2) + logging(comm, 'DEBUG', f'longitude max and min are {lon.max()}, {lon.min()}') logging(comm, 'DEBUG', f'Do DateTime calculation') otmct = container.get('variables/obsTimeMinusCycleTime') @@ -209,6 +210,7 @@ def create_obs_group(input_path, mapping_path, cycle_time, env): data = next(iter(iodaEncoder(description).encode(container).values())) logging(comm, 'INFO', f'Return the encoded data.') + return data