From fb9c3100c3d792602ad7a5579cab83efef287779 Mon Sep 17 00:00:00 2001 From: bekozi Date: Fri, 21 Aug 2020 17:01:47 -0600 Subject: [PATCH] ENH: Add netCDF 64-bit offset option to ocli (#506, #519) - Add 64bit_offset option to ocli - Pass large_file only when it is True allowing older ESMF versions to work - Add flag documentation --- doc/sphinx_examples/chunked_rwg_help.sh | 9 +++++++++ src/ocgis/messages.py | 3 ++- src/ocgis/ocli.py | 11 ++++++++--- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/doc/sphinx_examples/chunked_rwg_help.sh b/doc/sphinx_examples/chunked_rwg_help.sh index 3ba18ad6a..1091e9fc5 100644 --- a/doc/sphinx_examples/chunked_rwg_help.sh +++ b/doc/sphinx_examples/chunked_rwg_help.sh @@ -91,4 +91,13 @@ Options: and weight factor variables. WITHAUX adds auxiliary variables and additional file metadata to the output weight file. + --64bit_offset / --no_64bit_offset + If provided, create the weight file in + NetCDF using the 64-bit offset format to + allow variables larger than 2GB. Note the + 64-bit offset format is not supported in the + NetCDF version earlier than 3.6.0. An error + message will be generated if this flag is + specified while the application is linked + with a NetCDF library earlier than 3.6.0. --help Show this message and exit. \ No newline at end of file diff --git a/src/ocgis/messages.py b/src/ocgis/messages.py index 580ba1825..080ab3791 100644 --- a/src/ocgis/messages.py +++ b/src/ocgis/messages.py @@ -1,4 +1,5 @@ M1 = 'No dimensioned variables found. This typically means no target variables in the dataset have space and/or time dimensions associated with them. Consider using a dimension map if file metadata is non-standard. Overloading "variable" is also an option to avoid dimension checking.' M3 = 'Output path exists "{0}" and must be removed before proceeding. Set "overwrite" argument or env.OVERWRITE to True to overwrite.' M4 = """A level subset was requested but the target dataset does not have a level dimension. The dataset's alias is: {0}""" -M5 = "The ESMF FileMode constant value. BASIC (the default) only writes the factor index list and weight factor variables. WITHAUX adds auxiliary variables and additional file metadata to the output weight file." \ No newline at end of file +M5 = "The ESMF FileMode constant value. BASIC (the default) only writes the factor index list and weight factor variables. WITHAUX adds auxiliary variables and additional file metadata to the output weight file." +M6 = "If provided, create the weight file in NetCDF using the 64-bit offset format to allow variables larger than 2GB. Note the 64-bit offset format is not supported in the NetCDF version earlier than 3.6.0. An error message will be generated if this flag is specified while the application is linked with a NetCDF library earlier than 3.6.0." \ No newline at end of file diff --git a/src/ocgis/ocli.py b/src/ocgis/ocli.py index 427a6c339..847d514f0 100644 --- a/src/ocgis/ocli.py +++ b/src/ocgis/ocli.py @@ -14,7 +14,7 @@ from ocgis.base import grid_abstraction_scope, raise_if_empty from ocgis.constants import DriverKey, Topology, GridChunkerConstants, DecompositionType from ocgis.driver.nc_ugrid import DriverNetcdfUGRID -from ocgis.messages import M5 +from ocgis.messages import M5, M6 from ocgis.spatial.grid_chunker import GridChunker from ocgis.spatial.spatial_subset import SpatialSubsetOperation from ocgis.util.logging_ocgis import ocgis_lh @@ -94,9 +94,11 @@ def ocli(): @click.option('--loglvl', default="INFO", help='Verbosity level for standard out logging. Default is ' '"INFO". See Python logging level docs for additional values: https://docs.python.org/3/howto/logging.html') @click.option('--weightfilemode', default="BASIC", help=M5) +@click.option('--64bit_offset/--no_64bit_offset', 'large_file', default=False, help=M6) def chunked_rwg(source, destination, weight, nchunks_dst, merge, esmf_src_type, esmf_dst_type, genweights, esmf_regrid_method, spatial_subset, src_resolution, dst_resolution, buffer_distance, wd, persist, - eager, ignore_degenerate, data_variables, spatial_subset_path, verbose, loglvl, weightfilemode): + eager, ignore_degenerate, data_variables, spatial_subset_path, verbose, loglvl, weightfilemode, + large_file): # Used for creating the history string. the_locals = locals() @@ -172,7 +174,10 @@ def chunked_rwg(source, destination, weight, nchunks_dst, merge, esmf_src_type, # Arguments to ESMF regridding. esmf_kwargs = {'regrid_method': esmf_regrid_method, - 'ignore_degenerate': ignore_degenerate} + 'ignore_degenerate': ignore_degenerate,} + # Allow older versions of ESMF to work with a default large file flag. + if large_file: + esmf_kwargs['large_file'] = large_file # Create the chunked regridding object. This is used for both chunked regridding and a regrid with a spatial subset. gs = GridChunker(rd_src, rd_dst, nchunks_dst=nchunks_dst, src_grid_resolution=src_resolution, paths=paths,