From 03fb070838222028ef8d9a9e79bbaf43e636e12b Mon Sep 17 00:00:00 2001 From: Felipe Maza Date: Fri, 7 Jan 2022 13:18:19 +0100 Subject: [PATCH] add method outside to open_xarray_conn c> - Fix extent in `open_xarray_conn`. - Fix load env values. - Fix assert in filter test. - Add parameter `method=outside` to `open_xarray_conn`. --- CHANGELOG.md | 5 +++++ datahub/__init__.py | 2 +- datahub/catalog.py | 52 ++++++++++++++++++++++++++++++++++++++++++--- datahub/dataset.py | 50 +++++++++++++++++++++++++++++++++++++++++-- 4 files changed, 103 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index df15b46..5f1d0fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,9 @@ # Changelog +## 0.9.3 +- Fix extent in `open_xarray_conn`. +- Fix load env values. +- Fix assert in filter test. +- Add parameter `method=outside` to `open_xarray_conn`. ## 0.9.2 - Fix SECRET_AUTH env key is not required. diff --git a/datahub/__init__.py b/datahub/__init__.py index 7bcb23d..d6f5ebe 100644 --- a/datahub/__init__.py +++ b/datahub/__init__.py @@ -1,2 +1,2 @@ """ Datahub client""" -__version__ = "0.9.2" +__version__ = "0.9.3" diff --git a/datahub/catalog.py b/datahub/catalog.py index 8665460..5249faa 100644 --- a/datahub/catalog.py +++ b/datahub/catalog.py @@ -4,6 +4,7 @@ from io import StringIO import json import math +import numpy as np import os import requests import urllib.request @@ -286,27 +287,72 @@ def _coordinates_to_string(self, coordinates): logger.debug(f"coordinates={coordinates}") return text - def open_xarray_conn(self, dates=None, extent=None): + def open_xarray_conn(self, dates=None, extent=None, method=None): + + if method and method == "outside" and not extent: + raise Exception(f"Outside method requires an extent") list_conn = [dataset.opendap_url for dataset in self.datasets] logger.debug(f"opening: {','.join(list_conn)}") ds = self._join_datasets(list_conn) - # ds = self._join_datasets(list_conn) if dates: start = dates["start"] if "start" in dates else None end = dates["end"] if "end" in dates else None if "time" in ds.dims: + if method == "outside" and start: + start = ( + start + - np.diff(ds["time"].values) + .max() + .astype("timedelta64[s]") + .item() + ) + end = ( + end + + np.diff(ds["time"].values) + .max() + .astype("timedelta64[s]") + .item() + ) ds = ds.sel(time=slice(start, end)) elif "t" in ds.dims: + if method == "outside" and start: + start = ( + start + - np.diff(ds["t"].values).max().astype("timedelta64[s]").item() + ) + end = ( + end + + np.diff(ds["t"].values).max().astype("timedelta64[s]").item() + ) ds = ds.sel(t=slice(start, end)) if extent: - if "longitud" in ds.dims: + + if "longitude" in ds.dims: + if method == "outside": + extent["west"] = ( + extent["west"] - np.diff(ds["longitude"].values).max() + ) + extent["east"] = ( + extent["east"] - np.diff(ds["longitude"].values).max() + ) + extent["south"] = ( + extent["south"] - np.diff(ds["latitude"].values).max() + ) + extent["north"] = ( + extent["north"] - np.diff(ds["latitude"].values).max() + ) ds = ds.sel( longitude=slice(extent["west"], extent["east"]), latitude=slice(extent["south"], extent["north"]), ) elif "lon" in ds.dims: + if method == "outside": + extent["west"] = extent["west"] - np.diff(ds["lon"].values).max() + extent["east"] = extent["east"] - np.diff(ds["lon"].values).max() + extent["south"] = extent["south"] - np.diff(ds["lat"].values).max() + extent["north"] = extent["north"] - np.diff(ds["lat"].values).max() ds = ds.sel( lon=slice(extent["west"], extent["east"]), lat=slice(extent["south"], extent["north"]), diff --git a/datahub/dataset.py b/datahub/dataset.py index b8e2f58..4d292af 100644 --- a/datahub/dataset.py +++ b/datahub/dataset.py @@ -1,5 +1,6 @@ from bs4 import BeautifulSoup import logging +import numpy as np import requests import xarray @@ -190,23 +191,68 @@ def _coordinates_to_string(self, coordinates): logger.debug(f"coordinates={text}") return text - def open_xarray_conn(self, dates=None, extent=None): + def open_xarray_conn(self, dates=None, extent=None, method=None): + if method and method == "outside" and not extent: + raise Exception(f"Outside method requires an extent") logger.debug(f"opening {self.opendap_url}") ds = xarray.open_dataset(self.opendap_url) if dates: start = dates["start"] if "start" in dates else None end = dates["end"] if "end" in dates else None if "time" in ds.dims: + if method == "outside" and start: + start = ( + start + - np.diff(ds["time"].values) + .max() + .astype("timedelta64[s]") + .item() + ) + end = ( + end + + np.diff(ds["time"].values) + .max() + .astype("timedelta64[s]") + .item() + ) ds = ds.sel(time=slice(start, end)) elif "t" in ds.dims: + if method == "outside" and start: + start = ( + start + - np.diff(ds["t"].values).max().astype("timedelta64[s]").item() + ) + end = ( + end + + np.diff(ds["t"].values).max().astype("timedelta64[s]").item() + ) ds = ds.sel(t=slice(start, end)) if extent: - if "longitud" in ds.dims: + if "longitude" in ds.dims: + if method == "outside": + extent["west"] = ( + extent["west"] - np.diff(ds["longitude"].values).max() + ) + extent["east"] = ( + extent["east"] - np.diff(ds["longitude"].values).max() + ) + extent["south"] = ( + extent["south"] - np.diff(ds["latitude"].values).max() + ) + extent["north"] = ( + extent["north"] - np.diff(ds["latitude"].values).max() + ) ds = ds.sel( longitude=slice(extent["west"], extent["east"]), latitude=slice(extent["south"], extent["north"]), ) elif "lon" in ds.dims: + if method == "outside": + extent["west"] = extent["west"] - np.diff(ds["lon"].values).max() + extent["east"] = extent["east"] - np.diff(ds["lon"].values).max() + extent["south"] = extent["south"] - np.diff(ds["lat"].values).max() + extent["north"] = extent["north"] - np.diff(ds["lat"].values).max() + ds = ds.sel( lon=slice(extent["west"], extent["east"]), lat=slice(extent["south"], extent["north"]),