Merge pull request #67 from Kitware/esgf-data

Catalog modules: Pangeo & ESGF data
Kitware · Mar 8, 2024 · 0208640 · 0208640
2 parents 5cd9332 + 066f8e6
commit 0208640
Show file tree

Hide file tree

Showing 25 changed files with 990 additions and 353 deletions.
diff --git a/.gitattributes b/.gitattributes
@@ -19,3 +19,4 @@ docs/images/9a.png filter=lfs diff=lfs merge=lfs -text
 docs/images/0.png filter=lfs diff=lfs merge=lfs -text
 docs/images/11.png filter=lfs diff=lfs merge=lfs -text
 docs/images/12a.png filter=lfs diff=lfs merge=lfs -text
+examples/esgf_catalog.json filter=lfs diff=lfs merge=lfs -text
diff --git a/examples/example_config_esgf.json b/examples/example_config_esgf.json
@@ -0,0 +1,49 @@
+{
+    "data_origin": {
+        "source": "esgf",
+        "id": "CMIP6.PAMIP.MOHC.HadGEM3-GC31-MM.pdSST-futAntSIC.r102i1p1f2.day.sfcWind.gn.v20210311|eagle.alcf.anl.gov"
+    },
+    "data_array": {
+        "name": "sfcWind",
+        "x": "lon",
+        "y": "lat",
+        "t": "time",
+        "t_index": 0
+    },
+    "data_slices": {
+        "time": [
+            "Apr 01 2000 12:00",
+            "May 30 2001 12:00",
+            1
+        ],
+        "lat": [
+            -89.72,
+            89.72,
+            1
+        ],
+        "lon": [
+            0.42,
+            359.58,
+            1
+        ]
+    },
+    "ui": {
+        "main_drawer": false,
+        "axis_drawer": true,
+        "unapplied_changes": false,
+        "error_message": null,
+        "more_info_link": null,
+        "expanded_coordinates": [],
+        "current_time_string": "Apr 01 2000 12:00"
+    },
+    "render": {
+        "auto": true,
+        "x_scale": 1,
+        "y_scale": 1,
+        "z_scale": 1,
+        "scalar_warp": false,
+        "transparency": false,
+        "transparency_function": "linear",
+        "colormap": "cividis"
+    }
+}
diff --git a/examples/example_config_pangeo.json b/examples/example_config_pangeo.json
@@ -0,0 +1,55 @@
+{
+    "data_origin": {
+        "source": "pangeo",
+        "id": "MITgcm_channel_flatbottom_02km_run01_phys_snap15D"
+    },
+    "data_array": {
+        "name": "T",
+        "x": "YC",
+        "y": "XC",
+        "z": "Z",
+        "t": "time",
+        "t_index": 0
+    },
+    "data_slices": {
+        "time": [
+            142560000,
+            171072000,
+            1
+        ],
+        "Z": [
+            -2912.67,
+            -0.5,
+            1
+        ],
+        "YC": [
+            1000,
+            1999000,
+            1
+        ],
+        "XC": [
+            1000,
+            999000,
+            1
+        ]
+    },
+    "ui": {
+        "main_drawer": false,
+        "axis_drawer": false,
+        "unapplied_changes": false,
+        "error_message": null,
+        "more_info_link": null,
+        "expanded_coordinates": [],
+        "current_time_string": "139968000.0 seconds"
+    },
+    "render": {
+        "auto": true,
+        "x_scale": 1,
+        "y_scale": 1,
+        "z_scale": 100,
+        "scalar_warp": false,
+        "transparency": false,
+        "transparency_function": "linear",
+        "colormap": "viridis"
+    }
+}
diff --git a/examples/example_config_xarray.json b/examples/example_config_xarray.json
@@ -1,5 +1,8 @@
 {
-    "data_origin": "eraint_uvz",
+    "data_origin": {
+        "source": "xarray",
+        "id": "eraint_uvz"
+    },
     "data_array": {
         "name": "z",
         "x": "longitude",
@@ -30,4 +33,4 @@
         "axis_drawer": true,
         "expanded_coordinates": []
     }
-}
+}
diff --git a/examples/jupyter/advanced_pyvista_rendering.ipynb b/examples/jupyter/advanced_pyvista_rendering.ipynb
@@ -22,8 +22,8 @@
     "start = datetime.now()\n",
     "\n",
     "# Use Pan3D DatasetBuilder to import existing config file\n",
-    "config_path = '../example_config_cmip.json'\n",
-    "builder = DatasetBuilder()\n",
+    "config_path = '../example_config_esgf.json'\n",
+    "builder = DatasetBuilder(esgf=True)\n",
     "builder.import_config(config_path)\n",
     "\n",
     "# Access PyVista Mesh on builder\n",
@@ -43,26 +43,25 @@
     "plotter = pv.Plotter()\n",
     "\n",
     "# Open a GIF\n",
-    "plotter.open_gif(\"cmip.gif\")\n",
+    "plotter.open_gif(\"esgf.gif\")\n",
     "\n",
     "# Update T and write a frame for each updated position\n",
-    "# Pan3D Viewer showed that T ranges from Jan 16 1850 to Dec 16 2014, and it has 1980 time steps\n",
-    "# That equals one frame per month over 165 years; show one frame for every ten years.\n",
-    "n_time_frames = 1980\n",
-    "n_skip_frames = 120\n",
+    "# Pan3D Viewer showed that T ranges from Apr 01 2000 12:00 to May 30 2001 12:00, and it has 420 time steps\n",
+    "n_time_frames = 420\n",
+    "n_skip_frames = 1\n",
     "for i in range(0, n_time_frames, n_skip_frames):\n",
     "    plotter.clear()\n",
     "    builder.t_index = i\n",
-    "    actor = plotter.add_mesh(builder.mesh.warp_by_scalar(), render=False, clim=[-2, 2])\n",
+    "    actor = plotter.add_mesh(builder.mesh.warp_by_scalar(), render=False, clim=[0, 22])\n",
     "    \n",
     "    # Write a frame. This triggers a render.\n",
     "    plotter.write_frame()\n",
     "\n",
     "# Closes and finalizes GIF\n",
     "plotter.close()\n",
     "\n",
-    "# GIF generation takes about 8 mins for 16 frames (30 seconds to fetch and render each frame)\n",
-    "print(f'Saved cmip.gif. Took {datetime.now() - start} seconds.')"
+    "# GIF generation takes about 2 mins for 420 frames (approximately 0.3 seconds to fetch and render each frame)\n",
+    "print(f'Saved esgf.gif. Took {datetime.now() - start} seconds.')"
    ]
   }
  ],

diff --git a/examples/jupyter/import_config_esgf.ipynb b/examples/jupyter/import_config_esgf.ipynb
@@ -0,0 +1,52 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "bf9da680",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from pan3d import DatasetBuilder"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2f052f51-86f1-4b44-bfa9-0c6d75892571",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "config_path = '../example_config_esgf.json'\n",
+    "builder = DatasetBuilder(esgf=True, viewer=True)\n",
+    "\n",
+    "builder.import_config(config_path)\n",
+    "\n",
+    "# Show viewer in cell output\n",
+    "await builder.viewer.ready\n",
+    "builder.viewer.ui\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "pan3d",
+   "language": "python",
+   "name": "pan3d"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.18"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/examples/jupyter/import_config_pangeo.ipynb b/examples/jupyter/import_config_pangeo.ipynb
@@ -0,0 +1,52 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "bf9da680",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from pan3d import DatasetBuilder"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2f052f51-86f1-4b44-bfa9-0c6d75892571",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "config_path = '../example_config_pangeo.json'\n",
+    "builder = DatasetBuilder(pangeo=True, viewer=True)\n",
+    "\n",
+    "builder.import_config(config_path)\n",
+    "\n",
+    "# Show viewer in cell output\n",
+    "await builder.viewer.ready\n",
+    "builder.viewer.ui\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "pan3d",
+   "language": "python",
+   "name": "pan3d"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.18"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/examples/jupyter/import_config_xarray.ipynb b/examples/jupyter/import_config_xarray.ipynb
@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "id": "bf9da680",
    "metadata": {},
    "outputs": [],
@@ -18,16 +18,13 @@
    "outputs": [],
    "source": [
     "config_path = '../example_config_xarray.json'\n",
-    "builder = DatasetBuilder()\n",
-    "\n",
-    "# optional: to apply UI config, access viewer before importing\n",
-    "viewer = builder.viewer\n",
+    "builder = DatasetBuilder(viewer=True)\n",
     "\n",
     "builder.import_config(config_path)\n",
     "\n",
     "# Show viewer in cell output\n",
-    "await viewer.ready\n",
-    "viewer.ui\n"
+    "await builder.viewer.ready\n",
+    "builder.viewer.ui\n"
    ]
   }
  ],

diff --git a/examples/jupyter/manual_config.ipynb b/examples/jupyter/manual_config.ipynb
@@ -17,9 +17,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "builder = DatasetBuilder()\n",
+    "builder = DatasetBuilder(dataset='../example_dataset.nc')\n",
     "\n",
-    "builder.dataset_path = '../example_dataset.nc'\n",
     "builder.data_array_name = 'density'\n",
     "builder.x = 'length'\n",
     "builder.y = 'width'\n",

diff --git a/examples/jupyter/no_viewer.ipynb b/examples/jupyter/no_viewer.ipynb
@@ -22,8 +22,7 @@
     "# the Dataset Builder can be used alone to obtain data structures\n",
     "# and a corresponding mesh for rendering with PyVista\n",
     "\n",
-    "builder = DatasetBuilder()\n",
-    "builder.dataset_path = 'https://ncsa.osn.xsede.org/Pangeo/pangeo-forge/gpcp-feedstock/gpcp.zarr'\n",
+    "builder = DatasetBuilder(dataset='https://ncsa.osn.xsede.org/Pangeo/pangeo-forge/gpcp-feedstock/gpcp.zarr')\n",
     "builder.data_array_name = 'precip'\n",
     "builder.t_index = 2"
    ]
@@ -79,7 +78,7 @@
    "outputs": [],
    "source": [
     "# Use PyVista Plotter to display mesh rendering\n",
-    "plotter = pyvista.Plotter(notebook=True)\n",
+    "plotter = pyvista.Plotter()\n",
     "plotter.add_mesh(mesh, cmap=\"jet\")\n",
     "plotter.view_isometric()\n",
     "plotter.show()"

diff --git a/examples/jupyter/pangeo_forge.ipynb → examples/jupyter/url_config.ipynb b/examples/jupyter/pangeo_forge.ipynb → examples/jupyter/url_config.ipynb
diff --git a/examples/jupyter/viewer_state.ipynb b/examples/jupyter/viewer_state.ipynb
@@ -22,7 +22,10 @@
     "viewer = DatasetViewer()\n",
     "\n",
     "# Use Xarray example: Regional Arctic System Model\n",
-    "viewer.state.dataset_path = 'rasm'\n",
+    "viewer.state.dataset_info = {\n",
+    "    'source': 'xarray',\n",
+    "    'id': 'rasm',\n",
+    "}\n",
     "\n",
     "# Apply change, allow for auto array/coordinate selection\n",
     "viewer.state.flush()\n",