CONUS404 Temporal Aggregation

CONUS404 Temporal Aggregation#

Create daily averages from hourly data, write to a zarr dataset

import os
# Needed when boto3 >= 1.36.0 or the rechunking process will fail
# This needs to be set before the boto3 library gets loaded
# See: https://github.com/aws/aws-cli/issues/9214#issuecomment-2606619168
os.environ['AWS_REQUEST_CHECKSUM_CALCULATION'] = 'when_required'
os.environ['AWS_RESPONSE_CHECKSUM_VALIDATION'] = 'when_required'
import fsspec
import xarray as xr
import hvplot.xarray
import pystac
from packaging.version import Version
import zarr
import warnings
from dask.distributed import LocalCluster, Client
warnings.filterwarnings('ignore')

Open dataset from WMA STAC Catalog#

def get_children(catalog, collection_id=None):
    """
    This function retrieves a specified collection from a STAC catalog/collection and prints key metadata 
    for exploring/accessing the datasets contained within it.
    If there is no collection ID provided, the collections in the top level of the catalog will be printed.
    If a collection ID is provided, it will retrieve the collection with that ID from the input catalog/collection.
    If the collection ID points to a dataset, it will print the assets available for the dataset.
    If the collection ID points to another collection, it will list the child collections in the IDed collection.

    Args:
        catalog (pystac.Catalog | pystac.Collection): The STAC catalog/collection object.
        collection_id (str): The ID of the collection or dataset to retrieve from catalog.
    
    Returns:
        collection (pystac.Catalog | pystac.Collection): The collection object corresponding to the provided ID
                                                         or the top-level catalog if no ID is provided.
    """
    dataset = False
    if collection_id:
        collection = catalog.get_child(collection_id)
        if collection.assets:
            dataset = True
            print(f"{collection_id} is a dataset. Please review the assets below and select one to open.")

        else:
            print(f"{collection_id} is a collection. Please review the child items and select one to open in the next cell.")
    else:
        collection = catalog
    if dataset==True:
        # List the assets
        for asset in collection.assets:
            print(f"Asset ID: {asset}")
            print(f"    Title: {collection.assets[asset].title}")
            print(f"    Description: {collection.assets[asset].description}")
    else:
        collections = list(collection.get_collections())
        print(f"Number of collections: {len(collections)}")
        print("Collections IDs:")
        for child_collection in collections:
            id = child_collection.id
            cite_as = "Not available"
            for link in child_collection.links:
                if link.rel == "cite-as":
                    cite_as = link.target
            print(f"- {id}, Source: {cite_as}")
    return collection

# url for the WMA STAC Catalog
catalog_url = "https://api.water.usgs.gov/gdp/pygeoapi/stac/stac-collection/"

# use pystac to read the catalog
catalog = pystac.Catalog.from_file(catalog_url)

# list the collections in the catalog
catalog = get_children(catalog)

Number of collections: 43
Collections IDs:
- AIEM_permafrost, Source: Not available
- CA-BCM-2014, Source: https://doi.org/10.21429/dye8-h568
- FLET, Source: Not available
- GMO, Source: https://doi.org/10.21429/v7ys-6n72
- GMO_New, Source: https://doi.org/10.21429/c6s4-ve70
- LOCA2, Source: Not available
- PRISM_v2, Source: https://prism.oregonstate.edu/
- PuertoRico, Source: Not available
- RedRiver, Source: Not available
- SPEI, Source: Not available
- TTU_2019, Source: Not available
- TopoWx2017, Source: Not available
- WUS_HSP, Source: Not available
- alaska_et_2020, Source: Not available
- bcca, Source: Not available
- bcsd_mon_vic, Source: http://gdo-dcp.ucllnl.org/downscaled_cmip_projections/
- bcsd_obs, Source: http://gdo-dcp.ucllnl.org/downscaled_cmip_projections/
- cmip5_bcsd, Source: https://doi.org/10.21429/sbxv-1n90
- conus404-pgw, Source: Not available
- conus404, Source: Not available
- cooper, Source: Not available
- cprep, Source: Not available
- dcp_compressed, Source: https://doi.org/https%3A//doi.org/10.21429/j9f1-b218
- gridMET, Source: https://www.climatologylab.org/gridmet.html
- hawaii_2018, Source: Not available
- iclus, Source: Not available
- loca, Source: Not available
- maca-vic, Source: Not available
- macav2, Source: Not available
- maurer, Source: https://doi.org/10.21429/m7y0-xy02
- mows, Source: Not available
- nc-casc-snow, Source: Not available
- nlcd, Source: Not available
- notaro_2018, Source: Not available
- pacis, Source: Not available
- puerto_rico, Source: Not available
- red_river_2018, Source: https://doi.org/10.21429/em59-hn43
- serap, Source: Not available
- slr2d, Source: https://doi.org/10.21429/66gt-dm26
- snodas, Source: https://doi.org/10.7265/N5TB14TC
- ssebopeta, Source: Not available
- stageiv_combined, Source: https://pubs.usgs.gov/fs/2013/3035/
- wicci, Source: https://doi.org/10.21429/dtp5-z505

# select a collection from the catalog, replace the collection ID with the one you want to use:
collection = get_children(catalog, collection_id="conus404")

conus404 is a collection. Please review the child items and select one to open in the next cell.

Number of collections: 4
Collections IDs:
- conus404_daily, Source: https://doi.org/10.5066/P9PHPK4F
- conus404_hourly, Source: https://doi.org/10.5066/P9PHPK4F
- conus404_monthly, Source: https://doi.org/10.5066/P9PHPK4F
- conus404_xtrm_daily, Source: https://doi.org/10.5066/P9PHPK4F

# select a collection from the catalog, replace the collection ID with the one you want to use:
collection = get_children(collection, collection_id="conus404_hourly")

conus404_hourly is a dataset. Please review the assets below and select one to open.
Asset ID: zarr-s3-osn
    Title: Free access to zarr via S3 API
    Description: Free, public access to zarr data store via the S3 API. This data is stored on an Open Storage Network Pod.
Asset ID: zarr-disk-hovenweep
    Title: USGS internal access to zarr via Hovenweep
    Description: USGS internal access to zarr data store via disk storage attached to the Hovenweep supercomputer.
Asset ID: default
    Title: None
    Description: None

# replace with the asset ID you want to use:
selected_asset_id = "zarr-s3-osn"

# read the asset metadata
asset = collection.assets[selected_asset_id]

2) Set Up AWS Credentials (Optional)#

This notebook reads data from the OSN pod by default, which is object store data on a high speed internet connection that is free to access from any environment. If you change this notebook to use one of the CONUS404 datasets stored on S3 (options ending in -cloud), you will be pulling data from a requester-pays S3 bucket. This means you have to set up your AWS credentials, else we won’t be able to load the data. Please note that reading the -cloud data from S3 may incur charges if you are reading data outside of the us-west-2 region or running the notebook outside of the cloud altogether. If you would like to access one of the -cloud options, uncomment and run the following code snippet to set up your AWS credentials. You can find more info about this AWS helper function here.

# uncomment the lines below to read in your AWS credentials if you want to access data from a requester-pays bucket (-cloud)
# os.environ['AWS_PROFILE'] = 'default'
# %run ../../environment_set_up/Help_AWS_Credentials.ipynb

Parallelize with Dask#

Some of the steps we will take are aware of parallel clustered compute environments using dask. We’re going to start a cluster now so that future steps can take advantage of this ability.

This is an optional step, but speed ups data loading significantly, especially when accessing data from the cloud.

We have documentation on how to start a Dask Cluster in different computing environments here.

%run ../../environment_set_up/Start_Dask_Cluster_Nebari.ipynb
## If this notebook is not being run on Nebari, replace the above 
## path name with a helper appropriate to your compute environment.  Examples:
# %run ../../environment_set_up/Start_Dask_Cluster_Denali.ipynb
# %run ../../environment_set_up/Start_Dask_Cluster_Tallgrass.ipynb
# %run ../../environment_set_up/Start_Dask_Cluster_Desktop.ipynb

The 'cluster' object can be used to adjust cluster behavior.  i.e. 'cluster.adapt(minimum=10)'
The 'client' object can be used to directly interact with the cluster.  i.e. 'client.submit(func)' 
The link to view the client dashboard is:
>  https://nebari.chs.usgs.gov/gateway/clusters/nebari.2e13a3840c3c4fe1a76095b9e9534db0/status

Explore the dataset#

if Version(zarr.__version__) < Version("3.0.0"):
    ds = xr.open_dataset(
        asset.href,
        storage_options=asset.extra_fields['xarray:storage_options'],
        **asset.extra_fields['xarray:open_kwargs']
    )
else:
    ds = xr.open_dataset(
    asset.href,
    storage_options=asset.extra_fields['xarray:storage_options'],
    **asset.extra_fields['xarray:open_kwargs'],
    zarr_format=2
    )

ds

<xarray.Dataset> Size: 222TB
Dimensions:         (time: 376945, y: 1015, x: 1367, bottom_top_stag: 51,
                     bottom_top: 50, soil_layers_stag: 4, x_stag: 1368,
                     y_stag: 1016, snow_layers_stag: 3, snso_layers_stag: 7)
Coordinates:
    lat             (y, x) float32 6MB dask.array<chunksize=(175, 175), meta=np.ndarray>
    lat_u           (y, x_stag) float32 6MB dask.array<chunksize=(175, 175), meta=np.ndarray>
    lat_v           (y_stag, x) float32 6MB dask.array<chunksize=(175, 175), meta=np.ndarray>
    lon             (y, x) float32 6MB dask.array<chunksize=(175, 175), meta=np.ndarray>
    lon_u           (y, x_stag) float32 6MB dask.array<chunksize=(175, 175), meta=np.ndarray>
    lon_v           (y_stag, x) float32 6MB dask.array<chunksize=(175, 175), meta=np.ndarray>
  * time            (time) datetime64[ns] 3MB 1979-10-01 ... 2022-10-01
  * x               (x) float64 11kB -2.732e+06 -2.728e+06 ... 2.732e+06
  * y               (y) float64 8kB -2.028e+06 -2.024e+06 ... 2.028e+06
Dimensions without coordinates: bottom_top_stag, bottom_top, soil_layers_stag,
                                x_stag, y_stag, snow_layers_stag,
                                snso_layers_stag
Data variables: (12/153)
    ACDEWC          (time, y, x) float32 2TB dask.array<chunksize=(144, 175, 175), meta=np.ndarray>
    ACDRIPR         (time, y, x) float32 2TB dask.array<chunksize=(144, 175, 175), meta=np.ndarray>
    ACDRIPS         (time, y, x) float32 2TB dask.array<chunksize=(144, 175, 175), meta=np.ndarray>
    ACECAN          (time, y, x) float32 2TB dask.array<chunksize=(144, 175, 175), meta=np.ndarray>
    ACEDIR          (time, y, x) float32 2TB dask.array<chunksize=(144, 175, 175), meta=np.ndarray>
    ACETLSM         (time, y, x) float32 2TB dask.array<chunksize=(144, 175, 175), meta=np.ndarray>
    ...              ...
    ZNU             (bottom_top) float32 200B dask.array<chunksize=(50,), meta=np.ndarray>
    ZNW             (bottom_top_stag) float32 204B dask.array<chunksize=(51,), meta=np.ndarray>
    ZS              (soil_layers_stag) float32 16B dask.array<chunksize=(4,), meta=np.ndarray>
    ZSNSO           (time, snso_layers_stag, y, x) float32 15TB dask.array<chunksize=(144, 7, 175, 175), meta=np.ndarray>
    ZWT             (time, y, x) float32 2TB dask.array<chunksize=(144, 175, 175), meta=np.ndarray>
    crs             int64 8B ...
Attributes: (12/148)
    AER_ANGEXP_OPT:                  1
    AER_ANGEXP_VAL:                  1.2999999523162842
    AER_AOD550_OPT:                  1
    AER_AOD550_VAL:                  0.11999999731779099
    AER_ASY_OPT:                     1
    AER_ASY_VAL:                     0.8999999761581421
    ...                              ...
    WEST-EAST_PATCH_START_STAG:      1
    WEST-EAST_PATCH_START_UNSTAG:    1
    W_DAMPING:                       1
    YSU_TOPDOWN_PBLMIX:              0
    history:                         Tue Mar 29 16:35:22 2022: ncrcat -A -vW ...
    history_of_appended_files:       Tue Mar 29 16:35:22 2022: Appended file ...
ds.T2

<xarray.DataArray 'T2' (time: 376945, y: 1015, x: 1367)> Size: 2TB
dask.array<open_dataset-T2, shape=(376945, 1015, 1367), dtype=float32, chunksize=(144, 175, 175), chunktype=numpy.ndarray>
Coordinates:
    lat      (y, x) float32 6MB dask.array<chunksize=(175, 175), meta=np.ndarray>
    lon      (y, x) float32 6MB dask.array<chunksize=(175, 175), meta=np.ndarray>
  * time     (time) datetime64[ns] 3MB 1979-10-01 ... 2022-10-01
  * x        (x) float64 11kB -2.732e+06 -2.728e+06 ... 2.728e+06 2.732e+06
  * y        (y) float64 8kB -2.028e+06 -2.024e+06 ... 2.024e+06 2.028e+06
Attributes:
    description:   TEMP at 2 M
    grid_mapping:  crs
    long_name:     Temperature at 2 meters
    units:         K

Daily averages#

Time averages of any type are easy to do with xarray. Here we do 24 hour averages, and set the time offset to 12 hours, so that the time values are in the middle of the averaging period.

Digital Earth Africa has a great Working with Time in Xarray tutorial.

In the example below we just do a few days with a few variables as a quick demo.

%%time
ds_subset = ds[['T2','U10']].sel(time=slice('2017-01-02','2017-01-13'))

CPU times: user 9.45 ms, sys: 8.3 ms, total: 17.7 ms
Wall time: 17 ms

ds_subset_daily = ds_subset.resample(time="24H", offset="12h", label='right').mean()

ds_subset_daily

<xarray.Dataset> Size: 155MB
Dimensions:  (time: 13, y: 1015, x: 1367)
Coordinates:
    lat      (y, x) float32 6MB dask.array<chunksize=(175, 175), meta=np.ndarray>
    lon      (y, x) float32 6MB dask.array<chunksize=(175, 175), meta=np.ndarray>
  * x        (x) float64 11kB -2.732e+06 -2.728e+06 ... 2.728e+06 2.732e+06
  * y        (y) float64 8kB -2.028e+06 -2.024e+06 ... 2.024e+06 2.028e+06
  * time     (time) datetime64[ns] 104B 2017-01-02T12:00:00 ... 2017-01-14T12...
Data variables:
    T2       (time, y, x) float32 72MB dask.array<chunksize=(6, 175, 175), meta=np.ndarray>
    U10      (time, y, x) float32 72MB dask.array<chunksize=(6, 175, 175), meta=np.ndarray>
Attributes: (12/148)
    AER_ANGEXP_OPT:                  1
    AER_ANGEXP_VAL:                  1.2999999523162842
    AER_AOD550_OPT:                  1
    AER_AOD550_VAL:                  0.11999999731779099
    AER_ASY_OPT:                     1
    AER_ASY_VAL:                     0.8999999761581421
    ...                              ...
    WEST-EAST_PATCH_START_STAG:      1
    WEST-EAST_PATCH_START_UNSTAG:    1
    W_DAMPING:                       1
    YSU_TOPDOWN_PBLMIX:              0
    history:                         Tue Mar 29 16:35:22 2022: ncrcat -A -vW ...
    history_of_appended_files:       Tue Mar 29 16:35:22 2022: Appended file ...
ds_subset_daily.hvplot.quadmesh(x='lon', y='lat', rasterize=True, 
                             geo=True, tiles='OSM', alpha=0.7, cmap='turbo')

WARNING:param.load_subset: Callable raised "FutureCancelledError()".
Invoked as load_subset(np.datetime64('2017-01-02T12:00:00.000000000'))

WARNING:param.dynamic_operation: Callable raised "FutureCancelledError()".
Invoked as dynamic_operation(np.datetime64('2017-01-02T12:00:00.000000000'))

WARNING:param.dynamic_operation: Callable raised "FutureCancelledError()".
Invoked as dynamic_operation(np.datetime64('2017-01-02T12:00:00.000000000'), height=400, scale=1.0, width=400, x_range=None, y_range=None)

WARNING:param.dynamic_mul: Callable raised "FutureCancelledError()".
Invoked as dynamic_mul(np.datetime64('2017-01-02T12:00:00.000000000'))

WARNING:param.dynamic_operation: Callable raised "FutureCancelledError()".
Invoked as dynamic_operation(np.datetime64('2017-01-02T12:00:00.000000000'))

---------------------------------------------------------------------------
FutureCancelledError                      Traceback (most recent call last)
File /home/conda/hytest/f9b85f69-1756479041-247-pangeo/lib/python3.11/site-packages/IPython/core/formatters.py:1036, in MimeBundleFormatter.__call__(self, obj, include, exclude)
   1033     method = get_real_method(obj, self.print_method)
   1035     if method is not None:
-> 1036         return method(include=include, exclude=exclude)
   1037     return None
   1038 else:

File /home/conda/hytest/f9b85f69-1756479041-247-pangeo/lib/python3.11/site-packages/holoviews/core/dimension.py:1381, in Dimensioned._repr_mimebundle_(self, include, exclude)
   1374 def _repr_mimebundle_(self, include=None, exclude=None):
   1375     """Resolves the class hierarchy for the class rendering the
   1376     object using any display hooks registered on Store.display
   1377     hooks.  The output of all registered display_hooks is then
   1378     combined and returned.
   1379 
   1380     """
-> 1381     return Store.render(self)

File /home/conda/hytest/f9b85f69-1756479041-247-pangeo/lib/python3.11/site-packages/holoviews/core/options.py:1433, in Store.render(cls, obj)
   1431 data, metadata = {}, {}
   1432 for hook in hooks:
-> 1433     ret = hook(obj)
   1434     if ret is None:
   1435         continue

File /home/conda/hytest/f9b85f69-1756479041-247-pangeo/lib/python3.11/site-packages/holoviews/ipython/display_hooks.py:340, in pprint_display(obj)
    338 if not ip.display_formatter.formatters['text/plain'].pprint:
    339     return None
--> 340 return display(obj, raw_output=True)

File /home/conda/hytest/f9b85f69-1756479041-247-pangeo/lib/python3.11/site-packages/holoviews/ipython/display_hooks.py:314, in display(obj, raw_output, **kwargs)
    312 elif isinstance(obj, (HoloMap, DynamicMap)):
    313     with option_state(obj):
--> 314         output = map_display(obj)
    315 elif isinstance(obj, Plot):
    316     output = render(obj)

File /home/conda/hytest/f9b85f69-1756479041-247-pangeo/lib/python3.11/site-packages/holoviews/ipython/display_hooks.py:205, in display_hook.<locals>.wrapped(element)
    203 try:
    204     max_frames = OutputSettings.options['max_frames']
--> 205     mimebundle = fn(element, max_frames=max_frames)
    206     if mimebundle is None:
    207         return {}, {}

File /home/conda/hytest/f9b85f69-1756479041-247-pangeo/lib/python3.11/site-packages/holoviews/ipython/display_hooks.py:260, in map_display(vmap, max_frames)
    257     max_frame_warning(max_frames)
    258     return None
--> 260 return render(vmap)

File /home/conda/hytest/f9b85f69-1756479041-247-pangeo/lib/python3.11/site-packages/holoviews/ipython/display_hooks.py:76, in render(obj, **kwargs)
     73 if renderer.fig == 'pdf':
     74     renderer = renderer.instance(fig='png')
---> 76 return renderer.components(obj, **kwargs)

File /home/conda/hytest/f9b85f69-1756479041-247-pangeo/lib/python3.11/site-packages/holoviews/plotting/renderer.py:396, in Renderer.components(self, obj, fmt, comm, **kwargs)
    394 embed = (not (dynamic or streams or self.widget_mode == 'live') or config.embed)
    395 if embed or config.comms == 'default':
--> 396     return self._render_panel(plot, embed, comm)
    397 return self._render_ipywidget(plot)

File /home/conda/hytest/f9b85f69-1756479041-247-pangeo/lib/python3.11/site-packages/holoviews/plotting/renderer.py:403, in Renderer._render_panel(self, plot, embed, comm)
    401 doc = Document()
    402 with config.set(embed=embed):
--> 403     model = plot.layout._render_model(doc, comm)
    404 if embed:
    405     return render_model(model, comm)

File /home/conda/hytest/f9b85f69-1756479041-247-pangeo/lib/python3.11/site-packages/panel/viewable.py:768, in Viewable._render_model(self, doc, comm)
    766 if comm is None:
    767     comm = state._comm_manager.get_server_comm()
--> 768 model = self.get_root(doc, comm)
    770 if self._design and self._design.theme.bokeh_theme:
    771     doc.theme = self._design.theme.bokeh_theme

File /home/conda/hytest/f9b85f69-1756479041-247-pangeo/lib/python3.11/site-packages/panel/layout/base.py:329, in Panel.get_root(self, doc, comm, preprocess)
    325 def get_root(
    326     self, doc: Document | None = None, comm: Comm | None = None,
    327     preprocess: bool = True
    328 ) -> Model:
--> 329     root = super().get_root(doc, comm, preprocess)
    330     # ALERT: Find a better way to handle this
    331     if hasattr(root, 'styles') and 'overflow-x' in root.styles:

File /home/conda/hytest/f9b85f69-1756479041-247-pangeo/lib/python3.11/site-packages/panel/viewable.py:698, in Renderable.get_root(self, doc, comm, preprocess)
    696 wrapper = self._design._wrapper(self)
    697 if wrapper is self:
--> 698     root = self._get_model(doc, comm=comm)
    699     if preprocess:
    700         self._preprocess(root)

File /home/conda/hytest/f9b85f69-1756479041-247-pangeo/lib/python3.11/site-packages/panel/layout/base.py:313, in Panel._get_model(self, doc, root, parent, comm)
    311 root = root or model
    312 self._models[root.ref['id']] = (model, parent)
--> 313 objects, _ = self._get_objects(model, [], doc, root, comm)
    314 props = self._get_properties(doc)
    315 props[self._property_mapping['objects']] = objects

File /home/conda/hytest/f9b85f69-1756479041-247-pangeo/lib/python3.11/site-packages/panel/layout/base.py:295, in Panel._get_objects(self, model, old_objects, doc, root, comm)
    293 else:
    294     try:
--> 295         child = pane._get_model(doc, root, model, comm)
    296     except RerenderError as e:
    297         if e.layout is not None and e.layout is not self:

File /home/conda/hytest/f9b85f69-1756479041-247-pangeo/lib/python3.11/site-packages/panel/pane/holoviews.py:445, in HoloViews._get_model(self, doc, root, parent, comm)
    443     plot = self.object
    444 else:
--> 445     plot = self._render(doc, comm, root)
    447 plot.pane = self
    448 backend = plot.renderer.backend

File /home/conda/hytest/f9b85f69-1756479041-247-pangeo/lib/python3.11/site-packages/panel/pane/holoviews.py:539, in HoloViews._render(self, doc, comm, root)
    536     if comm:
    537         kwargs['comm'] = comm
--> 539 return renderer.get_plot(self.object, **kwargs)

File /home/conda/hytest/f9b85f69-1756479041-247-pangeo/lib/python3.11/site-packages/holoviews/plotting/bokeh/renderer.py:70, in BokehRenderer.get_plot(self_or_cls, obj, doc, renderer, **kwargs)
     63 @bothmethod
     64 def get_plot(self_or_cls, obj, doc=None, renderer=None, **kwargs):
     65     """Given a HoloViews Viewable return a corresponding plot instance.
     66     Allows supplying a document attach the plot to, useful when
     67     combining the bokeh model with another plot.
     68 
     69     """
---> 70     plot = super().get_plot(obj, doc, renderer, **kwargs)
     71     if plot.document is None:
     72         plot.document = Document() if self_or_cls.notebook_context else curdoc()

File /home/conda/hytest/f9b85f69-1756479041-247-pangeo/lib/python3.11/site-packages/holoviews/plotting/renderer.py:216, in Renderer.get_plot(self_or_cls, obj, doc, renderer, comm, **kwargs)
    213     raise SkipRendering(msg.format(dims=dims))
    215 # Initialize DynamicMaps with first data item
--> 216 initialize_dynamic(obj)
    218 if not renderer:
    219     renderer = self_or_cls

File /home/conda/hytest/f9b85f69-1756479041-247-pangeo/lib/python3.11/site-packages/holoviews/plotting/util.py:277, in initialize_dynamic(obj)
    275     continue
    276 if not len(dmap):
--> 277     dmap[dmap._initial_key()]

File /home/conda/hytest/f9b85f69-1756479041-247-pangeo/lib/python3.11/site-packages/holoviews/core/spaces.py:1302, in DynamicMap.__getitem__(self, key)
   1300 # Not a cross product and nothing cached so compute element.
   1301 if cache is not None: return cache
-> 1302 val = self._execute_callback(*tuple_key)
   1303 if data_slice:
   1304     val = self._dataslice(val, data_slice)

File /home/conda/hytest/f9b85f69-1756479041-247-pangeo/lib/python3.11/site-packages/holoviews/core/spaces.py:1050, in DynamicMap._execute_callback(self, *args)
   1047     kwargs['_memoization_hash_'] = hash_items
   1049 with dynamicmap_memoization(self.callback, self.streams):
-> 1050     retval = self.callback(*args, **kwargs)
   1051 return self._style(retval)

File /home/conda/hytest/f9b85f69-1756479041-247-pangeo/lib/python3.11/site-packages/holoviews/core/spaces.py:626, in Callable.__call__(self, *args, **kwargs)
    623     args, kwargs = (), dict(pos_kwargs, **kwargs)
    625 try:
--> 626     ret = self.callable(*args, **kwargs)
    627 except KeyError:
    628     # KeyError is caught separately because it is used to signal
    629     # invalid keys on DynamicMap and should not warn
    630     raise

File /home/conda/hytest/f9b85f69-1756479041-247-pangeo/lib/python3.11/site-packages/holoviews/util/__init__.py:1070, in Dynamic._dynamic_operation.<locals>.dynamic_operation(*key, **kwargs)
   1069 def dynamic_operation(*key, **kwargs):
-> 1070     key, obj = resolve(key, kwargs)
   1071     return apply(obj, *key, **kwargs)

File /home/conda/hytest/f9b85f69-1756479041-247-pangeo/lib/python3.11/site-packages/holoviews/util/__init__.py:1059, in Dynamic._dynamic_operation.<locals>.resolve(key, kwargs)
   1057 elif isinstance(map_obj, DynamicMap) and map_obj._posarg_keys and not key:
   1058     key = tuple(kwargs[k] for k in map_obj._posarg_keys)
-> 1059 return key, map_obj[key]

File /home/conda/hytest/f9b85f69-1756479041-247-pangeo/lib/python3.11/site-packages/holoviews/core/spaces.py:1302, in DynamicMap.__getitem__(self, key)
   1300 # Not a cross product and nothing cached so compute element.
   1301 if cache is not None: return cache
-> 1302 val = self._execute_callback(*tuple_key)
   1303 if data_slice:
   1304     val = self._dataslice(val, data_slice)

File /home/conda/hytest/f9b85f69-1756479041-247-pangeo/lib/python3.11/site-packages/holoviews/core/spaces.py:1050, in DynamicMap._execute_callback(self, *args)
   1047     kwargs['_memoization_hash_'] = hash_items
   1049 with dynamicmap_memoization(self.callback, self.streams):
-> 1050     retval = self.callback(*args, **kwargs)
   1051 return self._style(retval)

File /home/conda/hytest/f9b85f69-1756479041-247-pangeo/lib/python3.11/site-packages/holoviews/core/spaces.py:626, in Callable.__call__(self, *args, **kwargs)
    623     args, kwargs = (), dict(pos_kwargs, **kwargs)
    625 try:
--> 626     ret = self.callable(*args, **kwargs)
    627 except KeyError:
    628     # KeyError is caught separately because it is used to signal
    629     # invalid keys on DynamicMap and should not warn
    630     raise

File /home/conda/hytest/f9b85f69-1756479041-247-pangeo/lib/python3.11/site-packages/holoviews/core/overlay.py:36, in Overlayable.__mul__.<locals>.dynamic_mul(*args, **kwargs)
     35 def dynamic_mul(*args, **kwargs):
---> 36     element = other[args]
     37     return self * element

File /home/conda/hytest/f9b85f69-1756479041-247-pangeo/lib/python3.11/site-packages/holoviews/core/spaces.py:1302, in DynamicMap.__getitem__(self, key)
   1300 # Not a cross product and nothing cached so compute element.
   1301 if cache is not None: return cache
-> 1302 val = self._execute_callback(*tuple_key)
   1303 if data_slice:
   1304     val = self._dataslice(val, data_slice)

File /home/conda/hytest/f9b85f69-1756479041-247-pangeo/lib/python3.11/site-packages/holoviews/core/spaces.py:1050, in DynamicMap._execute_callback(self, *args)
   1047     kwargs['_memoization_hash_'] = hash_items
   1049 with dynamicmap_memoization(self.callback, self.streams):
-> 1050     retval = self.callback(*args, **kwargs)
   1051 return self._style(retval)

File /home/conda/hytest/f9b85f69-1756479041-247-pangeo/lib/python3.11/site-packages/holoviews/core/spaces.py:626, in Callable.__call__(self, *args, **kwargs)
    623     args, kwargs = (), dict(pos_kwargs, **kwargs)
    625 try:
--> 626     ret = self.callable(*args, **kwargs)
    627 except KeyError:
    628     # KeyError is caught separately because it is used to signal
    629     # invalid keys on DynamicMap and should not warn
    630     raise

File /home/conda/hytest/f9b85f69-1756479041-247-pangeo/lib/python3.11/site-packages/holoviews/util/__init__.py:1070, in Dynamic._dynamic_operation.<locals>.dynamic_operation(*key, **kwargs)
   1069 def dynamic_operation(*key, **kwargs):
-> 1070     key, obj = resolve(key, kwargs)
   1071     return apply(obj, *key, **kwargs)

File /home/conda/hytest/f9b85f69-1756479041-247-pangeo/lib/python3.11/site-packages/holoviews/util/__init__.py:1059, in Dynamic._dynamic_operation.<locals>.resolve(key, kwargs)
   1057 elif isinstance(map_obj, DynamicMap) and map_obj._posarg_keys and not key:
   1058     key = tuple(kwargs[k] for k in map_obj._posarg_keys)
-> 1059 return key, map_obj[key]

File /home/conda/hytest/f9b85f69-1756479041-247-pangeo/lib/python3.11/site-packages/holoviews/core/spaces.py:1302, in DynamicMap.__getitem__(self, key)
   1300 # Not a cross product and nothing cached so compute element.
   1301 if cache is not None: return cache
-> 1302 val = self._execute_callback(*tuple_key)
   1303 if data_slice:
   1304     val = self._dataslice(val, data_slice)

File /home/conda/hytest/f9b85f69-1756479041-247-pangeo/lib/python3.11/site-packages/holoviews/core/spaces.py:1050, in DynamicMap._execute_callback(self, *args)
   1047     kwargs['_memoization_hash_'] = hash_items
   1049 with dynamicmap_memoization(self.callback, self.streams):
-> 1050     retval = self.callback(*args, **kwargs)
   1051 return self._style(retval)

File /home/conda/hytest/f9b85f69-1756479041-247-pangeo/lib/python3.11/site-packages/holoviews/core/spaces.py:626, in Callable.__call__(self, *args, **kwargs)
    623     args, kwargs = (), dict(pos_kwargs, **kwargs)
    625 try:
--> 626     ret = self.callable(*args, **kwargs)
    627 except KeyError:
    628     # KeyError is caught separately because it is used to signal
    629     # invalid keys on DynamicMap and should not warn
    630     raise

File /home/conda/hytest/f9b85f69-1756479041-247-pangeo/lib/python3.11/site-packages/holoviews/util/__init__.py:1070, in Dynamic._dynamic_operation.<locals>.dynamic_operation(*key, **kwargs)
   1069 def dynamic_operation(*key, **kwargs):
-> 1070     key, obj = resolve(key, kwargs)
   1071     return apply(obj, *key, **kwargs)

File /home/conda/hytest/f9b85f69-1756479041-247-pangeo/lib/python3.11/site-packages/holoviews/util/__init__.py:1059, in Dynamic._dynamic_operation.<locals>.resolve(key, kwargs)
   1057 elif isinstance(map_obj, DynamicMap) and map_obj._posarg_keys and not key:
   1058     key = tuple(kwargs[k] for k in map_obj._posarg_keys)
-> 1059 return key, map_obj[key]

File /home/conda/hytest/f9b85f69-1756479041-247-pangeo/lib/python3.11/site-packages/holoviews/core/spaces.py:1302, in DynamicMap.__getitem__(self, key)
   1300 # Not a cross product and nothing cached so compute element.
   1301 if cache is not None: return cache
-> 1302 val = self._execute_callback(*tuple_key)
   1303 if data_slice:
   1304     val = self._dataslice(val, data_slice)

File /home/conda/hytest/f9b85f69-1756479041-247-pangeo/lib/python3.11/site-packages/holoviews/core/spaces.py:1050, in DynamicMap._execute_callback(self, *args)
   1047     kwargs['_memoization_hash_'] = hash_items
   1049 with dynamicmap_memoization(self.callback, self.streams):
-> 1050     retval = self.callback(*args, **kwargs)
   1051 return self._style(retval)

File /home/conda/hytest/f9b85f69-1756479041-247-pangeo/lib/python3.11/site-packages/holoviews/core/spaces.py:626, in Callable.__call__(self, *args, **kwargs)
    623     args, kwargs = (), dict(pos_kwargs, **kwargs)
    625 try:
--> 626     ret = self.callable(*args, **kwargs)
    627 except KeyError:
    628     # KeyError is caught separately because it is used to signal
    629     # invalid keys on DynamicMap and should not warn
    630     raise

File /home/conda/hytest/f9b85f69-1756479041-247-pangeo/lib/python3.11/site-packages/holoviews/core/data/__init__.py:1074, in Dataset.groupby.<locals>.load_subset(*args)
   1071 if np.isscalar(group):
   1072     return group_type(([group],), group=self.group,
   1073                       label=self.label, vdims=self.vdims)
-> 1074 data = group.reindex(kdims)
   1075 if drop_dim and self.interface.gridded:
   1076     data = data.columns()

File /home/conda/hytest/f9b85f69-1756479041-247-pangeo/lib/python3.11/site-packages/holoviews/core/data/__init__.py:201, in PipelineMeta.pipelined.<locals>.pipelined_fn(*args, **kwargs)
    198     inst._in_method = True
    200 try:
--> 201     result = method_fn(*args, **kwargs)
    202     if PipelineMeta.disable:
    203         return result

File /home/conda/hytest/f9b85f69-1756479041-247-pangeo/lib/python3.11/site-packages/holoviews/core/data/__init__.py:707, in Dataset.reindex(self, kdims, vdims)
    705 scalars = []
    706 if gridded:
--> 707     coords = [(d, self.interface.coords(self, d.name)) for d in self.kdims]
    708     scalars = [d for d, vs in coords if len(vs) == 1]
    710 if kdims is None:
    711     # If no key dimensions are defined and interface is gridded
    712     # drop all scalar key dimensions

File /home/conda/hytest/f9b85f69-1756479041-247-pangeo/lib/python3.11/site-packages/holoviews/core/data/__init__.py:707, in <listcomp>(.0)
    705 scalars = []
    706 if gridded:
--> 707     coords = [(d, self.interface.coords(self, d.name)) for d in self.kdims]
    708     scalars = [d for d, vs in coords if len(vs) == 1]
    710 if kdims is None:
    711     # If no key dimensions are defined and interface is gridded
    712     # drop all scalar key dimensions

File /home/conda/hytest/f9b85f69-1756479041-247-pangeo/lib/python3.11/site-packages/holoviews/core/data/xarray.py:376, in XArrayInterface.coords(cls, dataset, dimension, ordered, expanded, edges)
    373         data = cls._infer_interval_breaks(data, axis=1)
    374         data = cls._infer_interval_breaks(data, axis=0)
--> 376     return data.values if isinstance(data, xr.DataArray) else data
    378 data = np.atleast_1d(dataset.data[dim].data)
    379 if ordered and data.shape and np.all(data[1:] < data[:-1]):

File /home/conda/hytest/f9b85f69-1756479041-247-pangeo/lib/python3.11/site-packages/xarray/core/dataarray.py:814, in DataArray.values(self)
    801 @property
    802 def values(self) -> np.ndarray:
    803     """
    804     The array's data converted to numpy.ndarray.
    805 
   (...)    812     to this array may be reflected in the DataArray as well.
    813     """
--> 814     return self.variable.values

File /home/conda/hytest/f9b85f69-1756479041-247-pangeo/lib/python3.11/site-packages/xarray/core/variable.py:566, in Variable.values(self)
    563 @property
    564 def values(self) -> np.ndarray:
    565     """The variable's data as a numpy.ndarray"""
--> 566     return _as_array_or_item(self._data)

File /home/conda/hytest/f9b85f69-1756479041-247-pangeo/lib/python3.11/site-packages/xarray/core/variable.py:363, in _as_array_or_item(data)
    349 def _as_array_or_item(data):
    350     """Return the given values as a numpy array, or as an individual item if
    351     it's a 0d datetime64 or timedelta64 array.
    352 
   (...)    361     TODO: remove this (replace with np.asarray) once these issues are fixed
    362     """
--> 363     data = np.asarray(data)
    364     if data.ndim == 0:
    365         if data.dtype.kind == "M":

File /home/conda/hytest/f9b85f69-1756479041-247-pangeo/lib/python3.11/site-packages/dask/array/core.py:1729, in Array.__array__(self, dtype, copy, **kwargs)
   1722 if copy is False:
   1723     warnings.warn(
   1724         "Can't acquire a memory view of a Dask array. "
   1725         "This will raise in the future.",
   1726         FutureWarning,
   1727     )
-> 1729 x = self.compute()
   1731 # Apply requested dtype and convert non-numpy backends to numpy.
   1732 # If copy is True, numpy is going to perform its own deep copy
   1733 # after this method returns.
   1734 # If copy is None, finalize() ensures that the returned object
   1735 # does not share memory with an object stored in the graph or on a
   1736 # process-local Worker.
   1737 return np.asarray(x, dtype=dtype)

File /home/conda/hytest/f9b85f69-1756479041-247-pangeo/lib/python3.11/site-packages/dask/base.py:373, in DaskMethodsMixin.compute(self, **kwargs)
    349 def compute(self, **kwargs):
    350     """Compute this dask collection
    351 
    352     This turns a lazy Dask collection into its in-memory equivalent.
   (...)    371     dask.compute
    372     """
--> 373     (result,) = compute(self, traverse=False, **kwargs)
    374     return result

File /home/conda/hytest/f9b85f69-1756479041-247-pangeo/lib/python3.11/site-packages/dask/base.py:681, in compute(traverse, optimize_graph, scheduler, get, *args, **kwargs)
    678     expr = expr.optimize()
    679     keys = list(flatten(expr.__dask_keys__()))
--> 681     results = schedule(expr, keys, **kwargs)
    683 return repack(results)

File /home/conda/hytest/f9b85f69-1756479041-247-pangeo/lib/python3.11/site-packages/distributed/client.py:2417, in Client._gather(self, futures, errors, direct, local_worker)
   2415     exception = st.exception
   2416     traceback = st.traceback
-> 2417     raise exception.with_traceback(traceback)
   2418 if errors == "skip":
   2419     bad_keys.add(key)

FutureCancelledError: finalize-hlgfinalizecompute-204b4d5de9e14ac2b878a4f436f41865 cancelled for reason: scheduler-connection-lost.
Client lost the connection to the scheduler. Please check your connection and re-run your work.

:DynamicMap   [time]

Write daily values as a Zarr dataset (to onprem or cloud)#

You will need to to turn the following cell from raw to code and update the filepaths in order to save out your data.

Shutdown cluster#

client.close(); cluster.shutdown()
Contents