Skip to article frontmatterSkip to article content

Creating an Interactive Dashboard using hvPlot and Panel

import xarray as xr
import holoviews as hv
import panel as pn
import hvplot.xarray
from holoviews import opts

pn.extension()
Loading...

Data

In this notebook, we are going to load annual mean dataset of 2-m temperature from the ERA5 reanalysis that we preprocessed in Zarr format. Please see the preprocessing notebooks for the required steps.

rda_url = "https://data.rda.ucar.edu/"
annual_means = rda_url + "pythia_era5_24/annual_means/"
xrds = xr.open_dataset(annual_means + "temp_2m_annual_1940_2023.zarr", engine="zarr")
xrds
---------------------------------------------------------------------------
FileNotFoundError                         Traceback (most recent call last)
File ~/micromamba/envs/ERA5_interactive/lib/python3.13/site-packages/fsspec/mapping.py:155, in FSMap.__getitem__(self, key, default)
    154 try:
--> 155     result = self.fs.cat(k)
    156 except self.missing_exceptions as exc:

File ~/micromamba/envs/ERA5_interactive/lib/python3.13/site-packages/fsspec/asyn.py:118, in sync_wrapper.<locals>.wrapper(*args, **kwargs)
    117 self = obj or args[0]
--> 118 return sync(self.loop, func, *args, **kwargs)

File ~/micromamba/envs/ERA5_interactive/lib/python3.13/site-packages/fsspec/asyn.py:103, in sync(loop, func, timeout, *args, **kwargs)
    102 elif isinstance(return_result, BaseException):
--> 103     raise return_result
    104 else:

File ~/micromamba/envs/ERA5_interactive/lib/python3.13/site-packages/fsspec/asyn.py:56, in _runner(event, coro, result, timeout)
     55 try:
---> 56     result[0] = await coro
     57 except Exception as ex:

File ~/micromamba/envs/ERA5_interactive/lib/python3.13/site-packages/fsspec/asyn.py:464, in AsyncFileSystem._cat(self, path, recursive, on_error, batch_size, **kwargs)
    463     if ex:
--> 464         raise ex
    465 if (
    466     len(paths) > 1
    467     or isinstance(path, list)
    468     or paths[0] != self._strip_protocol(path)
    469 ):

File ~/micromamba/envs/ERA5_interactive/lib/python3.13/site-packages/fsspec/asyn.py:244, in _run_coros_in_chunks.<locals>._run_coro(coro, i)
    243 try:
--> 244     return await asyncio.wait_for(coro, timeout=timeout), i
    245 except Exception as e:

File ~/micromamba/envs/ERA5_interactive/lib/python3.13/asyncio/tasks.py:507, in wait_for(fut, timeout)
    506 async with timeouts.timeout(timeout):
--> 507     return await fut

File ~/micromamba/envs/ERA5_interactive/lib/python3.13/site-packages/fsspec/implementations/http.py:246, in HTTPFileSystem._cat_file(self, url, start, end, **kwargs)
    245     out = await r.read()
--> 246     self._raise_not_found_for_status(r, url)
    247 return out

File ~/micromamba/envs/ERA5_interactive/lib/python3.13/site-packages/fsspec/implementations/http.py:228, in HTTPFileSystem._raise_not_found_for_status(self, response, url)
    227 if response.status == 404:
--> 228     raise FileNotFoundError(url)
    229 response.raise_for_status()

FileNotFoundError: https://data.rda.ucar.edu/pythia_era5_24/annual_means/temp_2m_annual_1940_2023.zarr/.zmetadata

The above exception was the direct cause of the following exception:

KeyError                                  Traceback (most recent call last)
File ~/micromamba/envs/ERA5_interactive/lib/python3.13/site-packages/zarr/storage.py:1446, in FSStore.__getitem__(self, key)
   1445 try:
-> 1446     return self.map[key]
   1447 except self.exceptions as e:

File ~/micromamba/envs/ERA5_interactive/lib/python3.13/site-packages/fsspec/mapping.py:159, in FSMap.__getitem__(self, key, default)
    158         return default
--> 159     raise KeyError(key) from exc
    160 return result

KeyError: '.zmetadata'

The above exception was the direct cause of the following exception:

KeyError                                  Traceback (most recent call last)
File ~/micromamba/envs/ERA5_interactive/lib/python3.13/site-packages/xarray/backends/zarr.py:1862, in _get_open_params(store, mode, synchronizer, group, consolidated, consolidate_on_close, chunk_store, storage_options, zarr_version, use_zarr_fill_value_as_mask, zarr_format)
   1861 try:
-> 1862     zarr_root_group = zarr.open_consolidated(store, **open_kwargs)
   1863 except (ValueError, KeyError):
   1864     # ValueError in zarr-python 3.x, KeyError in 2.x.

File ~/micromamba/envs/ERA5_interactive/lib/python3.13/site-packages/zarr/convenience.py:1362, in open_consolidated(store, metadata_key, mode, **kwargs)
   1361 # setup metadata store
-> 1362 meta_store = ConsolidatedStoreClass(store, metadata_key=metadata_key)
   1364 # pass through

File ~/micromamba/envs/ERA5_interactive/lib/python3.13/site-packages/zarr/storage.py:3045, in ConsolidatedMetadataStore.__init__(self, store, metadata_key)
   3044 # retrieve consolidated metadata
-> 3045 meta = json_loads(self.store[metadata_key])
   3047 # check format of consolidated metadata

File ~/micromamba/envs/ERA5_interactive/lib/python3.13/site-packages/zarr/storage.py:1448, in FSStore.__getitem__(self, key)
   1447 except self.exceptions as e:
-> 1448     raise KeyError(key) from e

KeyError: '.zmetadata'

During handling of the above exception, another exception occurred:

GroupNotFoundError                        Traceback (most recent call last)
File ~/micromamba/envs/ERA5_interactive/lib/python3.13/site-packages/xarray/backends/zarr.py:1866, in _get_open_params(store, mode, synchronizer, group, consolidated, consolidate_on_close, chunk_store, storage_options, zarr_version, use_zarr_fill_value_as_mask, zarr_format)
   1865 try:
-> 1866     zarr_root_group = zarr.open_group(store, **open_kwargs)
   1867     emit_user_level_warning(
   1868         "Failed to open Zarr store with consolidated metadata, "
   1869         "but successfully read with non-consolidated metadata. "
   (...)   1879         RuntimeWarning,
   1880     )

File ~/micromamba/envs/ERA5_interactive/lib/python3.13/site-packages/zarr/hierarchy.py:1578, in open_group(store, mode, cache_attrs, synchronizer, path, chunk_store, storage_options, zarr_version, meta_array)
   1577             raise ContainsArrayError(path)
-> 1578         raise GroupNotFoundError(path)
   1580 elif mode == "w":

GroupNotFoundError: group not found at path ''

The above exception was the direct cause of the following exception:

FileNotFoundError                         Traceback (most recent call last)
Cell In[2], line 3
      1 rda_url = "https://data.rda.ucar.edu/"
      2 annual_means = rda_url + "pythia_era5_24/annual_means/"
----> 3 xrds = xr.open_dataset(annual_means + "temp_2m_annual_1940_2023.zarr", engine="zarr")
      4 xrds

File ~/micromamba/envs/ERA5_interactive/lib/python3.13/site-packages/xarray/backends/api.py:760, in open_dataset(filename_or_obj, engine, chunks, cache, decode_cf, mask_and_scale, decode_times, decode_timedelta, use_cftime, concat_characters, decode_coords, drop_variables, create_default_indexes, inline_array, chunked_array_type, from_array_kwargs, backend_kwargs, **kwargs)
    748 decoders = _resolve_decoders_kwargs(
    749     decode_cf,
    750     open_backend_dataset_parameters=backend.open_dataset_parameters,
   (...)    756     decode_coords=decode_coords,
    757 )
    759 overwrite_encoded_chunks = kwargs.pop("overwrite_encoded_chunks", None)
--> 760 backend_ds = backend.open_dataset(
    761     filename_or_obj,
    762     drop_variables=drop_variables,
    763     **decoders,
    764     **kwargs,
    765 )
    766 ds = _dataset_from_backend_dataset(
    767     backend_ds,
    768     filename_or_obj,
   (...)    779     **kwargs,
    780 )
    781 return ds

File ~/micromamba/envs/ERA5_interactive/lib/python3.13/site-packages/xarray/backends/zarr.py:1654, in ZarrBackendEntrypoint.open_dataset(self, filename_or_obj, mask_and_scale, decode_times, concat_characters, decode_coords, drop_variables, use_cftime, decode_timedelta, group, mode, synchronizer, consolidated, chunk_store, storage_options, zarr_version, zarr_format, store, engine, use_zarr_fill_value_as_mask, cache_members)
   1652 filename_or_obj = _normalize_path(filename_or_obj)
   1653 if not store:
-> 1654     store = ZarrStore.open_group(
   1655         filename_or_obj,
   1656         group=group,
   1657         mode=mode,
   1658         synchronizer=synchronizer,
   1659         consolidated=consolidated,
   1660         consolidate_on_close=False,
   1661         chunk_store=chunk_store,
   1662         storage_options=storage_options,
   1663         zarr_version=zarr_version,
   1664         use_zarr_fill_value_as_mask=None,
   1665         zarr_format=zarr_format,
   1666         cache_members=cache_members,
   1667     )
   1669 store_entrypoint = StoreBackendEntrypoint()
   1670 with close_on_error(store):

File ~/micromamba/envs/ERA5_interactive/lib/python3.13/site-packages/xarray/backends/zarr.py:714, in ZarrStore.open_group(cls, store, mode, synchronizer, group, consolidated, consolidate_on_close, chunk_store, storage_options, append_dim, write_region, safe_chunks, align_chunks, zarr_version, zarr_format, use_zarr_fill_value_as_mask, write_empty, cache_members)
    688 @classmethod
    689 def open_group(
    690     cls,
   (...)    707     cache_members: bool = True,
    708 ):
    709     (
    710         zarr_group,
    711         consolidate_on_close,
    712         close_store_on_close,
    713         use_zarr_fill_value_as_mask,
--> 714     ) = _get_open_params(
    715         store=store,
    716         mode=mode,
    717         synchronizer=synchronizer,
    718         group=group,
    719         consolidated=consolidated,
    720         consolidate_on_close=consolidate_on_close,
    721         chunk_store=chunk_store,
    722         storage_options=storage_options,
    723         zarr_version=zarr_version,
    724         use_zarr_fill_value_as_mask=use_zarr_fill_value_as_mask,
    725         zarr_format=zarr_format,
    726     )
    728     return cls(
    729         zarr_group,
    730         mode,
   (...)    739         cache_members=cache_members,
    740     )

File ~/micromamba/envs/ERA5_interactive/lib/python3.13/site-packages/xarray/backends/zarr.py:1882, in _get_open_params(store, mode, synchronizer, group, consolidated, consolidate_on_close, chunk_store, storage_options, zarr_version, use_zarr_fill_value_as_mask, zarr_format)
   1867             emit_user_level_warning(
   1868                 "Failed to open Zarr store with consolidated metadata, "
   1869                 "but successfully read with non-consolidated metadata. "
   (...)   1879                 RuntimeWarning,
   1880             )
   1881         except missing_exc as err:
-> 1882             raise FileNotFoundError(
   1883                 f"No such file or directory: '{store}'"
   1884             ) from err
   1886 # but the user should still receive a DataTree whose root is the group they asked for
   1887 if group and group != "/":

FileNotFoundError: No such file or directory: 'https://data.rda.ucar.edu/pythia_era5_24/annual_means/temp_2m_annual_1940_2023.zarr'
# Select the time range of interest
xrds = xrds.sel(time=slice('2017-01-01', '2023-12-31'))
xrds.load();

Panel Widgets

Panel provides a variety of widgets that can be used to build interactive dashboards. In this notebook, we are going to use some of these widgets. For the complete list of widgets, please see the Panel documentation.

The panel widgets that we are using are:

  • pn.widgets.Select for selecting the variable

  • pn.widgets.DatePicker for selecting the date

  • pn.widgets.Player for making time series animations


w_var = pn.widgets.Select(name="Data Variable", options=list(xrds.data_vars))

dataset_controls = pn.WidgetBox(
    "## Dataset Controls",
    w_var,
)
dataset_controls

Now, let’s create dropdown widgets for selecting the colormap and plot type.

w_cmap = pn.widgets.Select(name="Colormap", options=["inferno", "plasma", "coolwarm"])


w_plot_type = pn.widgets.Select(
    name="Plot Type", options=["Color Plot", "Contour"]
)


plot_controls = pn.WidgetBox(
    "## Plot Controls",
    w_plot_type,
    w_cmap,
)
plot_controls

Now, let’s put together all the controls and the plot in a panel layout using pn.Row and pn.Column:

controls = pn.Column(dataset_controls, plot_controls)
controls
w_player = pn.widgets.Player(
    value=0,
    start=0,
    end=len(xrds.time) - 1,
    name="Year",
    loop_policy="loop",
    interval=300,
    align="center",
    width_policy="fit",
)
w_player

Plotting Function

def plot_ds(time, var, cmap, plot_type):
    clim = (xrds[var].values.min(), xrds[var].values.max())

    if plot_type == "Color Plot":
        return (
            xrds[var]
            .isel(time=time)
            .hvplot(
                cmap=cmap,
                title=str(f"{var} year {time}"),
                clim=clim,
                dynamic=False,
                rasterize=True,
                precompute=True,
            )
            .opts(framewise=False)
        )

    elif plot_type == "Contour":
        return (
            xrds[var]
            .isel(time=time)
            .hvplot.contour(
                cmap=cmap,
                dynamic=False,
                rasterize=True,
                title=str(f"{var} Year: {time}"),
                clim=clim,
                precompute=True,
            )
            .opts(framewise=False)
        )

app = pn.Row(
    controls,
    pn.Column(
        pn.panel(
            hv.DynamicMap(
                pn.bind(
                    plot_ds,
                    time=w_player,
                    var=w_var,
                    cmap=w_cmap,
                    plot_type=w_plot_type,
                )
            )
        ),
        w_player,
    ),
).servable()

app

Please note how the above dashboard is servable. You can deploy the dashboard by running the following command in the terminal:

panel serve --show 04_dashboard.ipynb --allow-websocket-origin=projectpythia.2i2c.cloud

This will open a new tab in your default web browser with the dashboard. The allow websocket origin flag is required to allow traffic to flow to the site. This should be update to reflect the base URL where the application is launched. A wildcard can be used, *, to allow traffic from any site to connect.

On the Project Pythia 2i2c hosted JupyterHub the link to use to access the panel serve command is https://projectpythia.2i2c.cloud/hub/user-redirect/proxy/5006/04_dashboard

References
  1. C3S. (2018). ERA5 hourly data on single levels from 1940 to present. Copernicus Climate Change Service (C3S) Climate Data Store (CDS). 10.24381/CDS.ADBB2D47
Visualization Notebooks
Interactive Visualuzation using hvPlot
Preprocessing Notebooks for NCAR's GDEX
Generate annual/yearly zarr stores from hourly ERA5 NetCDF files on NCAR’s Geoscience Data Exchange