Skip to article frontmatterSkip to article content

Along Track Altimetry Analysis

Project Pythia Logo Pangeo Logo

Along Track Altimetry Analysis


Overview

  1. Using CNES altimetry data

  2. Visualizing data using hvplot

  3. Use xhistogram to plot multidimensional data

Prerequisites

ConceptsImportanceNotes
Intro to PandasHelpful
Using hvplotHelpfulMatplotlib knowledge also helpful
DaskHelpful
xhistogramHelpful
  • Time to learn: 15 minutes

Imports


import fsspec
import xarray as xr
import numpy as np
import hvplot
import hvplot.dask
import hvplot.pandas
import hvplot.xarray
from xhistogram.xarray import histogram
from intake import open_catalog
Loading...

Load Data

The analysis ready along-track altimetry data were prepared by CNES. They are catalogged in the Pangeo Cloud Data Catalog here: https://catalog.pangeo.io/browse/master/ocean/altimetry/

We will work with Jason 3.

cat = open_catalog("https://raw.githubusercontent.com/pangeo-data/pangeo-datastore/master/intake-catalogs/ocean/altimetry.yaml")
print(list(cat))
ds = cat['j3'].to_dask()
ds
['al', 'alg', 'c2', 'e1', 'e1g', 'e2', 'en', 'enn', 'g2', 'h2', 'j1', 'j1g', 'j1n', 'j2', 'j2g', 'j2n', 'j3', 's3a', 's3b', 'tp', 'tpn']
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
Cell In[2], line 3
      1 cat = open_catalog("https://raw.githubusercontent.com/pangeo-data/pangeo-datastore/master/intake-catalogs/ocean/altimetry.yaml")
      2 print(list(cat))
----> 3 ds = cat['j3'].to_dask()
      4 ds

File ~/micromamba/envs/po-cookbook-dev/lib/python3.13/site-packages/intake_xarray/base.py:8, in IntakeXarraySourceAdapter.to_dask(self)
      6 def to_dask(self):
      7     if "chunks" not in self.reader.kwargs:
----> 8         return self.reader(chunks={}).read()
      9     else:
     10         return self.reader.read()

File ~/micromamba/envs/po-cookbook-dev/lib/python3.13/site-packages/intake/readers/readers.py:121, in BaseReader.read(self, *args, **kwargs)
    119 kw.update(kwargs)
    120 args = kw.pop("args", ()) or args
--> 121 return self._read(*args, **kw)

File ~/micromamba/envs/po-cookbook-dev/lib/python3.13/site-packages/intake/readers/readers.py:1327, in XArrayDatasetReader._read(self, data, open_local, **kw)
   1325         f = fsspec.open(data.url, **(data.storage_options or {})).open()
   1326         return open_dataset(f, **kw)
-> 1327 return open_dataset(data.url, **kw)

File ~/micromamba/envs/po-cookbook-dev/lib/python3.13/site-packages/xarray/backends/api.py:760, in open_dataset(filename_or_obj, engine, chunks, cache, decode_cf, mask_and_scale, decode_times, decode_timedelta, use_cftime, concat_characters, decode_coords, drop_variables, create_default_indexes, inline_array, chunked_array_type, from_array_kwargs, backend_kwargs, **kwargs)
    748 decoders = _resolve_decoders_kwargs(
    749     decode_cf,
    750     open_backend_dataset_parameters=backend.open_dataset_parameters,
   (...)    756     decode_coords=decode_coords,
    757 )
    759 overwrite_encoded_chunks = kwargs.pop("overwrite_encoded_chunks", None)
--> 760 backend_ds = backend.open_dataset(
    761     filename_or_obj,
    762     drop_variables=drop_variables,
    763     **decoders,
    764     **kwargs,
    765 )
    766 ds = _dataset_from_backend_dataset(
    767     backend_ds,
    768     filename_or_obj,
   (...)    779     **kwargs,
    780 )
    781 return ds

File ~/micromamba/envs/po-cookbook-dev/lib/python3.13/site-packages/xarray/backends/zarr.py:1654, in ZarrBackendEntrypoint.open_dataset(self, filename_or_obj, mask_and_scale, decode_times, concat_characters, decode_coords, drop_variables, use_cftime, decode_timedelta, group, mode, synchronizer, consolidated, chunk_store, storage_options, zarr_version, zarr_format, store, engine, use_zarr_fill_value_as_mask, cache_members)
   1652 filename_or_obj = _normalize_path(filename_or_obj)
   1653 if not store:
-> 1654     store = ZarrStore.open_group(
   1655         filename_or_obj,
   1656         group=group,
   1657         mode=mode,
   1658         synchronizer=synchronizer,
   1659         consolidated=consolidated,
   1660         consolidate_on_close=False,
   1661         chunk_store=chunk_store,
   1662         storage_options=storage_options,
   1663         zarr_version=zarr_version,
   1664         use_zarr_fill_value_as_mask=None,
   1665         zarr_format=zarr_format,
   1666         cache_members=cache_members,
   1667     )
   1669 store_entrypoint = StoreBackendEntrypoint()
   1670 with close_on_error(store):

File ~/micromamba/envs/po-cookbook-dev/lib/python3.13/site-packages/xarray/backends/zarr.py:714, in ZarrStore.open_group(cls, store, mode, synchronizer, group, consolidated, consolidate_on_close, chunk_store, storage_options, append_dim, write_region, safe_chunks, align_chunks, zarr_version, zarr_format, use_zarr_fill_value_as_mask, write_empty, cache_members)
    688 @classmethod
    689 def open_group(
    690     cls,
   (...)    707     cache_members: bool = True,
    708 ):
    709     (
    710         zarr_group,
    711         consolidate_on_close,
    712         close_store_on_close,
    713         use_zarr_fill_value_as_mask,
--> 714     ) = _get_open_params(
    715         store=store,
    716         mode=mode,
    717         synchronizer=synchronizer,
    718         group=group,
    719         consolidated=consolidated,
    720         consolidate_on_close=consolidate_on_close,
    721         chunk_store=chunk_store,
    722         storage_options=storage_options,
    723         zarr_version=zarr_version,
    724         use_zarr_fill_value_as_mask=use_zarr_fill_value_as_mask,
    725         zarr_format=zarr_format,
    726     )
    728     return cls(
    729         zarr_group,
    730         mode,
   (...)    739         cache_members=cache_members,
    740     )

File ~/micromamba/envs/po-cookbook-dev/lib/python3.13/site-packages/xarray/backends/zarr.py:1858, in _get_open_params(store, mode, synchronizer, group, consolidated, consolidate_on_close, chunk_store, storage_options, zarr_version, use_zarr_fill_value_as_mask, zarr_format)
   1854 group = open_kwargs.pop("path")
   1856 if consolidated:
   1857     # TODO: an option to pass the metadata_key keyword
-> 1858     zarr_root_group = zarr.open_consolidated(store, **open_kwargs)
   1859 elif consolidated is None:
   1860     # same but with more error handling in case no consolidated metadata found
   1861     try:

File ~/micromamba/envs/po-cookbook-dev/lib/python3.13/site-packages/zarr/api/synchronous.py:231, in open_consolidated(use_consolidated, *args, **kwargs)
    226 def open_consolidated(*args: Any, use_consolidated: Literal[True] = True, **kwargs: Any) -> Group:
    227     """
    228     Alias for :func:`open_group` with ``use_consolidated=True``.
    229     """
    230     return Group(
--> 231         sync(async_api.open_consolidated(*args, use_consolidated=use_consolidated, **kwargs))
    232     )

File ~/micromamba/envs/po-cookbook-dev/lib/python3.13/site-packages/zarr/core/sync.py:163, in sync(coro, loop, timeout)
    160 return_result = next(iter(finished)).result()
    162 if isinstance(return_result, BaseException):
--> 163     raise return_result
    164 else:
    165     return return_result

File ~/micromamba/envs/po-cookbook-dev/lib/python3.13/site-packages/zarr/core/sync.py:119, in _runner(coro)
    114 """
    115 Await a coroutine and return the result of running it. If awaiting the coroutine raises an
    116 exception, the exception will be returned.
    117 """
    118 try:
--> 119     return await coro
    120 except Exception as ex:
    121     return ex

File ~/micromamba/envs/po-cookbook-dev/lib/python3.13/site-packages/zarr/api/asynchronous.py:408, in open_consolidated(use_consolidated, *args, **kwargs)
    403 if use_consolidated is not True:
    404     raise TypeError(
    405         "'use_consolidated' must be 'True' in 'open_consolidated'. Use 'open' with "
    406         "'use_consolidated=False' to bypass consolidated metadata."
    407     )
--> 408 return await open_group(*args, use_consolidated=use_consolidated, **kwargs)

File ~/micromamba/envs/po-cookbook-dev/lib/python3.13/site-packages/zarr/api/asynchronous.py:857, in open_group(store, mode, cache_attrs, synchronizer, path, chunk_store, storage_options, zarr_version, zarr_format, meta_array, attributes, use_consolidated)
    855 try:
    856     if mode in _READ_MODES:
--> 857         return await AsyncGroup.open(
    858             store_path, zarr_format=zarr_format, use_consolidated=use_consolidated
    859         )
    860 except (KeyError, FileNotFoundError):
    861     pass

File ~/micromamba/envs/po-cookbook-dev/lib/python3.13/site-packages/zarr/core/group.py:559, in AsyncGroup.open(cls, store, zarr_format, use_consolidated)
    552         raise FileNotFoundError(store_path)
    553 elif zarr_format is None:
    554     (
    555         zarr_json_bytes,
    556         zgroup_bytes,
    557         zattrs_bytes,
    558         maybe_consolidated_metadata_bytes,
--> 559     ) = await asyncio.gather(
    560         (store_path / ZARR_JSON).get(),
    561         (store_path / ZGROUP_JSON).get(),
    562         (store_path / ZATTRS_JSON).get(),
    563         (store_path / str(consolidated_key)).get(),
    564     )
    565     if zarr_json_bytes is not None and zgroup_bytes is not None:
    566         # warn and favor v3
    567         msg = f"Both zarr.json (Zarr format 3) and .zgroup (Zarr format 2) metadata objects exist at {store_path}. Zarr format 3 will be used."

File ~/micromamba/envs/po-cookbook-dev/lib/python3.13/site-packages/zarr/storage/_common.py:168, in StorePath.get(self, prototype, byte_range)
    166 if prototype is None:
    167     prototype = default_buffer_prototype()
--> 168 return await self.store.get(self.path, prototype=prototype, byte_range=byte_range)

File ~/micromamba/envs/po-cookbook-dev/lib/python3.13/site-packages/zarr/storage/_fsspec.py:299, in FsspecStore.get(self, key, prototype, byte_range)
    297 try:
    298     if byte_range is None:
--> 299         value = prototype.buffer.from_bytes(await self.fs._cat_file(path))
    300     elif isinstance(byte_range, RangeByteRequest):
    301         value = prototype.buffer.from_bytes(
    302             await self.fs._cat_file(
    303                 path,
   (...)    306             )
    307         )

File ~/micromamba/envs/po-cookbook-dev/lib/python3.13/site-packages/gcsfs/core.py:1119, in GCSFileSystem._cat_file(self, path, start, end, **kwargs)
   1117 else:
   1118     head = {}
-> 1119 headers, out = await self._call("GET", u2, headers=head)
   1120 return out

File ~/micromamba/envs/po-cookbook-dev/lib/python3.13/site-packages/gcsfs/core.py:483, in GCSFileSystem._call(self, method, path, json_out, info_out, *args, **kwargs)
    479 async def _call(
    480     self, method, path, *args, json_out=False, info_out=False, **kwargs
    481 ):
    482     logger.debug(f"{method.upper()}: {path}, {args}, {kwargs.get('headers')}")
--> 483     status, headers, info, contents = await self._request(
    484         method, path, *args, **kwargs
    485     )
    486     if json_out:
    487         return json.loads(contents)

File ~/micromamba/envs/po-cookbook-dev/lib/python3.13/site-packages/decorator.py:224, in decorate.<locals>.fun(*args, **kw)
    222 if not kwsyntax:
    223     args, kw = fix(args, kw, sig)
--> 224 return await caller(func, *(extras + args), **kw)

File ~/micromamba/envs/po-cookbook-dev/lib/python3.13/site-packages/gcsfs/retry.py:135, in retry_request(func, retries, *args, **kwargs)
    133     if retry > 0:
    134         await asyncio.sleep(min(random.random() + 2 ** (retry - 1), 32))
--> 135     return await func(*args, **kwargs)
    136 except (
    137     HttpError,
    138     requests.exceptions.RequestException,
   (...)    141     aiohttp.client_exceptions.ClientError,
    142 ) as e:
    143     if (
    144         isinstance(e, HttpError)
    145         and e.code == 400
    146         and "requester pays" in e.message
    147     ):

File ~/micromamba/envs/po-cookbook-dev/lib/python3.13/site-packages/gcsfs/core.py:476, in GCSFileSystem._request(self, method, path, headers, json, data, *args, **kwargs)
    473 info = r.request_info  # for debug only
    474 contents = await r.read()
--> 476 validate_response(status, contents, path, args)
    477 return status, headers, info, contents

File ~/micromamba/envs/po-cookbook-dev/lib/python3.13/site-packages/gcsfs/retry.py:120, in validate_response(status, content, path, args)
    118     raise requests.exceptions.ProxyError()
    119 elif "invalid" in str(msg):
--> 120     raise ValueError(f"Bad Request: {path}\n{msg}")
    121 elif error and not isinstance(error, str):
    122     raise HttpError(error)

ValueError: Bad Request: https://storage.googleapis.com/download/storage/v1/b/pangeo-cnes/o/alti%2Fj3%2Fzarr.json?alt=media
User project specified in the request is invalid.

Load some data into memory:

# Select latitude, longitude, and sea level anomaly
ds_ll = ds[['latitude', 'longitude', 'sla_filtered']].reset_coords().astype('f4').load()
ds_ll

Convert to pandas dataframe:

df = ds_ll.to_dataframe()
df

Visualize with hvplot

df.hvplot.scatter(x='longitude', y='latitude', datashade=True)

Bin using xhistogram

lon_bins = np.arange(0, 361, 2)
lat_bins = np.arange(-70, 71, 2)

# helps with memory management
ds_ll_chunked = ds_ll.chunk({'time': '5MB'})

sla_variance = histogram(ds_ll_chunked.longitude, ds_ll_chunked.latitude,
                         bins=[lon_bins, lat_bins],
                         weights=ds_ll_chunked.sla_filtered.fillna(0.)**2)

norm = histogram(ds_ll_chunked.longitude, ds_ll_chunked.latitude,
                         bins=[lon_bins, lat_bins])


# let's get at least 200 points in a box for it to be unmasked
thresh = 200
sla_variance = sla_variance / norm.where(norm > thresh)
sla_variance
sla_variance.load()
# plot the sea level anomaly variance
sla_variance.plot(x='longitude_bin', figsize=(12, 6), vmax=0.2)

Summary


In this example we visualized sea level anomalies using along-track altimetry data using hvplot. Then, we used xhistogram to calculate and plot the variance of the data.

What’s next?

Other examples will look at other datasets to visualize sea surface temeratures, ocean depth, and currents.

Resources and references