Overview¶
Within this notebook, we will cover:
How to load a Kerchunk pre-generated reference file into Xarray as if it were a Zarr store.
Prerequisites¶
| Concepts | Importance | Notes |
|---|---|---|
| Kerchunk Basics | Required | Core |
| Xarray Tutorial | Required | Core |
Time to learn: 45 minutes
Opening Reference Dataset with Fsspec and Xarray¶
One way of using our reference dataset is opening it with Xarray. To do this, we will create an fsspec filesystem and pass it to Xarray.
# create an fsspec reference filesystem from the Kerchunk output
import fsspec
import xarray as xr
fs = fsspec.filesystem(
"reference",
fo="references/ARG_combined.json",
remote_protocol="s3",
remote_options={"anon": True},
skip_instance_cache=True,
)
m = fs.get_mapper("")
ds = xr.open_dataset(m, engine="zarr", backend_kwargs={"consolidated": False})---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
Cell In[1], line 13
9 remote_options={"anon": True},
10 skip_instance_cache=True,
11 )
12 m = fs.get_mapper("")
---> 13 ds = xr.open_dataset(m, engine="zarr", backend_kwargs={"consolidated": False})
File ~/micromamba/envs/kerchunk-cookbook/lib/python3.14/site-packages/xarray/backends/api.py:613, in open_dataset(filename_or_obj, engine, chunks, cache, decode_cf, mask_and_scale, decode_times, decode_timedelta, use_cftime, concat_characters, decode_coords, drop_variables, create_default_indexes, inline_array, chunked_array_type, from_array_kwargs, backend_kwargs, **kwargs)
601 decoders = _resolve_decoders_kwargs(
602 decode_cf,
603 open_backend_dataset_parameters=backend.open_dataset_parameters,
(...) 609 decode_coords=decode_coords,
610 )
612 overwrite_encoded_chunks = kwargs.pop("overwrite_encoded_chunks", None)
--> 613 backend_ds = backend.open_dataset(
614 filename_or_obj,
615 drop_variables=drop_variables,
616 **decoders,
617 **kwargs,
618 )
619 ds = _dataset_from_backend_dataset(
620 backend_ds,
621 filename_or_obj,
(...) 632 **kwargs,
633 )
634 return ds
File ~/micromamba/envs/kerchunk-cookbook/lib/python3.14/site-packages/xarray/backends/zarr.py:1741, in ZarrBackendEntrypoint.open_dataset(self, filename_or_obj, mask_and_scale, decode_times, concat_characters, decode_coords, drop_variables, use_cftime, decode_timedelta, group, mode, synchronizer, consolidated, chunk_store, storage_options, zarr_version, zarr_format, store, engine, use_zarr_fill_value_as_mask, cache_members)
1739 filename_or_obj = _normalize_path(filename_or_obj)
1740 if not store:
-> 1741 store = ZarrStore.open_group(
1742 filename_or_obj,
1743 group=group,
1744 mode=mode,
1745 synchronizer=synchronizer,
1746 consolidated=consolidated,
1747 consolidate_on_close=False,
1748 chunk_store=chunk_store,
1749 storage_options=storage_options,
1750 zarr_version=zarr_version,
1751 use_zarr_fill_value_as_mask=None,
1752 zarr_format=zarr_format,
1753 cache_members=cache_members,
1754 )
1756 store_entrypoint = StoreBackendEntrypoint()
1757 with close_on_error(store):
File ~/micromamba/envs/kerchunk-cookbook/lib/python3.14/site-packages/xarray/backends/zarr.py:771, in ZarrStore.open_group(cls, store, mode, synchronizer, group, consolidated, consolidate_on_close, chunk_store, storage_options, append_dim, write_region, safe_chunks, align_chunks, zarr_version, zarr_format, use_zarr_fill_value_as_mask, write_empty, cache_members)
745 @classmethod
746 def open_group(
747 cls,
(...) 764 cache_members: bool = True,
765 ):
766 (
767 zarr_group,
768 consolidate_on_close,
769 close_store_on_close,
770 use_zarr_fill_value_as_mask,
--> 771 ) = _get_open_params(
772 store=store,
773 mode=mode,
774 synchronizer=synchronizer,
775 group=group,
776 consolidated=consolidated,
777 consolidate_on_close=consolidate_on_close,
778 chunk_store=chunk_store,
779 storage_options=storage_options,
780 zarr_version=zarr_version,
781 use_zarr_fill_value_as_mask=use_zarr_fill_value_as_mask,
782 zarr_format=zarr_format,
783 )
785 return cls(
786 zarr_group,
787 mode,
(...) 796 cache_members=cache_members,
797 )
File ~/micromamba/envs/kerchunk-cookbook/lib/python3.14/site-packages/xarray/backends/zarr.py:1983, in _get_open_params(store, mode, synchronizer, group, consolidated, consolidate_on_close, chunk_store, storage_options, zarr_version, use_zarr_fill_value_as_mask, zarr_format)
1979 if _zarr_v3():
1980 # we have determined that we don't want to use consolidated metadata
1981 # so we set that to False to avoid trying to read it
1982 open_kwargs["use_consolidated"] = False
-> 1983 zarr_group = zarr.open_group(store, **open_kwargs)
1985 close_store_on_close = zarr_group.store is not store
1987 # we use this to determine how to handle fill_value
File ~/micromamba/envs/kerchunk-cookbook/lib/python3.14/site-packages/zarr/api/synchronous.py:552, in open_group(store, mode, cache_attrs, synchronizer, path, chunk_store, storage_options, zarr_format, meta_array, attributes, use_consolidated)
482 def open_group(
483 store: StoreLike | None = None,
484 *,
(...) 494 use_consolidated: bool | str | None = None,
495 ) -> Group:
496 """Open a group using file-mode-like semantics.
497
498 Parameters
(...) 549 The new group.
550 """
551 return Group(
--> 552 sync(
553 async_api.open_group(
554 store=store,
555 mode=mode,
556 cache_attrs=cache_attrs,
557 synchronizer=synchronizer,
558 path=path,
559 chunk_store=chunk_store,
560 storage_options=storage_options,
561 zarr_format=zarr_format,
562 meta_array=meta_array,
563 attributes=attributes,
564 use_consolidated=use_consolidated,
565 )
566 )
567 )
File ~/micromamba/envs/kerchunk-cookbook/lib/python3.14/site-packages/zarr/core/sync.py:156, in sync(coro, loop, timeout)
153 return_result = next(iter(finished)).result()
155 if isinstance(return_result, BaseException):
--> 156 raise return_result
157 else:
158 return return_result
File ~/micromamba/envs/kerchunk-cookbook/lib/python3.14/site-packages/zarr/core/sync.py:116, in _runner(coro)
111 """
112 Await a coroutine and return the result of running it. If awaiting the coroutine raises an
113 exception, the exception will be returned.
114 """
115 try:
--> 116 return await coro
117 except Exception as ex: # noqa: BLE001 -- the caller re-raises the returned exception
118 return ex
File ~/micromamba/envs/kerchunk-cookbook/lib/python3.14/site-packages/zarr/api/asynchronous.py:859, in open_group(store, mode, cache_attrs, synchronizer, path, chunk_store, storage_options, zarr_format, meta_array, attributes, use_consolidated)
794 """Open a group using file-mode-like semantics.
795
796 Parameters
(...) 847 The new group.
848 """
850 _warn_unimplemented_kwargs(
851 {
852 "cache_attrs": cache_attrs,
(...) 856 }
857 )
--> 859 store_path = await make_store_path(store, mode=mode, storage_options=storage_options, path=path)
860 if attributes is None:
861 attributes = {}
File ~/micromamba/envs/kerchunk-cookbook/lib/python3.14/site-packages/zarr/storage/_common.py:482, in make_store_path(store_like, path, mode, storage_options)
477 raise ValueError(
478 "'path' was provided but is not used for FSMap store_like objects. Specify the path when creating the FSMap instance instead."
479 )
481 else:
--> 482 store = await make_store(store_like, mode=mode, storage_options=storage_options)
483 return await StorePath.open(store, path=path_normalized, mode=mode)
File ~/micromamba/envs/kerchunk-cookbook/lib/python3.14/site-packages/zarr/storage/_common.py:414, in make_store(store_like, mode, storage_options)
409 return FsspecStore.from_url(
410 store_like, storage_options=storage_options, read_only=_read_only
411 )
413 elif _has_fsspec and isinstance(store_like, FSMap):
--> 414 return FsspecStore.from_mapper(store_like, read_only=_read_only)
416 else:
417 raise TypeError(f"Unsupported type for store_like: '{type(store_like).__name__}'")
File ~/micromamba/envs/kerchunk-cookbook/lib/python3.14/site-packages/zarr/storage/_fsspec.py:209, in FsspecStore.from_mapper(cls, fs_map, read_only, allowed_exceptions)
185 @classmethod
186 def from_mapper(
187 cls,
(...) 190 allowed_exceptions: tuple[type[Exception], ...] = ALLOWED_EXCEPTIONS,
191 ) -> FsspecStore:
192 """
193 Create an FsspecStore from an FSMap object.
194
(...) 207 FsspecStore
208 """
--> 209 fs = _make_async(fs_map.fs)
210 return cls(
211 fs=fs,
212 path=fs_map.root,
213 read_only=read_only,
214 allowed_exceptions=allowed_exceptions,
215 )
File ~/micromamba/envs/kerchunk-cookbook/lib/python3.14/site-packages/zarr/storage/_fsspec.py:56, in _make_async(fs)
51 return fs
52 if fs.async_impl:
53 # Convert sync instance of an async fs to an async instance. Reuse the original
54 # constructor arguments rather than round-tripping through JSON, since storage
55 # options may hold objects that are not JSON-serializable (e.g. credentials).
---> 56 return type(fs)(*fs.storage_args, **{**fs.storage_options, "asynchronous": True})
58 if fsspec_version < parse_version("2024.12.0"):
59 raise ImportError(
60 f"The filesystem '{fs}' is synchronous, and the required "
61 "AsyncFileSystemWrapper is not available. Upgrade fsspec to version "
62 "2024.12.0 or later to enable this functionality."
63 )
File ~/micromamba/envs/kerchunk-cookbook/lib/python3.14/site-packages/fsspec/spec.py:128, in _Cached.__call__(cls, *args, **kwargs)
125 if inst is not None:
126 return inst
--> 128 obj = super().__call__(*args, **kwargs, **strip_tokenize_options)
129 # Setting _fs_token here causes some static linters to complain.
130 obj._fs_token_ = token
File ~/micromamba/envs/kerchunk-cookbook/lib/python3.14/site-packages/fsspec/implementations/reference.py:785, in ReferenceFileSystem.__init__(self, fo, target, ref_storage_args, target_protocol, target_options, remote_protocol, remote_options, fs, template_overrides, simple_templates, max_gap, max_block, cache_size, **kwargs)
783 self.fss[k] = AsyncFileSystemWrapper(f, asynchronous=self.asynchronous)
784 elif self.asynchronous ^ f.asynchronous:
--> 785 raise ValueError(
786 "Reference-FS's target filesystem must have same value "
787 "of asynchronous"
788 )
ValueError: Reference-FS's target filesystem must have same value of asynchronousOpening Reference Dataset with Xarray and the Kerchunk Engine¶
As of writing, the latest version of Kerchunk supports opening an reference dataset with Xarray without specifically creating an fsspec filesystem. This is the same behavior as the example above, just a few less lines of code.
storage_options = {
"remote_protocol": "s3",
"skip_instance_cache": True,
"remote_options": {"anon": True}
} # options passed to fsspec
open_dataset_options = {"chunks": {}} # opens passed to xarray
ds = xr.open_dataset(
"references/ARG_combined.json",
engine="kerchunk",
storage_options=storage_options,
open_dataset_options=open_dataset_options,
)