Overview¶
Within this notebook, we will cover:
How to load a Kerchunk pre-generated reference file into Xarray as if it were a Zarr store.
Prerequisites¶
| Concepts | Importance | Notes |
|---|---|---|
| Kerchunk Basics | Required | Core |
| Xarray Tutorial | Required | Core |
Time to learn: 45 minutes
Opening Reference Dataset with Fsspec and Xarray¶
One way of using our reference dataset is opening it with Xarray. To do this, we will create an fsspec filesystem and pass it to Xarray.
# create an fsspec reference filesystem from the Kerchunk output
import fsspec
import xarray as xr
fs = fsspec.filesystem(
"reference",
fo="references/ARG_combined.json",
remote_protocol="s3",
remote_options={"anon": True},
skip_instance_cache=True,
)
m = fs.get_mapper("")
ds = xr.open_dataset(m, engine="zarr", backend_kwargs={"consolidated": False})---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
Cell In[1], line 13
5 fs = fsspec.filesystem(
6 "reference",
7 fo="references/ARG_combined.json",
(...) 10 skip_instance_cache=True,
11 )
12 m = fs.get_mapper("")
---> 13 ds = xr.open_dataset(m, engine="zarr", backend_kwargs={"consolidated": False})
File ~/micromamba/envs/kerchunk-cookbook/lib/python3.14/site-packages/xarray/backends/api.py:606, in open_dataset(filename_or_obj, engine, chunks, cache, decode_cf, mask_and_scale, decode_times, decode_timedelta, use_cftime, concat_characters, decode_coords, drop_variables, create_default_indexes, inline_array, chunked_array_type, from_array_kwargs, backend_kwargs, **kwargs)
594 decoders = _resolve_decoders_kwargs(
595 decode_cf,
596 open_backend_dataset_parameters=backend.open_dataset_parameters,
(...) 602 decode_coords=decode_coords,
603 )
605 overwrite_encoded_chunks = kwargs.pop("overwrite_encoded_chunks", None)
--> 606 backend_ds = backend.open_dataset(
607 filename_or_obj,
608 drop_variables=drop_variables,
609 **decoders,
610 **kwargs,
611 )
612 ds = _dataset_from_backend_dataset(
613 backend_ds,
614 filename_or_obj,
(...) 625 **kwargs,
626 )
627 return ds
File ~/micromamba/envs/kerchunk-cookbook/lib/python3.14/site-packages/xarray/backends/zarr.py:1660, in ZarrBackendEntrypoint.open_dataset(self, filename_or_obj, mask_and_scale, decode_times, concat_characters, decode_coords, drop_variables, use_cftime, decode_timedelta, group, mode, synchronizer, consolidated, chunk_store, storage_options, zarr_version, zarr_format, store, engine, use_zarr_fill_value_as_mask, cache_members)
1658 filename_or_obj = _normalize_path(filename_or_obj)
1659 if not store:
-> 1660 store = ZarrStore.open_group(
1661 filename_or_obj,
1662 group=group,
1663 mode=mode,
1664 synchronizer=synchronizer,
1665 consolidated=consolidated,
1666 consolidate_on_close=False,
1667 chunk_store=chunk_store,
1668 storage_options=storage_options,
1669 zarr_version=zarr_version,
1670 use_zarr_fill_value_as_mask=None,
1671 zarr_format=zarr_format,
1672 cache_members=cache_members,
1673 )
1675 store_entrypoint = StoreBackendEntrypoint()
1676 with close_on_error(store):
File ~/micromamba/envs/kerchunk-cookbook/lib/python3.14/site-packages/xarray/backends/zarr.py:714, in ZarrStore.open_group(cls, store, mode, synchronizer, group, consolidated, consolidate_on_close, chunk_store, storage_options, append_dim, write_region, safe_chunks, align_chunks, zarr_version, zarr_format, use_zarr_fill_value_as_mask, write_empty, cache_members)
688 @classmethod
689 def open_group(
690 cls,
(...) 707 cache_members: bool = True,
708 ):
709 (
710 zarr_group,
711 consolidate_on_close,
712 close_store_on_close,
713 use_zarr_fill_value_as_mask,
--> 714 ) = _get_open_params(
715 store=store,
716 mode=mode,
717 synchronizer=synchronizer,
718 group=group,
719 consolidated=consolidated,
720 consolidate_on_close=consolidate_on_close,
721 chunk_store=chunk_store,
722 storage_options=storage_options,
723 zarr_version=zarr_version,
724 use_zarr_fill_value_as_mask=use_zarr_fill_value_as_mask,
725 zarr_format=zarr_format,
726 )
728 return cls(
729 zarr_group,
730 mode,
(...) 739 cache_members=cache_members,
740 )
File ~/micromamba/envs/kerchunk-cookbook/lib/python3.14/site-packages/xarray/backends/zarr.py:1902, in _get_open_params(store, mode, synchronizer, group, consolidated, consolidate_on_close, chunk_store, storage_options, zarr_version, use_zarr_fill_value_as_mask, zarr_format)
1898 if _zarr_v3():
1899 # we have determined that we don't want to use consolidated metadata
1900 # so we set that to False to avoid trying to read it
1901 open_kwargs["use_consolidated"] = False
-> 1902 zarr_group = zarr.open_group(store, **open_kwargs)
1904 close_store_on_close = zarr_group.store is not store
1906 # we use this to determine how to handle fill_value
File ~/micromamba/envs/kerchunk-cookbook/lib/python3.14/site-packages/zarr/api/synchronous.py:549, in open_group(store, mode, cache_attrs, synchronizer, path, chunk_store, storage_options, zarr_version, zarr_format, meta_array, attributes, use_consolidated)
478 def open_group(
479 store: StoreLike | None = None,
480 *,
(...) 491 use_consolidated: bool | str | None = None,
492 ) -> Group:
493 """Open a group using file-mode-like semantics.
494
495 Parameters
(...) 546 The new group.
547 """
548 return Group(
--> 549 sync(
550 async_api.open_group(
551 store=store,
552 mode=mode,
553 cache_attrs=cache_attrs,
554 synchronizer=synchronizer,
555 path=path,
556 chunk_store=chunk_store,
557 storage_options=storage_options,
558 zarr_version=zarr_version,
559 zarr_format=zarr_format,
560 meta_array=meta_array,
561 attributes=attributes,
562 use_consolidated=use_consolidated,
563 )
564 )
565 )
File ~/micromamba/envs/kerchunk-cookbook/lib/python3.14/site-packages/zarr/core/sync.py:159, in sync(coro, loop, timeout)
156 return_result = next(iter(finished)).result()
158 if isinstance(return_result, BaseException):
--> 159 raise return_result
160 else:
161 return return_result
File ~/micromamba/envs/kerchunk-cookbook/lib/python3.14/site-packages/zarr/core/sync.py:119, in _runner(coro)
114 """
115 Await a coroutine and return the result of running it. If awaiting the coroutine raises an
116 exception, the exception will be returned.
117 """
118 try:
--> 119 return await coro
120 except Exception as ex:
121 return ex
File ~/micromamba/envs/kerchunk-cookbook/lib/python3.14/site-packages/zarr/api/asynchronous.py:860, in open_group(store, mode, cache_attrs, synchronizer, path, chunk_store, storage_options, zarr_version, zarr_format, meta_array, attributes, use_consolidated)
857 if chunk_store is not None:
858 warnings.warn("chunk_store is not yet implemented", ZarrRuntimeWarning, stacklevel=2)
--> 860 store_path = await make_store_path(store, mode=mode, storage_options=storage_options, path=path)
861 if attributes is None:
862 attributes = {}
File ~/micromamba/envs/kerchunk-cookbook/lib/python3.14/site-packages/zarr/storage/_common.py:422, in make_store_path(store_like, path, mode, storage_options)
417 raise ValueError(
418 "'path' was provided but is not used for FSMap store_like objects. Specify the path when creating the FSMap instance instead."
419 )
421 else:
--> 422 store = await make_store(store_like, mode=mode, storage_options=storage_options)
423 return await StorePath.open(store, path=path_normalized, mode=mode)
File ~/micromamba/envs/kerchunk-cookbook/lib/python3.14/site-packages/zarr/storage/_common.py:354, in make_store(store_like, mode, storage_options)
351 return await make_store(Path(store_like), mode=mode, storage_options=storage_options)
353 elif _has_fsspec and isinstance(store_like, FSMap):
--> 354 return FsspecStore.from_mapper(store_like, read_only=_read_only)
356 else:
357 raise TypeError(f"Unsupported type for store_like: '{type(store_like).__name__}'")
File ~/micromamba/envs/kerchunk-cookbook/lib/python3.14/site-packages/zarr/storage/_fsspec.py:197, in FsspecStore.from_mapper(cls, fs_map, read_only, allowed_exceptions)
173 @classmethod
174 def from_mapper(
175 cls,
(...) 178 allowed_exceptions: tuple[type[Exception], ...] = ALLOWED_EXCEPTIONS,
179 ) -> FsspecStore:
180 """
181 Create a FsspecStore from a FSMap object.
182
(...) 195 FsspecStore
196 """
--> 197 fs = _make_async(fs_map.fs)
198 return cls(
199 fs=fs,
200 path=fs_map.root,
201 read_only=read_only,
202 allowed_exceptions=allowed_exceptions,
203 )
File ~/micromamba/envs/kerchunk-cookbook/lib/python3.14/site-packages/zarr/storage/_fsspec.py:57, in _make_async(fs)
55 fs_dict = json.loads(fs.to_json())
56 fs_dict["asynchronous"] = True
---> 57 return fsspec.AbstractFileSystem.from_json(json.dumps(fs_dict))
59 if fsspec_version < parse_version("2024.12.0"):
60 raise ImportError(
61 f"The filesystem '{fs}' is synchronous, and the required "
62 "AsyncFileSystemWrapper is not available. Upgrade fsspec to version "
63 "2024.12.0 or later to enable this functionality."
64 )
File ~/micromamba/envs/kerchunk-cookbook/lib/python3.14/site-packages/fsspec/spec.py:1491, in AbstractFileSystem.from_json(blob)
1470 """
1471 Recreate a filesystem instance from JSON representation.
1472
(...) 1487 at import time.
1488 """
1489 from .json import FilesystemJSONDecoder
-> 1491 return json.loads(blob, cls=FilesystemJSONDecoder)
File ~/micromamba/envs/kerchunk-cookbook/lib/python3.14/json/__init__.py:365, in loads(s, cls, object_hook, parse_float, parse_int, parse_constant, object_pairs_hook, **kw)
363 if parse_constant is not None:
364 kw['parse_constant'] = parse_constant
--> 365 return cls(**kw).decode(s)
File ~/micromamba/envs/kerchunk-cookbook/lib/python3.14/json/decoder.py:345, in JSONDecoder.decode(self, s, _w)
340 def decode(self, s, _w=WHITESPACE.match):
341 """Return the Python representation of ``s`` (a ``str`` instance
342 containing a JSON document).
343
344 """
--> 345 obj, end = self.raw_decode(s, idx=_w(s, 0).end())
346 end = _w(s, end).end()
347 if end != len(s):
File ~/micromamba/envs/kerchunk-cookbook/lib/python3.14/json/decoder.py:361, in JSONDecoder.raw_decode(self, s, idx)
352 """Decode a JSON document from ``s`` (a ``str`` beginning with
353 a JSON document) and return a 2-tuple of the Python
354 representation and the index in ``s`` where the document ended.
(...) 358
359 """
360 try:
--> 361 obj, end = self.scan_once(s, idx)
362 except StopIteration as err:
363 raise JSONDecodeError("Expecting value", s, err.value) from None
File ~/micromamba/envs/kerchunk-cookbook/lib/python3.14/site-packages/fsspec/json.py:92, in FilesystemJSONDecoder.custom_object_hook(self, dct)
90 if "cls" in dct:
91 if (obj_cls := self.try_resolve_fs_cls(dct)) is not None:
---> 92 return AbstractFileSystem.from_dict(dct)
93 if (obj_cls := self.try_resolve_path_cls(dct)) is not None:
94 return obj_cls(dct["str"])
File ~/micromamba/envs/kerchunk-cookbook/lib/python3.14/site-packages/fsspec/spec.py:1567, in AbstractFileSystem.from_dict(dct)
1564 dct.pop("cls", None)
1565 dct.pop("protocol", None)
-> 1567 return cls(
1568 *json_decoder.unmake_serializable(dct.pop("args", ())),
1569 **json_decoder.unmake_serializable(dct),
1570 )
File ~/micromamba/envs/kerchunk-cookbook/lib/python3.14/site-packages/fsspec/spec.py:84, in _Cached.__call__(cls, *args, **kwargs)
82 return cls._cache[token]
83 else:
---> 84 obj = super().__call__(*args, **kwargs, **strip_tokenize_options)
85 # Setting _fs_token here causes some static linters to complain.
86 obj._fs_token_ = token
File ~/micromamba/envs/kerchunk-cookbook/lib/python3.14/site-packages/fsspec/implementations/reference.py:776, in ReferenceFileSystem.__init__(self, fo, target, ref_storage_args, target_protocol, target_options, remote_protocol, remote_options, fs, template_overrides, simple_templates, max_gap, max_block, cache_size, **kwargs)
774 self.fss[k] = AsyncFileSystemWrapper(f, asynchronous=self.asynchronous)
775 elif self.asynchronous ^ f.asynchronous:
--> 776 raise ValueError(
777 "Reference-FS's target filesystem must have same value "
778 "of asynchronous"
779 )
ValueError: Reference-FS's target filesystem must have same value of asynchronousOpening Reference Dataset with Xarray and the Kerchunk Engine¶
As of writing, the latest version of Kerchunk supports opening an reference dataset with Xarray without specifically creating an fsspec filesystem. This is the same behavior as the example above, just a few less lines of code.
storage_options = {
"remote_protocol": "s3",
"skip_instance_cache": True,
"remote_options": {"anon": True}
} # options passed to fsspec
open_dataset_options = {"chunks": {}} # opens passed to xarray
ds = xr.open_dataset(
"references/ARG_combined.json",
engine="kerchunk",
storage_options=storage_options,
open_dataset_options=open_dataset_options,
)