Skip to article frontmatterSkip to article content

Advanced Baseline Imager (ABI) data with Satpy

pythia ncar

This notebook is developed during the pythia cook-off at NCAR Mesa-Lab Boulder Colorado, June 12-14, 2024

Participants in the workshop event have the chance to practice collaborative problem-solving and hands-on learning in the field of Python programming.

This notebook is part of the Breakout Topic: Geostationary on AWS, lead by Jorge Humberto Bravo Mendez jbravo2@stevens.edu, from Stevens Institute of Technology

Advanced Baseline Imager (ABI) data with Satpy

Using Satpy to read and Advanced Baseline Imager (ABI) data from GOES-R satellites. Here’s a step-by-step guide:


Imports

import warnings
warnings.filterwarnings('ignore')
warnings.simplefilter('ignore', SyntaxWarning)

from satpy.scene import Scene
from satpy.utils import debug_on

from datetime import datetime

from glob import glob

Starting to create satpy scenes

sat_files = glob("input/G16_ABI-L1b-RadC/*")
sat_files
['input/G16_ABI-L1b-RadC/OR_ABI-L1b-RadC-M6C12_G16_s20232561536173_e20232561538552_c20232561539031.nc', 'input/G16_ABI-L1b-RadC/OR_ABI-L1b-RadC-M6C09_G16_s20232561536173_e20232561538551_c20232561538580.nc', 'input/G16_ABI-L1b-RadC/OR_ABI-L1b-RadC-M6C04_G16_s20232561536173_e20232561538546_c20232561539001.nc', 'input/G16_ABI-L1b-RadC/OR_ABI-L1b-RadC-M6C03_G16_s20232561536173_e20232561538546_c20232561538589.nc', 'input/G16_ABI-L1b-RadC/OR_ABI-L1b-RadC-M6C15_G16_s20232561536173_e20232561538551_c20232561539007.nc', 'input/G16_ABI-L1b-RadC/OR_ABI-L1b-RadC-M6C13_G16_s20232561536173_e20232561538557_c20232561539004.nc', 'input/G16_ABI-L1b-RadC/OR_ABI-L1b-RadC-M6C14_G16_s20232561536173_e20232561538546_c20232561538592.nc', 'input/G16_ABI-L1b-RadC/OR_ABI-L1b-RadC-M6C02_G16_s20232561536173_e20232561538546_c20232561538578.nc', 'input/G16_ABI-L1b-RadC/OR_ABI-L1b-RadC-M6C16_G16_s20232561536173_e20232561538557_c20232561539036.nc', 'input/G16_ABI-L1b-RadC/OR_ABI-L1b-RadC-M6C01_G16_s20232561536173_e20232561538548_c20232561538585.nc', 'input/G16_ABI-L1b-RadC/OR_ABI-L1b-RadC-M6C10_G16_s20232561536173_e20232561538557_c20232561539020.nc', 'input/G16_ABI-L1b-RadC/OR_ABI-L1b-RadC-M6C05_G16_s20232561536173_e20232561538546_c20232561539016.nc', 'input/G16_ABI-L1b-RadC/OR_ABI-L1b-RadC-M6C07_G16_s20232561536173_e20232561538557_c20232561539026.nc', 'input/G16_ABI-L1b-RadC/OR_ABI-L1b-RadC-M6C11_G16_s20232561536173_e20232561538546_c20232561539034.nc', 'input/G16_ABI-L1b-RadC/OR_ABI-L1b-RadC-M6C08_G16_s20232561536173_e20232561538546_c20232561538598.nc', 'input/G16_ABI-L1b-RadC/OR_ABI-L1b-RadC-M6C06_G16_s20232561536173_e20232561538551_c20232561539011.nc']
scn = Scene(filenames = sat_files, reader='abi_l1b')

dataset_names = scn.all_dataset_names()

print(dataset_names)
---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
File ~/micromamba/envs/geostationary-cookbook/lib/python3.14/site-packages/xarray/backends/file_manager.py:219, in CachingFileManager._acquire_with_cache_info(self, needs_lock)
    218 try:
--> 219     file = self._cache[self._key]
    220 except KeyError:

File ~/micromamba/envs/geostationary-cookbook/lib/python3.14/site-packages/xarray/backends/lru_cache.py:56, in LRUCache.__getitem__(self, key)
     55 with self._lock:
---> 56     value = self._cache[key]
     57     self._cache.move_to_end(key)

KeyError: [<class 'netCDF4._netCDF4.Dataset'>, ('/home/runner/work/geostationary-cookbook/geostationary-cookbook/notebooks/input/G16_ABI-L1b-RadC/OR_ABI-L1b-RadC-M6C02_G16_s20232561536173_e20232561538546_c20232561538578.nc',), 'r', (('clobber', True), ('diskless', False), ('format', 'NETCDF4'), ('persist', False)), 'f3d77038-089f-44ae-935f-27baee946511']

During handling of the above exception, another exception occurred:

OSError                                   Traceback (most recent call last)
Cell In[3], line 1
----> 1 scn = Scene(filenames = sat_files, reader='abi_l1b')
      3 dataset_names = scn.all_dataset_names()
      5 print(dataset_names)

File ~/micromamba/envs/geostationary-cookbook/lib/python3.14/site-packages/satpy/scene.py:155, in Scene.__init__(self, filenames, reader, filter_parameters, reader_kwargs)
    152 if filenames:
    153     filenames = convert_remote_files_to_fsspec(filenames, storage_options)
--> 155 self._readers = self._create_reader_instances(filenames=filenames,
    156                                               reader=reader,
    157                                               reader_kwargs=cleaned_reader_kwargs)
    158 self._datasets = DatasetDict()
    159 self._wishlist = set()

File ~/micromamba/envs/geostationary-cookbook/lib/python3.14/site-packages/satpy/scene.py:176, in Scene._create_reader_instances(self, filenames, reader, reader_kwargs)
    171 def _create_reader_instances(self,
    172                              filenames=None,
    173                              reader=None,
    174                              reader_kwargs=None):
    175     """Find readers and return their instances."""
--> 176     return load_readers(filenames=filenames,
    177                         reader=reader,
    178                         reader_kwargs=reader_kwargs)

File ~/micromamba/envs/geostationary-cookbook/lib/python3.14/site-packages/satpy/readers/core/loading.py:65, in load_readers(filenames, reader, reader_kwargs)
     63 loadables = reader_instance.select_files_from_pathnames(readers_files)
     64 if loadables:
---> 65     reader_instance.create_storage_items(
     66             loadables,
     67             fh_kwargs=reader_kwargs_without_filter[None if reader is None else reader[idx]])
     68     reader_instances[reader_instance.name] = reader_instance
     69     remaining_filenames -= set(loadables)

File ~/micromamba/envs/geostationary-cookbook/lib/python3.14/site-packages/satpy/readers/core/yaml_reader.py:618, in FileYAMLReader.create_storage_items(self, files, **kwargs)
    616 def create_storage_items(self, files, **kwargs):
    617     """Create the storage items."""
--> 618     return self.create_filehandlers(files, **kwargs)

File ~/micromamba/envs/geostationary-cookbook/lib/python3.14/site-packages/satpy/readers/core/yaml_reader.py:630, in FileYAMLReader.create_filehandlers(self, filenames, fh_kwargs)
    628 # load files that we know about by creating the file handlers
    629 for filetype, filetype_info in self.sorted_filetype_items():
--> 630     filehandlers = self._new_filehandlers_for_filetype(filetype_info,
    631                                                        filename_set,
    632                                                        fh_kwargs=fh_kwargs)
    634     if filehandlers:
    635         created_fhs[filetype] = filehandlers

File ~/micromamba/envs/geostationary-cookbook/lib/python3.14/site-packages/satpy/readers/core/yaml_reader.py:613, in FileYAMLReader._new_filehandlers_for_filetype(self, filetype_info, filenames, fh_kwargs)
    609 filehandler_iter = self._new_filehandler_instances(filetype_info,
    610                                                    filename_iter,
    611                                                    fh_kwargs=fh_kwargs)
    612 filtered_iter = self.filter_fh_by_metadata(filehandler_iter)
--> 613 return list(filtered_iter)

File ~/micromamba/envs/geostationary-cookbook/lib/python3.14/site-packages/satpy/readers/core/yaml_reader.py:595, in FileYAMLReader.filter_fh_by_metadata(self, filehandlers)
    593 def filter_fh_by_metadata(self, filehandlers):
    594     """Filter out filehandlers using provide filter parameters."""
--> 595     for filehandler in filehandlers:
    596         filehandler.metadata["start_time"] = filehandler.start_time
    597         filehandler.metadata["end_time"] = filehandler.end_time

File ~/micromamba/envs/geostationary-cookbook/lib/python3.14/site-packages/satpy/readers/core/yaml_reader.py:591, in FileYAMLReader._new_filehandler_instances(self, filetype_info, filename_items, fh_kwargs)
    588     warnings.warn(str(err) + " for {}".format(filename), stacklevel=4)
    589     continue
--> 591 yield filetype_cls(filename, filename_info, filetype_info, *req_fh, **fh_kwargs)

File ~/micromamba/envs/geostationary-cookbook/lib/python3.14/site-packages/satpy/readers/abi_l1b.py:40, in NC_ABI_L1B.__init__(self, filename, filename_info, filetype_info, clip_negative_radiances)
     38 def __init__(self, filename, filename_info, filetype_info, clip_negative_radiances=None):
     39     """Open the NetCDF file with xarray and prepare the Dataset for reading."""
---> 40     super().__init__(filename, filename_info, filetype_info)
     41     if clip_negative_radiances is None:
     42         clip_negative_radiances = satpy.config.get("readers.clip_negative_radiances")

File ~/micromamba/envs/geostationary-cookbook/lib/python3.14/site-packages/satpy/readers/core/abi.py:58, in NC_ABI_BASE.__init__(self, filename, filename_info, filetype_info, **kwargs)
     55 platform_shortname = filename_info["platform_shortname"]
     56 self.platform_name = PLATFORM_NAMES.get(platform_shortname.lower())
---> 58 self.nlines = self.nc["y"].size
     59 self.ncols = self.nc["x"].size
     61 self.coords = {}

File ~/micromamba/envs/geostationary-cookbook/lib/python3.14/functools.py:1127, in cached_property.__get__(self, instance, owner)
   1125 val = cache.get(self.attrname, _NOT_FOUND)
   1126 if val is _NOT_FOUND:
-> 1127     val = self.func(instance)
   1128     try:
   1129         cache[self.attrname] = val

File ~/micromamba/envs/geostationary-cookbook/lib/python3.14/site-packages/satpy/readers/core/abi.py:69, in NC_ABI_BASE.nc(self)
     67 with dask.config.set({"array.chunk-size": chunk_bytes}):
     68     f_obj = open_file_or_filename(self.filename)
---> 69     nc = xr.open_dataset(f_obj,
     70                          decode_cf=True,
     71                          mask_and_scale=False,
     72                          chunks="auto")
     73 nc = self._rename_dims(nc)
     74 return nc

File ~/micromamba/envs/geostationary-cookbook/lib/python3.14/site-packages/xarray/backends/api.py:596, in open_dataset(filename_or_obj, engine, chunks, cache, decode_cf, mask_and_scale, decode_times, decode_timedelta, use_cftime, concat_characters, decode_coords, drop_variables, create_default_indexes, inline_array, chunked_array_type, from_array_kwargs, backend_kwargs, **kwargs)
    584 decoders = _resolve_decoders_kwargs(
    585     decode_cf,
    586     open_backend_dataset_parameters=backend.open_dataset_parameters,
   (...)    592     decode_coords=decode_coords,
    593 )
    595 overwrite_encoded_chunks = kwargs.pop("overwrite_encoded_chunks", None)
--> 596 backend_ds = backend.open_dataset(
    597     filename_or_obj,
    598     drop_variables=drop_variables,
    599     **decoders,
    600     **kwargs,
    601 )
    602 ds = _dataset_from_backend_dataset(
    603     backend_ds,
    604     filename_or_obj,
   (...)    615     **kwargs,
    616 )
    617 return ds

File ~/micromamba/envs/geostationary-cookbook/lib/python3.14/site-packages/xarray/backends/netCDF4_.py:744, in NetCDF4BackendEntrypoint.open_dataset(self, filename_or_obj, mask_and_scale, decode_times, concat_characters, decode_coords, drop_variables, use_cftime, decode_timedelta, group, mode, format, clobber, diskless, persist, auto_complex, lock, autoclose)
    722 def open_dataset(
    723     self,
    724     filename_or_obj: T_PathFileOrDataStore,
   (...)    741     autoclose=False,
    742 ) -> Dataset:
    743     filename_or_obj = _normalize_path(filename_or_obj)
--> 744     store = NetCDF4DataStore.open(
    745         filename_or_obj,
    746         mode=mode,
    747         format=format,
    748         group=group,
    749         clobber=clobber,
    750         diskless=diskless,
    751         persist=persist,
    752         auto_complex=auto_complex,
    753         lock=lock,
    754         autoclose=autoclose,
    755     )
    757     store_entrypoint = StoreBackendEntrypoint()
    758     with close_on_error(store):

File ~/micromamba/envs/geostationary-cookbook/lib/python3.14/site-packages/xarray/backends/netCDF4_.py:524, in NetCDF4DataStore.open(cls, filename, mode, format, group, clobber, diskless, persist, auto_complex, lock, lock_maker, autoclose)
    520 else:
    521     manager = CachingFileManager(
    522         netCDF4.Dataset, filename, mode=mode, kwargs=kwargs
    523     )
--> 524 return cls(manager, group=group, mode=mode, lock=lock, autoclose=autoclose)

File ~/micromamba/envs/geostationary-cookbook/lib/python3.14/site-packages/xarray/backends/netCDF4_.py:428, in NetCDF4DataStore.__init__(self, manager, group, mode, lock, autoclose)
    426 self._group = group
    427 self._mode = mode
--> 428 self.format = self.ds.data_model
    429 self._filename = self.ds.filepath()
    430 self.is_remote = is_remote_uri(self._filename)

File ~/micromamba/envs/geostationary-cookbook/lib/python3.14/site-packages/xarray/backends/netCDF4_.py:533, in NetCDF4DataStore.ds(self)
    531 @property
    532 def ds(self):
--> 533     return self._acquire()

File ~/micromamba/envs/geostationary-cookbook/lib/python3.14/site-packages/xarray/backends/netCDF4_.py:527, in NetCDF4DataStore._acquire(self, needs_lock)
    526 def _acquire(self, needs_lock=True):
--> 527     with self._manager.acquire_context(needs_lock) as root:
    528         ds = _nc4_require_group(root, self._group, self._mode)
    529     return ds

File ~/micromamba/envs/geostationary-cookbook/lib/python3.14/contextlib.py:141, in _GeneratorContextManager.__enter__(self)
    139 del self.args, self.kwds, self.func
    140 try:
--> 141     return next(self.gen)
    142 except StopIteration:
    143     raise RuntimeError("generator didn't yield") from None

File ~/micromamba/envs/geostationary-cookbook/lib/python3.14/site-packages/xarray/backends/file_manager.py:207, in CachingFileManager.acquire_context(self, needs_lock)
    204 @contextmanager
    205 def acquire_context(self, needs_lock: bool = True) -> Iterator[T_File]:
    206     """Context manager for acquiring a file."""
--> 207     file, cached = self._acquire_with_cache_info(needs_lock)
    208     try:
    209         yield file

File ~/micromamba/envs/geostationary-cookbook/lib/python3.14/site-packages/xarray/backends/file_manager.py:225, in CachingFileManager._acquire_with_cache_info(self, needs_lock)
    223     kwargs = kwargs.copy()
    224     kwargs["mode"] = self._mode
--> 225 file = self._opener(*self._args, **kwargs)
    226 if self._mode == "w":
    227     # ensure file doesn't get overridden when opened again
    228     self._mode = "a"

File src/netCDF4/_netCDF4.pyx:2517, in netCDF4._netCDF4.Dataset.__init__()

File src/netCDF4/_netCDF4.pyx:2154, in netCDF4._netCDF4._ensure_nc_success()

OSError: [Errno -101] NetCDF: HDF error: '/home/runner/work/geostationary-cookbook/geostationary-cookbook/notebooks/input/G16_ABI-L1b-RadC/OR_ABI-L1b-RadC-M6C02_G16_s20232561536173_e20232561538546_c20232561538578.nc'
scn.load([f'C{x:02d}' for x in range(1, 17)])
print(scn.available_composite_names())
rgb_im = 'airmass'

scn.load([rgb_im])
### Uncomment to show it
#scn.show(rgb_im)
result = scn[rgb_im]

#result
keys = scn.keys()

#keys
area_info = scn["C13"].area

#area_info
area_info = scn["C01"].area

#area_info
area_info = scn["C02"].area

#area_info
scn.load(["natural_color"])
rs = scn["C13"].area

lscn = scn.resample(rs)
lscn.load(["natural_color"])

### Uncomment to show it
#lscn.show("natural_color")
lscn.load(['true_color'])

### Uncomment to show it
#lscn.show('true_color')