
This notebook is developed during the pythia cook-off at NCAR Mesa-Lab Boulder Colorado, June 12-14, 2024
Participants in the workshop event have the chance to practice collaborative problem-solving and hands-on learning in the field of Python programming.
This notebook is part of the Breakout Topic: Geostationary on AWS, lead by Jorge Humberto Bravo Mendez jbravo2@stevens.edu, from Stevens Institute of Technology
Advanced Baseline Imager (ABI) data with Satpy¶
Using Satpy to read and Advanced Baseline Imager (ABI) data from GOES-R satellites. Here’s a step-by-step guide:
Imports¶
import warnings
warnings.filterwarnings('ignore')
from satpy.scene import Scene
from satpy.utils import debug_on
from datetime import datetime
from glob import glob
Starting to create satpy scenes¶
sat_files = glob("input/G18_ABI-L1b-RadC/*")
sat_files
['input/G18_ABI-L1b-RadC/OR_ABI-L1b-RadC-M6C10_G18_s20230041816176_e20230041818563_c20230041819021.nc',
'input/G18_ABI-L1b-RadC/OR_ABI-L1b-RadC-M6C03_G18_s20230041816176_e20230041818550_c20230041818584.nc',
'input/G18_ABI-L1b-RadC/OR_ABI-L1b-RadC-M6C05_G18_s20230041816176_e20230041818550_c20230041819005.nc',
'input/G18_ABI-L1b-RadC/OR_ABI-L1b-RadC-M6C12_G18_s20230041816176_e20230041818555_c20230041819009.nc',
'input/G18_ABI-L1b-RadC/OR_ABI-L1b-RadC-M6C09_G18_s20230041816176_e20230041818555_c20230041819014.nc',
'input/G18_ABI-L1b-RadC/OR_ABI-L1b-RadC-M6C16_G18_s20230041816176_e20230041818561_c20230041819025.nc',
'input/G18_ABI-L1b-RadC/OR_ABI-L1b-RadC-M6C04_G18_s20230041816176_e20230041818549_c20230041818580.nc',
'input/G18_ABI-L1b-RadC/OR_ABI-L1b-RadC-M6C08_G18_s20230041816176_e20230041818549_c20230041818599.nc',
'input/G18_ABI-L1b-RadC/OR_ABI-L1b-RadC-M6C01_G18_s20230041816176_e20230041818551_c20230041818587.nc',
'input/G18_ABI-L1b-RadC/OR_ABI-L1b-RadC-M6C02_G18_s20230041816176_e20230041818549_c20230041818582.nc',
'input/G18_ABI-L1b-RadC/OR_ABI-L1b-RadC-M6C15_G18_s20230041816176_e20230041818555_c20230041819030.nc',
'input/G18_ABI-L1b-RadC/OR_ABI-L1b-RadC-M6C13_G18_s20230041816176_e20230041818561_c20230041819034.nc',
'input/G18_ABI-L1b-RadC/OR_ABI-L1b-RadC-M6C06_G18_s20230041816176_e20230041818555_c20230041819032.nc',
'input/G18_ABI-L1b-RadC/OR_ABI-L1b-RadC-M6C07_G18_s20230041816176_e20230041818562_c20230041818592.nc',
'input/G18_ABI-L1b-RadC/OR_ABI-L1b-RadC-M6C11_G18_s20230041816176_e20230041818549_c20230041818594.nc',
'input/G18_ABI-L1b-RadC/OR_ABI-L1b-RadC-M6C14_G18_s20230041816176_e20230041818549_c20230041819017.nc']
scn = Scene(filenames = sat_files, reader='abi_l1b')
dataset_names = scn.all_dataset_names()
print(dataset_names)
['C01', 'C02', 'C03', 'C04', 'C05', 'C06', 'C07', 'C08', 'C09', 'C10', 'C11', 'C12', 'C13', 'C14', 'C15', 'C16']
scn.load([f'C{x:02d}' for x in range(1, 17)])
print(scn.available_composite_names())
['24h_microphysics', 'airmass', 'ash', 'blowing_snow', 'cimss_cloud_type', 'cimss_cloud_type_raw', 'cimss_green', 'cimss_green_sunz', 'cimss_green_sunz_rayleigh', 'cimss_true_color', 'cimss_true_color_sunz', 'cimss_true_color_sunz_rayleigh', 'cira_day_convection', 'cira_fire_temperature', 'cloud_phase', 'cloud_phase_distinction', 'cloud_phase_distinction_raw', 'cloud_phase_raw', 'cloudtop', 'color_infrared', 'colorized_ir_clouds', 'convection', 'day_cloud_type', 'day_microphysics', 'day_microphysics_abi', 'day_microphysics_eum', 'day_severe_storms', 'day_severe_storms_tropical', 'dust', 'fire_temperature_awips', 'fog', 'geo_color', 'geo_color_background_with_low_clouds', 'geo_color_high_clouds', 'geo_color_low_clouds', 'geo_color_night', 'green', 'green_crefl', 'green_nocorr', 'green_raw', 'green_snow', 'highlight_C14', 'ir108_3d', 'ir_cloud_day', 'land_cloud', 'land_cloud_fire', 'natural_color', 'natural_color_nocorr', 'natural_color_raw', 'natural_color_raw_with_night_ir', 'night_fog', 'night_ir_alpha', 'night_ir_with_background', 'night_ir_with_background_hires', 'night_microphysics', 'night_microphysics_eum', 'night_microphysics_tropical', 'overshooting_tops', 'overview', 'overview_raw', 'rocket_plume_day', 'rocket_plume_night', 'snow', 'snow_fog', 'so2', 'tropical_airmass', 'true_color', 'true_color_crefl', 'true_color_nocorr', 'true_color_raw', 'true_color_reproduction', 'true_color_reproduction_corr', 'true_color_reproduction_uncorr', 'true_color_with_night_fires', 'true_color_with_night_fires_nocorr', 'true_color_with_night_ir', 'true_color_with_night_ir_hires', 'water_vapors1', 'water_vapors2']
rgb_im = 'airmass'
scn.load([rgb_im])
### Uncomment to show it
#scn.show(rgb_im)
result = scn[rgb_im]
result
Loading...
keys = scn.keys()
keys
[DataID(name='C01', wavelength=WavelengthRange(min=0.45, central=0.47, max=0.49, unit='µm'), resolution=1000, calibration=<1>, modifiers=()),
DataID(name='C02', wavelength=WavelengthRange(min=0.59, central=0.64, max=0.69, unit='µm'), resolution=500, calibration=<1>, modifiers=()),
DataID(name='C03', wavelength=WavelengthRange(min=0.8455, central=0.865, max=0.8845, unit='µm'), resolution=1000, calibration=<1>, modifiers=()),
DataID(name='C04', wavelength=WavelengthRange(min=1.3705, central=1.378, max=1.3855, unit='µm'), resolution=2000, calibration=<1>, modifiers=()),
DataID(name='C05', wavelength=WavelengthRange(min=1.58, central=1.61, max=1.64, unit='µm'), resolution=1000, calibration=<1>, modifiers=()),
DataID(name='C06', wavelength=WavelengthRange(min=2.225, central=2.25, max=2.275, unit='µm'), resolution=2000, calibration=<1>, modifiers=()),
DataID(name='C07', wavelength=WavelengthRange(min=3.8, central=3.9, max=4.0, unit='µm'), resolution=2000, calibration=<2>, modifiers=()),
DataID(name='C08', wavelength=WavelengthRange(min=5.77, central=6.185, max=6.6, unit='µm'), resolution=2000, calibration=<2>, modifiers=()),
DataID(name='C09', wavelength=WavelengthRange(min=6.75, central=6.95, max=7.15, unit='µm'), resolution=2000, calibration=<2>, modifiers=()),
DataID(name='C10', wavelength=WavelengthRange(min=7.24, central=7.34, max=7.44, unit='µm'), resolution=2000, calibration=<2>, modifiers=()),
DataID(name='C11', wavelength=WavelengthRange(min=8.3, central=8.5, max=8.7, unit='µm'), resolution=2000, calibration=<2>, modifiers=()),
DataID(name='C12', wavelength=WavelengthRange(min=9.42, central=9.61, max=9.8, unit='µm'), resolution=2000, calibration=<2>, modifiers=()),
DataID(name='C13', wavelength=WavelengthRange(min=10.1, central=10.35, max=10.6, unit='µm'), resolution=2000, calibration=<2>, modifiers=()),
DataID(name='C14', wavelength=WavelengthRange(min=10.8, central=11.2, max=11.6, unit='µm'), resolution=2000, calibration=<2>, modifiers=()),
DataID(name='C15', wavelength=WavelengthRange(min=11.8, central=12.3, max=12.8, unit='µm'), resolution=2000, calibration=<2>, modifiers=()),
DataID(name='C16', wavelength=WavelengthRange(min=13.0, central=13.3, max=13.6, unit='µm'), resolution=2000, calibration=<2>, modifiers=()),
DataID(name='airmass', resolution=2000)]
area_info = scn["C13"].area
area_info
Loading...
area_info = scn["C01"].area
area_info
Loading...
area_info = scn["C02"].area
area_info
Loading...
scn.load(["natural_color"])
The following datasets were not created and may require resampling to be generated: DataID(name='natural_color')
rs = scn["C13"].area
lscn = scn.resample(rs)
lscn.load(["natural_color"])
### Uncomment to show it
#lscn.show("natural_color")
lscn.load(['true_color'])
### Uncomment to show it
#lscn.show('true_color')
0%| | 0/52 [00:00<?, ?kB/s]
100%|██████████| 52/52 [00:00<00:00, 880.53kB/s]
---------------------------------------------------------------------------
EOFError Traceback (most recent call last)
Cell In[16], line 1
----> 1 lscn.load(['true_color'])
3 ### Uncomment to show it
4 #lscn.show('true_color')
File ~/micromamba/envs/geostationary-cookbook/lib/python3.13/site-packages/satpy/scene.py:1476, in Scene.load(self, wishlist, calibration, resolution, polarization, level, modifiers, generate, unload, **kwargs)
1474 self._read_datasets_from_storage(**kwargs)
1475 if generate:
-> 1476 self.generate_possible_composites(unload)
File ~/micromamba/envs/geostationary-cookbook/lib/python3.13/site-packages/satpy/scene.py:1539, in Scene.generate_possible_composites(self, unload)
1532 def generate_possible_composites(self, unload):
1533 """See which composites can be generated and generate them.
1534
1535 Args:
1536 unload (bool): if the dependencies of the composites
1537 should be unloaded after successful generation.
1538 """
-> 1539 keepables = self._generate_composites_from_loaded_datasets()
1541 if self.missing_datasets:
1542 self._remove_failed_datasets(keepables)
File ~/micromamba/envs/geostationary-cookbook/lib/python3.13/site-packages/satpy/scene.py:1558, in Scene._generate_composites_from_loaded_datasets(self)
1555 trunk_nodes = self._dependency_tree.trunk(limit_nodes_to=self.missing_datasets,
1556 limit_children_to=self._datasets.keys())
1557 needed_comp_nodes = set(self._filter_loaded_datasets_from_trunk_nodes(trunk_nodes))
-> 1558 return self._generate_composites_nodes_from_loaded_datasets(needed_comp_nodes)
File ~/micromamba/envs/geostationary-cookbook/lib/python3.13/site-packages/satpy/scene.py:1564, in Scene._generate_composites_nodes_from_loaded_datasets(self, compositor_nodes)
1562 keepables = set()
1563 for node in compositor_nodes:
-> 1564 self._generate_composite(node, keepables)
1565 return keepables
File ~/micromamba/envs/geostationary-cookbook/lib/python3.13/site-packages/satpy/scene.py:1588, in Scene._generate_composite(self, comp_node, keepables)
1586 try:
1587 delayed_prereq = False
-> 1588 prereq_datasets = self._get_prereq_datasets(
1589 comp_node.name,
1590 prereqs,
1591 keepables,
1592 )
1593 except DelayedGeneration:
1594 # if we are missing a required dependency that could be generated
1595 # later then we need to wait to return until after we've also
1596 # processed the optional dependencies
1597 delayed_prereq = True
File ~/micromamba/envs/geostationary-cookbook/lib/python3.13/site-packages/satpy/scene.py:1670, in Scene._get_prereq_datasets(self, comp_id, prereq_nodes, keepables, skip)
1667 prereq_id = prereq_node.name
1668 if prereq_id not in self._datasets and prereq_id not in keepables \
1669 and isinstance(prereq_node, CompositorNode):
-> 1670 self._generate_composite(prereq_node, keepables)
1672 # composite generation may have updated the DataID
1673 prereq_id = prereq_node.name
File ~/micromamba/envs/geostationary-cookbook/lib/python3.13/site-packages/satpy/scene.py:1622, in Scene._generate_composite(self, comp_node, keepables)
1619 return
1621 try:
-> 1622 composite = compositor(prereq_datasets,
1623 optional_datasets=optional_datasets,
1624 **comp_node.name.to_dict())
1625 cid = DataID.new_id_from_dataarray(composite)
1626 self._datasets[cid] = composite
File ~/micromamba/envs/geostationary-cookbook/lib/python3.13/site-packages/satpy/modifiers/atmosphere.py:107, in PSPRayleighReflectance.__call__(self, projectables, optional_datasets, **info)
102 reduce_strength = np.clip(self.attrs.get("reduce_strength", 0), 0, 1).astype(vis.dtype)
104 logger.info("Removing Rayleigh scattering with atmosphere '%s' and "
105 "aerosol type '%s' for '%s'",
106 atmosphere, aerosol_type, vis.attrs["name"])
--> 107 corrector = Rayleigh(vis.attrs["platform_name"], vis.attrs["sensor"],
108 atmosphere=atmosphere,
109 aerosol_type=aerosol_type)
111 try:
112 refl_cor_band = corrector.get_reflectance(sunz, satz, ssadiff,
113 vis.attrs["name"],
114 red.data)
File ~/micromamba/envs/geostationary-cookbook/lib/python3.13/site-packages/pyspectral/rayleigh.py:173, in Rayleigh.__init__(self, platform_name, sensor, **kwargs)
171 if not self._lutfiles_version_uptodate and self.do_download:
172 LOG.info("Will download from internet...")
--> 173 download_luts(aerosol_type=aerosol_type)
175 if (not os.path.exists(self.reflectance_lut_filename) or
176 not os.path.isfile(self.reflectance_lut_filename)):
177 raise IOError('pyspectral file for Rayleigh scattering correction ' +
178 'does not exist! Filename = ' +
179 str(self.reflectance_lut_filename))
File ~/micromamba/envs/geostationary-cookbook/lib/python3.13/site-packages/pyspectral/utils.py:392, in download_luts(aerosol_types, dry_run, aerosol_type)
389 continue
391 local_tarball_pathname = os.path.join(subdir_path, "pyspectral_rayleigh_correction_luts.tgz")
--> 392 _download_tarball_and_extract(lut_tarball_url, local_tarball_pathname, subdir_path)
File ~/micromamba/envs/geostationary-cookbook/lib/python3.13/site-packages/pyspectral/utils.py:421, in _download_tarball_and_extract(tarball_url, local_pathname, extract_dir)
419 tar = tarfile.open(local_pathname)
420 tar_kwargs = {} if sys.version_info < (3, 12) else {"filter": "data"}
--> 421 tar.extractall(extract_dir, **tar_kwargs)
422 tar.close()
423 os.remove(local_pathname)
File ~/micromamba/envs/geostationary-cookbook/lib/python3.13/tarfile.py:2352, in TarFile.extractall(self, path, members, numeric_owner, filter)
2347 if tarinfo.isdir():
2348 # For directories, delay setting attributes until later,
2349 # since permissions can interfere with extraction and
2350 # extracting contents can reset mtime.
2351 directories.append(unfiltered)
-> 2352 self._extract_one(tarinfo, path, set_attrs=not tarinfo.isdir(),
2353 numeric_owner=numeric_owner,
2354 filter_function=filter_function)
2356 # Reverse sort directories.
2357 directories.sort(key=lambda a: a.name, reverse=True)
File ~/micromamba/envs/geostationary-cookbook/lib/python3.13/tarfile.py:2455, in TarFile._extract_one(self, tarinfo, path, set_attrs, numeric_owner, filter_function)
2452 self._check("r")
2454 try:
-> 2455 self._extract_member(tarinfo, os.path.join(path, tarinfo.name),
2456 set_attrs=set_attrs,
2457 numeric_owner=numeric_owner,
2458 filter_function=filter_function,
2459 extraction_root=path)
2460 except (OSError, UnicodeEncodeError) as e:
2461 self._handle_fatal_error(e)
File ~/micromamba/envs/geostationary-cookbook/lib/python3.13/tarfile.py:2544, in TarFile._extract_member(self, tarinfo, targetpath, set_attrs, numeric_owner, filter_function, extraction_root)
2541 self._dbg(1, tarinfo.name)
2543 if tarinfo.isreg():
-> 2544 self.makefile(tarinfo, targetpath)
2545 elif tarinfo.isdir():
2546 self.makedir(tarinfo, targetpath)
File ~/micromamba/envs/geostationary-cookbook/lib/python3.13/tarfile.py:2601, in TarFile.makefile(self, tarinfo, targetpath)
2599 target.truncate()
2600 else:
-> 2601 copyfileobj(source, target, tarinfo.size, ReadError, bufsize)
File ~/micromamba/envs/geostationary-cookbook/lib/python3.13/tarfile.py:250, in copyfileobj(src, dst, length, exception, bufsize)
248 blocks, remainder = divmod(length, bufsize)
249 for b in range(blocks):
--> 250 buf = src.read(bufsize)
251 if len(buf) < bufsize:
252 raise exception("unexpected end of data")
File ~/micromamba/envs/geostationary-cookbook/lib/python3.13/gzip.py:340, in GzipFile.read(self, size)
338 import errno
339 raise OSError(errno.EBADF, "read() on write-only GzipFile object")
--> 340 return self._buffer.read(size)
File ~/micromamba/envs/geostationary-cookbook/lib/python3.13/_compression.py:68, in DecompressReader.readinto(self, b)
66 def readinto(self, b):
67 with memoryview(b) as view, view.cast("B") as byte_view:
---> 68 data = self.read(len(byte_view))
69 byte_view[:len(data)] = data
70 return len(data)
File ~/micromamba/envs/geostationary-cookbook/lib/python3.13/gzip.py:566, in _GzipReader.read(self, size)
564 break
565 if buf == b"":
--> 566 raise EOFError("Compressed file ended before the "
567 "end-of-stream marker was reached")
569 self._crc = zlib.crc32(uncompress, self._crc)
570 self._stream_size += len(uncompress)
EOFError: Compressed file ended before the end-of-stream marker was reached