Preserving a spatial quantity of interest (QoI) with safeguards¶
In this example, we compute the relative vorticity on a dataset of wind u, v vectors, which requires taking the derivative along both variables. We compare how three different lossy compressors (ZFP, SZ3, and SPERR) affect the derived relative vorticity when compressing the u and v variables (stacked into one variable). Next, we apply safeguards to guarantee an error bound on the derived relative vorticity. We also compare the safeguards with the compressor configuration auto-tuner OptZConfig.
Stacking u and v into one variable that is then compressed is possible because u and v have very similar data distributions.
This example also showcases how to deal with boundary conditions in spatial data. For instance, the longitude coordinate is periodic and computing a derivative along the longitude needs to handle the periodic coordinates.
QPET supports mean error bounds over non-overlapping blocks of data, but not over overlapping windows of data with boundary conditions, which are required to preserve an error over a finite-difference approximated spatial derivative. To be safe, QPET would need to be configured with a maximally conservative pointwise error bound, which is no different than configuring SPERR (or SZ3 or ZFP) with this error bound. Therefore, we do not compare with QPET in this example.
import ssl
ssl._create_default_https_context = ssl._create_stdlib_context
import copy
from pathlib import Path
import earthkit.plots
import humanize
import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import xarray as xr
from matplotlib import patheffects as PathEffects
# Retrieve the data
ERA5 = xr.open_dataset(Path() / "data" / "era5-uv" / "data.nc")
ERA5 = ERA5.sel(valid_time="2024-04-02T12:00:00", pressure_level=500)
def compute_relative_vorticity(ERA5: xr.Dataset) -> xr.DataArray:
# reimplementation of np.deg2rad that matches the safeguards QoI
def deg2rad(a: np.ndarray) -> np.ndarray:
# np.deg2rad(a) = a * a.dtype.type(np.pi / 180)
return a * a.dtype.type(np.pi) / a.dtype.type(180)
earth_radius = 6371000 # [m], globally averaged
# computing the derivative with a finite difference requires extending the
# data domain and tricking xarray for the coordinates
# e.g. the data needs to be wrapped along the longitude axis,
# but the longitude coordinate needs to be extended with odd reflection
# ([0, 0.25, ..., 359.75, 360] -> [-0.25, 0, 0.25, ..., 359.75, 360, 360.25])
# since xarray cannot handle differentiating along a proper periodic axis
# along the latitude axis, xr.differentiate -> np.gradient falls back to
# forward/backwards differences at the boundaries (poles)
ERA5_wrapped = ERA5.pad(longitude=1, mode="wrap").assign_coords(
longitude=ERA5.longitude.pad(longitude=1, mode="reflect", reflect_type="odd"),
)
# compute the relative vorticity
ERA5_dUdTheta = (
ERA5_wrapped["u"]
* np.cos(deg2rad(ERA5_wrapped["latitude"].astype(ERA5_wrapped["u"].dtype)))
).differentiate("latitude")
ERA5_dVdPhi = ERA5_wrapped["v"].differentiate("longitude")
ERA5_VOR = (ERA5_dVdPhi - ERA5_dUdTheta) / (
earth_radius
* np.cos(deg2rad(ERA5_wrapped["latitude"].astype(ERA5_wrapped["u"].dtype)))
)
# remove the padding to extract just the valid values
ERA5_VOR = ERA5_VOR.sel(longitude=slice(0, 359.9))
ERA5_VOR.attrs.update(long_name="Relative vorticity", units="s**-1")
return ERA5_VOR
ERA5_VOR = compute_relative_vorticity(ERA5)
def compute_corrections_percentage(my_ERA5: xr.Dataset, orig_ERA5: xr.Dataset) -> float:
neq = np.sum(my_ERA5 != orig_ERA5)
return int(neq.u + neq.v) / int(orig_ERA5.u.size + orig_ERA5.v.size)
old_cmap_and_norm = earthkit.plots.styles.colors.cmap_and_norm
def my_cmap_and_norm(colors, levels, normalize=True, extend=None, extend_levels=True):
return old_cmap_and_norm(colors, levels, normalize, extend, True)
earthkit.plots.styles.colors.cmap_and_norm = my_cmap_and_norm
def plot_relative_vorticity(
my_ERA5: xr.Dataset,
cr,
chart,
title,
span,
vor_eb_abs,
error=False,
corr=None,
my_ERA5_it=None,
cr_it=None,
inset=True,
):
my_ERA5_VOR = compute_relative_vorticity(my_ERA5)
if error:
with xr.set_options(keep_attrs=True):
da = (my_ERA5_VOR - ERA5_VOR).compute()
da.attrs.update(long_name=f"Absolute error over {da.long_name.lower()}")
else:
da = my_ERA5_VOR
# compute the default style that earthkit.maps would apply
source = earthkit.plots.sources.XarraySource(da)
style = copy.deepcopy(
earthkit.plots.styles.auto.guess_style(
source,
units=source.units,
)
)
style._levels = earthkit.plots.styles.levels.Levels(np.linspace(-span, span, 22))
style._legend_kwargs["ticks"] = np.linspace(-span, span, 5)
style._colors = "coolwarm" if error else "BrBG"
extend_left = np.nanmin(da) < -span
extend_right = np.nanmax(da) > span
extend = {
(False, False): "neither",
(True, False): "min",
(False, True): "max",
(True, True): "both",
}[(extend_left, extend_right)]
style._legend_kwargs["extend"] = extend
if error:
chart.pcolormesh(da, style=style, zorder=-12)
if corr is not None:
da_hatch = (my_ERA5["u"] == corr["u"]) & (my_ERA5["v"] == corr["v"])
if my_ERA5_it is None:
da_corr = da_hatch.astype(float)
else:
with xr.set_options(keep_attrs=True):
da_hatch_it = (my_ERA5_it["u"] == corr["u"]) & (
my_ERA5_it["v"] == corr["v"]
)
da_corr = (~da_hatch).astype(float) + (~da_hatch_it).astype(float)
old_process_projection_requirements = (
chart.ax.get_figure()._process_projection_requirements
)
def _process_projection_requirements(
*, axes_class=None, polar=False, projection=None, **kwargs
):
if axes_class is not None and projection is not None:
return axes_class, dict(projection=projection, **kwargs)
return old_process_projection_requirements(
axes_class=axes_class, polar=polar, projection=projection, **kwargs
)
chart.ax.get_figure()._process_projection_requirements = (
_process_projection_requirements
)
axin = chart.ax.inset_axes(
[0.025, 0.05, 1 / 3, 1 / 3],
xticklabels=[],
yticklabels=[],
axes_class=type(chart.ax),
projection=chart.ax.projection,
)
axin.pcolormesh(
da_corr.longitude.values,
da_corr.latitude.values,
np.squeeze(da_corr.values),
cmap=mpl.colors.ListedColormap(["white", "green", "lawngreen"]),
vmin=0,
vmax=2,
rasterized=True,
)
axin.coastlines(color="#555555")
axin.spines["geo"].set_edgecolor("black")
axin.set_title(
"Corrections",
path_effects=[PathEffects.withStroke(linewidth=3, foreground="white")],
)
elif inset:
da_err = ~(np.abs(da) <= vor_eb_abs)
old_process_projection_requirements = (
chart.ax.get_figure()._process_projection_requirements
)
def _process_projection_requirements(
*, axes_class=None, polar=False, projection=None, **kwargs
):
if axes_class is not None and projection is not None:
return axes_class, dict(projection=projection, **kwargs)
return old_process_projection_requirements(
axes_class=axes_class, polar=polar, projection=projection, **kwargs
)
chart.ax.get_figure()._process_projection_requirements = (
_process_projection_requirements
)
axin = chart.ax.inset_axes(
[0.025, 0.05, 1 / 3, 1 / 3],
xticklabels=[],
yticklabels=[],
axes_class=type(chart.ax),
projection=chart.ax.projection,
)
axin.pcolormesh(
da_err.longitude.values,
da_err.latitude.values,
np.squeeze(da_err.values),
cmap=mpl.colors.ListedColormap(["white", "red"]),
vmin=0,
vmax=1,
rasterized=True,
)
axin.coastlines(color="#555555")
axin.spines["geo"].set_edgecolor("black")
axin.set_title(
"Violations",
path_effects=[PathEffects.withStroke(linewidth=3, foreground="white")],
)
else:
chart.pcolormesh(da, style=style, zorder=-11)
chart.ax.set_rasterization_zorder(-10)
chart.title(title)
if error:
err_v = np.mean(~(np.abs(my_ERA5_VOR - ERA5_VOR) <= vor_eb_abs))
err_v = (
0
if err_v == 0
else np.format_float_positional(100 * err_v, precision=1, min_digits=1)
+ "%"
)
if err_v == "0.0%":
err_v = "<0.05%"
t = chart.ax.text(
0.95,
0.1,
f"V={err_v}",
ha="right",
va="bottom",
transform=chart.ax.transAxes,
)
t.set_bbox(dict(facecolor="white", alpha=0.75, edgecolor="black"))
t = chart.ax.text(
0.95,
0.9,
(
rf"$\times$ {np.round(cr, 2)}"
+ ("" if cr_it is None else rf" ($\times$ {np.round(cr_it, 2)})")
)
if error
else humanize.naturalsize(ERA5["u"].nbytes + ERA5["v"].nbytes, binary=True),
ha="right",
va="top",
transform=chart.ax.transAxes,
)
t.set_bbox(dict(facecolor="white", alpha=0.75, edgecolor="black"))
for m in earthkit.plots.schemas.schema.quickmap_subplot_workflow:
if m != "title":
getattr(chart, m)()
for m in earthkit.plots.schemas.schema.quickmap_figure_workflow:
getattr(chart, m)()
counts, bins = np.histogram(da.values.flatten(), range=(-span, span), bins=21)
midpoints = bins[:-1] + np.diff(bins) / 2
cb = chart.ax.collections[0].colorbar
if error:
if extend_left:
cb._extend_patches[0].set_hatch("xx")
cb._extend_patches[0].set_ec("white")
cb.ax.fill_between(
[-span, -vor_eb_abs], *cb.ax.get_ylim(), hatch="xx", ec="w", fc="none", lw=0
)
cb.ax.fill_between(
[vor_eb_abs, span], *cb.ax.get_ylim(), hatch="xx", ec="w", fc="none", lw=0
)
if extend_right:
cb._extend_patches[-1].set_hatch("xx")
cb._extend_patches[-1].set_ec("white")
extend_width = (bins[-1] - bins[-2]) / (bins[-1] - bins[0])
cax = cb.ax.inset_axes(
[
0.0 - extend_width * extend_left,
1.25,
1.0 + extend_width * (0 + extend_left + extend_right),
1.0,
]
)
cax.bar(
midpoints,
height=counts,
width=(bins[-1] - bins[0]) / len(counts),
color=cb.cmap(cb.norm(midpoints)),
**(
dict(
hatch=["xx" if np.abs(m) > vor_eb_abs else "" for m in midpoints],
ec="white",
lw=0,
)
if error
else dict()
),
)
if extend_left:
cax.bar(
bins[0] - (bins[1] - bins[0]) / 2,
height=np.sum(da < -span),
width=(bins[-1] - bins[0]) / len(counts),
color=cb.cmap(cb.norm(midpoints[0])),
**(
dict(
hatch="xx",
ec="white",
lw=0,
)
if error
else dict()
),
)
if extend_right:
cax.bar(
bins[-1] + (bins[-1] - bins[-2]) / 2,
height=np.sum(da > span),
width=(bins[-1] - bins[0]) / len(counts),
color=cb.cmap(cb.norm(midpoints[-1])),
**(
dict(
hatch="xx",
ec="white",
lw=0,
)
if error
else dict()
),
)
q1, q2, q3 = da.quantile([0.25, 0.5, 0.75]).values
cax.axvline(da.mean().item(), ls=":", ymin=0.1, ymax=0.9, c="w", lw=2)
cax.axvline(q1, ymin=0.25, ymax=0.75, c="w", lw=2)
cax.axvline(q2, ymin=0.1, ymax=0.9, c="w", lw=2)
cax.axvline(q3, ymin=0.25, ymax=0.75, c="w", lw=2)
cax.axvline(da.mean().item(), ymin=0.1, ymax=0.9, ls=":", c="k", lw=1)
cax.axvline(q1, ymin=0.25, ymax=0.75, c="k", lw=1)
cax.axvline(q2, ymin=0.1, ymax=0.9, c="k", lw=1)
cax.axvline(q3, ymin=0.25, ymax=0.75, c="k", lw=1)
cax.set_xlim(
-span - (bins[-1] - bins[-2]) * extend_left,
span + (bins[-1] - bins[-2]) * extend_right,
)
cax.set_xticks([])
cax.set_yticks([])
cax.spines[:].set_visible(False)
def table_relative_vorticity(
my_ERA5: xr.Dataset,
cr,
title,
vor_eb_abs,
corr,
) -> pd.DataFrame:
my_ERA5_VOR = compute_relative_vorticity(my_ERA5)
err_inf_U = np.amax(np.abs(my_ERA5["u"] - ERA5["u"]))
err_inf_V = np.amax(np.abs(my_ERA5["v"] - ERA5["v"]))
err_inf_VOR = np.amax(np.abs(my_ERA5_VOR - ERA5_VOR))
err_2_VOR = np.sqrt(np.mean(np.square(my_ERA5_VOR - ERA5_VOR)))
err_v = np.mean(~(np.abs(my_ERA5_VOR - ERA5_VOR) <= vor_eb_abs))
err_v = (
0
if err_v == 0
else np.format_float_positional(100 * err_v, precision=1, min_digits=1) + "%"
)
if err_v == "0.0%":
err_v = "<0.05%"
corr = None if corr is None else compute_corrections_percentage(my_ERA5, corr)
corr = (
""
if corr is None
else (
0
if corr == 0
else np.format_float_positional(100 * corr, precision=1, min_digits=1) + "%"
)
)
if corr == "0.0%":
corr = "<0.05%"
return pd.DataFrame(
{
"Compressor": [title[0]],
"Safeguarded": [title[1]],
"Corrections": [title[2]],
r"$L_{\infty}(\hat{u})$": [
f"{err_inf_U:.03}",
],
r"$L_{\infty}(\hat{v})$": [
f"{err_inf_V:.03}",
],
r"$L_{\infty}(\hat{\zeta})$": [
f"{err_inf_VOR:.03}",
],
r"$L_{2}(\hat{\zeta})$": [
f"{err_2_VOR:.03}",
],
"V": [err_v],
"C": [corr],
"CR": [
rf"$\times$ {np.round(cr, 2)}",
],
}
)
# Since numcodecs-safeguards only supports single-variable safeguarding, we
# stack the u and v variables into a combined variable.
ERA5_UV = np.stack([ERA5["u"].values, ERA5["v"].values], axis=0)
ERA5.u.dims, ERA5_UV.shape
(('latitude', 'longitude'), (2, 721, 1440))
import observe
observations = []
Lossless compression¶
We first compress the data losslessly with ZStandard at level 22, which gives maximum compression, to provide a baseline.
from numcodecs_wasm_zstd import Zstd
zstd = Zstd(level=22)
with observe.observe(zstd, observations):
ERA5_UV_zstd_enc = zstd.encode(ERA5_UV)
ERA5_UV_zstd = zstd.decode(ERA5_UV_zstd_enc)
ERA5_zstd = ERA5.copy(data=dict(u=ERA5_UV_zstd[0], v=ERA5_UV_zstd[1]))
ERA5_zstd_cr = ERA5_UV.nbytes / ERA5_UV_zstd_enc.nbytes
Compressing u and v with lossy compressors¶
We configure each compressor with an absolute error bound of 0.01 m/s over the u-v array, which (mostly) produces errors on the order of $10^{-8}$ on the derived relative vorticity.
eb_abs = 0.01
from numcodecs_wasm_zfp import Zfp
zfp = Zfp(mode="fixed-accuracy", tolerance=eb_abs)
with observe.observe(zfp, observations):
ERA5_UV_zfp_enc = zfp.encode(ERA5_UV)
ERA5_UV_zfp = zfp.decode(ERA5_UV_zfp_enc)
ERA5_zfp = ERA5.copy(data=dict(u=ERA5_UV_zfp[0], v=ERA5_UV_zfp[1]))
ERA5_zfp_cr = ERA5_UV.nbytes / ERA5_UV_zfp_enc.nbytes
from numcodecs_wasm_sz3 import Sz3
sz3 = Sz3(eb_mode="abs", eb_abs=eb_abs)
with observe.observe(sz3, observations):
ERA5_UV_sz3_enc = sz3.encode(ERA5_UV)
ERA5_UV_sz3 = sz3.decode(ERA5_UV_sz3_enc)
ERA5_sz3 = ERA5.copy(data=dict(u=ERA5_UV_sz3[0], v=ERA5_UV_sz3[1]))
ERA5_sz3_cr = ERA5_UV.nbytes / ERA5_UV_sz3_enc.nbytes
from numcodecs_wasm_sperr import Sperr
sperr = Sperr(mode="pwe", pwe=eb_abs)
with observe.observe(sperr, observations):
ERA5_UV_sperr_enc = sperr.encode(ERA5_UV)
ERA5_UV_sperr = sperr.decode(ERA5_UV_sperr_enc)
ERA5_sperr = ERA5.copy(data=dict(u=ERA5_UV_sperr[0], v=ERA5_UV_sperr[1]))
ERA5_sperr_cr = ERA5_UV.nbytes / ERA5_UV_sperr_enc.nbytes
from numcodecs_zero import ZeroCodec
zero = ZeroCodec()
with observe.observe(zero, observations):
ERA5_UV_zero_enc = zero.encode(ERA5_UV)
ERA5_UV_zero = zero.decode(ERA5_UV_zero_enc)
ERA5_zero = ERA5.copy(data=dict(u=ERA5_UV_zero[0], v=ERA5_UV_zero[1]))
Compressing u and v using the safeguarded lossy compressors¶
We configure the safeguards to bound the pointwise absolute error on the derived relative vorticity, choosing an error bound of $10^{-8}$ 1/s that is reasonable for the range of the computed baseline relative vorticity.
The relative vorticity computation is translated into a quantity of interest over a small local neighbourhood, in which the first-order, second-order-accuracy finite difference is used to approximate the spatial derivatives. For the derivative along the longitude axis, we specify that the coordinates are periodic with a period of 360 degrees to ensure the finite difference on the arbitrary grid is not confused by the coordinate jump at the longitude wrap-around from 0 to 360 degrees.
vor_eb_abs = 1e-8
from compression_safeguards import SafeguardKind
qoi_eb_stencil = SafeguardKind.qoi_eb_stencil.value(
qoi="""
V["earth_radius"] = 6371000; # [m], globally averaged
# extract u and v from the first stacked dimension
V["u"] = X[0, I[1], I[2]];
V["v"] = X[1, I[1], I[2]];
# convert latitude in degrees to radians
V["latRad"] = c["lat"] * pi / 180;
# approximate the spatial derivatives with finite differences
# (1) along the latitude axis, we need to handle the polar boundary
V["dUdTheta"] = where(
abs(c["lat"]) < 90,
# use central difference where possible
finite_difference(
V["u"] * cos(V["latRad"]),
order=1, accuracy=2, type=0, axis=1,
grid_centre=c["lat"],
),
# at the poles, use forward/backwards difference
where(
c["lat"] == -90,
finite_difference(
V["u"] * cos(V["latRad"]),
order=1, accuracy=1, type=-1, axis=1,
grid_centre=c["lat"],
),
finite_difference(
V["u"] * cos(V["latRad"]),
order=1, accuracy=1, type=+1, axis=1,
grid_centre=c["lat"],
),
),
);
# (2) along the longitude axis, we handle the 360 degree periodic boundary
V["dVdPhi"] = finite_difference(
V["v"],
order=1, accuracy=2, type=0, axis=2,
grid_centre=c["lon"], grid_period=360,
);
# compute the relative vorticity
return (V["dVdPhi"] - V["dUdTheta"]) / (
V["earth_radius"] * cos(V["latRad"])
);
""",
type="abs",
eb=vor_eb_abs,
neighbourhood=[
# [u, v]: stacked variables
dict(axis=0, before=0, after=1, boundary="valid"),
# latitude: edge boundary is good enough (since we ensure in the QoI
# that the edge-extended values are not used)
dict(axis=1, before=1, after=1, boundary="edge"),
# longitude: wrapping boundary
dict(axis=2, before=1, after=1, boundary="wrap"),
],
)
First, we ensure that the compute_relative_vorticity function and the relative vorticity quantity of interest produce equivalent results, to ensure that bounding the quantity of interest also bounds the relative vorticity we later compute.
from compression_safeguards.utils.bindings import Bindings
vor_py = compute_relative_vorticity(ERA5)
vor_qoi = qoi_eb_stencil.evaluate_qoi(
ERA5_UV,
late_bound=Bindings(
lat=ERA5.latitude.values.reshape(1, -1, 1),
lon=ERA5.longitude.values.reshape(1, 1, -1),
),
)
assert np.all(vor_py == vor_qoi.squeeze())
Next, we use the numcodecs-safeguards frontend to wrap the safeguards around several different lossy compressors.
from numcodecs_safeguards import SafeguardedCodec
ERA5_sg = dict()
ERA5_sg_cr = dict()
for codec in [
zero,
zfp,
sz3,
sperr,
]:
sg = SafeguardedCodec(
codec=codec,
safeguards=[qoi_eb_stencil],
fixed_constants=dict(
lat=ERA5.latitude.values.reshape(1, -1, 1),
lon=ERA5.longitude.values.reshape(1, 1, -1),
),
)
with observe.observe(sg, observations):
ERA5_UV_sg_enc = sg.encode(ERA5_UV)
ERA5_UV_sg = sg.decode(ERA5_UV_sg_enc)
ERA5_sg[codec.codec_id] = ERA5.copy(data=dict(u=ERA5_UV_sg[0], v=ERA5_UV_sg[1]))
ERA5_sg_cr[codec.codec_id] = ERA5_UV.nbytes / np.asarray(ERA5_UV_sg_enc).nbytes
ERA5_sg_it = dict()
ERA5_sg_it_cr = dict()
for codec in [
zero,
zfp,
sz3,
sperr,
]:
sg = SafeguardedCodec(
codec=codec,
safeguards=[qoi_eb_stencil],
fixed_constants=dict(
lat=ERA5.latitude.values.reshape(1, -1, 1),
lon=ERA5.longitude.values.reshape(1, 1, -1),
),
# use iteration to refine the corrections
compute=dict(unstable_iterative=True),
)
with observe.observe(sg, observations):
ERA5_UV_sg_it_enc = sg.encode(ERA5_UV)
ERA5_UV_sg_it = sg.decode(ERA5_UV_sg_it_enc)
ERA5_sg_it[codec.codec_id] = ERA5.copy(
data=dict(u=ERA5_UV_sg_it[0], v=ERA5_UV_sg_it[1])
)
ERA5_sg_it_cr[codec.codec_id] = (
ERA5_UV.nbytes / np.asarray(ERA5_UV_sg_it_enc).nbytes
)
ERA5_sg_lossless = dict()
ERA5_sg_lossless_cr = dict()
for codec in [
zero,
zfp,
sz3,
sperr,
]:
sg = SafeguardedCodec(
codec=codec,
safeguards=[qoi_eb_stencil],
fixed_constants=dict(
lat=ERA5.latitude.values.reshape(1, -1, 1),
lon=ERA5.longitude.values.reshape(1, 1, -1),
),
# produce lossless corrections and refine them with iteration
compute=dict(unstable_iterative=True, unstable_lossless_corrections=True),
)
with observe.observe(sg, observations):
ERA5_UV_sg_lossless_enc = sg.encode(ERA5_UV)
ERA5_UV_sg_lossless = sg.decode(ERA5_UV_sg_lossless_enc)
ERA5_sg_lossless[codec.codec_id] = ERA5.copy(
data=dict(u=ERA5_UV_sg_lossless[0], v=ERA5_UV_sg_lossless[1])
)
ERA5_sg_lossless_cr[codec.codec_id] = (
ERA5_UV.nbytes / np.asarray(ERA5_UV_sg_lossless_enc).nbytes
)
Compressing u and v using OptZConfig¶
We configure OptZConfig with a custom safety violations metric, implemented in Python, that computes the percentage of violations $V$. We then maximise the score
$$ \textrm{score} = \begin{cases} -\textrm{V} \quad &\text{if } \textrm{V} > 0 \\ \textrm{CR} \quad &\text{otherwise} \end{cases} $$
using the FRAZ search algorithm with 25 iterations, where CR is the achieved compression ratio. Since FRAZ seems to struggle with finding sufficient absolute error bounds spread across several orders of magnitude, we search for bounds in logarithmic space by wrapping each codec in an Exponential<CODEC> meta-codec.
import numcodecs
class SafetyViolationsMetric(numcodecs.abc.Codec):
codec_id = "safety-violations-metric"
def __init__(self):
self._data = None
def encode(self, buf):
# store the original data for later
self._data = np.array(buf, copy=True)
# return no metric
return np.empty(0, dtype=np.float64)
def decode(self, buf, out=None):
# compute the violations
data_VOR = compute_relative_vorticity(
ERA5.copy(data=dict(u=self._data[0], v=self._data[1]))
)
buf_VOR = compute_relative_vorticity(ERA5.copy(data=dict(u=buf[0], v=buf[1])))
violations = np.mean(~(np.abs(buf_VOR - data_VOR) <= vor_eb_abs))
self._data = None
# return the violations score metric
return numcodecs.compat.ndarray_copy(np.float64(violations), out)
numcodecs.registry.register_codec(SafetyViolationsMetric)
class ExponentialZfp(Zfp):
codec_id = "e-zfp.rs"
def __new__(cls, tolerance: float, **kwargs):
codec = super().__new__(cls, tolerance=np.exp(tolerance), **kwargs)
codec._tolerance = tolerance
return codec
def get_config(self):
return {
**super().get_config(),
"id": type(self).codec_id,
"tolerance": self._tolerance,
}
class ExponentialSz3(Sz3):
codec_id = "e-sz3.rs"
def __new__(cls, eb_abs: float, **kwargs):
codec = super().__new__(cls, eb_abs=np.exp(eb_abs), **kwargs)
codec._eb_abs = eb_abs
return codec
def get_config(self):
return {
**super().get_config(),
"id": type(self).codec_id,
"eb_abs": self._eb_abs,
}
class ExponentialSperr(Sperr):
codec_id = "e-sperr.rs"
def __new__(cls, pwe: float, **kwargs):
codec = super().__new__(cls, pwe=np.exp(pwe), **kwargs)
codec._pwe = pwe
return codec
def get_config(self):
return {**super().get_config(), "id": type(self).codec_id, "pwe": self._pwe}
numcodecs.registry.register_codec(ExponentialZfp)
numcodecs.registry.register_codec(ExponentialSz3)
numcodecs.registry.register_codec(ExponentialSperr)
from numcodecs_wasm_pressio import Pressio
ERA5_optzconfig = dict()
ERA5_optzconfig_cr = dict()
for codec, parameter, lower_bound in [
(zfp, "tolerance", 1e-4), # decent guess
(sz3, "eb_abs", 1e-8), # crash for lower bounds
(sperr, "pwe", 1e-10), # crash for lower bounds
]:
optzconfig = Pressio(
compressor_id="opt",
compressor_config={
"opt:output": ["composite:score"],
"opt:inputs": [f"numcodecs.rs:{parameter}"],
"opt:lower_bound": np.log(lower_bound),
"opt:upper_bound": np.log(eb_abs),
"opt:max_iterations": 25,
"opt:objective_mode_name": "max",
},
early_config={
"opt:compressor": "pressio",
"pressio:compressor": "numcodecs.rs",
**{
f"numcodecs.rs:{k}": f"e-{v}" if k == "id" else v
for k, v in codec.get_config().items()
},
"opt:search": "fraz",
"pressio:metric": "composite",
"composite:plugins": ["size", "numcodecs.rs-metric"],
"composite:scripts": [
"""
violations = metrics["numcodecs.rs-metric:decompression"]
if violations > 0 then
return "score", -violations
else
return "score", metrics["size:compression_ratio"]
end
"""
],
"numcodecs.rs-metric:id": "safety-violations-metric",
},
)
with observe.observe(optzconfig, observations):
ERA5_UV_optzconfig_enc = optzconfig.encode(ERA5_UV)
ERA5_UV_optzconfig = optzconfig.decode(ERA5_UV_optzconfig_enc)
ERA5_optzconfig[codec.codec_id] = ERA5.copy(
data=dict(u=ERA5_UV_optzconfig[0], v=ERA5_UV_optzconfig[1])
)
ERA5_optzconfig_cr[codec.codec_id] = ERA5_UV.nbytes / ERA5_UV_optzconfig_enc.nbytes
rank={0,1,} iter={0} input={-6.90776,} output={-0.0032584,} objective={-0.0032584}
rank={0,1,} iter={1} input={-8.13608,} output={-0.00089382,} objective={-0.00089382}
rank={0,1,} iter={2} input={-5.69821,} output={-0.00448066,} objective={-0.00448066}
rank={0,1,} iter={3} input={-8.23592,} output={-0.00089382,} objective={-0.00089382}
rank={0,1,} iter={4} input={-4.91958,} output={-0.00769861,} objective={-0.00769861}
rank={0,1,} iter={5} input={-8.89969,} output={1.11299,} objective={1.11299}
rank={0,1,} iter={6} input={-9.21034,} output={1.04082,} objective={1.04082}
rank={0,1,} iter={7} input={-9.03247,} output={1.04082,} objective={1.04082}
rank={0,1,} iter={8} input={-8.53698,} output={1.11299,} objective={1.11299}
rank={0,1,} iter={9} input={-7.52202,} output={-0.00193501,} objective={-0.00193501}
rank={0,1,} iter={10} input={-8.71833,} output={1.11299,} objective={1.11299}
rank={0,1,} iter={11} input={-6.30335,} output={-0.0032584,} objective={-0.0032584}
rank={0,1,} iter={12} input={-8.80901,} output={1.11299,} objective={1.11299}
rank={0,1,} iter={13} input={-8.62649,} output={1.11299,} objective={1.11299}
rank={0,1,} iter={14} input={-5.30798,} output={-0.00769861,} objective={-0.00769861}
rank={0,1,} iter={15} input={-9.12072,} output={1.04082,} objective={1.04082}
rank={0,1,} iter={16} input={-8.95677,} output={1.11299,} objective={1.11299}
rank={0,1,} iter={17} input={-8.67231,} output={1.11299,} objective={1.11299}
rank={0,1,} iter={18} input={-8.85447,} output={1.11299,} objective={1.11299}
rank={0,1,} iter={19} input={-8.76379,} output={1.11299,} objective={1.11299}
rank={0,1,} iter={20} input={-8.58171,} output={1.11299,} objective={1.11299}
rank={0,1,} iter={21} input={-8.92796,} output={1.11299,} objective={1.11299}
rank={0,1,} iter={22} input={-8.9852,} output={1.11299,} objective={1.11299}
rank={0,1,} iter={23} input={-9.16489,} output={1.04082,} objective={1.04082}
rank={0,1,} iter={24} input={-9.0765,} output={1.04082,} objective={1.04082}
final_iter={25} inputs={-8.89969,} output={1.11299,}
rank={0,1,} iter={0} input={-11.5129,} output={-0.00133014,} objective={-0.00133014}
rank={0,1,} iter={1} input={-15.1979,} output={-0.000223455,} objective={-0.000223455}
rank={0,1,} iter={2} input={-7.88428,} output={-0.00632224,} objective={-0.00632224}
rank={0,1,} iter={3} input={-15.4974,} output={-2.88951e-06,} objective={-2.88951e-06}
rank={0,1,} iter={4} input={-5.54841,} output={-0.0467676,} objective={-0.0467676}
rank={0,1,} iter={5} input={-17.4887,} output={1.66174,} objective={1.66174}
rank={0,1,} iter={6} input={-18.4207,} output={1.66174,} objective={1.66174}
rank={0,1,} iter={7} input={-17.9522,} output={1.66174,} objective={1.66174}
rank={0,1,} iter={8} input={-18.1858,} output={1.66174,} objective={1.66174}
rank={0,1,} iter={9} input={-17.7204,} output={1.66174,} objective={1.66174}
rank={0,1,} iter={10} input={-18.3037,} output={1.66174,} objective={1.66174}
rank={0,1,} iter={11} input={-18.0664,} output={1.66174,} objective={1.66174}
rank={0,1,} iter={12} input={-17.8375,} output={1.66174,} objective={1.66174}
rank={0,1,} iter={13} input={-17.6033,} output={1.66174,} objective={1.66174}
rank={0,1,} iter={14} input={-18.1249,} output={1.66174,} objective={1.66174}
rank={0,1,} iter={15} input={-18.2447,} output={1.66174,} objective={1.66174}
rank={0,1,} iter={16} input={-17.6614,} output={1.66174,} objective={1.66174}
rank={0,1,} iter={17} input={-17.7792,} output={1.66174,} objective={1.66174}
rank={0,1,} iter={18} input={-18.3619,} output={1.66174,} objective={1.66174}
rank={0,1,} iter={19} input={-17.8954,} output={1.66174,} objective={1.66174}
rank={0,1,} iter={20} input={-18.0095,} output={1.66174,} objective={1.66174}
rank={0,1,} iter={21} input={-17.5481,} output={1.66174,} objective={1.66174}
rank={0,1,} iter={22} input={-18.1555,} output={1.66174,} objective={1.66174}
rank={0,1,} iter={23} input={-18.2742,} output={1.66174,} objective={1.66174}
rank={0,1,} iter={24} input={-17.5186,} output={1.66174,} objective={1.66174}
final_iter={25} inputs={-17.4887,} output={1.66174,}
rank={0,1,} iter={0} input={-13.8155,} output={-0.0027614,} objective={-0.0027614}
rank={0,1,} iter={1} input={-18.7288,} output={-0.00173948,} objective={-0.00173948}
rank={0,1,} iter={2} input={-8.97731,} output={-0.00290299,} objective={-0.00290299}
rank={0,1,} iter={3} input={-19.1282,} output={-0.00130509,} objective={-0.00130509}
rank={0,1,} iter={4} input={-5.86282,} output={-0.0199019,} objective={-0.0199019}
rank={0,1,} iter={5} input={-21.7833,} output={1.14644,} objective={1.14644}
rank={0,1,} iter={6} input={-23.0259,} output={1.07729,} objective={1.07729}
rank={0,1,} iter={7} input={-22.325,} output={1.11528,} objective={1.11528}
rank={0,1,} iter={8} input={-20.3324,} output={-0.000217676,} objective={-0.000217676}
rank={0,1,} iter={9} input={-16.2726,} output={-0.00267857,} objective={-0.00267857}
rank={0,1,} iter={10} input={-21.0578,} output={-2.02265e-05,} objective={-2.02265e-05}
rank={0,1,} iter={11} input={-11.3979,} output={-0.00277296,} objective={-0.00277296}
rank={0,1,} iter={12} input={-22.0319,} output={1.13189,} objective={1.13189}
rank={0,1,} iter={13} input={-7.42181,} output={-0.00698008,} objective={-0.00698008}
rank={0,1,} iter={14} input={-21.6019,} output={1.15738,} objective={1.15738}
rank={0,1,} iter={15} input={-4.61353,} output={-0.0611217,} objective={-0.0611217}
rank={0,1,} iter={16} input={-21.2392,} output={-5.77901e-06,} objective={-5.77901e-06}
rank={0,1,} iter={17} input={-15.0428,} output={-0.00274118,} objective={-0.00274118}
rank={0,1,} iter={18} input={-21.6875,} output={1.15216,} objective={1.15216}
rank={0,1,} iter={19} input={-17.5006,} output={-0.00243296,} objective={-0.00243296}
rank={0,1,} iter={20} input={-10.1873,} output={-0.00277393,} objective={-0.00277393}
rank={0,1,} iter={21} input={-12.6069,} output={-0.00276911,} objective={-0.00276911}
rank={0,1,} iter={22} input={-6.64237,} output={-0.0127947,} objective={-0.0127947}
rank={0,1,} iter={23} input={-8.20425,} output={-0.00400196,} objective={-0.00400196}
rank={0,1,} iter={24} input={-22.6686,} output={1.09629,} objective={1.09629}
final_iter={25} inputs={-21.6019,} output={1.15738,}
Visual comparison of the error distributions for the derived relative vorticity¶
fig = earthkit.plots.Figure(
size=(10, 23),
rows=6,
columns=2,
)
plot_relative_vorticity(
ERA5, 1.0, fig.add_map(0, 0), "Original", span=5e-6, vor_eb_abs=vor_eb_abs
)
plot_relative_vorticity(
ERA5_zfp,
ERA5_zfp_cr,
fig.add_map(1, 0),
r"ZFP($\epsilon_{{abs}}$)",
span=vor_eb_abs,
vor_eb_abs=vor_eb_abs,
error=True,
)
plot_relative_vorticity(
ERA5_sz3,
ERA5_sz3_cr,
fig.add_map(2, 0),
r"SZ3($\epsilon_{{abs}}$)",
span=vor_eb_abs,
vor_eb_abs=vor_eb_abs,
error=True,
)
plot_relative_vorticity(
ERA5_sperr,
ERA5_sperr_cr,
fig.add_map(3, 0),
r"SPERR($\epsilon_{{abs}}$)",
span=vor_eb_abs,
vor_eb_abs=vor_eb_abs,
error=True,
)
plot_relative_vorticity(
ERA5_sg["zero"],
ERA5_sg_cr["zero"],
fig.add_map(0, 1),
r"Safeguarded(0, $\epsilon_{{QoI,abs}}$)",
span=vor_eb_abs,
vor_eb_abs=vor_eb_abs,
error=True,
corr=ERA5_zero,
my_ERA5_it=ERA5_sg_it["zero"],
cr_it=ERA5_sg_it_cr["zero"],
)
plot_relative_vorticity(
ERA5_sg["zfp.rs"],
ERA5_sg_cr["zfp.rs"],
fig.add_map(1, 1),
r"Safeguarded(ZFP, $\epsilon_{{QoI,abs}}$)",
span=vor_eb_abs,
vor_eb_abs=vor_eb_abs,
error=True,
corr=ERA5_zfp,
my_ERA5_it=ERA5_sg_it["zfp.rs"],
cr_it=ERA5_sg_it_cr["zfp.rs"],
)
plot_relative_vorticity(
ERA5_sg["sz3.rs"],
ERA5_sg_cr["sz3.rs"],
fig.add_map(2, 1),
r"Safeguarded(SZ3, $\epsilon_{{QoI,abs}}$)",
span=vor_eb_abs,
vor_eb_abs=vor_eb_abs,
error=True,
corr=ERA5_sz3,
my_ERA5_it=ERA5_sg_it["sz3.rs"],
cr_it=ERA5_sg_it_cr["sz3.rs"],
)
plot_relative_vorticity(
ERA5_sg["sperr.rs"],
ERA5_sg_cr["sperr.rs"],
fig.add_map(3, 1),
r"Safeguarded(SPERR, $\epsilon_{{QoI,abs}}$)",
span=vor_eb_abs,
vor_eb_abs=vor_eb_abs,
error=True,
corr=ERA5_sperr,
my_ERA5_it=ERA5_sg_it["sperr.rs"],
cr_it=ERA5_sg_it_cr["sperr.rs"],
)
plot_relative_vorticity(
ERA5_optzconfig["zfp.rs"],
ERA5_optzconfig_cr["zfp.rs"],
fig.add_map(4, 0),
r"OptZConfig(ZFP, $\epsilon_{{QoI,abs}}$)",
span=vor_eb_abs,
vor_eb_abs=vor_eb_abs,
error=True,
inset=False,
)
plot_relative_vorticity(
ERA5_optzconfig["sz3.rs"],
ERA5_optzconfig_cr["sz3.rs"],
fig.add_map(4, 1),
r"OptZConfig(SZ3, $\epsilon_{{QoI,abs}}$)",
span=vor_eb_abs,
vor_eb_abs=vor_eb_abs,
error=True,
inset=False,
)
plot_relative_vorticity(
ERA5_optzconfig["sperr.rs"],
ERA5_optzconfig_cr["sperr.rs"],
fig.add_map(5, 0),
r"OptZConfig(SPERR, $\epsilon_{{QoI,abs}}$)",
span=vor_eb_abs,
vor_eb_abs=vor_eb_abs,
error=True,
inset=False,
)
fig.save(Path("plots") / "vorticity.pdf")
vor_sg_table = pd.concat(
[
table_relative_vorticity(
ERA5_sg_lossless["zero"],
ERA5_sg_lossless_cr["zero"],
["0", r"$\epsilon_{QoI,abs}$", "lossless"],
vor_eb_abs,
ERA5_zero,
),
table_relative_vorticity(
ERA5_sg["zero"],
ERA5_sg_cr["zero"],
["0", r"$\epsilon_{QoI,abs}$", "one-shot"],
vor_eb_abs,
ERA5_zero,
),
table_relative_vorticity(
ERA5_sg_it["zero"],
ERA5_sg_it_cr["zero"],
["0", r"$\epsilon_{QoI,abs}$", "iterative"],
vor_eb_abs,
ERA5_zero,
),
table_relative_vorticity(
ERA5_zfp,
ERA5_zfp_cr,
[r"ZFP($\epsilon_{abs}$)", "-", ""],
vor_eb_abs,
None,
),
table_relative_vorticity(
ERA5_sg_lossless["zfp.rs"],
ERA5_sg_lossless_cr["zfp.rs"],
[r"ZFP($\epsilon_{abs}$)", r"$\epsilon_{QoI,abs}$", "lossless"],
vor_eb_abs,
ERA5_zfp,
),
table_relative_vorticity(
ERA5_sg["zfp.rs"],
ERA5_sg_cr["zfp.rs"],
[r"ZFP($\epsilon_{abs}$)", r"$\epsilon_{QoI,abs}$", "one-shot"],
vor_eb_abs,
ERA5_zfp,
),
table_relative_vorticity(
ERA5_sg_it["zfp.rs"],
ERA5_sg_it_cr["zfp.rs"],
[r"ZFP($\epsilon_{abs}$)", r"$\epsilon_{QoI,abs}$", "iterative"],
vor_eb_abs,
ERA5_zfp,
),
table_relative_vorticity(
ERA5_optzconfig["zfp.rs"],
ERA5_optzconfig_cr["zfp.rs"],
[r"OptZConfig(ZFP)", r"$\epsilon_{QoI,abs}$", ""],
vor_eb_abs,
None,
),
table_relative_vorticity(
ERA5_sz3,
ERA5_sz3_cr,
[r"SZ3($\epsilon_{abs}$)", "-", ""],
vor_eb_abs,
None,
),
table_relative_vorticity(
ERA5_sg_lossless["sz3.rs"],
ERA5_sg_lossless_cr["sz3.rs"],
[r"SZ3($\epsilon_{abs}$)", r"$\epsilon_{QoI,abs}$", "lossless"],
vor_eb_abs,
ERA5_sz3,
),
table_relative_vorticity(
ERA5_sg["sz3.rs"],
ERA5_sg_cr["sz3.rs"],
[r"SZ3($\epsilon_{abs}$)", r"$\epsilon_{QoI,abs}$", "one-shot"],
vor_eb_abs,
ERA5_sz3,
),
table_relative_vorticity(
ERA5_sg_it["sz3.rs"],
ERA5_sg_it_cr["sz3.rs"],
[r"SZ3($\epsilon_{abs}$)", r"$\epsilon_{QoI,abs}$", "iterative"],
vor_eb_abs,
ERA5_sz3,
),
table_relative_vorticity(
ERA5_optzconfig["sz3.rs"],
ERA5_optzconfig_cr["sz3.rs"],
[r"OptZConfig(SZ3)", r"$\epsilon_{QoI,abs}$", ""],
vor_eb_abs,
None,
),
table_relative_vorticity(
ERA5_sperr,
ERA5_sperr_cr,
[r"SPERR($\epsilon_{abs}$)", "-", ""],
vor_eb_abs,
None,
),
table_relative_vorticity(
ERA5_sg_lossless["sperr.rs"],
ERA5_sg_lossless_cr["sperr.rs"],
[r"SPERR($\epsilon_{abs}$)", r"$\epsilon_{QoI,abs}$", "lossless"],
vor_eb_abs,
ERA5_sperr,
),
table_relative_vorticity(
ERA5_sg["sperr.rs"],
ERA5_sg_cr["sperr.rs"],
[r"SPERR($\epsilon_{abs}$)", r"$\epsilon_{QoI,abs}$", "one-shot"],
vor_eb_abs,
ERA5_sperr,
),
table_relative_vorticity(
ERA5_sg_it["sperr.rs"],
ERA5_sg_it_cr["sperr.rs"],
[r"SPERR($\epsilon_{abs}$)", r"$\epsilon_{QoI,abs}$", "iterative"],
vor_eb_abs,
ERA5_sperr,
),
table_relative_vorticity(
ERA5_optzconfig["sperr.rs"],
ERA5_optzconfig_cr["sperr.rs"],
[r"OptZConfig(SPERR)", r"$\epsilon_{QoI,abs}$", ""],
vor_eb_abs,
None,
),
table_relative_vorticity(
ERA5_zstd,
ERA5_zstd_cr,
["ZSTD(22)", "-", ""],
vor_eb_abs,
None,
),
]
).set_index(["Compressor", "Safeguarded", "Corrections"])
Path("tables").joinpath("vorticity.tex").write_text(
vor_sg_table.to_latex(escape=False)
.replace("%", r"\%")
.replace("\\cline{1-10} \\cline{2-10}\n\\bottomrule", "\\bottomrule")
)
vor_sg_table
| $L_{\infty}(\hat{u})$ | $L_{\infty}(\hat{v})$ | $L_{\infty}(\hat{\zeta})$ | $L_{2}(\hat{\zeta})$ | V | C | CR | |||
|---|---|---|---|---|---|---|---|---|---|
| Compressor | Safeguarded | Corrections | |||||||
| 0 | $\epsilon_{QoI,abs}$ | lossless | 0.0 | 0.0396 | 9.16e-09 | 2.2e-11 | 0 | 100.0% | $\times$ 2.66 |
| one-shot | 0.00795 | 0.00728 | 9.59e-09 | 2.71e-09 | 0 | 99.9% | $\times$ 3.97 | ||
| iterative | 0.00795 | 0.0396 | 9.59e-09 | 2.71e-09 | 0 | 99.9% | $\times$ 3.97 | ||
| ZFP($\epsilon_{abs}$) | - | 0.00197 | 0.00232 | 0.0127 | 0.000142 | 1.1% | $\times$ 1.9 | ||
| $\epsilon_{QoI,abs}$ | lossless | 0.00197 | 0.00232 | 1e-08 | 9.14e-10 | 0 | 1.7% | $\times$ 1.88 | |
| one-shot | 0.00197 | 0.00232 | 7.45e-09 | 6.91e-10 | 0 | 1.7% | $\times$ 1.89 | ||
| iterative | 0.00197 | 0.00232 | 1e-08 | 9.42e-10 | 0 | 1.0% | $\times$ 1.89 | ||
| OptZConfig(ZFP) | $\epsilon_{QoI,abs}$ | 0.0 | 0.0 | 0.0 | 0.0 | 0 | $\times$ 1.11 | ||
| SZ3($\epsilon_{abs}$) | - | 0.01 | 0.01 | 0.0815 | 0.00016 | 12.8% | $\times$ 6.43 | ||
| $\epsilon_{QoI,abs}$ | lossless | 0.01 | 0.01 | 1e-08 | 3.63e-09 | 0 | 22.9% | $\times$ 3.34 | |
| one-shot | 0.00797 | 0.00796 | 9.73e-09 | 2.66e-09 | 0 | 30.3% | $\times$ 4.5 | ||
| iterative | 0.01 | 0.01 | 1e-08 | 3.85e-09 | 0 | 10.9% | $\times$ 5.38 | ||
| OptZConfig(SZ3) | $\epsilon_{QoI,abs}$ | 0.0 | 0.0 | 0.0 | 0.0 | 0 | $\times$ 1.66 | ||
| SPERR($\epsilon_{abs}$) | - | 0.01 | 0.01 | 0.112 | 0.00132 | 6.2% | $\times$ 9.33 | ||
| $\epsilon_{QoI,abs}$ | lossless | 0.01 | 0.01 | 1e-08 | 3.07e-09 | 0 | 11.2% | $\times$ 5.61 | |
| one-shot | 0.00796 | 0.00796 | 9.57e-09 | 2.33e-09 | 0 | 14.8% | $\times$ 6.72 | ||
| iterative | 0.01 | 0.01 | 1e-08 | 3.16e-09 | 0 | 4.9% | $\times$ 8.06 | ||
| OptZConfig(SPERR) | $\epsilon_{QoI,abs}$ | 4.66e-10 | 4.66e-10 | 7.45e-09 | 3.96e-11 | 0 | $\times$ 1.16 | ||
| ZSTD(22) | - | 0.0 | 0.0 | 0.0 | 0.0 | 0 | $\times$ 2.12 |
fig = earthkit.plots.Figure(
size=(10, 5),
rows=1,
columns=2,
)
chart = fig.add_map(0, 0)
# Original
da = ERA5_VOR.sel(longitude=slice(180, 300))
# compute the default style that earthkit.maps would apply
source_original = earthkit.plots.sources.XarraySource(da)
style_original = copy.deepcopy(
earthkit.plots.styles.auto.guess_style(
source_original,
units=source_original.units,
)
)
style_original._levels = earthkit.plots.styles.levels.Levels(
np.linspace(-5e-6, 5e-6, 22)
)
style_original._legend_kwargs["ticks"] = np.linspace(-5e-6, 5e-6, 5)
style_original._colors = "BrBG"
style_original._legend_kwargs["extend"] = "both"
chart.pcolormesh(da, style=style_original, zorder=-11)
t = chart.ax.text(
1 / 6,
0.9,
"Original",
ha="center",
va="top",
transform=chart.ax.transAxes,
)
t.set_bbox(dict(facecolor="white", alpha=0.75, edgecolor="black"))
chart.legend()
# SPERR
my_ERA5_VOR = compute_relative_vorticity(ERA5_sperr)
with xr.set_options(keep_attrs=True):
da = (my_ERA5_VOR - ERA5_VOR).compute()
da.attrs.update(long_name=f"Absolute error over {da.long_name.lower()}")
# compute the default style that earthkit.maps would apply
source_error = earthkit.plots.sources.XarraySource(da)
style_error = copy.deepcopy(
earthkit.plots.styles.auto.guess_style(
source_error,
units=source_error.units,
)
)
style_error._levels = earthkit.plots.styles.levels.Levels(
np.linspace(-vor_eb_abs, vor_eb_abs, 22)
)
style_error._legend_kwargs["ticks"] = np.linspace(-vor_eb_abs, vor_eb_abs, 5)
style_error._colors = "coolwarm"
style_error._legend_kwargs["extend"] = "both"
chart.pcolormesh(da.sel(longitude=slice(300, 360)), style=style_error, zorder=-11)
chart.pcolormesh(da.sel(longitude=slice(0, 60)), style=style_error, zorder=-11)
t = chart.ax.text(
0.5,
0.9,
"SPERR",
ha="center",
va="top",
transform=chart.ax.transAxes,
)
t.set_bbox(dict(facecolor="white", alpha=0.75, edgecolor="black"))
t = chart.ax.text(
0.5,
0.5,
rf"$\times$ {np.round(ERA5_sperr_cr, 2)}",
ha="center",
va="center",
transform=chart.ax.transAxes,
color="mistyrose",
fontsize=20,
)
t.set_path_effects([PathEffects.withStroke(linewidth=3, foreground="black")])
err_v_sperr = np.mean(~(np.abs(my_ERA5_VOR - ERA5_VOR) <= vor_eb_abs))
err_v_sperr = (
0
if err_v_sperr == 0
else np.format_float_positional(100 * err_v_sperr, precision=1, min_digits=1) + "%"
)
if err_v_sperr == "0.0%":
err_v_sperr = "<0.05%"
t = chart.ax.text(
0.5,
0.1,
f"V={err_v_sperr}",
ha="center",
va="bottom",
transform=chart.ax.transAxes,
)
t.set_bbox(dict(facecolor="white", alpha=0.75, edgecolor="black"))
SPERR_ERA5_VOR = my_ERA5_VOR
# OptZConfig(SPERR)
my_ERA5_VOR = compute_relative_vorticity(ERA5_optzconfig["sperr.rs"])
with xr.set_options(keep_attrs=True):
da = (my_ERA5_VOR - ERA5_VOR).compute()
da.attrs.update(long_name=f"Absolute error over {da.long_name.lower()}")
da = da.sel(longitude=slice(60, 180))
chart.pcolormesh(da, style=style_error, zorder=-11)
t = chart.ax.text(
5 / 6,
0.9,
"OptZConfig(SPERR)",
ha="center",
va="top",
transform=chart.ax.transAxes,
)
t.set_bbox(dict(facecolor="white", alpha=0.75, edgecolor="black"))
t = chart.ax.text(
5 / 6,
0.5,
rf"$\times$ {np.round(ERA5_optzconfig_cr['sperr.rs'], 2)}",
ha="center",
va="center",
transform=chart.ax.transAxes,
color="lightgreen",
fontsize=20,
)
t.set_path_effects([PathEffects.withStroke(linewidth=3, foreground="black")])
err_v_optzconfig_sperr = np.mean(~(np.abs(my_ERA5_VOR - ERA5_VOR) <= vor_eb_abs))
err_v_optzconfig_sperr = (
0
if err_v_optzconfig_sperr == 0
else np.format_float_positional(
100 * err_v_optzconfig_sperr, precision=1, min_digits=1
)
+ "%"
)
if err_v_optzconfig_sperr == "0.0%":
err_v_optzconfig_sperr = "<0.05%"
t = chart.ax.text(
5 / 6,
0.1,
f"V={err_v_optzconfig_sperr}",
ha="center",
va="bottom",
transform=chart.ax.transAxes,
)
t.set_bbox(dict(facecolor="white", alpha=0.75, edgecolor="black"))
chart.ax.set_rasterization_zorder(-10)
chart.ax.axvline(-60, c="white", ls=(2, (4, 4)), lw=2)
chart.ax.axvline(-60, c="black", ls=(6, (4, 4)), lw=2)
chart.ax.axvline(+60, c="white", ls=(2, (4, 4)), lw=2)
chart.ax.axvline(+60, c="black", ls=(6, (4, 4)), lw=2)
chart.title("Without safeguards")
for m in earthkit.plots.schemas.schema.quickmap_subplot_workflow:
if m != "title":
getattr(chart, m)()
for m in earthkit.plots.schemas.schema.quickmap_figure_workflow:
if m != "legend":
getattr(chart, m)()
chart = fig.add_map(0, 1)
# Corrections and Errors
is_err = ~(np.abs(SPERR_ERA5_VOR - ERA5_VOR) <= vor_eb_abs)
is_corr = (ERA5_sg["sperr.rs"]["u"] != ERA5_sperr["u"]) | (
ERA5_sg["sperr.rs"]["v"] != ERA5_sperr["v"]
)
is_corr_it = (ERA5_sg_it["sperr.rs"]["u"] != ERA5_sperr["u"]) | (
ERA5_sg_it["sperr.rs"]["v"] != ERA5_sperr["v"]
)
chart.pcolormesh(
is_corr.sel(longitude=slice(300, 340)),
no_style=True,
cmap=mpl.colors.ListedColormap(["white", "green"]),
zorder=-11,
legend_style=None,
)
chart.pcolormesh(
is_err.sel(longitude=slice(340, 360)),
no_style=True,
cmap=mpl.colors.ListedColormap(["white", "red"]),
zorder=-11,
legend_style=None,
)
chart.pcolormesh(
is_err.sel(longitude=slice(0, 20)),
no_style=True,
cmap=mpl.colors.ListedColormap(["white", "red"]),
zorder=-11,
legend_style=None,
)
chart.pcolormesh(
is_corr_it.sel(longitude=slice(20, 60)),
no_style=True,
cmap=mpl.colors.ListedColormap(["white", "limegreen"]),
zorder=-11,
legend_style=None,
)
t = chart.ax.text(
0.5,
0.9,
"SPERR",
ha="center",
va="top",
transform=chart.ax.transAxes,
)
t.set_bbox(dict(facecolor="white", alpha=0.75, edgecolor="black"))
t = chart.ax.text(
0.5,
0.5,
f"V={err_v_sperr}",
ha="center",
va="center",
transform=chart.ax.transAxes,
rotation=90,
)
t.set_bbox(dict(facecolor="white", alpha=0.75, edgecolor="black"))
corr = compute_corrections_percentage(ERA5_sg["sperr.rs"], ERA5_sperr)
corr = (
0
if corr == 0
else np.format_float_positional(100 * corr, precision=1, min_digits=1) + "%"
)
if corr == "0.0%":
corr = "<0.05%"
t = chart.ax.text(
7 / 18,
0.9,
"Sg",
ha="center",
va="top",
transform=chart.ax.transAxes,
)
t.set_bbox(dict(facecolor="white", alpha=0.75, edgecolor="black"))
t = chart.ax.text(
7 / 18,
0.5,
f"C={corr}",
ha="center",
va="center",
transform=chart.ax.transAxes,
rotation=90,
)
t.set_bbox(dict(facecolor="white", alpha=0.75, edgecolor="black"))
corr_it = compute_corrections_percentage(ERA5_sg_it["sperr.rs"], ERA5_sperr)
corr_it = (
0
if corr_it == 0
else np.format_float_positional(100 * corr_it, precision=1, min_digits=1) + "%"
)
if corr_it == "0.0%":
corr_it = "<0.05%"
t = chart.ax.text(
11 / 18,
0.9,
"Sg[it]",
ha="center",
va="top",
transform=chart.ax.transAxes,
)
t.set_bbox(dict(facecolor="white", alpha=0.75, edgecolor="black"))
t = chart.ax.text(
11 / 18,
0.5,
f"C={corr_it}",
ha="center",
va="center",
transform=chart.ax.transAxes,
rotation=90,
)
t.set_bbox(dict(facecolor="white", alpha=0.75, edgecolor="black"))
# Safeguarded(SPERR)
my_ERA5_VOR = compute_relative_vorticity(ERA5_sg["sperr.rs"])
with xr.set_options(keep_attrs=True):
da = (my_ERA5_VOR - ERA5_VOR).compute()
da.attrs.update(long_name=f"Absolute error over {da.long_name.lower()}")
da = da.sel(longitude=slice(180, 300))
chart.pcolormesh(da, style=style_error, zorder=-11)
t = chart.ax.text(
1 / 6,
0.9,
"Sg(SPERR)",
ha="center",
va="top",
transform=chart.ax.transAxes,
)
t.set_bbox(dict(facecolor="white", alpha=0.75, edgecolor="black"))
t = chart.ax.text(
1 / 6,
0.5,
rf"$\times$ {np.round(ERA5_sg_cr['sperr.rs'], 2)}",
ha="center",
va="center",
transform=chart.ax.transAxes,
color="lightgreen",
fontsize=20,
)
t.set_path_effects([PathEffects.withStroke(linewidth=3, foreground="black")])
err_v = np.mean(~(np.abs(my_ERA5_VOR - ERA5_VOR) <= vor_eb_abs))
err_v = (
0
if err_v == 0
else np.format_float_positional(100 * err_v, precision=1, min_digits=1) + "%"
)
if err_v == "0.0%":
err_v = "<0.05%"
t = chart.ax.text(
1 / 6,
0.1,
f"V={err_v}",
ha="center",
va="bottom",
transform=chart.ax.transAxes,
)
t.set_bbox(dict(facecolor="white", alpha=0.75, edgecolor="black"))
chart.legend(extend="neither")
# Safeguarded[it](SPERR)
my_ERA5_VOR = compute_relative_vorticity(ERA5_sg_it["sperr.rs"])
with xr.set_options(keep_attrs=True):
da = (my_ERA5_VOR - ERA5_VOR).compute()
da.attrs.update(long_name=f"Absolute error over {da.long_name.lower()}")
da = da.sel(longitude=slice(60, 180))
chart.pcolormesh(da, style=style_error, zorder=-11)
t = chart.ax.text(
5 / 6,
0.9,
"Sg[it](SPERR)",
ha="center",
va="top",
transform=chart.ax.transAxes,
)
t.set_bbox(dict(facecolor="white", alpha=0.75, edgecolor="black"))
t = chart.ax.text(
5 / 6,
0.5,
rf"$\times$ {np.round(ERA5_sg_it_cr['sperr.rs'], 2)}",
ha="center",
va="center",
transform=chart.ax.transAxes,
color="lightgreen",
fontsize=20,
)
t.set_path_effects([PathEffects.withStroke(linewidth=3, foreground="black")])
err_v = np.mean(~(np.abs(my_ERA5_VOR - ERA5_VOR) <= vor_eb_abs))
err_v = (
0
if err_v == 0
else np.format_float_positional(100 * err_v, precision=1, min_digits=1) + "%"
)
if err_v == "0.0%":
err_v = "<0.05%"
t = chart.ax.text(
5 / 6,
0.1,
f"V={err_v}",
ha="center",
va="bottom",
transform=chart.ax.transAxes,
)
t.set_bbox(dict(facecolor="white", alpha=0.75, edgecolor="black"))
chart.ax.set_rasterization_zorder(-10)
chart.ax.axvline(-20, c="white", ls=(2, (4, 4)), lw=2)
chart.ax.axvline(-20, c="black", ls=(6, (4, 4)), lw=2)
chart.ax.axvline(+20, c="white", ls=(2, (4, 4)), lw=2)
chart.ax.axvline(+20, c="black", ls=(6, (4, 4)), lw=2)
chart.ax.axvline(-60, c="black", lw=1)
chart.ax.axvline(+60, c="black", lw=1)
chart.title("Safeguarded: preserve relative vorticity")
for m in earthkit.plots.schemas.schema.quickmap_subplot_workflow:
if m != "title":
getattr(chart, m)()
for m in earthkit.plots.schemas.schema.quickmap_figure_workflow:
if m != "legend":
getattr(chart, m)()
fig.save(Path("plots") / "vorticity-summary.pdf")
Warning: Style not set. Warning: Style not set. Warning: Style not set. Warning: Style not set.
fig = earthkit.plots.Figure(
size=(5, 5),
rows=1,
columns=1,
)
chart = fig.add_map(0, 0)
# Original
da = ERA5_VOR.sel(longitude=slice(180, 300))
# compute the default style that earthkit.maps would apply
source_original = earthkit.plots.sources.XarraySource(da)
style_original = copy.deepcopy(
earthkit.plots.styles.auto.guess_style(
source_original,
units=source_original.units,
)
)
style_original._levels = earthkit.plots.styles.levels.Levels(
np.linspace(-5e-6, 5e-6, 22)
)
style_original._legend_kwargs["ticks"] = np.linspace(-5e-6, 5e-6, 5)
style_original._colors = "BrBG"
style_original._legend_kwargs["extend"] = "both"
chart.pcolormesh(da, style=style_original, zorder=-11)
# # SPERR
my_ERA5_VOR = compute_relative_vorticity(ERA5_sperr)
with xr.set_options(keep_attrs=True):
da = (my_ERA5_VOR - ERA5_VOR).compute()
da.attrs.update(long_name=f"Absolute error over {da.long_name.lower()}")
# compute the default style that earthkit.maps would apply
source_error = earthkit.plots.sources.XarraySource(da)
style_error = copy.deepcopy(
earthkit.plots.styles.auto.guess_style(
source_error,
units=source_error.units,
)
)
style_error._levels = earthkit.plots.styles.levels.Levels(
np.linspace(-vor_eb_abs, vor_eb_abs, 22)
)
style_error._legend_kwargs["ticks"] = np.linspace(-vor_eb_abs, vor_eb_abs, 5)
style_error._colors = "coolwarm"
style_error._legend_kwargs["extend"] = "both"
chart.pcolormesh(da.sel(longitude=slice(300, 360)), style=style_error, zorder=-11)
chart.pcolormesh(da.sel(longitude=slice(0, 60)), style=style_error, zorder=-11)
t = chart.ax.text(
0.5,
0.5,
rf"$\times$ {np.round(ERA5_sperr_cr, 2)}",
ha="center",
va="center",
transform=chart.ax.transAxes,
color="mistyrose",
fontsize=20,
)
t.set_path_effects([PathEffects.withStroke(linewidth=3, foreground="black")])
# Safeguarded[it](SPERR)
my_ERA5_VOR = compute_relative_vorticity(ERA5_sg_it["sperr.rs"])
with xr.set_options(keep_attrs=True):
da = (my_ERA5_VOR - ERA5_VOR).compute()
da.attrs.update(long_name=f"Absolute error over {da.long_name.lower()}")
da = da.sel(longitude=slice(60, 180))
chart.pcolormesh(da, style=style_error, zorder=-11)
t = chart.ax.text(
5 / 6,
0.5,
rf"$\times$ {np.round(ERA5_sg_it_cr['sperr.rs'], 2)}",
ha="center",
va="center",
transform=chart.ax.transAxes,
color="lightgreen",
fontsize=20,
)
t.set_path_effects([PathEffects.withStroke(linewidth=3, foreground="black")])
chart.ax.set_rasterization_zorder(-10)
chart.ax.axvline(-60, c="white", ls=(2, (4, 4)), lw=2)
chart.ax.axvline(-60, c="black", ls=(6, (4, 4)), lw=2)
chart.ax.axvline(+60, c="white", ls=(2, (4, 4)), lw=2)
chart.ax.axvline(+60, c="black", ls=(6, (4, 4)), lw=2)
for m in earthkit.plots.schemas.schema.quickmap_subplot_workflow:
if m != "title":
getattr(chart, m)()
for m in earthkit.plots.schemas.schema.quickmap_figure_workflow:
if m != "legend":
getattr(chart, m)()
cax1 = chart.distinct_legend_layers[1].legend(location="bottom")
cax2 = chart.distinct_legend_layers[0].legend(location="bottom")
fig.fig.patch.set_facecolor("#002346")
cax1.ax.tick_params(axis="x", labelcolor="#CCCCCC")
cax1.ax.xaxis.label.set_color("white")
cax2.ax.tick_params(axis="x", labelcolor="#CCCCCC")
cax2.ax.xaxis.label.set_color("white")
chart.ax.artists[0].xlabel_style = dict(color="#CCCCCC")
chart.ax.artists[0].ylabel_style = dict(color="#CCCCCC")
# manual fig.save to override the dpi
fig._release_queue()
plt.savefig(
Path("plots") / "vorticity-egu26.pdf",
bbox_inches="tight",
dpi=400,
facecolor="#002346",
)
import json
with Path("observations").joinpath("vorticity.json").open("w") as f:
json.dump(observations, f)