Preserving a multi-variable pointwise quantity of interest (QoI) with safeguards¶
In this example, we compute the wind kinetic energy per unit mass from a dataset of wind components u and v. We compare how three different lossy compressors (ZFP, SZ3, and SPERR) affect the derived pointwise kinetic energy when compressing the u and v variables (stacked into one variable). Finally, we apply safeguards to guarantee an error bound on the derived kinetic energy. We also compare the safeguards with the QoI-aware QPET-SPERR compressor and the compressor configuration auto-tuner OptZConfig.
Stacking u and v into one variable that is then compressed is possible because u and v have very similar data distributions.
import ssl
ssl._create_default_https_context = ssl._create_stdlib_context
import copy
from pathlib import Path
import earthkit.plots
import humanize
import matplotlib as mpl
import numpy as np
import pandas as pd
import xarray as xr
from matplotlib import patheffects as PathEffects
# Retrieve the data
ERA5 = xr.open_dataset(Path() / "data" / "era5-uv" / "data.nc")
ERA5 = ERA5.sel(valid_time="2024-04-02T12:00:00", pressure_level=500)
def compute_kinetic_energy(ERA5: xr.Dataset) -> xr.DataArray:
ERA5_KE = 0.5 * (np.square(ERA5["u"]) + np.square(ERA5["v"]))
ERA5_KE.attrs.update(
long_name="wind kinetic energy per unit mass", units="m**2 s**-2"
)
return ERA5_KE
ERA5_KE = compute_kinetic_energy(ERA5)
def compute_corrections_percentage(my_ERA5: xr.Dataset, orig_ERA5: xr.Dataset) -> float:
neq = np.sum(my_ERA5 != orig_ERA5)
return int(neq.u + neq.v) / int(orig_ERA5.u.size + orig_ERA5.v.size)
old_cmap_and_norm = earthkit.plots.styles.colors.cmap_and_norm
def my_cmap_and_norm(colors, levels, normalize=True, extend=None, extend_levels=True):
return old_cmap_and_norm(colors, levels, normalize, extend, True)
earthkit.plots.styles.colors.cmap_and_norm = my_cmap_and_norm
def plot_kinetic_energy(
my_ERA5: xr.Dataset,
cr,
chart,
title,
span,
ke_eb_abs,
error=False,
corr=None,
my_ERA5_it=None,
cr_it=None,
inset=True,
):
my_ERA5_KE = compute_kinetic_energy(my_ERA5)
if error:
with xr.set_options(keep_attrs=True):
da = (my_ERA5_KE - ERA5_KE).compute()
da.attrs.update(long_name=f"Absolute error over {da.long_name}")
else:
# plot the square root of kinetic energy to better capture scale
da = np.sqrt(my_ERA5_KE)
da.attrs.update(long_name=f"sqrt({da.long_name})", units="m**1 s**-1")
# compute the default style that earthkit.maps would apply
source = earthkit.plots.sources.XarraySource(da)
style = copy.deepcopy(
earthkit.plots.styles.auto.guess_style(
source,
units=source.units,
)
)
if error:
style._levels = earthkit.plots.styles.levels.Levels(
np.linspace(-span, span, 21)
)
style._legend_kwargs["ticks"] = np.linspace(-span, span, 5)
style._colors = "coolwarm"
else:
style._levels = earthkit.plots.styles.levels.Levels(np.linspace(0, span, 21))
style._legend_kwargs["ticks"] = np.linspace(0, span, 5)
style._colors = "viridis"
extend_left = np.nanmin(da) < (-span if error else 0)
extend_right = np.nanmax(da) > span
extend = {
(False, False): "neither",
(True, False): "min",
(False, True): "max",
(True, True): "both",
}[(extend_left, extend_right)]
if error:
style._legend_kwargs["extend"] = extend
chart.pcolormesh(da, style=style, zorder=-12)
if corr is not None:
da_hatch = (my_ERA5["u"] == corr["u"]) & (my_ERA5["v"] == corr["v"])
if my_ERA5_it is None:
da_corr = da_hatch.astype(float)
else:
with xr.set_options(keep_attrs=True):
da_hatch_it = (my_ERA5_it["u"] == corr["u"]) & (
my_ERA5_it["v"] == corr["v"]
)
da_corr = (~da_hatch).astype(float) + (~da_hatch_it).astype(float)
old_process_projection_requirements = (
chart.ax.get_figure()._process_projection_requirements
)
def _process_projection_requirements(
*, axes_class=None, polar=False, projection=None, **kwargs
):
if axes_class is not None and projection is not None:
return axes_class, dict(projection=projection, **kwargs)
return old_process_projection_requirements(
axes_class=axes_class, polar=polar, projection=projection, **kwargs
)
chart.ax.get_figure()._process_projection_requirements = (
_process_projection_requirements
)
axin = chart.ax.inset_axes(
[0.025, 0.05, 1 / 3, 1 / 3],
xticklabels=[],
yticklabels=[],
axes_class=type(chart.ax),
projection=chart.ax.projection,
)
axin.pcolormesh(
da_corr.longitude.values,
da_corr.latitude.values,
np.squeeze(da_corr.values),
cmap=mpl.colors.ListedColormap(["white", "green", "lawngreen"]),
vmin=0,
vmax=2,
rasterized=True,
)
axin.coastlines(color="#555555")
axin.spines["geo"].set_edgecolor("black")
axin.set_title(
"Corrections",
path_effects=[PathEffects.withStroke(linewidth=3, foreground="white")],
)
elif inset:
da_err = ~(np.abs(da) <= ke_eb_abs)
old_process_projection_requirements = (
chart.ax.get_figure()._process_projection_requirements
)
def _process_projection_requirements(
*, axes_class=None, polar=False, projection=None, **kwargs
):
if axes_class is not None and projection is not None:
return axes_class, dict(projection=projection, **kwargs)
return old_process_projection_requirements(
axes_class=axes_class, polar=polar, projection=projection, **kwargs
)
chart.ax.get_figure()._process_projection_requirements = (
_process_projection_requirements
)
axin = chart.ax.inset_axes(
[0.025, 0.05, 1 / 3, 1 / 3],
xticklabels=[],
yticklabels=[],
axes_class=type(chart.ax),
projection=chart.ax.projection,
)
axin.pcolormesh(
da_err.longitude.values,
da_err.latitude.values,
np.squeeze(da_err.values),
cmap=mpl.colors.ListedColormap(["white", "red"]),
vmin=0,
vmax=1,
rasterized=True,
)
axin.coastlines(color="#555555")
axin.spines["geo"].set_edgecolor("black")
axin.set_title(
"Violations",
path_effects=[PathEffects.withStroke(linewidth=3, foreground="white")],
)
else:
chart.quickplot(da, style=style, extend=extend, zorder=-11)
chart.ax.set_rasterization_zorder(-10)
chart.title(title)
if error:
err_v = np.mean(~(np.abs(my_ERA5_KE - ERA5_KE) <= ke_eb_abs))
err_v = (
0
if err_v == 0
else np.format_float_positional(100 * err_v, precision=1, min_digits=1)
+ "%"
)
if err_v == "0.0%":
err_v = "<0.05%"
t = chart.ax.text(
0.95,
0.1,
f"V={err_v}",
ha="right",
va="bottom",
transform=chart.ax.transAxes,
)
t.set_bbox(dict(facecolor="white", alpha=0.75, edgecolor="black"))
t = chart.ax.text(
0.95,
0.9,
(
rf"$\times$ {np.round(cr, 2)}"
+ ("" if cr_it is None else rf" ($\times$ {np.round(cr_it, 2)})")
)
if error
else humanize.naturalsize(ERA5["u"].nbytes + ERA5["v"].nbytes, binary=True),
ha="right",
va="top",
transform=chart.ax.transAxes,
)
t.set_bbox(dict(facecolor="white", alpha=0.75, edgecolor="black"))
for m in earthkit.plots.schemas.schema.quickmap_subplot_workflow:
if m != "title":
getattr(chart, m)()
for m in earthkit.plots.schemas.schema.quickmap_figure_workflow:
getattr(chart, m)()
counts, bins = np.histogram(
da.values.flatten(), range=(-span if error else 0, span), bins=20
)
midpoints = bins[:-1] + np.diff(bins) / 2
cb = chart.ax.collections[0].colorbar
if error:
if extend_left:
cb._extend_patches[0].set_hatch("xx")
cb._extend_patches[0].set_ec("white")
cb.ax.fill_between(
[-span, -ke_eb_abs], *cb.ax.get_ylim(), hatch="xx", ec="w", fc="none", lw=0
)
cb.ax.fill_between(
[ke_eb_abs, span], *cb.ax.get_ylim(), hatch="xx", ec="w", fc="none", lw=0
)
if extend_right:
cb._extend_patches[-1].set_hatch("xx")
cb._extend_patches[-1].set_ec("white")
extend_width = (bins[-1] - bins[-2]) / (bins[-1] - bins[0])
cax = cb.ax.inset_axes(
[
0.0 - extend_width * extend_left,
1.25,
1.0 + extend_width * (0 + extend_left + extend_right),
1.0,
]
)
cax.bar(
midpoints,
height=counts,
width=(bins[-1] - bins[0]) / len(counts),
color=cb.cmap(cb.norm(midpoints)),
**(
dict(
hatch=["xx" if np.abs(m) > ke_eb_abs else "" for m in midpoints],
ec="white",
lw=0,
)
if error
else dict()
),
)
if extend_left:
cax.bar(
bins[0] - (bins[1] - bins[0]) / 2,
height=np.sum(da < (-span if error else 0)),
width=(bins[-1] - bins[0]) / len(counts),
color=cb.cmap(cb.norm(midpoints[0])),
**(
dict(
hatch="xx",
ec="white",
lw=0,
)
if error
else dict()
),
)
if extend_right:
cax.bar(
bins[-1] + (bins[-1] - bins[-2]) / 2,
height=np.sum(da > span),
width=(bins[-1] - bins[0]) / len(counts),
color=cb.cmap(cb.norm(midpoints[-1])),
**(
dict(
hatch="xx",
ec="white",
lw=0,
)
if error
else dict()
),
)
q1, q2, q3 = da.quantile([0.25, 0.5, 0.75]).values
cax.axvline(da.mean().item(), ls=":", ymin=0.1, ymax=0.9, c="w", lw=2)
cax.axvline(q1, ymin=0.25, ymax=0.75, c="w", lw=2)
cax.axvline(q2, ymin=0.1, ymax=0.9, c="w", lw=2)
cax.axvline(q3, ymin=0.25, ymax=0.75, c="w", lw=2)
cax.axvline(da.mean().item(), ymin=0.1, ymax=0.9, ls=":", c="k", lw=1)
cax.axvline(q1, ymin=0.25, ymax=0.75, c="k", lw=1)
cax.axvline(q2, ymin=0.1, ymax=0.9, c="k", lw=1)
cax.axvline(q3, ymin=0.25, ymax=0.75, c="k", lw=1)
cax.set_xlim(
(-span if error else 0) - (bins[-1] - bins[-2]) * extend_left,
span + (bins[-1] - bins[-2]) * extend_right,
)
cax.set_xticks([])
cax.set_yticks([])
cax.spines[:].set_visible(False)
def table_kinetic_energy(
my_ERA5: xr.Dataset,
cr,
title,
ke_eb_abs,
corr,
) -> pd.DataFrame:
my_ERA5_KE = compute_kinetic_energy(my_ERA5)
err_inf_U = np.amax(np.abs(my_ERA5["u"] - ERA5["u"]))
err_inf_V = np.amax(np.abs(my_ERA5["v"] - ERA5["v"]))
err_inf_KE = np.amax(np.abs(my_ERA5_KE - ERA5_KE))
err_2_KE = np.sqrt(np.mean(np.square(my_ERA5_KE - ERA5_KE)))
err_v = np.mean(~(np.abs(my_ERA5_KE - ERA5_KE) <= ke_eb_abs))
err_v = (
0
if err_v == 0
else np.format_float_positional(100 * err_v, precision=1, min_digits=1) + "%"
)
if err_v == "0.0%":
err_v = "<0.05%"
corr = None if corr is None else compute_corrections_percentage(my_ERA5, corr)
corr = (
""
if corr is None
else (
0
if corr == 0
else np.format_float_positional(100 * corr, precision=1, min_digits=1) + "%"
)
)
if corr == "0.0%":
corr = "<0.05%"
return pd.DataFrame(
{
"Compressor": [title[0]],
"Safeguarded": [title[1]],
"Corrections": [title[2]],
r"$L_{\infty}(\hat{u})$": [
f"{err_inf_U:.03}",
],
r"$L_{\infty}(\hat{v})$": [
f"{err_inf_V:.03}",
],
r"$L_{\infty}(\hat{\mathrm{KE}})$": [
f"{err_inf_KE:.03}",
],
r"$L_{2}(\hat{\mathrm{KE}})$": [
f"{err_2_KE:.03}",
],
"V": [err_v],
"C": [corr],
"CR": [
rf"$\times$ {np.round(cr, 2)}",
],
}
)
# Since numcodecs-safeguards only supports single-variable safeguarding, we
# stack the u and v variables into a combined variable.
ERA5_UV = np.stack([ERA5["u"].values, ERA5["v"].values], axis=0)
ERA5.u.dims, ERA5_UV.shape
(('latitude', 'longitude'), (2, 721, 1440))
import observe
observations = []
Lossless compression¶
We first compress the data losslessly with ZStandard at level 22, which gives maximum compression, to provide a baseline.
from numcodecs_wasm_zstd import Zstd
zstd = Zstd(level=22)
with observe.observe(zstd, observations):
ERA5_UV_zstd_enc = zstd.encode(ERA5_UV)
ERA5_UV_zstd = zstd.decode(ERA5_UV_zstd_enc)
ERA5_zstd = ERA5.copy(data=dict(u=ERA5_UV_zstd[0], v=ERA5_UV_zstd[1]))
ERA5_zstd_cr = ERA5_UV.nbytes / ERA5_UV_zstd_enc.nbytes
Compressing u and v with lossy compressors¶
We configure each compressor with an absolute error bound of 0.125 m/s over the u-v array, which seems to provide similar errors on the derived kinetic energy.
eb_abs = 0.125
from numcodecs_wasm_zfp import Zfp
zfp = Zfp(mode="fixed-accuracy", tolerance=eb_abs)
with observe.observe(zfp, observations):
ERA5_UV_zfp_enc = zfp.encode(ERA5_UV)
ERA5_UV_zfp = zfp.decode(ERA5_UV_zfp_enc)
ERA5_zfp = ERA5.copy(data=dict(u=ERA5_UV_zfp[0], v=ERA5_UV_zfp[1]))
ERA5_zfp_cr = ERA5_UV.nbytes / ERA5_UV_zfp_enc.nbytes
from numcodecs_wasm_sz3 import Sz3
sz3 = Sz3(eb_mode="abs", eb_abs=eb_abs)
with observe.observe(sz3, observations):
ERA5_UV_sz3_enc = sz3.encode(ERA5_UV)
ERA5_UV_sz3 = sz3.decode(ERA5_UV_sz3_enc)
ERA5_sz3 = ERA5.copy(data=dict(u=ERA5_UV_sz3[0], v=ERA5_UV_sz3[1]))
ERA5_sz3_cr = ERA5_UV.nbytes / ERA5_UV_sz3_enc.nbytes
from numcodecs_wasm_sperr import Sperr
sperr = Sperr(mode="pwe", pwe=eb_abs)
with observe.observe(sperr, observations):
ERA5_UV_sperr_enc = sperr.encode(ERA5_UV)
ERA5_UV_sperr = sperr.decode(ERA5_UV_sperr_enc)
ERA5_sperr = ERA5.copy(data=dict(u=ERA5_UV_sperr[0], v=ERA5_UV_sperr[1]))
ERA5_sperr_cr = ERA5_UV.nbytes / ERA5_UV_sperr_enc.nbytes
from numcodecs_zero import ZeroCodec
zero = ZeroCodec()
with observe.observe(zero, observations):
ERA5_UV_zero_enc = zero.encode(ERA5_UV)
ERA5_UV_zero = zero.decode(ERA5_UV_zero_enc)
ERA5_zero = ERA5.copy(data=dict(u=ERA5_UV_zero[0], v=ERA5_UV_zero[1]))
Compressing u and v using the safeguarded lossy compressors¶
We configure the safeguards to bound the pointwise absolute error on the derived kinetic energy, choosing an error bound that is comparable to the errors produced by the lossy compression methods above.
The kinetic energy computation is translated into a quantity of interest. Even though kinetic energy is a pointwise quantity, stacking u and v into a single variable means that we now compute the kinetic energy over a two-element (u, v) local neighbourhood.
ke_eb_abs = 1.0
from numcodecs_safeguards import SafeguardedCodec
ERA5_sg = dict()
ERA5_sg_cr = dict()
for codec in [
zero,
zfp,
sz3,
sperr,
]:
sg = SafeguardedCodec(
codec=codec,
safeguards=[
dict(
kind="qoi_eb_stencil",
qoi="0.5 * (square(X[0]) + square(X[1]))",
type="abs",
eb=ke_eb_abs,
neighbourhood=[
dict(axis=0, before=0, after=1, boundary="valid"),
],
)
],
)
with observe.observe(sg, observations):
ERA5_UV_sg_enc = sg.encode(ERA5_UV)
ERA5_UV_sg = sg.decode(ERA5_UV_sg_enc)
ERA5_sg[codec.codec_id] = ERA5.copy(data=dict(u=ERA5_UV_sg[0], v=ERA5_UV_sg[1]))
ERA5_sg_cr[codec.codec_id] = ERA5_UV.nbytes / np.asarray(ERA5_UV_sg_enc).nbytes
ERA5_sg_it = dict()
ERA5_sg_it_cr = dict()
for codec in [
ZeroCodec(),
zfp,
sz3,
sperr,
]:
sg_it = SafeguardedCodec(
codec=codec,
safeguards=[
dict(
kind="qoi_eb_stencil",
qoi="0.5 * (square(X[0]) + square(X[1]))",
type="abs",
eb=ke_eb_abs,
neighbourhood=[
dict(axis=0, before=0, after=1, boundary="valid"),
],
)
],
# use iteration to refine the corrections
compute=dict(unstable_iterative=True),
)
with observe.observe(sg_it, observations):
ERA5_UV_sg_it_enc = sg_it.encode(ERA5_UV)
ERA5_UV_sg_it = sg_it.decode(ERA5_UV_sg_it_enc)
ERA5_sg_it[codec.codec_id] = ERA5.copy(
data=dict(u=ERA5_UV_sg_it[0], v=ERA5_UV_sg_it[1])
)
ERA5_sg_it_cr[codec.codec_id] = (
ERA5_UV.nbytes / np.asarray(ERA5_UV_sg_it_enc).nbytes
)
ERA5_sg_lossless = dict()
ERA5_sg_lossless_cr = dict()
for codec in [
ZeroCodec(),
zfp,
sz3,
sperr,
]:
sg_lossless = SafeguardedCodec(
codec=codec,
safeguards=[
dict(
kind="qoi_eb_stencil",
qoi="0.5 * (square(X[0]) + square(X[1]))",
type="abs",
eb=ke_eb_abs,
neighbourhood=[
dict(axis=0, before=0, after=1, boundary="valid"),
],
)
],
# produce lossless corrections and refine them with iteration
compute=dict(unstable_iterative=True, unstable_lossless_corrections=True),
)
with observe.observe(sg_lossless, observations):
ERA5_UV_sg_lossless_enc = sg_lossless.encode(ERA5_UV)
ERA5_UV_sg_lossless = sg_lossless.decode(ERA5_UV_sg_lossless_enc)
ERA5_sg_lossless[codec.codec_id] = ERA5.copy(
data=dict(u=ERA5_UV_sg_lossless[0], v=ERA5_UV_sg_lossless[1])
)
ERA5_sg_lossless_cr[codec.codec_id] = (
ERA5_UV.nbytes / np.asarray(ERA5_UV_sg_lossless_enc).nbytes
)
Compressing u and v using QPET-SPERR¶
We similarly configure QPET-SPERR to bound the pointwise absolute error on the derived kinetic energy, choosing an error bound that is comparable to the errors produced by the lossy compression methods above.
The kinetic energy computation is translated into a quantity of interest. We utilize QPET-SPERR's block-mean mode to bound the mean of $x^2$ over the stacked $[u, v]$ axis, which bounds $0.5 \cdot (u^2 + v^2)$, i.e. the kinetic energy.
from numcodecs_wasm_qpet_sperr import QpetSperr
qpet = QpetSperr(
mode="qoi-symbolic", qoi="x^2", qoi_block_size=(2, 1, 1), qoi_pwe=ke_eb_abs
)
with observe.observe(qpet, observations):
ERA5_UV_qpet_enc = qpet.encode(ERA5_UV)
ERA5_UV_qpet = qpet.decode(ERA5_UV_qpet_enc)
ERA5_qpet = ERA5.copy(data=dict(u=ERA5_UV_qpet[0], v=ERA5_UV_qpet[1]))
ERA5_qpet_cr = ERA5_UV.nbytes / ERA5_UV_qpet_enc.nbytes
Tuning eb with qoi current_eb = 0.0385615, current_br = 2.08203 current_eb = 0.02754, current_br = 2.26562 current_eb = 0.02137, current_br = 2.4375 current_eb = 0.0176515, current_br = 2.58984 current_eb = 0.0162742, current_br = 2.64062 current_eb = 0.0154391, current_br = 2.68359 current_eb = 0.0145696, current_br = 2.70703 Selected quantile: 0.200005 Best abs eb: 0.0385615 Tuning eb with qoi current_eb = 0.0385615, current_br = 2.12109 current_eb = 0.0273392, current_br = 2.34375 current_eb = 0.0213517, current_br = 2.49219 current_eb = 0.0186124, current_br = 2.59375 current_eb = 0.0170938, current_br = 2.66406 current_eb = 0.0153922, current_br = 2.76172 current_eb = 0.0140332, current_br = 2.8125 Selected quantile: 0.200005 Best abs eb: 0.0385615 Tuning eb with qoi current_eb = 0.0294802, current_br = 2.58594 current_eb = 0.0213552, current_br = 2.5 current_eb = 0.0166734, current_br = 2.73047 current_eb = 0.0139531, current_br = 2.85938 current_eb = 0.0129703, current_br = 2.89453 current_eb = 0.011896, current_br = 2.98047 current_eb = 0.0105038, current_br = 3.0625 Selected quantile: 0.0999985 Best abs eb: 0.0213552 Tuning eb with qoi current_eb = 0.0213552, current_br = 2.35547 current_eb = 0.0213552, current_br = 2.35547 current_eb = 0.0196951, current_br = 2.42969 current_eb = 0.0174219, current_br = 2.49609 current_eb = 0.0163573, current_br = 2.56641 current_eb = 0.0153228, current_br = 2.57031 current_eb = 0.0141949, current_br = 2.63281 Selected quantile: 0.0999985 Best abs eb: 0.0213552 Tuning eb with qoi current_eb = 0.0213552, current_br = 2.98828 current_eb = 0.0213552, current_br = 2.98828 current_eb = 0.0181022, current_br = 3.10156 current_eb = 0.0158601, current_br = 3.19922 current_eb = 0.0149439, current_br = 3.21484 current_eb = 0.0140495, current_br = 3.32031 current_eb = 0.0131764, current_br = 3.36328 Selected quantile: 0.0999985 Best abs eb: 0.0213552 Tuning eb with qoi current_eb = 0.0213552, current_br = 2.5625 current_eb = 0.0213552, current_br = 2.5625 current_eb = 0.0213552, current_br = 2.5625 current_eb = 0.0187288, current_br = 2.66016 current_eb = 0.0173352, current_br = 2.76172 current_eb = 0.0162173, current_br = 2.79297 current_eb = 0.0140113, current_br = 2.92578 Selected quantile: 0.05 Best abs eb: 0.0213552 Tuning eb with qoi current_eb = 0.0213552, current_br = 3.82031 current_eb = 0.0213552, current_br = 3.82031 current_eb = 0.0213552, current_br = 3.82031 current_eb = 0.0213552, current_br = 3.82031 current_eb = 0.0213552, current_br = 3.82031 current_eb = 0.0213552, current_br = 3.82031 current_eb = 0.0213552, current_br = 3.82031 Selected quantile: 0.0019989 Best abs eb: 0.0213552 Tuning eb with qoi current_eb = 0.0213552, current_br = 3.53906 current_eb = 0.0213552, current_br = 3.53906 current_eb = 0.0213552, current_br = 3.53906 current_eb = 0.0213552, current_br = 3.53906 current_eb = 0.0213552, current_br = 3.53906 current_eb = 0.0213552, current_br = 3.53906 current_eb = 0.020427, current_br = 3.58594 Selected quantile: 0.0019989 Best abs eb: 0.020427 Tuning eb with qoi current_eb = 0.020427, current_br = 2.82422 current_eb = 0.020427, current_br = 2.82422 current_eb = 0.020427, current_br = 2.82422 current_eb = 0.020427, current_br = 2.82422 current_eb = 0.020427, current_br = 2.82422 current_eb = 0.020427, current_br = 2.82422 current_eb = 0.020427, current_br = 2.82422 Selected quantile: 0.0019989 Best abs eb: 0.020427 Tuning eb with qoi current_eb = 0.020427, current_br = 3.07422 current_eb = 0.020427, current_br = 3.07422 current_eb = 0.020427, current_br = 3.07422 current_eb = 0.020427, current_br = 3.07422 current_eb = 0.020427, current_br = 3.07422 current_eb = 0.019976, current_br = 3.10938 current_eb = 0.0173694, current_br = 3.25 Selected quantile: 0.00499725 Best abs eb: 0.019976 Tuning eb with qoi current_eb = 0.019976, current_br = 5.33594 current_eb = 0.019976, current_br = 5.33594 current_eb = 0.019976, current_br = 5.33594 current_eb = 0.019976, current_br = 5.33594 current_eb = 0.0181529, current_br = 5.69531 current_eb = 0.0162117, current_br = 5.69922 current_eb = 0.0139566, current_br = 5.85938 Selected quantile: 0.0199966 Best abs eb: 0.019976 Tuning eb with qoi current_eb = 0.019976, current_br = 3.375 current_eb = 0.019976, current_br = 3.375 current_eb = 0.019976, current_br = 3.375 current_eb = 0.019976, current_br = 3.375 current_eb = 0.019976, current_br = 3.375 current_eb = 0.019976, current_br = 3.375 current_eb = 0.019976, current_br = 3.375 Selected quantile: 0.00198975 Best abs eb: 0.019976 Tuning eb with qoi current_eb = 0.019976, current_br = 3 current_eb = 0.019976, current_br = 3 current_eb = 0.0141789, current_br = 3.15234 current_eb = 0.0111776, current_br = 3.39062 current_eb = 0.0104769, current_br = 3.47656 current_eb = 0.0100804, current_br = 3.50781 current_eb = 0.00986025, current_br = 3.54688 Selected quantile: 0.100002 Best abs eb: 0.019976 Tuning eb with qoi current_eb = 0.019976, current_br = 2.78516 current_eb = 0.0190553, current_br = 2.8125 current_eb = 0.0153439, current_br = 2.95703 current_eb = 0.0132674, current_br = 3.13672 current_eb = 0.0124521, current_br = 3.23047 current_eb = 0.0117992, current_br = 3.29297 current_eb = 0.010319, current_br = 3.46875 Selected quantile: 0.100002 Best abs eb: 0.0190553 Tuning eb with qoi current_eb = 0.0190553, current_br = 2.84375 current_eb = 0.0190553, current_br = 2.84375 current_eb = 0.0172441, current_br = 2.91016 current_eb = 0.0145963, current_br = 3.07812 current_eb = 0.0136844, current_br = 3.14453 current_eb = 0.0129884, current_br = 3.18359 current_eb = 0.0122608, current_br = 3.24219 Selected quantile: 0.100002 Best abs eb: 0.0190553 Tuning eb with qoi current_eb = 0.0190553, current_br = 2.87891 current_eb = 0.0190553, current_br = 2.87891 current_eb = 0.0165124, current_br = 3.04297 current_eb = 0.014258, current_br = 3.23438 current_eb = 0.0132623, current_br = 3.31641 current_eb = 0.0126522, current_br = 3.38281 current_eb = 0.0120816, current_br = 3.44531 Selected quantile: 0.100002 Best abs eb: 0.0190553 Tuning eb with qoi current_eb = 0.0190553, current_br = 2.81641 current_eb = 0.0190553, current_br = 2.81641 current_eb = 0.0190553, current_br = 2.81641 current_eb = 0.0175791, current_br = 2.98438 current_eb = 0.0152917, current_br = 3.12109 current_eb = 0.014218, current_br = 3.16797 current_eb = 0.0134137, current_br = 3.25781 Selected quantile: 0.0499963 Best abs eb: 0.0190553 Tuning eb with qoi current_eb = 0.0190553, current_br = 3.8125 current_eb = 0.0190553, current_br = 3.8125 current_eb = 0.0190553, current_br = 3.8125 current_eb = 0.0158133, current_br = 4.00391 current_eb = 0.0135989, current_br = 4.23828 current_eb = 0.0124878, current_br = 4.32422 current_eb = 0.0117148, current_br = 4.37891 Selected quantile: 0.05 Best abs eb: 0.0190553
Compressing u and v using OptZConfig¶
We configure OptZConfig with a custom safety violations metric, implemented in Python, that computes the percentage of violations $V$. We then maximise the score
$$ \textrm{score} = \begin{cases} -\textrm{V} \quad &\text{if } \textrm{V} > 0 \\ \textrm{CR} \quad &\text{otherwise} \end{cases} $$
using the FRAZ search algorithm with 25 iterations, where CR is the achieved compression ratio. Since FRAZ seems to struggle with finding sufficient absolute error bounds spread across several orders of magnitude, we search for bounds in logarithmic space by wrapping each codec in an Exponential<CODEC> meta-codec.
import numcodecs
class SafetyViolationsMetric(numcodecs.abc.Codec):
codec_id = "safety-violations-metric"
def __init__(self):
self._data = None
def encode(self, buf):
# store the original data for later
self._data = np.array(buf, copy=True)
# return no metric
return np.empty(0, dtype=np.float64)
def decode(self, buf, out=None):
# compute the violations
data_KE = compute_kinetic_energy(
ERA5.copy(data=dict(u=self._data[0], v=self._data[1]))
)
buf_KE = compute_kinetic_energy(ERA5.copy(data=dict(u=buf[0], v=buf[1])))
violations = np.mean(~(np.abs(buf_KE - data_KE) <= ke_eb_abs))
self._data = None
# return the violations score metric
return numcodecs.compat.ndarray_copy(np.float64(violations), out)
numcodecs.registry.register_codec(SafetyViolationsMetric)
class ExponentialZfp(Zfp):
codec_id = "e-zfp.rs"
def __new__(cls, tolerance: float, **kwargs):
codec = super().__new__(cls, tolerance=np.exp(tolerance), **kwargs)
codec._tolerance = tolerance
return codec
def get_config(self):
return {
**super().get_config(),
"id": type(self).codec_id,
"tolerance": self._tolerance,
}
class ExponentialSz3(Sz3):
codec_id = "e-sz3.rs"
def __new__(cls, eb_abs: float, **kwargs):
codec = super().__new__(cls, eb_abs=np.exp(eb_abs), **kwargs)
codec._eb_abs = eb_abs
return codec
def get_config(self):
return {
**super().get_config(),
"id": type(self).codec_id,
"eb_abs": self._eb_abs,
}
class ExponentialSperr(Sperr):
codec_id = "e-sperr.rs"
def __new__(cls, pwe: float, **kwargs):
codec = super().__new__(cls, pwe=np.exp(pwe), **kwargs)
codec._pwe = pwe
return codec
def get_config(self):
return {**super().get_config(), "id": type(self).codec_id, "pwe": self._pwe}
numcodecs.registry.register_codec(ExponentialZfp)
numcodecs.registry.register_codec(ExponentialSz3)
numcodecs.registry.register_codec(ExponentialSperr)
from numcodecs_wasm_pressio import Pressio
ERA5_optzconfig = dict()
ERA5_optzconfig_cr = dict()
for codec, parameter, lower_bound in [
(zfp, "tolerance", 1e-3), # initial guess
(sz3, "eb_abs", 1e-3), # initial guess
(sperr, "pwe", 1e-3), # initial guess
]:
optzconfig = Pressio(
compressor_id="opt",
compressor_config={
"opt:output": ["composite:score"],
"opt:inputs": [f"numcodecs.rs:{parameter}"],
"opt:lower_bound": np.log(lower_bound),
"opt:upper_bound": np.log(eb_abs),
"opt:max_iterations": 25,
"opt:objective_mode_name": "max",
},
early_config={
"opt:compressor": "pressio",
"pressio:compressor": "numcodecs.rs",
**{
f"numcodecs.rs:{k}": f"e-{v}" if k == "id" else v
for k, v in codec.get_config().items()
},
"opt:search": "fraz",
"pressio:metric": "composite",
"composite:plugins": ["size", "numcodecs.rs-metric"],
"composite:scripts": [
"""
violations = metrics["numcodecs.rs-metric:decompression"]
if violations > 0 then
return "score", -violations
else
return "score", metrics["size:compression_ratio"]
end
"""
],
"numcodecs.rs-metric:id": "safety-violations-metric",
},
)
with observe.observe(optzconfig, observations):
ERA5_UV_optzconfig_enc = optzconfig.encode(ERA5_UV)
ERA5_UV_optzconfig = optzconfig.decode(ERA5_UV_optzconfig_enc)
ERA5_optzconfig[codec.codec_id] = ERA5.copy(
data=dict(u=ERA5_UV_optzconfig[0], v=ERA5_UV_optzconfig[1])
)
ERA5_optzconfig_cr[codec.codec_id] = ERA5_UV.nbytes / ERA5_UV_optzconfig_enc.nbytes
rank={0,1,} iter={0} input={-4.4936,} output={1.90014,} objective={1.90014}
rank={0,1,} iter={1} input={-5.78144,} output={1.53997,} objective={1.53997}
rank={0,1,} iter={2} input={-3.22544,} output={2.44998,} objective={2.44998}
rank={0,1,} iter={3} input={-2.07944,} output={-1.54107e-05,} objective={-1.54107e-05}
rank={0,1,} iter={4} input={-6.90629,} output={1.40538,} objective={1.40538}
rank={0,1,} iter={5} input={-3.65599,} output={2.14691,} objective={2.14691}
rank={0,1,} iter={6} input={-5.05317,} output={1.70242,} objective={1.70242}
rank={0,1,} iter={7} input={-4.01707,} output={2.14691,} objective={2.14691}
rank={0,1,} iter={8} input={-2.61839,} output={2.82804,} objective={2.82804}
rank={0,1,} iter={9} input={-6.3311,} output={1.40538,} objective={1.40538}
rank={0,1,} iter={10} input={-2.86112,} output={2.44998,} objective={2.44998}
rank={0,1,} iter={11} input={-5.40251,} output={1.70242,} objective={1.70242}
rank={0,1,} iter={12} input={-3.41208,} output={2.44998,} objective={2.44998}
rank={0,1,} iter={13} input={-3.04288,} output={2.44998,} objective={2.44998}
rank={0,1,} iter={14} input={-2.31486,} output={2.82804,} objective={2.82804}
rank={0,1,} iter={15} input={-4.76529,} output={1.90014,} objective={1.90014}
rank={0,1,} iter={16} input={-2.46663,} output={2.82804,} objective={2.82804}
rank={0,1,} iter={17} input={-4.24504,} output={1.90014,} objective={1.90014}
rank={0,1,} iter={18} input={-2.54251,} output={2.82804,} objective={2.82804}
rank={0,1,} iter={19} input={-6.61825,} output={1.40538,} objective={1.40538}
rank={0,1,} iter={20} input={-6.05081,} output={1.53997,} objective={1.53997}
rank={0,1,} iter={21} input={-3.83644,} output={2.14691,} objective={2.14691}
rank={0,1,} iter={22} input={-2.72434,} output={2.82804,} objective={2.82804}
rank={0,1,} iter={23} input={-5.58389,} output={1.53997,} objective={1.53997}
rank={0,1,} iter={24} input={-5.22766,} output={1.70242,} objective={1.70242}
final_iter={25} inputs={-2.61839,} output={2.82804,}
rank={0,1,} iter={0} input={-4.4936,} output={6.65809,} objective={6.65809}
rank={0,1,} iter={1} input={-5.78144,} output={4.72472,} objective={4.72472}
rank={0,1,} iter={2} input={-3.22544,} output={-0.0202911,} objective={-0.0202911}
rank={0,1,} iter={3} input={-4.85335,} output={6.10991,} objective={6.10991}
rank={0,1,} iter={4} input={-6.90629,} output={3.46382,} objective={3.46382}
rank={0,1,} iter={5} input={-5.18654,} output={5.55666,} objective={5.55666}
rank={0,1,} iter={6} input={-3.88187,} output={-0.000616428,} objective={-0.000616428}
rank={0,1,} iter={7} input={-2.07953,} output={-0.20619,} objective={-0.20619}
rank={0,1,} iter={8} input={-4.61382,} output={6.41075,} objective={6.41075}
rank={0,1,} iter={9} input={-6.28567,} output={4.11987,} objective={4.11987}
rank={0,1,} iter={10} input={-4.34067,} output={-1.92634e-06,} objective={-1.92634e-06}
rank={0,1,} iter={11} input={-2.65299,} output={-0.0860052,} objective={-0.0860052}
rank={0,1,} iter={12} input={-4.54755,} output={6.54557,} objective={6.54557}
rank={0,1,} iter={13} input={-5.47352,} output={5.13337,} objective={5.13337}
rank={0,1,} iter={14} input={-4.45537,} output={6.7308,} objective={6.7308}
rank={0,1,} iter={15} input={-6.58868,} output={3.84495,} objective={3.84495}
rank={0,1,} iter={16} input={-4.3789,} output={-9.63168e-07,} objective={-9.63168e-07}
rank={0,1,} iter={17} input={-3.55369,} output={-0.00505374,} objective={-0.00505374}
rank={0,1,} iter={18} input={-4.47327,} output={6.69199,} objective={6.69199}
rank={0,1,} iter={19} input={-6.02979,} output={4.41478,} objective={4.41478}
rank={0,1,} iter={20} input={-4.43625,} output={6.77247,} objective={6.77247}
rank={0,1,} iter={21} input={-2.36615,} output={-0.13977,} objective={-0.13977}
rank={0,1,} iter={22} input={-4.39802,} output={6.82193,} objective={6.82193}
rank={0,1,} iter={23} input={-2.93911,} output={-0.0462436,} objective={-0.0462436}
rank={0,1,} iter={24} input={-4.41703,} output={6.81978,} objective={6.81978}
final_iter={25} inputs={-4.39802,} output={6.82193,}
rank={0,1,} iter={0} input={-4.4936,} output={9.68056,} objective={9.68056}
rank={0,1,} iter={1} input={-5.78144,} output={6.57552,} objective={6.57552}
rank={0,1,} iter={2} input={-3.22544,} output={-0.00457698,} objective={-0.00457698}
rank={0,1,} iter={3} input={-4.83076,} output={8.67178,} objective={8.67178}
rank={0,1,} iter={4} input={-6.90629,} output={4.94341,} objective={4.94341}
rank={0,1,} iter={5} input={-4.43616,} output={9.87254,} objective={9.87254}
rank={0,1,} iter={6} input={-5.1777,} output={7.79598,} objective={7.79598}
rank={0,1,} iter={7} input={-3.21271,} output={-0.00491601,} objective={-0.00491601}
rank={0,1,} iter={8} input={-4.60027,} output={9.34452,} objective={9.34452}
rank={0,1,} iter={9} input={-3.82444,} output={-9.728e-05,} objective={-9.728e-05}
rank={0,1,} iter={10} input={-2.08173,} output={-0.0910782,} objective={-0.0910782}
rank={0,1,} iter={11} input={-4.1303,} output={-3.85267e-06,} objective={-3.85267e-06}
rank={0,1,} iter={12} input={-6.3166,} output={5.72159,} objective={5.72159}
rank={0,1,} iter={13} input={-4.28323,} output={10.4146,} objective={10.4146}
rank={0,1,} iter={14} input={-2.64701,} output={-0.0311527,} objective={-0.0311527}
rank={0,1,} iter={15} input={-4.35213,} output={10.1663,} objective={10.1663}
rank={0,1,} iter={16} input={-5.46972,} output={7.16234,} objective={7.16234}
rank={0,1,} iter={17} input={-4.31211,} output={10.3088,} objective={10.3088}
rank={0,1,} iter={18} input={-6.60648,} output={5.30946,} objective={5.30946}
rank={0,1,} iter={19} input={-4.20676,} output={-2.88951e-06,} objective={-2.88951e-06}
rank={0,1,} iter={20} input={-6.04545,} output={6.13601,} objective={6.13601}
rank={0,1,} iter={21} input={-4.245,} output={10.5574,} objective={10.5574}
rank={0,1,} iter={22} input={-3.5249,} output={-0.000965095,} objective={-0.000965095}
rank={0,1,} iter={23} input={-4.2636,} output={-9.63168e-07,} objective={-9.63168e-07}
rank={0,1,} iter={24} input={-2.92939,} output={-0.014044,} objective={-0.014044}
final_iter={25} inputs={-4.245,} output={10.5574,}
Visual comparison of the error distributions for the derived kinetic energy¶
fig = earthkit.plots.Figure(
size=(10, 23),
rows=6,
columns=2,
)
plot_kinetic_energy(
ERA5, 1.0, fig.add_map(0, 0), "Original", span=50, ke_eb_abs=ke_eb_abs
)
plot_kinetic_energy(
ERA5_zfp,
ERA5_zfp_cr,
fig.add_map(1, 0),
r"ZFP($\epsilon_{{abs}}$)",
span=0.25,
ke_eb_abs=ke_eb_abs,
error=True,
)
plot_kinetic_energy(
ERA5_sz3,
ERA5_sz3_cr,
fig.add_map(2, 0),
r"SZ3($\epsilon_{{abs}}$)",
span=2.5,
ke_eb_abs=ke_eb_abs,
error=True,
)
plot_kinetic_energy(
ERA5_sperr,
ERA5_sperr_cr,
fig.add_map(3, 0),
r"SPERR($\epsilon_{{abs}}$)",
span=2.5,
ke_eb_abs=ke_eb_abs,
error=True,
)
plot_kinetic_energy(
ERA5_sg["zero"],
ERA5_sg_cr["zero"],
fig.add_map(0, 1),
r"Safeguarded(0, $\epsilon_{{QoI,abs}}$)",
span=ke_eb_abs,
ke_eb_abs=ke_eb_abs,
error=True,
corr=ERA5_zero,
my_ERA5_it=ERA5_sg_it["zero"],
cr_it=ERA5_sg_it_cr["zero"],
)
plot_kinetic_energy(
ERA5_sg["zfp.rs"],
ERA5_sg_cr["zfp.rs"],
fig.add_map(1, 1),
r"Safeguarded(ZFP, $\epsilon_{{QoI,abs}}$)",
span=ke_eb_abs,
ke_eb_abs=ke_eb_abs,
error=True,
corr=ERA5_zfp,
my_ERA5_it=ERA5_sg_it["zfp.rs"],
cr_it=ERA5_sg_it_cr["zfp.rs"],
)
plot_kinetic_energy(
ERA5_sg["sz3.rs"],
ERA5_sg_cr["sz3.rs"],
fig.add_map(2, 1),
r"Safeguarded(SZ3, $\epsilon_{{QoI,abs}}$)",
span=ke_eb_abs,
ke_eb_abs=ke_eb_abs,
error=True,
corr=ERA5_sz3,
my_ERA5_it=ERA5_sg_it["sz3.rs"],
cr_it=ERA5_sg_it_cr["sz3.rs"],
)
plot_kinetic_energy(
ERA5_sg["sperr.rs"],
ERA5_sg_cr["sperr.rs"],
fig.add_map(3, 1),
r"Safeguarded(SPERR, $\epsilon_{{QoI,abs}}$)",
span=ke_eb_abs,
ke_eb_abs=ke_eb_abs,
error=True,
corr=ERA5_sperr,
my_ERA5_it=ERA5_sg_it["sperr.rs"],
cr_it=ERA5_sg_it_cr["sperr.rs"],
)
plot_kinetic_energy(
ERA5_optzconfig["zfp.rs"],
ERA5_optzconfig_cr["zfp.rs"],
fig.add_map(4, 0),
r"OptZConfig(ZFP, $\epsilon_{{QoI,abs}}$)",
span=ke_eb_abs,
ke_eb_abs=ke_eb_abs,
error=True,
inset=False,
)
plot_kinetic_energy(
ERA5_optzconfig["sz3.rs"],
ERA5_optzconfig_cr["sz3.rs"],
fig.add_map(4, 1),
r"OptZConfig(SZ3, $\epsilon_{{QoI,abs}}$)",
span=ke_eb_abs,
ke_eb_abs=ke_eb_abs,
error=True,
inset=False,
)
plot_kinetic_energy(
ERA5_optzconfig["sperr.rs"],
ERA5_optzconfig_cr["sperr.rs"],
fig.add_map(5, 0),
r"OptZConfig(SPERR, $\epsilon_{{QoI,abs}}$)",
span=ke_eb_abs,
ke_eb_abs=ke_eb_abs,
error=True,
inset=False,
)
plot_kinetic_energy(
ERA5_qpet,
ERA5_qpet_cr,
fig.add_map(5, 1),
r"QPET-SPERR($\epsilon_{{QoI,abs}}$)",
span=ke_eb_abs,
ke_eb_abs=ke_eb_abs,
error=True,
inset=False,
)
fig.save(Path("plots") / "kinetic-energy.pdf")
ke_sg_table = pd.concat(
[
table_kinetic_energy(
ERA5_sg_lossless["zero"],
ERA5_sg_lossless_cr["zero"],
["0", r"$\epsilon_{QoI,abs}$", "lossless"],
ke_eb_abs,
ERA5_zero,
),
table_kinetic_energy(
ERA5_sg["zero"],
ERA5_sg_cr["zero"],
["0", r"$\epsilon_{QoI,abs}$", "one-shot"],
ke_eb_abs,
ERA5_zero,
),
table_kinetic_energy(
ERA5_sg_it["zero"],
ERA5_sg_it_cr["zero"],
["0", r"$\epsilon_{QoI,abs}$", "iterative"],
ke_eb_abs,
ERA5_zero,
),
table_kinetic_energy(
ERA5_zfp,
ERA5_zfp_cr,
[r"ZFP($\epsilon_{abs}$)", "-", ""],
ke_eb_abs,
None,
),
table_kinetic_energy(
ERA5_sg_lossless["zfp.rs"],
ERA5_sg_lossless_cr["zfp.rs"],
[r"ZFP($\epsilon_{abs}$)", r"$\epsilon_{QoI,abs}$", "lossless"],
ke_eb_abs,
ERA5_zfp,
),
table_kinetic_energy(
ERA5_sg["zfp.rs"],
ERA5_sg_cr["zfp.rs"],
[r"ZFP($\epsilon_{abs}$)", r"$\epsilon_{QoI,abs}$", "one-shot"],
ke_eb_abs,
ERA5_zfp,
),
table_kinetic_energy(
ERA5_sg_it["zfp.rs"],
ERA5_sg_it_cr["zfp.rs"],
[r"ZFP($\epsilon_{abs}$)", r"$\epsilon_{QoI,abs}$", "iterative"],
ke_eb_abs,
ERA5_zfp,
),
table_kinetic_energy(
ERA5_optzconfig["zfp.rs"],
ERA5_optzconfig_cr["zfp.rs"],
[r"OptZConfig(ZFP)", r"$\epsilon_{QoI,abs}$", ""],
ke_eb_abs,
None,
),
table_kinetic_energy(
ERA5_sz3,
ERA5_sz3_cr,
[r"SZ3($\epsilon_{abs}$)", "-", ""],
ke_eb_abs,
None,
),
table_kinetic_energy(
ERA5_sg_lossless["sz3.rs"],
ERA5_sg_lossless_cr["sz3.rs"],
[r"SZ3($\epsilon_{abs}$)", r"$\epsilon_{QoI,abs}$", "lossless"],
ke_eb_abs,
ERA5_sz3,
),
table_kinetic_energy(
ERA5_sg["sz3.rs"],
ERA5_sg_cr["sz3.rs"],
[r"SZ3($\epsilon_{abs}$)", r"$\epsilon_{QoI,abs}$", "one-shot"],
ke_eb_abs,
ERA5_sz3,
),
table_kinetic_energy(
ERA5_sg_it["sz3.rs"],
ERA5_sg_it_cr["sz3.rs"],
[r"SZ3($\epsilon_{abs}$)", r"$\epsilon_{QoI,abs}$", "iterative"],
ke_eb_abs,
ERA5_sz3,
),
table_kinetic_energy(
ERA5_optzconfig["sz3.rs"],
ERA5_optzconfig_cr["sz3.rs"],
[r"OptZConfig(SZ3)", r"$\epsilon_{QoI,abs}$", ""],
ke_eb_abs,
None,
),
table_kinetic_energy(
ERA5_sperr,
ERA5_sperr_cr,
[r"SPERR($\epsilon_{abs}$)", "-", ""],
ke_eb_abs,
None,
),
table_kinetic_energy(
ERA5_sg_lossless["sperr.rs"],
ERA5_sg_lossless_cr["sperr.rs"],
[r"SPERR($\epsilon_{abs}$)", r"$\epsilon_{QoI,abs}$", "lossless"],
ke_eb_abs,
ERA5_sperr,
),
table_kinetic_energy(
ERA5_sg["sperr.rs"],
ERA5_sg_cr["sperr.rs"],
[r"SPERR($\epsilon_{abs}$)", r"$\epsilon_{QoI,abs}$", "one-shot"],
ke_eb_abs,
ERA5_sperr,
),
table_kinetic_energy(
ERA5_sg_it["sperr.rs"],
ERA5_sg_it_cr["sperr.rs"],
[r"SPERR($\epsilon_{abs}$)", r"$\epsilon_{QoI,abs}$", "iterative"],
ke_eb_abs,
ERA5_sperr,
),
table_kinetic_energy(
ERA5_optzconfig["sperr.rs"],
ERA5_optzconfig_cr["sperr.rs"],
[r"OptZConfig(SPERR)", r"$\epsilon_{QoI,abs}$", ""],
ke_eb_abs,
None,
),
table_kinetic_energy(
ERA5_qpet,
ERA5_qpet_cr,
["QPET-SPERR", r"$\epsilon_{QoI,abs}$", ""],
ke_eb_abs,
None,
),
table_kinetic_energy(
ERA5_zstd,
ERA5_zstd_cr,
["ZSTD(22)", "-", ""],
ke_eb_abs,
None,
),
]
).set_index(["Compressor", "Safeguarded", "Corrections"])
Path("tables").joinpath("kinetic-energy.tex").write_text(
ke_sg_table.to_latex(escape=False)
.replace("%", r"\%")
.replace("\\cline{1-10} \\cline{2-10}\n\\bottomrule", "\\bottomrule")
)
ke_sg_table
| $L_{\infty}(\hat{u})$ | $L_{\infty}(\hat{v})$ | $L_{\infty}(\hat{\mathrm{KE}})$ | $L_{2}(\hat{\mathrm{KE}})$ | V | C | CR | |||
|---|---|---|---|---|---|---|---|---|---|
| Compressor | Safeguarded | Corrections | |||||||
| 0 | $\epsilon_{QoI,abs}$ | lossless | 1.41 | 1.41 | 1.0 | 0.199 | 0 | 89.5% | $\times$ 2.98 |
| one-shot | 1.0 | 0.999 | 0.999 | 0.375 | 0 | 89.7% | $\times$ 8.5 | ||
| iterative | 1.41 | 1.73 | 1.0 | 0.4 | 0 | 87.1% | $\times$ 8.57 | ||
| ZFP($\epsilon_{abs}$) | - | 0.028 | 0.0349 | 1.19 | 0.103 | <0.05% | $\times$ 3.31 | ||
| $\epsilon_{QoI,abs}$ | lossless | 0.028 | 0.0349 | 0.998 | 0.103 | 0 | <0.05% | $\times$ 3.31 | |
| one-shot | 0.028 | 0.0349 | 0.926 | 0.1 | 0 | 0.1% | $\times$ 3.3 | ||
| iterative | 0.028 | 0.0349 | 0.998 | 0.103 | 0 | <0.05% | $\times$ 3.31 | ||
| OptZConfig(ZFP) | $\epsilon_{QoI,abs}$ | 0.0145 | 0.0148 | 0.641 | 0.0533 | 0 | $\times$ 2.83 | ||
| SZ3($\epsilon_{abs}$) | - | 0.125 | 0.125 | 9.29 | 0.985 | 20.6% | $\times$ 15.95 | ||
| $\epsilon_{QoI,abs}$ | lossless | 0.125 | 0.125 | 1.0 | 0.428 | 0 | 13.1% | $\times$ 5.83 | |
| one-shot | 0.125 | 0.125 | 0.998 | 0.327 | 0 | 26.8% | $\times$ 9.15 | ||
| iterative | 0.125 | 0.125 | 1.0 | 0.451 | 0 | 11.9% | $\times$ 11.27 | ||
| OptZConfig(SZ3) | $\epsilon_{QoI,abs}$ | 0.0123 | 0.0123 | 0.971 | 0.111 | 0 | $\times$ 6.82 | ||
| SPERR($\epsilon_{abs}$) | - | 0.125 | 0.125 | 8.36 | 0.62 | 9.1% | $\times$ 28.19 | ||
| $\epsilon_{QoI,abs}$ | lossless | 0.125 | 0.125 | 1.0 | 0.359 | 0 | 5.3% | $\times$ 12.3 | |
| one-shot | 0.125 | 0.125 | 0.999 | 0.277 | 0 | 13.8% | $\times$ 15.88 | ||
| iterative | 0.125 | 0.125 | 1.0 | 0.372 | 0 | 5.0% | $\times$ 20.71 | ||
| OptZConfig(SPERR) | $\epsilon_{QoI,abs}$ | 0.0143 | 0.0143 | 0.93 | 0.0888 | 0 | $\times$ 10.56 | ||
| QPET-SPERR | $\epsilon_{QoI,abs}$ | 0.0385 | 0.0386 | 0.999 | 0.128 | 0 | $\times$ 11.4 | ||
| ZSTD(22) | - | 0.0 | 0.0 | 0.0 | 0.0 | 0 | $\times$ 2.12 |
fig = earthkit.plots.Figure(
size=(10, 5),
rows=1,
columns=2,
)
chart = fig.add_map(0, 0)
# Original
da = ERA5_KE.sel(longitude=slice(180, 300))
# plot the square root of kinetic energy to better capture scale
da = np.sqrt(da)
da.attrs.update(long_name=f"sqrt({da.long_name})", units="m**1 s**-1")
# compute the default style that earthkit.maps would apply
source_original = earthkit.plots.sources.XarraySource(da)
style_original = copy.deepcopy(
earthkit.plots.styles.auto.guess_style(
source_original,
units=source_original.units,
)
)
style_original._levels = earthkit.plots.styles.levels.Levels(np.linspace(0, 50, 21))
style_original._legend_kwargs["ticks"] = np.linspace(0, 50, 5)
style_original._colors = "viridis"
style_original._legend_kwargs["extend"] = "neither"
chart.pcolormesh(da, style=style_original, zorder=-11)
t = chart.ax.text(
1 / 6,
0.9,
"Original",
ha="center",
va="top",
transform=chart.ax.transAxes,
)
t.set_bbox(dict(facecolor="white", alpha=0.75, edgecolor="black"))
chart.legend()
# # SPERR
my_ERA5_KE = compute_kinetic_energy(ERA5_sperr)
with xr.set_options(keep_attrs=True):
da = (my_ERA5_KE - ERA5_KE).compute()
da.attrs.update(long_name=f"Absolute error over {da.long_name}")
is_err = ~(np.abs(my_ERA5_KE - ERA5_KE) <= ke_eb_abs)
# compute the default style that earthkit.maps would apply
source_error = earthkit.plots.sources.XarraySource(da)
style_error = copy.deepcopy(
earthkit.plots.styles.auto.guess_style(
source_error,
units=source_error.units,
)
)
style_error._levels = earthkit.plots.styles.levels.Levels(
np.linspace(-ke_eb_abs, ke_eb_abs, 21)
)
style_error._legend_kwargs["ticks"] = np.linspace(-ke_eb_abs, ke_eb_abs, 5)
style_error._colors = "coolwarm"
style_error._legend_kwargs["extend"] = "both"
chart.pcolormesh(da.sel(longitude=slice(300, 360)), style=style_error, zorder=-11)
chart.pcolormesh(da.sel(longitude=slice(0, 60)), style=style_error, zorder=-11)
t = chart.ax.text(
0.5,
0.9,
"SPERR",
ha="center",
va="top",
transform=chart.ax.transAxes,
)
t.set_bbox(dict(facecolor="white", alpha=0.75, edgecolor="black"))
t = chart.ax.text(
0.5,
0.5,
rf"$\times$ {np.round(ERA5_sperr_cr, 2)}",
ha="center",
va="center",
transform=chart.ax.transAxes,
color="mistyrose",
fontsize=20,
)
t.set_path_effects([PathEffects.withStroke(linewidth=3, foreground="black")])
err_v_sperr = np.mean(~(np.abs(my_ERA5_KE - ERA5_KE) <= ke_eb_abs))
err_v_sperr = (
0
if err_v_sperr == 0
else np.format_float_positional(100 * err_v_sperr, precision=1, min_digits=1) + "%"
)
if err_v_sperr == "0.0%":
err_v_sperr = "<0.05%"
t = chart.ax.text(
0.5,
0.1,
f"V={err_v_sperr}",
ha="center",
va="bottom",
transform=chart.ax.transAxes,
)
t.set_bbox(dict(facecolor="white", alpha=0.75, edgecolor="black"))
# QPET-SPERR
my_ERA5_KE = compute_kinetic_energy(ERA5_qpet)
with xr.set_options(keep_attrs=True):
da = (my_ERA5_KE - ERA5_KE).compute()
da.attrs.update(long_name=f"Absolute error over {da.long_name}")
da = da.sel(longitude=slice(60, 180))
chart.pcolormesh(da, style=style_error, zorder=-11)
t = chart.ax.text(
5 / 6,
0.9,
"QPET-SPERR",
ha="center",
va="top",
transform=chart.ax.transAxes,
)
t.set_bbox(dict(facecolor="white", alpha=0.75, edgecolor="black"))
t = chart.ax.text(
5 / 6,
0.5,
rf"$\times$ {np.round(ERA5_qpet_cr, 2)}",
ha="center",
va="center",
transform=chart.ax.transAxes,
color="lightgreen",
fontsize=20,
)
t.set_path_effects([PathEffects.withStroke(linewidth=3, foreground="black")])
err_v = np.mean(~(np.abs(my_ERA5_KE - ERA5_KE) <= ke_eb_abs))
err_v = (
0
if err_v == 0
else np.format_float_positional(100 * err_v, precision=1, min_digits=1) + "%"
)
if err_v == "0.0%":
err_v = "<0.05%"
t = chart.ax.text(
5 / 6,
0.1,
f"V={err_v}",
ha="center",
va="bottom",
transform=chart.ax.transAxes,
)
t.set_bbox(dict(facecolor="white", alpha=0.75, edgecolor="black"))
chart.ax.set_rasterization_zorder(-10)
chart.ax.axvline(-60, c="white", ls=(2, (4, 4)), lw=2)
chart.ax.axvline(-60, c="black", ls=(6, (4, 4)), lw=2)
chart.ax.axvline(+60, c="white", ls=(2, (4, 4)), lw=2)
chart.ax.axvline(+60, c="black", ls=(6, (4, 4)), lw=2)
chart.title("Without safeguards")
for m in earthkit.plots.schemas.schema.quickmap_subplot_workflow:
if m != "title":
getattr(chart, m)()
for m in earthkit.plots.schemas.schema.quickmap_figure_workflow:
if m != "legend":
getattr(chart, m)()
chart = fig.add_map(0, 1)
# Corrections and Errors
is_corr = (ERA5_sg["sperr.rs"]["u"] != ERA5_sperr["u"]) | (
ERA5_sg["sperr.rs"]["v"] != ERA5_sperr["v"]
)
is_corr_it = (ERA5_sg_it["sperr.rs"]["u"] != ERA5_sperr["u"]) | (
ERA5_sg_it["sperr.rs"]["v"] != ERA5_sperr["v"]
)
chart.pcolormesh(
is_corr.sel(longitude=slice(300, 340)),
no_style=True,
cmap=mpl.colors.ListedColormap(["white", "green"]),
zorder=-11,
legend_style=None,
)
chart.pcolormesh(
is_err.sel(longitude=slice(340, 360)),
no_style=True,
cmap=mpl.colors.ListedColormap(["white", "red"]),
zorder=-11,
legend_style=None,
)
chart.pcolormesh(
is_err.sel(longitude=slice(0, 20)),
no_style=True,
cmap=mpl.colors.ListedColormap(["white", "red"]),
zorder=-11,
legend_style=None,
)
chart.pcolormesh(
is_corr_it.sel(longitude=slice(20, 60)),
no_style=True,
cmap=mpl.colors.ListedColormap(["white", "limegreen"]),
zorder=-11,
legend_style=None,
)
t = chart.ax.text(
0.5,
0.9,
"SPERR",
ha="center",
va="top",
transform=chart.ax.transAxes,
)
t.set_bbox(dict(facecolor="white", alpha=0.75, edgecolor="black"))
t = chart.ax.text(
0.5,
0.5,
f"V={err_v_sperr}",
ha="center",
va="center",
transform=chart.ax.transAxes,
rotation=90,
)
t.set_bbox(dict(facecolor="white", alpha=0.75, edgecolor="black"))
corr = compute_corrections_percentage(ERA5_sg["sperr.rs"], ERA5_sperr)
corr = (
0
if corr == 0
else np.format_float_positional(100 * corr, precision=1, min_digits=1) + "%"
)
if corr == "0.0%":
corr = "<0.05%"
t = chart.ax.text(
7 / 18,
0.9,
"Sg",
ha="center",
va="top",
transform=chart.ax.transAxes,
)
t.set_bbox(dict(facecolor="white", alpha=0.75, edgecolor="black"))
t = chart.ax.text(
7 / 18,
0.5,
f"C={corr}",
ha="center",
va="center",
transform=chart.ax.transAxes,
rotation=90,
)
t.set_bbox(dict(facecolor="white", alpha=0.75, edgecolor="black"))
corr_it = compute_corrections_percentage(ERA5_sg_it["sperr.rs"], ERA5_sperr)
corr_it = (
0
if corr_it == 0
else np.format_float_positional(100 * corr_it, precision=1, min_digits=1) + "%"
)
if corr_it == "0.0%":
corr_it = "<0.05%"
t = chart.ax.text(
11 / 18,
0.9,
"Sg[it]",
ha="center",
va="top",
transform=chart.ax.transAxes,
)
t.set_bbox(dict(facecolor="white", alpha=0.75, edgecolor="black"))
t = chart.ax.text(
11 / 18,
0.5,
f"C={corr_it}",
ha="center",
va="center",
transform=chart.ax.transAxes,
rotation=90,
)
t.set_bbox(dict(facecolor="white", alpha=0.75, edgecolor="black"))
# Safeguarded(SPERR)
my_ERA5_KE = compute_kinetic_energy(ERA5_sg["sperr.rs"])
with xr.set_options(keep_attrs=True):
da = (my_ERA5_KE - ERA5_KE).compute()
da.attrs.update(long_name=f"Absolute error over {da.long_name}")
da = da.sel(longitude=slice(180, 300))
chart.pcolormesh(da, style=style_error, zorder=-11)
t = chart.ax.text(
1 / 6,
0.9,
"Sg(SPERR)",
ha="center",
va="top",
transform=chart.ax.transAxes,
)
t.set_bbox(dict(facecolor="white", alpha=0.75, edgecolor="black"))
t = chart.ax.text(
1 / 6,
0.5,
rf"$\times$ {np.round(ERA5_sg_cr['sperr.rs'], 2)}",
ha="center",
va="center",
transform=chart.ax.transAxes,
color="lightgreen",
fontsize=20,
)
t.set_path_effects([PathEffects.withStroke(linewidth=3, foreground="black")])
err_v = np.mean(~(np.abs(my_ERA5_KE - ERA5_KE) <= ke_eb_abs))
err_v = (
0
if err_v == 0
else np.format_float_positional(100 * err_v, precision=1, min_digits=1) + "%"
)
if err_v == "0.0%":
err_v = "<0.05%"
t = chart.ax.text(
1 / 6,
0.1,
f"V={err_v}",
ha="center",
va="bottom",
transform=chart.ax.transAxes,
)
t.set_bbox(dict(facecolor="white", alpha=0.75, edgecolor="black"))
chart.legend(extend="neither")
# Safeguarded[it](SPERR)
my_ERA5_KE = compute_kinetic_energy(ERA5_sg_it["sperr.rs"])
with xr.set_options(keep_attrs=True):
da = (my_ERA5_KE - ERA5_KE).compute()
da.attrs.update(long_name=f"Absolute error over {da.long_name}")
da = da.sel(longitude=slice(60, 180))
chart.pcolormesh(da, style=style_error, zorder=-11)
t = chart.ax.text(
5 / 6,
0.9,
"Sg[it](SPERR)",
ha="center",
va="top",
transform=chart.ax.transAxes,
)
t.set_bbox(dict(facecolor="white", alpha=0.75, edgecolor="black"))
t = chart.ax.text(
5 / 6,
0.5,
rf"$\times$ {np.round(ERA5_sg_it_cr['sperr.rs'], 2)}",
ha="center",
va="center",
transform=chart.ax.transAxes,
color="lightgreen",
fontsize=20,
)
t.set_path_effects([PathEffects.withStroke(linewidth=3, foreground="black")])
err_v = np.mean(~(np.abs(my_ERA5_KE - ERA5_KE) <= ke_eb_abs))
err_v = (
0
if err_v == 0
else np.format_float_positional(100 * err_v, precision=1, min_digits=1) + "%"
)
if err_v == "0.0%":
err_v = "<0.05%"
t = chart.ax.text(
5 / 6,
0.1,
f"V={err_v}",
ha="center",
va="bottom",
transform=chart.ax.transAxes,
)
t.set_bbox(dict(facecolor="white", alpha=0.75, edgecolor="black"))
chart.ax.set_rasterization_zorder(-10)
chart.ax.axvline(-20, c="white", ls=(2, (4, 4)), lw=2)
chart.ax.axvline(-20, c="black", ls=(6, (4, 4)), lw=2)
chart.ax.axvline(+20, c="white", ls=(2, (4, 4)), lw=2)
chart.ax.axvline(+20, c="black", ls=(6, (4, 4)), lw=2)
chart.ax.axvline(-60, c="black", lw=1)
chart.ax.axvline(+60, c="black", lw=1)
chart.title("Safeguarded: preserve kinetic energy")
for m in earthkit.plots.schemas.schema.quickmap_subplot_workflow:
if m != "title":
getattr(chart, m)()
for m in earthkit.plots.schemas.schema.quickmap_figure_workflow:
if m != "legend":
getattr(chart, m)()
fig.save(Path("plots") / "kinetic-energy-summary.pdf")
Warning: Style not set. Warning: Style not set. Warning: Style not set. Warning: Style not set.
import json
with Path("observations").joinpath("kinetic-energy.json").open("w") as f:
json.dump(observations, f)