ESA CCI CMIP7 recipe example¶

Shows a bundled CMIP7 plugin recipe for a tiny ESA CCI water-vapour dataset.

In [1]:

import numpy as np
import woodpecker_cmip7_plugin  # noqa: F401 - imports plugin fixes for editable installs

import woodpecker
from woodpecker.testing import make_cmip7

import numpy as np import woodpecker_cmip7_plugin # noqa: F401 - imports plugin fixes for editable installs import woodpecker from woodpecker.testing import make_cmip7

Create an ESA CCI-like input with names, metadata, and latitude order that need repair.

In [2]:

source_name = "ESACCI-WATERVAPOUR-L3C-TCWV-meris-005deg-2002-2017-fv3.2.zarr"
dataset = make_cmip7(variable="prw", overrides={"source_name": source_name}, seed=7)
dataset = dataset.isel(lat=slice(None, None, -1))
dataset = dataset.assign_coords(bnds=[0, 1])
dataset["lat_bnds"] = (
    ("lat", "bnds"),
    np.column_stack([dataset["lat"].values - 0.5, dataset["lat"].values + 0.5]),
)

dataset

source_name = "ESACCI-WATERVAPOUR-L3C-TCWV-meris-005deg-2002-2017-fv3.2.zarr" dataset = make_cmip7(variable="prw", overrides={"source_name": source_name}, seed=7) dataset = dataset.isel(lat=slice(None, None, -1)) dataset = dataset.assign_coords(bnds=[0, 1]) dataset["lat_bnds"] = ( ("lat", "bnds"), np.column_stack([dataset["lat"].values - 0.5, dataset["lat"].values + 0.5]), ) dataset

Out[2]:

<xarray.Dataset> Size: 32kB
Dimensions:   (time: 12, lat: 18, lon: 36, bnds: 2)
Coordinates:
  * time      (time) datetime64[s] 96B 2000-01-01 2000-02-01 ... 2000-12-01
  * lat       (lat) float64 144B 85.0 75.0 65.0 55.0 ... -55.0 -65.0 -75.0 -85.0
  * lon       (lon) float64 288B 0.0 10.0 20.0 30.0 ... 320.0 330.0 340.0 350.0
  * bnds      (bnds) int64 16B 0 1
Data variables:
    prw       (time, lat, lon) float32 31kB 2.101 2.13 2.169 ... 1.913 1.942
    lat_bnds  (lat, bnds) float64 288B 84.5 85.5 74.5 75.5 ... -74.5 -85.5 -84.5
Attributes: (12/15)
    project_id:          CMIP7
    dataset_id:          CMIP7.CMIP.MOHC.UKESM2-1.historical.r1i1p1f1.Amon.pr...
    source_file:         CMIP7.CMIP.MOHC.UKESM2-1.historical.r1i1p1f1.Amon.pr...
    source_id:           UKESM2-1
    source_name:         ESACCI-WATERVAPOUR-L3C-TCWV-meris-005deg-2002-2017-f...
    mip_era:             CMIP7
    ...                  ...
    variable_id:         prw
    table_id:            Amon
    units:               1
    frequency:           mon
    grid_label:          gn
    nominal_resolution:  100 km

xarray.Dataset

• Dimensions:
  • time: 12
  • lat: 18
  • lon: 36
  • bnds: 2
• Coordinates: (4)
  • time
    (time)
    datetime64[s]
    2000-01-01 ... 2000-12-01

    axis :

    T

    standard_name :

    time

    array(['2000-01-01T00:00:00', '2000-02-01T00:00:00', '2000-03-01T00:00:00',
           '2000-04-01T00:00:00', '2000-05-01T00:00:00', '2000-06-01T00:00:00',
           '2000-07-01T00:00:00', '2000-08-01T00:00:00', '2000-09-01T00:00:00',
           '2000-10-01T00:00:00', '2000-11-01T00:00:00', '2000-12-01T00:00:00'],
          dtype='datetime64[s]')

  • lat
    (lat)
    float64
    85.0 75.0 65.0 ... -75.0 -85.0

    axis :

    Y

    standard_name :

    latitude

    units :

    degrees_north

    array([ 85.,  75.,  65.,  55.,  45.,  35.,  25.,  15.,   5.,  -5., -15., -25.,
           -35., -45., -55., -65., -75., -85.])

  • lon
    (lon)
    float64
    0.0 10.0 20.0 ... 330.0 340.0 350.0

    axis :

    X

    standard_name :

    longitude

    units :

    degrees_east

    array([  0.,  10.,  20.,  30.,  40.,  50.,  60.,  70.,  80.,  90., 100., 110.,
           120., 130., 140., 150., 160., 170., 180., 190., 200., 210., 220., 230.,
           240., 250., 260., 270., 280., 290., 300., 310., 320., 330., 340., 350.])

  • bnds
    (bnds)
    int64
    0 1

    array([0, 1])

• Data variables: (2)
  • prw
    (time, lat, lon)
    float32
    2.101 2.13 2.169 ... 1.913 1.942

    standard_name :

    prw

    units :

    1

    array([[[2.1006863, 2.1299899, 2.1685736, ..., 2.1554322, 2.1553924,
             2.1069613],
            [2.3234065, 2.3564806, 2.451858 , ..., 2.371485 , 2.4330938,
             2.4072785],
            [2.6385028, 2.6414766, 2.6260262, ..., 2.5073013, 2.6527877,
             2.5593188],
            ...,
            [2.6107688, 2.6290631, 2.696778 , ..., 2.6027303, 2.6441977,
             2.6585782],
            [2.3866024, 2.4324481, 2.3590486, ..., 2.3484964, 2.4205737,
             2.2886076],
            [2.1307952, 2.145671 , 2.1170268, ..., 2.0902917, 2.1837785,
             2.0903568]],
    
           [[2.3426175, 2.3520176, 2.2954812, ..., 2.2883632, 2.3923256,
             2.3600202],
            [2.6209102, 2.6512473, 2.5702348, ..., 2.6896303, 2.5656567,
             2.5265803],
            [2.8065405, 2.845276 , 2.865964 , ..., 2.899191 , 2.8452604,
             2.7579186],
    ...
            [2.2679636, 2.3006077, 2.257001 , ..., 2.3265393, 2.2625225,
             2.3339534],
            [1.9898797, 2.108566 , 1.9793998, ..., 2.0616384, 2.0563881,
             2.011986 ],
            [1.8067367, 1.8236928, 1.8712323, ..., 1.8635817, 1.7452581,
             1.758769 ]],
    
           [[1.9013596, 1.918799 , 1.8729172, ..., 1.9153534, 1.9674724,
             1.9585377],
            [2.1345463, 2.1565955, 2.2150626, ..., 2.1671193, 2.2132986,
             2.2707636],
            [2.3905864, 2.4242573, 2.4238603, ..., 2.4202633, 2.3560967,
             2.5040846],
            ...,
            [2.3824544, 2.4487472, 2.3498225, ..., 2.4324057, 2.45668  ,
             2.3432837],
            [2.188815 , 2.2785237, 2.2177572, ..., 2.1643305, 2.1699286,
             2.131081 ],
            [1.9431614, 1.9427598, 1.9074578, ..., 1.8943526, 1.9125551,
             1.9416133]]], shape=(12, 18, 36), dtype=float32)

  • lat_bnds
    (lat, bnds)
    float64
    84.5 85.5 74.5 ... -85.5 -84.5

    array([[ 84.5,  85.5],
           [ 74.5,  75.5],
           [ 64.5,  65.5],
           [ 54.5,  55.5],
           [ 44.5,  45.5],
           [ 34.5,  35.5],
           [ 24.5,  25.5],
           [ 14.5,  15.5],
           [  4.5,   5.5],
           [ -5.5,  -4.5],
           [-15.5, -14.5],
           [-25.5, -24.5],
           [-35.5, -34.5],
           [-45.5, -44.5],
           [-55.5, -54.5],
           [-65.5, -64.5],
           [-75.5, -74.5],
           [-85.5, -84.5]])

• Attributes: (15)

  project_id :

  CMIP7

  dataset_id :

  CMIP7.CMIP.MOHC.UKESM2-1.historical.r1i1p1f1.Amon.prw.gn.v20260101

  source_file :

  CMIP7.CMIP.MOHC.UKESM2-1.historical.r1i1p1f1.Amon.prw.gn.v20260101.nc

  source_id :

  UKESM2-1

  source_name :

  ESACCI-WATERVAPOUR-L3C-TCWV-meris-005deg-2002-2017-fv3.2.zarr

  mip_era :

  CMIP7

  activity_id :

  CMIP

  experiment_id :

  historical

  variant_label :

  r1i1p1f1

  variable_id :

  prw

  table_id :

  Amon

  units :

  1

  frequency :

  mon

  grid_label :

  gn

  nominal_resolution :

  100 km

Load the bundled ESA CCI recipe and inspect its match rules and fix steps.

In [3]:

recipe = woodpecker.recipe.get("cmip7.esa_cci_water_vapour_zarr")

recipe.model_dump()

recipe = woodpecker.recipe.get("cmip7.esa_cci_water_vapour_zarr") recipe.model_dump()

Out[3]:

{'id': 'cmip7.esa_cci_water_vapour_zarr',
 'aliases': [],
 'description': 'CMIP7/ESA CCI zarr-style inputs',
 'match': {'attrs': {},
  'dataset_id_patterns': [],
  'path_patterns': ['*ESACCI-WATERVAPOUR-*.zarr']},
 'steps': [{'id': 'cmip7.configurable_reformat_bridge',
   'phase': 'apply',
   'options': {'realm': 'atmos',
    'branded_variable': 'prw_tavg-u-hxy-u',
    'dim_map': {'bnds': 'nv'},
    'variable_map': {'prw': 'tcwv'},
    'keep_global_attrs': True},
   'links': []},
  {'id': 'woodpecker.ensure_latitude_is_increasing',
   'phase': 'apply',
   'options': {},
   'links': []}],
 'links': [{'rel': 'self',
   'href': 'https://github.com/roocs/woodpecker/blob/main/plugins/woodpecker-cmip7-plugin/src/woodpecker_cmip7_plugin/recipes/esa_cci_water_vapour_recipe.json',
   'title': 'Source recipe document'},
  {'rel': 'issue',
   'href': 'https://github.com/roocs/woodpecker/issues',
   'title': 'Track ESA CCI fix discussions'}]}

In [4]:

recipe.match.model_dump(), [step.id for step in recipe.steps]

recipe.match.model_dump(), [step.id for step in recipe.steps]

Out[4]:

({'attrs': {},
  'dataset_id_patterns': [],
  'path_patterns': ['*ESACCI-WATERVAPOUR-*.zarr']},
 ['cmip7.configurable_reformat_bridge',
  'woodpecker.ensure_latitude_is_increasing'])

In [5]:

findings = woodpecker.recipe.check(dataset, recipe)
findings.fix_ids

findings = woodpecker.recipe.check(dataset, recipe) findings.fix_ids

Out[5]:

('cmip7.configurable_reformat_bridge',
 'woodpecker.ensure_latitude_is_increasing')

Dry-run previews the repair without changing the dataset.

In [6]:

result = woodpecker.recipe.apply(dataset, recipe, dry_run=True)

result.stats, result.preview, tuple(dataset.data_vars), tuple(dataset.dims)

result = woodpecker.recipe.apply(dataset, recipe, dry_run=True) result.stats, result.preview, tuple(dataset.data_vars), tuple(dataset.dims)

Out[6]:

({'attempted': 2,
  'changed': 2,
  'persist_attempted': 0,
  'persisted': 0,
  'persist_failed': 0,
  'preview': [{'path': 'ESACCI-WATERVAPOUR-L3C-TCWV-meris-005deg-2002-2017-fv3.2.zarr',
    'fix_id': 'cmip7.configurable_reformat_bridge',
    'name': 'Configurable CMIP7 reformat bridge (plugin)',
    'labels': ['risk.workflow_transformation'],
    'label_titles': ['careful: workflow transformation'],
    'label_metadata': [{'id': 'risk.workflow_transformation',
      'title': 'careful: workflow transformation',
      'description': 'Applies a composed workflow with structural and metadata changes.',
      'category': 'risk-high'}],
    'changed': True},
   {'path': 'ESACCI-WATERVAPOUR-L3C-TCWV-meris-005deg-2002-2017-fv3.2.zarr',
    'fix_id': 'woodpecker.ensure_latitude_is_increasing',
    'name': 'Ensure latitude is increasing',
    'labels': ['risk.coordinate_reordering'],
    'label_titles': ['careful: coordinate reordering'],
    'label_metadata': [{'id': 'risk.coordinate_reordering',
      'title': 'careful: coordinate reordering',
      'description': 'Reorders coordinate-dependent data.',
      'category': 'risk-medium'}],
    'changed': True}]},
 ({'path': 'ESACCI-WATERVAPOUR-L3C-TCWV-meris-005deg-2002-2017-fv3.2.zarr',
   'fix_id': 'cmip7.configurable_reformat_bridge',
   'name': 'Configurable CMIP7 reformat bridge (plugin)',
   'labels': ['risk.workflow_transformation'],
   'label_titles': ['careful: workflow transformation'],
   'label_metadata': [{'id': 'risk.workflow_transformation',
     'title': 'careful: workflow transformation',
     'description': 'Applies a composed workflow with structural and metadata changes.',
     'category': 'risk-high'}],
   'changed': True},
  {'path': 'ESACCI-WATERVAPOUR-L3C-TCWV-meris-005deg-2002-2017-fv3.2.zarr',
   'fix_id': 'woodpecker.ensure_latitude_is_increasing',
   'name': 'Ensure latitude is increasing',
   'labels': ['risk.coordinate_reordering'],
   'label_titles': ['careful: coordinate reordering'],
   'label_metadata': [{'id': 'risk.coordinate_reordering',
     'title': 'careful: coordinate reordering',
     'description': 'Reorders coordinate-dependent data.',
     'category': 'risk-medium'}],
   'changed': True}),
 ('prw', 'lat_bnds'),
 ('time', 'lat', 'lon', 'bnds'))

Apply the recipe in memory and re-check.

In [7]:

write = woodpecker.recipe.apply(dataset, recipe, dry_run=False)

(
    write.stats,
    tuple(dataset.data_vars),
    tuple(dataset.dims),
    dataset.attrs["realm"],
    dataset.attrs["branded_variable"],
    float(dataset["lat"].values[0]) < float(dataset["lat"].values[-1]),
)

write = woodpecker.recipe.apply(dataset, recipe, dry_run=False) ( write.stats, tuple(dataset.data_vars), tuple(dataset.dims), dataset.attrs["realm"], dataset.attrs["branded_variable"], float(dataset["lat"].values[0]) < float(dataset["lat"].values[-1]), )

Out[7]:

({'attempted': 2,
  'changed': 2,
  'persist_attempted': 1,
  'persisted': 1,
  'persist_failed': 0,
  'preview': [{'path': 'ESACCI-WATERVAPOUR-L3C-TCWV-meris-005deg-2002-2017-fv3.2.zarr',
    'fix_id': 'cmip7.configurable_reformat_bridge',
    'name': 'Configurable CMIP7 reformat bridge (plugin)',
    'labels': ['risk.workflow_transformation'],
    'label_titles': ['careful: workflow transformation'],
    'label_metadata': [{'id': 'risk.workflow_transformation',
      'title': 'careful: workflow transformation',
      'description': 'Applies a composed workflow with structural and metadata changes.',
      'category': 'risk-high'}],
    'changed': True},
   {'path': 'ESACCI-WATERVAPOUR-L3C-TCWV-meris-005deg-2002-2017-fv3.2.zarr',
    'fix_id': 'woodpecker.ensure_latitude_is_increasing',
    'name': 'Ensure latitude is increasing',
    'labels': ['risk.coordinate_reordering'],
    'label_titles': ['careful: coordinate reordering'],
    'label_metadata': [{'id': 'risk.coordinate_reordering',
      'title': 'careful: coordinate reordering',
      'description': 'Reorders coordinate-dependent data.',
      'category': 'risk-medium'}],
    'changed': True}]},
 ('lat_bnds', 'tcwv'),
 ('time', 'lon', 'lat', 'nv'),
 'atmos',
 'prw_tavg-u-hxy-u',
 True)

In [8]:

recheck = woodpecker.recipe.check(dataset, recipe)
bool(recheck)

recheck = woodpecker.recipe.check(dataset, recipe) bool(recheck)

Out[8]:

False