xMIP plugin demo¶

Shows an xMIP-derived plugin recipe as small, inspectable Woodpecker fixes composed into one workflow.

In [1]:

import numpy as np
import woodpecker_xmip_plugin  # noqa: F401 - imports plugin fixes for editable installs

import woodpecker
from woodpecker.testing import make_cmip6

import numpy as np import woodpecker_xmip_plugin # noqa: F401 - imports plugin fixes for editable installs import woodpecker from woodpecker.testing import make_cmip6

Create a tiny CMIP6 dataset and add xMIP-style issues: i/j axes, longitude/latitude names, negative longitudes, centimeter vertical units, and GFDL-CM4 metadata.

In [2]:

dataset = make_cmip6(
    overrides={"source_id": "GFDL-CM4", "experiment_id": "historical"},
    seed=7,
)
dataset = dataset.isel(time=slice(0, 2), lat=slice(0, 2), lon=slice(0, 3))
dataset = dataset.rename({"lat": "j", "lon": "i"})
dataset = dataset.expand_dims({"lev": [0.0, 100.0]})
dataset["lev"].attrs["units"] = "centimeters"

longitude = np.broadcast_to(np.array([-10.0, 0.0, 10.0]), (dataset.sizes["j"], 3))
latitude = np.broadcast_to(
    np.asarray(dataset["j"].values)[:, None],
    (dataset.sizes["j"], dataset.sizes["i"]),
)
dataset["longitude"] = (("j", "i"), longitude)
dataset["latitude"] = (("j", "i"), latitude)

dataset

dataset = make_cmip6( overrides={"source_id": "GFDL-CM4", "experiment_id": "historical"}, seed=7, ) dataset = dataset.isel(time=slice(0, 2), lat=slice(0, 2), lon=slice(0, 3)) dataset = dataset.rename({"lat": "j", "lon": "i"}) dataset = dataset.expand_dims({"lev": [0.0, 100.0]}) dataset["lev"].attrs["units"] = "centimeters" longitude = np.broadcast_to(np.array([-10.0, 0.0, 10.0]), (dataset.sizes["j"], 3)) latitude = np.broadcast_to( np.asarray(dataset["j"].values)[:, None], (dataset.sizes["j"], dataset.sizes["i"]), ) dataset["longitude"] = (("j", "i"), longitude) dataset["latitude"] = (("j", "i"), latitude) dataset

Out[2]:

<xarray.Dataset> Size: 264B
Dimensions:    (lev: 2, time: 2, j: 2, i: 3)
Coordinates:
  * lev        (lev) float64 16B 0.0 100.0
  * time       (time) datetime64[s] 16B 2000-01-01 2000-02-01
  * j          (j) float64 16B -85.0 -75.0
  * i          (i) float64 24B 0.0 10.0 20.0
Data variables:
    tas        (lev, time, j, i) float32 96B 251.3 251.4 251.5 ... 255.5 255.7
    longitude  (j, i) float64 48B -10.0 0.0 10.0 -10.0 0.0 10.0
    latitude   (j, i) float64 48B -85.0 -85.0 -85.0 -75.0 -75.0 -75.0
Attributes: (12/15)
    project_id:          CMIP6
    dataset_id:          CMIP6.CMIP.MOHC.HadGEM3-GC31-LL.historical.r1i1p1f3....
    source_file:         CMIP6.CMIP.MOHC.HadGEM3-GC31-LL.historical.r1i1p1f3....
    source_id:           GFDL-CM4
    source_name:         HadGEM3-GC31-LL
    mip_era:             CMIP6
    ...                  ...
    variable_id:         tas
    table_id:            Amon
    units:               K
    frequency:           mon
    grid_label:          gn
    nominal_resolution:  250 km

xarray.Dataset

• Dimensions:
  • lev: 2
  • time: 2
  • j: 2
  • i: 3
• Coordinates: (4)
  • lev
    (lev)
    float64
    0.0 100.0

    units :

    centimeters

    array([  0., 100.])

  • time
    (time)
    datetime64[s]
    2000-01-01 2000-02-01

    axis :

    T

    standard_name :

    time

    array(['2000-01-01T00:00:00', '2000-02-01T00:00:00'], dtype='datetime64[s]')

  • j
    (j)
    float64
    -85.0 -75.0

    axis :

    Y

    standard_name :

    latitude

    units :

    degrees_north

    array([-85., -75.])

  • i
    (i)
    float64
    0.0 10.0 20.0

    axis :

    X

    standard_name :

    longitude

    units :

    degrees_east

    array([ 0., 10., 20.])

• Data variables: (3)
  • tas
    (lev, time, j, i)
    float32
    251.3 251.4 251.5 ... 255.5 255.7

    standard_name :

    air_temperature

    units :

    K

    array([[[[251.25006, 251.40385, 251.5099 ],
             [256.74838, 256.93314, 256.99445]],
    
            [[249.9617 , 250.03717, 250.2387 ],
             [255.39217, 255.49796, 255.74046]]],
    
    
           [[[251.25006, 251.40385, 251.5099 ],
             [256.74838, 256.93314, 256.99445]],
    
            [[249.9617 , 250.03717, 250.2387 ],
             [255.39217, 255.49796, 255.74046]]]], dtype=float32)

  • longitude
    (j, i)
    float64
    -10.0 0.0 10.0 -10.0 0.0 10.0

    array([[-10.,   0.,  10.],
           [-10.,   0.,  10.]])

  • latitude
    (j, i)
    float64
    -85.0 -85.0 -85.0 -75.0 -75.0 -75.0

    array([[-85., -85., -85.],
           [-75., -75., -75.]])

• Attributes: (15)

  project_id :

  CMIP6

  dataset_id :

  CMIP6.CMIP.MOHC.HadGEM3-GC31-LL.historical.r1i1p1f3.Amon.tas.gn.v20200101

  source_file :

  CMIP6.CMIP.MOHC.HadGEM3-GC31-LL.historical.r1i1p1f3.Amon.tas.gn.v20200101.nc

  source_id :

  GFDL-CM4

  source_name :

  HadGEM3-GC31-LL

  mip_era :

  CMIP6

  activity_id :

  CMIP

  experiment_id :

  historical

  variant_label :

  r1i1p1f3

  variable_id :

  tas

  table_id :

  Amon

  units :

  K

  frequency :

  mon

  grid_label :

  gn

  nominal_resolution :

  250 km

In [3]:

{
    "dims": dict(dataset.sizes),
    "data_vars": list(dataset.data_vars),
    "coords": list(dataset.coords),
    "lev_units": dataset["lev"].attrs.get("units"),
    "longitude_min": float(dataset["longitude"].min()),
    "branch_time_in_parent": dataset.attrs.get("branch_time_in_parent"),
}

{ "dims": dict(dataset.sizes), "data_vars": list(dataset.data_vars), "coords": list(dataset.coords), "lev_units": dataset["lev"].attrs.get("units"), "longitude_min": float(dataset["longitude"].min()), "branch_time_in_parent": dataset.attrs.get("branch_time_in_parent"), }

Out[3]:

{'dims': {'lev': 2, 'time': 2, 'j': 2, 'i': 3},
 'data_vars': ['tas', 'longitude', 'latitude'],
 'coords': ['lev', 'time', 'j', 'i'],
 'lev_units': 'centimeters',
 'longitude_min': -10.0,
 'branch_time_in_parent': None}

Load the bundled xmip.cmip6_preprocessing recipe.

In [4]:

recipe = woodpecker.recipe.get("xmip.cmip6_preprocessing")

recipe.model_dump()

recipe = woodpecker.recipe.get("xmip.cmip6_preprocessing") recipe.model_dump()

Out[4]:

{'id': 'xmip.cmip6_preprocessing',
 'aliases': [],
 'description': 'Woodpecker-native xMIP-style CMIP6 preprocessing. This recipe composes small, inspectable fixes instead of hiding the workflow in one monolithic preprocessing function.',
 'match': {'attrs': {'project_id': 'CMIP6'},
  'dataset_id_patterns': [],
  'path_patterns': []},
 'steps': [{'id': 'woodpecker.rename_variables',
   'phase': 'apply',
   'options': {'mapping': {'x': ['i', 'ni', 'xh', 'nlon'],
     'y': ['j', 'nj', 'yh', 'nlat'],
     'lev': ['deptht', 'olevel', 'zlev', 'olev', 'depth'],
     'bnds': ['bnds', 'axis_nbounds', 'd2'],
     'vertex': ['vertex', 'nvertex', 'vertices', 'nvertices'],
     'lon': ['longitude', 'nav_lon'],
     'lat': ['latitude', 'nav_lat'],
     'lev_bounds': ['deptht_bounds', 'lev_bnds', 'olevel_bounds', 'zlev_bnds'],
     'lon_bounds': ['bounds_lon',
      'bounds_nav_lon',
      'lon_bnds',
      'x_bnds',
      'vertices_longitude',
      'longitude_bnds'],
     'lat_bounds': ['bounds_lat',
      'bounds_nav_lat',
      'lat_bnds',
      'y_bnds',
      'vertices_latitude',
      'latitude_bnds'],
     'time_bounds': ['time_bnds']}},
   'links': []},
  {'id': 'woodpecker.promote_missing_dimension_coords',
   'phase': 'apply',
   'options': {},
   'links': []},
  {'id': 'woodpecker.set_coordinate_variables',
   'phase': 'apply',
   'options': {'coordinates': ['x',
     'y',
     'lon',
     'lat',
     'lev',
     'bnds',
     'lev_bounds',
     'lon_bounds',
     'lat_bounds',
     'time_bounds',
     'lat_verticies',
     'lon_verticies']},
   'links': []},
  {'id': 'xmip.broadcast_lon_lat',
   'phase': 'apply',
   'options': {},
   'links': []},
  {'id': 'woodpecker.normalize_longitude_convention',
   'phase': 'apply',
   'options': {'coordinate': 'lon',
    'target': '0_360',
    'bounds': ['lon_bounds'],
    'mask_abs_gt': 1000},
   'links': []},
  {'id': 'woodpecker.convert_units',
   'phase': 'apply',
   'options': {'units': {'lev': 'm'}, 'overrides': {'so': None}},
   'links': []},
  {'id': 'xmip.normalize_lon_lat_bounds',
   'phase': 'apply',
   'options': {},
   'links': []},
  {'id': 'xmip.sort_vertex_order',
   'phase': 'apply',
   'options': {},
   'links': []},
  {'id': 'xmip.convert_bounds_to_vertices',
   'phase': 'apply',
   'options': {},
   'links': []},
  {'id': 'xmip.convert_vertices_to_bounds',
   'phase': 'apply',
   'options': {},
   'links': []},
  {'id': 'xmip.fix_known_cmip6_metadata',
   'phase': 'apply',
   'options': {},
   'links': []},
  {'id': 'woodpecker.drop_variables',
   'phase': 'apply',
   'options': {'variables': ['bnds', 'vertex'], 'errors': 'ignore'},
   'links': []}],
 'links': [{'rel': 'source',
   'href': 'https://github.com/jbusecke/xMIP',
   'title': 'Original xMIP project'}]}

Check and dry-run the recipe. Dry-run checks do not simulate earlier steps, so the first report only includes fixes visible on the original dataset.

In [5]:

findings = woodpecker.recipe.check(dataset, recipe)

findings.fix_ids

findings = woodpecker.recipe.check(dataset, recipe) findings.fix_ids

/home/runner/work/woodpecker/woodpecker/woodpecker/fixes/common/common_0005.py:223: UserWarning: Import(s) unavailable to set up matplotlib support...skipping this portion of the setup.
  import cf_xarray.units  # noqa: F401

Out[5]:

('woodpecker.rename_variables',
 'woodpecker.convert_units',
 'xmip.fix_known_cmip6_metadata')

In [6]:

result = woodpecker.recipe.apply(dataset, recipe, dry_run=True)

result.stats, result.preview

result = woodpecker.recipe.apply(dataset, recipe, dry_run=True) result.stats, result.preview

Out[6]:

({'attempted': 3,
  'changed': 3,
  'persist_attempted': 0,
  'persisted': 0,
  'persist_failed': 0,
  'preview': [{'path': 'HadGEM3-GC31-LL',
    'fix_id': 'woodpecker.rename_variables',
    'name': 'Rename variables and dimensions',
    'labels': ['risk.reversible_rename'],
    'label_titles': ['safe: reversible rename'],
    'label_metadata': [{'id': 'risk.reversible_rename',
      'title': 'safe: reversible rename',
      'description': 'Renames variables, coordinates, or dimensions without changing values.',
      'category': 'risk-low'}],
    'changed': True},
   {'path': 'HadGEM3-GC31-LL',
    'fix_id': 'woodpecker.convert_units',
    'name': 'Convert variable units',
    'labels': ['risk.value_transformation'],
    'label_titles': ['careful: value transformation'],
    'label_metadata': [{'id': 'risk.value_transformation',
      'title': 'careful: value transformation',
      'description': 'Transforms data or coordinate values.',
      'category': 'risk-medium'}],
    'changed': True},
   {'path': 'HadGEM3-GC31-LL',
    'fix_id': 'xmip.fix_known_cmip6_metadata',
    'name': 'Fix known CMIP6 metadata',
    'labels': ['risk.metadata_only'],
    'label_titles': ['safe: metadata only'],
    'label_metadata': [{'id': 'risk.metadata_only',
      'title': 'safe: metadata only',
      'description': 'Changes metadata without changing data values.',
      'category': 'risk-low'}],
    'changed': True}]},
 ({'path': 'HadGEM3-GC31-LL',
   'fix_id': 'woodpecker.rename_variables',
   'name': 'Rename variables and dimensions',
   'labels': ['risk.reversible_rename'],
   'label_titles': ['safe: reversible rename'],
   'label_metadata': [{'id': 'risk.reversible_rename',
     'title': 'safe: reversible rename',
     'description': 'Renames variables, coordinates, or dimensions without changing values.',
     'category': 'risk-low'}],
   'changed': True},
  {'path': 'HadGEM3-GC31-LL',
   'fix_id': 'woodpecker.convert_units',
   'name': 'Convert variable units',
   'labels': ['risk.value_transformation'],
   'label_titles': ['careful: value transformation'],
   'label_metadata': [{'id': 'risk.value_transformation',
     'title': 'careful: value transformation',
     'description': 'Transforms data or coordinate values.',
     'category': 'risk-medium'}],
   'changed': True},
  {'path': 'HadGEM3-GC31-LL',
   'fix_id': 'xmip.fix_known_cmip6_metadata',
   'name': 'Fix known CMIP6 metadata',
   'labels': ['risk.metadata_only'],
   'label_titles': ['safe: metadata only'],
   'label_metadata': [{'id': 'risk.metadata_only',
     'title': 'safe: metadata only',
     'description': 'Changes metadata without changing data values.',
     'category': 'risk-low'}],
   'changed': True}))

Apply the recipe in memory and inspect the result.

In [7]:

result = woodpecker.recipe.apply(dataset, recipe, dry_run=False)

result.stats

result = woodpecker.recipe.apply(dataset, recipe, dry_run=False) result.stats

Out[7]:

{'attempted': 5,
 'changed': 5,
 'persist_attempted': 1,
 'persisted': 1,
 'persist_failed': 0,
 'preview': [{'path': 'HadGEM3-GC31-LL',
   'fix_id': 'woodpecker.rename_variables',
   'name': 'Rename variables and dimensions',
   'labels': ['risk.reversible_rename'],
   'label_titles': ['safe: reversible rename'],
   'label_metadata': [{'id': 'risk.reversible_rename',
     'title': 'safe: reversible rename',
     'description': 'Renames variables, coordinates, or dimensions without changing values.',
     'category': 'risk-low'}],
   'changed': True},
  {'path': 'HadGEM3-GC31-LL',
   'fix_id': 'woodpecker.set_coordinate_variables',
   'name': 'Set coordinate variables',
   'labels': ['risk.metadata_only'],
   'label_titles': ['safe: metadata only'],
   'label_metadata': [{'id': 'risk.metadata_only',
     'title': 'safe: metadata only',
     'description': 'Changes metadata without changing data values.',
     'category': 'risk-low'}],
   'changed': True},
  {'path': 'HadGEM3-GC31-LL',
   'fix_id': 'woodpecker.normalize_longitude_convention',
   'name': 'Normalize longitude convention',
   'labels': ['risk.coordinate_transformation'],
   'label_titles': ['careful: coordinate transformation'],
   'label_metadata': [{'id': 'risk.coordinate_transformation',
     'title': 'careful: coordinate transformation',
     'description': 'Transforms coordinate values, bounds, or geometry.',
     'category': 'risk-medium'}],
   'changed': True},
  {'path': 'HadGEM3-GC31-LL',
   'fix_id': 'woodpecker.convert_units',
   'name': 'Convert variable units',
   'labels': ['risk.value_transformation'],
   'label_titles': ['careful: value transformation'],
   'label_metadata': [{'id': 'risk.value_transformation',
     'title': 'careful: value transformation',
     'description': 'Transforms data or coordinate values.',
     'category': 'risk-medium'}],
   'changed': True},
  {'path': 'HadGEM3-GC31-LL',
   'fix_id': 'xmip.fix_known_cmip6_metadata',
   'name': 'Fix known CMIP6 metadata',
   'labels': ['risk.metadata_only'],
   'label_titles': ['safe: metadata only'],
   'label_metadata': [{'id': 'risk.metadata_only',
     'title': 'safe: metadata only',
     'description': 'Changes metadata without changing data values.',
     'category': 'risk-low'}],
   'changed': True}]}

In [8]:

{
    "dims": dict(dataset.sizes),
    "data_vars": list(dataset.data_vars),
    "coords": list(dataset.coords),
    "lev_values": dataset["lev"].values.tolist(),
    "lev_units": dataset["lev"].attrs.get("units"),
    "lon_min": float(dataset["lon"].min()),
    "branch_time_in_parent": dataset.attrs.get("branch_time_in_parent"),
}

{ "dims": dict(dataset.sizes), "data_vars": list(dataset.data_vars), "coords": list(dataset.coords), "lev_values": dataset["lev"].values.tolist(), "lev_units": dataset["lev"].attrs.get("units"), "lon_min": float(dataset["lon"].min()), "branch_time_in_parent": dataset.attrs.get("branch_time_in_parent"), }

Out[8]:

{'dims': {'lev': 2, 'time': 2, 'y': 2, 'x': 3},
 'data_vars': ['tas'],
 'coords': ['lev', 'time', 'y', 'x', 'lon', 'lat'],
 'lev_values': [0.0, 1.0],
 'lev_units': 'm',
 'lon_min': 0.0,
 'branch_time_in_parent': 91250}

In [9]:

woodpecker.recipe.check(dataset, recipe)

woodpecker.recipe.check(dataset, recipe)

Out[9]:

CheckResult(findings=())