Atlas recipe example¶

Shows a bundled Atlas plugin recipe on a tiny synthetic dataset.

Flow: create data -> load recipe -> check -> dry-run -> apply -> re-check.

In [1]:

import woodpecker_atlas_plugin  # noqa: F401 - imports plugin fixes for editable installs

import woodpecker
from woodpecker.testing import make_atlas

import woodpecker_atlas_plugin # noqa: F401 - imports plugin fixes for editable installs import woodpecker from woodpecker.testing import make_atlas

Create an Atlas-like dataset with missing project_id metadata and overly strong compression.

In [2]:

dataset = make_atlas(missing=["project_id"], seed=7)
dataset["pr"].encoding["complevel"] = 5

dataset

dataset = make_atlas(missing=["project_id"], seed=7) dataset["pr"].encoding["complevel"] = 5 dataset

Out[2]:

<xarray.Dataset> Size: 32kB
Dimensions:  (time: 12, lat: 18, lon: 36)
Coordinates:
  * time     (time) datetime64[s] 96B 2000-01-01 2000-02-01 ... 2000-12-01
  * lat      (lat) float64 144B -85.0 -75.0 -65.0 -55.0 ... 55.0 65.0 75.0 85.0
  * lon      (lon) float64 288B 0.0 10.0 20.0 30.0 ... 320.0 330.0 340.0 350.0
Data variables:
    pr       (time, lat, lon) float32 31kB 2.131 2.146 2.117 ... 1.967 1.959
Attributes:
    dataset_id:     c3s-ipcc-atlas.cmip6.historical.ssp245.pr.monthly.global
    ds_id:          c3s-ipcc-atlas.cmip6.historical.ssp245.pr.monthly.global
    source_name:    c3s-ipcc-atlas.cmip6.historical.ssp245.pr.monthly.global.nc
    experiment_id:  ssp245
    variable_id:    pr
    table_id:       Amon
    units:          kg m-2 s-1
    frequency:      mon
    grid_label:     gr
    atlas_region:   global

xarray.Dataset

• Dimensions:
  • time: 12
  • lat: 18
  • lon: 36
• Coordinates: (3)
  • time
    (time)
    datetime64[s]
    2000-01-01 ... 2000-12-01

    axis :

    T

    standard_name :

    time

    array(['2000-01-01T00:00:00', '2000-02-01T00:00:00', '2000-03-01T00:00:00',
           '2000-04-01T00:00:00', '2000-05-01T00:00:00', '2000-06-01T00:00:00',
           '2000-07-01T00:00:00', '2000-08-01T00:00:00', '2000-09-01T00:00:00',
           '2000-10-01T00:00:00', '2000-11-01T00:00:00', '2000-12-01T00:00:00'],
          dtype='datetime64[s]')

  • lat
    (lat)
    float64
    -85.0 -75.0 -65.0 ... 75.0 85.0

    axis :

    Y

    standard_name :

    latitude

    units :

    degrees_north

    array([-85., -75., -65., -55., -45., -35., -25., -15.,  -5.,   5.,  15.,  25.,
            35.,  45.,  55.,  65.,  75.,  85.])

  • lon
    (lon)
    float64
    0.0 10.0 20.0 ... 330.0 340.0 350.0

    axis :

    X

    standard_name :

    longitude

    units :

    degrees_east

    array([  0.,  10.,  20.,  30.,  40.,  50.,  60.,  70.,  80.,  90., 100., 110.,
           120., 130., 140., 150., 160., 170., 180., 190., 200., 210., 220., 230.,
           240., 250., 260., 270., 280., 290., 300., 310., 320., 330., 340., 350.])

• Data variables: (1)
  • pr
    (time, lat, lon)
    float32
    2.131 2.146 2.117 ... 1.967 1.959

    standard_name :

    precipitation_flux

    units :

    kg m-2 s-1

    array([[[2.1307952, 2.145671 , 2.1170268, ..., 2.0902917, 2.1837785,
             2.0903568],
            [2.3866024, 2.4324481, 2.3590486, ..., 2.3484964, 2.4205737,
             2.2886076],
            [2.6107688, 2.6290631, 2.696778 , ..., 2.6027303, 2.6441977,
             2.6585782],
            ...,
            [2.6385028, 2.6414766, 2.6260262, ..., 2.5073013, 2.6527877,
             2.5593188],
            [2.3234065, 2.3564806, 2.451858 , ..., 2.371485 , 2.4330938,
             2.4072785],
            [2.1006863, 2.1299899, 2.1685736, ..., 2.1554322, 2.1553924,
             2.1069613]],
    
           [[2.3821805, 2.3187356, 2.3855553, ..., 2.3139884, 2.3556695,
             2.2862306],
            [2.570143 , 2.5370078, 2.6448169, ..., 2.6066222, 2.5511594,
             2.571719 ],
            [2.8037019, 2.8168488, 2.7184513, ..., 2.8385255, 2.8121667,
             2.8297384],
    ...
            [2.3898225, 2.2893002, 2.3579347, ..., 2.3311307, 2.309217 ,
             2.3084476],
            [2.086894 , 1.9906682, 2.0509331, ..., 2.0129004, 2.0722415,
             2.0879805],
            [1.7945877, 1.7707531, 1.7055819, ..., 1.7709614, 1.7776444,
             1.8267434]],
    
           [[1.9431614, 1.9427598, 1.9074578, ..., 1.8943526, 1.9125551,
             1.9416133],
            [2.188815 , 2.2785237, 2.2177572, ..., 2.1643305, 2.1699286,
             2.131081 ],
            [2.3824544, 2.4487472, 2.3498225, ..., 2.4324057, 2.45668  ,
             2.3432837],
            ...,
            [2.3905864, 2.4242573, 2.4238603, ..., 2.4202633, 2.3560967,
             2.5040846],
            [2.1345463, 2.1565955, 2.2150626, ..., 2.1671193, 2.2132986,
             2.2707636],
            [1.9013596, 1.918799 , 1.8729172, ..., 1.9153534, 1.9674724,
             1.9585377]]], shape=(12, 18, 36), dtype=float32)

• Attributes: (10)

  dataset_id :

  c3s-ipcc-atlas.cmip6.historical.ssp245.pr.monthly.global

  ds_id :

  c3s-ipcc-atlas.cmip6.historical.ssp245.pr.monthly.global

  source_name :

  c3s-ipcc-atlas.cmip6.historical.ssp245.pr.monthly.global.nc

  experiment_id :

  ssp245

  variable_id :

  pr

  table_id :

  Amon

  units :

  kg m-2 s-1

  frequency :

  mon

  grid_label :

  gr

  atlas_region :

  global

Load the bundled Atlas recipe and inspect its match rules and fix steps.

In [3]:

recipe = woodpecker.recipe.get("c3s.atlas")

recipe.model_dump()

recipe = woodpecker.recipe.get("c3s.atlas") recipe.model_dump()

Out[3]:

{'id': 'c3s.atlas',
 'aliases': [],
 'description': 'C3S/CDS adaptation recipe for Atlas NetCDF datasets',
 'match': {'attrs': {},
  'dataset_id_patterns': [],
  'path_patterns': ['*atlas*.nc']},
 'steps': [{'id': 'atlas.encoding_cleanup',
   'phase': 'apply',
   'options': {},
   'links': []},
  {'id': 'atlas.project_id_normalization',
   'phase': 'apply',
   'options': {},
   'links': []}],
 'links': [{'rel': 'self',
   'href': 'https://github.com/roocs/woodpecker/blob/main/plugins/woodpecker-atlas-plugin/src/woodpecker_atlas_plugin/recipes/atlas_basic_recipe.json',
   'title': 'Source recipe document'}]}

In [4]:

recipe.match.model_dump(), [step.id for step in recipe.steps]

recipe.match.model_dump(), [step.id for step in recipe.steps]

Out[4]:

({'attrs': {}, 'dataset_id_patterns': [], 'path_patterns': ['*atlas*.nc']},
 ['atlas.encoding_cleanup', 'atlas.project_id_normalization'])

In [5]:

findings = woodpecker.recipe.check(dataset, recipe)
findings.fix_ids

findings = woodpecker.recipe.check(dataset, recipe) findings.fix_ids

Out[5]:

('atlas.encoding_cleanup',
 'atlas.encoding_cleanup',
 'atlas.project_id_normalization')

Dry-run previews the repair without changing the dataset.

In [6]:

result = woodpecker.recipe.apply(dataset, recipe, dry_run=True)

result.stats, result.preview, dataset.attrs.get("project_id"), dataset["pr"].encoding["complevel"]

result = woodpecker.recipe.apply(dataset, recipe, dry_run=True) result.stats, result.preview, dataset.attrs.get("project_id"), dataset["pr"].encoding["complevel"]

Out[6]:

({'attempted': 2,
  'changed': 2,
  'persist_attempted': 0,
  'persisted': 0,
  'persist_failed': 0,
  'preview': [{'path': 'c3s-ipcc-atlas.cmip6.historical.ssp245.pr.monthly.global.nc',
    'fix_id': 'atlas.encoding_cleanup',
    'name': 'ATLAS encoding cleanup',
    'labels': ['risk.encoding_metadata'],
    'label_titles': ['safe: encoding metadata'],
    'label_metadata': [{'id': 'risk.encoding_metadata',
      'title': 'safe: encoding metadata',
      'description': 'Changes encoding or persistence metadata without changing data values.',
      'category': 'risk-low'}],
    'changed': True},
   {'path': 'c3s-ipcc-atlas.cmip6.historical.ssp245.pr.monthly.global.nc',
    'fix_id': 'atlas.project_id_normalization',
    'name': 'ATLAS project_id normalization',
    'labels': ['risk.metadata_only'],
    'label_titles': ['safe: metadata only'],
    'label_metadata': [{'id': 'risk.metadata_only',
      'title': 'safe: metadata only',
      'description': 'Changes metadata without changing data values.',
      'category': 'risk-low'}],
    'changed': True}]},
 ({'path': 'c3s-ipcc-atlas.cmip6.historical.ssp245.pr.monthly.global.nc',
   'fix_id': 'atlas.encoding_cleanup',
   'name': 'ATLAS encoding cleanup',
   'labels': ['risk.encoding_metadata'],
   'label_titles': ['safe: encoding metadata'],
   'label_metadata': [{'id': 'risk.encoding_metadata',
     'title': 'safe: encoding metadata',
     'description': 'Changes encoding or persistence metadata without changing data values.',
     'category': 'risk-low'}],
   'changed': True},
  {'path': 'c3s-ipcc-atlas.cmip6.historical.ssp245.pr.monthly.global.nc',
   'fix_id': 'atlas.project_id_normalization',
   'name': 'ATLAS project_id normalization',
   'labels': ['risk.metadata_only'],
   'label_titles': ['safe: metadata only'],
   'label_metadata': [{'id': 'risk.metadata_only',
     'title': 'safe: metadata only',
     'description': 'Changes metadata without changing data values.',
     'category': 'risk-low'}],
   'changed': True}),
 None,
 5)

Apply the recipe in memory and re-check.

In [7]:

write = woodpecker.recipe.apply(dataset, recipe, dry_run=False)

(
    write.stats,
    dataset.attrs["project_id"],
    dataset["pr"].encoding["complevel"],
    dataset["pr"].encoding["zlib"],
    dataset["pr"].encoding["shuffle"],
)

write = woodpecker.recipe.apply(dataset, recipe, dry_run=False) ( write.stats, dataset.attrs["project_id"], dataset["pr"].encoding["complevel"], dataset["pr"].encoding["zlib"], dataset["pr"].encoding["shuffle"], )

Out[7]:

({'attempted': 2,
  'changed': 2,
  'persist_attempted': 1,
  'persisted': 1,
  'persist_failed': 0,
  'preview': [{'path': 'c3s-ipcc-atlas.cmip6.historical.ssp245.pr.monthly.global.nc',
    'fix_id': 'atlas.encoding_cleanup',
    'name': 'ATLAS encoding cleanup',
    'labels': ['risk.encoding_metadata'],
    'label_titles': ['safe: encoding metadata'],
    'label_metadata': [{'id': 'risk.encoding_metadata',
      'title': 'safe: encoding metadata',
      'description': 'Changes encoding or persistence metadata without changing data values.',
      'category': 'risk-low'}],
    'changed': True},
   {'path': 'c3s-ipcc-atlas.cmip6.historical.ssp245.pr.monthly.global.nc',
    'fix_id': 'atlas.project_id_normalization',
    'name': 'ATLAS project_id normalization',
    'labels': ['risk.metadata_only'],
    'label_titles': ['safe: metadata only'],
    'label_metadata': [{'id': 'risk.metadata_only',
      'title': 'safe: metadata only',
      'description': 'Changes metadata without changing data values.',
      'category': 'risk-low'}],
    'changed': True}]},
 'c3s-ipcc-atlas',
 1,
 True,
 True)

In [8]:

recheck = woodpecker.recipe.check(dataset, recipe)
bool(recheck)

recheck = woodpecker.recipe.check(dataset, recipe) bool(recheck)

Out[8]:

False