DuckDB recipe store example¶

Shows an in-memory DuckDB recipe store. Use DuckDBRecipeStore("recipes.duckdb") for a persistent file.

In [1]:

from woodpecker.recipes import RecipeLoader
from woodpecker.stores import DuckDBRecipeStore
from woodpecker.testing import make_atlas, make_cmip6, make_cmip6_decadal, make_cmip7

from woodpecker.recipes import RecipeLoader from woodpecker.stores import DuckDBRecipeStore from woodpecker.testing import make_atlas, make_cmip6, make_cmip6_decadal, make_cmip7

Load discovered recipes into the DuckDB store.

In [2]:

store = DuckDBRecipeStore()

for recipe in RecipeLoader().catalog().list_recipes():
    store.save_recipe(recipe)

[recipe.id for recipe in store.list_recipes()]

store = DuckDBRecipeStore() for recipe in RecipeLoader().catalog().list_recipes(): store.save_recipe(recipe) [recipe.id for recipe in store.list_recipes()]

Out[2]:

['c3s.atlas',
 'c3s.cmip6_decadal',
 'cmip6.core_units',
 'cmip7.esa_cci_water_vapour_dotname',
 'cmip7.esa_cci_water_vapour_zarr',
 'xmip.cmip6_preprocessing',
 'xmip.cmip6_preprocessing_nominal_xy']

Query representative datasets by metadata and source path.

In [3]:

query_cases = [
    (
        "CMIP6 core",
        make_cmip6(overrides={"units": "degC"}, seed=7),
        "CMIP6.CMIP.MOHC.HadGEM3-GC31-LL.historical.r1i1p1f3.Amon.tas.gn.v20200101.nc",
    ),
    (
        "CMIP6 decadal",
        make_cmip6_decadal(seed=7),
        "CMIP6.DCPP.MPI-M.MPI-ESM1-2-HR.dcppA-hindcast.s1960-r1i1p1f1.Omon.tos.gn.v20200101.nc",
    ),
    (
        "Atlas",
        make_atlas(seed=7),
        "c3s-ipcc-atlas.cmip6.historical.ssp245.pr.monthly.global.nc",
    ),
    (
        "ESA CCI water vapour",
        make_cmip7(variable="prw", seed=7),
        "ESACCI-WATERVAPOUR-L3C-TCWV-meris-005deg-2002-2017-fv3.2.zarr",
    ),
]

query_cases = [ ( "CMIP6 core", make_cmip6(overrides={"units": "degC"}, seed=7), "CMIP6.CMIP.MOHC.HadGEM3-GC31-LL.historical.r1i1p1f3.Amon.tas.gn.v20200101.nc", ), ( "CMIP6 decadal", make_cmip6_decadal(seed=7), "CMIP6.DCPP.MPI-M.MPI-ESM1-2-HR.dcppA-hindcast.s1960-r1i1p1f1.Omon.tos.gn.v20200101.nc", ), ( "Atlas", make_atlas(seed=7), "c3s-ipcc-atlas.cmip6.historical.ssp245.pr.monthly.global.nc", ), ( "ESA CCI water vapour", make_cmip7(variable="prw", seed=7), "ESACCI-WATERVAPOUR-L3C-TCWV-meris-005deg-2002-2017-fv3.2.zarr", ), ]

In [4]:

matches = []
for label, dataset, source_path in query_cases:
    matched_plans = store.lookup(dataset, path=source_path)
    matches.append(
        {
            "case": label,
            "dataset_id": dataset.attrs.get("dataset_id", ""),
            "source_path": source_path,
            "recipe_ids": [recipe.id for recipe in matched_plans],
        }
    )

matches

matches = [] for label, dataset, source_path in query_cases: matched_plans = store.lookup(dataset, path=source_path) matches.append( { "case": label, "dataset_id": dataset.attrs.get("dataset_id", ""), "source_path": source_path, "recipe_ids": [recipe.id for recipe in matched_plans], } ) matches

Out[4]:

[{'case': 'CMIP6 core',
  'dataset_id': 'CMIP6.CMIP.MOHC.HadGEM3-GC31-LL.historical.r1i1p1f3.Amon.tas.gn.v20200101',
  'source_path': 'CMIP6.CMIP.MOHC.HadGEM3-GC31-LL.historical.r1i1p1f3.Amon.tas.gn.v20200101.nc',
  'recipe_ids': ['cmip6.core_units',
   'xmip.cmip6_preprocessing',
   'xmip.cmip6_preprocessing_nominal_xy']},
 {'case': 'CMIP6 decadal',
  'dataset_id': 'CMIP6.DCPP.MPI-M.MPI-ESM1-2-HR.dcppA-hindcast.s1960-r1i1p1f1.Omon.tos.gn.v20200101',
  'source_path': 'CMIP6.DCPP.MPI-M.MPI-ESM1-2-HR.dcppA-hindcast.s1960-r1i1p1f1.Omon.tos.gn.v20200101.nc',
  'recipe_ids': ['c3s.cmip6_decadal',
   'xmip.cmip6_preprocessing',
   'xmip.cmip6_preprocessing_nominal_xy']},
 {'case': 'Atlas',
  'dataset_id': 'c3s-ipcc-atlas.cmip6.historical.ssp245.pr.monthly.global',
  'source_path': 'c3s-ipcc-atlas.cmip6.historical.ssp245.pr.monthly.global.nc',
  'recipe_ids': ['c3s.atlas']},
 {'case': 'ESA CCI water vapour',
  'dataset_id': 'CMIP7.CMIP.MOHC.UKESM2-1.historical.r1i1p1f1.Amon.prw.gn.v20260101',
  'source_path': 'ESACCI-WATERVAPOUR-L3C-TCWV-meris-005deg-2002-2017-fv3.2.zarr',
  'recipe_ids': ['cmip7.esa_cci_water_vapour_zarr']}]

Inspect one selected recipe and its ordered fix ids.

In [5]:

selected = store.get_recipe("c3s.cmip6_decadal")

{
    "id": selected.id,
    "description": selected.description,
    "match": selected.match.model_dump() if selected.match else None,
    "steps": [step.id for step in selected.steps],
}

selected = store.get_recipe("c3s.cmip6_decadal") { "id": selected.id, "description": selected.description, "match": selected.match.model_dump() if selected.match else None, "steps": [step.id for step in selected.steps], }

Out[5]:

{'id': 'c3s.cmip6_decadal',
 'description': 'Apply C3S/CDS-ready metadata/encoding normalization for CMIP6-decadal hindcast datasets',
 'match': {'attrs': {'project_id': 'CMIP6',
   'activity_id': 'DCPP',
   'experiment_id': 'dcppA-hindcast'},
  'dataset_id_patterns': ['CMIP6.DCPP.*.dcppA-hindcast.*'],
  'path_patterns': []},
 'steps': ['cmip6_decadal.calendar_normalization',
  'cmip6_decadal.time_metadata',
  'cmip6_decadal.realization_variable',
  'cmip6_decadal.coordinates_encoding_cleanup',
  'cmip6_decadal.realization_comment_normalization',
  'cmip6_decadal.realization_dtype_normalization',
  'cmip6_decadal.fillvalue_encoding_cleanup',
  'cmip6_decadal.further_info_url_normalization',
  'cmip6_decadal.start_token_normalization',
  'cmip6_decadal.realization_long_name_normalization',
  'cmip6_decadal.realization_index_normalization',
  'cmip6_decadal.leadtime_metadata_normalization',
  'cmip6_decadal.model_global_attributes',
  'cmip6_decadal.reftime_coordinate',
  'cmip6_decadal.leadtime_coordinate']}

In [6]:

store.close()

store.close()