Skip to content

Commit

Permalink
itemsize hardcode
Browse files Browse the repository at this point in the history
  • Loading branch information
norlandrhagen committed Sep 11, 2024
1 parent 160c0e4 commit 6219ef1
Showing 1 changed file with 3 additions and 4 deletions.
7 changes: 3 additions & 4 deletions xbeam_virtualizarr/recipe.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,15 +25,14 @@ def run(argv=None, save_main_session=True):

combined_ds = xr.open_dataset(reference_path, engine="kerchunk", chunks={})
# subset the reference zarr
source_dataset = combined_ds.isel(day=slice(0, 10000))[
source_dataset = combined_ds.isel(day=slice(0, 2000))[
["air_temperature"]
] # all vars
# source_chunks = dict(source_dataset.sizes) # this is total size. Hardcode for now
source_chunks = {"day": 61, "lat": 98, "lon": 231}
target_chunks = {"day": 16, "lat": 585, "lon": 1386} # ~ full map 100MB chunks
template = xbeam.make_template(source_dataset).isel(day=slice(0, 10000), drop=True)

itemsize = max(variable.dtype.itemsize for variable in template.values())
itemsize = 8

with beam.Pipeline(options=pipeline_options) as p:
(
Expand All @@ -45,7 +44,7 @@ def run(argv=None, save_main_session=True):
target_chunks,
itemsize=itemsize,
)
| xbeam.ChunksToZarr(output_path, template, target_chunks)
| xbeam.ChunksToZarr(output_path, target_chunks)
)


Expand Down

0 comments on commit 6219ef1

Please sign in to comment.