try experimental min

leap-stc · Sep 11, 2024 · a186891 · a186891
1 parent 0821e97
commit a186891
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 3 deletions.
diff --git a/.github/workflows/deploy_recipe.yaml b/.github/workflows/deploy_recipe.yaml
@@ -48,7 +48,8 @@ jobs:
           --job_name $JOB_NAME \
           --region us-central1 \
           --machine_type t2d-standard-16	 \
-          --disk_size_gb 50 \
-          --num_workers 5 \
+          --disk_size_gb 50
+          --max_num_workers 100 \
+          --experiments=minNumWorkers=5 \
           --temp_location gs://leap-persistent/norlandrhagen/dataflow_temp/temp/ \
           --staging_location gs://leap-persistent/norlandrhagen/dataflow_temp/staging/
diff --git a/xbeam_virtualizarr/recipe.py b/xbeam_virtualizarr/recipe.py
@@ -25,7 +25,7 @@ def run(argv=None, save_main_session=True):
 
     combined_ds = xr.open_dataset(reference_path, engine="kerchunk", chunks=None)
     # subset the reference zarr
-    source_dataset = combined_ds.isel(day=slice(0, 10000))[
+    source_dataset = combined_ds.isel(day=slice(0, 20000))[
         ["air_temperature"]
     ]  # all vars
     # source_chunks = dict(source_dataset.sizes) # this is total size. Hardcode for now