Dask Demo

Dask Demo

import xarray as xr
import glob
from distributed import Client
import xwrf

client = Client("tcp://127.0.0.1:40447")
client

Client

Client-614d1692-1920-11ef-89c6-c28b38d965fe

Connection method: Direct
Dashboard: http://127.0.0.1:45625/status

Scheduler Info

Scheduler

Scheduler-d1419b47-801a-4bea-911c-1cc588232f9b

Comm: tcp://127.0.0.1:40447 Workers: 4
Dashboard: http://127.0.0.1:45625/status Total threads: 8
Started: 1 minute ago Total memory: 755.55 GiB

Workers

Worker: 0
Comm: tcp://127.0.0.1:39319 Total threads: 2
Dashboard: http://127.0.0.1:37355/status Memory: 188.89 GiB
Nanny: tcp://127.0.0.1:35871
Local directory: /tmp/dask-scratch-space/worker-fvktrm37
Tasks executing: Tasks in memory:
Tasks ready: Tasks in flight:
CPU usage: 0.0% Last seen: Just now
Memory usage: 178.00 MiB Spilled bytes: 0 B
Read bytes: 21.39 kiB Write bytes: 24.83 kiB

Worker: 1
Comm: tcp://127.0.0.1:32889 Total threads: 2
Dashboard: http://127.0.0.1:44767/status Memory: 188.89 GiB
Nanny: tcp://127.0.0.1:42265
Local directory: /tmp/dask-scratch-space/worker-dvt_j5bn
Tasks executing: Tasks in memory:
Tasks ready: Tasks in flight:
CPU usage: 2.0% Last seen: Just now
Memory usage: 177.64 MiB Spilled bytes: 0 B
Read bytes: 22.07 kiB Write bytes: 25.50 kiB

Worker: 2
Comm: tcp://127.0.0.1:34273 Total threads: 2
Dashboard: http://127.0.0.1:40483/status Memory: 188.89 GiB
Nanny: tcp://127.0.0.1:46181
Local directory: /tmp/dask-scratch-space/worker-b2to9jfc
Tasks executing: Tasks in memory:
Tasks ready: Tasks in flight:
CPU usage: 0.0% Last seen: Just now
Memory usage: 176.97 MiB Spilled bytes: 0 B
Read bytes: 21.04 kiB Write bytes: 24.47 kiB

Worker: 3
Comm: tcp://127.0.0.1:38297 Total threads: 2
Dashboard: http://127.0.0.1:36527/status Memory: 188.89 GiB
Nanny: tcp://127.0.0.1:43807
Local directory: /tmp/dask-scratch-space/worker-4rba7qoj
Tasks executing: Tasks in memory:
Tasks ready: Tasks in flight:
CPU usage: 2.0% Last seen: Just now
Memory usage: 177.78 MiB Spilled bytes: 0 B
Read bytes: 20.13 kiB Write bytes: 23.57 kiB 
files = glob.glob("/data/project/ARM_Summer_School_2024_Data/lasso_tutorial/cacti/lasso-cacti/20190129/eda09/base/les/subset_d4/corlasso_met_*")

ds = xr.open_dataset(files[0])

ds = xr.open_mfdataset(files[:10],
                       chunks={},
                       coords="minimal")

/opt/conda/lib/python3.11/site-packages/distributed/client.py:3163: UserWarning: Sending large graph of size 28.39 MiB.
This may cause some slowdown.
Consider scattering data ahead of time and using futures.
  warnings.warn(
/opt/conda/lib/python3.11/site-packages/distributed/client.py:3163: UserWarning: Sending large graph of size 28.39 MiB.
This may cause some slowdown.
Consider scattering data ahead of time and using futures.
  warnings.warn(
/opt/conda/lib/python3.11/site-packages/distributed/client.py:3163: UserWarning: Sending large graph of size 28.39 MiB.
This may cause some slowdown.
Consider scattering data ahead of time and using futures.
  warnings.warn(
/opt/conda/lib/python3.11/site-packages/distributed/client.py:3163: UserWarning: Sending large graph of size 28.39 MiB.
This may cause some slowdown.
Consider scattering data ahead of time and using futures.
  warnings.warn(
/opt/conda/lib/python3.11/site-packages/distributed/client.py:3163: UserWarning: Sending large graph of size 28.39 MiB.
This may cause some slowdown.
Consider scattering data ahead of time and using futures.
  warnings.warn(
/opt/conda/lib/python3.11/site-packages/distributed/client.py:3163: UserWarning: Sending large graph of size 28.39 MiB.
This may cause some slowdown.
Consider scattering data ahead of time and using futures.
  warnings.warn(
/opt/conda/lib/python3.11/site-packages/distributed/client.py:3163: UserWarning: Sending large graph of size 28.39 MiB.
This may cause some slowdown.
Consider scattering data ahead of time and using futures.
  warnings.warn(
/opt/conda/lib/python3.11/site-packages/distributed/client.py:3163: UserWarning: Sending large graph of size 28.39 MiB.
This may cause some slowdown.
Consider scattering data ahead of time and using futures.
  warnings.warn(
/opt/conda/lib/python3.11/site-packages/distributed/client.py:3163: UserWarning: Sending large graph of size 28.39 MiB.
This may cause some slowdown.
Consider scattering data ahead of time and using futures.
  warnings.warn(
/opt/conda/lib/python3.11/site-packages/distributed/client.py:3163: UserWarning: Sending large graph of size 28.39 MiB.
This may cause some slowdown.
Consider scattering data ahead of time and using futures.
  warnings.warn(
/opt/conda/lib/python3.11/site-packages/distributed/client.py:3163: UserWarning: Sending large graph of size 28.39 MiB.
This may cause some slowdown.
Consider scattering data ahead of time and using futures.
  warnings.warn(
/opt/conda/lib/python3.11/site-packages/distributed/client.py:3163: UserWarning: Sending large graph of size 28.39 MiB.
This may cause some slowdown.
Consider scattering data ahead of time and using futures.
  warnings.warn(
/opt/conda/lib/python3.11/site-packages/distributed/client.py:3163: UserWarning: Sending large graph of size 28.39 MiB.
This may cause some slowdown.
Consider scattering data ahead of time and using futures.
  warnings.warn(
/opt/conda/lib/python3.11/site-packages/distributed/client.py:3163: UserWarning: Sending large graph of size 28.39 MiB.
This may cause some slowdown.
Consider scattering data ahead of time and using futures.
  warnings.warn(
/opt/conda/lib/python3.11/site-packages/distributed/client.py:3163: UserWarning: Sending large graph of size 28.39 MiB.
This may cause some slowdown.
Consider scattering data ahead of time and using futures.
  warnings.warn(
/opt/conda/lib/python3.11/site-packages/distributed/client.py:3163: UserWarning: Sending large graph of size 28.39 MiB.
This may cause some slowdown.
Consider scattering data ahead of time and using futures.
  warnings.warn(
/opt/conda/lib/python3.11/site-packages/distributed/client.py:3163: UserWarning: Sending large graph of size 28.39 MiB.
This may cause some slowdown.
Consider scattering data ahead of time and using futures.
  warnings.warn(
/opt/conda/lib/python3.11/site-packages/distributed/client.py:3163: UserWarning: Sending large graph of size 28.39 MiB.
This may cause some slowdown.
Consider scattering data ahead of time and using futures.
  warnings.warn(

area_mean_t = ds.mean(dim=["south_north", "west_east"])

subset = area_mean_t[["HAMSL", "TEMPERATURE", "QVAPOR", "UA", "VA", "REFL_10CM_MAX"]]

area_mean_t = subset.compute()

area_mean_t = area_mean_t.set_coords("HAMSL")

area_mean_t.UA.plot(x='Time', y='HAMSL')

<matplotlib.collections.QuadMesh at 0x7f8609211850>
../../_images/e9e896019f62d25d45fc996303075853fda7d0def28e62a2dbe38fa9bf235af3.png 
client.shutdown()