0

I'm dealing with CMIP6 Scenariomip data, I have eight ESMs precipitation grid that each one is composed by multiple netcdf files. There is one ESMs's netcdf files are seperately reserved every decade, while others are entire file spanning from 2010 to 2100. Here comes the difference. I used the xarray open_mfdataset() to open that ESMs file, and temperolly merge it at the same time, while i just used open_dataset otherwise. I have to do some calculation, while the one i use open_mfdataset() is like 100 times slower than the others, while the dataset's resolution have no difference. I know it's something wrong with the dask imbeded in open_mfdataset, i want to know is there anyway not to use dask in this problem, or anything else can make it faster. Here is my problematic code:

import numpy as np
import pandas as pd
import xarray as xr
import rioxarray as rio

 def nc_analysis(self):
        # clip
        # SSP245
        ds245 = xr.open_mfdataset(self.SSP245dir,
                               concat_dim="time",
                               combine= "nested"
                                  )
        # SSP370
        ds370 = xr.open_mfdataset(self.SSP370dir,
                               concat_dim="time",
                               combine= "nested")
        # SSP585
        ds585 = xr.open_mfdataset(self.SSP585dir,
                               concat_dim="time",
                               combine= "nested")
        # resample
        ds = xr.open_dataset(template_nc)  # template
        ds245_interp = ds245.interp_like(ds.drop_dims("time"), kwargs={"fill_value": "extrapolate"},method="nearest")  
        ds370_interp = ds370.interp_like(ds.drop_dims("time"), kwargs={"fill_value": "extrapolate"},method="nearest")  
        ds585_interp = ds585.interp_like(ds.drop_dims("time"), kwargs={"fill_value": "extrapolate"},method="nearest")  
        lon_bnds, lat_bnds = (73, 105), (25, 40)
        gain = 86400
        ds245_pr = ds245_interp.pr.sel(lon=slice(*lon_bnds), lat=slice(*lat_bnds)) * gain
        ds370_pr = ds370_interp.pr.sel(lon=slice(*lon_bnds), lat=slice(*lat_bnds)) * gain
        ds585_pr = ds585_interp.pr.sel(lon=slice(*lon_bnds), lat=slice(*lat_bnds)) * gain
        ########  below is the part that way slower than others    #########
        for i,time in tqdm(enumerate(ds245_pr.time), total = 1032):
            year = int(time.dt.year)
            month = int(time.dt.month)
            ds245_pr.sel(time=time).values = ds245_pr.sel(time=time) * monthrange(year,month)[1]
            ds370_pr.sel(time=time).values = ds370_pr.sel(time=time) * monthrange(year,month)[1]
            ds585_pr.sel(time=time).values = ds585_pr.sel(time=time) * monthrange(year,month)[1]

other code, without performance issue.

    def nc_analysis(self):
        # clip
        # SSP245
        ds245 = xr.open_dataset(self.SSP245dir)
        # SSP370
        ds370 = xr.open_dataset(self.SSP370dir)
        # SSP585
        ds585 = xr.open_dataset(self.SSP585dir)
        lon_bnds, lat_bnds = (73, 105), (25, 40)
        # resample
        ds = xr.open_dataset(template_nc)  # 100km template
        ds245_interp = ds245.interp_like(ds.drop_dims("time"), kwargs={"fill_value": "extrapolate"},method="nearest")  
        ds370_interp = ds370.interp_like(ds.drop_dims("time"), kwargs={"fill_value": "extrapolate"},method="nearest")  
        ds585_interp = ds585.interp_like(ds.drop_dims("time"), kwargs={"fill_value": "extrapolate"},method="nearest")  
        lon_bnds, lat_bnds = (73, 105), (25, 40)
        gain = 86400
        ds245_pr = ds245_interp.pr.sel(lon=slice(*lon_bnds), lat=slice(*lat_bnds)) * gain
        ds370_pr = ds370_interp.pr.sel(lon=slice(*lon_bnds), lat=slice(*lat_bnds)) * gain
        ds585_pr = ds585_interp.pr.sel(lon=slice(*lon_bnds), lat=slice(*lat_bnds)) * gain
        for i,time in enumerate(ds245_pr.time):
            year = int(time.dt.year)
            month = int(time.dt.month)
            ds245_pr.sel(time=time).values = ds245_pr.sel(time=time) * monthrange(year, month)[1]
            ds370_pr.sel(time=time).values = ds370_pr.sel(time=time) * monthrange(year, month)[1]
            ds585_pr.sel(time=time).values = ds585_pr.sel(time=time) * monthrange(year, month)[1]

0

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.

Browse other questions tagged or ask your own question.