#!/usr/bin/env python

import subprocess

mnths = ['01','02','03','04','05','06','07','08','09','10','11','12']
yrs = range(1979,2018)
#yrs = range(1979,1980)

# EXTRACT THE REGION
for i, y in enumerate(yrs):
    # Extract
    print('Extracting regional data for ' + str(y) + '...')
    for ii, m in enumerate(mnths):
        subprocess.call(['ncks','-d','latitude,-90.,30.','era5_1deg_hrly_' + str(y) + '_' + str(m) + '.nc','era5_1deg_hrly_' + str(y) + '_' + str(m) + '_sh.nc'])
        # Unpack from short variable to double precision so that there's no offset for each file
        subprocess.call(['ncpdq','-U','-O','era5_1deg_hrly_' + str(y) + '_' + str(m) + '_sh.nc','era5_1deg_hrly_' + str(y) + '_' + str(m) + '_sh.nc'])
        # Make time the record dimension
        subprocess.call(['ncks','-O','--mk_rec_dmn','time','era5_1deg_hrly_' + str(y) + '_' + str(m) + '_sh.nc','era5_1deg_hrly_' + str(y) + '_' + str(m) + '_sh.nc'])

    # CONCATENATE TO ANNUAL FILES
    print('Concatenate to annual files...')
    subprocess.call(['ncrcat','era5_1deg_hrly_' + str(y) + '_01_sh.nc',
        'era5_1deg_hrly_' + str(y) + '_02_sh.nc',
        'era5_1deg_hrly_' + str(y) + '_03_sh.nc',
        'era5_1deg_hrly_' + str(y) + '_04_sh.nc',
        'era5_1deg_hrly_' + str(y) + '_05_sh.nc',
        'era5_1deg_hrly_' + str(y) + '_06_sh.nc',
        'era5_1deg_hrly_' + str(y) + '_07_sh.nc',
        'era5_1deg_hrly_' + str(y) + '_08_sh.nc',
        'era5_1deg_hrly_' + str(y) + '_09_sh.nc',
        'era5_1deg_hrly_' + str(y) + '_10_sh.nc',
        'era5_1deg_hrly_' + str(y) + '_11_sh.nc',
        'era5_1deg_hrly_' + str(y) + '_12_sh.nc',
        'era5_1deg_hrly_' + str(y) + '_sh.nc',
        ])
    
    # REMOVE UNNECESSARY MONTHLY FILES
    for ii, m in enumerate(mnths):
        subprocess.call(['rm','-f','era5_1deg_hrly_' + str(y) + '_' + str(m) + '_sh.nc'])

    # EXTRACT EACH VARIABLE AND PACK IT BACK DOWN 
    print('Extract and pack down...')
    #ncks -C -v time,pressure in.nc out.nc
    subprocess.call(['ncks','-C','-v','msl,time,latitude,longitude','era5_1deg_hrly_' + str(y) + '_sh.nc','era5_msl_sh_hrly_' + str(y) + '.nc'])
#    subprocess.call(['ncpdq','-O','era5_msl_sh_hrly_' + str(y) + '.nc','era5_msl_sh_hrly_' + str(y) + '.nc'])
    subprocess.call(['ncks','-C','-v','tp,time,latitude,longitude','era5_1deg_hrly_' + str(y) + '_sh.nc','era5_tp_sh_hrly_' + str(y) + '.nc'])
 #   subprocess.call(['ncpdq','-O','era5_tp_sh_hrly_' + str(y) + '.nc','era5_tp_sh_hrly_' + str(y) + '.nc'])
    subprocess.call(['ncks','-C','-v','t2m,time,latitude,longitude','era5_1deg_hrly_' + str(y) + '_sh.nc','era5_t2m_sh_hrly_' + str(y) + '.nc'])
#    subprocess.call(['ncpdq','-O','era5_t2m_sh_hrly_' + str(y) + '.nc','era5_t2m_sh_hrly_' + str(y) + '.nc'])

    # REMOVE UNNECESSARY MONTHLY FILES
    subprocess.call(['rm','-f','era5_1deg_hrly_' + str(y) + '_sh.nc'])



# COMPUTE DAILY AVERAGES OR SUMS FOR EACH VARIABLE
for i, y in enumerate(yrs):
    print('Computing to daily data ' + str(y) + '...')

    subprocess.call(['cdo','daymean','era5_msl_sh_hrly_' + str(y) + '.nc','era5_msl_sh_daily_' + str(y) + '.nc'])
    subprocess.call(['cdo','daymean','era5_t2m_sh_hrly_' + str(y) + '.nc','era5_t2m_sh_daily_' + str(y) + '.nc'])
    subprocess.call(['cdo','daysum','era5_tp_sh_hrly_' + str(y) + '.nc','era5_tp_sh_daily_' + str(y) + '.nc'])

# CONCATENATE TO ANNUAL FILES
print('Concatenate msl to annual files...')
subprocess.call(['cdo','cat',
'era5_msl_sh_daily_1979.nc',
'era5_msl_sh_daily_1980.nc',
'era5_msl_sh_daily_1981.nc',
'era5_msl_sh_daily_1982.nc',
'era5_msl_sh_daily_1983.nc',
'era5_msl_sh_daily_1984.nc',
'era5_msl_sh_daily_1985.nc',
'era5_msl_sh_daily_1986.nc',
'era5_msl_sh_daily_1987.nc',
'era5_msl_sh_daily_1988.nc',
'era5_msl_sh_daily_1989.nc',
'era5_msl_sh_daily_1990.nc',
'era5_msl_sh_daily_1991.nc',
'era5_msl_sh_daily_1992.nc',
'era5_msl_sh_daily_1993.nc',
'era5_msl_sh_daily_1994.nc',
'era5_msl_sh_daily_1995.nc',
'era5_msl_sh_daily_1996.nc',
'era5_msl_sh_daily_1997.nc',
'era5_msl_sh_daily_1998.nc',
'era5_msl_sh_daily_1999.nc',
'era5_msl_sh_daily_2000.nc',
'era5_msl_sh_daily_2001.nc',
'era5_msl_sh_daily_2002.nc',
'era5_msl_sh_daily_2003.nc',
'era5_msl_sh_daily_2004.nc',
'era5_msl_sh_daily_2005.nc',
'era5_msl_sh_daily_2006.nc',
'era5_msl_sh_daily_2007.nc',
'era5_msl_sh_daily_2008.nc',
'era5_msl_sh_daily_2009.nc',
'era5_msl_sh_daily_2010.nc',
'era5_msl_sh_daily_2011.nc',
'era5_msl_sh_daily_2012.nc',
'era5_msl_sh_daily_2013.nc',
'era5_msl_sh_daily_2014.nc',
'era5_msl_sh_daily_2015.nc',
'era5_msl_sh_daily_2016.nc',
'era5_msl_sh_daily_2017.nc',
'era5_msl_sh_daily_1979_2017.nc'])


print('Concatenate t2m to annual files...')
subprocess.call(['cdo','cat',
'era5_t2m_sh_daily_1979.nc',
'era5_t2m_sh_daily_1980.nc',
'era5_t2m_sh_daily_1981.nc',
'era5_t2m_sh_daily_1982.nc',
'era5_t2m_sh_daily_1983.nc',
'era5_t2m_sh_daily_1984.nc',
'era5_t2m_sh_daily_1985.nc',
'era5_t2m_sh_daily_1986.nc',
'era5_t2m_sh_daily_1987.nc',
'era5_t2m_sh_daily_1988.nc',
'era5_t2m_sh_daily_1989.nc',
'era5_t2m_sh_daily_1990.nc',
'era5_t2m_sh_daily_1991.nc',
'era5_t2m_sh_daily_1992.nc',
'era5_t2m_sh_daily_1993.nc',
'era5_t2m_sh_daily_1994.nc',
'era5_t2m_sh_daily_1995.nc',
'era5_t2m_sh_daily_1996.nc',
'era5_t2m_sh_daily_1997.nc',
'era5_t2m_sh_daily_1998.nc',
'era5_t2m_sh_daily_1999.nc',
'era5_t2m_sh_daily_2000.nc',
'era5_t2m_sh_daily_2001.nc',
'era5_t2m_sh_daily_2002.nc',
'era5_t2m_sh_daily_2003.nc',
'era5_t2m_sh_daily_2004.nc',
'era5_t2m_sh_daily_2005.nc',
'era5_t2m_sh_daily_2006.nc',
'era5_t2m_sh_daily_2007.nc',
'era5_t2m_sh_daily_2008.nc',
'era5_t2m_sh_daily_2009.nc',
'era5_t2m_sh_daily_2010.nc',
'era5_t2m_sh_daily_2011.nc',
'era5_t2m_sh_daily_2012.nc',
'era5_t2m_sh_daily_2013.nc',
'era5_t2m_sh_daily_2014.nc',
'era5_t2m_sh_daily_2015.nc',
'era5_t2m_sh_daily_2016.nc',
'era5_t2m_sh_daily_2017.nc',
'era5_t2m_sh_daily_1979_2017.nc'])

print('Concatenate tp to annual files...')
subprocess.call(['cdo','cat',
'era5_tp_sh_daily_1979.nc', 
'era5_tp_sh_daily_1980.nc', 
'era5_tp_sh_daily_1981.nc', 
'era5_tp_sh_daily_1982.nc', 
'era5_tp_sh_daily_1983.nc', 
'era5_tp_sh_daily_1984.nc', 
'era5_tp_sh_daily_1985.nc', 
'era5_tp_sh_daily_1986.nc', 
'era5_tp_sh_daily_1987.nc', 
'era5_tp_sh_daily_1988.nc', 
'era5_tp_sh_daily_1989.nc', 
'era5_tp_sh_daily_1990.nc', 
'era5_tp_sh_daily_1991.nc', 
'era5_tp_sh_daily_1992.nc', 
'era5_tp_sh_daily_1993.nc', 
'era5_tp_sh_daily_1994.nc', 
'era5_tp_sh_daily_1995.nc', 
'era5_tp_sh_daily_1996.nc', 
'era5_tp_sh_daily_1997.nc', 
'era5_tp_sh_daily_1998.nc', 
'era5_tp_sh_daily_1999.nc', 
'era5_tp_sh_daily_2000.nc', 
'era5_tp_sh_daily_2001.nc', 
'era5_tp_sh_daily_2002.nc', 
'era5_tp_sh_daily_2003.nc', 
'era5_tp_sh_daily_2004.nc', 
'era5_tp_sh_daily_2005.nc', 
'era5_tp_sh_daily_2006.nc', 
'era5_tp_sh_daily_2007.nc', 
'era5_tp_sh_daily_2008.nc', 
'era5_tp_sh_daily_2009.nc', 
'era5_tp_sh_daily_2010.nc', 
'era5_tp_sh_daily_2011.nc', 
'era5_tp_sh_daily_2012.nc', 
'era5_tp_sh_daily_2013.nc', 
'era5_tp_sh_daily_2014.nc', 
'era5_tp_sh_daily_2015.nc', 
'era5_tp_sh_daily_2016.nc', 
'era5_tp_sh_daily_2017.nc', 
'era5_tp_sh_daily_1979_2017.nc']) 

    
# EXTRACT EACH VARIABLE AND PACK IT BACK DOWN 
#print('Pack down...')
#ncks -C -v time,pressure in.nc out.nc
#subprocess.call(['ncpdq','-O','era5_msl_sh_hrly_1979_2017.nc','era5_msl_sh_hrly_1979_2017.nc'])
#subprocess.call(['ncpdq','-O','era5_tp_sh_hrly_1979_2017.nc','era5_tp_sh_hrly_1979_2017.nc'])
#subprocess.call(['ncpdq','-O','era5_t2m_sh_hrly_1979_2017.nc','era5_t2m_sh_hrly_1979_2017.nc'])

# REMOVE PACKING FILES
for i, y in enumerate(yrs):
    subprocess.call(['rm','-f','era5_msl_sh_daily_' + str(y) + '.nc'])
    subprocess.call(['rm','-f','era5_t2m_sh_daily_' + str(y) + '.nc'])
    subprocess.call(['rm','-f','era5_tp_sh_daily_' + str(y) + '.nc'])
    subprocess.call(['rm','-f','era5_msl_sh_hrly_' + str(y) + '.nc'])
    subprocess.call(['rm','-f','era5_t2m_sh_hrly_' + str(y) + '.nc'])
    subprocess.call(['rm','-f','era5_tp_sh_hrly_' + str(y) + '.nc'])