Read Granules#

[1]:

import starepandas
import netCDF4
import matplotlib.pyplot as plt
import numpy
import pandas
import time
import copy
import geopandas
import pystare
import pyhdf

Manually reading#

[2]:

vnp03_name = '../tests/data/granules/VNP03DNB.A2020219.0742.001.2020219124651.nc'
vnp03 = starepandas.io.granules.VNP03DNB(vnp03_name)
vnp03.read_latlon()
vnp03.read_data()
vnp03 = vnp03.to_df(xy=True)

MOD09GA#

[3]:

file_name = '../tests/data/granules/MOD09GA.A2020009.h00v08.006.2020011025435.hdf'

[4]:

starepandas.read_granule(file_name, xy=True)

[4]:

	x	y	sur_refl_b01_1	sur_refl_b02_1	sur_refl_b03_1	sur_refl_b04_1	sur_refl_b05_1	sur_refl_b06_1	sur_refl_b07_1	QC_500m_1	obscov_500m_1
0	0	0	NaN	NaN	NaN	NaN	NaN	NaN	NaN	<NA>	NaN
1	1	0	NaN	NaN	NaN	NaN	NaN	NaN	NaN	<NA>	NaN
2	2	0	NaN	NaN	NaN	NaN	NaN	NaN	NaN	<NA>	NaN
3	3	0	NaN	NaN	NaN	NaN	NaN	NaN	NaN	<NA>	NaN
4	4	0	NaN	NaN	NaN	NaN	NaN	NaN	NaN	<NA>	NaN
...	...	...	...	...	...	...	...	...	...	...	...
5759995	2395	2399	0.2020	0.1939	0.2187	0.2053	0.1872	0.1359	0.0817	1073741824	0.15
5759996	2396	2399	0.2122	0.2043	0.2320	0.2174	0.2015	0.1356	0.0900	1073741824	0.19
5759997	2397	2399	0.2122	0.2043	0.2320	0.2174	0.2015	0.1356	0.0900	1073741824	0.19
5759998	2398	2399	0.2174	0.2094	0.2342	0.2225	0.2074	0.1464	0.0867	1073741824	0.11
5759999	2399	2399	0.2174	0.2094	0.2342	0.2225	0.2074	0.1464	0.0867	1073741824	0.14

5760000 rows × 11 columns

[5]:

mod09ga = starepandas.io.granules.Mod09GA(file_name)
mod09ga.read_data()

ds_name = 'state_1km_1'
mod09ga.read_dataset(ds_name, resample_factor=2)
mod09ga.decode_state('state_1km_1')
mod09ga.to_df()

[5]:

	sur_refl_b01_1	sur_refl_b02_1	sur_refl_b03_1	sur_refl_b04_1	sur_refl_b05_1	sur_refl_b06_1	sur_refl_b07_1	QC_500m_1	obscov_500m_1	state_1km_1	cloud	cloud_shadow	cloud_internal	snow_mod35	snow_internal
0	NaN	NaN	NaN	NaN	NaN	NaN	NaN	<NA>	NaN	<NA>	<NA>	True	True	True	True
1	NaN	NaN	NaN	NaN	NaN	NaN	NaN	<NA>	NaN	<NA>	<NA>	True	True	True	True
2	NaN	NaN	NaN	NaN	NaN	NaN	NaN	<NA>	NaN	<NA>	<NA>	True	True	True	True
3	NaN	NaN	NaN	NaN	NaN	NaN	NaN	<NA>	NaN	<NA>	<NA>	True	True	True	True
4	NaN	NaN	NaN	NaN	NaN	NaN	NaN	<NA>	NaN	<NA>	<NA>	True	True	True	True
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
5759995	0.2020	0.1939	0.2187	0.2053	0.1872	0.1359	0.0817	1073741824	0.15	1849	1	False	True	False	False
5759996	0.2122	0.2043	0.2320	0.2174	0.2015	0.1356	0.0900	1073741824	0.19	1849	1	False	True	False	False
5759997	0.2122	0.2043	0.2320	0.2174	0.2015	0.1356	0.0900	1073741824	0.19	1849	1	False	True	False	False
5759998	0.2174	0.2094	0.2342	0.2225	0.2074	0.1464	0.0867	1073741824	0.11	1849	1	False	True	False	False
5759999	0.2174	0.2094	0.2342	0.2225	0.2074	0.1464	0.0867	1073741824	0.14	1849	1	False	True	False	False

5760000 rows × 15 columns

MOD09#

[6]:

file_name = '../tests/data/granules/MOD09.A2002299.0710.006.2015151173939.hdf'
mod09 = starepandas.read_granule(file_name)

[7]:

starepandas.io.granules.modis.decode_state(mod09['1km Reflectance Data State QA'])

[7]:

	cloud	cloud_shadow	cloud_internal	snow_mod35	snow_internal
0	00	False	False	False	False
1	00	False	False	False	False
2	00	False	False	False	False
3	00	False	False	False	False
4	00	False	False	False	False
...	...	...	...	...	...
2748615	00	False	False	False	False
2748616	00	False	False	False	False
2748617	00	False	False	False	False
2748618	00	False	False	False	False
2748619	00	False	False	False	False

2748620 rows × 5 columns

MOD05#

[8]:

fname = '../tests/data/granules/MOD05_L2.A2019336.0000.061.2019336211522.hdf'
mod05 = starepandas.read_granule(fname, sidecar=True, latlon=True, xy=True, read_timestamp=True)
sids = mod05.to_array(mod05._sid_column_name)
#mod05.to_sidecar('test.nc')

[9]:

mod05

[9]:

	lat	lon	sids	ts_start	ts_end	x	y	Scan_Start_Time	Solar_Zenith	Solar_Azimuth	Sensor_Zenith	Sensor_Azimuth	Water_Vapor_Infrared
0	53.201778	-15.933996	4298473764500464809	2019-12-02	2019-12-02 00:05:00	0	0	8.493984e+08	147.089997	-22.959999	65.339999	-89.519998	NaN
1	53.203171	-16.288101	4298458168380511209	2019-12-02	2019-12-02 00:05:00	1	0	8.493984e+08	146.999997	-23.539999	64.719999	-89.799998	NaN
2	53.203518	-16.629105	4297394569014717897	2019-12-02	2019-12-02 00:05:00	2	0	8.493984e+08	146.919997	-24.099999	64.109999	-90.079998	NaN
3	53.202934	-16.957928	4297300698872999369	2019-12-02	2019-12-02 00:05:00	3	0	8.493984e+08	146.839997	-24.639999	63.509999	-90.339998	NaN
4	53.201508	-17.275316	4297290857922121161	2019-12-02	2019-12-02 00:05:00	4	0	8.493984e+08	146.759997	-25.159999	62.909999	-90.589998	NaN
...	...	...	...	...	...	...	...	...	...	...	...	...	...
109615	64.084297	-66.281410	3727686638370615689	2019-12-02	2019-12-02 00:05:00	265	405	8.493987e+08	121.569997	-74.679998	62.229999	47.999999	NaN
109616	63.960953	-66.591354	3727661374545790857	2019-12-02	2019-12-02 00:05:00	266	405	8.493987e+08	121.469997	-75.049998	62.819999	47.719999	NaN
109617	63.831799	-66.911255	3727838256925064969	2019-12-02	2019-12-02 00:05:00	267	405	8.493987e+08	121.369997	-75.439998	63.419999	47.429999	NaN
109618	63.698635	-67.236229	3727843063731949801	2019-12-02	2019-12-02 00:05:00	268	405	8.493987e+08	121.259997	-75.829998	64.019999	47.139999	NaN
109619	63.551617	-67.589508	3727853163225616425	2019-12-02	2019-12-02 00:05:00	269	405	8.493987e+08	121.149997	-76.249998	64.639999	46.819999	NaN

109620 rows × 13 columns

Subset Loading#

[10]:

import shapely
roi = shapely.geometry.Polygon([[-120, 37],[-120, 38],[-119, 38],[-119, 37]])
roi = geopandas.GeoDataFrame(geometry=[roi], crs=4326)
roi = starepandas.STAREDataFrame(roi)
roi['sids'] = roi.make_sids(level=10)
sids = roi.sids.iloc[0]

[11]:

roi.plot(trixels=False)

[11]:

<AxesSubplot:>

../../_images/examples_notebooks_read_granules_16_1.png

[12]:

fname = '../tests/data/granules/MOD05_L2.A2019336.0000.061.2019336211522.hdf'
mod = starepandas.io.granules.Mod05(fname)
mod.read_sidecar_index()

[13]:

intersects = pystare.intersects(sids, mod.sids.flatten())
intersects = numpy.reshape(intersects, mod.sids.shape)
intersects

[13]:

array([[False, False, False, ..., False, False, False],
       [False, False, False, ..., False, False, False],
       [False, False, False, ..., False, False, False],
       ...,
       [False, False, False, ..., False, False, False],
       [False, False, False, ..., False, False, False],
       [False, False, False, ..., False, False, False]])

VNP02#

[14]:

vnp02_name = '../tests/data/granules/VNP02DNB.A2020219.0742.001.2020219125654.nc'
vnp02 = starepandas.read_granule(vnp02_name, sidecar=False, latlon=False)

[15]:

vnp02 = vnp03.join(vnp02)

[16]:

vnp02[(vnp02.land_water_mask==2) | (vnp02.land_water_mask==4) | (vnp02.land_water_mask==6) | (vnp02.land_water_mask==7)]

[16]:

	lat	lon	x	y	moon_illumination_fraction	land_water_mask	quality_flag	DNB_observations	DNB_quality_flags
267	41.680256	-102.596710	267	0	92.790001	2	0	3.503916e-09	16
368	41.663277	-101.661072	368	0	92.799995	2	0	6.539059e-09	16
369	41.663055	-101.652069	369	0	92.799995	2	0	7.083299e-09	16
403	41.654938	-101.342972	403	0	92.799995	2	0	8.231430e-09	16
883	41.460007	-96.951828	883	0	92.820000	2	0	2.414932e-09	16
...	...	...	...	...	...	...	...	...	...
13134843	16.577330	-78.576134	4059	3231	92.879997	7	0	2.087984e-09	0
13134844	16.575632	-78.569000	4060	3231	92.879997	7	0	1.579183e-09	0
13134845	16.573931	-78.561844	4061	3231	92.879997	7	0	1.579183e-09	0
13134846	16.572226	-78.554688	4062	3231	92.879997	7	0	2.393264e-09	0
13134847	16.570518	-78.547523	4063	3231	92.879997	7	0	1.986224e-09	0

4150456 rows × 9 columns

VNP CLDMSK#

[17]:

cldmsk_name = '../tests/data/granules/CLDMSK_L2_VIIRS_SNPP.A2020219.0742.001.2020219190616.nc'

[18]:

cldmsk = starepandas.io.granules.CLDMSKL2VIIRS(cldmsk_name)

[19]:

cldmsk.read_latlon()
cldmsk.read_data()
cldmsk.to_df()

[19]:

	lat	lon	Integer_Cloud_Mask
0	41.747662	-104.969498	3
1	41.747696	-104.950218	3
2	41.747730	-104.930237	3
3	41.747761	-104.910843	3
4	41.747784	-104.891747	3
...	...	...	...
10342395	16.522247	-78.595093	2
10342396	16.518646	-78.580162	1
10342397	16.515034	-78.565208	0
10342398	16.511414	-78.550224	1
10342399	16.507788	-78.535187	1

10342400 rows × 3 columns

Plotting#

[20]:

path = '../tests/data/granules/VNP03DNB.A2020219.0742.001.2020219124651.nc'
vnp03dnb = starepandas.io.granules.VNP03DNB(path)
vnp03dnb.read_latlon()

path = '../tests/data/granules/VNP03MOD.A2020219.0742.001.2020219124651.nc'
vnp03mod = starepandas.io.granules.VNP03MOD(path)
vnp03mod.read_latlon()

path = '../tests/data/granules/CLDMSK_L2_VIIRS_SNPP.A2020219.0742.001.2020219190616.nc'
cldmsk = starepandas.io.granules.CLDMSKL2VIIRS(path)
cldmsk.read_latlon()

[21]:

fig, ax = plt.subplots(figsize=(5,5), dpi=100)

n = 20
ns = 0
nt = 0

ax.grid('on')

ax.plot(cldmsk.lon[ns:ns+n, nt:nt+n].flatten(), cldmsk.lat[ns:ns+n, nt:nt+n].flatten(),
        color='r', lw=0, marker='.')

ax.plot(vnp03mod.lon[ns:ns+n, nt:nt+n].flatten(), vnp03mod.lat[ns:ns+n, nt:nt+n].flatten(),
        color='g',lw=0, marker='.')

ax.plot(vnp03dnb.lon[ns:ns+n, nt:nt+n].flatten(), vnp03dnb.lat[ns:ns+n, nt:nt+n].flatten(),
        color='b',lw=0, marker='.')

[21]:

[<matplotlib.lines.Line2D at 0x7f62d8da2470>]

../../_images/examples_notebooks_read_granules_29_1.png

[ ]: