Tuesday, June 1, 2010

Getting MODIS Image Automatically From FTP in Python

# This is a Python script that automatically downloads historical
# MODIS data from the LP DAAC FTP site
# This version should work for all of the tiled datasets
# It is currently hard-coded to downloaded specific MODIS tiles for
# the northern Great Plains & upper midwest

# Initailly historical date for Data transfer must be set on "lpdacc.txt".



import os, ftplib,sys,string


# Login information for accessing the LP DAAC FTP site
Hostname = "e4ftl01u.ecs.nasa.gov"
Username = "anonymous"
Password = "@anonymous"

# Get user inputs
# Base directory for the MODIS data

# Basedir = input("Enter the LP DAAC directory containing the dataset you want to download:")
Basedir="MOLT/MOD11A2.005"
print "The LP DAAC directory containing the dataset you want to download" +str(Basedir)


# Local directory for data storage
#Hdfdir = input("Enter the local directory where you want to store the hdf files:")
Hdfdir=r"H:\MODIS_LST_NDVI\MOD11A2\\"
print "The local directory where you want to store the hdf files" +str(Hdfdir)


# Empty lists for the collector functions
Dirlist = []
Filelist = []

mylog=open(r"H:\MODIS_LST_NDVI\MOD11A2\mylog.txt",'w',)

# Assigning Global ariables
i=0
k=0
flag=0

try:

# Define helper functions that are used to read files/subdirectories from the
# ftp site and store them as lists
def collector(line = ''):
global Dirlist
Dirlist.append(line)

def collector2(line = ''):
global Filelist
Filelist.append(line)



# Open ftp connection
ftp = ftplib.FTP(Hostname,Username,Password)

# Go to the directory containing the dataset of interest
ftp.cwd(Basedir)

# Involke the LIST ftp function, calling the collector function to store the
# results to Dirlist in list format
ftp.retrlines("LIST", collector)

# Get Directory listing only (Without including the sub directories)
mainDirlist=[]
myDirlist=[]
ftp.dir(mainDirlist.append)
myDirlist=mainDirlist[1:]
# parsing the Directory list
dirInfo=""


for mainDir in myDirlist:
#parsing the directory name only[2002.12.03]
mainDirname= mainDir[37:47]
dd=mainDirname[8:10]
mm=mainDirname[5:7]
yyyy=mainDirname[0:4]
#Extracging yyyy mm dd to compare with log file information
dirInfo=str(yyyy)+str(mm)+str(dd)
print mainDirname
print dirInfo

# Read the log file to retrive the information of latest downloaded data
logFileread=open(r"H:\MODIS_LST_NDVI\MOD11A2\lpdaac.txt",'r')
logFileread.seek(0)
logInfo=int(logFileread.read())
logFileread.close()

print"Local Drive Recent Log Dir # "+str(logInfo)
mylog.write("Local Drive Recent Log Dir # "+str(logInfo)+'\n')

# Coparing the logfile(already downloaded data) with recent datas in the ftp
if(int(logInfo) print "current path -->"+str(ftp.pwd())
if(flag==1): # if flag matches the criteria reset the counters
ftp.cwd("..")
k=0
flag=0
Filelist = []

# List all the files in the subdirectory
path=str(mainDirname)
ftp.cwd(path)
print "New path -->"+str(ftp.pwd())
# Filelist = ftp.dir()
##FTP Directory bhitra chire pni file ma chire ko chhina
ftp.retrlines("LIST", collector2)
#ftp.retrlines("LIST")
# Download data from the MODIS tiles that we are interested in
for Currow2 in Filelist:
Splitrow2 = Currow2.split()
Permissions = Splitrow2[0]

# Skip over the jpeg browse images - some of these cause problems
if Permissions[0:3] == "-rw":
Directories = Splitrow2[1]
Group = Splitrow2[2]
Size = Splitrow2[3]
Month = Splitrow2[4]
Date = Splitrow2[5]
Time = Splitrow2[6]
Filename = Splitrow2[7]
mylog.write(Filename)
LocalFile = Hdfdir + Filename
Splitfname = Filename.split(".")
# Split the header file name into its various components
Splitfname = Filename.split(".")
Mdataset = Splitfname[0]
Maqdate = Splitfname[1]
Mlocation = Splitfname[2]
Mprocdate = Splitfname[3]
Mext1 = Splitfname[4]
Mext2 = Splitfname[5]


# Pull out the horizontal and vertical tile numbers
Htile = Mlocation[1:3]
Vtile = Mlocation[4:6]


# Only retrieve data for the three tiles covering the NGP/Upper Midwest
if (((Htile == "09") & (Vtile == "04"))|((Htile == "10") & (Vtile == "04"))|((Htile == "11") & (Vtile == "04"))|((Htile == "12") & (Vtile == "04"))|((Htile == "09") & (Vtile == "05"))|((Htile == "10") & (Vtile == "05"))|((Htile == "11") & (Vtile == "05"))):
# Retrieve the hdf and xml files and place them in the local directory

ftp.retrbinary("RETR " + Filename, open(LocalFile, "wb").write)

# Write download information in the log file
logFwrite=open(r"H:\MODIS_LST_NDVI\MOD11A2\lpdaac.txt",'w')
logFwrite.seek(0)
logFwrite.write(dirInfo)
logFwrite.close()

k=k+1
if(k==14): # value of k should be no of tiles*2[for *.hdf and *.hdf.xml ]
flag=1

else:
print i
i=i+1
print "loginfor-->"+str(logInfo)
print "dirInfor-->"+str(dirInfo)
print "Dirname-->"+str(mainDirname)
print "Filename-->"+str(Filename)
mylog.write("loginfor-->"+str(logInfo)+'\n')
mylog.write("dirInfor-->"+str(dirInfo)+'\n')
mylog.write("Dirname-->"+str(mainDirname)+'\n')
mylog.write("Filename-->"+str(Filename)+'\n')



else:
print "Already downloaded in our local drive" +str(mainDirname)
mylog.write( "Already downloaded in our local drive" +str(mainDirname)+'\n')



finally:
ftp.quit()
ftp.close()
print "Closing FTP"
mylog.write("Closing FTP")
mylog.close()

MODIS , Python , remote sensing

7 comments :

  1. How can I get 'lpdacc.txt' file. I don't know how to creat it. Thanks for any help!

    ReplyDelete
    Replies
    1. It could be any file to store the record of latest download date, in my case I used text file for its simplicity. It has the date in MMDDYYYY format and will update record on each download and then the record will be compare with MODIS archive to get updated files.

      Delete
    2. but at the first time, this file don't have anything to read. It's make script can't run anymore :(.
      Is that's right?

      Delete
    3. How do I get to know the area/tile/coordinate form the file name? e.g.if the file name is MOD04_3K.A2018002.0545.006.2018002134322.hdf,
      then how do I identify the tiles/coordinates from this file name?

      Delete
    4. You can not get tile info directly from the file name- its naming convention is -
      MOD04_3K.AYYYYDDD.HHMM.CCC.YYYYDDDHHMMSS.hdf
      AYYYYDDD = Year and Day of Year of acquisition
      HHMM = Hour and Minute of acquisition
      CCC = Collection number
      YYYYDDDHHMMSS = Production Date and Time

      Delete
  2. Yes you are partially correct at initial create a lpdaac.txt, write 12012008 and save it to download data after December 01 2008.

    ReplyDelete
    Replies
    1. thank you! now I understood. I still have some error but I'll fix it. :)

      Delete

 

© 2011 GIS and Remote Sensing Tools, Tips and more .. ToS | Privacy Policy | Sitemap

About Me