Connect to FTP Server with username and password. Go to remote pre-determined directory and obtain listing according to given criteria. For each file in the listing, check if we do not already have the (complete) file from a previous run. If not, download it to the target directory and then record this action. Arguments: None
| Return Type | Function name | Arguments |
|---|---|---|
| hzEcode | hzFtpHost::GetAll | (void) |
Declared in file: hzFtpClient.h
Defined in file : hzFtpClient.cpp
Function Logic:
Function body:
hzEcode hzFtpHost::GetAll (void)
{
// Connect to FTP Server with username and password. Go to remote pre-determined directory and obtain listing according to given criteria. For each file in the
// listing, check if we do not already have the (complete) file from a previous run. If not, download it to the target directory and then record this action.
//
// Arguments: None
//
// Returns: E_HOSTFAIL If the collection is cut off and cannot be re-established within an acceptable period.
// E_PROTOCOL If the action fails for reasons of protocol.
// E_OK If the collection runs to completion
_hzfunc("hzFtpSite::GetAll") ;
hzMapS <hzString,hzDirent> mapFilesCurr ; // Files already downloaded for the current issue (in name order)
hzVect <hzDirent> vecFilesCurr ; // Files already downloaded for the current issue
hzVect <hzDirent> listing ; // The list of files
hzFtpClient ftp ; // The FTP client connection
hzDirent de ; // Directory entry
hzDirent de_a ; // Directory entry of file we have already (if applicable)
hzString localfile ; // Target file
hzString rento ; // Rename server file to ...
uint32_t nIndex ; // Directory entry iterator
uint32_t nAlready = 0; // When a file is skipped (it is in the listing for the criteria but we already have it), we
// increment this value.
uint32_t nDnloads = 0; // Number of files downloaded (this attempt).
uint32_t nListed = 0; // Set from the count of server files meeting the criteria. At the end we add up nAlready and
// nDnloads and only when these add up to nListed can the status be set to ACTION_COLLECTED
// otherwise status should be ACTION_STARTED
int32_t sys_rc ; // Return from system call
char* cmd ; // For system call command when needed
hzEcode rc = E_OK ; // Return from FTP functions
/*
** ** Check we have everything we need: The publication and the olddata and newdata dirs
** */
if (!m_bInit)
Fatal("Permenant Failure: No FTP Parameters\n") ;
threadLog("Collecting from server [%s,%s,(%s)] (user=%s, pass=%s) to dir %s ->", *m_Host, *m_Source, *m_Criteria, *m_Username, *m_Password, *m_Repos) ;
threadLog("OK GO!\n") ;
// Get a combined listing of the historic and current data input directories
dnh.Load(mapFilesCurr, m_Repos) ;
rc = ReadDir(vecFilesCurr, *m_Repos) ;
if (rc != E_OK)
{
threadLog("Failed to read local dir (%s)\n", *m_Repos) ;
return rc ;
}
for (nIndex = 0; nIndex < vecFilesCurr.Count() ; nIndex++)
{
de = vecFilesCurr[nIndex] ;
mapFilesCurr.Insert(de.strName(), de) ;
}
// Set up FTP params
rc = ftp.Initialize(m_Host, m_Username, m_Password) ;
if (rc != E_OK)
{
threadLog("Could not INIT FTP session to server [%s] (user=%s, pass=%s). Error is %s\n", *m_Host, *m_Username, *m_Password, Err2Txt(rc)) ;
return rc ;
}
// Start FTP session
rc = ftp.StartSession() ;
if (rc != E_OK)
{
threadLog("Could not start FTP session to server [%s] (user=%s, pass=%s). Error is %s\n", *m_Host, *m_Username, *m_Password, Err2Txt(rc)) ;
return rc ;
}
// Go to the relevent remote directory
rc = ftp.SetRemoteDir(m_Source) ;
if (rc != E_OK)
{
if (rc == E_NOTFOUND)
{
threadLog("Permenant Failure: Expected source directory %s has not been found on the server\n", *m_Source) ;
ftp.QuitSession() ;
return rc ;
}
threadLog("Temporary error: Could not CD to %s\n", *m_Source) ;
return rc ;
}
rc = ftp.SetLocalDir(m_Repos) ;
if (rc != E_OK)
{
threadLog("Permenant Failure: Expected target directory %s has not been found or is otherwise in error on the server\n",
*m_Repos) ;
ftp.QuitSession() ;
return rc ;
}
// Get the server directory listing
rc = ftp.GetDirList(listing, m_Criteria) ;
if (rc != E_OK)
{
if (rc == E_PROTOCOL)
{
threadLog("Permanent error: Could not get a listing. Protocol error\n") ;
return rc ;
}
threadLog("We have %d files in server listing but an error of %s - will rerun\n", listing.Count(), Err2Txt(rc)) ;
return rc ;
}
// Got listing but is there anything in it
if (listing.Count() == 0)
{
// Expected files were not there so try again later
threadLog("Temporary error: Got directory listing but there were no files matching [%s]\n", *m_Criteria) ;
ftp.QuitSession() ;
return rc ;
}
// Compare files and download new ones
nListed = listing.Count() ;
for (nIndex = 0; nIndex < listing.Count() ; nIndex++)
{
de = listing[nIndex] ;
if (!mapFilesCurr.Exists(de.strName()))
threadLog("Downloading file (%d of %d) %s\n", nIndex, listing.Count(), de.txtName()) ;
else
{
de_a = mapFilesCurr[de.strName()] ;
if (de_a.Size() == de.Size())
{
if (de_a.Mtime() >&eq; de.Mtime())
{
threadLog("Skiping file (%d of %d) %s\n", nIndex, listing.Count(), de.txtName()) ;
nAlready++ ;
continue ;
}
}
threadLog("Re-fetching file (%d of %d) %s\n", nIndex, listing.Count(), de.txtName()) ;
}
if (!de.strName())
{
threadLog("Critical error: An Un-named meta file has been found in the listing. Corruption suspected\n") ;
ftp.QuitSession() ;
return rc ;
}
if (!de.Size())
continue ;
// We have a new file so fetch
localfile = m_Repos + "/" + de.strName() ;
rc = ftp.FileDownload(de) ;
if (rc != E_OK)
{
if (rc == E_OPENFAIL || rc == E_WRITEFAIL)
{
threadLog("Problem opening/writing file (%s) (error=%s)\n", de.txtName(), Err2Txt(rc)) ;
return rc ;
}
ftp.QuitSession() ;
return rc ;
}
// The file is OK - Add it to download history
dnh.Append(de.Mtime(), de.Size(), de.strName()) ;
nDnloads++ ;
// Deal with case where file is zipped
if (strstr(de.txtName(), ".zip"))
{
cmd = new char[300];
sprintf(cmd, "/usr/bin/unzip -q -j -n -d %s %s", *m_Repos, *localfile) ;
sys_rc = system(cmd) ;
if (sys_rc)
{
threadLog("System call [%s] returns %d\n", cmd, sys_rc) ;
delete cmd ;
return rc ;
}
threadLog("System call [%s] returns OK\n", cmd) ;
unlink(*localfile) ;
delete cmd ;
}
}
// Quit FTP session, return to original dir and finish.
ftp.QuitSession() ;
if ((nDnloads + nAlready) == nListed)
{
rc = ReadDir(vecFilesCurr, *m_Repos) ;
if (rc != E_OK)
{
threadLog("Failed to read target dir (%s) after download - delaying for 15 mins\n", *m_Repos) ;
return rc ;
}
else
{
hzXDate now ; // System date and time
uint32_t eHi ; // Latest file found
uint32_t eIs ; // System date and time as epoch
now.SysDateTime() ;
eIs = now.AsEpoch() ;
eHi = 0;
for (nIndex = 0; nIndex < vecFilesCurr.Count() ; nIndex++)
{
de = vecFilesCurr[nIndex] ;
if (eHi < de.Mtime())
eHi = de.Mtime() ;
}
if (eHi > (eIs - 600))
threadLog("Oldest file less than 10 mins old - delaying just in case\n") ;
}
}
else
{
threadLog("Total of %d listed on server. We have %d already and %d were downloaded. %d files remain outstanding\n",
nListed, nAlready, nDnloads, (nListed - nAlready - nDnloads)) ;
}
return rc ;
}