Declared in file: hzFtpClient.h
Defined in file : hzFtpClient.cpp

Function Logic:

Function body:

hzEcode hzFtpHost::GetAll (void)
{
   _hzfunc("hzFtpSite::GetAll") ;
   hzMapS  <hzString,hzDirent> mapFilesCurr ;
   hzVect  <hzDirent>          vecFilesCurr ;
   hzVect  <hzDirent>  listing ;
   hzFtpClient ftp ;
   hzDirent    de ;
   hzDirent    de_a ;
   hzString    localfile ;
   hzString    rento ;
   uint32_t    nIndex ;
   uint32_t    nAlready = 0;
   uint32_t    nDnloads = 0;
   uint32_t    nListed = 0;
   int32_t     sys_rc ;
   char*       cmd ;
   hzEcode     rc = E_OK ;
   /*
   **  ** Check we have everything we need: The publication and the olddata and newdata dirs
   **      */
   if (!m_bInit)
       Fatal("Permenant Failure: No FTP Parameters\n") ;
   threadLog("Collecting from server [%s,%s,(%s)] (user=%s, pass=%s) to dir %s ->", *m_Host, *m_Source, *m_Criteria, *m_Username, *m_Password, *m_Repos) ;
   threadLog("OK GO!\n") ;
   dnh.Load(mapFilesCurr, m_Repos) ;
   rc = ReadDir(vecFilesCurr, *m_Repos) ;
   if (rc != E_OK)
   {
       threadLog("Failed to read local dir (%s)\n", *m_Repos) ;
       return rc ;
   }
   for (nIndex = 0; nIndex < vecFilesCurr.Count() ; nIndex++)
   {
       de = vecFilesCurr[nIndex] ;
       mapFilesCurr.Insert(de.strName(), de) ;
   }
   rc = ftp.Initialize(m_Host, m_Username, m_Password) ;
   if (rc != E_OK)
   {
       threadLog("Could not INIT FTP session to server [%s] (user=%s, pass=%s). Error is %s\n", *m_Host, *m_Username, *m_Password, Err2Txt(rc)) ;
       return rc ;
   }
   rc = ftp.StartSession() ;
   if (rc != E_OK)
   {
       threadLog("Could not start FTP session to server [%s] (user=%s, pass=%s). Error is %s\n", *m_Host, *m_Username, *m_Password, Err2Txt(rc)) ;
       return rc ;
   }
   rc = ftp.SetRemoteDir(m_Source) ;
   if (rc != E_OK)
   {
       if (rc == E_NOTFOUND)
       {
           threadLog("Permenant Failure: Expected source directory %s has not been found on the server\n", *m_Source) ;
           ftp.QuitSession() ;
           return rc ;
       }
       threadLog("Temporary error: Could not CD to %s\n", *m_Source) ;
       return rc ;
   }
   rc = ftp.SetLocalDir(m_Repos) ;
   if (rc != E_OK)
   {
       threadLog("Permenant Failure: Expected target directory %s has not been found or is otherwise in error on the server\n",
           *m_Repos) ;
       ftp.QuitSession() ;
       return rc ;
   }
   rc = ftp.GetDirList(listing, m_Criteria) ;
   if (rc != E_OK)
   {
       if (rc == E_PROTOCOL)
       {
           threadLog("Permanent error: Could not get a listing. Protocol error\n") ;
           return rc ;
       }
       threadLog("We have %d files in server listing but an error of %s - will rerun\n", listing.Count(), Err2Txt(rc)) ;
       return rc ;
   }
   if (listing.Count() == 0)
   {
       threadLog("Temporary error: Got directory listing but there were no files matching [%s]\n", *m_Criteria) ;
       ftp.QuitSession() ;
       return rc ;
   }
   nListed = listing.Count() ;
   for (nIndex = 0; nIndex < listing.Count() ; nIndex++)
   {
       de = listing[nIndex] ;
       if (!mapFilesCurr.Exists(de.strName()))
           threadLog("Downloading file (%d of %d) %s\n", nIndex, listing.Count(), de.txtName()) ;
       else
       {
           de_a = mapFilesCurr[de.strName()] ;
           if (de_a.Size() == de.Size())
           {
               if (de_a.Mtime() >&eq; de.Mtime())
               {
                   threadLog("Skiping file (%d of %d) %s\n", nIndex, listing.Count(), de.txtName()) ;
                   nAlready++ ;
                   continue ;
               }
           }
           threadLog("Re-fetching file (%d of %d) %s\n", nIndex, listing.Count(), de.txtName()) ;
       }
       if (!de.strName())
       {
           threadLog("Critical error: An Un-named meta file has been found in the listing. Corruption suspected\n") ;
           ftp.QuitSession() ;
           return rc ;
       }
       if (!de.Size())
           continue ;
       localfile = m_Repos + "/" + de.strName() ;
       rc = ftp.FileDownload(de) ;
       if (rc != E_OK)
       {
           if (rc == E_OPENFAIL || rc == E_WRITEFAIL)
           {
               threadLog("Problem opening/writing file (%s) (error=%s)\n", de.txtName(), Err2Txt(rc)) ;
               return rc ;
           }
           ftp.QuitSession() ;
           return rc ;
       }
       dnh.Append(de.Mtime(), de.Size(), de.strName()) ;
       nDnloads++ ;
       if (strstr(de.txtName(), ".zip"))
       {
           cmd = new char[300];
           sprintf(cmd, "/usr/bin/unzip -q -j -n -d %s %s", *m_Repos, *localfile) ;
           sys_rc = system(cmd) ;
           if (sys_rc)
           {
               threadLog("System call [%s] returns %d\n", cmd, sys_rc) ;
               delete cmd ;
               return rc ;
           }
           threadLog("System call [%s] returns OK\n", cmd) ;
           unlink(*localfile) ;
           delete cmd ;
       }
   }
   ftp.QuitSession() ;
   if ((nDnloads + nAlready) == nListed)
   {
       rc = ReadDir(vecFilesCurr, *m_Repos) ;
       if (rc != E_OK)
       {
           threadLog("Failed to read target dir (%s) after download - delaying for 15 mins\n", *m_Repos) ;
           return rc ;
       }
       else
       {
           hzXDate     now ;
           uint32_t    eHi ;
           uint32_t    eIs ;
           now.SysDateTime() ;
           eIs = now.AsEpoch() ;
           eHi = 0;
           for (nIndex = 0; nIndex < vecFilesCurr.Count() ; nIndex++)
           {
               de = vecFilesCurr[nIndex] ;
               if (eHi < de.Mtime())
                   eHi = de.Mtime() ;
           }
           if (eHi > (eIs - 600))
               threadLog("Oldest file less than 10 mins old - delaying just in case\n") ;
       }
   }
   else
   {
       threadLog("Total of %d listed on server. We have %d already and %d were downloaded. %d files remain outstanding\n",
           nListed, nAlready, nDnloads, (nListed - nAlready - nDnloads)) ;
   }
   return rc ;
}