Epoll server in edge-triggered mode. In this mode, all client connections are non-blocking and notification that a socket has data available for reaading, is only given at the point the data arrives. Consequently, sockets must always be read until no more data is available. Arguments: None Returns: None Note: Call only once for all listening sockets! This function will not return until the server is shut down
| Return Type | Function name | Arguments |
|---|---|---|
| void | hzIpServer::Serve | (void) |
Declared in file: hzIpServer.h
Defined in file : hzIpServer.cpp
Function Logic:
Function body:
void hzIpServer::Serve (void)
{
// Category: Internet Server
//
// Epoll server in edge-triggered mode. In this mode, all client connections are non-blocking and notification that a socket has data available for reaading, is only given at
// the point the data arrives. Consequently, sockets must always be read until no more data is available.
//
// Arguments: None
// Returns: None
//
// Note: Call only once for all listening sockets! This function will not return until the server is shut down
_hzfunc("hzIpServer::Serve") ;
hzList <hzIpListen*>::Iter I ; // Listening sockets iterator
struct epoll_event epEventNew ; // Epoll event for new connections
hzPacket tbuf ; // Fixed buffer for single IP packet
// SOCKADDR cliAddr ; // Client address
SOCKADDRIN cliAddrInet ; // INET client address
SOCKADDRUN cliAddrUnix ; // UNIX client address
socklen_t cliLenInet ; // INET client address length
socklen_t cliLenUnix ; // UNIX client address length
hzIpListen* pLS ; // Listening socket
hzIpConnex* pCC ; // Connected client
SSL* pUDP_SSL ; // SSL session (if applicable)
hzProcInfo proc_data ; // Process data (Client socket & IP address, must be destructed by the called thread function)
hzIpinfo* ipi ; // IP address for testing against black/white lists
hzChain errMsg ; // For building error message for logs
hzXDate now ; // Time now (used to log connections)
pthread_attr_t tattr ; // Thread attribute (to support thread invokation)
timeval tv ; // Time limit for epoll call
// uint64_t nsThen ; // Before the wait
// uint64_t nsNow ; // After the wait
hzIpaddr ipa ; // IP address
uint32_t nCliSeq ; // Client connection id allocator
// uint32_t nLoop ; // Number of times round the epoll event loop
uint32_t nSlot ; // Connections iterator
uint32_t nEpollEvents ; // Result of the epoll call
uint32_t nError ; // Errno of the epoll call
uint32_t cSock ; // Client socket (from accept)
uint32_t cPort ; // Client socket (from accept)
// uint32_t prc ; // Return from pthread_create
// uint32_t nBannedAttempts ; // Connection attempts by blocked IP address
int32_t flags ; // Flags to make socket non-blocking
int32_t nRecv ; // No bytes read from client socket after epoll (in one read)
int32_t nRecvTotal ; // No bytes read from client socket after epoll (in a series of reads)
int32_t sys_rc ; // Return from system library calls
int32_t sys_err ; // Derived error e.g. SSL_get_error()
int32_t xmitState ; // Return from _xmit() call
// int32_t nTries ; // Attempts to call SSL_accept()
bool bCloseSSL ; // Need to close underlying connection because of a non-recoverable SSL_accept failure
hzTcpCode trc ; // Return code
char ipbuf[44]; // Client IP address textform buffer
// Pre-define thread attributes
pthread_attr_init(&tattr) ;
pthread_attr_setdetachstate(&tattr, PTHREAD_CREATE_DETACHED) ;
// Init connected client regime
for (nSlot = 0; nSlot < 1024;nSlot++)
currCC[nSlot] = 0;
cliLenInet = sizeof(cliAddrInet) ;
cliLenUnix = sizeof(cliAddrUnix) ;
// Create epoll socket and and listening sockets to the controller
s_epollSocket = epoll_create1(0);
if (s_epollSocket == -1)
Fatal("Could not create epoll socket\n") ;
m_pLog->Log("Epoll socket is %d\n", s_epollSocket) ;
for (I = m_LS ; I.Valid() ; I++)
{
pLS = I.Element() ;
epEventNew.data.fd = pLS->GetSocket() ;
epEventNew.events = EPOLLIN ;
if (epoll_ctl(s_epollSocket, EPOLL_CTL_ADD, pLS->GetSocket(), &epEventNew) < 0)
Fatal("Could not add listening socket %d to the epoll controller\n", pLS->GetSocket()) ;
m_pLog->Log("Added listening socket is %d\n", pLS->GetSocket()) ;
m_mapLS.Insert(pLS->GetSocket(), pLS) ;
}
/*
** ** Main loop - waiting for events
** */
// m_nLoop = nCliSeq = nBannedAttempts = 0 ;
m_nLoop = nCliSeq = 0;
for (;;)
{
// Check for shutdown condition
if (m_bShutdown)
{
// Check for outstanding connections
for (nSlot = 0; nSlot < 1024;nSlot++)
{
if (currCC[nSlot])
break ;
}
if (nSlot == 1024)
break ;
}
// Check for expired connections
for (nSlot = 0; nSlot < ConnInError.Count() ; nSlot++)
{
pCC = ConnInError.GetObj(nSlot) ;
}
// Wait for events
// nsThen = RealtimeNano() ;
nEpollEvents = epoll_wait(s_epollSocket, m_arEvents, MAXEVENTS, 60000);
// nsNow = RealtimeNano() ;
m_nLoop++ ;
if (!nEpollEvents)
m_pLog->Log("Loop %u No action\n", m_nLoop) ;
// else
// m_pLog->Log("Loop %u: Waited %lu nanoseconds for %d events\n", m_nLoop, nsNow - nsThen, nEpollEvents) ;
for (nSlot = 0; nSlot < nEpollEvents ; nSlot++)
{
/*
** ** Check for hangups and other connection errors
** */
if (m_arEvents[nSlot].events & EPOLLHUP)
{
// Hangup signal
cSock = m_arEvents[nSlot].data.fd ;
pCC = currCC[cSock] ;
nError = 0;
if (!pCC) { m_bShutdown = true ; m_pLog->Log("Loop %u slot %u: HANGUP CORRUPT Sock %d No connector\n", m_nLoop, nSlot, cSock) ; continue ; }
if (!pCC->CliSocket()) { m_bShutdown = true ; m_pLog->Log("Loop %u slot %u: HANGUP CORRUPT Sock %d Connector defunct\n", m_nLoop, nSlot, cSock) ; continue ; }
if (getsockopt(cSock, SOL_SOCKET, SO_ERROR, (void *)&nError, &cliLenInet) == 0)
m_pLog->Log("Loop %u slot %u: HANGUP on socket %d (%s)\n", m_nLoop, nSlot, cSock, strerror(nError)) ;
else
m_pLog->Log("Loop %u slot %u: HANGUP on socket %d (no details)\n", m_nLoop, nSlot, cSock) ;
// Remove the connection
pCC->Terminate() ;
delete pCC ;
currCC[cSock] = 0;
continue ;
}
if (m_arEvents[nSlot].events & EPOLLERR)
{
// An error has occured on this fd, or the socket is not ready for reading
cSock = m_arEvents[nSlot].data.fd ;
pCC = currCC[cSock] ;
nError = 0;
if (!pCC) { m_bShutdown = true ; m_pLog->Log("Loop %u slot %u: EPOLLERR CORRUPT Sock %d No connector\n", m_nLoop, nSlot, cSock) ; continue ; }
if (!pCC->CliSocket()) { m_bShutdown = true ; m_pLog->Log("Loop %u slot %u: EPOLLERR CORRUPT Sock %d Connector defunct\n", m_nLoop, nSlot, cSock) ; continue ; }
if (getsockopt(cSock, SOL_SOCKET, SO_ERROR, (void *)&nError, &cliLenInet) == 0)
m_pLog->Log("Loop %u slot %u: EPOLLERR on socket %d (%s)\n", m_nLoop, nSlot, cSock, strerror(nError)) ;
else
m_pLog->Log("Loop %u slot %u: EPOLLERR on socket %d (no details)\n", m_nLoop, nSlot, cSock) ;
// Remove the connection
pCC->Terminate() ;
delete pCC ;
currCC[cSock] = 0;
continue ;
}
/*
** ** Events on listening sockets
** */
if (m_mapLS.Exists(m_arEvents[nSlot].data.fd))
{
// We have a notification on the listening socket so either a new incoming connection or if the listening socket is UDP, an incoming UDP packet.
pLS = m_mapLS[m_arEvents[nSlot].data.fd] ;
pUDP_SSL = 0;
if (pLS->UseUDP())
{
// Listening socket expects UDP clients
cSock = pLS->GetSocket() ;
flags = fcntl(pLS->GetSocket(), F_GETFL, 0);
if (flags == -1)
{
m_pLog->Log("UDP: Loop %u: NOTE: Could not make client socket %d non blocking (case 1)\n", m_nLoop, pLS->GetSocket()) ;
if (close(pLS->GetSocket()) < 0)
m_pLog->Log("Loop %u: NOTE: Could not close socket %d after non_block_fail(1) errno=%d\n", m_nLoop, pLS->GetSocket(), errno) ;
continue ;
}
flags |= O_NONBLOCK ;
if (fcntl(pLS->GetSocket(), F_SETFL, flags) == -1)
{
m_pLog->Log("UDP: Loop %u: NOTE: Could not make client socket %d non blocking (case 2)\n", m_nLoop, pLS->GetSocket()) ;
if (close(pLS->GetSocket()) < 0)
m_pLog->Log("Loop %u: NOTE: Could not close socket %d after non_block_fail(2) errno=%d\n", m_nLoop, pLS->GetSocket(), errno) ;
continue ;
}
if ((nRecv = recvfrom(pLS->GetSocket(), tbuf.m_data, HZ_MAXPACKET, 0,(SOCKADDR*) &cliAddrInet, &cliLenInet)) < 0)
{
m_pLog->Log("UDP: Loop %u: NOTE: Could not read from UDP client\n", m_nLoop) ;
continue ;
}
// Get client IP address
ipa = (uint32_t) cliAddrInet.sin_addr.s_addr ;
cPort = ntohs(cliAddrInet.sin_port) ;
tbuf.m_data[nRecv] = 0;
pCC = udpClients[ipa] ;
if (!pCC)
{
pCC = new hzIpConnex(m_pLog) ;
udpClients.Insert(ipa, pCC) ;
// Initialize and oxygenate the connection
pCC->Initialize(pLS, pUDP_SSL, ipa, cSock, cPort, nCliSeq++) ;
if (pCC->m_OnConnect)
pCC->m_OnConnect(pCC) ;
}
m_pLog->Out("Received UDP message %s from port %d internet address %s\n", tbuf.m_data, cPort, *ipa) ;
if (pCC->MsgReady())
trc = pCC->m_OnIngress(pCC->InputZone(), pCC) ;
else
trc = TCP_INCOMPLETE ;
switch (trc)
{
case TCP_TERMINATE:
if (pCC->_isxmit())
{
// If outgoing data not complete, transmit some more.
xmitState = pCC->_xmit(tbuf) ;
if (xmitState < 0)
{
m_pLog->Log("UDP: Loop %u slot %u: TCP_TERMINATE: Write error - Sock %d/%d to be removed\n", m_nLoop, nSlot, cSock, pCC->CliPort()) ;
pCC->Terminate() ;
udpClients.Delete(ipa) ;
delete pCC ;
break ;
}
if (xmitState > 0)
{
// We have EAGAIN or EWOULDBLOCK and so now we make epoll entry to look for write event (when ocket becomes writable)
m_pLog->Log("UDP: Loop %u slot %u: TCP_TERMINATE: Write delay - Sock %d /%d, looking for write event\n",
m_nLoop, nSlot, cSock, pCC->CliPort()) ;
break ;
}
}
// No more outgoing data so close
m_pLog->Log("UDP: Loop %u slot %u: TCP_TERMINATE: Normal termination, sock %d/%d to be removed\n", m_nLoop, nSlot, cSock, pCC->CliPort()) ;
pCC->Terminate() ;
udpClients.Delete(ipa) ;
delete pCC ;
break ;
case TCP_KEEPALIVE:
m_pLog->Log("UDP: Client %d SERVER - data processed, awaiting further messages\n", pCC->EventNo()) ;
pCC->Oxygen() ;
if (pCC->_isxmit())
{
xmitState = pCC->_xmit(tbuf) ;
if (xmitState < 0)
{
m_pLog->Log("UDP: Loop %u slot %u: TCP_KEEPALIVE: Write error, sock %d/%d to be removed\n", m_nLoop, nSlot, cSock, pCC->CliPort()) ;
pCC->Terminate() ;
udpClients.Delete(ipa) ;
delete pCC ;
break ;
}
if (xmitState > 0)
{
// We have EAGAIN or EWOULDBLOCK and so now we make epoll entry to look for write event (when ocket becomes writable)
m_pLog->Log("UDP: Loop %u slot %u: TCP_KEEPALIVE: Write delay - Sock %d/%d retained\n", m_nLoop, nSlot, cSock, pCC->CliPort()) ;
break ;
}
// We have xmitState of 0 which means the message was written in full to the socket
m_pLog->Log("UDP: Loop %u slot %u: TCP_KEEPALIVE: Write OK, sock %d/%d retained\n", m_nLoop, nSlot, cSock, pCC->CliPort()) ;
}
if (pCC->IsCliTerm())
{
// No longer any incoming requests and as we have finished writing, close
m_pLog->Log("UDP: Loop %u loop %u: TCP_KEEPALIVE: Client done so sock %d/%d to be removed\n", m_nLoop, nSlot, cSock, pCC->CliPort()) ;
pCC->Terminate() ;
udpClients.Delete(ipa) ;
delete pCC ;
break ;
}
break ;
case TCP_INCOMPLETE:
m_pLog->Log("UDP: Client %d SERVER - data incomplete\n", pCC->EventNo()) ;
pCC->Oxygen() ;
break ;
case TCP_INVALID: // The message processor cannot dechipher the message. Kill the connection
if (m_pLog)
m_pLog->Log("UDP: Loop %u: Sock %d/%d INVALID TCP code\n", m_nLoop, cSock, pCC->CliPort()) ;
pCC->Terminate() ;
udpClients.Delete(ipa) ;
delete pCC ;
break ;
}
// End of UDP section
continue ;
}
if (pLS->UseUNIX())
{
cSock = accept(pLS->GetSocket(), (SOCKADDR*) &cliAddrUnix, &cliLenUnix) ;
if (cSock < 0)
{
m_pLog->Log("Loop %u: NOTE: Listening socket %d ignored local client on port %d\n", m_nLoop, pLS->GetSocket(), pLS->GetPort()) ;
if (errno && errno != EAGAIN && errno != EWOULDBLOCK)
{
m_bShutdown = true ;
m_pLog->Log("Loop %u: SHUTDOWN: Listening socket %d failure: Port %d (errno=%d)\n", m_nLoop, pLS->GetSocket(), pLS->GetPort(), errno) ;
}
continue ;
}
if (_hzGlobal_Debug & HZ_DEBUG_SERVER)
m_pLog->Log("Loop %u: Accepted UNIX connection: socket %u port %u\n", m_nLoop, cSock, pLS->GetPort()) ;
pCC = new hzIpConnex(m_pLog) ;
if (!pCC)
{
m_pLog->Log("ERROR: No memory for UNIX client on socket %d. Closing conection\n", cSock) ;
if (close(cSock) < 0)
m_pLog->Log("NOTE: Could not close socket %d after memory allocation failure. errno=%d\n", cSock, errno) ;
m_bShutdown = true ;
continue ;
}
if (_nonblock(cSock) != E_OK)
continue ;
strcpy(ipbuf, "127.0.0.1") ;
// End of UNIX socket section
// continue ;
}
else
{
// Handle new TCP client connections
// Accept the client no matter what, then if there is a problem close the connection.
cSock = accept(pLS->GetSocket(), (SOCKADDR*) &cliAddrInet, &cliLenInet) ;
if (cSock < 0)
{
m_pLog->Log("Loop %u: NOTE: Listening socket %d ignored client on port %d\n", m_nLoop, pLS->GetSocket(), pLS->GetPort()) ;
if (errno && errno != EAGAIN && errno != EWOULDBLOCK)
{
m_bShutdown = true ;
m_pLog->Log("Loop %u: SHUTDOWN: Listening socket %d failure: Port %d (errno=%d)\n", m_nLoop, pLS->GetSocket(), pLS->GetPort(), errno) ;
}
continue ;
}
// Get client IP address and port
inet_ntop(AF_INET, &cliAddrInet.sin_addr, ipbuf, 16);
ipa = ipbuf ;
cPort = ntohs(cliAddrInet.sin_port) ;
// Check if client IP is blocked. Note this does not work when connections are coming via a local proxy service as the IP address will always be 127.0.0.1
if (_hzGlobal_StatusIP.Exists(ipa))
{
ipi = &(_hzGlobal_StatusIP[ipa]) ;
if (!(ipi->m_bInfo & HZ_IPSTATUS_WHITE))
{
if (!(ipi->m_nSince%100))
m_pLog->Log("BLOCKED IP %s reaches %u attempts\n", *ipa, ipi->m_nSince) ;
if (close(cSock) < 0)
m_pLog->Log("ERROR: Could not close socket %d after blocked IP address detected. errno=%d\n", cSock, errno) ;
ipi->m_nSince++ ;
// ipi->m_nTotal++ ;
// nBannedAttempts++ ;
// if (!(nBannedAttempts%10000))
// m_pLog->Log("BLOCKED IP TOTAL reaches %u attempts\n", nBannedAttempts) ;
continue ;
}
}
if (_hzGlobal_Debug & HZ_DEBUG_SERVER)
m_pLog->Log("Loop %u: Accepted connection (1): socket %d/%d host %s port %d\n", m_nLoop, cSock, cPort, ipbuf, pLS->GetPort()) ;
}
/*
** ** Allocate the connected client object
** */
// Check that slot in the connected client array (as indicated by the client socket), is actually free
pCC = currCC[cSock] ;
if (pCC)
{
m_pLog->Log("Loop %u slot %u: CORRUPT: Existing client connection handler on sock %d/%d.\n", m_nLoop, nSlot, cSock, pCC->CliPort()) ;
if (close(cSock) < 0)
m_pLog->Log("NOTE: Could not close socket %d after memory allocation failure. errno=%d\n", cSock, errno) ;
m_bShutdown = true ;
continue ;
}
// Allocate the connected client object
pCC = new hzIpConnex(m_pLog) ;
if (!pCC)
{
m_pLog->Log("ERROR: No memory for client %s on socket %d. Closing conection\n", ipbuf, cSock) ;
if (close(cSock) < 0)
m_pLog->Log("NOTE: Could not close socket %d after memory allocation failure. errno=%d\n", cSock, errno) ;
m_bShutdown = true ;
continue ;
}
if (_nonblock(cSock) != E_OK)
continue ;
if (pLS->UseSSL())
{
// Set timeout to 0.1 secs.
tv.tv_sec = 0;
tv.tv_usec = 100000;
if (setsockopt(cSock, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof(tv)) < 0)
{
m_pLog->Log("Loop %u: NOTE: Could not set send socket options\n", m_nLoop) ;
if (close(cSock) < 0)
m_pLog->Log("Loop %u: NOTE: Could not close socket %d after setop_send errno=%d\n", m_nLoop, cSock, errno) ;
continue ;
}
if (setsockopt(cSock, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)) < 0)
{
m_pLog->Log("Loop %u: NOTE: Could not set recv socket options\n", m_nLoop) ;
if (close(cSock) < 0)
m_pLog->Log("Loop %u: NOTE: Could not close socket %d after setop_recv errno=%d\n", cSock, errno) ;
continue ;
}
// Create a new SSL session instance
m_pLog->Log("Loop %u: NOTE: SSL instance. Client %s:%d\n", m_nLoop, ipbuf, pLS->GetPort()) ;
pCC->m_pSSL = SSL_new(s_SSL_svrRegime->m_svrCTX) ; // s_svrCTX) ;
if (!pCC->m_pSSL)
{
m_pLog->Log("Loop %u: NOTE: Failed to allocate an SSL instance on port %d\n", m_nLoop, pLS->GetPort()) ;
if (close(cSock) < 0)
{
m_pLog->Log("NOTE: Could not close socket %d after memory allocation failure. errno=%d\n", cSock, errno) ;
m_bShutdown = true ;
}
continue ;
}
m_pLog->Log("Loop %u: Allocated SSL instance on port %d\n", m_nLoop, pLS->GetPort()) ;
// Set the SSL session socket
SSL_set_accept_state(pCC->m_pSSL) ;
sys_rc = SSL_set_fd(pCC->m_pSSL, cSock) ;
errMsg.Clear() ;
errMsg.Printf("Loop %u: SSL_set_fd %d - ", m_nLoop, sys_rc) ;
bCloseSSL = false ;
sys_rc = SSL_accept(pCC->m_pSSL) ;
if (sys_rc >&eq; 1)
{
pCC->m_bAcceptSSL = false ;
errMsg << "SSL_accept OK\n" ;
}
else
{
pCC->m_bAcceptSSL = true ;
sys_err = SSL_get_error(pCC->m_pSSL, sys_rc) ;
errMsg.Printf("Failed to accept SSL client port %d sock %d ret %d err=%d ", pLS->GetPort(), cSock, sys_rc, sys_err) ;
switch (sys_err)
{
case SSL_ERROR_ZERO_RETURN: errMsg << "SSL_ERROR_ZERO_RETURN\n" ; bCloseSSL = true ; break ;
case SSL_ERROR_WANT_READ: errMsg << "SSL_ERROR_WANT_READ\n" ; break ;
case SSL_ERROR_WANT_WRITE: errMsg << "SSL_ERROR_WANT_WRITE\n" ; break ;
case SSL_ERROR_WANT_CONNECT: errMsg << "SSL_ERROR_WANT_CONNECT\n" ; break ;
case SSL_ERROR_WANT_ACCEPT: errMsg << "SSL_ERROR_WANT_ACCEPT\n" ; break ;
case SSL_ERROR_WANT_X509_LOOKUP: errMsg << "SSL_ERROR_WANT_ACCEPT\n" ; break ;
case SSL_ERROR_WANT_ASYNC: errMsg << "SSL_ERROR_WANT_ASYNC\n" ; break ;
case SSL_ERROR_WANT_ASYNC_JOB: errMsg << "SSL_ERROR_WANT_ASYNC_JOB\n" ; break ;
case SSL_ERROR_WANT_CLIENT_HELLO_CB: errMsg << "SSL_ERROR_WANT_CLIENT_HELLO_CB\n" ; break ;
case SSL_ERROR_SYSCALL: errMsg << "SSL_ERROR_SYSCALL\n" ; bCloseSSL = true ; break ;
case SSL_ERROR_SSL: errMsg << "SSL_ERROR_SSL\n" ; bCloseSSL = true ; break ;
}
}
m_pLog->Log(errMsg) ;
if (bCloseSSL)
{
SSL_free(pCC->m_pSSL) ;
if (close(cSock) < 0)
{
m_pLog->Log("Loop %u: NOTE: Could not close socket %d after SSL no_alloc errno=%d\n", m_nLoop, cSock, errno) ;
m_bShutdown = true ;
}
continue ;
}
}
// Deal with the case where we have too many connections or we are shutting down.
if (m_bShutdown || pLS->GetActConn() >&eq; pLS->GetMaxConn())
{
m_pLog->Log("Loop %u: NOTE: System too busy: Curr %d Max %d stat %d\n", m_nLoop, pLS->GetActConn(), m_nMaxClients, m_bShutdown ? 1: 0);
if (close(cSock) < 0)
m_pLog->Log("Loop %u: NOTE: Could not close socket %d after sys_too_busy errno=%d\n", m_nLoop, cSock, errno) ;
continue ;
}
// Client is already connected. Do we allow further connections?
/*
** if (_ipcheck(ipbuf, pLS->GetPort()))
** {
** m_pLog->Log("Loop %u: NOTE: Lorris Attack from IP %s port %d\n", ipbuf, pLS->GetPort()) ;
** if (close(cSock) < 0)
** m_pLog->Log("Loop %u: NOTE: Could not close socket %d after lorris_attack errno=%d\n", m_nLoop, cSock, errno) ;
** continue ;
** }
** */
// All is well so set socket options
tv.tv_sec = 20;
tv.tv_usec = 0;
if (setsockopt(cSock, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof(tv)) < 0)
{
m_pLog->Log("Loop %u: NOTE: Could not set send socket options\n", m_nLoop) ;
if (close(cSock) < 0)
m_pLog->Log("Loop %u: NOTE: Could not close socket %d after setop_send errno=%d\n", m_nLoop, cSock, errno) ;
continue ;
}
if (setsockopt(cSock, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)) < 0)
{
m_pLog->Log("Loop %u: NOTE: Could not set recv socket options\n", m_nLoop) ;
if (close(cSock) < 0)
m_pLog->Log("Loop %u: NOTE: Could not close socket %d after setop_recv errno=%d\n", cSock, errno) ;
continue ;
}
// Add new client socket to the epoll control
epEventNew.data.fd = cSock ;
epEventNew.events = EPOLLIN ;
if (epoll_ctl(s_epollSocket, EPOLL_CTL_ADD, cSock, &epEventNew) < 0)
{
m_pLog->Log("Loop %u slot %u: EPOLL ERROR: Could not add client connection handler on sock %d/%d. Error=%s\n",
m_nLoop, nSlot, cSock, pCC->CliPort(), strerror(errno)) ;
if (close(cSock) < 0)
m_pLog->Log("NOTE: Could not close socket %d after memory allocation failure. errno=%d\n", cSock, errno) ;
m_bShutdown = true ;
delete pCC ;
continue ;
}
// Initialize and oxygenate the connection
currCC[cSock] = pCC ;
pCC->Initialize(pLS, pCC->m_pSSL, ipa, cSock, cPort, nCliSeq++) ;
if (pCC->m_OnConnect)
pCC->m_OnConnect(pCC) ;
now.SysDateTime() ;
m_pLog->Log("Loop %u slot %u: NEW Connection on socket %d/%d from %s:%u\n", m_nLoop, nSlot, cSock, pCC->CliPort(), ipbuf, pLS->GetPort()) ;
continue ;
// End of listening socket handling.
}
/*
** ** WRITE Events
** */
if (m_arEvents[nSlot].events & EPOLLOUT)
{
// Subsequent to a call to hzIpConnex::SendData to place a response in the output queue and to modify the epoll entry for the connection so the socket is monitored
// for writability, we now have notification that the socket is writable.
cSock = m_arEvents[nSlot].data.fd ;
pCC = currCC[cSock] ;
if (!pCC)
{
m_pLog->Log("Loop %u slot %u: CORRUPT: Write event on socket %d/%d but no connector!\n", m_nLoop, nSlot, cSock, pCC->CliPort()) ;
m_bShutdown = true ;
continue ;
}
if (cSock != pCC->CliSocket())
{
m_pLog->Log("Loop %u slot %u: CORRUPT: Write event on sock %d/%d but attached client object has socket of %d\n",
m_nLoop, nSlot, cSock, pCC->CliPort(), pCC->CliSocket()) ;
m_bShutdown = true ;
continue ;
}
if (pCC->IsCliBad())
{
m_pLog->Log("Loop %u slot %u: BAD Client, connection on socket %d/%d to be removed\n", m_nLoop, nSlot, cSock, pCC->CliPort()) ;
pCC->Terminate() ;
delete pCC ;
currCC[cSock] = 0;
continue ;
}
if (pCC->m_bAcceptSSL)
{
// Not Complete - Call SSL_accept() again
// SSL_set_accept_state(pCC->m_pSSL) ;
// sys_rc = SSL_set_fd(pCC->m_pSSL, cSock) ;
// m_pLog->Log("Subsequent SSL_set_fd returns %d\n", sys_rc) ;
sys_rc = SSL_accept(pCC->m_pSSL) ;
if (sys_rc > 0)
{
pCC->m_bAcceptSSL = false ;
m_pLog->Log("Subsequent SSL_accept OK %d\n", sys_rc) ;
}
else
{
sys_err = SSL_get_error(pCC->m_pSSL, sys_rc) ;
m_pLog->Log("Loop %u: NOTE: Subsequent Failed to accept SSL client port %d sock %d ret %d err=%d\n", m_nLoop, pLS->GetPort(), cSock, sys_rc, sys_err) ;
switch (sys_err)
{
case SSL_ERROR_ZERO_RETURN: bCloseSSL = true ; m_pLog->Log("SSL_ERROR_ZERO_RETURN\n") ; break ;
case SSL_ERROR_WANT_READ: bCloseSSL = false ; m_pLog->Log("SSL_ERROR_WANT_READ\n") ; break ;
case SSL_ERROR_WANT_WRITE: bCloseSSL = false ; m_pLog->Log("SSL_ERROR_WANT_WRITE\n") ; break ;
case SSL_ERROR_WANT_CONNECT: bCloseSSL = false ; m_pLog->Log("SSL_ERROR_WANT_CONNECT\n") ; break ;
case SSL_ERROR_WANT_ACCEPT: bCloseSSL = false ; m_pLog->Log("SSL_ERROR_WANT_ACCEPT\n") ; break ;
case SSL_ERROR_WANT_X509_LOOKUP: bCloseSSL = false ; m_pLog->Log("SSL_ERROR_WANT_ACCEPT\n") ; break ;
case SSL_ERROR_WANT_ASYNC: bCloseSSL = false ; m_pLog->Log("SSL_ERROR_WANT_ASYNC\n") ; break ;
case SSL_ERROR_WANT_ASYNC_JOB: bCloseSSL = false ; m_pLog->Log("SSL_ERROR_WANT_ASYNC_JOB\n") ; break ;
case SSL_ERROR_WANT_CLIENT_HELLO_CB: bCloseSSL = false ; m_pLog->Log("SSL_ERROR_WANT_CLIENT_HELLO_CB\n") ; break ;
case SSL_ERROR_SYSCALL: bCloseSSL = true ; m_pLog->Log("SSL_ERROR_SYSCALL\n") ; break ;
case SSL_ERROR_SSL: bCloseSSL = true ; m_pLog->Log("SSL_ERROR_SSL\n") ; break ;
}
if (bCloseSSL)
{
// SSL_free(pCC->m_pSSL) ;
pCC->Terminate() ;
delete pCC ;
currCC[cSock] = 0;
// if (close(cSock) < 0)
// {
// m_pLog->Log("Loop %u: NOTE: Could not close socket %d after SSL no_alloc errno=%d\n", m_nLoop, cSock, errno) ;
// m_bShutdown = true ;
// }
}
continue ;
}
}
if (!pCC->_isxmit() && _hzGlobal_Debug & HZ_DEBUG_SERVER)
m_pLog->Log("Loop %u slot %u: Write event with nothing socket %d/%d\n", m_nLoop, nSlot, cSock, pCC->CliPort()) ;
if (pCC->_isxmit())
{
// m_Track.Printf("Loop %u slot %u: Write event with output on socket %d/%d\n", m_nLoop, nSlot, cSock, pCC->CliPort()) ;
xmitState = pCC->_xmit(tbuf) ;
if (xmitState < 0)
{
m_pLog->Log("Loop %u slot %u: EPOLLOUT Write error, connection on socket %d/%d to be removed\n", m_nLoop, nSlot, cSock, pCC->CliPort()) ;
pCC->Terminate() ;
delete pCC ;
currCC[cSock] = 0;
continue ;
}
}
}
/*
** ** READ Events
** */
if (m_arEvents[nSlot].events & EPOLLIN)
{
// At this point we have data on the socket to be read
cSock = m_arEvents[nSlot].data.fd ;
pCC = currCC[cSock] ;
if (!pCC)
{
m_pLog->Log("Loop %u: CORRUPT: Read event on socket %d/%d but no connector!\n", m_nLoop, cSock, pCC->CliPort()) ;
m_bShutdown = true ;
continue ;
}
if (cSock != pCC->CliSocket())
{
m_pLog->Log("Loop %u slot %u: CORRUPT: Read event on sock %d/%d but attached client object has socket of %d\n",
m_nLoop, nSlot, cSock, pCC->CliPort(), pCC->CliSocket()) ;
m_bShutdown = true ;
continue ;
}
// Test if client connection is using SSL and if so, is the SSL_accept() process complete?
if (pCC->m_bAcceptSSL)
{
// Not Complete - Call SSL_accept() again
bCloseSSL = false ;
sys_rc = SSL_accept(pCC->m_pSSL) ;
if (sys_rc > 0)
{
// pCC->m_bAcceptSSL = false ;
m_pLog->Log("Subsequent W SSL_accept OK %d\n", sys_rc) ;
}
else
{
sys_err = SSL_get_error(pCC->m_pSSL, sys_rc) ;
m_pLog->Log("Loop %u: NOTE: Subsequent W Failed to accept SSL client port %d sock %d ret %d err=%d\n", m_nLoop, pLS->GetPort(), cSock, sys_rc, sys_err) ;
switch (sys_err)
{
case SSL_ERROR_ZERO_RETURN: bCloseSSL = true ; m_pLog->Log("SSL_ERROR_ZERO_RETURN\n") ; break ;
case SSL_ERROR_WANT_READ: bCloseSSL = false ; m_pLog->Log("SSL_ERROR_WANT_READ\n") ; break ;
case SSL_ERROR_WANT_WRITE: bCloseSSL = false ; m_pLog->Log("SSL_ERROR_WANT_WRITE\n") ; break ;
case SSL_ERROR_WANT_CONNECT: bCloseSSL = false ; m_pLog->Log("SSL_ERROR_WANT_CONNECT\n") ; break ;
case SSL_ERROR_WANT_ACCEPT: bCloseSSL = false ; m_pLog->Log("SSL_ERROR_WANT_ACCEPT\n") ; break ;
case SSL_ERROR_WANT_X509_LOOKUP: bCloseSSL = false ; m_pLog->Log("SSL_ERROR_WANT_ACCEPT\n") ; break ;
case SSL_ERROR_WANT_ASYNC: bCloseSSL = false ; m_pLog->Log("SSL_ERROR_WANT_ASYNC\n") ; break ;
case SSL_ERROR_WANT_ASYNC_JOB: bCloseSSL = false ; m_pLog->Log("SSL_ERROR_WANT_ASYNC_JOB\n") ; break ;
case SSL_ERROR_WANT_CLIENT_HELLO_CB: bCloseSSL = false ; m_pLog->Log("SSL_ERROR_WANT_CLIENT_HELLO_CB\n") ; break ;
case SSL_ERROR_SYSCALL: bCloseSSL = true ; m_pLog->Log("SSL_ERROR_SYSCALL\n") ; break ;
case SSL_ERROR_SSL: bCloseSSL = true ; m_pLog->Log("SSL_ERROR_SSL\n") ; break ;
}
if (bCloseSSL)
{
pCC->Terminate() ;
delete pCC ;
currCC[cSock] = 0;
}
continue ;
}
}
// Now because we are in edge-triggered mode, continue reading from the socket until we get -1
for (nRecvTotal = 0;; nRecvTotal += nRecv)
{
nRecv = pCC->Recv(tbuf) ;
if (nRecv <&eq; 0)
break ;
}
if (pCC->m_bAcceptSSL)
{
pCC->m_bAcceptSSL = false ;
if (!nRecvTotal)
continue ;
}
if (_hzGlobal_Debug & HZ_DEBUG_SERVER)
{
if (!pCC->IsCliTerm())
m_pLog->Log("Loop %u slot %u: Client LIVE sock %d/%d. Recv %d Have %d\n", m_nLoop, nSlot, cSock, pCC->CliPort(), nRecvTotal, pCC->SizeIn()) ;
else
m_pLog->Log("Loop %u slot %u: Client DONE sock %d/%d. Revc %d Have %d\n", m_nLoop, nSlot, cSock, pCC->CliPort(), nRecvTotal, pCC->SizeIn()) ;
}
// Data to be processed has come in
if (!nRecvTotal)
{
m_pLog->Log("Loop %u slot %u: Read event with zero data on socket %d - disconnecting\n", m_nLoop, nSlot, cSock) ;
if (!pCC->_isxmit())
{
// Nothing has come in and nothing is due to go out so clear off the connection
m_pLog->Log("Loop %u slot %u: Terminating unused connection on socket %d\n", m_nLoop, nSlot, cSock) ;
pCC->Terminate() ;
delete pCC ;
currCC[cSock] = 0;
}
}
else
{
if (pCC->m_pProxy)
{
threadLog("Sending back response from proxy\n") ;
pCC->m_pProxy->SendData(pCC->InputZone()) ;
pCC->InputZone().Clear() ;
continue ;
}
if (pCC->MsgReady())
trc = pCC->m_OnIngress(pCC->InputZone(), pCC) ;
else
{
m_pLog->Log("Loop %u slot %u: NOTE: Client message not yet complete on socket %d/%d\n", m_nLoop, nSlot, cSock, pCC->CliPort()) ;
trc = TCP_INCOMPLETE ;
}
switch (trc)
{
case TCP_TERMINATE: // Directive is to close after outgoing message is complete
if (pCC->_isxmit())
{
// If outgoing data not complete, transmit some more.
xmitState = pCC->_xmit(tbuf) ;
if (xmitState < 0)
{
m_pLog->Log("Loop %u slot %u: TCP_TERMINATE: Write error - Sock %d/%d doomed\n", m_nLoop, nSlot, cSock, pCC->CliPort()) ;
pCC->Terminate() ;
delete pCC ;
currCC[cSock] = 0;
break ;
}
}
if (!pCC->_isxmit())
{
// No longer any outgoing data so close
m_pLog->Log("Loop %u slot %u: TCP_TERMINATE: Normal termination, sock %d/%d to be removed\n", m_nLoop, nSlot, cSock, pCC->CliPort()) ;
pCC->Terminate() ;
delete pCC ;
currCC[cSock] = 0;
break ;
}
m_pLog->Log("Loop %u slot %u: TCP_TERMINATE: Write delay - Sock %d/%d\n", m_nLoop, nSlot, cSock, pCC->CliPort()) ;
break ;
case TCP_KEEPALIVE: // Directive is to keep open after outgoing message is complete
if (pCC->_isxmit())
{
xmitState = pCC->_xmit(tbuf) ;
if (xmitState < 0)
{
m_pLog->Log("Loop %u slot %u: TCP_KEEPALIVE: Write error, sock %d/%d doomed\n", m_nLoop, nSlot, cSock, pCC->CliPort()) ;
pCC->Terminate() ;
delete pCC ;
currCC[cSock] = 0;
break ;
}
if (xmitState > 0)
{
// We have EAGAIN or EWOULDBLOCK and so now we make epoll entry to look for write event (when ocket becomes writable)
m_pLog->Log("Loop %u slot %u: TCP_KEEPALIVE: Write delay - Sock %d/%d\n", m_nLoop, nSlot, cSock, pCC->CliPort()) ;
}
}
if (pCC->IsCliTerm())
{
// No longer any outgoing data so close
m_pLog->Log("Loop %u slot %u: TCP_KEEPALIVE: Client terminated so sock %d/%d to be removed\n", m_nLoop, nSlot, cSock, pCC->CliPort()) ;
pCC->Terminate() ;
delete pCC ;
currCC[cSock] = 0;
break ;
}
pCC->Oxygen() ;
break ;
case TCP_INCOMPLETE: // The message processor has indicated it is still waiting for more imput
pCC->Oxygen() ;
break ;
case TCP_INVALID: // The message processor cannot dechipher the message. Kill the connection
m_pLog->Log("Loop %u: Sock %d/%d INVALID TCP code\n", m_nLoop, cSock, pCC->CliPort()) ;
pCC->Terminate() ;
delete pCC ;
currCC[cSock] = 0;
break ;
}
}
}
}
}
threadLog("SHUTDOWN COMPLETE\n") ;
}