Act as server based on the epoll method. This function is strickly for use in multi threaded servers. In the epoll method, the epoll socket (the one which activates the polling) is allocated and used to set up the controlling array of sockets. This array includes listening sockets as well as the sockets of connected clients. Arguments: None Returns: None Note: Call only once for all listening sockets! This function will not return until the server is shut down
| Return Type | Function name | Arguments |
|---|---|---|
| void | hzIpServer::ServeX | (void) |
Declared in file: hzIpServer.h
Defined in file : hzIpServer.cpp
Function Logic:
Function body:
void hzIpServer::ServeX (void)
{
// Category: Internet Server
//
// Act as server based on the epoll method. This function is strickly for use in multi threaded servers. In the epoll method, the epoll socket
// (the one which activates the polling) is allocated and used to set up the controlling array of sockets. This array includes listening sockets
// as well as the sockets of connected clients.
//
// Arguments: None
//
// Returns: None
//
// Note: Call only once for all listening sockets! This function will not return until the server is shut down
_hzfunc("hzIpServer::ServeX") ;
// Here we stay in a loop consiting of three stages:-
// 1) Setup all array of all active entities for the epoll system call
// 2) Call epoll to wait for something to come in.
// 3) Go through all active entries to see which one recieved a message
hzMapS<uint32_t,hzIpListen*> ls ; // Listening sockets map
hzMapS<uint32_t,hzIpConnex*> allCC ; // Connected clients
hzMapS<uint32_t,hzIpConnex*> duff ; // Failed clients
hzList<hzIpListen*>::Iter I ; // Listening sockets iterator
// struct epoll_event m_arEvents[100] ; // Array of epoll events
struct epoll_event epEventNew ; // Epoll event for new connections
// struct epoll_event epEventDead ; // Epoll event for dead connections
hzPacket tbuf ; // Fixed buffer for single IP packet
SOCKADDR cliAddr ; // Client address
SOCKADDRIN cliAddrIn ; // Client address
socklen_t cliLen ; // Client address length
hzIpListen* pLS ; // Listening socket
hzIpConnex* pCC ; // Connected client
// hzProcInfo* pPD ; // Process data (Client socket & IP address, must be destructed by the called thread function)
SSL* pSSL ; // SSL session (if applicable)
// X509* client_cert ; // Client certificate
// char* clicert_str ; // Client certificate string
// const char* errstr ; // Either 'hangup' or 'error'
timeval tv ; // Time limit for epoll call
pthread_attr_t tattr ; // Thread attribute (to support thread invokation)
// pthread_t tid ; // Needed for multi-threading
// uint64_t then ; // Before the wait
// uint64_t now ; // After the wait
uint64_t nsThen ; // Before the wait
uint64_t nsNow ; // After the wait
hzIpaddr ipa ; // IP address
uint32_t nBannedAttempts ; // Total connection attempts by banned IP addresses
uint32_t nCliSeq = 1; // Client connection id allocator
uint32_t nX ; // Dead/failed connections counter
// uint32_t nLoop ; // Number of times round the epoll event loop
uint32_t nSlot ; // Connections iterator
uint32_t nError ; // Errno of the epoll call
uint32_t cSock ; // Client socket (from accept)
uint32_t cPort ; // Client socket (from accept)
int32_t aSock ; // Client socket (validated)
// int32_t epollSocket ; // The epoll 'master' socket
int32_t flags ; // Flags to mak socket non-blocking
int32_t nC ; // Connections iterator
int32_t nRecv ; // No bytes read from client socket after epoll
int32_t nEpollEvents ; // Result of the epoll call
int32_t sys_rc ; // Return from system library calls
hzEcode err ; // Returns by called functions
char ipbuf[44]; // Client IP address textform buffer
// ls.SetDefaultObj(0) ;
// allCC.SetDefaultObj(0) ;
// Pre-define thread attributes
pthread_attr_init(&tattr) ;
pthread_attr_setdetachstate(&tattr, PTHREAD_CREATE_DETACHED) ;
// Create epoll socket and and listening sockets to the controller
s_epollSocket = epoll_create1(0);
if (s_epollSocket == -1)
Fatal("Could not create epoll socket\n") ;
m_pLog->Log("Epoll socket is %d\n", s_epollSocket) ;
for (I = m_LS ; I.Valid() ; I++)
{
pLS = I.Element() ;
epEventNew.data.fd = pLS->GetSocket() ;
epEventNew.events = EPOLLIN ;
if (epoll_ctl(s_epollSocket, EPOLL_CTL_ADD, pLS->GetSocket(), &epEventNew) < 0)
Fatal("Could not add listening socket %d to the epoll controller\n", pLS->GetSocket()) ;
m_pLog->Log("Added listening socket is %d\n", pLS->GetSocket()) ;
ls.Insert(pLS->GetSocket(), pLS) ;
}
// Main loop - waiting for events
m_nLoop = nCliSeq = nBannedAttempts = 0;
for (;;)
{
nsThen = RealtimeNano() ;
nEpollEvents = epoll_wait(s_epollSocket, m_arEvents, MAXEVENTS, 30000);
nsNow = RealtimeNano() ;
for (nC = 0; nC < nEpollEvents ; nC++)
{
if (_hzGlobal_Debug & HZ_DEBUG_SERVER)
{
if (nSlot)
m_pLog->Log("Loop %u slot %u: Event on socket %d\n", m_nLoop, nSlot, m_arEvents[nSlot].data.fd) ;
else
m_pLog->Log("Loop %u: Waited %lu nanoseconds for %d events: 1st sock %u\n", m_nLoop, nsNow - nsThen, nEpollEvents, m_arEvents[0].data.fd);
}
// Hangup?
if (m_arEvents[nSlot].events & EPOLLHUP)
{
// Hangup signal
cSock = m_arEvents[nSlot].data.fd ;
pCC = currCC[cSock] ;
nError = 0;
if (!pCC) { m_bShutdown = true ; m_pLog->Log("Loop %u slot %u: HANGUP CORRUPT Sock %d No connector\n", m_nLoop, nSlot, cSock) ; continue ; }
if (!pCC->CliSocket()) { m_bShutdown = true ; m_pLog->Log("Loop %u slot %u: HANGUP CORRUPT Sock %d Connector defunct\n", m_nLoop, nSlot, cSock) ; continue ; }
if (getsockopt(cSock, SOL_SOCKET, SO_ERROR, (void *)&nError, &cliLen) == 0)
m_pLog->Log("Loop %u slot %u: HANGUP on socket %d (%s)\n", m_nLoop, nSlot, cSock, strerror(nError)) ;
else
m_pLog->Log("Loop %u slot %u: HANGUP on socket %d (no details)\n", m_nLoop, nSlot, cSock) ;
}
// General connection error?
if (m_arEvents[nSlot].events & EPOLLERR)
{
// An error has occured on this fd, or the socket is not ready for reading
cSock = m_arEvents[nSlot].data.fd ;
pCC = currCC[cSock] ;
nError = 0;
if (!pCC) { m_bShutdown = true ; m_pLog->Log("Loop %u slot %u: EPOLLERR CORRUPT Sock %d No connector\n", m_nLoop, nSlot, cSock) ; continue ; }
if (!pCC->CliSocket()) { m_bShutdown = true ; m_pLog->Log("Loop %u slot %u: EPOLLERR CORRUPT Sock %d Connector defunct\n", m_nLoop, nSlot, cSock) ; continue ; }
if (getsockopt(cSock, SOL_SOCKET, SO_ERROR, (void *)&nError, &cliLen) == 0)
m_pLog->Log("Loop %u slot %u: EPOLLERR on socket %d (%s)\n", m_nLoop, nSlot, cSock, strerror(nError)) ;
else
m_pLog->Log("Loop %u slot %u: EPOLLERR on socket %d (no details)\n", m_nLoop, nSlot, cSock) ;
}
// Listening socket event?
if (ls.Exists(m_arEvents[nC].data.fd))
{
// We have a notification on the listening socket, which means one or more incoming connections.
// Firstly accept the client no matter what. If there is a problem we will imeadiately close the connection.
pLS = ls[m_arEvents[nC].data.fd] ;
pSSL = 0;
cliLen = sizeof(SOCKADDRIN) ;
if ((aSock = accept(pLS->GetSocket(), &cliAddr, &cliLen)) < 0)
{
m_pLog->Log("Listening socket %d will not accept client on port %d\n", aSock, pLS->GetPort()) ;
continue ;
}
// Make the incoming socket non-blocking and add it to the list of fds to monitor.
cSock = aSock ;
flags = fcntl(cSock, F_GETFL, 0);
if (flags == -1)
Fatal("Could not make client socket %d non blocking (case 1)\n", cSock) ;
flags |= O_NONBLOCK ;
if (fcntl(cSock, F_SETFL, flags) == -1)
Fatal("Could not make client socket %d non blocking (case 2)\n", cSock) ;
// Add client socker to the epoll control
epEventNew.data.fd = cSock ;
epEventNew.events = EPOLLIN | EPOLLET ;
if (epoll_ctl(s_epollSocket, EPOLL_CTL_ADD, cSock, &epEventNew) < 0)
Fatal("Could not add client socket %d to control\n", cSock) ;
// Get the IP address
inet_ntop(AF_INET, &cliAddrIn.sin_addr, ipbuf, 16);
ipa = ipbuf ;
if (m_bShutdown)
{
// Shutdown in progress
m_pLog->Log("NOTE: Shutdown in progress. New connections refused\n") ;
if (close(cSock) < 0)
m_pLog->Log("NOTE: Could not close socket %d\n", cSock) ;
continue ;
}
if (pLS->GetActConn() >&eq; pLS->GetMaxConn())
{
// Too many connections
m_pLog->Log("NOTE: System too busy: Curr %d Max %d stat %d\n", m_nCurrentClients, m_nMaxClients, m_bShutdown ? 1: 0);
if (close(cSock) < 0)
m_pLog->Log("NOTE: Could not close socket %d\n", cSock) ;
continue ;
}
// Client is already connected. Do we allow further connections?
/*
** if (_ipcheck(ipbuf, pLS->GetPort()))
** {
** m_pLog->Log("Lorris Attack from IP %s port %d\n", ipbuf, pLS->GetPort()) ;
** if (close(cSock) < 0)
** m_pLog->Log("NOTE: Could not close socket %d after lorris_attack\n", cSock) ;
** continue ;
** }
** */
// All is well so set socket options
tv.tv_sec = 1;
tv.tv_usec = 0;
if (setsockopt(cSock, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof(tv)) < 0)
{ m_pLog->Log("Could not set send socket options\n") ; continue ; }
if (setsockopt(cSock, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)) < 0)
{ m_pLog->Log("Could not set recv socket options\n") ; continue ; }
// Do we have SSL on top of connection?
if (pLS->UseSSL())
{
// Create a new SSL session instance
// pSSL = SSL_new(s_svrCTX) ;
pSSL = SSL_new(s_SSL_svrRegime->m_svrCTX) ;
if (!pSSL)
{ m_pLog->Log("Failed to allocate an SSL instance on port %d\n", pLS->GetPort()) ; continue ; }
m_pLog->Log("Allocated SSL instance on port %d\n", pLS->GetPort()) ;
// Set the SSL session socket
SSL_set_fd(pSSL, cSock) ;
sys_rc = SSL_accept(pSSL) ;
if (sys_rc <&eq; 0)
{
m_pLog->Log("NOTE: Failed to accept SSL client on port %d socket %d (error=%d,%d)\n",
pLS->GetPort(), cSock, sys_rc, SSL_get_error(pSSL, sys_rc)) ;
SSL_free(pSSL) ;
if (close(cSock) < 0)
m_pLog->Log("NOTE: Could not close socket %d after SSL fail_accept\n", cSock) ;
m_pLog->Log("Closed socket\n") ;
continue ;
}
m_pLog->Log("SSL connection using %s\n", SSL_get_cipher(pSSL)) ;
/*
** if (verify_callback)
** {
** // Get the client's certificate (optional)
** client_cert = SSL_get_peer_certificate(pSSL);
**
** if (client_cert != NULL)
** {
** clicert_str = X509_NAME_oneline(X509_get_subject_name(client_cert), 0, 0);
** if (!clicert_str)
** { m_pLog->Log("No Client Certificate found\n") ; continue ; }
** m_pLog->Log("Subject of Client Cert: %s\n", clicert_str) ;
** free(clicert_str) ;
**
** clicert_str = X509_NAME_oneline(X509_get_issuer_name(client_cert), 0, 0);
** if (!clicert_str)
** { m_pLog->Log("No Client Cert Issuer found\n") ; continue ; }
** m_pLog->Log("Issuer of Client Cert: %s\n", clicert_str) ;
** free(clicert_str) ;
**
** X509_free(client_cert);
** }
** else
** printf("The SSL client does not have certificate.\n");
** }
** */
}
// If the listening socket is for a session handler, deal with this here
/*
** ** Allocate the connected client object
** */
pCC = allCC[cSock] ;
if (!pCC)
{
pCC = new hzIpConnex(m_pLog) ;
if (!pCC)
hzexit(E_MEMORY, "No memory for new client connection\n") ;
allCC.Insert(cSock, pCC) ;
}
m_pLog->Log("New client connection on socket %d\n", cSock) ;
// Initialize and oxygenate the connection
pCC->Initialize(pLS, pSSL, ipa, cSock, cPort, nCliSeq++) ;
if (pCC->m_OnConnect)
{
if (_hzGlobal_Debug & HZ_DEBUG_SERVER)
m_pLog->Log("Client %d (sock %d): Issuing server hello\n", pCC->EventNo(), pCC->CliSocket()) ;
pCC->m_OnConnect(pCC) ;
}
continue ;
}
// Deal with connected clients here.
if (m_arEvents[nC].events & EPOLLOUT)
{
// Should not get this as we are not asking for these notifications
cSock = m_arEvents[nC].data.fd ;
m_pLog->Log("WHAT??? write event on socket %d\n", cSock) ;
continue ;
}
if (m_arEvents[nC].events & EPOLLIN)
{
// At this point we have date on the socket to be read, at least in theory
cSock = m_arEvents[nC].data.fd ;
pCC = allCC[cSock] ;
if (!pCC)
{
m_pLog->Log("WHAT??? read event on socket %d but no connector!\n", cSock) ;
continue ;
}
if (pCC->IsCliTerm())
m_pLog->Log("NOTE: Client %d (sock %d) has a read event after sending 0 byte packet\n", pCC->EventNo(), pCC->CliSocket()) ;
// Now because we are in edge-triggered mode, continue reading from the socket until we get -1
// m_pLog->Log("Client %d (sock %d): Got %d bytes\n", pCC->EventNo(), pCC->CliSocket(), nRecv) ;
for (;;)
{
nRecv = pCC->Recv(tbuf) ;
m_pLog->Log("Client %d (sock %d): Got %d bytes\n", pCC->EventNo(), pCC->CliSocket(), nRecv) ;
if (nRecv <&eq; 0)
break ;
}
if (!pCC->SizeIn())
{
m_pLog->Log("Client %d (sock %d): Final read - NO DATA\n", pCC->EventNo(), pCC->CliSocket()) ;
if (nsNow > pCC->Expires())
{
m_pLog->Log("Client %d (sock %d): Final read - NO DATA deleted\n", pCC->EventNo(), pCC->CliSocket()) ;
pCC->Terminate() ;
}
}
else
{
// Data to be processed has come in
// if (_hzGlobal_Debug & HZ_DEBUG_SERVER)
// {
// m_pLog->Log("Client %d (sock %d): Triggered with %d bytes\n[\n", pCC->EventNo(), pCC->CliSocket(), pCC->SizeIn()) ;
// m_pLog->Out(pCC->InputZone()) ;
// m_pLog->Out("]\n") ;
// }
s_queRequests.Push(pCC) ;
pthread_cond_signal(&s_request_cond) ;
}
}
}
// Kill off any inactive and out of date keep-alive connections - but only if there is a delay between polls
if ((nsNow - nsThen) > 100000000)
{
if (_hzGlobal_Debug & HZ_DEBUG_SERVER)
m_pLog->Log("Purging ...\n") ;
for (nX = 0; nX < allCC.Count() ; nX++)
{
pCC = allCC.GetObj(nX) ;
if (!pCC)
continue ;
if (!pCC->CliSocket())
continue ;
if (pCC->_isxmit())
{
err = pCC->_xmit(tbuf) ;
continue ;
}
/*
** if (pCC->State() == HZCONNEX_FAIL)
** {
** if (now < pCC->Expires())
** continue ;
**
** //pCC->StateDone() ;
** pCC->SetState(*_fn, HZCONNEX_DONE) ;
** if (m_pLog)
** m_pLog->Log("Client (ev %d sock %d state %d) vgn %d: Failed, removed\n",
** pCC->EventNo(), pCC->CliSocket(), pCC->State(), pCC->IsVirgin() ? 1 : 0) ;
** pCC->Terminate() ;
** continue ;
** }
** */
// No incoming data or connection expired?
if (!pCC->SizeIn() && (nsNow > pCC->Expires()))
{
// pCC->StateDone() ;
// pCC->SetState(*_fn, HZCONNEX_DONE) ;
// m_pLog->Log("Client (ev %d sock %d state %d) vgn %d: Inactive, removed\n",
// pCC->EventNo(), pCC->CliSocket(), pCC->State(), pCC->IsVirgin() ? 1 : 0) ;
m_pLog->Log("Client (ev %d sock %d) vgn %d: Inactive, removed\n", pCC->EventNo(), pCC->CliSocket(), pCC->IsVirgin() ? 1: 0);
pCC->Terminate() ;
}
}
}
}
threadLog("Shutdown\n") ;
}