I am using epoll to implement a socket-based server program. When I use multithreading, some requests fail to respond. The test results using webbench are as follows. enter image description here
//This is the code to listen on the socket.
//Omit the creation code of listenfd.
while (!stopserver)
{
int num = epoll_wait(epollfd, eve, 10000, -1);
for (int i = 0; i < num; ++i)
{
int fd = eve[i].data.fd;
if (fd == listenfd)
{
sockaddr_in caddr;
socklen_t tmplen = sizeof(caddr);
connfd = accept(listenfd, (sockaddr*)&caddr, &tmplen);
assert(connfd != -1);
//The true here is to use EPOLLONESHOT.
addFd(epollfd, connfd, true);
}
else
{
worker.init(eve[i].data.fd, epollfd);
threadpool->add(&worker);
}
}
}
bool ThreadPool::add(Worker* worker)
{
std::unique_lock<std::mutex> locker(mutex);
if (workerdeque.size() > maxrequestsnum)
{
locker.unlock();
return false;
}
workerdeque.push_back(worker);
locker.unlock();
sem_post(&sem);
return true;
}
void ThreadPool::run()
{
while (true)
{
sem_wait(&sem);
std::unique_lock<std::mutex> locker(mutex);
if (workerdeque.empty())
{
locker.unlock();
continue;
}
Worker* request = workerdeque.front();
workerdeque.pop_front();
locker.unlock();
if (!request)
continue;
std::cout << "work: " << request->connfd << std::endl;
request->work();
}
}
void Worker::init(const int& connfd, const int& epollfd)
{
this->connfd = connfd;
this->epollfd = epollfd;
}
void Worker::work()
{
int len = 0, totle = 0;
char buf[4096]{ '0' };
char temp[4096]{ '0' };
while ((len = recv(connfd, temp, sizeof(temp), NULL)) > 0)
{
if (totle + len < sizeof(buf))
{
memcpy(buf + totle, temp, len);
totle += len;
}
}
if (len == -1 && errno == EAGAIN)
{
//Re-register EPOLLONESHOT event
modfd(epollfd, connfd);
//Parse after reading
parser.getLine(buf, tline);
parser.getStatus(tline, tstatus);
parser.getFile(tline, tfile);
std::cout << "close:" << connfd << std::endl;
close(connfd);
}
else if (len == 0)
{
std::cout << "close" << std::endl;
epoll_ctl(epollfd, EPOLL_CTL_DEL, connfd, NULL);
close(connfd);
}
else
{
std::cout << "errno:" << errno << " " << connfd << std::endl;
}
}
These are the wrong output information. These are the wrong output information. It seems that they are all trying to read data from a confd (but confd should contain no data, which causes errno to be set to 9).
//SendResponse represents that the response was successfully sent.
sendResponse
close:480
work: 362
close362
work: 362
errno:9 362
work: 227
close227
work: 227
errno:9 227
work: 227
errno:9 227
work: 227
errno:9 227
work: 227
errno:9 227
work: 227
errno:9 227
work: 227
errno:9 227
work: 227
errno:9 227
work: 227
errno:9 227
work: 227
errno:9 227
work: 360
close360
work: 147
close147
This is the normal output.But I can't understand why there are so many outputs of the same connfd. Is it because multiple threads use the same connfd?
sendResponse
close:480
work: 480
sendResponse
close:480
work: 480
Worker::init
, I actually suspect the error rather there. Actually you should post theWorker
class itself, too, and maybe you even can remove some code that doesn't contribute to the problem (-> minimal reproducible example!).