toffaletti / state-threads Goto Github PK

Probably forgotten: results in multiple worker pthreads sharing key counters. Cured with this simple patch:

diff --git a/key.c b/key.c
index 9708c35..9133e65 100644
--- a/key.c
+++ b/key.c
@@ -47,8 +47,8 @@
/*

• Destructor table for per-thread private data
  */
  -static _st_destructor_t _st_destructors[ST_KEYS_MAX];
  -static int key_max = 0;
  +static __thread _st_destructor_t _st_destructors[ST_KEYS_MAX];
  +static __thread int key_max = 0;

As a result, st_usleep(999) is equivalent to st_usleep(0).

Proposed solution:

diff --git a/event.c b/event.c
index 6ee32f7..f5ea4d9 100644
--- a/event.c
+++ b/event.c
@@ -1288,7 +1288,7 @@ ST_HIDDEN void _st_epoll_dispatch(void)

     /* Check for I/O operations */
     nfd = epoll_wait(_st_epoll_data->epfd, _st_epoll_data->evtlist,
-                     _st_epoll_data->evtlist_size, timeout);
+                     _st_epoll_data->evtlist_size, timeout == -1 ? -1 : 0);

     if (nfd > 0) {
         for (i = 0; i < nfd; i++) {
@@ -1359,6 +1359,14 @@ ST_HIDDEN void _st_epoll_dispatch(void)
             }
         }
     }
+    else if (nfd == 0) {
+       if (timeout >= 0) {
+           struct timespec t;
+           t.tv_sec = min_timeout / 1000000;
+           t.tv_nsec = min_timeout * 1000;
+           nanosleep(&t, NULL);
+       }
+    }
 }

I came across a performance issue in epoll mode, when there were thousands concurrent connections. Profiling shows that _st_epoll_dispatch() consumed a lot of CPU.

After reviewing the function, I think I've found the reason: there's a loop that enumerates ALL threads in the I/O queue.

    for (q = _ST_IOQ.next; q != &_ST_IOQ; q = q->next) {

As I'm using one thread per connection model, I believe this loop make epoll mode degraded effectively to select mode.

This causes CLOSE_WAIT-state sockets to linger. This fix cures it:

@@ -142,7 +142,7 @@ static __thread struct _st_epolldata {
#define _ST_EPOLL_EXCEP_CNT(fd) (_st_epoll_data->fd_data[fd].ex_ref_cnt)
#define _ST_EPOLL_REVENTS(fd) (_st_epoll_data->fd_data[fd].revents)

-#define _ST_EPOLL_READ_BIT(fd) (_ST_EPOLL_READ_CNT(fd) ? EPOLLIN : 0)
+#define _ST_EPOLL_READ_BIT(fd) (_ST_EPOLL_READ_CNT(fd) ? (EPOLLIN|EPOLLRDHUP) : 0)
#define _ST_EPOLL_WRITE_BIT(fd) (_ST_EPOLL_WRITE_CNT(fd) ? EPOLLOUT : 0)
#define _ST_EPOLL_EXCEP_BIT(fd) (_ST_EPOLL_EXCEP_CNT(fd) ? EPOLLPRI : 0)
#define _ST_EPOLL_EVENTS(fd) \

commit accd04f in my fork.