RA09040 died on an mblk explosion of unknown origin; dump in thoth is df694f9bd93066367f57508f7431f65c
Former user commented on 2014-01-03T04:11:41.000-0500:
These appear to be mblks enqueued from the so_rcv_q_head members of various sonode structures – with many, many sockets having a so_rcv_queued of SO_MAX_BUF. Moreover, these appear to all be sockets from haproxy. For whatever reason, haproxy appears to not be reading ready data on some (large) number of sockets, corking the kernel with SO_MAX_BUF buffers.
Former user commented on 2014-01-07T12:29:31.000-0500:
As it turns out, the problem here is – in the end – straightforward:
POLLHUP is not generated (ever) for disconnected sockets. This
is not seen more broadly because POLLIN is generated for these
sockets – and most will poll on POLLIN as well as POLLOUT. In haproxy's
case, however, it is (legitimately) polling only on POLLOUT; should
a socket go through a clean but disorderly shutdown (e.g., an RST),
haproxy will never be made aware of the fact. This, in turn, causes
the backend-facing socket associated with the connection to backup
with accumulated data. We will therefore hold onto kernel memory at
a rate proportional to the the rate of RSTs – currently on the
order of tens of per day (which, at 4MB of buffered data per, amounts
to on the order of a hundred megabytes per day). The workaround is
to restart haproxy; the fix is to generate POLLHUP when the socket
is clearly disconnected and can no longer read or write.
Former user commented on 2014-02-24T12:05:46.000-0500:
illumos-joyent commit 24130af (branch master, by Bryan Cantrill)
OS-2689#icft=OS-2689 POLLHUP not generated for disconnected sockets
Former user commented on 2014-02-24T12:18:27.000-0500:
I used the test programs from http://oss.sgi.com/cgi-bin/extract-mesg.cgi?a=netdev&m=2002-08&i=20020816010809.GA28446%40anakin.wychk.org:
Here's poll.c:
#include <poll.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h> /* strdup */
#include <unistd.h> /* getopt */
#include <strings.h>
#define PORT (22)
#define BACKLOG 20
#ifndef INFTIM
#define INFTIM (-1)
#endif
extern char *optarg;
int main(argc, argv)
int argc;
char *argv[];
{
struct pollfd nfd;
struct sockaddr_in saddr, caddr;
int fd;
int c;
int clientfd;
size_t len;
unsigned short port = PORT;
bzero(&saddr, sizeof(struct sockaddr_in));
bzero(&caddr, sizeof(struct sockaddr_in));
while ((c = getopt(argc, argv, "h:p:")) != -1) {
switch (c) {
case 'p':
port = atoi(optarg);
break;
default:
break;
}
}
saddr.sin_family = AF_INET;
saddr.sin_port = htons(port);
saddr.sin_addr.s_addr = INADDR_ANY;
fd = socket(AF_INET, SOCK_STREAM, 0);
if (fd < 0) {
printf("socket\n");
exit(1);
}
if (bind(fd, (struct sockaddr *)&saddr, sizeof(struct sockaddr_in)) < 0) {
printf("bind\n");
exit(1);
}
if (listen(fd, BACKLOG) < 0) {
printf("listen\n");
exit(1);
}
len = sizeof(struct sockaddr_in);
if ((clientfd = accept(fd, (struct sockaddr *)&caddr, &len)) < 0) {
printf("accept: %s\n", strerror(errno)), exit(1);
}
printf("accept ok.\n");
/* reset */
nfd.fd = clientfd;
nfd.events = 0;
nfd.revents = 0;
if (poll(&nfd, 1, INFTIM) < 0)
printf("poll: %s\n", strerror(errno)), exit(1);
if (nfd.revents & POLLERR)
printf("poll: POLLERR set\n");
if (nfd.revents & POLLHUP)
printf("poll: POLLHUP set\n");
if (nfd.revents & POLLNVAL)
printf("poll: POLLNVAL set\n"); /* should not happen */
close(clientfd);
printf("sever terminating\n");
exit(0);
}And linger.c:
#include <sys/socket.h>
#include <netinet/in.h>
#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h> /* getopt */
#include <strings.h>
#define HOST "192.168.0.1"
#define PORT (22)
int main(argc, argv)
int argc;
char *argv[];
{
int sockfd;
int c;
struct linger ling;
struct sockaddr_in servaddr;
char *host = HOST;
unsigned short port = PORT;
while ((c = getopt(argc, argv, "h:p:")) != -1) {
switch (c) {
case 'h':
host = (char *)strdup(optarg);
break;
case 'p':
port = atoi(optarg);
break;
default:
break;
}
}
sockfd = socket(AF_INET, SOCK_STREAM, 0);
if (sockfd < 0) {
printf("socket\n");
exit(1);
}
bzero(&servaddr, sizeof(struct sockaddr_in));
servaddr.sin_family = AF_INET;
servaddr.sin_port = htons(port);
servaddr.sin_addr.s_addr = inet_addr(host);
ling.l_onoff = 1;
ling.l_linger = 0;
if (setsockopt(sockfd, SOL_SOCKET, SO_LINGER, &ling, sizeof(ling)) < 0) {
printf("setsockopt\n");
exit(1);
}
printf("connecting ..\n");
if (connect(sockfd, (struct sockaddr *)&servaddr, sizeof(struct sockaddr_in)) < 0) {
printf("connect: %s\n", strerror(errno));
exit(1);
}
sleep(10);
printf("closing ...\n");
close(sockfd); /* send RST */
printf("closed.\n");
exit(0);
}I ran them over localhost ("./poll -p 8000" and "./linger -h 127.0.0.1 -p 8000") and confirmed that Linux behavior now is what it was 12 years ago – and that our behavior now matches it with respect to POLLHUP.
Former user commented on 2015-02-05T19:37:23.000-0500:
We seem to have hit this in the us-east Manta again, twice in two days. Two haproxy instances in muskie zones (not the external-facing loadbalancers) stopped responding to requests. Connecting with curl(1) and fetching the /ping endpoint causes curl to block indefinitely. The first one of these died before I had a chance to dig in, but today I got so far as to see that with truss, it's looping like this:
pollsys(0x0879CE08, 65537, 0x08047C88, 0x00000000) (sleeping...)
pollsys(0x0879CE08, 65537, 0x08047C88, 0x00000000) = 0
pollsys(0x0879CE08, 65537, 0x08047C88, 0x00000000) (sleeping...)
pollsys(0x0879CE08, 65537, 0x08047C88, 0x00000000) = 0
pollsys(0x0879CE08, 65537, 0x08047C88, 0x00000000) (sleeping...)
pollsys(0x0879CE08, 65537, 0x08047C88, 0x00000000) = 0
pollsys(0x0879CE08, 65537, 0x08047C88, 0x00000000) (sleeping...)
pollsys(0x0879CE08, 65537, 0x08047C88, 0x00000000) = 0
pollsys(0x0879CE08, 65537, 0x08047C88, 0x00000000) (sleeping...)
pollsys(0x0879CE08, 65537, 0x08047C88, 0x00000000) = 0
pollsys(0x0879CE08, 65537, 0x08047C88, 0x00000000) (sleeping...)
pollsys(0x0879CE08, 65537, 0x08047C88, 0x00000000) = 0
pollsys(0x0879CE08, 65537, 0x08047C88, 0x00000000) (sleeping...)
pollsys(0x0879CE08, 65537, 0x08047C88, 0x00000000) = 0
pollsys(0x0879CE08, 65537, 0x08047C88, 0x00000000) (sleeping...)
pollsys(0x0879CE08, 65537, 0x08047C88, 0x00000000) = 0
pollsys(0x0879CE08, 65537, 0x08047C88, 0x00000000) = 0
pollsys(0x0879CE08, 65537, 0x08047C88, 0x00000000) = 0
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130386
fcntl(130386, F_SETFL, FNONBLOCK) = 0
setsockopt(130386, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130386, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130393
fcntl(130393, F_SETFL, FNONBLOCK) = 0
setsockopt(130393, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130393, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130410
fcntl(130410, F_SETFL, FNONBLOCK) = 0
setsockopt(130410, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130410, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130412
fcntl(130412, F_SETFL, FNONBLOCK) = 0
setsockopt(130412, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130412, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130414
fcntl(130414, F_SETFL, FNONBLOCK) = 0
setsockopt(130414, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130414, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130416
fcntl(130416, F_SETFL, FNONBLOCK) = 0
setsockopt(130416, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130416, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130420
fcntl(130420, F_SETFL, FNONBLOCK) = 0
setsockopt(130420, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130420, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130421
fcntl(130421, F_SETFL, FNONBLOCK) = 0
setsockopt(130421, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130421, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130423
fcntl(130423, F_SETFL, FNONBLOCK) = 0
setsockopt(130423, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130423, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130425
fcntl(130425, F_SETFL, FNONBLOCK) = 0
setsockopt(130425, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130425, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130426
fcntl(130426, F_SETFL, FNONBLOCK) = 0
setsockopt(130426, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130426, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130428
fcntl(130428, F_SETFL, FNONBLOCK) = 0
setsockopt(130428, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130428, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130430
fcntl(130430, F_SETFL, FNONBLOCK) = 0
setsockopt(130430, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130430, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130432
fcntl(130432, F_SETFL, FNONBLOCK) = 0
setsockopt(130432, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130432, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130433
fcntl(130433, F_SETFL, FNONBLOCK) = 0
setsockopt(130433, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130433, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130435
fcntl(130435, F_SETFL, FNONBLOCK) = 0
setsockopt(130435, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130435, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130436
fcntl(130436, F_SETFL, FNONBLOCK) = 0
setsockopt(130436, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130436, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130437
fcntl(130437, F_SETFL, FNONBLOCK) = 0
setsockopt(130437, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130437, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130439
fcntl(130439, F_SETFL, FNONBLOCK) = 0
setsockopt(130439, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130439, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130440
fcntl(130440, F_SETFL, FNONBLOCK) = 0
setsockopt(130440, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130440, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130442
fcntl(130442, F_SETFL, FNONBLOCK) = 0
setsockopt(130442, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130442, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130443
fcntl(130443, F_SETFL, FNONBLOCK) = 0
setsockopt(130443, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130443, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130444
fcntl(130444, F_SETFL, FNONBLOCK) = 0
setsockopt(130444, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130444, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130445
fcntl(130445, F_SETFL, FNONBLOCK) = 0
setsockopt(130445, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130445, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130446
fcntl(130446, F_SETFL, FNONBLOCK) = 0
setsockopt(130446, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130446, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130448
fcntl(130448, F_SETFL, FNONBLOCK) = 0
setsockopt(130448, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130448, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130450
fcntl(130450, F_SETFL, FNONBLOCK) = 0
setsockopt(130450, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130450, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130451
fcntl(130451, F_SETFL, FNONBLOCK) = 0
setsockopt(130451, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130451, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130452
fcntl(130452, F_SETFL, FNONBLOCK) = 0
setsockopt(130452, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130452, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130453
fcntl(130453, F_SETFL, FNONBLOCK) = 0
setsockopt(130453, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130453, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130454
fcntl(130454, F_SETFL, FNONBLOCK) = 0
setsockopt(130454, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130454, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130455
fcntl(130455, F_SETFL, FNONBLOCK) = 0
setsockopt(130455, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130455, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
pollsys(0x0879CE08, 65569, 0x08047C88, 0x00000000) = 32
connect(130386, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130393, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130410, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130412, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130414, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130416, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130420, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130421, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130423, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130425, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130426, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130428, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130430, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130432, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130433, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130435, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130436, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130437, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130439, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130440, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130442, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130443, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130444, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130445, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130446, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130448, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130450, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130451, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130452, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130453, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130454, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130455, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
close(130386) = 0
close(130393) = 0
close(130410) = 0
close(130412) = 0
close(130414) = 0
close(130416) = 0
close(130420) = 0
close(130421) = 0
close(130423) = 0
close(130425) = 0
close(130426) = 0
close(130428) = 0
close(130430) = 0
close(130432) = 0
close(130433) = 0
close(130435) = 0
close(130436) = 0
close(130437) = 0
close(130439) = 0
close(130440) = 0
close(130442) = 0
close(130443) = 0
close(130444) = 0
close(130445) = 0
close(130446) = 0
close(130448) = 0
close(130450) = 0
close(130451) = 0
close(130452) = 0
close(130453) = 0
close(130454) = 0
close(130455) = 0
pollsys(0x0879CE08, 65537, 0x08047C88, 0x00000000) (sleeping...)
pollsys(0x0879CE08, 65537, 0x08047C88, 0x00000000) = 0
pollsys(0x0879CE08, 65537, 0x08047C88, 0x00000000) = 1
recv(128911, "8184\0\0\0\0 p i n g", 16384, 0) = 10
recv(128911, 0x8E14799A, 16374, 0) Err#11 EAGAIN
getsockopt(129018, SOL_SOCKET, SO_ERROR, 0x08047BF8, 0x08047BFC, SOV_DEFAULT) = 0
send(129018, "8184\0\0\0\0 p i n g", 10, 128) = 10
pollsys(0x0879CE08, 65537, 0x08047C88, 0x00000000) (sleeping...)
pollsys(0x0879CE08, 65537, 0x08047C88, 0x00000000) = 0
pollsys(0x0879CE08, 65537, 0x08047C88, 0x00000000) (sleeping...)
pollsys(0x0879CE08, 65537, 0x08047C88, 0x00000000) = 0
pollsys(0x0879CE08, 65537, 0x08047C88, 0x00000000) (sleeping...)
pollsys(0x0879CE08, 65537, 0x08047C88, 0x00000000) = 0
pollsys(0x0879CE08, 65537, 0x08047C88, 0x00000000) (sleeping...)
pollsys(0x0879CE08, 65537, 0x08047C88, 0x00000000) = 0
pollsys(0x0879CE08, 65537, 0x08047C88, 0x00000000) (sleeping...)
pollsys(0x0879CE08, 65537, 0x08047C88, 0x00000000) = 0
pollsys(0x0879CE08, 65537, 0x08047C88, 0x00000000) (sleeping...)
pollsys(0x0879CE08, 65537, 0x08047C88, 0x00000000) = 0
pollsys(0x0879CE08, 65537, 0x08047C88, 0x00000000) (sleeping...)
pollsys(0x0879CE08, 65537, 0x08047C88, 0x00000000) = 0
pollsys(0x0879CE08, 65537, 0x08047C88, 0x00000000) (sleeping...)
pollsys(0x0879CE08, 65537, 0x08047C88, 0x00000000) = 0
pollsys(0x0879CE08, 65537, 0x08047C88, 0x00000000) = 0
pollsys(0x0879CE08, 65537, 0x08047C88, 0x00000000) = 0
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130386
fcntl(130386, F_SETFL, FNONBLOCK) = 0
setsockopt(130386, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130386, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130393
fcntl(130393, F_SETFL, FNONBLOCK) = 0
setsockopt(130393, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130393, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130410
fcntl(130410, F_SETFL, FNONBLOCK) = 0
setsockopt(130410, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130410, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130412
fcntl(130412, F_SETFL, FNONBLOCK) = 0
setsockopt(130412, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130412, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130414
fcntl(130414, F_SETFL, FNONBLOCK) = 0
setsockopt(130414, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130414, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130416
fcntl(130416, F_SETFL, FNONBLOCK) = 0
setsockopt(130416, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130416, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130420
fcntl(130420, F_SETFL, FNONBLOCK) = 0
setsockopt(130420, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130420, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130421
fcntl(130421, F_SETFL, FNONBLOCK) = 0
setsockopt(130421, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130421, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130423
fcntl(130423, F_SETFL, FNONBLOCK) = 0
setsockopt(130423, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130423, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130425
fcntl(130425, F_SETFL, FNONBLOCK) = 0
setsockopt(130425, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130425, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130426
fcntl(130426, F_SETFL, FNONBLOCK) = 0
setsockopt(130426, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130426, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130428
fcntl(130428, F_SETFL, FNONBLOCK) = 0
setsockopt(130428, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130428, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130430
fcntl(130430, F_SETFL, FNONBLOCK) = 0
setsockopt(130430, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130430, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130432
fcntl(130432, F_SETFL, FNONBLOCK) = 0
setsockopt(130432, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130432, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130433
fcntl(130433, F_SETFL, FNONBLOCK) = 0
setsockopt(130433, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130433, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130435
fcntl(130435, F_SETFL, FNONBLOCK) = 0
setsockopt(130435, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130435, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130436
fcntl(130436, F_SETFL, FNONBLOCK) = 0
setsockopt(130436, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130436, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130437
fcntl(130437, F_SETFL, FNONBLOCK) = 0
setsockopt(130437, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130437, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130439
fcntl(130439, F_SETFL, FNONBLOCK) = 0
setsockopt(130439, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130439, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130440
fcntl(130440, F_SETFL, FNONBLOCK) = 0
setsockopt(130440, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130440, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130442
fcntl(130442, F_SETFL, FNONBLOCK) = 0
setsockopt(130442, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130442, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130443
fcntl(130443, F_SETFL, FNONBLOCK) = 0
setsockopt(130443, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130443, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130444
fcntl(130444, F_SETFL, FNONBLOCK) = 0
setsockopt(130444, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130444, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130445
fcntl(130445, F_SETFL, FNONBLOCK) = 0
setsockopt(130445, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130445, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130446
fcntl(130446, F_SETFL, FNONBLOCK) = 0
setsockopt(130446, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130446, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130448
fcntl(130448, F_SETFL, FNONBLOCK) = 0
setsockopt(130448, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130448, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130450
fcntl(130450, F_SETFL, FNONBLOCK) = 0
setsockopt(130450, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130450, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130451
fcntl(130451, F_SETFL, FNONBLOCK) = 0
setsockopt(130451, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130451, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130452
fcntl(130452, F_SETFL, FNONBLOCK) = 0
setsockopt(130452, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130452, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130453
fcntl(130453, F_SETFL, FNONBLOCK) = 0
setsockopt(130453, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130453, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130454
fcntl(130454, F_SETFL, FNONBLOCK) = 0
setsockopt(130454, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130454, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130455
fcntl(130455, F_SETFL, FNONBLOCK) = 0
setsockopt(130455, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130455, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
pollsys(0x0879CE08, 65569, 0x08047C88, 0x00000000) = 32
connect(130386, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130393, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130410, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130412, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130414, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130416, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130420, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130421, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130423, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130425, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130426, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130428, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130430, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130432, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130433, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130435, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130436, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130437, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130439, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130440, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130442, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130443, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130444, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130445, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130446, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130448, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130450, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130451, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130452, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130453, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130454, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
connect(130455, 0x08047C68, 16, SOV_DEFAULT) Err#133 EISCONN
close(130386) = 0
close(130393) = 0
close(130410) = 0
close(130412) = 0
close(130414) = 0
close(130416) = 0
close(130420) = 0
close(130421) = 0
close(130423) = 0
close(130425) = 0
close(130426) = 0
close(130428) = 0
close(130430) = 0
close(130432) = 0
close(130433) = 0
close(130435) = 0
close(130436) = 0
close(130437) = 0
close(130439) = 0
close(130440) = 0
close(130442) = 0
close(130443) = 0
close(130444) = 0
close(130445) = 0
close(130446) = 0
close(130448) = 0
close(130450) = 0
close(130451) = 0
close(130452) = 0
close(130453) = 0
close(130454) = 0
close(130455) = 0
pollsys(0x0879CE08, 65537, 0x08047C88, 0x00000000) (sleeping...)
pollsys(0x0879CE08, 65537, 0x08047C88, 0x00000000) = 0
pollsys(0x0879CE08, 65537, 0x08047C88, 0x00000000) (sleeping...)
pollsys(0x0879CE08, 65537, 0x08047C88, 0x00000000) = 0
pollsys(0x0879CE08, 65537, 0x08047C88, 0x00000000) (sleeping...)
pollsys(0x0879CE08, 65537, 0x08047C88, 0x00000000) = 0
pollsys(0x0879CE08, 65537, 0x08047C88, 0x00000000) (sleeping...)
pollsys(0x0879CE08, 65537, 0x08047C88, 0x00000000) = 0
pollsys(0x0879CE08, 65537, 0x08047C88, 0x00000000) (sleeping...)
pollsys(0x0879CE08, 65537, 0x08047C88, 0x00000000) = 0
pollsys(0x0879CE08, 65537, 0x08047C88, 0x00000000) (sleeping...)
pollsys(0x0879CE08, 65537, 0x08047C88, 0x00000000) = 0
pollsys(0x0879CE08, 65537, 0x08047C88, 0x00000000) = 1
recv(128911, "8184\0\0\0\0 p i n g", 16384, 0) = 10
recv(128911, 0x8E14799A, 16374, 0) Err#11 EAGAIN
getsockopt(129018, SOL_SOCKET, SO_ERROR, 0x08047BF8, 0x08047BFC, SOV_DEFAULT) = 0
send(129018, "8184\0\0\0\0 p i n g", 10, 128) = 10
pollsys(0x0879CE08, 65537, 0x08047C88, 0x00000000) (sleeping...)
pollsys(0x0879CE08, 65537, 0x08047C88, 0x00000000) = 0
pollsys(0x0879CE08, 65537, 0x08047C88, 0x00000000) (sleeping...)
pollsys(0x0879CE08, 65537, 0x08047C88, 0x00000000) = 0
pollsys(0x0879CE08, 65537, 0x08047C88, 0x00000000) (sleeping...)
pollsys(0x0879CE08, 65537, 0x08047C88, 0x00000000) = 0
pollsys(0x0879CE08, 65537, 0x08047C88, 0x00000000) = 0
pollsys(0x0879CE08, 65537, 0x08047C88, 0x00000000) = 0
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130386
fcntl(130386, F_SETFL, FNONBLOCK) = 0
setsockopt(130386, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130386, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130393
fcntl(130393, F_SETFL, FNONBLOCK) = 0
setsockopt(130393, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130393, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130410
fcntl(130410, F_SETFL, FNONBLOCK) = 0
setsockopt(130410, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130410, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130412
fcntl(130412, F_SETFL, FNONBLOCK) = 0
setsockopt(130412, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130412, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130414
fcntl(130414, F_SETFL, FNONBLOCK) = 0
setsockopt(130414, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130414, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130416
fcntl(130416, F_SETFL, FNONBLOCK) = 0
setsockopt(130416, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130416, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130420
fcntl(130420, F_SETFL, FNONBLOCK) = 0
setsockopt(130420, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130420, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130421
fcntl(130421, F_SETFL, FNONBLOCK) = 0
setsockopt(130421, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130421, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130423
fcntl(130423, F_SETFL, FNONBLOCK) = 0
setsockopt(130423, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130423, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130425
fcntl(130425, F_SETFL, FNONBLOCK) = 0
setsockopt(130425, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130425, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130426
fcntl(130426, F_SETFL, FNONBLOCK) = 0
setsockopt(130426, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130426, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130428
fcntl(130428, F_SETFL, FNONBLOCK) = 0
setsockopt(130428, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130428, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130430
fcntl(130430, F_SETFL, FNONBLOCK) = 0
setsockopt(130430, tcp, TCP_NODELAY, 0x0809BE18, 4, SOV_DEFAULT) = 0
connect(130430, 0x08047C70, 16, SOV_DEFAULT) Err#150 EINPROGRESS
so_socket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 0x00000000, SOV_DEFAULT) = 130432
fcntl(130432, F_SETFL, FNONBLOCK)
Former user commented on 2015-02-05T19:46:20.000-0500:
We've hit this on a third haproxy in us-east, too. It's also huge, also on an old platform, and also started on October 16:
[root@1a1ff4e5 (webapi) ~]$ ps -opid,rss,vsz,args -p 56448
PID RSS VSZ COMMAND
56448 2198032 2205144 /opt/local/sbin/haproxy -f /opt/smartdc/muskie/etc/haproxy.cfg -D
[root@1a1ff4e5 (webapi) ~]$ uname -v
joyent_20130723T015243Z
[root@1a1ff4e5 (webapi) ~]$ svcs -p haproxy
STATE STIME FMRI
online Oct_16 svc:/manta/haproxy:default
Oct_16 56448 haproxy