0

I wrote a man-in-the-middle/proxy server initially on my mac. Essentially the proxy creates a socket and waits for a connection, then connects to another application. This works flawlessly on in OS X and in OpenBSD; however, when porting the proxy to Ubuntu, it doesn't work as intended.

There is two instances of this proxy running, listening on two separate ports. When this is ran on Ubuntu, all the traffic goes through a single port. I also run into a problem when setting the socket to nonblocking (through fcntl) that sometimes it fails with "Invalid Argument"

I am using sys/socket.

Any pitfalls during this port that I am missing?

EDIT:

I believe there are two problems. One being the Invalid Argument, the other being that traffic is being pushed to different ports.

traffic issue

Service 1 binds to Proxy instance 1 which then binds back to the appropriate service on the black box, which kicks off service 2. However, for some reason on Ubuntu it connects to the Instance 1 which is listening on the incorrect port.

EDIT Solution to Invalid Argument for fcntl:

Found out why i was getting the invalid argument, sadly i'm still having the other issue. fcntl(fd, cmd, arg)

cmd - F_SETFL(long)

I was passing in a pointer to an int instead of the long primitive.

EDIT:

#include <stdio.h>
#include <unistd.h>
#include <stdlib.h>
#include <errno.h>
#include <netdb.h>
#include <string.h>
#include <signal.h>
#include <assert.h>
#include <syslog.h>


#include <sys/types.h>
#include <sys/select.h>
#include <sys/file.h>
#include <sys/ioctl.h>
#include <sys/param.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <sys/time.h>
#include <sys/wait.h>

#include <netinet/in.h>

#include <arpa/ftp.h>
#include <arpa/inet.h>
#include <arpa/telnet.h>


void cleanup(int sig)
{
    syslog(LOG_INFO, "Cleaning up...");
    exit(0);
}

void sigreap(int sig)
{
    int status;
    pid_t p;
    while ((p = waitpid(-1, &status, WNOHANG)) > 0) {
    syslog(LOG_INFO, "sigreap: pid=%d, status=%d\n", (int) p, status);
    }
    /* doh! */
    signal(SIGCHLD, sigreap);
}

void set_nonblock(int fd)
{
    long fl;
    int x;
    fl = fcntl(fd, F_GETFL);
    if (fl < 0) {
    syslog(LOG_ERR, "fcntl F_GETFL: FD %d: %s", fd, strerror(errno));
    exit(1);
    }
    fl |= O_NONBLOCK;
    x = fcntl(fd, F_SETFL, fl);
    if (x < 0) {
    syslog(LOG_ERR, "fcntl F_SETFL: FD %d: %s", fd, strerror(errno));
    exit(1);
    }
}


int create_server_sock(char *addr, int port)
{
    int addrlen, s, on = 1, x;
    static struct sockaddr_in client_addr;

    s = socket(AF_INET, SOCK_STREAM, 0);
    if (s < 0)
        perror("socket"), exit(1);

    addrlen = sizeof(client_addr);
    memset(&client_addr, '\0', addrlen);
    client_addr.sin_family = AF_INET;
    client_addr.sin_addr.s_addr = INADDR_ANY; //inet_addr(addr);
    client_addr.sin_port = htons(port);
    setsockopt(s, SOL_SOCKET, SO_REUSEADDR, &on, 4);
    x = bind(s, (struct sockaddr *) &client_addr, addrlen);
    if (x < 0)
        perror("bind"), exit(1);

    x = listen(s, 5);
    if (x < 0)
        perror("listen"), exit(1);

    return s;
}

int open_remote_host(char *host, int port)
{
    struct sockaddr_in rem_addr;
    int len, s, x;
    struct hostent *H;
    int on = 1;

    H = gethostbyname(host);
    if (!H)
    return (-2);

    len = sizeof(rem_addr);

    s = socket(AF_INET, SOCK_STREAM, 0);
    if (s < 0)
    return s;

    setsockopt(s, SOL_SOCKET, SO_REUSEADDR, &on, 4);

    len = sizeof(rem_addr);
    memset(&rem_addr, '\0', len);
    rem_addr.sin_family = AF_INET;
    memcpy(&rem_addr.sin_addr, H->h_addr, H->h_length);
    rem_addr.sin_port = htons(port);
    x = connect(s, (struct sockaddr *) &rem_addr, len);
    if (x < 0) {
    close(s);
    return x;
    }
    set_nonblock(s);
    return s;
}

int get_hinfo_from_sockaddr(struct sockaddr_in addr, int len, char *fqdn)
{
    struct hostent *hostinfo;

    hostinfo = gethostbyaddr((char *) &addr.sin_addr.s_addr, len, AF_INET);
    if (!hostinfo) {
    sprintf(fqdn, "%s", inet_ntoa(addr.sin_addr));
    return 0;
    }
    if (hostinfo && fqdn)
    sprintf(fqdn, "%s [%s]", hostinfo->h_name, inet_ntoa(addr.sin_addr));
    return 0;
}


int wait_for_connection(int s)
{
    int newsock;
    socklen_t len;
    static struct sockaddr_in peer;

    len = sizeof(struct sockaddr);
    newsock = accept(s, (struct sockaddr *) &peer, &len);
    /* dump_sockaddr (peer, len); */
    if (newsock < 0) {
    if (errno != EINTR)
        perror("accept");
    }
    get_hinfo_from_sockaddr(peer, len, client_hostname);
    set_nonblock(newsock);
    return (newsock);
}



static int print_bytes(char * buf, ssize_t length)
{
    int i = 0, ascii_off = 0, hex_off = 0;
    char * hex_bytes = (char *) calloc(32*2,1);
    char * ascii_bytes = (char *) calloc(32*2,1);

    for( i = 0; i < length; i++)
    {
        hex_off += sprintf(hex_bytes+hex_off,"%02X ",(unsigned char)buf[i]);
        if(buf[i] >= '!' && buf[i] <= '~')
            ascii_off += sprintf(ascii_bytes+ascii_off,"%c ",buf[i]);
        else
            ascii_off += sprintf(ascii_bytes+ascii_off,". ");
        if( ((i+1) % 16 == 0) || i == length-1 )
        {
            fprintf(stderr,"%-48s\t%s\n",hex_bytes,ascii_bytes);
            free(hex_bytes);
            free(ascii_bytes);
            hex_bytes = (char *) calloc(32*2,1);
            ascii_bytes = (char *) calloc(32*2,1);
            ascii_off = 0;
            hex_off = 0;
            if(i != length-1)
                fprintf(stderr,"\t");
        }


    }
    free(hex_bytes);
    free(ascii_bytes);
    return 0;
}


int mywrite(int fd, char *buf, int *len)
{
    int x = write(fd, buf, *len);
    print_bytes(buf,*len);
    if (x < 0)
        return x;
    if (x == 0)
        return x;
    if (x != *len)
        memmove(buf, buf+x, (*len)-x);
    *len -= x;
    return x;
}


void service_client(int fd1, int fd2, int injfd)
{
    int maxfd;

    cfd = fd1;
    sfd = fd2;
    char *sbuf;
    char *cbuf;
    int x, n;
    int cbo = 0;
    int sbo = 0;
    int ibo = 0;
    fd_set R;
    int max_clients = 30;
    int i = 0,s = 0, addrlen;
    struct sockaddr_in address;

    sbuf = malloc(BUF_SIZE);
    cbuf = malloc(BUF_SIZE);
    cntrlbuf = calloc(1,BUF_SIZE);
    char * injbuf = malloc(BUF_SIZE);
    maxfd = cfd > sfd ? cfd : sfd; 
    maxfd = injfd > maxfd ? injfd : maxfd;
    maxfd++;
    maxfd++;

    struct inj_con * ptr;

    while (1) {
        struct timeval to;
        if (cbo) {
            process_packet(cbuf,&cbo,sfd);
        }
        if (sbo) {
            process_packet(sbuf,&sbo,cfd);
        }
        if (ibo) {
            process_packet(injbuf,&ibo,cfd);
        }
        if (cntrlo) {
            fprintf(stderr,"\033[33;1mControl->(%d), len = 0x%x (%d):\033[0m\n\t",cfd,cntrlo,cntrlo);
            if (mywrite(cfd, cntrlbuf, &cntrlo) < 0 && errno != EWOULDBLOCK) {
                syslog(LOG_ERR, "write %d: %s", cfd, strerror(errno));
                exit(1);
            }
        }
        FD_ZERO(&R);
        if (cbo < BUF_SIZE)
            FD_SET(cfd, &R);
        if (sbo < BUF_SIZE)
            FD_SET(sfd, &R);
        if (ibo < BUF_SIZE)
            FD_SET(injfd, &R);



        to.tv_sec = 0;
        to.tv_usec = 1000;
        x = select(max_clients+3, &R, 0, 0, &to);
        if (x > 0 || cntrl_q->item_count > 0) {
            if (FD_ISSET(injfd, &R)) {
                int new_socket;
                if((new_socket = accept(injfd, (struct sockaddr *) &address, (socklen_t *) &addrlen)) < 0)
                {
                    perror("accept");
                    exit(1);
                }


                // Truncated
                //
            }

            char * temp_pkt; 

            if (FD_ISSET(cfd, &R)) {
                temp_pkt = (char *) calloc(BUF_SIZE,1);
                n = read(cfd, temp_pkt, BUF_SIZE);
                syslog(LOG_INFO, "read %d bytes from CLIENT (%d)", n, cfd);
                if (n > 0) {
                    push_msg(s_q,temp_pkt,n);
                } else {
                    free(temp_pkt);
                    close(cfd);
                    close(sfd);
                    close_injection_sockets(); 
                    close(injfd);
                    _exit(0);
                }
            } 
            if (FD_ISSET(sfd, &R)) {
                temp_pkt = (char *) calloc(BUF_SIZE,1);
                n = read(sfd, temp_pkt, BUF_SIZE);
                syslog(LOG_INFO, "read %d bytes from SERVER (%d)\n", n, sfd);
                if (n > 0) {
                    push_msg(c_q,temp_pkt,n);
                } else {
                    free(temp_pkt);
                    close(sfd);
                    close(cfd);
                    close_injection_sockets(); 
                    close(injfd);
                    _exit(0);
                }
            }

            if(cntrlo == 0 && cntrl_q->front != NULL)
            {
                struct msg * tmp = next_msg(cntrl_q);
                if(tmp != NULL)
                {
                    memcpy(cntrlbuf,tmp->msg,tmp->len);
                    cntrlo += tmp->len;
                    free(tmp->msg);
                    free(tmp);
                }
            }
            if(sbo == 0 && c_q->front != NULL)
            {
                struct msg * tmp = next_msg(c_q);
                if(tmp != NULL)
                {
                    memcpy(sbuf,tmp->msg,tmp->len);
                    sbo += tmp->len;
                    free(tmp->msg);
                    free(tmp);
                }
            }
            if(cbo == 0 && s_q->front != NULL)
            {
                struct msg * tmp = next_msg(s_q);
                if(tmp != NULL)
                {
                    memcpy(cbuf,tmp->msg,tmp->len);
                    cbo += tmp->len;
                    free(tmp->msg);
                    free(tmp);
                }
            }
            if(ibo == 0 && inj_q->front != NULL)
            {
                struct msg * tmp = next_msg(inj_q);
                if(tmp != NULL)
                {
                    memcpy(injbuf,tmp->msg,tmp->len);
                    ibo += tmp->len;
                    free(tmp->msg);
                    free(tmp);
                }
            }

        } else if (x < 0 && errno != EINTR) {
            close(sfd);
            close(cfd);
            _exit(0);
        }

    }
}


static int create_injection_sock(int injectionport)
{
    struct sockaddr_in serv_addr;
    int portno = injectionport;

    int sockfd = socket(AF_INET, SOCK_STREAM, 0);

    if (sockfd <0)
    {
        perror("ERROR: opening socket");
        exit(1);
    }

    bzero((char *) &serv_addr, sizeof(serv_addr));
    serv_addr.sin_family = AF_INET;
    serv_addr.sin_addr.s_addr = INADDR_ANY;
    serv_addr.sin_port = htons(portno);

    if (bind(sockfd, (struct sockaddr *) &serv_addr, sizeof(serv_addr)) < 0)
    {
        perror("ERROR: on bind");
        exit(1);
    }

    if (listen(sockfd,5) < 0 )
    {
        perror("listen injection");
        exit(1);
    }
    return sockfd;
}


int main(int argc, char *argv[])
{

    if (!(5 == argc || 6 == argc)) {
        fprintf(stderr, "usage: %s laddr lport rhost rport [injectionport]\n", argv[0]);
        exit(1);
    }

    char *localaddr = strdup(argv[1]);
    int localport = atoi(argv[2]);
    char *remoteaddr = strdup(argv[3]);
    int remoteport = atoi(argv[4]);
    int injectionport;
    if(argc == 6)
        injectionport  = atoi(argv[5]);
    int client, server;
    int master_sock;
    int injection_sock = -1;

    cntrl_q = (struct item_queue *) calloc(1,sizeof(struct item_queue));
    inj_q = (struct item_queue *) calloc(1,sizeof(struct item_queue));
    s_q = (struct item_queue *) calloc(1,sizeof(struct item_queue));
    c_q = (struct item_queue *) calloc(1,sizeof(struct item_queue));
    identities = (struct item_queue *) calloc(1,sizeof(struct item_queue));


    assert(localaddr);
    assert(localport > 0);
    assert(remoteaddr);
    assert(remoteport > 0);
    if(argc == 6)
        assert(injectionport > 0);

    openlog(argv[0], LOG_PID, LOG_LOCAL4);

    signal(SIGINT, cleanup);
    signal(SIGCHLD, sigreap);
    if(argc == 6)
        injection_sock = create_injection_sock(injectionport);
    master_sock = create_server_sock(localaddr, localport);
    for (;;) {
        if ((client = wait_for_connection(master_sock)) < 0)
            continue;
        if ((server = open_remote_host(remoteaddr, remoteport)) < 0)
            continue;
        if (!fork()) {
            service_client(client, server, injection_sock);
            }
            close(client);
            close(server);
        }

}
6
  • The non-blocking invalid argument sounds suspiciously like a bug of some kind. It might behoove you to post some relevant code.
    – Duck
    Jan 8, 2014 at 16:45
  • added code for reference
    – jbh
    Jan 8, 2014 at 17:52
  • 1
    Post your actual code, not the starting point reference. Which call to fcntl() is failing, the one that calls F_GETFL or F_SETFL? Jan 8, 2014 at 18:04
  • F_SETFL is where it fails, but oddly not all the time.
    – jbh
    Jan 8, 2014 at 18:22
  • 1
    Do you actually care about writing this low-level socket code? If not, take a look at libevent or libev. I haven't used libev in anger, but I have used libevent for something like what you're doing and found it to be very efficient. Avoiding that memove will help too.
    – kfsone
    Jan 8, 2014 at 18:23

2 Answers 2

0

Just a guess, but I think your problem is the way you're handling the injection socket. I don't know what you deleted in the // TRUNCATED code block, but basically what you're doing is this:

// Parent process -- bind & listen on the injection socket
injfd = socket(...);
bind(injfd, ...);
listen(injfd, ...);
...
while (1)
{
    client = wait_for_connection();
    ...
    if (!fork())
    {
        // Each child process
        ...
        if (stuff && FD_ISSET(injfd, &R)) {
            new_socket = accept(injfd);
            ...
        }
        ...
    }
    ...
}

In other words, all of your child processes are listening on the same injection socket and trying to accept connections. I'm not sure if this even well-defined behavior, but even in the best case, when a new connection arrives on the injection port, the process that is going to accept the connection could be random and uncontrollable. It could be any of the child processes or even the parent process.

In order to control that behavior, you need to decide who should be listening for connections on the injection socket. If only the parent should be listening, then only the parent should call accept() on it or pass it as an argument to select(). Likewise, if only a particular child should be listening, then only that child should call accept() or pass it to select(). All other processes that are ignoring that socket should close() it at their earliest convenience (e.g. immediately after fork() returns) to avoid leaking file descriptors.

1
  • This was my initial thought on the issue, and i think this pertains to the nonblocking Invalid Argument issue. So earlier I removed the fork, and I still ran into the issue of all the traffic being funneled to a single port instead of the two ports as it should, and how it functions on BSD based systems.
    – jbh
    Jan 8, 2014 at 19:44
0

Turns out that SO_REUSEADDR socket option was the issue. I removed me setting that socket option and everything worked out.

This Lead me to the solution.

1
  • Impossible to believe.
    – user207421
    Oct 24, 2017 at 9:29

Your Answer

Reminder: Answers generated by Artificial Intelligence tools are not allowed on Stack Overflow. Learn more

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge that you have read and understand our privacy policy and code of conduct.

Not the answer you're looking for? Browse other questions tagged or ask your own question.