SCTP Multihoming

Question

I've been developing this simple client - server application with C where the client is just sending random data to the server and the server just listens to what the client sends. The protocol I'm using is SCTP and I'm interested on how to implement the multihoming feature to it.

I've been searching through the internet about SCTP and multihoming and haven't been able to find any examples about how to instruct SCTP to use multiple addresses for communication. I've only managed to find what commands one should use when trying to setup SCTP with multihoming and it should be quite straightforward.

I've created a client and a server which both use my computers two WLAN interfaces as their connection points. Both adapters are connected to the same AP. The server is listening for data from the client from these interfaces and the client sends data through them. The problem is that when I disconnect the primary WLAN adapter the client is sending data to, the transmission just halts when it should fallback to the secondary connection. I've traced the packets with Wireshark and the first INIT and INIT_ACK packets report that both the client and the server are using the WLAN adapters as their communication links.

When I reconnect the primary WLAN connection the transmission continues after a little while and bursts a huge load of packets to the server which isn't right. The packets should have been transmitted over the secondary connection. On many sites it is said that SCTP switches between connections automagically but in my case that's not happening. So do you guys have any clues why the transmission doesn't fallback to the secondary connection when the primary link is down even though the client and the server knows each others addresses including the secondary address?

About the server:

The server creates a SOCK_SEQPACKET type socket and binds all interfaces found with INADDR_ANY. getladdrs reports that the server is bounded to 3 addresses (including 127.0.0.1). After that the server listens to the socket and waits the client to send data. Server reads the data with sctp_recvmsg call.

About the client:

The client creates also a SEQPACKET socket and connects to an IP-address specified by a commandline argument. getladdrs in this case returns also 3 addresses like in the servers case. After that the client just starts to send data to the server with one second delay to the server until the user interrupts the send with Ctrl-C.

Here's some source code:

Server:

#define BUFFER_SIZE (1 << 16)
#define PORT 10000   

int sock, ret, flags;
int i;
int addr_count = 0;
char buffer[BUFFER_SIZE];
socklen_t from_len;

struct sockaddr_in addr;
struct sockaddr_in *laddr[10];
struct sockaddr_in *paddrs[10];
struct sctp_sndrcvinfo sinfo;
struct sctp_event_subscribe event;  
struct sctp_prim prim_addr; 
struct sctp_paddrparams heartbeat;
struct sigaction sig_handler;

void handle_signal(int signum);

int main(void)
{
    if((sock = socket(AF_INET, SOCK_SEQPACKET, IPPROTO_SCTP)) < 0)
        perror("socket");

    memset(&addr, 0, sizeof(struct sockaddr_in));
    memset((void*)&event, 1, sizeof(struct sctp_event_subscribe));

    addr.sin_family = AF_INET;
    addr.sin_addr.s_addr = htonl(INADDR_ANY);
    addr.sin_port = htons(PORT);

    from_len = (socklen_t)sizeof(struct sockaddr_in);

    sig_handler.sa_handler = handle_signal;
    sig_handler.sa_flags = 0;

    if(sigaction(SIGINT, &sig_handler, NULL) == -1)
        perror("sigaction");

    if(setsockopt(sock, IPPROTO_SCTP, SCTP_EVENTS, &event, sizeof(struct sctp_event_subscribe)) < 0)
        perror("setsockopt");

    if(setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &reuse, sizeof(int))< 0)
        perror("setsockopt");

    if(bind(sock, (struct sockaddr*)&addr, sizeof(struct sockaddr)) < 0)
        perror("bind");

    if(listen(sock, 2) < 0)
        perror("listen");

    addr_count = sctp_getladdrs(sock, 0, (struct sockaddr**)laddr);
    printf("Addresses binded: %d\n", addr_count);

    for(i = 0; i < addr_count; i++)
         printf("Address %d: %s:%d\n", i +1, inet_ntoa((*laddr)[i].sin_addr), (*laddr)[i].sin_port);

    sctp_freeladdrs((struct sockaddr*)*laddr);

    while(1)
    {
        flags = 0;

        ret = sctp_recvmsg(sock, buffer, BUFFER_SIZE, (struct sockaddr*)&addr, &from_len, NULL, &flags);

        if(flags & MSG_NOTIFICATION)
        printf("Notification received from %s:%u\n",  inet_ntoa(addr.sin_addr), ntohs(addr.sin_port));

        printf("%d bytes received from %s:%u\n", ret, inet_ntoa(addr.sin_addr), ntohs(addr.sin_port));      
    }

    if(close(sock) < 0)
        perror("close");
}   

void handle_signal(int signum)
{
    switch(signum)
    {
        case SIGINT:
            if(close(sock) != 0)
                perror("close");
            exit(0);
            break;  
        default: exit(0);
            break;
    }
}

And the Client:

#define PORT 10000
#define MSG_SIZE 1000
#define NUMBER_OF_MESSAGES 1000
#define PPID 1234

int sock;
struct sockaddr_in *paddrs[10];
struct sockaddr_in *laddrs[10];

void handle_signal(int signum);

int main(int argc, char **argv)
{
    int i;
    int counter = 1;
    int ret;
    int addr_count;
    char address[16];
    char buffer[MSG_SIZE];
    sctp_assoc_t id;
    struct sockaddr_in addr;
    struct sctp_status status;
    struct sctp_initmsg initmsg;
    struct sctp_event_subscribe events;
    struct sigaction sig_handler;

    memset((void*)&buffer,  'j', MSG_SIZE);
    memset((void*)&initmsg, 0, sizeof(initmsg));
    memset((void*)&addr,    0, sizeof(struct sockaddr_in));
    memset((void*)&events, 1, sizeof(struct sctp_event_subscribe));

    if(argc != 2 || (inet_addr(argv[1]) == -1))
    {
        puts("Usage: client [IP ADDRESS in form xxx.xxx.xxx.xxx] ");        
        return 0;
    }

    strncpy(address, argv[1], 15);
    address[15] = 0;

    addr.sin_family = AF_INET;
    inet_aton(address, &(addr.sin_addr));
    addr.sin_port = htons(PORT);

    initmsg.sinit_num_ostreams = 2;
    initmsg.sinit_max_instreams = 2;
    initmsg.sinit_max_attempts = 5;

    sig_handler.sa_handler = handle_signal;
    sig_handler.sa_flags = 0;

    if(sigaction(SIGINT, &sig_handler, NULL) == -1)
        perror("sigaction");

    if((sock = socket(AF_INET, SOCK_SEQPACKET, IPPROTO_SCTP)) < 0)
        perror("socket");

    if((setsockopt(sock, SOL_SCTP, SCTP_INITMSG, &initmsg, sizeof(initmsg))) != 0)
        perror("setsockopt");

    if((setsockopt(sock, SOL_SCTP, SCTP_EVENTS, (const void *)&events, sizeof(events))) != 0)
        perror("setsockopt");

    if(sendto(sock, buffer, MSG_SIZE, 0, (struct sockaddr*)&addr, sizeof(struct sockaddr)) == -1)
        perror("sendto");

    addr_count = sctp_getpaddrs(sock, 0, (struct sockaddr**)paddrs);
    printf("\nPeer addresses: %d\n", addr_count);

    for(i = 0; i < addr_count; i++)
    printf("Address %d: %s:%d\n", i +1, inet_ntoa((*paddrs)[i].sin_addr), (*paddrs)[i].sin_port);

    sctp_freepaddrs((struct sockaddr*)*paddrs);

    addr_count = sctp_getladdrs(sock, 0, (struct sockaddr**)laddrs);
    printf("\nLocal addresses: %d\n", addr_count);

    for(i = 0; i < addr_count; i++)
    printf("Address %d: %s:%d\n", i +1, inet_ntoa((*laddrs)[i].sin_addr), (*laddrs)[i].sin_port);

    sctp_freeladdrs((struct sockaddr*)*laddrs);

    i = sizeof(status);
    if((ret = getsockopt(sock, SOL_SCTP, SCTP_STATUS, &status, (socklen_t *)&i)) != 0)
        perror("getsockopt");

    printf("\nSCTP Status:\n--------\n");
    printf("assoc id  = %d\n", status.sstat_assoc_id);
    printf("state     = %d\n", status.sstat_state);
    printf("instrms   = %d\n", status.sstat_instrms);
    printf("outstrms  = %d\n--------\n\n", status.sstat_outstrms);

    for(i = 0; i < NUMBER_OF_MESSAGES; i++)
    {
        counter++;
        printf("Sending data chunk #%d...", counter);
        if((ret = sendto(sock, buffer, MSG_SIZE, 0, (struct sockaddr*)&addr, sizeof(struct sockaddr))) == -1)
            perror("sendto");

        printf("Sent %d bytes to peer\n",ret);

        sleep(1);
    }

    if(close(sock) != 0)
        perror("close");
}


void handle_signal(int signum)
{

    switch(signum)
    {
        case SIGINT:
            if(close(sock) != 0)
                perror("close");
            exit(0);
            break;  
        default: exit(0);
            break;
    }

}

So do you guys have any clues what I'm doing wrong?

Answer 1

I have never tried SCTP but according to this site it supports two models.

one-to-one style and one-to-many style.

As far as I understand multihoming works for one-to-one style.

In order to open a one-to-one style socket you need a call like :

connSock = socket( AF_INET, SOCK_STREAM, IPPROTO_SCTP );

Note that SOCK_STREAM is different than your SOCK_SEQPACKET

When you do

sock = socket(AF_INET, SOCK_SEQPACKET, IPPROTO_SCTP)

it seems to open one-to-many style socket which I could not understand if it supports multihoming or not.

So try your code with SOCK_STREAM parameter.

Also here is an example that uses SOCK_STREAM.

Answer 2

Ok I resolved the multihoming problem finally. Here's what I did.

I adjusted the heartbeat value to 5000 ms with sctp_paddrparams struct. The flags variable located in the struct has to in SPP_HB_ENABLE mode because otherwise SCTP ignores the heartbeat value when trying to set the value with setsockopt().

That was the reason why SCTP didn't send heartbeats as often as I wanted. The reason, why I didn't notice the flag variable, was the obsolete reference guide to SCTP I was reading, which stated that there didn't exist a flags variable inside the struct! Newer reference revealed that there was. So heartbeat problem solved!

Another thing was to modify the rto_max value to, for example, 2000 ms or so. Lowering the value tells SCTP to change the path much sooner. The default value was 60 000 ms which was too high (1 minute before it starts to change the path). rto_max value can be adjusted with sctp_rtoinfo struct.

With these two modifications the multihoming started to work. Oh and a another thing. Client has to be in STREAM mode when the Server is in SEQPACKET mode. Client sends data to server with normal send() command and Server read data with sctp_recvmsg() where addr struct is set to NULL.

I hope that this information helps other guys struggling with the multihoming of SCTP. Cheers guys for your opinions, those were a big help for me! Here is some code example so this maybe the first multihoming simple example in the net if you ask me (didnt find any examples than multistreaming examples)

Server:

#include <sys/types.h>
#include <sys/socket.h>
#include <signal.h>
#include <netinet/in.h>
#include <netinet/sctp.h>
#include <arpa/inet.h>
#include <string.h>
#include <unistd.h>
#include <stdio.h>
#include <sys/ioctl.h>
#include <net/if.h>
#include <stdlib.h>
#include <pthread.h>

#define BUFFER_SIZE (1 << 16)
#define PORT 10000   

int sock, ret, flags;
int i, reuse = 1;
int addr_count = 0;
char buffer[BUFFER_SIZE];
socklen_t from_len;

struct sockaddr_in addr;
struct sockaddr_in *laddr[10];
struct sockaddr_in *paddrs[10];
struct sctp_sndrcvinfo sinfo;
struct sctp_event_subscribe event;  
struct sctp_prim prim_addr; 
struct sctp_paddrparams heartbeat;
struct sigaction sig_handler;
struct sctp_rtoinfo rtoinfo;

void handle_signal(int signum);

int main(void)
{
    if((sock = socket(AF_INET, SOCK_SEQPACKET, IPPROTO_SCTP)) < 0)
        perror("socket");

    memset(&addr,       0, sizeof(struct sockaddr_in));
    memset(&event,      1, sizeof(struct sctp_event_subscribe));
    memset(&heartbeat,  0, sizeof(struct sctp_paddrparams));
    memset(&rtoinfo,    0, sizeof(struct sctp_rtoinfo));

    addr.sin_family = AF_INET;
    addr.sin_addr.s_addr = htonl(INADDR_ANY);
    addr.sin_port = htons(PORT);

    from_len = (socklen_t)sizeof(struct sockaddr_in);

    sig_handler.sa_handler = handle_signal;
    sig_handler.sa_flags = 0;

    heartbeat.spp_flags = SPP_HB_ENABLE;
    heartbeat.spp_hbinterval = 5000;
    heartbeat.spp_pathmaxrxt = 1;

    rtoinfo.srto_max = 2000;

    /*Set Heartbeats*/
    if(setsockopt(sock, SOL_SCTP, SCTP_PEER_ADDR_PARAMS , &heartbeat, sizeof(heartbeat)) != 0)
        perror("setsockopt");

    /*Set rto_max*/
    if(setsockopt(sock, SOL_SCTP, SCTP_RTOINFO , &rtoinfo, sizeof(rtoinfo)) != 0)
        perror("setsockopt");

    /*Set Signal Handler*/
    if(sigaction(SIGINT, &sig_handler, NULL) == -1)
        perror("sigaction");

    /*Set Events */
    if(setsockopt(sock, IPPROTO_SCTP, SCTP_EVENTS, &event, sizeof(struct sctp_event_subscribe)) < 0)
        perror("setsockopt");

    /*Set the Reuse of Address*/
    if(setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &reuse, sizeof(int))< 0)
        perror("setsockopt");

    /*Bind the Addresses*/
    if(bind(sock, (struct sockaddr*)&addr, sizeof(struct sockaddr)) < 0)
        perror("bind");

    if(listen(sock, 2) < 0)
        perror("listen");

    /*Get Heartbeat Value*/
    i = (sizeof heartbeat);
    getsockopt(sock, SOL_SCTP, SCTP_PEER_ADDR_PARAMS, &heartbeat, (socklen_t*)&i);
    printf("Heartbeat interval %d\n", heartbeat.spp_hbinterval);

    /*Print Locally Binded Addresses*/
    addr_count = sctp_getladdrs(sock, 0, (struct sockaddr**)laddr);
    printf("Addresses binded: %d\n", addr_count);
    for(i = 0; i < addr_count; i++)
         printf("Address %d: %s:%d\n", i +1, inet_ntoa((*laddr)[i].sin_addr), (*laddr)[i].sin_port);
    sctp_freeladdrs((struct sockaddr*)*laddr);

    while(1)
    {
        flags = 0;

        ret = sctp_recvmsg(sock, buffer, BUFFER_SIZE, NULL, 0, NULL, &flags);

        if(flags & MSG_NOTIFICATION)
        printf("Notification received from %s:%u\n",  inet_ntoa(addr.sin_addr), ntohs(addr.sin_port));

        printf("%d bytes received from %s:%u\n", ret, inet_ntoa(addr.sin_addr), ntohs(addr.sin_port));      
    }

    if(close(sock) < 0)
        perror("close");
}   

void handle_signal(int signum)
{
    switch(signum)
    {
        case SIGINT:
            if(close(sock) != 0)
                perror("close");
            exit(0);
            break;  
        default: exit(0);
            break;
    }
}

Client:

#include <sys/types.h>
#include <sys/socket.h>
#include <signal.h>
#include <netinet/in.h>
#include <netinet/sctp.h>
#include <arpa/inet.h>
#include <string.h>
#include <unistd.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/ioctl.h>

#define PORT 11000
#define MSG_SIZE 1000
#define NUMBER_OF_MESSAGES 1000

int sock;
struct sockaddr_in *paddrs[5];
struct sockaddr_in *laddrs[5];

void handle_signal(int signum);

int main(int argc, char **argv)
{
    int i;
    int counter = 0;
    int asconf = 1;
    int ret;
    int addr_count;
    char address[16];
    char buffer[MSG_SIZE];
    sctp_assoc_t id;
    struct sockaddr_in addr;
    struct sctp_status status;
    struct sctp_initmsg initmsg;
    struct sctp_event_subscribe events;
    struct sigaction sig_handler;
    struct sctp_paddrparams heartbeat;
    struct sctp_rtoinfo rtoinfo;

    memset(&buffer,     'j', MSG_SIZE);
    memset(&initmsg,    0,   sizeof(struct sctp_initmsg));
    memset(&addr,       0,   sizeof(struct sockaddr_in));
    memset(&events,     1,   sizeof(struct sctp_event_subscribe));
    memset(&status,     0,   sizeof(struct sctp_status));
    memset(&heartbeat,  0,   sizeof(struct sctp_paddrparams));
    memset(&rtoinfo,    0, sizeof(struct sctp_rtoinfo))

    if(argc != 2 || (inet_addr(argv[1]) == -1))
    {
        puts("Usage: client [IP ADDRESS in form xxx.xxx.xxx.xxx] ");        
        return 0;
    }

    strncpy(address, argv[1], 15);
    address[15] = 0;

    addr.sin_family = AF_INET;
    inet_aton(address, &(addr.sin_addr));
    addr.sin_port = htons(PORT);

    initmsg.sinit_num_ostreams = 2;
    initmsg.sinit_max_instreams = 2;
    initmsg.sinit_max_attempts = 1;

    heartbeat.spp_flags = SPP_HB_ENABLE;
    heartbeat.spp_hbinterval = 5000;
    heartbeat.spp_pathmaxrxt = 1;

    rtoinfo.srto_max = 2000;

    sig_handler.sa_handler = handle_signal;
    sig_handler.sa_flags = 0;

    /*Handle SIGINT in handle_signal Function*/
    if(sigaction(SIGINT, &sig_handler, NULL) == -1)
        perror("sigaction");

    /*Create the Socket*/
    if((ret = (sock = socket(AF_INET, SOCK_STREAM, IPPROTO_SCTP))) < 0)
        perror("socket");

    /*Configure Heartbeats*/
    if((ret = setsockopt(sock, SOL_SCTP, SCTP_PEER_ADDR_PARAMS , &heartbeat, sizeof(heartbeat))) != 0)
        perror("setsockopt");

    /*Set rto_max*/
    if((ret = setsockopt(sock, SOL_SCTP, SCTP_RTOINFO , &rtoinfo, sizeof(rtoinfo))) != 0)
        perror("setsockopt");

    /*Set SCTP Init Message*/
    if((ret = setsockopt(sock, SOL_SCTP, SCTP_INITMSG, &initmsg, sizeof(initmsg))) != 0)
        perror("setsockopt");

    /*Enable SCTP Events*/
    if((ret = setsockopt(sock, SOL_SCTP, SCTP_EVENTS, (void *)&events, sizeof(events))) != 0)
        perror("setsockopt");

    /*Get And Print Heartbeat Interval*/
    i = (sizeof heartbeat);
    getsockopt(sock, SOL_SCTP, SCTP_PEER_ADDR_PARAMS, &heartbeat, (socklen_t*)&i);

    printf("Heartbeat interval %d\n", heartbeat.spp_hbinterval);

    /*Connect to Host*/
    if(((ret = connect(sock, (struct sockaddr*)&addr, sizeof(struct sockaddr)))) < 0)
    {
        perror("connect");
        close(sock);
        exit(0);
    }

    /*Get Peer Addresses*/
    addr_count = sctp_getpaddrs(sock, 0, (struct sockaddr**)paddrs);
    printf("\nPeer addresses: %d\n", addr_count);

    /*Print Out Addresses*/
    for(i = 0; i < addr_count; i++)
        printf("Address %d: %s:%d\n", i +1, inet_ntoa((*paddrs)[i].sin_addr), (*paddrs)[i].sin_port);

    sctp_freepaddrs((struct sockaddr*)*paddrs);

    /*Start to Send Data*/
    for(i = 0; i < NUMBER_OF_MESSAGES; i++)
    {
        counter++;
        printf("Sending data chunk #%d...", counter);

        if((ret = send(sock, buffer, MSG_SIZE, 0)) == -1)
            perror("write");

        printf("Sent %d bytes to peer\n",ret);

        sleep(1);
    }

    if(close(sock) != 0)
        perror("close");
}


void handle_signal(int signum)
{
    switch(signum)
    {
        case SIGINT:
            if(close(sock) != 0)
                perror("close");
            exit(0);
            break;  
        default: exit(0);
            break;
    }
}

Answer 3

You client opens an assosiation by using sendto() call. It is OK. But after that you should not use sendto() anymore. Because sendto() will propably force SCTP to use interface of the given IP. So, use write() or send() instead of sendto():

counter++;
printf("Sending data chunk #%d...", counter);

if((ret = write(sock, buffer, MSG_SIZE) == -1)
    perror("write");