Boost asio: Unable to read URL Body (JSON)

Question

I have test the following Boost::Asio minimal HTTP server example: (Minimal HTTP Server based on Boost:Asio)

I'm able to successfully collect the Header information such as Content-Length but the example hangs when I try to read the Body information. Here is the function that attempts to read the Body information:

static void read_body(std::shared_ptr<session> pThis) {
            info("read_body");

            int nbuffer = pThis->headers.content_length();
            std::shared_ptr<std::vector<char>> bufptr = std::make_shared<std::vector<char>>(nbuffer);
            asio::async_read(pThis->socket, boost::asio::buffer(*bufptr, nbuffer),
                    [pThis](const error_code& e, std::size_t s)
                    {
                        info("read body complete");
                    });
        }

I have successfully read the same Body information utilizing a solution with libmicrohttpd.

Is there a more correct means to read the Body (JSON) information with Boost:ASIO?

Answer 1

The essence of the problem is twofold:

• firstly, there is Undefined Behaviour because you fail to capture bufptr in the completion handler, meaning that the vector is already freed by the time that the syscall is writing to it...

• Secondly, you're "junking" the first part of the body, as you have already received it when reading the headers. You should add code to preserve the part that was already received in read_next_line.

The real "misguidance" is probably in the use of

asio::async_read_until(pThis->socket, pThis->buff, '\r', ...

as if it will somehow receive exactly a single line. Sockets do not work that way. You'll receive a packet. The TCP stack(s) decide what constitutes a packet. Asio just promises to read "until" the packet >contains< the delimiter. It doesn't say it won't receive more, it just doesn't schedule another receive operation.

So, to really fix it, you might simply read all headers at once:

asio::async_read_until(pThis->socket, pThis->buff, "\r\n\r\n", 
    [pThis](const error_code &e, std::size_t s) {
        if (e) { std::cerr << "Error:" << __LINE__ << " " << e.message() << "\n"; return; }
        std::cout << __FILE__ << ":" << __LINE__ << " received:" << s << "\n";
        std::istream is(&pThis->buff);

        std::string line, ignore;

        if (getline(is, line, '\r') && is.ignore(1, '\n'))
            pThis->headers.on_read_request_line(line);

        while (getline(is, line, '\r') && is.ignore(1, '\n') && !line.empty())
            pThis->headers.on_read_header(line);

NOTE Again, it's important not to assume that the end-of-headers coincides with the packet boundary. Therefore, start read_body() with draining the available input already received:

    std::shared_ptr<std::vector<char> > bufptr = std::make_shared<std::vector<char> >(nbuffer);

    auto partial = std::copy(
            std::istreambuf_iterator<char>(&pThis->buff), {}, 
            bufptr->begin());

    std::size_t already_received = std::distance(bufptr->begin(), partial);

    assert(nbuffer >= already_received);
    nbuffer -= already_received;

FIXED UP DEMO

Live On Coliru

#include <boost/asio.hpp>
#include <boost/algorithm/string/trim.hpp>
#include <string>
#include <memory>
#include <iostream>
#include <fstream>

using namespace boost;
using namespace boost::system;
using namespace boost::asio;

unsigned char *get_icon(int *pOutSize);

class session;

class http_headers {
    std::string method;
    std::string url;
    std::string version;

    std::map<std::string, std::string> headers;

public:
    std::string get_response() {
        std::stringstream ssOut;
        if (url == "/favicon.ico") {
            int nSize = 0;
            unsigned char *data = get_icon(&nSize);

            ssOut << "HTTP/1.1 200 OK" << std::endl;
            ssOut << "content-type: image/vnd.microsoft.icon" << std::endl;
            ssOut << "Content-Length: " << nSize << std::endl;
            ssOut << std::endl;

            ssOut.write((char *)data, nSize);
        } else if (url == "/") {
            std::string sHTML = "<html><body><h1>Hello World</h1><p>This is a test web server in c++</p></body></html>";
            ssOut << "HTTP/1.1 200 OK" << std::endl;
            ssOut << "Content-Type: text/html" << std::endl;
            ssOut << "Content-Length: " << sHTML.length() << std::endl;
            ssOut << std::endl;
            ssOut << sHTML;
        } else {
            std::string sHTML = "<html><body><h1>404 Not Found</h1><p>There's nothing here.</p></body></html>";
            ssOut << "HTTP/1.1 404 Not Found" << std::endl;
            ssOut << "Content-Type: text/html" << std::endl;
            ssOut << "Content-Length: " << sHTML.length() << std::endl;
            ssOut << std::endl;
            ssOut << sHTML;
        }
        return ssOut.str();
    }

    size_t content_length() {
        auto request = headers.find("Content-Length");
        if (request != headers.end()) {
            std::stringstream ssLength(request->second);
            size_t content_length;
            ssLength >> content_length;
            return content_length;
        }
        return 0;
    }

    void on_read_header(std::string line) {
        std::cout << "header: '" << line << "'\n";

        std::stringstream ssHeader(line);
        std::string headerName;
        std::getline(ssHeader, headerName, ':');

        std::string value;
        std::getline(ssHeader, value);
        headers[headerName] = value;
    }

    void on_read_request_line(std::string line) {
        std::stringstream ssRequestLine(line);
        ssRequestLine >> method;
        ssRequestLine >> url;
        ssRequestLine >> version;

        std::cout << "request for resource: " << url << std::endl;
    }
};

namespace {
    static void info(std::string s) { std::cout << "INFO:" << s << "\n"; }
}

class session {
    asio::streambuf buff;
    http_headers headers;

    static void read_body(std::shared_ptr<session> pThis) {

        info("read_body");

        size_t nbuffer = pThis->headers.content_length();
        std::cout << __FILE__ << ":" << __LINE__ << " nbuffer:" << nbuffer << "\n";

        std::shared_ptr<std::vector<char> > bufptr = std::make_shared<std::vector<char> >(nbuffer);

        auto partial = std::copy(
                std::istreambuf_iterator<char>(&pThis->buff), {}, 
                bufptr->begin());

        std::size_t already_received = std::distance(bufptr->begin(), partial);

        assert(nbuffer >= already_received);
        nbuffer -= already_received;

        asio::async_read(pThis->socket, boost::asio::buffer(&*bufptr->begin() + already_received, nbuffer),
            [=](const error_code &e, std::size_t s) {
                info("read body complete"); 
                // EOF is to be expected on client disconnect
                if (e && e != boost::asio::error::eof) { 
                    std::cerr << "Error:" << __LINE__ << " " << e.message() << "\n"; return; 
                }

                std::cout << __FILE__ << ":" << __LINE__ << " received:" << s << "/" << nbuffer << "\n";

                std::string body(&*bufptr->begin(), already_received + s);

                std::string::size_type p = 0;
                for (int i = 0; i<2; ++i)
                    p = body.find_last_of("\r\n", p-1);

                std::cout << "Tail: '" << body.substr(p+1) << "'\n";

                {
                    std::ofstream ofs("debug.txt", std::ios::binary);
                    ofs << body;
                    ofs << "\n" << __FILE__ << ":" << __LINE__ << " received:" << s << "/" << nbuffer << "\n";
                }
            });
    }

    static void read_headers(std::shared_ptr<session> pThis) {
        asio::async_read_until(pThis->socket, pThis->buff, "\r\n\r\n", 
            [pThis](const error_code &e, std::size_t s) {
                if (e) { std::cerr << "Error:" << __LINE__ << " " << e.message() << "\n"; return; }
                std::cout << __FILE__ << ":" << __LINE__ << " received:" << s << "\n";
                std::istream is(&pThis->buff);

                std::string line, ignore;

                if (getline(is, line, '\r') && is.ignore(1, '\n'))
                    pThis->headers.on_read_request_line(line);

                while (getline(is, line, '\r') && is.ignore(1, '\n') && !line.empty())
                    pThis->headers.on_read_header(line);

                if (pThis->headers.content_length()) {
                    pThis->read_body(pThis);

                    auto str = std::make_shared<std::string>("HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n");
                    asio::async_write(
                        pThis->socket, boost::asio::buffer(*str),
                        [pThis, str](const error_code &e, std::size_t s) {
                            std::cout << "done" << std::endl; 
                        });
                } else {
                    std::shared_ptr<std::string> str = std::make_shared<std::string>(pThis->headers.get_response());
                    asio::async_write(
                        pThis->socket, boost::asio::buffer(*str),
                        [pThis, str](const error_code &e, std::size_t s) {
                            std::cout << "done" << std::endl; 
                        });
                }
            });
    }

public:
    ip::tcp::socket socket;

    session(io_service &io_service) : socket(io_service) {}

    static void interact(std::shared_ptr<session> pThis) { read_headers(pThis); }
};

void accept_and_run(ip::tcp::acceptor &acceptor, io_service &io_service) {
    std::shared_ptr<session> sesh = std::make_shared<session>(io_service);
    acceptor.async_accept(sesh->socket, [sesh, &acceptor, &io_service](const error_code &accept_error) {
        accept_and_run(acceptor, io_service);
        if (accept_error) {
            std::cerr << "Accept error: " << accept_error.message() << "\n";
        } else {
            session::interact(sesh);
        }
    });
}

int main() {
    io_service io_service;
    ip::tcp::endpoint endpoint{ ip::tcp::v4(), 8181 };
    ip::tcp::acceptor acceptor{ io_service, endpoint };

    acceptor.listen();
    accept_and_run(acceptor, io_service);

    io_service.run();
}

unsigned char icon_data[] = {
    // reserved
    0x00, 0x00,
    // icon type (1 = icon)
    0x01, 0x00,
    // number of images (1)
    0x01, 0x00,
    // width, height (16x16)
    0x10, 0x10,
    // size of colour palette
    0x00,
    // reserved
    0x00,
    // colour planes (1)
    0x01, 0x00,
    // bits per pixel (32)
    0x20, 0x00,

    // size of data in bytes
    0x28, 0x04, 0x00, 0x00,

    // offset of bitmap data
    0x16, 0x00, 0x00, 0x00,

    // BEGIN BITMAPINFOHEADER
    // bcsize
    0x28, 0x00, 0x00, 0x00, // biSize
    0x10, 0x00, 0x00, 0x00, // biWidth
    0x20, 0x00, 0x00, 0x00, // biHeight (with both AND and XOR mask? wtf?)

    0x01, 0x00, // biPlanes
    0x20, 0x00, // biBitCount (32)

    0x00, 0x00, 0x00, 0x00, // biCompression
    0x00, 0x00, 0x00, 0x00, // biSizeImage
    0x00, 0x00, 0x00, 0x00, // biXPelsPerMeter
    0x00, 0x00, 0x00, 0x00, // biYPelsPerMeter
    0x00, 0x00, 0x00, 0x00, // biClrUsed
    0x00, 0x00, 0x00, 0x00, // biClrImportant
    // END BITMAPINFOHEADER

    // BITMAP DATA (4 bytes per pixel)
    0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF,
    0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF,
    0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF,
    0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF,

    0x00, 0xFF, 0x00, 0xFF, 0x00, 0x00, 0xFF, 0xFF, 0x00, 0x00, 0xFF, 0xFF, 0x00, 0x00, 0xFF, 0xFF,
    0x00, 0x00, 0xFF, 0xFF, 0x00, 0x00, 0xFF, 0xFF, 0x00, 0x00, 0xFF, 0xFF, 0x00, 0x00, 0xFF, 0xFF,
    0x00, 0x00, 0xFF, 0xFF, 0x00, 0x00, 0xFF, 0xFF, 0x00, 0x00, 0xFF, 0xFF, 0x00, 0x00, 0xFF, 0xFF,
    0x00, 0x00, 0xFF, 0xFF, 0x00, 0x00, 0xFF, 0xFF, 0x00, 0x00, 0xFF, 0xFF, 0x00, 0xFF, 0x00, 0xFF,

    0x00, 0xFF, 0x00, 0xFF, 0x00, 0x00, 0xFF, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF,
    0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF,
    0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF,
    0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0x00, 0xFF, 0xFF, 0x00, 0xFF, 0x00, 0xFF,

    0x00, 0xFF, 0x00, 0xFF, 0x00, 0x00, 0xFF, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF,
    0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF,
    0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF,
    0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0x00, 0xFF, 0xFF, 0x00, 0xFF, 0x00, 0xFF,

    0x00, 0xFF, 0x00, 0xFF, 0x00, 0x00, 0xFF, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF,
    0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF,
    0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF,
    0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0x00, 0xFF, 0xFF, 0x00, 0xFF, 0x00, 0xFF,

    0x00, 0xFF, 0x00, 0xFF, 0x00, 0x00, 0xFF, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF,
    0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF,
    0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF,
    0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0x00, 0xFF, 0xFF, 0x00, 0xFF, 0x00, 0xFF,

    0x00, 0xFF, 0x00, 0xFF, 0x00, 0x00, 0xFF, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF,
    0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF,
    0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF,
    0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0x00, 0xFF, 0xFF, 0x00, 0xFF, 0x00, 0xFF,

    0x00, 0xFF, 0x00, 0xFF, 0x00, 0x00, 0xFF, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF,
    0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF,
    0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF,
    0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0x00, 0xFF, 0xFF, 0x00, 0xFF, 0x00, 0xFF,

    0x00, 0xFF, 0x00, 0xFF, 0x00, 0x00, 0xFF, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF,
    0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF,
    0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF,
    0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0x00, 0xFF, 0xFF, 0x00, 0xFF, 0x00, 0xFF,

    0x00, 0xFF, 0x00, 0xFF, 0x00, 0x00, 0xFF, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF,
    0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF,
    0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF,
    0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0x00, 0xFF, 0xFF, 0x00, 0xFF, 0x00, 0xFF,

    0x00, 0xFF, 0x00, 0xFF, 0x00, 0x00, 0xFF, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF,
    0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF,
    0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF,
    0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0x00, 0xFF, 0xFF, 0x00, 0xFF, 0x00, 0xFF,

    0x00, 0xFF, 0x00, 0xFF, 0x00, 0x00, 0xFF, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF,
    0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF,
    0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF,
    0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0x00, 0xFF, 0xFF, 0x00, 0xFF, 0x00, 0xFF,

    0x00, 0xFF, 0x00, 0xFF, 0x00, 0x00, 0xFF, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF,
    0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF,
    0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF,
    0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0x00, 0xFF, 0xFF, 0x00, 0xFF, 0x00, 0xFF,

    0x00, 0xFF, 0x00, 0xFF, 0x00, 0x00, 0xFF, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF,
    0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF,
    0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF,
    0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0x00, 0xFF, 0xFF, 0x00, 0xFF, 0x00, 0xFF,

    0x00, 0xFF, 0x00, 0xFF, 0x00, 0x00, 0xFF, 0xFF, 0x00, 0x00, 0xFF, 0xFF, 0x00, 0x00, 0xFF, 0xFF,
    0x00, 0x00, 0xFF, 0xFF, 0x00, 0x00, 0xFF, 0xFF, 0x00, 0x00, 0xFF, 0xFF, 0x00, 0x00, 0xFF, 0xFF,
    0x00, 0x00, 0xFF, 0xFF, 0x00, 0x00, 0xFF, 0xFF, 0x00, 0x00, 0xFF, 0xFF, 0x00, 0x00, 0xFF, 0xFF,
    0x00, 0x00, 0xFF, 0xFF, 0x00, 0x00, 0xFF, 0xFF, 0x00, 0x00, 0xFF, 0xFF, 0x00, 0xFF, 0x00, 0xFF,

    0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF,
    0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF,
    0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF,
    0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00, 0xFF
};

unsigned char *get_icon(int *pOut) {
    *pOut = sizeof(icon_data);
    return icon_data;
}