Why doesn't XMLHttpRequest payload reach Node.

Question

For my hobby project, I've been trying to convert a webpage to HTTPS completely.

The webpage consists of:

• An apache server hosting the HTML / JS / CSS
• A Node.JS http server handling all functionality
• A Socket.IO plugin

All are now on port 443, with a proxy configured in Apache, sending the web page stuff to localhost:8091, where the html file is accessible on the internal network, sending the Node.JS and Socket.IO stuff to localhost:8000.

Everything seems to be working fine, except for one single thing: The payload from XMLHttpRequests doesn't seem to be reaching the Node.JS server anymore.

XMLHttpRequest:

function doHttp(content, callback, url, contentType) {

    var action;
    var doUpdate;
    if (content != null) {
        action = JSON.parse(content).action;
    }

    var xhttp = new XMLHttpRequest();
    if (url === null || url === undefined) {
        // Uses a proxy in Apache to forward url/sprinklers/node to localhost:8000 on the server
        var url = window.location.href + "node"
        doUpdate = (action != "load" && action != "checkRow");
    }

    console.log("doHttp url", url)

    xhttp.open("POST", url, true);
    if (contentType === null || contentType === undefined) {
        contentType = "application/json";
    }

    console.log("doHttp contentType", contentType)
    xhttp.setRequestHeader("Content-Type", contentType);

    xhttp.onload = function() {
        if (callback != null) {
            callback(xhttp.responseText);
        }
        if (doUpdate) {
            doDomoticz("update");
        }
    };
    xhttp.onerror = function() {
        throw xhttp.response;
    };

    xhttp.send(content);
}

Upon loading the page, this function is called with:

• content = {action: "load"}
• callback = A fitting cb function
• url = https://myUrl.com/sprinklers/node

The log statements are as follows:

doHttp url https://myUrl.com/sprinklers/node
doHttp contentType application/json

So far, so good. When I check the network tab in the Chrome inspector, I can see {action: "load"} in the 'Request Payload' section, so that seems to be working fine.

On to the proxy settings:

<VirtualHost *:8091>
    DocumentRoot /var/www/SprinklerServer

    ErrorLog ${APACHE_LOG_DIR}/port_8091_error.log
    CustomLog ${APACHE_LOG_DIR}/port_8091_access.log combined

    DirectoryIndex sprinklers.html

#    <Directory "/var/www/SprinklerServer">
#        AuthType Basic
#        AuthName "Restricted Content"
#        AuthUserFile /var/www/SprinklerServer/.htpasswd
#        Require valid-user
#    </Directory>
</VirtualHost>
# Password disabled for testing period

<VirtualHost *:443>
    ServerName myUrl.com

    ErrorLog ${APACHE_LOG_DIR}/port_443_error.log
    CustomLog ${APACHE_LOG_DIR}/port_443_access.log combined

    SSLEngine on
    SSLCertificateFile      /etc/letsencrypt/live/myUrl.com/cert.pem
    SSLCertificateKeyFile   /etc/letsencrypt/live/myUrl.com/privkey.pem

    SSLProxyEngine on
    ProxyPreserveHost On
    ProxyRequests Off

    RewriteEngine on
    # Domoticz rule can be ignored for this question
    RewriteRule ^/domoticz$ /domoticz/ [R]
    RewriteRule ^/sprinklers/node$ /sprinklers/node/ [R]
    RewriteRule ^/sprinklers$ /sprinklers/ [R]

    # When Upgrade:websocket header is present, redirect to ws
    # Using NC flag (case-insensitive) as some browsers will pass Websocket
    RewriteCond %{HTTP:Upgrade} websocket [NC]
    RewriteCond %{HTTP:Connection} upgrade [NC]
    RewriteRule .* ws://127.0.0.1:8000/socket.io%{REQUEST_URI}  [P]

    # Domoticz Proxys can be ignored for this question
    ProxyPassMatch "^/domoticz/(.*)$" "https://127.0.0.1:444/$1"
    ProxyPassReverse "^/domoticz/(.*)$" "https://127.0.0.1:444/$1"
    ProxyPassMatch "^/sprinklers/node/(.*)$" "http://127.0.0.1:8000/$1"
    ProxyPassReverse "^/sprinklers/node/(.*)$" "http://127.0.0.1:8000/$1"
    ProxyPassMatch "^/sprinklers/(.*)$" "http://127.0.0.1:8091/$1"
    ProxyPassReverse "^/sprinklers/(.*)$" "http://127.0.0.1:8091/$1"

</VirtualHost>

(Not sure what syntax highlighting to use here. Suggestions?)

As you can see, the actual webpage is hosted at myUrl.com/sprinklers, and I route all Node.JS and Socket.IO stuff via myUrl.com/sprinklers/node.

On to the Node.JS server script:"

var server = http.createServer(function(request, response) {

    console.log("REQUEST")
    console.log(request.url);

    response.setHeader("Access-Control-Allow-Origin", "*");
    response.setHeader("Access-Control-Allow-Methods", "POST, GET");
    response.setHeader("Access-Control-Allow-Headers", "Content-Type");

    if (!fs.existsSync(filePath)) {
        try {
            fs.writeFileSync(filePath, JSON.stringify({data:{}}, null, 4));
        } catch (e) {
            console.error("Can't write file", e);
        }
    }
    fileString = fs.readFileSync(filePath, "utf8");

    try {
      fileJSON = JSON.parse(fileString);
    } catch(e) {
        console.error(e)
    }

    httpString = "";
    httpJSON;

    request.on("error", function(error) {

        console.error(err);


    }).on("data", function(chunk) {

        console.log("CHUNK", chunk)

        httpString += chunk;


    }).on("end", function() {

        if (httpString.length > 0) {

            console.log("HTTPSTRING", httpString)

            httpJSON = JSON.parse(httpString);

            // Call function based on action name
            if (funcs[String(httpJSON.action)] != null) {
                funcs[String(httpJSON.action)](response)

            } else {
                response.statusCode = 500;
                response.write("Received data is either corrupted, or provided action \"" + String(httpJSON.action) + "\" doesn't have a matching function.", "utf8");

            }
        }

        response.end();

    })

}).listen(8000) // Using a proxy that directs all url/node things to localhost:8000
const io = require("socket.io").listen(server)

The log statements in there only produce the following on the console where I'm running the script:

REQUEST
/

CHUNK or HTTPSTRING are never reached, so it seems like it's never getting any data. I'm assuming this is because the request payload is filtered out somewhere. My guess would be the proxy, but I haven't got a clue how to test this. I'm pretty new to proxies and HTTPS. It has taken me quite some time to even get to the point I'm currently at, which was only possible with lots of trial and error.

I'm usually pretty OK at figuring out what's causing the issue, and struggling as long as required until it's fixed, but this time I'm really stuck. Does anyone know what could be causing this issue?

Recap

Web page used to run on HTTP, worked fine. Switched over to HTTPS, put an Apache proxy in the middle. Everything works, except for XMLHttpRequest data not reaching the Node.JS server

Answer 1

I had an entire answer here, including a fix I thought I found, but sadly, it doesn't work. So instead, I'll just answer my original question: Why does this happen.

It's simple really: I'm using RewriteRule to rewrite the urls in case they're missing the trailing slash. Sadly, this doesn't support POST data, which I require. Since this was a broad question, focused on the why, I'm gonna mark this answer as conclusive, and open a new question that's specific on how to modify the url to include the trailing slash without removing the POST data. I'll post a link here when it's up.

EDIT:

I've already solved my new question. Turns out all you need to do is make sure that RewriteRule uses HTTP code 307. More detail in the other question: How to rewrite a URL while keeping POST data?