Stream child process output in flowing mode

Question

I have custom command line written using Python which prints its output using "print" statement. I am using it from Node.js by spawning a child process and sending commands to it using child.stdin.write method. Here's source:

var childProcess = require('child_process'),
    spawn = childProcess.spawn;

var child = spawn('./custom_cli', ['argument_1', 'argument_2']);

child.stdout.on('data', function (d) {
  console.log('out: ' + d);
});

child.stderr.on('data', function (d) {
  console.log('err: ' + d);
});

//execute first command after 1sec
setTimeout(function () {
  child.stdin.write('some_command' + '\n');
}, 1000);

//execute "quit" command after 2sec
//to terminate the command line
setTimeout(function () {
  child.stdin.write('quit' + '\n');
}, 2000);

Now the issue is I am not receiving the output in flowing mode. I want get the output from child process as soon as it's printed but I am receiving the output of all the commands only when child process is terminated (using custom cli's quit command).

Answer 1

In my case in Python I'm using sys.stdin.readline and yielding last line:

def read_stdin():
    '''
        read standard input
        yeld next line
    '''
    try:
        readline = sys.stdin.readline()
        while readline:
            yield readline
            readline = sys.stdin.readline()
    except:
        # LP: avoid to exit(1) at stdin end
        pass

 for line in read_stdin():
     out = process(line)
     ofp.write(out)
     sys.stdout.flush()

and when in Node.js

var child = spawn(binpath, args);

    // register child process signals
    child.stdout.on('data', function (_data) {
        var data = Buffer.from(_data, 'utf-8').toString().trim();
        console.log(data);
    });
    child.stderr.on('data', function (data) {
        console.warn('pid:%s stderr:%s', child.pid, data);
    });
    child.stdout.on('exit', function (_) {
        console.warn('pid:%s exit', child.pid);
    });
    child.stdout.on('end', function (_) {
        console.warn('pid:%s ended', child.pid);
    });
    child.on('error', function (error) {
        console.error(error);
    });
    child.on('close', (code, signal) => { // called after `end`
        console.warn('pid:%s terminated with code:%d due to receipt of signal:%s with ', child.pid, code, signal);
    });
    child.on('uncaughtException', function (error) {
        console.warn('pid:%s terminated due to receipt of error:%s', child.pid, error);
    });

Answer 2

You need to flush the output in the child process.

Probably you think this isn't necessary because when testing and letting the output happen on a terminal, then the library flushes itself (e. g. when a line is complete). This is not done when printing goes to a pipe (due to performance reasons).

Flush yourself:

#!/usr/bin/env python

import sys, time

while True:
  print "foo"
  sys.stdout.flush()
  time.sleep(2)

Answer 3

The best way is to use unbuffered mode of python standard output. It will force python to write output to output streams without need to flush yourself.

For example:

var spawn = require('child_process').spawn,
child = spawn('python',['-u', 'myscript.py']); // Or in custom_cli add python -u myscript.py

child.stdout.on('data', function (data) {
    console.log('stdout: ' + data);
});

child.stderr.on('data', function (data) {
    console.log('stderr: ' + data);
});