How to capture output of a shell script running in

Question

A long running shell script produces stdout and stderr, which I would like to show on a textctrl in a GUI. This is possible using threading and separating the GUI thread from the shell script's thread. However when I implement multiprocessing, I hit a roadblock. Here's my -stripped down- code:

#!/usr/bin/env python

import wx
import sys, subprocess
from multiprocessing import Process, Queue
from Queue import Empty

class MyFrame(wx.Frame):
    def __init__(self, *args, **kwds):
        wx.Frame.__init__(self, *args, **kwds)

        self.button = wx.Button(self, -1 , "Run")
        self.output = wx.TextCtrl(self, -1, '', style=wx.TE_MULTILINE|\
                                                      wx.TE_READONLY|wx.HSCROLL)

        self.Bind(wx.EVT_BUTTON, self.OnButton, self.button)

        sizer = wx.BoxSizer(wx.VERTICAL)
        sizer.Add(self.output, -1, wx.EXPAND, 0)
        sizer.Add(self.button)
        self.SetSizerAndFit(sizer)
        self.Centre()

    def OnButton(self, event):
        numtasks = 4 # total number of tasks to run
        numprocs = 2 # number of processors = number of parallel tasks
        work_queue = Queue()
        for i in xrange(numtasks):
             work_queue.put(i)
        processes = [Process(target=self.doWork, args=(work_queue, ))
                     for i in range(numprocs)]
        for p in processes:
            p.daemon = True
            p.start()

    def doWork(self, work_queue):
        while True:
            try:
                x = work_queue.get(block=False)
                self.runScript(x)
            except Empty:
                print "Queue Empty"
                break

    def runScript(self, taskno):
        print '## Now Running: ', taskno
        command = ['./script.sh']  
        proc = subprocess.Popen(command, shell=True, 
                                         stdout=subprocess.PIPE, 
                                         stderr=subprocess.STDOUT)
        while True:
            stdout = proc.stdout.readline()
            if not stdout:
                break
            #sys.stdout.flush() #no need for flush, apparently it is embedded in the multiprocessing module
            self.output.AppendText(stdout.rstrip()) #this is the part that doesn't work.

if __name__ == "__main__":
    app = wx.App(0)
    frame = MyFrame(None, title="shell2textctrl", size=(500,500))
    frame.Show(True)
    app.MainLoop()

I've tried many solutions upon others' suggestions. Among these are: using wx.CallAfter or wx.lib.delayedresult to make the GUI responsive; several threading recipes (eg. wxApplication Development Cookbook, threadpool, others...) and have the multiprocess() run from a separate thread; redefining sys.stdout.write to write to the textctrl; and whatnot. None of them were successfull. Can anyone please provide a solution together with working code? You can use this script written somewhere here by I forgot who:

#!/bin/tcsh -f
@ i = 1
while ($i <= 20)
 echo $i
 @ i += 1
 sleep 0.2
end

Answer 1

I found a solution which implements a modified output Queue on top of Roger Stuckey's Simpler wxPython Multiprocessing Example framework. The code below is even Simpler than his; it could also be cleaned out a little. Hope it helps someone else. I still have a nagging feeling that there should be a more straightforward way of doing this though.

import getopt, math, random, sys, time, types, wx, subprocess

from multiprocessing import Process, Queue, cpu_count, current_process, freeze_support
from Queue import Empty

class MyFrame(wx.Frame):
    def __init__(self, parent, id, title):
        wx.Frame.__init__(self, parent, id, title, wx.Point(700, 500), wx.Size(300, 200))

        self.panel = wx.Panel(self, wx.ID_ANY)

        #widgets
        self.start_bt = wx.Button(self.panel, wx.ID_ANY, "Start")
        self.Bind(wx.EVT_BUTTON, self.OnButton, self.start_bt)

        self.output_tc = wx.TextCtrl(self.panel, wx.ID_ANY, style=wx.TE_MULTILINE|wx.TE_READONLY)

        # sizer
        self.sizer = wx.GridBagSizer(5, 5)
        self.sizer.Add(self.start_bt, (0, 0), flag=wx.ALIGN_CENTER|wx.LEFT|wx.TOP|wx.RIGHT, border=5)
        self.sizer.Add(self.output_tc, (1, 0), flag=wx.EXPAND|wx.LEFT|wx.RIGHT|wx.BOTTOM, border=5)
        self.sizer.AddGrowableCol(0)
        self.sizer.AddGrowableRow(1)
        self.panel.SetSizer(self.sizer)

        # Set some program flags
        self.keepgoing = True
        self.i = 0
        self.j = 0


    def OnButton(self, event):
        self.start_bt.Enable(False)

        self.numtasks = 4
        self.numproc = 2
        #self.numproc = cpu_count()
        self.output_tc.AppendText('Number of processes = %d\n' % self.numproc)

        # Create the queues
        self.taskQueue = Queue()
        self.outputQueue = Queue()

        # Create the task list
        self.Tasks = range(self.numtasks)

        # The worker processes...
        for n in range(self.numproc):
            process = Process(target=self.worker, args=(self.taskQueue, self.outputQueue))
            process.start()

        # Start processing tasks
        self.processTasks(self.update)

        if (self.keepgoing):
            self.start_bt.Enable(True)

    def processTasks(self, resfunc=None):
        self.keepgoing = True

        # Submit first set of tasks
        numprocstart = min(self.numproc, self.numtasks)
        for self.i in range(numprocstart):
            self.taskQueue.put(self.Tasks[self.i])

        self.j = -1 # done queue index
        self.i = numprocstart - 1 # task queue index
        while (self.j < self.i):
            # Get and print results
            self.j += 1
            output = None
            while output != 'STOP!':
                try:
                    output = self.outputQueue.get() 
                    if output != 'STOP!': 
                        resfunc(output)
                except Empty:
                    break

            if ((self.keepgoing) and (self.i + 1 < self.numtasks)):
                # Submit another task
                self.i += 1
                self.taskQueue.put(self.Tasks[self.i])

    def update(self, output):
        self.output_tc.AppendText('%s PID=%d Task=%d : %s\n' % output)
        wx.YieldIfNeeded()

    def worker(self, inputq, outputq):
        while True:
            try:
                tasknum = inputq.get()
                print '## Now Running: ', tasknum #this goes to terminal/console. Add it to outputq if you'd like it on the TextCtrl.
                command = ['./script.sh']
                p = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
                while True:
                    r = p.stdout.readline()
                    if not r:
                        outputq.put('STOP!')
                        break
                    outputq.put(( current_process().name, current_process().pid, tasknum, r.rstrip()))
            except Empty:
                break

    # The worker must not require any existing object for execution!
    worker = classmethod(worker)

class MyApp(wx.App):
    def OnInit(self):
        self.frame = MyFrame(None, -1, 'stdout to GUI using multiprocessing')
        self.frame.Show(True)
        self.frame.Center()
        return True

if __name__ == '__main__':
    freeze_support()
    app = MyApp(0)
    app.MainLoop()

Answer 2

I don't know if this will help you or not, but I have an example of something like this using subprocess here:

http://www.blog.pythonlibrary.org/2010/06/05/python-running-ping-traceroute-and-more/

See the pingIP and tracertIP methods. If your shell script is written in Python, you can probably add some kind of generator to it like the ones I am using in that article to post back to the GUI. You probably already read the LongRunningProcess wiki article, but I took that and made my own tutorial here:

http://www.blog.pythonlibrary.org/2010/05/22/wxpython-and-threads/