Popen Mixed Data Stream?

I've got problem with Popen and Pipes on Windows. I think Popen mixed data between stdout and stderr. When my program reads only stdout everything is OK but when I read stdout and

Solution 1:

from subproccess import *
x = Popen('sh /root/script.sh', stdout=PIPE, stderr=PIPE, stdin=PIPE, shell=True)
print x.stdout.readline()
print x.stderr.readline()

Since you don't posted any relevant code, here's how it should look. Note that readline will still give you output even tho there is none to fetch, as to stderr sometimes "hangs" waiting for output, but this is the way to go if you're using subprocess.

from subprocess import PIPE, Popen
x = Popen(['ls', '-l'], stdout=PIPE, stderr=PIPE, stdin=PIPE, shell=False)
while 1:
    if x.poll() != None:
        break
    _queue.put(x.stdout.readline())
    sleep(0.2)

Something like that should work. If you don't need to separate the two outputs, and you just want to read the data and att it to a queue, you can do:

x = Popen(['ls', '-l'], stdout=PIPE, stderr=STDOUT, stdin=PIPE, shell=False)

Here's what i normally do

(and i'm not saying this is the best practice of things, but it works)

from threading import Thread, enumerate
from subprocess import Popen, PIPE
from time import sleep

class nonBlockingStderr(Thread):
    def __init__(self, handle):
        self.handle = handle
        self.stderrOutput = []
        Thread.__init__(self)
        self.start()
    def stderr(self):
        if len(self.stderrOutput) <= 0:
            return ''
        else:
            ret = self.stderrOutput[0]
            self.stderrOutput = self.stderrOutput[1:]
            return ret
    def run(self):
        while 1:
            line = self.handle.readline()
            if len(line) > 0:
                self.stderrOutput.append(line)
            sleep(0.1)

from subprocess import PIPE, Popen
x = Popen(['ls', '-l'], stdout=PIPE, stderr=PIPE, stdin=PIPE, shell=False)
errHandle = nonBlockingStderr(x.stderr)
while 1:
    if x.poll() != None:
        break
    _queue.put(errHandle.stderr())
    _queue.put(x.stdout.readline())
    sleep(0.2)

Solution 2:

A simple way to separate output for stdout/stderr is to use .communicate():

from subprocess import Popen, PIPE

p = Popen(command, stdout=PIPE, stderr=PIPE)
out, err = p.communicate()

It reads all output in memory.

If you want to process the output while the subprocess is running; you need threads on Windows:

from Queue import Empty, Queue
from subprocess import Popen, PIPE
from threading import Thread

def reader_thread(pipe, process_line):
    for line in iter(pipe.readline, b''):
        process_line(line)
    pipe.close()

def start_reader_thread(pipe, process_line):
    t = Thread(target=reader_thread, args=[pipe, process_line])
    t.daemon = True
    t.start()

q = Queue()
p = Popen(command, stdout=PIPE, stderr=PIPE, bufsize=1)
start_reader_thread(p.stdout, lambda line: q.put(('stdout', line)))
start_reader_thread(p.stderr, lambda line: q.put(('stderr', line)))

while p.poll() is None: # while subprocess is running
    try:
        source, line = q.get_nowait()
    except Empty:
        pass
    else:
        # use source, line here
        print(source, line)
q.put(None)
for source, line in iter(q.get, None): # process the rest
    # use source, line here
    print("after", source, line)