バッファなしのPython Subrocess PIPE

Question

私は次のようにサブプロセスのラッパーを実装しようとしている：

def ans_cmd_stream_color(inputcmd): 
"""Driver function for local ansible commands. 

Stream stdout to stdout and log file with color. 
Runs <inputcmd> via subprocess. 
Returns return code, stdout, stderr as dict. 
""" 
fullcmd = inputcmd 
create_debug('Enabling colorful ansible output.', LOGGER) 
create_info('Running command: ' + fullcmd, LOGGER, True) 
p = subprocess.Popen('export ANSIBLE_FORCE_COLOR=true; ' + fullcmd, 
        stdout=subprocess.PIPE, 
        stderr=subprocess.PIPE, 
        shell=True) 
stdout_l = [] 
stderr_l = [] 
rcode = 0 
# Regex black magic 
ansi_escape = re.compile(r'\x1b[^m]*m') 
# Get the unbuffered IO action going. 
try: 
    # Non blocking 
    reads = [p.stdout.fileno(), p.stderr.fileno()] 
    ret = select.select(reads, [], []) 
    # Print line by line 
    while True: 
     for fd in ret[0]: 
      if fd == p.stdout.fileno(): 
       line = p.stdout.readline() 
       sys.stdout.write(line.encode('utf-8')) 
       stdout_l.append(ansi_escape.sub('', 
               line.encode('utf-8')) 
           ) 
      if fd == p.stderr.fileno(): 
       line = p.stdout.readline() 
       sys.stderr.write(line.encode('utf-8')) 
       stderr_l.append(ansi_escape.sub('', 
               line.encode('utf-8')) 
           ) 
     # Break when the process is done. 
     if p.poll() is not None: 
      rcode = p.returncode 
      break 
except BaseException as e: 
    raise e 
outstr = ''.join(stdout_l) 
errstr = ''.join(stderr_l) 
outstr, errstr = str(outstr).rstrip('\n'), str(errstr).rstrip('\n') 
expstr = errstr.strip('ERROR: ') 
if len(expstr) >= 1: 
    create_info('Command: ' + str(fullcmd) + ': ' + expstr + '\n', LOGGER, 
       True) 
    if rcode == 0: 
     rcode = 1 
else: 
    create_info(outstr + '\n', LOGGER) 
    if rcode == 0: 
     create_info('Command: ' + fullcmd + ' ran successfully.', LOGGER, 
        True) 
    expstr = False 
ret_dict = {inputcmd: {}} 
ret_dict[inputcmd]['rcode'] = rcode 
ret_dict[inputcmd]['stdout'] = outstr 
ret_dict[inputcmd]['stderr'] = expstr 
return copy.deepcopy(ret_dict) 

アイデアは、サブプロセスコマンドのストリーミング出力を印刷して、関数、ユーザーに情報を返すことです。問題は、私は設定しない限りさえ直接io.openを使用して、サブプロセスのPIPがまだバッファリングされていることである。

理想的ではありません

os.environ["PYTHONUNBUFFERED"] = "1" 

。任意のアイデアや誰かがこの問題に遭遇しましたか？

UPDATEは：ansibleを使用すると、バッファリング設定を称えるためにサブプロセスのためのバッファリングを無効にする必要があります：あなたはおそらく、直接サブプロセスのパイプから読むべき

def ans_cmd_stream_color(inputcmd): 
"""Driver function for local ansible commands. 

Stream stdout to stdout and log file with color. 
Runs <inputcmd> via subprocess. 
Returns return code, stdout, stderr as dict. 
""" 
fullcmd = inputcmd 
create_debug('Enabling colorful ansible output.', LOGGER) 
create_info('Running command: ' + fullcmd, LOGGER, True) 
p = subprocess.Popen('export ANSIBLE_FORCE_COLOR=true; ' + 
        'export PYTHONUNBUFFERED=1; ' + fullcmd, 
        stdout=subprocess.PIPE, 
        stderr=subprocess.PIPE, 
        shell=True) 
stdout_l = [] 
stderr_l = [] 
rcode = 0 
# Regex black magic 
ansi_escape = re.compile(r'\x1b[^m]*m') 
# Get the unbuffered IO action going. 
try: 
    # Non blocking 
    reads = [p.stdout.fileno(), p.stderr.fileno()] 
    ret = select.select(reads, [], []) 
    # Print line by line 
    while True: 
     for fd in ret[0]: 
      if fd == p.stdout.fileno(): 
       line = p.stdout.readline() 
       sys.stdout.write(line.encode('utf-8')) 
       stdout_l.append(ansi_escape.sub('', 
               line.encode('utf-8')) 
           ) 
      if fd == p.stderr.fileno(): 
       line = p.stdout.readline() 
       sys.stderr.write(line.encode('utf-8')) 
       stderr_l.append(ansi_escape.sub('', 
               line.encode('utf-8')) 
           ) 
     # Break when the process is done. 
     if p.poll() is not None: 
      rcode = p.returncode 
      break 
except BaseException as e: 
    raise e 
outstr = ''.join(stdout_l) 
errstr = ''.join(stderr_l) 
outstr, errstr = str(outstr).rstrip('\n'), str(errstr).rstrip('\n') 
expstr = errstr.strip('ERROR: ') 
if len(expstr) >= 1: 
    create_info('Command: ' + str(fullcmd) + ': ' + expstr + '\n', LOGGER, 
       True) 
    if rcode == 0: 
     rcode = 1 
else: 
    create_info(outstr + '\n', LOGGER) 
    if rcode == 0: 
     create_info('Command: ' + fullcmd + ' ran successfully.', LOGGER, 
        True) 
    expstr = False 
ret_dict = {inputcmd: {}} 
ret_dict[inputcmd]['rcode'] = rcode 
ret_dict[inputcmd]['stdout'] = outstr 
ret_dict[inputcmd]['stderr'] = expstr 
return copy.deepcopy(ret_dict) 

Answer 1

。以下のようなものは標準から情報ロガーに、標準エラーはエラーロガーに読み込まれます。

---

import logging, subprocess 
logging.basicConfig(level=logging.INFO) 
proc = subprocess.Popen(
     cmd, stdout=subprocess.PIPE, 
     stderr=subprocess.PIPE 
    ) 
cont = True 
while cont: 
    cont = False 
    line = proc.stdout.readline() 
    if not line == b"": 
     out = line.decode("utf-8").rstrip() 
     logging.info(out) 
     cont = True 

    line = proc.stderr.readline() 
    if not line == b"": 
     out = line.decode("utf-8").rstrip() 
     logging.error(out) 
     cont = True 

    if not cont and proc.poll() is not None: 
     break 

は this questionごとに、バッファリングの問題に対処するには、いずれかの下位のPythonスクリプトは、明示的にバッファをフラッシュしなければならない、または環境変数 PYTHONUNBUFFEREDは、非空の文字列に設定する必要があります。