Using module 'subprocess' with timeout
下面是python代码,用于运行返回其
1 2 3 4 5 | proc = subprocess.Popen( cmd, stderr=subprocess.STDOUT, # Merge stdout and stderr stdout=subprocess.PIPE, shell=True) |
1 | stdoutdata, stderrdata = proc.communicate() |
在要在Windows和Linux上运行的python程序中实现超时的最简单方法是什么?
我对低层次的细节不太了解,但考虑到这一点,python 2.6 API提供了等待线程和终止进程,在单独的线程?
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 | import subprocess, threading class Command(object): def __init__(self, cmd): self.cmd = cmd self.process = None def run(self, timeout): def target(): print 'Thread started' self.process = subprocess.Popen(self.cmd, shell=True) self.process.communicate() print 'Thread finished' thread = threading.Thread(target=target) thread.start() thread.join(timeout) if thread.is_alive(): print 'Terminating process' self.process.terminate() thread.join() print self.process.returncode command = Command("echo 'Process started'; sleep 2; echo 'Process finished'") command.run(timeout=3) command.run(timeout=1) |
我的计算机中此代码段的输出是:
1 2 3 4 5 6 7 8 9 10 | Thread started Process started Process finished Thread finished 0 Thread started Process started Terminating process Thread finished -15 |
可以看出,在第一次执行过程中,正确完成(返回代码0),而在第二个代码中进程已终止(返回代码-15)。
我没有在Windows中进行测试;但是,除了更新示例之外司令部,我想应该能用,因为我在记录任何表示thread.join或process.terminate的内容不支持。
Python中的3.3升:
1 2 3 | from subprocess import STDOUT, check_output output = check_output(cmd, stderr=STDOUT, timeout=seconds) |
这是一个
本规范提出的
我的情况是经常使用的,因为它
在超时功能是可用的在线维护,通过
可以使用threading.timer类简化jcollado的答案:
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 | import shlex from subprocess import Popen, PIPE from threading import Timer def run(cmd, timeout_sec): proc = Popen(shlex.split(cmd), stdout=PIPE, stderr=PIPE) timer = Timer(timeout_sec, proc.kill) try: timer.start() stdout, stderr = proc.communicate() finally: timer.cancel() # Examples: both take 1 second run("sleep 1", 5) # process ends normally at 1 second run("sleep 5", 1) # timeout happens at 1 second |
如果你在Unix上,
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 | import signal ... class Alarm(Exception): pass def alarm_handler(signum, frame): raise Alarm signal.signal(signal.SIGALRM, alarm_handler) signal.alarm(5*60) # 5 minutes try: stdoutdata, stderrdata = proc.communicate() signal.alarm(0) # reset the alarm except Alarm: print"Oops, taking too long!" # whatever else |
这是亚历克斯·马泰利的解决方案,作为一个适当的过程杀戮模块。其他方法不起作用,因为它们不使用proc.communication()。因此,如果您有一个产生大量输出的进程,它将填充其输出缓冲区,然后阻塞,直到您从中读取某些内容为止。
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 | from os import kill from signal import alarm, signal, SIGALRM, SIGKILL from subprocess import PIPE, Popen def run(args, cwd = None, shell = False, kill_tree = True, timeout = -1, env = None): ''' Run a command with a timeout after which it will be forcibly killed. ''' class Alarm(Exception): pass def alarm_handler(signum, frame): raise Alarm p = Popen(args, shell = shell, cwd = cwd, stdout = PIPE, stderr = PIPE, env = env) if timeout != -1: signal(SIGALRM, alarm_handler) alarm(timeout) try: stdout, stderr = p.communicate() if timeout != -1: alarm(0) except Alarm: pids = [p.pid] if kill_tree: pids.extend(get_process_children(p.pid)) for pid in pids: # process might have died before getting to this line # so wrap to avoid OSError: no such process try: kill(pid, SIGKILL) except OSError: pass return -9, '', '' return p.returncode, stdout, stderr def get_process_children(pid): p = Popen('ps --no-headers -o pid --ppid %d' % pid, shell = True, stdout = PIPE, stderr = PIPE) stdout, stderr = p.communicate() return [int(p) for p in stdout.split()] if __name__ == '__main__': print run('find /', shell = True, timeout = 3) print run('find', shell = True) |
我修改了Sussudio的答案。现在函数返回:(
1 2 3 4 5 6 7 8 9 10 11 12 | def kill_proc(proc, timeout): timeout["value"] = True proc.kill() def run(cmd, timeout_sec): proc = subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE, stderr=subprocess.PIPE) timeout = {"value": False} timer = Timer(timeout_sec, kill_proc, [proc, timeout]) timer.start() stdout, stderr = proc.communicate() timer.cancel() return proc.returncode, stdout.decode("utf-8"), stderr.decode("utf-8"), timeout["value"] |
没有人提到的surprised利用
这才是每一个用例显然是工作,但如果你处理简单的脚本,这是很难击败。
也可用路径中的gtimeout coreutils
另一个选项是写入临时文件以防止stdout阻塞,而不需要使用communication()进行轮询。这对我很有用,而其他答案却不适用;例如在Windows上。
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 | outFile = tempfile.SpooledTemporaryFile() errFile = tempfile.SpooledTemporaryFile() proc = subprocess.Popen(args, stderr=errFile, stdout=outFile, universal_newlines=False) wait_remaining_sec = timeout while proc.poll() is None and wait_remaining_sec > 0: time.sleep(1) wait_remaining_sec -= 1 if wait_remaining_sec <= 0: killProc(proc.pid) raise ProcessIncompleteError(proc, timeout) # read temp streams from start outFile.seek(0); errFile.seek(0); out = outFile.read() err = errFile.read() outFile.close() errFile.close() |
1 2 3 | import subprocess subprocess.call("command", timeout=20, shell=True) |
这将使该呼叫的指令和异常
1 | subprocess.TimeoutExpired |
如果该命令不是20秒后完成。
然后你可以继续你的例外处理的代码,这样的事情:
1 2 3 4 | try: subprocess.call("command", timeout=20, shell=True) except subprocess.TimeoutExpired: # insert code here |
希望这帮助。
这里是我的解决方案是使用线程和事件:I
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 | import subprocess from threading import Thread, Event def kill_on_timeout(done, timeout, proc): if not done.wait(timeout): proc.kill() def exec_command(command, timeout): done = Event() proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) watcher = Thread(target=kill_on_timeout, args=(done, timeout, proc)) watcher.daemon = True watcher.start() data, stderr = proc.communicate() done.set() return data, stderr, proc.returncode |
在行动:
1 2 3 4 5 | In [2]: exec_command(['sleep', '10'], 5) Out[2]: ('', '', -9) In [3]: exec_command(['sleep', '10'], 11) Out[3]: ('', '', 0) |
我不知道为什么,但它是不是由于Python mentionned 3.5,有一个新的命令(这是普遍的
subprocess.run(args, *, stdin=None, input=None, stdout=None, stderr=None, shell=False, cwd=None, timeout=None, check=False, encoding=None, errors=None)
1 | Run the command described by args. Wait for command to complete, then return a CompletedProcess instance. |
它的提出是对
我添加了从
安装:
1 | pip install easyprocess |
例子:
1 2 3 4 5 | from easyprocess import Proc # shell is not supported! stdout=Proc('ping localhost').call(timeout=1.5).stdout print stdout |
我使用的解决方案是在shell命令前面加上时间限制。如果命令占用的时间太长,TimeLimit将停止它,Popen将按TimeLimit设置返回代码。如果大于128,则表示时间限制终止了进程。
另请参见具有超时和大输出(>64K)的python子进程。
如果你是用Python 2,给它尝试
1 2 3 4 5 6 | import subprocess32 try: output = subprocess32.check_output(command, shell=True, timeout=3) except subprocess32.TimeoutExpired as e: print e |
一旦您了解了在*Unix中运行的全过程机器,您将很容易找到更简单的解决方案:
考虑一下这个简单的例子,如何使用select.select()使timeoutable communication()方法(现在在*nix上也可以使用)。这也可以用epoll/poll/kqueue编写,但是select.select()变量可能是一个很好的例子。select.select()(speed和1024 max fds)的主要限制不适用于您的任务。
这在*nix下工作,不创建线程,不使用信号,可以从任何线程(不仅是主线程)中发出,并且可以快速从我的机器上的stdout(i5 2.3ghz)读取250MB/s的数据。
在通信结束时连接stdout/stderr时出现问题。如果您有大量的程序输出,这可能会导致大量的内存使用。但您可以使用较小的超时多次调用communication()。
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 | class Popen(subprocess.Popen): def communicate(self, input=None, timeout=None): if timeout is None: return subprocess.Popen.communicate(self, input) if self.stdin: # Flush stdio buffer, this might block if user # has been writing to .stdin in an uncontrolled # fashion. self.stdin.flush() if not input: self.stdin.close() read_set, write_set = [], [] stdout = stderr = None if self.stdin and input: write_set.append(self.stdin) if self.stdout: read_set.append(self.stdout) stdout = [] if self.stderr: read_set.append(self.stderr) stderr = [] input_offset = 0 deadline = time.time() + timeout while read_set or write_set: try: rlist, wlist, xlist = select.select(read_set, write_set, [], max(0, deadline - time.time())) except select.error as ex: if ex.args[0] == errno.EINTR: continue raise if not (rlist or wlist): # Just break if timeout # Since we do not close stdout/stderr/stdin, we can call # communicate() several times reading data by smaller pieces. break if self.stdin in wlist: chunk = input[input_offset:input_offset + subprocess._PIPE_BUF] try: bytes_written = os.write(self.stdin.fileno(), chunk) except OSError as ex: if ex.errno == errno.EPIPE: self.stdin.close() write_set.remove(self.stdin) else: raise else: input_offset += bytes_written if input_offset >= len(input): self.stdin.close() write_set.remove(self.stdin) # Read stdout / stderr by 1024 bytes for fn, tgt in ( (self.stdout, stdout), (self.stderr, stderr), ): if fn in rlist: data = os.read(fn.fileno(), 1024) if data == '': fn.close() read_set.remove(fn) tgt.append(data) if stdout is not None: stdout = ''.join(stdout) if stderr is not None: stderr = ''.join(stderr) return (stdout, stderr) |
我已经实现了我能从其中收集到的一些东西。这在Windows中有效,由于这是一个社区wiki,我想我也会共享我的代码:
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 | class Command(threading.Thread): def __init__(self, cmd, outFile, errFile, timeout): threading.Thread.__init__(self) self.cmd = cmd self.process = None self.outFile = outFile self.errFile = errFile self.timed_out = False self.timeout = timeout def run(self): self.process = subprocess.Popen(self.cmd, stdout = self.outFile, \ stderr = self.errFile) while (self.process.poll() is None and self.timeout > 0): time.sleep(1) self.timeout -= 1 if not self.timeout > 0: self.process.terminate() self.timed_out = True else: self.timed_out = False |
然后从另一个类或文件:
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 | outFile = tempfile.SpooledTemporaryFile() errFile = tempfile.SpooledTemporaryFile() executor = command.Command(c, outFile, errFile, timeout) executor.daemon = True executor.start() executor.join() if executor.timed_out: out = 'timed out' else: outFile.seek(0) errFile.seek(0) out = outFile.read() err = errFile.read() outFile.close() errFile.close() |
你可以利用这一
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 | import subprocess from datetime import datetime from select import select def call_with_timeout(cmd, timeout): started = datetime.now() sp = subprocess.Popen(cmd, stdout=subprocess.PIPE) while True: p = select([sp.stdout], [], [], timeout) if p[0]: p[0][0].read() ret = sp.poll() if ret is not None: return ret if (datetime.now()-started).total_seconds() > timeout: sp.kill() return None |
《Linux命令
1 2 3 | cmd ="timeout 20"+ cmd subprocess.Popen(cmd.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE) (output, err) = p.communicate() |
虽然我没有广泛地研究过它,但我在ActiveState找到的这个装饰器似乎对这类事情非常有用。除了
我在Windows、Linux和Mac上成功地使用了KillableProcess。如果您使用的是cygwin-python,那么您将需要osaf版本的killableprocess,因为否则本机Windows进程不会被杀死。
有一个一类的类的概念延伸到它与一些popen decorators的简单方法。Let’s call it expirablepopen。
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 | from logging import error from subprocess import Popen from threading import Event from threading import Thread class ExpirablePopen(Popen): def __init__(self, *args, **kwargs): self.timeout = kwargs.pop('timeout', 0) self.timer = None self.done = Event() Popen.__init__(self, *args, **kwargs) def __tkill(self): timeout = self.timeout if not self.done.wait(timeout): error('Terminating process {} by timeout of {} secs.'.format(self.pid, timeout)) self.kill() def expirable(func): def wrapper(self, *args, **kwargs): # zero timeout means call of parent method if self.timeout == 0: return func(self, *args, **kwargs) # if timer is None, need to start it if self.timer is None: self.timer = thr = Thread(target=self.__tkill) thr.daemon = True thr.start() result = func(self, *args, **kwargs) self.done.set() return result return wrapper wait = expirable(Popen.wait) communicate = expirable(Popen.communicate) if __name__ == '__main__': from subprocess import PIPE print ExpirablePopen('ssh -T [email protected]', stdout=PIPE, timeout=1).communicate() |
我有问题,我在想如果它终止了多线程subprocess比给定的超时时间长度。我想在
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 | import os import sys import signal import subprocess from multiprocessing import Pool cores_for_parallelization = 4 timeout_time = 15 # seconds def main(): jobs = [...YOUR_JOB_LIST...] with Pool(cores_for_parallelization) as p: p.map(run_parallel_jobs, jobs) def run_parallel_jobs(args): # Define the arguments including the paths initial_terminal_command = 'C:\\Python34\\python.exe' # Python executable function_to_start = 'C:\\temp\\xyz.py' # The multithreading script final_list = [initial_terminal_command, function_to_start] final_list.extend(args) # Start the subprocess and determine the process PID subp = subprocess.Popen(final_list) # starts the process pid = subp.pid # Wait until the return code returns from the function by considering the timeout. # If not, terminate the process. try: returncode = subp.wait(timeout=timeout_time) # should be zero if accomplished except subprocess.TimeoutExpired: # Distinguish between Linux and Windows and terminate the process if # the timeout has been expired if sys.platform == 'linux2': os.kill(pid, signal.SIGTERM) elif sys.platform == 'win32': subp.terminate() if __name__ == '__main__': main() |
不幸的是,我受到雇主关于源代码公开的非常严格的政策的约束,所以我不能提供实际的代码。但依我看,最好的解决方案是创建一个覆盖
http:/ / / / subprocess2 pypi.python.org pypi Python扩展模块而提供的subprocess允许你到一定的等待期,否则终止。
所以,等待10秒到的进程的终止,否则杀死。
1 2 3 4 5 | pipe = subprocess.Popen('...') timeout = 10 results = pipe.waitOrTerminate(timeout) |
这是一个Windows和Unix兼容的。"结果是一个字典,它包含一个"返回",这是一returncode的应用程序(或没有,如果它已被杀),以及"actiontaken"。"这将是subprocess2 _ _过程完成,如果正常的过程完成的"面具",或subprocess2端_ _过程"和subprocess2 _过程取决于采取的行动_杀死(详情见文档)
该解决方案的过程中杀死了树的情况下壳= true,通过参数的过程(或不),有超时和错误的过程中得到的输出和输出的电话(它使用的过程psutil杀死_ _树)。该解决方案是基于在多操作系统的发布,包括jcollado。张贴评论和响应通过在jcollado jradice女士的回答。2012年srvr测试在Windows和Ubuntu的14。请注意,如果你需要改变parent.children Ubuntu(……)_儿童parent.get call to(……)。
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 | def kill_proc_tree(pid, including_parent=True): parent = psutil.Process(pid) children = parent.children(recursive=True) for child in children: child.kill() psutil.wait_procs(children, timeout=5) if including_parent: parent.kill() parent.wait(5) def run_with_timeout(cmd, current_dir, cmd_parms, timeout): def target(): process = subprocess.Popen(cmd, cwd=current_dir, shell=True, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE) # wait for the process to terminate if (cmd_parms ==""): out, err = process.communicate() else: out, err = process.communicate(cmd_parms) errcode = process.returncode thread = Thread(target=target) thread.start() thread.join(timeout) if thread.is_alive(): me = os.getpid() kill_proc_tree(me, including_parent=False) thread.join() |
在Python 2.6),使用gevent
1 2 3 4 5 6 7 8 9 10 11 12 13 14 | from gevent.subprocess import Popen, PIPE, STDOUT def call_sys(cmd, timeout): p= Popen(cmd, shell=True, stdout=PIPE) output, _ = p.communicate(timeout=timeout) assert p.returncode == 0, p. returncode return output call_sys('./t.sh', 2) # t.sh example sleep 5 echo done exit 1 |
Python 2.7
1 2 3 4 5 6 7 8 9 10 11 12 13 | import time import subprocess def run_command(cmd, timeout=0): start_time = time.time() df = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) while timeout and df.poll() == None: if time.time()-start_time >= timeout: df.kill() return -1,"" output = ' '.join(df.communicate()).strip() return df.returncode, output |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 | import subprocess, optparse, os, sys, re, datetime, threading, time, glob, shutil, xml.dom.minidom, traceback class OutputManager: def __init__(self, filename, mode, console, logonly): self.con = console self.logtoconsole = True self.logtofile = False if filename: try: self.f = open(filename, mode) self.logtofile = True if logonly == True: self.logtoconsole = False except IOError: print (sys.exc_value) print ("Switching to console only output... ") self.logtofile = False self.logtoconsole = True def write(self, data): if self.logtoconsole == True: self.con.write(data) if self.logtofile == True: self.f.write(data) sys.stdout.flush() def getTimeString(): return time.strftime("%Y-%m-%d", time.gmtime()) def runCommand(command): ''' Execute a command in new thread and return the stdout and stderr content of it. ''' try: Output = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True).communicate()[0] except Exception as e: print ("runCommand failed :%s" % (command)) print (str(e)) sys.stdout.flush() return None return Output def GetOs(): Os ="" if sys.platform.startswith('win32'): Os ="win" elif sys.platform.startswith('linux'): Os ="linux" elif sys.platform.startswith('darwin'): Os ="mac" return Os def check_output(*popenargs, **kwargs): try: if 'stdout' in kwargs: raise ValueError('stdout argument not allowed, it will be overridden.') # Get start time. startTime = datetime.datetime.now() timeoutValue=3600 cmd = popenargs[0] if sys.platform.startswith('win32'): process = subprocess.Popen( cmd, stdout=subprocess.PIPE, shell=True) elif sys.platform.startswith('linux'): process = subprocess.Popen( cmd , stdout=subprocess.PIPE, shell=True ) elif sys.platform.startswith('darwin'): process = subprocess.Popen( cmd , stdout=subprocess.PIPE, shell=True ) stdoutdata, stderrdata = process.communicate( timeout = timeoutValue ) retcode = process.poll() #################################### # Catch crash error and log it. #################################### OutputHandle = None try: if retcode >= 1: OutputHandle = OutputManager( 'CrashJob_' + getTimeString() + '.txt', 'a+', sys.stdout, False) OutputHandle.write( cmd ) print (stdoutdata) print (stderrdata) sys.stdout.flush() except Exception as e: print (str(e)) except subprocess.TimeoutExpired: #################################### # Catch time out error and log it. #################################### Os = GetOs() if Os == 'win': killCmd ="taskkill /FI "IMAGENAME eq {0}" /T /F" elif Os == 'linux': killCmd ="pkill {0)" elif Os == 'mac': # Linux, Mac OS killCmd ="killall -KILL {0}" runCommand(killCmd.format("java")) runCommand(killCmd.format("YouApp")) OutputHandle = None try: OutputHandle = OutputManager( 'KillJob_' + getTimeString() + '.txt', 'a+', sys.stdout, False) OutputHandle.write( cmd ) except Exception as e: print (str(e)) except Exception as e: for frame in traceback.extract_tb(sys.exc_info()[2]): fname,lineno,fn,text = frame print"Error in %s on line %d" % (fname, lineno) |
只是想写的东西是simpler。
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 | #!/usr/bin/python from subprocess import Popen, PIPE import datetime import time popen = Popen(["/bin/sleep","10"]); pid = popen.pid sttime = time.time(); waittime = 3 print"Start time %s"%(sttime) while True: popen.poll(); time.sleep(1) rcode = popen.returncode now = time.time(); if [ rcode is None ] and [ now > (sttime + waittime) ] : print"Killing it now" popen.kill() |