183 lines
5.4 KiB
Python
183 lines
5.4 KiB
Python
###############################################################################
|
|
# Popen for LokyProcess.
|
|
#
|
|
# author: Thomas Moreau and Olivier Grisel
|
|
#
|
|
import os
|
|
import sys
|
|
import signal
|
|
import pickle
|
|
from io import BytesIO
|
|
from multiprocessing import util, process
|
|
from multiprocessing.connection import wait
|
|
from multiprocessing.context import set_spawning_popen
|
|
|
|
from . import reduction, resource_tracker, spawn
|
|
|
|
|
|
__all__ = ['Popen']
|
|
|
|
|
|
#
|
|
# Wrapper for an fd used while launching a process
|
|
#
|
|
|
|
class _DupFd:
|
|
def __init__(self, fd):
|
|
self.fd = reduction._mk_inheritable(fd)
|
|
|
|
def detach(self):
|
|
return self.fd
|
|
|
|
|
|
#
|
|
# Start child process using subprocess.Popen
|
|
#
|
|
|
|
class Popen:
|
|
method = 'loky'
|
|
DupFd = _DupFd
|
|
|
|
def __init__(self, process_obj):
|
|
sys.stdout.flush()
|
|
sys.stderr.flush()
|
|
self.returncode = None
|
|
self._fds = []
|
|
self._launch(process_obj)
|
|
|
|
def duplicate_for_child(self, fd):
|
|
self._fds.append(fd)
|
|
return reduction._mk_inheritable(fd)
|
|
|
|
def poll(self, flag=os.WNOHANG):
|
|
if self.returncode is None:
|
|
while True:
|
|
try:
|
|
pid, sts = os.waitpid(self.pid, flag)
|
|
except OSError:
|
|
# Child process not yet created. See #1731717
|
|
# e.errno == errno.ECHILD == 10
|
|
return None
|
|
else:
|
|
break
|
|
if pid == self.pid:
|
|
if os.WIFSIGNALED(sts):
|
|
self.returncode = -os.WTERMSIG(sts)
|
|
else:
|
|
assert os.WIFEXITED(sts)
|
|
self.returncode = os.WEXITSTATUS(sts)
|
|
return self.returncode
|
|
|
|
def wait(self, timeout=None):
|
|
if self.returncode is None:
|
|
if timeout is not None:
|
|
if not wait([self.sentinel], timeout):
|
|
return None
|
|
# This shouldn't block if wait() returned successfully.
|
|
return self.poll(os.WNOHANG if timeout == 0.0 else 0)
|
|
return self.returncode
|
|
|
|
def terminate(self):
|
|
if self.returncode is None:
|
|
try:
|
|
os.kill(self.pid, signal.SIGTERM)
|
|
except ProcessLookupError:
|
|
pass
|
|
except OSError:
|
|
if self.wait(timeout=0.1) is None:
|
|
raise
|
|
|
|
def _launch(self, process_obj):
|
|
|
|
tracker_fd = resource_tracker._resource_tracker.getfd()
|
|
|
|
fp = BytesIO()
|
|
set_spawning_popen(self)
|
|
try:
|
|
prep_data = spawn.get_preparation_data(
|
|
process_obj._name,
|
|
getattr(process_obj, "init_main_module", True))
|
|
reduction.dump(prep_data, fp)
|
|
reduction.dump(process_obj, fp)
|
|
|
|
finally:
|
|
set_spawning_popen(None)
|
|
|
|
try:
|
|
parent_r, child_w = os.pipe()
|
|
child_r, parent_w = os.pipe()
|
|
# for fd in self._fds:
|
|
# _mk_inheritable(fd)
|
|
|
|
cmd_python = [sys.executable]
|
|
cmd_python += ['-m', self.__module__]
|
|
cmd_python += ['--process-name', str(process_obj.name)]
|
|
cmd_python += ['--pipe', str(reduction._mk_inheritable(child_r))]
|
|
reduction._mk_inheritable(child_w)
|
|
reduction._mk_inheritable(tracker_fd)
|
|
self._fds += [child_r, child_w, tracker_fd]
|
|
if sys.version_info >= (3, 8) and os.name == 'posix':
|
|
mp_tracker_fd = prep_data['mp_tracker_args']['fd']
|
|
self.duplicate_for_child(mp_tracker_fd)
|
|
|
|
from .fork_exec import fork_exec
|
|
pid = fork_exec(cmd_python, self._fds, env=process_obj.env)
|
|
util.debug(
|
|
f"launched python with pid {pid} and cmd:\n{cmd_python}"
|
|
)
|
|
self.sentinel = parent_r
|
|
|
|
method = 'getbuffer'
|
|
if not hasattr(fp, method):
|
|
method = 'getvalue'
|
|
with os.fdopen(parent_w, 'wb') as f:
|
|
f.write(getattr(fp, method)())
|
|
self.pid = pid
|
|
finally:
|
|
if parent_r is not None:
|
|
util.Finalize(self, os.close, (parent_r,))
|
|
for fd in (child_r, child_w):
|
|
if fd is not None:
|
|
os.close(fd)
|
|
|
|
@staticmethod
|
|
def thread_is_spawning():
|
|
return True
|
|
|
|
|
|
if __name__ == '__main__':
|
|
import argparse
|
|
parser = argparse.ArgumentParser('Command line parser')
|
|
parser.add_argument('--pipe', type=int, required=True,
|
|
help='File handle for the pipe')
|
|
parser.add_argument('--process-name', type=str, default=None,
|
|
help='Identifier for debugging purpose')
|
|
|
|
args = parser.parse_args()
|
|
|
|
info = {}
|
|
exitcode = 1
|
|
try:
|
|
with os.fdopen(args.pipe, 'rb') as from_parent:
|
|
process.current_process()._inheriting = True
|
|
try:
|
|
prep_data = pickle.load(from_parent)
|
|
spawn.prepare(prep_data)
|
|
process_obj = pickle.load(from_parent)
|
|
finally:
|
|
del process.current_process()._inheriting
|
|
|
|
exitcode = process_obj._bootstrap()
|
|
except Exception:
|
|
print('\n\n' + '-' * 80)
|
|
print(f'{args.process_name} failed with traceback: ')
|
|
print('-' * 80)
|
|
import traceback
|
|
print(traceback.format_exc())
|
|
print('\n' + '-' * 80)
|
|
finally:
|
|
if from_parent is not None:
|
|
from_parent.close()
|
|
|
|
sys.exit(exitcode)
|