|
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410 |
- # -*- test-case-name: twisted.runner.test.test_procmon -*-
- # Copyright (c) Twisted Matrix Laboratories.
- # See LICENSE for details.
-
- """
- Support for starting, monitoring, and restarting child process.
- """
- from typing import Dict, List, Optional
-
- import attr
- import incremental
-
- from twisted.application import service
- from twisted.internet import error, protocol, reactor as _reactor
- from twisted.logger import Logger
- from twisted.protocols import basic
- from twisted.python import deprecate
-
-
- @attr.s(frozen=True, auto_attribs=True)
- class _Process:
- """
- The parameters of a process to be restarted.
-
- @ivar args: command-line arguments (including name of command as first one)
- @type args: C{list}
-
- @ivar uid: user-id to run process as, or None (which means inherit uid)
- @type uid: C{int}
-
- @ivar gid: group-id to run process as, or None (which means inherit gid)
- @type gid: C{int}
-
- @ivar env: environment for process
- @type env: C{dict}
-
- @ivar cwd: initial working directory for process or None
- (which means inherit cwd)
- @type cwd: C{str}
- """
-
- args: List[str]
- uid: Optional[int] = None
- gid: Optional[int] = None
- env: Dict[str, str] = attr.ib(default=attr.Factory(dict))
- cwd: Optional[str] = None
-
- @deprecate.deprecated(incremental.Version("Twisted", 18, 7, 0))
- def toTuple(self):
- """
- Convert process to tuple.
-
- Convert process to tuple that looks like the legacy structure
- of processes, for potential users who inspected processes
- directly.
-
- This was only an accidental feature, and will be removed. If
- you need to remember what processes were added to a process monitor,
- keep track of that when they are added. The process list
- inside the process monitor is no longer a public API.
-
- This allows changing the internal structure of the process list,
- when warranted by bug fixes or additional features.
-
- @return: tuple representation of process
- """
- return (self.args, self.uid, self.gid, self.env)
-
-
- class DummyTransport:
-
- disconnecting = 0
-
-
- transport = DummyTransport()
-
-
- class LineLogger(basic.LineReceiver):
-
- tag = None
- stream = None
- delimiter = b"\n"
- service = None
-
- def lineReceived(self, line):
- try:
- line = line.decode("utf-8")
- except UnicodeDecodeError:
- line = repr(line)
-
- self.service.log.info(
- "[{tag}] {line}", tag=self.tag, line=line, stream=self.stream
- )
-
-
- class LoggingProtocol(protocol.ProcessProtocol):
-
- service = None
- name = None
-
- def connectionMade(self):
- self._output = LineLogger()
- self._output.tag = self.name
- self._output.stream = "stdout"
- self._output.service = self.service
- self._outputEmpty = True
-
- self._error = LineLogger()
- self._error.tag = self.name
- self._error.stream = "stderr"
- self._error.service = self.service
- self._errorEmpty = True
-
- self._output.makeConnection(transport)
- self._error.makeConnection(transport)
-
- def outReceived(self, data):
- self._output.dataReceived(data)
- self._outputEmpty = data[-1] == b"\n"
-
- def errReceived(self, data):
- self._error.dataReceived(data)
- self._errorEmpty = data[-1] == b"\n"
-
- def processEnded(self, reason):
- if not self._outputEmpty:
- self._output.dataReceived(b"\n")
- if not self._errorEmpty:
- self._error.dataReceived(b"\n")
- self.service.connectionLost(self.name)
-
- @property
- def output(self):
- return self._output
-
- @property
- def empty(self):
- return self._outputEmpty
-
-
- class ProcessMonitor(service.Service):
- """
- ProcessMonitor runs processes, monitors their progress, and restarts
- them when they die.
-
- The ProcessMonitor will not attempt to restart a process that appears to
- die instantly -- with each "instant" death (less than 1 second, by
- default), it will delay approximately twice as long before restarting
- it. A successful run will reset the counter.
-
- The primary interface is L{addProcess} and L{removeProcess}. When the
- service is running (that is, when the application it is attached to is
- running), adding a process automatically starts it.
-
- Each process has a name. This name string must uniquely identify the
- process. In particular, attempting to add two processes with the same
- name will result in a C{KeyError}.
-
- @type threshold: C{float}
- @ivar threshold: How long a process has to live before the death is
- considered instant, in seconds. The default value is 1 second.
-
- @type killTime: C{float}
- @ivar killTime: How long a process being killed has to get its affairs
- in order before it gets killed with an unmaskable signal. The
- default value is 5 seconds.
-
- @type minRestartDelay: C{float}
- @ivar minRestartDelay: The minimum time (in seconds) to wait before
- attempting to restart a process. Default 1s.
-
- @type maxRestartDelay: C{float}
- @ivar maxRestartDelay: The maximum time (in seconds) to wait before
- attempting to restart a process. Default 3600s (1h).
-
- @type _reactor: L{IReactorProcess} provider
- @ivar _reactor: A provider of L{IReactorProcess} and L{IReactorTime}
- which will be used to spawn processes and register delayed calls.
-
- @type log: L{Logger}
- @ivar log: The logger used to propagate log messages from spawned
- processes.
-
- """
-
- threshold = 1
- killTime = 5
- minRestartDelay = 1
- maxRestartDelay = 3600
- log = Logger()
-
- def __init__(self, reactor=_reactor):
- self._reactor = reactor
-
- self._processes = {}
- self.protocols = {}
- self.delay = {}
- self.timeStarted = {}
- self.murder = {}
- self.restart = {}
-
- @deprecate.deprecatedProperty(incremental.Version("Twisted", 18, 7, 0))
- def processes(self):
- """
- Processes as dict of tuples
-
- @return: Dict of process name to monitored processes as tuples
- """
- return {name: process.toTuple() for name, process in self._processes.items()}
-
- @deprecate.deprecated(incremental.Version("Twisted", 18, 7, 0))
- def __getstate__(self):
- dct = service.Service.__getstate__(self)
- del dct["_reactor"]
- dct["protocols"] = {}
- dct["delay"] = {}
- dct["timeStarted"] = {}
- dct["murder"] = {}
- dct["restart"] = {}
- del dct["_processes"]
- dct["processes"] = self.processes
- return dct
-
- def addProcess(self, name, args, uid=None, gid=None, env={}, cwd=None):
- """
- Add a new monitored process and start it immediately if the
- L{ProcessMonitor} service is running.
-
- Note that args are passed to the system call, not to the shell. If
- running the shell is desired, the common idiom is to use
- C{ProcessMonitor.addProcess("name", ['/bin/sh', '-c', shell_script])}
-
- @param name: A name for this process. This value must be
- unique across all processes added to this monitor.
- @type name: C{str}
- @param args: The argv sequence for the process to launch.
- @param uid: The user ID to use to run the process. If L{None},
- the current UID is used.
- @type uid: C{int}
- @param gid: The group ID to use to run the process. If L{None},
- the current GID is used.
- @type uid: C{int}
- @param env: The environment to give to the launched process. See
- L{IReactorProcess.spawnProcess}'s C{env} parameter.
- @type env: C{dict}
- @param cwd: The initial working directory of the launched process.
- The default of C{None} means inheriting the laucnhing process's
- working directory.
- @type env: C{dict}
- @raise KeyError: If a process with the given name already exists.
- """
- if name in self._processes:
- raise KeyError(f"remove {name} first")
- self._processes[name] = _Process(args, uid, gid, env, cwd)
- self.delay[name] = self.minRestartDelay
- if self.running:
- self.startProcess(name)
-
- def removeProcess(self, name):
- """
- Stop the named process and remove it from the list of monitored
- processes.
-
- @type name: C{str}
- @param name: A string that uniquely identifies the process.
- """
- self.stopProcess(name)
- del self._processes[name]
-
- def startService(self):
- """
- Start all monitored processes.
- """
- service.Service.startService(self)
- for name in list(self._processes):
- self.startProcess(name)
-
- def stopService(self):
- """
- Stop all monitored processes and cancel all scheduled process restarts.
- """
- service.Service.stopService(self)
-
- # Cancel any outstanding restarts
- for name, delayedCall in list(self.restart.items()):
- if delayedCall.active():
- delayedCall.cancel()
-
- for name in list(self._processes):
- self.stopProcess(name)
-
- def connectionLost(self, name):
- """
- Called when a monitored processes exits. If
- L{service.IService.running} is L{True} (ie the service is started), the
- process will be restarted.
- If the process had been running for more than
- L{ProcessMonitor.threshold} seconds it will be restarted immediately.
- If the process had been running for less than
- L{ProcessMonitor.threshold} seconds, the restart will be delayed and
- each time the process dies before the configured threshold, the restart
- delay will be doubled - up to a maximum delay of maxRestartDelay sec.
-
- @type name: C{str}
- @param name: A string that uniquely identifies the process
- which exited.
- """
- # Cancel the scheduled _forceStopProcess function if the process
- # dies naturally
- if name in self.murder:
- if self.murder[name].active():
- self.murder[name].cancel()
- del self.murder[name]
-
- del self.protocols[name]
-
- if self._reactor.seconds() - self.timeStarted[name] < self.threshold:
- # The process died too fast - backoff
- nextDelay = self.delay[name]
- self.delay[name] = min(self.delay[name] * 2, self.maxRestartDelay)
-
- else:
- # Process had been running for a significant amount of time
- # restart immediately
- nextDelay = 0
- self.delay[name] = self.minRestartDelay
-
- # Schedule a process restart if the service is running
- if self.running and name in self._processes:
- self.restart[name] = self._reactor.callLater(
- nextDelay, self.startProcess, name
- )
-
- def startProcess(self, name):
- """
- @param name: The name of the process to be started
- """
- # If a protocol instance already exists, it means the process is
- # already running
- if name in self.protocols:
- return
-
- process = self._processes[name]
-
- proto = LoggingProtocol()
- proto.service = self
- proto.name = name
- self.protocols[name] = proto
- self.timeStarted[name] = self._reactor.seconds()
- self._reactor.spawnProcess(
- proto,
- process.args[0],
- process.args,
- uid=process.uid,
- gid=process.gid,
- env=process.env,
- path=process.cwd,
- )
-
- def _forceStopProcess(self, proc):
- """
- @param proc: An L{IProcessTransport} provider
- """
- try:
- proc.signalProcess("KILL")
- except error.ProcessExitedAlready:
- pass
-
- def stopProcess(self, name):
- """
- @param name: The name of the process to be stopped
- """
- if name not in self._processes:
- raise KeyError(f"Unrecognized process name: {name}")
-
- proto = self.protocols.get(name, None)
- if proto is not None:
- proc = proto.transport
- try:
- proc.signalProcess("TERM")
- except error.ProcessExitedAlready:
- pass
- else:
- self.murder[name] = self._reactor.callLater(
- self.killTime, self._forceStopProcess, proc
- )
-
- def restartAll(self):
- """
- Restart all processes. This is useful for third party management
- services to allow a user to restart servers because of an outside change
- in circumstances -- for example, a new version of a library is
- installed.
- """
- for name in self._processes:
- self.stopProcess(name)
-
- def __repr__(self) -> str:
- lst = []
- for name, proc in self._processes.items():
- uidgid = ""
- if proc.uid is not None:
- uidgid = str(proc.uid)
- if proc.gid is not None:
- uidgid += ":" + str(proc.gid)
-
- if uidgid:
- uidgid = "(" + uidgid + ")"
- lst.append(f"{name!r}{uidgid}: {proc.args!r}")
- return "<" + self.__class__.__name__ + " " + " ".join(lst) + ">"
|