This commit is contained in:
Mai Gillmann
2019-12-17 14:09:10 +01:00
parent 66e908fc8a
commit 4791d00a43
2122 changed files with 423791 additions and 0 deletions
@@ -0,0 +1,168 @@
# -*- test-case-name: twisted.test.test_stdio -*-
"""Standard input/out/err support.
Future Plans::
support for stderr, perhaps
Rewrite to use the reactor instead of an ad-hoc mechanism for connecting
protocols to transport.
Maintainer: James Y Knight
"""
from zope.interface import implementer
from twisted.internet import process, error, interfaces
from twisted.python import log, failure
@implementer(interfaces.IAddress)
class PipeAddress(object):
pass
@implementer(interfaces.ITransport, interfaces.IProducer,
interfaces.IConsumer, interfaces.IHalfCloseableDescriptor)
class StandardIO(object):
_reader = None
_writer = None
disconnected = False
disconnecting = False
def __init__(self, proto, stdin=0, stdout=1, reactor=None):
if reactor is None:
from twisted.internet import reactor
self.protocol = proto
self._writer = process.ProcessWriter(reactor, self, 'write', stdout)
self._reader = process.ProcessReader(reactor, self, 'read', stdin)
self._reader.startReading()
self.protocol.makeConnection(self)
# ITransport
# XXX Actually, see #3597.
def loseWriteConnection(self):
if self._writer is not None:
self._writer.loseConnection()
def write(self, data):
if self._writer is not None:
self._writer.write(data)
def writeSequence(self, data):
if self._writer is not None:
self._writer.writeSequence(data)
def loseConnection(self):
self.disconnecting = True
if self._writer is not None:
self._writer.loseConnection()
if self._reader is not None:
# Don't loseConnection, because we don't want to SIGPIPE it.
self._reader.stopReading()
def getPeer(self):
return PipeAddress()
def getHost(self):
return PipeAddress()
# Callbacks from process.ProcessReader/ProcessWriter
def childDataReceived(self, fd, data):
self.protocol.dataReceived(data)
def childConnectionLost(self, fd, reason):
if self.disconnected:
return
if reason.value.__class__ == error.ConnectionDone:
# Normal close
if fd == 'read':
self._readConnectionLost(reason)
else:
self._writeConnectionLost(reason)
else:
self.connectionLost(reason)
def connectionLost(self, reason):
self.disconnected = True
# Make sure to cleanup the other half
_reader = self._reader
_writer = self._writer
protocol = self.protocol
self._reader = self._writer = None
self.protocol = None
if _writer is not None and not _writer.disconnected:
_writer.connectionLost(reason)
if _reader is not None and not _reader.disconnected:
_reader.connectionLost(reason)
try:
protocol.connectionLost(reason)
except:
log.err()
def _writeConnectionLost(self, reason):
self._writer=None
if self.disconnecting:
self.connectionLost(reason)
return
p = interfaces.IHalfCloseableProtocol(self.protocol, None)
if p:
try:
p.writeConnectionLost()
except:
log.err()
self.connectionLost(failure.Failure())
def _readConnectionLost(self, reason):
self._reader=None
p = interfaces.IHalfCloseableProtocol(self.protocol, None)
if p:
try:
p.readConnectionLost()
except:
log.err()
self.connectionLost(failure.Failure())
else:
self.connectionLost(reason)
# IConsumer
def registerProducer(self, producer, streaming):
if self._writer is None:
producer.stopProducing()
else:
self._writer.registerProducer(producer, streaming)
def unregisterProducer(self):
if self._writer is not None:
self._writer.unregisterProducer()
# IProducer
def stopProducing(self):
self.loseConnection()
def pauseProducing(self):
if self._reader is not None:
self._reader.pauseProducing()
def resumeProducing(self):
if self._reader is not None:
self._reader.resumeProducing()
def stopReading(self):
"""Compatibility only, don't use. Call pauseProducing."""
self.pauseProducing()
def startReading(self):
"""Compatibility only, don't use. Call resumeProducing."""
self.resumeProducing()
@@ -0,0 +1,133 @@
# -*- test-case-name: twisted.internet.test.test_win32serialport -*-
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
Serial port support for Windows.
Requires PySerial and pywin32.
"""
from __future__ import division, absolute_import
# system imports
from serial import PARITY_NONE
from serial import STOPBITS_ONE
from serial import EIGHTBITS
from serial.serialutil import to_bytes
import win32file, win32event
# twisted imports
from twisted.internet import abstract
# sibling imports
from twisted.internet.serialport import BaseSerialPort
class SerialPort(BaseSerialPort, abstract.FileDescriptor):
"""A serial device, acting as a transport, that uses a win32 event."""
connected = 1
def __init__(self, protocol, deviceNameOrPortNumber, reactor,
baudrate = 9600, bytesize = EIGHTBITS, parity = PARITY_NONE,
stopbits = STOPBITS_ONE, xonxoff = 0, rtscts = 0):
self._serial = self._serialFactory(
deviceNameOrPortNumber, baudrate=baudrate, bytesize=bytesize,
parity=parity, stopbits=stopbits, timeout=None,
xonxoff=xonxoff, rtscts=rtscts)
self.flushInput()
self.flushOutput()
self.reactor = reactor
self.protocol = protocol
self.outQueue = []
self.closed = 0
self.closedNotifies = 0
self.writeInProgress = 0
self.protocol = protocol
self._overlappedRead = win32file.OVERLAPPED()
self._overlappedRead.hEvent = win32event.CreateEvent(None, 1, 0, None)
self._overlappedWrite = win32file.OVERLAPPED()
self._overlappedWrite.hEvent = win32event.CreateEvent(None, 0, 0, None)
self.reactor.addEvent(self._overlappedRead.hEvent, self, 'serialReadEvent')
self.reactor.addEvent(self._overlappedWrite.hEvent, self, 'serialWriteEvent')
self.protocol.makeConnection(self)
self._finishPortSetup()
def _finishPortSetup(self):
"""
Finish setting up the serial port.
This is a separate method to facilitate testing.
"""
flags, comstat = self._clearCommError()
rc, self.read_buf = win32file.ReadFile(self._serial._port_handle,
win32file.AllocateReadBuffer(1),
self._overlappedRead)
def _clearCommError(self):
return win32file.ClearCommError(self._serial._port_handle)
def serialReadEvent(self):
#get that character we set up
n = win32file.GetOverlappedResult(self._serial._port_handle, self._overlappedRead, 0)
first = to_bytes(self.read_buf[:n])
#now we should get everything that is already in the buffer
flags, comstat = self._clearCommError()
if comstat.cbInQue:
win32event.ResetEvent(self._overlappedRead.hEvent)
rc, buf = win32file.ReadFile(self._serial._port_handle,
win32file.AllocateReadBuffer(comstat.cbInQue),
self._overlappedRead)
n = win32file.GetOverlappedResult(self._serial._port_handle, self._overlappedRead, 1)
#handle all the received data:
self.protocol.dataReceived(first + to_bytes(buf[:n]))
else:
#handle all the received data:
self.protocol.dataReceived(first)
#set up next one
win32event.ResetEvent(self._overlappedRead.hEvent)
rc, self.read_buf = win32file.ReadFile(self._serial._port_handle,
win32file.AllocateReadBuffer(1),
self._overlappedRead)
def write(self, data):
if data:
if self.writeInProgress:
self.outQueue.append(data)
else:
self.writeInProgress = 1
win32file.WriteFile(self._serial._port_handle, data, self._overlappedWrite)
def serialWriteEvent(self):
try:
dataToWrite = self.outQueue.pop(0)
except IndexError:
self.writeInProgress = 0
return
else:
win32file.WriteFile(self._serial._port_handle, dataToWrite, self._overlappedWrite)
def connectionLost(self, reason):
"""
Called when the serial port disconnects.
Will call C{connectionLost} on the protocol that is handling the
serial data.
"""
self.reactor.removeEvent(self._overlappedRead.hEvent)
self.reactor.removeEvent(self._overlappedWrite.hEvent)
abstract.FileDescriptor.connectionLost(self, reason)
self._serial.close()
self.protocol.connectionLost(reason)
@@ -0,0 +1,133 @@
# -*- test-case-name: twisted.test.test_stdio -*-
"""
Windows-specific implementation of the L{twisted.internet.stdio} interface.
"""
from __future__ import absolute_import, division
import win32api
import os
import msvcrt
from zope.interface import implementer
from twisted.internet.interfaces import (IHalfCloseableProtocol, ITransport,
IConsumer, IPushProducer, IAddress)
from twisted.internet import _pollingfile, main
from twisted.python.failure import Failure
@implementer(IAddress)
class Win32PipeAddress(object):
pass
@implementer(ITransport, IConsumer, IPushProducer)
class StandardIO(_pollingfile._PollingTimer):
disconnecting = False
disconnected = False
def __init__(self, proto, reactor=None):
"""
Start talking to standard IO with the given protocol.
Also, put it stdin/stdout/stderr into binary mode.
"""
if reactor is None:
from twisted.internet import reactor
for stdfd in range(0, 1, 2):
msvcrt.setmode(stdfd, os.O_BINARY)
_pollingfile._PollingTimer.__init__(self, reactor)
self.proto = proto
hstdin = win32api.GetStdHandle(win32api.STD_INPUT_HANDLE)
hstdout = win32api.GetStdHandle(win32api.STD_OUTPUT_HANDLE)
self.stdin = _pollingfile._PollableReadPipe(
hstdin, self.dataReceived, self.readConnectionLost)
self.stdout = _pollingfile._PollableWritePipe(
hstdout, self.writeConnectionLost)
self._addPollableResource(self.stdin)
self._addPollableResource(self.stdout)
self.proto.makeConnection(self)
def dataReceived(self, data):
self.proto.dataReceived(data)
def readConnectionLost(self):
if IHalfCloseableProtocol.providedBy(self.proto):
self.proto.readConnectionLost()
self.checkConnLost()
def writeConnectionLost(self):
if IHalfCloseableProtocol.providedBy(self.proto):
self.proto.writeConnectionLost()
self.checkConnLost()
connsLost = 0
def checkConnLost(self):
self.connsLost += 1
if self.connsLost >= 2:
self.disconnecting = True
self.disconnected = True
self.proto.connectionLost(Failure(main.CONNECTION_DONE))
# ITransport
def write(self, data):
self.stdout.write(data)
def writeSequence(self, seq):
self.stdout.write(b''.join(seq))
def loseConnection(self):
self.disconnecting = True
self.stdin.close()
self.stdout.close()
def getPeer(self):
return Win32PipeAddress()
def getHost(self):
return Win32PipeAddress()
# IConsumer
def registerProducer(self, producer, streaming):
return self.stdout.registerProducer(producer, streaming)
def unregisterProducer(self):
return self.stdout.unregisterProducer()
# def write() above
# IProducer
def stopProducing(self):
self.stdin.stopProducing()
# IPushProducer
def pauseProducing(self):
self.stdin.pauseProducing()
def resumeProducing(self):
self.stdin.resumeProducing()
@@ -0,0 +1,546 @@
# -*- test-case-name: twisted.test.test_abstract -*-
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
Support for generic select()able objects.
"""
from __future__ import division, absolute_import
from socket import AF_INET, AF_INET6, inet_pton, error
from zope.interface import implementer
# Twisted Imports
from twisted.python.compat import unicode, lazyByteSlice, _PY3
from twisted.python import reflect, failure
from twisted.internet import interfaces, main
if _PY3:
# Python 3.4+ can join bytes and memoryviews; using a
# memoryview prevents the slice from copying
def _concatenate(bObj, offset, bArray):
return b''.join([memoryview(bObj)[offset:]] + bArray)
else:
from __builtin__ import buffer
def _concatenate(bObj, offset, bArray):
# Avoid one extra string copy by using a buffer to limit what
# we include in the result.
return buffer(bObj, offset) + b"".join(bArray)
class _ConsumerMixin(object):
"""
L{IConsumer} implementations can mix this in to get C{registerProducer} and
C{unregisterProducer} methods which take care of keeping track of a
producer's state.
Subclasses must provide three attributes which L{_ConsumerMixin} will read
but not write:
- connected: A C{bool} which is C{True} as long as the consumer has
someplace to send bytes (for example, a TCP connection), and then
C{False} when it no longer does.
- disconnecting: A C{bool} which is C{False} until something like
L{ITransport.loseConnection} is called, indicating that the send buffer
should be flushed and the connection lost afterwards. Afterwards,
C{True}.
- disconnected: A C{bool} which is C{False} until the consumer no longer
has a place to send bytes, then C{True}.
Subclasses must also override the C{startWriting} method.
@ivar producer: L{None} if no producer is registered, otherwise the
registered producer.
@ivar producerPaused: A flag indicating whether the producer is currently
paused.
@type producerPaused: L{bool}
@ivar streamingProducer: A flag indicating whether the producer was
registered as a streaming (ie push) producer or not (ie a pull
producer). This will determine whether the consumer may ever need to
pause and resume it, or if it can merely call C{resumeProducing} on it
when buffer space is available.
@ivar streamingProducer: C{bool} or C{int}
"""
producer = None
producerPaused = False
streamingProducer = False
def startWriting(self):
"""
Override in a subclass to cause the reactor to monitor this selectable
for write events. This will be called once in C{unregisterProducer} if
C{loseConnection} has previously been called, so that the connection can
actually close.
"""
raise NotImplementedError("%r did not implement startWriting")
def registerProducer(self, producer, streaming):
"""
Register to receive data from a producer.
This sets this selectable to be a consumer for a producer. When this
selectable runs out of data on a write() call, it will ask the producer
to resumeProducing(). When the FileDescriptor's internal data buffer is
filled, it will ask the producer to pauseProducing(). If the connection
is lost, FileDescriptor calls producer's stopProducing() method.
If streaming is true, the producer should provide the IPushProducer
interface. Otherwise, it is assumed that producer provides the
IPullProducer interface. In this case, the producer won't be asked to
pauseProducing(), but it has to be careful to write() data only when its
resumeProducing() method is called.
"""
if self.producer is not None:
raise RuntimeError(
"Cannot register producer %s, because producer %s was never "
"unregistered." % (producer, self.producer))
if self.disconnected:
producer.stopProducing()
else:
self.producer = producer
self.streamingProducer = streaming
if not streaming:
producer.resumeProducing()
def unregisterProducer(self):
"""
Stop consuming data from a producer, without disconnecting.
"""
self.producer = None
if self.connected and self.disconnecting:
self.startWriting()
@implementer(interfaces.ILoggingContext)
class _LogOwner(object):
"""
Mixin to help implement L{interfaces.ILoggingContext} for transports which
have a protocol, the log prefix of which should also appear in the
transport's log prefix.
"""
def _getLogPrefix(self, applicationObject):
"""
Determine the log prefix to use for messages related to
C{applicationObject}, which may or may not be an
L{interfaces.ILoggingContext} provider.
@return: A C{str} giving the log prefix to use.
"""
if interfaces.ILoggingContext.providedBy(applicationObject):
return applicationObject.logPrefix()
return applicationObject.__class__.__name__
def logPrefix(self):
"""
Override this method to insert custom logging behavior. Its
return value will be inserted in front of every line. It may
be called more times than the number of output lines.
"""
return "-"
@implementer(
interfaces.IPushProducer, interfaces.IReadWriteDescriptor,
interfaces.IConsumer, interfaces.ITransport,
interfaces.IHalfCloseableDescriptor)
class FileDescriptor(_ConsumerMixin, _LogOwner):
"""
An object which can be operated on by select().
This is an abstract superclass of all objects which may be notified when
they are readable or writable; e.g. they have a file-descriptor that is
valid to be passed to select(2).
"""
connected = 0
disconnected = 0
disconnecting = 0
_writeDisconnecting = False
_writeDisconnected = False
dataBuffer = b""
offset = 0
SEND_LIMIT = 128*1024
def __init__(self, reactor=None):
"""
@param reactor: An L{IReactorFDSet} provider which this descriptor will
use to get readable and writeable event notifications. If no value
is given, the global reactor will be used.
"""
if not reactor:
from twisted.internet import reactor
self.reactor = reactor
self._tempDataBuffer = [] # will be added to dataBuffer in doWrite
self._tempDataLen = 0
def connectionLost(self, reason):
"""The connection was lost.
This is called when the connection on a selectable object has been
lost. It will be called whether the connection was closed explicitly,
an exception occurred in an event handler, or the other end of the
connection closed it first.
Clean up state here, but make sure to call back up to FileDescriptor.
"""
self.disconnected = 1
self.connected = 0
if self.producer is not None:
self.producer.stopProducing()
self.producer = None
self.stopReading()
self.stopWriting()
def writeSomeData(self, data):
"""
Write as much as possible of the given data, immediately.
This is called to invoke the lower-level writing functionality, such
as a socket's send() method, or a file's write(); this method
returns an integer or an exception. If an integer, it is the number
of bytes written (possibly zero); if an exception, it indicates the
connection was lost.
"""
raise NotImplementedError("%s does not implement writeSomeData" %
reflect.qual(self.__class__))
def doRead(self):
"""
Called when data is available for reading.
Subclasses must override this method. The result will be interpreted
in the same way as a result of doWrite().
"""
raise NotImplementedError("%s does not implement doRead" %
reflect.qual(self.__class__))
def doWrite(self):
"""
Called when data can be written.
@return: L{None} on success, an exception or a negative integer on
failure.
@see: L{twisted.internet.interfaces.IWriteDescriptor.doWrite}.
"""
if len(self.dataBuffer) - self.offset < self.SEND_LIMIT:
# If there is currently less than SEND_LIMIT bytes left to send
# in the string, extend it with the array data.
self.dataBuffer = _concatenate(
self.dataBuffer, self.offset, self._tempDataBuffer)
self.offset = 0
self._tempDataBuffer = []
self._tempDataLen = 0
# Send as much data as you can.
if self.offset:
l = self.writeSomeData(lazyByteSlice(self.dataBuffer, self.offset))
else:
l = self.writeSomeData(self.dataBuffer)
# There is no writeSomeData implementation in Twisted which returns
# < 0, but the documentation for writeSomeData used to claim negative
# integers meant connection lost. Keep supporting this here,
# although it may be worth deprecating and removing at some point.
if isinstance(l, Exception) or l < 0:
return l
self.offset += l
# If there is nothing left to send,
if self.offset == len(self.dataBuffer) and not self._tempDataLen:
self.dataBuffer = b""
self.offset = 0
# stop writing.
self.stopWriting()
# If I've got a producer who is supposed to supply me with data,
if self.producer is not None and ((not self.streamingProducer)
or self.producerPaused):
# tell them to supply some more.
self.producerPaused = False
self.producer.resumeProducing()
elif self.disconnecting:
# But if I was previously asked to let the connection die, do
# so.
return self._postLoseConnection()
elif self._writeDisconnecting:
# I was previously asked to half-close the connection. We
# set _writeDisconnected before calling handler, in case the
# handler calls loseConnection(), which will want to check for
# this attribute.
self._writeDisconnected = True
result = self._closeWriteConnection()
return result
return None
def _postLoseConnection(self):
"""Called after a loseConnection(), when all data has been written.
Whatever this returns is then returned by doWrite.
"""
# default implementation, telling reactor we're finished
return main.CONNECTION_DONE
def _closeWriteConnection(self):
# override in subclasses
pass
def writeConnectionLost(self, reason):
# in current code should never be called
self.connectionLost(reason)
def readConnectionLost(self, reason):
# override in subclasses
self.connectionLost(reason)
def _isSendBufferFull(self):
"""
Determine whether the user-space send buffer for this transport is full
or not.
When the buffer contains more than C{self.bufferSize} bytes, it is
considered full. This might be improved by considering the size of the
kernel send buffer and how much of it is free.
@return: C{True} if it is full, C{False} otherwise.
"""
return len(self.dataBuffer) + self._tempDataLen > self.bufferSize
def _maybePauseProducer(self):
"""
Possibly pause a producer, if there is one and the send buffer is full.
"""
# If we are responsible for pausing our producer,
if self.producer is not None and self.streamingProducer:
# and our buffer is full,
if self._isSendBufferFull():
# pause it.
self.producerPaused = True
self.producer.pauseProducing()
def write(self, data):
"""Reliably write some data.
The data is buffered until the underlying file descriptor is ready
for writing. If there is more than C{self.bufferSize} data in the
buffer and this descriptor has a registered streaming producer, its
C{pauseProducing()} method will be called.
"""
if isinstance(data, unicode): # no, really, I mean it
raise TypeError("Data must not be unicode")
if not self.connected or self._writeDisconnected:
return
if data:
self._tempDataBuffer.append(data)
self._tempDataLen += len(data)
self._maybePauseProducer()
self.startWriting()
def writeSequence(self, iovec):
"""
Reliably write a sequence of data.
Currently, this is a convenience method roughly equivalent to::
for chunk in iovec:
fd.write(chunk)
It may have a more efficient implementation at a later time or in a
different reactor.
As with the C{write()} method, if a buffer size limit is reached and a
streaming producer is registered, it will be paused until the buffered
data is written to the underlying file descriptor.
"""
for i in iovec:
if isinstance(i, unicode): # no, really, I mean it
raise TypeError("Data must not be unicode")
if not self.connected or not iovec or self._writeDisconnected:
return
self._tempDataBuffer.extend(iovec)
for i in iovec:
self._tempDataLen += len(i)
self._maybePauseProducer()
self.startWriting()
def loseConnection(self, _connDone=failure.Failure(main.CONNECTION_DONE)):
"""Close the connection at the next available opportunity.
Call this to cause this FileDescriptor to lose its connection. It will
first write any data that it has buffered.
If there is data buffered yet to be written, this method will cause the
transport to lose its connection as soon as it's done flushing its
write buffer. If you have a producer registered, the connection won't
be closed until the producer is finished. Therefore, make sure you
unregister your producer when it's finished, or the connection will
never close.
"""
if self.connected and not self.disconnecting:
if self._writeDisconnected:
# doWrite won't trigger the connection close anymore
self.stopReading()
self.stopWriting()
self.connectionLost(_connDone)
else:
self.stopReading()
self.startWriting()
self.disconnecting = 1
def loseWriteConnection(self):
self._writeDisconnecting = True
self.startWriting()
def stopReading(self):
"""Stop waiting for read availability.
Call this to remove this selectable from being notified when it is
ready for reading.
"""
self.reactor.removeReader(self)
def stopWriting(self):
"""Stop waiting for write availability.
Call this to remove this selectable from being notified when it is ready
for writing.
"""
self.reactor.removeWriter(self)
def startReading(self):
"""Start waiting for read availability.
"""
self.reactor.addReader(self)
def startWriting(self):
"""Start waiting for write availability.
Call this to have this FileDescriptor be notified whenever it is ready for
writing.
"""
self.reactor.addWriter(self)
# Producer/consumer implementation
# first, the consumer stuff. This requires no additional work, as
# any object you can write to can be a consumer, really.
producer = None
bufferSize = 2**2**2**2
def stopConsuming(self):
"""Stop consuming data.
This is called when a producer has lost its connection, to tell the
consumer to go lose its connection (and break potential circular
references).
"""
self.unregisterProducer()
self.loseConnection()
# producer interface implementation
def resumeProducing(self):
if self.connected and not self.disconnecting:
self.startReading()
def pauseProducing(self):
self.stopReading()
def stopProducing(self):
self.loseConnection()
def fileno(self):
"""File Descriptor number for select().
This method must be overridden or assigned in subclasses to
indicate a valid file descriptor for the operating system.
"""
return -1
def isIPAddress(addr, family=AF_INET):
"""
Determine whether the given string represents an IP address of the given
family; by default, an IPv4 address.
@type addr: C{str}
@param addr: A string which may or may not be the decimal dotted
representation of an IPv4 address.
@param family: The address family to test for; one of the C{AF_*} constants
from the L{socket} module. (This parameter has only been available
since Twisted 17.1.0; previously L{isIPAddress} could only test for IPv4
addresses.)
@type family: C{int}
@rtype: C{bool}
@return: C{True} if C{addr} represents an IPv4 address, C{False} otherwise.
"""
if isinstance(addr, bytes):
try:
addr = addr.decode("ascii")
except UnicodeDecodeError:
return False
if family == AF_INET6:
# On some platforms, inet_ntop fails unless the scope ID is valid; this
# is a test for whether the given string *is* an IP address, so strip
# any potential scope ID before checking.
addr = addr.split(u"%", 1)[0]
elif family == AF_INET:
# On Windows, where 3.5+ implement inet_pton, "0" is considered a valid
# IPv4 address, but we want to ensure we have all 4 segments.
if addr.count(u".") != 3:
return False
else:
raise ValueError("unknown address family {!r}".format(family))
try:
# This might be a native implementation or the one from
# twisted.python.compat.
inet_pton(family, addr)
except (ValueError, error):
return False
return True
def isIPv6Address(addr):
"""
Determine whether the given string represents an IPv6 address.
@param addr: A string which may or may not be the hex
representation of an IPv6 address.
@type addr: C{str}
@return: C{True} if C{addr} represents an IPv6 address, C{False}
otherwise.
@rtype: C{bool}
"""
return isIPAddress(addr, AF_INET6)
__all__ = ["FileDescriptor", "isIPAddress", "isIPv6Address"]
@@ -0,0 +1,249 @@
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
An epoll() based implementation of the twisted main loop.
To install the event loop (and you should do this before any connections,
listeners or connectors are added)::
from twisted.internet import epollreactor
epollreactor.install()
"""
from __future__ import division, absolute_import
from select import epoll, EPOLLHUP, EPOLLERR, EPOLLIN, EPOLLOUT
import errno
from zope.interface import implementer
from twisted.internet.interfaces import IReactorFDSet
from twisted.python import log
from twisted.internet import posixbase
@implementer(IReactorFDSet)
class EPollReactor(posixbase.PosixReactorBase, posixbase._PollLikeMixin):
"""
A reactor that uses epoll(7).
@ivar _poller: A C{epoll} which will be used to check for I/O
readiness.
@ivar _selectables: A dictionary mapping integer file descriptors to
instances of C{FileDescriptor} which have been registered with the
reactor. All C{FileDescriptors} which are currently receiving read or
write readiness notifications will be present as values in this
dictionary.
@ivar _reads: A set containing integer file descriptors. Values in this
set will be registered with C{_poller} for read readiness notifications
which will be dispatched to the corresponding C{FileDescriptor}
instances in C{_selectables}.
@ivar _writes: A set containing integer file descriptors. Values in this
set will be registered with C{_poller} for write readiness
notifications which will be dispatched to the corresponding
C{FileDescriptor} instances in C{_selectables}.
@ivar _continuousPolling: A L{_ContinuousPolling} instance, used to handle
file descriptors (e.g. filesystem files) that are not supported by
C{epoll(7)}.
"""
# Attributes for _PollLikeMixin
_POLL_DISCONNECTED = (EPOLLHUP | EPOLLERR)
_POLL_IN = EPOLLIN
_POLL_OUT = EPOLLOUT
def __init__(self):
"""
Initialize epoll object, file descriptor tracking dictionaries, and the
base class.
"""
# Create the poller we're going to use. The 1024 here is just a hint
# to the kernel, it is not a hard maximum. After Linux 2.6.8, the size
# argument is completely ignored.
self._poller = epoll(1024)
self._reads = set()
self._writes = set()
self._selectables = {}
self._continuousPolling = posixbase._ContinuousPolling(self)
posixbase.PosixReactorBase.__init__(self)
def _add(self, xer, primary, other, selectables, event, antievent):
"""
Private method for adding a descriptor from the event loop.
It takes care of adding it if new or modifying it if already added
for another state (read -> read/write for example).
"""
fd = xer.fileno()
if fd not in primary:
flags = event
# epoll_ctl can raise all kinds of IOErrors, and every one
# indicates a bug either in the reactor or application-code.
# Let them all through so someone sees a traceback and fixes
# something. We'll do the same thing for every other call to
# this method in this file.
if fd in other:
flags |= antievent
self._poller.modify(fd, flags)
else:
self._poller.register(fd, flags)
# Update our own tracking state *only* after the epoll call has
# succeeded. Otherwise we may get out of sync.
primary.add(fd)
selectables[fd] = xer
def addReader(self, reader):
"""
Add a FileDescriptor for notification of data available to read.
"""
try:
self._add(reader, self._reads, self._writes, self._selectables,
EPOLLIN, EPOLLOUT)
except IOError as e:
if e.errno == errno.EPERM:
# epoll(7) doesn't support certain file descriptors,
# e.g. filesystem files, so for those we just poll
# continuously:
self._continuousPolling.addReader(reader)
else:
raise
def addWriter(self, writer):
"""
Add a FileDescriptor for notification of data available to write.
"""
try:
self._add(writer, self._writes, self._reads, self._selectables,
EPOLLOUT, EPOLLIN)
except IOError as e:
if e.errno == errno.EPERM:
# epoll(7) doesn't support certain file descriptors,
# e.g. filesystem files, so for those we just poll
# continuously:
self._continuousPolling.addWriter(writer)
else:
raise
def _remove(self, xer, primary, other, selectables, event, antievent):
"""
Private method for removing a descriptor from the event loop.
It does the inverse job of _add, and also add a check in case of the fd
has gone away.
"""
fd = xer.fileno()
if fd == -1:
for fd, fdes in selectables.items():
if xer is fdes:
break
else:
return
if fd in primary:
if fd in other:
flags = antievent
# See comment above modify call in _add.
self._poller.modify(fd, flags)
else:
del selectables[fd]
# See comment above _control call in _add.
self._poller.unregister(fd)
primary.remove(fd)
def removeReader(self, reader):
"""
Remove a Selectable for notification of data available to read.
"""
if self._continuousPolling.isReading(reader):
self._continuousPolling.removeReader(reader)
return
self._remove(reader, self._reads, self._writes, self._selectables,
EPOLLIN, EPOLLOUT)
def removeWriter(self, writer):
"""
Remove a Selectable for notification of data available to write.
"""
if self._continuousPolling.isWriting(writer):
self._continuousPolling.removeWriter(writer)
return
self._remove(writer, self._writes, self._reads, self._selectables,
EPOLLOUT, EPOLLIN)
def removeAll(self):
"""
Remove all selectables, and return a list of them.
"""
return (self._removeAll(
[self._selectables[fd] for fd in self._reads],
[self._selectables[fd] for fd in self._writes]) +
self._continuousPolling.removeAll())
def getReaders(self):
return ([self._selectables[fd] for fd in self._reads] +
self._continuousPolling.getReaders())
def getWriters(self):
return ([self._selectables[fd] for fd in self._writes] +
self._continuousPolling.getWriters())
def doPoll(self, timeout):
"""
Poll the poller for new events.
"""
if timeout is None:
timeout = -1 # Wait indefinitely.
try:
# Limit the number of events to the number of io objects we're
# currently tracking (because that's maybe a good heuristic) and
# the amount of time we block to the value specified by our
# caller.
l = self._poller.poll(timeout, len(self._selectables))
except IOError as err:
if err.errno == errno.EINTR:
return
# See epoll_wait(2) for documentation on the other conditions
# under which this can fail. They can only be due to a serious
# programming error on our part, so let's just announce them
# loudly.
raise
_drdw = self._doReadOrWrite
for fd, event in l:
try:
selectable = self._selectables[fd]
except KeyError:
pass
else:
log.callWithLogger(selectable, _drdw, selectable, fd, event)
doIteration = doPoll
def install():
"""
Install the epoll() reactor.
"""
p = EPollReactor()
from twisted.internet.main import installReactor
installReactor(p)
__all__ = ["EPollReactor", "install"]
@@ -0,0 +1,517 @@
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
Exceptions and errors for use in twisted.internet modules.
"""
from __future__ import division, absolute_import
import socket
from twisted.python import deprecate
from incremental import Version
class BindError(Exception):
"""An error occurred binding to an interface"""
def __str__(self):
s = self.__doc__
if self.args:
s = '%s: %s' % (s, ' '.join(self.args))
s = '%s.' % s
return s
class CannotListenError(BindError):
"""
This gets raised by a call to startListening, when the object cannotstart
listening.
@ivar interface: the interface I tried to listen on
@ivar port: the port I tried to listen on
@ivar socketError: the exception I got when I tried to listen
@type socketError: L{socket.error}
"""
def __init__(self, interface, port, socketError):
BindError.__init__(self, interface, port, socketError)
self.interface = interface
self.port = port
self.socketError = socketError
def __str__(self):
iface = self.interface or 'any'
return "Couldn't listen on %s:%s: %s." % (iface, self.port,
self.socketError)
class MulticastJoinError(Exception):
"""
An attempt to join a multicast group failed.
"""
class MessageLengthError(Exception):
"""Message is too long to send"""
def __str__(self):
s = self.__doc__
if self.args:
s = '%s: %s' % (s, ' '.join(self.args))
s = '%s.' % s
return s
class DNSLookupError(IOError):
"""DNS lookup failed"""
def __str__(self):
s = self.__doc__
if self.args:
s = '%s: %s' % (s, ' '.join(self.args))
s = '%s.' % s
return s
class ConnectInProgressError(Exception):
"""A connect operation was started and isn't done yet."""
# connection errors
class ConnectError(Exception):
"""An error occurred while connecting"""
def __init__(self, osError=None, string=""):
self.osError = osError
Exception.__init__(self, string)
def __str__(self):
s = self.__doc__ or self.__class__.__name__
if self.osError:
s = '%s: %s' % (s, self.osError)
if self.args[0]:
s = '%s: %s' % (s, self.args[0])
s = '%s.' % s
return s
class ConnectBindError(ConnectError):
"""Couldn't bind"""
class UnknownHostError(ConnectError):
"""Hostname couldn't be looked up"""
class NoRouteError(ConnectError):
"""No route to host"""
class ConnectionRefusedError(ConnectError):
"""Connection was refused by other side"""
class TCPTimedOutError(ConnectError):
"""TCP connection timed out"""
class BadFileError(ConnectError):
"""File used for UNIX socket is no good"""
class ServiceNameUnknownError(ConnectError):
"""Service name given as port is unknown"""
class UserError(ConnectError):
"""User aborted connection"""
class TimeoutError(UserError):
"""User timeout caused connection failure"""
class SSLError(ConnectError):
"""An SSL error occurred"""
class VerifyError(Exception):
"""Could not verify something that was supposed to be signed.
"""
class PeerVerifyError(VerifyError):
"""The peer rejected our verify error.
"""
class CertificateError(Exception):
"""
We did not find a certificate where we expected to find one.
"""
try:
import errno
errnoMapping = {
errno.ENETUNREACH: NoRouteError,
errno.ECONNREFUSED: ConnectionRefusedError,
errno.ETIMEDOUT: TCPTimedOutError,
}
if hasattr(errno, "WSAECONNREFUSED"):
errnoMapping[errno.WSAECONNREFUSED] = ConnectionRefusedError
errnoMapping[errno.WSAENETUNREACH] = NoRouteError
except ImportError:
errnoMapping = {}
def getConnectError(e):
"""Given a socket exception, return connection error."""
if isinstance(e, Exception):
args = e.args
else:
args = e
try:
number, string = args
except ValueError:
return ConnectError(string=e)
if hasattr(socket, 'gaierror') and isinstance(e, socket.gaierror):
# Only works in 2.2 in newer. Really that means always; #5978 covers
# this and other weirdnesses in this function.
klass = UnknownHostError
else:
klass = errnoMapping.get(number, ConnectError)
return klass(number, string)
class ConnectionClosed(Exception):
"""
Connection was closed, whether cleanly or non-cleanly.
"""
class ConnectionLost(ConnectionClosed):
"""Connection to the other side was lost in a non-clean fashion"""
def __str__(self):
s = self.__doc__.strip().splitlines()[0]
if self.args:
s = '%s: %s' % (s, ' '.join(self.args))
s = '%s.' % s
return s
class ConnectionAborted(ConnectionLost):
"""
Connection was aborted locally, using
L{twisted.internet.interfaces.ITCPTransport.abortConnection}.
@since: 11.1
"""
def __str__(self):
s = [(
"Connection was aborted locally using"
" ITCPTransport.abortConnection"
)]
if self.args:
s.append(': ')
s.append(' '.join(self.args))
s.append('.')
return ''.join(s)
class ConnectionDone(ConnectionClosed):
"""Connection was closed cleanly"""
def __str__(self):
s = self.__doc__
if self.args:
s = '%s: %s' % (s, ' '.join(self.args))
s = '%s.' % s
return s
class FileDescriptorOverrun(ConnectionLost):
"""
A mis-use of L{IUNIXTransport.sendFileDescriptor} caused the connection to
be closed.
Each file descriptor sent using C{sendFileDescriptor} must be associated
with at least one byte sent using L{ITransport.write}. If at any point
fewer bytes have been written than file descriptors have been sent, the
connection is closed with this exception.
"""
class ConnectionFdescWentAway(ConnectionLost):
"""Uh""" #TODO
class AlreadyCalled(ValueError):
"""Tried to cancel an already-called event"""
def __str__(self):
s = self.__doc__
if self.args:
s = '%s: %s' % (s, ' '.join(self.args))
s = '%s.' % s
return s
class AlreadyCancelled(ValueError):
"""Tried to cancel an already-cancelled event"""
def __str__(self):
s = self.__doc__
if self.args:
s = '%s: %s' % (s, ' '.join(self.args))
s = '%s.' % s
return s
class PotentialZombieWarning(Warning):
"""
Emitted when L{IReactorProcess.spawnProcess} is called in a way which may
result in termination of the created child process not being reported.
Deprecated in Twisted 10.0.
"""
MESSAGE = (
"spawnProcess called, but the SIGCHLD handler is not "
"installed. This probably means you have not yet "
"called reactor.run, or called "
"reactor.run(installSignalHandler=0). You will probably "
"never see this process finish, and it may become a "
"zombie process.")
deprecate.deprecatedModuleAttribute(
Version("Twisted", 10, 0, 0),
"There is no longer any potential for zombie process.",
__name__,
"PotentialZombieWarning")
class ProcessDone(ConnectionDone):
"""A process has ended without apparent errors"""
def __init__(self, status):
Exception.__init__(self, "process finished with exit code 0")
self.exitCode = 0
self.signal = None
self.status = status
class ProcessTerminated(ConnectionLost):
"""
A process has ended with a probable error condition
@ivar exitCode: See L{__init__}
@ivar signal: See L{__init__}
@ivar status: See L{__init__}
"""
def __init__(self, exitCode=None, signal=None, status=None):
"""
@param exitCode: The exit status of the process. This is roughly like
the value you might pass to L{os.exit}. This is L{None} if the
process exited due to a signal.
@type exitCode: L{int} or L{None}
@param signal: The exit signal of the process. This is L{None} if the
process did not exit due to a signal.
@type signal: L{int} or L{None}
@param status: The exit code of the process. This is a platform
specific combination of the exit code and the exit signal. See
L{os.WIFEXITED} and related functions.
@type status: L{int}
"""
self.exitCode = exitCode
self.signal = signal
self.status = status
s = "process ended"
if exitCode is not None: s = s + " with exit code %s" % exitCode
if signal is not None: s = s + " by signal %s" % signal
Exception.__init__(self, s)
class ProcessExitedAlready(Exception):
"""
The process has already exited and the operation requested can no longer
be performed.
"""
class NotConnectingError(RuntimeError):
"""The Connector was not connecting when it was asked to stop connecting"""
def __str__(self):
s = self.__doc__
if self.args:
s = '%s: %s' % (s, ' '.join(self.args))
s = '%s.' % s
return s
class NotListeningError(RuntimeError):
"""The Port was not listening when it was asked to stop listening"""
def __str__(self):
s = self.__doc__
if self.args:
s = '%s: %s' % (s, ' '.join(self.args))
s = '%s.' % s
return s
class ReactorNotRunning(RuntimeError):
"""
Error raised when trying to stop a reactor which is not running.
"""
class ReactorNotRestartable(RuntimeError):
"""
Error raised when trying to run a reactor which was stopped.
"""
class ReactorAlreadyRunning(RuntimeError):
"""
Error raised when trying to start the reactor multiple times.
"""
class ReactorAlreadyInstalledError(AssertionError):
"""
Could not install reactor because one is already installed.
"""
class ConnectingCancelledError(Exception):
"""
An C{Exception} that will be raised when an L{IStreamClientEndpoint} is
cancelled before it connects.
@ivar address: The L{IAddress} that is the destination of the
cancelled L{IStreamClientEndpoint}.
"""
def __init__(self, address):
"""
@param address: The L{IAddress} that is the destination of the
L{IStreamClientEndpoint} that was cancelled.
"""
Exception.__init__(self, address)
self.address = address
class NoProtocol(Exception):
"""
An C{Exception} that will be raised when the factory given to a
L{IStreamClientEndpoint} returns L{None} from C{buildProtocol}.
"""
class UnsupportedAddressFamily(Exception):
"""
An attempt was made to use a socket with an address family (eg I{AF_INET},
I{AF_INET6}, etc) which is not supported by the reactor.
"""
class UnsupportedSocketType(Exception):
"""
An attempt was made to use a socket of a type (eg I{SOCK_STREAM},
I{SOCK_DGRAM}, etc) which is not supported by the reactor.
"""
class AlreadyListened(Exception):
"""
An attempt was made to listen on a file descriptor which can only be
listened on once.
"""
class InvalidAddressError(ValueError):
"""
An invalid address was specified (i.e. neither IPv4 or IPv6, or expected
one and got the other).
@ivar address: See L{__init__}
@ivar message: See L{__init__}
"""
def __init__(self, address, message):
"""
@param address: The address that was provided.
@type address: L{bytes}
@param message: A native string of additional information provided by
the calling context.
@type address: L{str}
"""
self.address = address
self.message = message
__all__ = [
'BindError', 'CannotListenError', 'MulticastJoinError',
'MessageLengthError', 'DNSLookupError', 'ConnectInProgressError',
'ConnectError', 'ConnectBindError', 'UnknownHostError', 'NoRouteError',
'ConnectionRefusedError', 'TCPTimedOutError', 'BadFileError',
'ServiceNameUnknownError', 'UserError', 'TimeoutError', 'SSLError',
'VerifyError', 'PeerVerifyError', 'CertificateError',
'getConnectError', 'ConnectionClosed', 'ConnectionLost',
'ConnectionDone', 'ConnectionFdescWentAway', 'AlreadyCalled',
'AlreadyCancelled', 'PotentialZombieWarning', 'ProcessDone',
'ProcessTerminated', 'ProcessExitedAlready', 'NotConnectingError',
'NotListeningError', 'ReactorNotRunning', 'ReactorAlreadyRunning',
'ReactorAlreadyInstalledError', 'ConnectingCancelledError',
'UnsupportedAddressFamily', 'UnsupportedSocketType', 'InvalidAddressError']
@@ -0,0 +1,118 @@
# -*- test-case-name: twisted.test.test_fdesc -*-
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
Utility functions for dealing with POSIX file descriptors.
"""
import os
import errno
try:
import fcntl
except ImportError:
fcntl = None
# twisted imports
from twisted.internet.main import CONNECTION_LOST, CONNECTION_DONE
def setNonBlocking(fd):
"""
Set the file description of the given file descriptor to non-blocking.
"""
flags = fcntl.fcntl(fd, fcntl.F_GETFL)
flags = flags | os.O_NONBLOCK
fcntl.fcntl(fd, fcntl.F_SETFL, flags)
def setBlocking(fd):
"""
Set the file description of the given file descriptor to blocking.
"""
flags = fcntl.fcntl(fd, fcntl.F_GETFL)
flags = flags & ~os.O_NONBLOCK
fcntl.fcntl(fd, fcntl.F_SETFL, flags)
if fcntl is None:
# fcntl isn't available on Windows. By default, handles aren't
# inherited on Windows, so we can do nothing here.
_setCloseOnExec = _unsetCloseOnExec = lambda fd: None
else:
def _setCloseOnExec(fd):
"""
Make a file descriptor close-on-exec.
"""
flags = fcntl.fcntl(fd, fcntl.F_GETFD)
flags = flags | fcntl.FD_CLOEXEC
fcntl.fcntl(fd, fcntl.F_SETFD, flags)
def _unsetCloseOnExec(fd):
"""
Make a file descriptor close-on-exec.
"""
flags = fcntl.fcntl(fd, fcntl.F_GETFD)
flags = flags & ~fcntl.FD_CLOEXEC
fcntl.fcntl(fd, fcntl.F_SETFD, flags)
def readFromFD(fd, callback):
"""
Read from file descriptor, calling callback with resulting data.
If successful, call 'callback' with a single argument: the
resulting data.
Returns same thing FileDescriptor.doRead would: CONNECTION_LOST,
CONNECTION_DONE, or None.
@type fd: C{int}
@param fd: non-blocking file descriptor to be read from.
@param callback: a callable which accepts a single argument. If
data is read from the file descriptor it will be called with this
data. Handling exceptions from calling the callback is up to the
caller.
Note that if the descriptor is still connected but no data is read,
None will be returned but callback will not be called.
@return: CONNECTION_LOST on error, CONNECTION_DONE when fd is
closed, otherwise None.
"""
try:
output = os.read(fd, 8192)
except (OSError, IOError) as ioe:
if ioe.args[0] in (errno.EAGAIN, errno.EINTR):
return
else:
return CONNECTION_LOST
if not output:
return CONNECTION_DONE
callback(output)
def writeToFD(fd, data):
"""
Write data to file descriptor.
Returns same thing FileDescriptor.writeSomeData would.
@type fd: C{int}
@param fd: non-blocking file descriptor to be written to.
@type data: C{str} or C{buffer}
@param data: bytes to write to fd.
@return: number of bytes written, or CONNECTION_LOST.
"""
try:
return os.write(fd, data)
except (OSError, IOError) as io:
if io.errno in (errno.EAGAIN, errno.EINTR):
return 0
return CONNECTION_LOST
__all__ = ["setNonBlocking", "setBlocking", "readFromFD", "writeToFD"]
@@ -0,0 +1,188 @@
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
This module provides support for Twisted to interact with the glib
mainloop via GObject Introspection.
In order to use this support, simply do the following::
from twisted.internet import gireactor
gireactor.install()
If you wish to use a GApplication, register it with the reactor::
from twisted.internet import reactor
reactor.registerGApplication(app)
Then use twisted.internet APIs as usual.
On Python 3, pygobject v3.4 or later is required.
"""
from __future__ import division, absolute_import
from twisted.python.compat import _PY3
from twisted.internet.error import ReactorAlreadyRunning
from twisted.internet import _glibbase
from twisted.python import runtime
if _PY3:
# We require a sufficiently new version of pygobject, so always exists:
_pygtkcompatPresent = True
else:
# We can't just try to import gi.pygtkcompat, because that would import
# gi, and the goal here is to not import gi in cases where that would
# cause segfault.
from twisted.python.modules import theSystemPath
_pygtkcompatPresent = True
try:
theSystemPath["gi.pygtkcompat"]
except KeyError:
_pygtkcompatPresent = False
# Modules that we want to ensure aren't imported if we're on older versions of
# GI:
_PYGTK_MODULES = ['gobject', 'glib', 'gio', 'gtk']
def _oldGiInit():
"""
Make sure pygtk and gi aren't loaded at the same time, and import Glib if
possible.
"""
# We can't immediately prevent imports, because that confuses some buggy
# code in gi:
_glibbase.ensureNotImported(
_PYGTK_MODULES,
"Introspected and static glib/gtk bindings must not be mixed; can't "
"import gireactor since pygtk2 module is already imported.")
global GLib
from gi.repository import GLib
if getattr(GLib, "threads_init", None) is not None:
GLib.threads_init()
_glibbase.ensureNotImported([], "",
preventImports=_PYGTK_MODULES)
if not _pygtkcompatPresent:
# Older versions of gi don't have compatibility layer, so just enforce no
# imports of pygtk and gi at same time:
_oldGiInit()
else:
# Newer version of gi, so we can try to initialize compatibility layer; if
# real pygtk was already imported we'll get ImportError at this point
# rather than segfault, so unconditional import is fine.
import gi.pygtkcompat
gi.pygtkcompat.enable()
# At this point importing gobject will get you gi version, and importing
# e.g. gtk will either fail in non-segfaulty way or use gi version if user
# does gi.pygtkcompat.enable_gtk(). So, no need to prevent imports of
# old school pygtk modules.
from gi.repository import GLib
if getattr(GLib, "threads_init", None) is not None:
GLib.threads_init()
class GIReactor(_glibbase.GlibReactorBase):
"""
GObject-introspection event loop reactor.
@ivar _gapplication: A C{Gio.Application} instance that was registered
with C{registerGApplication}.
"""
_POLL_DISCONNECTED = (GLib.IOCondition.HUP | GLib.IOCondition.ERR |
GLib.IOCondition.NVAL)
_POLL_IN = GLib.IOCondition.IN
_POLL_OUT = GLib.IOCondition.OUT
# glib's iochannel sources won't tell us about any events that we haven't
# asked for, even if those events aren't sensible inputs to the poll()
# call.
INFLAGS = _POLL_IN | _POLL_DISCONNECTED
OUTFLAGS = _POLL_OUT | _POLL_DISCONNECTED
# By default no Application is registered:
_gapplication = None
def __init__(self, useGtk=False):
_gtk = None
if useGtk is True:
from gi.repository import Gtk as _gtk
_glibbase.GlibReactorBase.__init__(self, GLib, _gtk, useGtk=useGtk)
def registerGApplication(self, app):
"""
Register a C{Gio.Application} or C{Gtk.Application}, whose main loop
will be used instead of the default one.
We will C{hold} the application so it doesn't exit on its own. In
versions of C{python-gi} 3.2 and later, we exit the event loop using
the C{app.quit} method which overrides any holds. Older versions are
not supported.
"""
if self._gapplication is not None:
raise RuntimeError(
"Can't register more than one application instance.")
if self._started:
raise ReactorAlreadyRunning(
"Can't register application after reactor was started.")
if not hasattr(app, "quit"):
raise RuntimeError("Application registration is not supported in"
" versions of PyGObject prior to 3.2.")
self._gapplication = app
def run():
app.hold()
app.run(None)
self._run = run
self._crash = app.quit
class PortableGIReactor(_glibbase.PortableGlibReactorBase):
"""
Portable GObject Introspection event loop reactor.
"""
def __init__(self, useGtk=False):
_gtk = None
if useGtk is True:
from gi.repository import Gtk as _gtk
_glibbase.PortableGlibReactorBase.__init__(self, GLib, _gtk,
useGtk=useGtk)
def registerGApplication(self, app):
"""
Register a C{Gio.Application} or C{Gtk.Application}, whose main loop
will be used instead of the default one.
"""
raise NotImplementedError("GApplication is not currently supported on Windows.")
def install(useGtk=False):
"""
Configure the twisted mainloop to be run inside the glib mainloop.
@param useGtk: should GTK+ rather than glib event loop be
used (this will be slightly slower but does support GUI).
"""
if runtime.platform.getType() == 'posix':
reactor = GIReactor(useGtk=useGtk)
else:
reactor = PortableGIReactor(useGtk=useGtk)
from twisted.internet.main import installReactor
installReactor(reactor)
return reactor
__all__ = ['install']
@@ -0,0 +1,10 @@
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
I/O Completion Ports reactor
"""
from twisted.internet.iocpreactor.reactor import install
__all__ = ['install']
@@ -0,0 +1,399 @@
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
Abstract file handle class
"""
from twisted.internet import main, error, interfaces
from twisted.internet.abstract import _ConsumerMixin, _LogOwner
from twisted.python import failure
from twisted.python.compat import unicode
from zope.interface import implementer
import errno
from twisted.internet.iocpreactor.const import ERROR_HANDLE_EOF
from twisted.internet.iocpreactor.const import ERROR_IO_PENDING
from twisted.internet.iocpreactor import iocpsupport as _iocp
@implementer(interfaces.IPushProducer, interfaces.IConsumer,
interfaces.ITransport, interfaces.IHalfCloseableDescriptor)
class FileHandle(_ConsumerMixin, _LogOwner):
"""
File handle that can read and write asynchronously
"""
# read stuff
maxReadBuffers = 16
readBufferSize = 4096
reading = False
dynamicReadBuffers = True # set this to false if subclass doesn't do iovecs
_readNextBuffer = 0
_readSize = 0 # how much data we have in the read buffer
_readScheduled = None
_readScheduledInOS = False
def startReading(self):
self.reactor.addActiveHandle(self)
if not self._readScheduled and not self.reading:
self.reading = True
self._readScheduled = self.reactor.callLater(0,
self._resumeReading)
def stopReading(self):
if self._readScheduled:
self._readScheduled.cancel()
self._readScheduled = None
self.reading = False
def _resumeReading(self):
self._readScheduled = None
if self._dispatchData() and not self._readScheduledInOS:
self.doRead()
def _dispatchData(self):
"""
Dispatch previously read data. Return True if self.reading and we don't
have any more data
"""
if not self._readSize:
return self.reading
size = self._readSize
full_buffers = size // self.readBufferSize
while self._readNextBuffer < full_buffers:
self.dataReceived(self._readBuffers[self._readNextBuffer])
self._readNextBuffer += 1
if not self.reading:
return False
remainder = size % self.readBufferSize
if remainder:
self.dataReceived(self._readBuffers[full_buffers][0:remainder])
if self.dynamicReadBuffers:
total_buffer_size = self.readBufferSize * len(self._readBuffers)
# we have one buffer too many
if size < total_buffer_size - self.readBufferSize:
del self._readBuffers[-1]
# we filled all buffers, so allocate one more
elif (size == total_buffer_size and
len(self._readBuffers) < self.maxReadBuffers):
self._readBuffers.append(bytearray(self.readBufferSize))
self._readNextBuffer = 0
self._readSize = 0
return self.reading
def _cbRead(self, rc, data, evt):
self._readScheduledInOS = False
if self._handleRead(rc, data, evt):
self.doRead()
def _handleRead(self, rc, data, evt):
"""
Returns False if we should stop reading for now
"""
if self.disconnected:
return False
# graceful disconnection
if (not (rc or data)) or rc in (errno.WSAEDISCON, ERROR_HANDLE_EOF):
self.reactor.removeActiveHandle(self)
self.readConnectionLost(failure.Failure(main.CONNECTION_DONE))
return False
# XXX: not handling WSAEWOULDBLOCK
# ("too many outstanding overlapped I/O requests")
elif rc:
self.connectionLost(failure.Failure(
error.ConnectionLost("read error -- %s (%s)" %
(errno.errorcode.get(rc, 'unknown'), rc))))
return False
else:
assert self._readSize == 0
assert self._readNextBuffer == 0
self._readSize = data
return self._dispatchData()
def doRead(self):
evt = _iocp.Event(self._cbRead, self)
evt.buff = buff = self._readBuffers
rc, numBytesRead = self.readFromHandle(buff, evt)
if not rc or rc == ERROR_IO_PENDING:
self._readScheduledInOS = True
else:
self._handleRead(rc, numBytesRead, evt)
def readFromHandle(self, bufflist, evt):
raise NotImplementedError() # TODO: this should default to ReadFile
def dataReceived(self, data):
raise NotImplementedError
def readConnectionLost(self, reason):
self.connectionLost(reason)
# write stuff
dataBuffer = b''
offset = 0
writing = False
_writeScheduled = None
_writeDisconnecting = False
_writeDisconnected = False
writeBufferSize = 2**2**2**2
def loseWriteConnection(self):
self._writeDisconnecting = True
self.startWriting()
def _closeWriteConnection(self):
# override in subclasses
pass
def writeConnectionLost(self, reason):
# in current code should never be called
self.connectionLost(reason)
def startWriting(self):
self.reactor.addActiveHandle(self)
if not self._writeScheduled and not self.writing:
self.writing = True
self._writeScheduled = self.reactor.callLater(0,
self._resumeWriting)
def stopWriting(self):
if self._writeScheduled:
self._writeScheduled.cancel()
self._writeScheduled = None
self.writing = False
def _resumeWriting(self):
self._writeScheduled = None
self.doWrite()
def _cbWrite(self, rc, numBytesWritten, evt):
if self._handleWrite(rc, numBytesWritten, evt):
self.doWrite()
def _handleWrite(self, rc, numBytesWritten, evt):
"""
Returns false if we should stop writing for now
"""
if self.disconnected or self._writeDisconnected:
return False
# XXX: not handling WSAEWOULDBLOCK
# ("too many outstanding overlapped I/O requests")
if rc:
self.connectionLost(failure.Failure(
error.ConnectionLost("write error -- %s (%s)" %
(errno.errorcode.get(rc, 'unknown'), rc))))
return False
else:
self.offset += numBytesWritten
# If there is nothing left to send,
if self.offset == len(self.dataBuffer) and not self._tempDataLen:
self.dataBuffer = b""
self.offset = 0
# stop writing
self.stopWriting()
# If I've got a producer who is supposed to supply me with data
if self.producer is not None and ((not self.streamingProducer)
or self.producerPaused):
# tell them to supply some more.
self.producerPaused = True
self.producer.resumeProducing()
elif self.disconnecting:
# But if I was previously asked to let the connection die,
# do so.
self.connectionLost(failure.Failure(main.CONNECTION_DONE))
elif self._writeDisconnecting:
# I was previously asked to half-close the connection.
self._writeDisconnected = True
self._closeWriteConnection()
return False
else:
return True
def doWrite(self):
if len(self.dataBuffer) - self.offset < self.SEND_LIMIT:
# If there is currently less than SEND_LIMIT bytes left to send
# in the string, extend it with the array data.
self.dataBuffer = (self.dataBuffer[self.offset:] +
b"".join(self._tempDataBuffer))
self.offset = 0
self._tempDataBuffer = []
self._tempDataLen = 0
evt = _iocp.Event(self._cbWrite, self)
# Send as much data as you can.
if self.offset:
sendView = memoryview(self.dataBuffer)
evt.buff = buff = sendView[self.offset:]
else:
evt.buff = buff = self.dataBuffer
rc, data = self.writeToHandle(buff, evt)
if rc and rc != ERROR_IO_PENDING:
self._handleWrite(rc, data, evt)
def writeToHandle(self, buff, evt):
raise NotImplementedError() # TODO: this should default to WriteFile
def write(self, data):
"""Reliably write some data.
The data is buffered until his file descriptor is ready for writing.
"""
if isinstance(data, unicode): # no, really, I mean it
raise TypeError("Data must not be unicode")
if not self.connected or self._writeDisconnected:
return
if data:
self._tempDataBuffer.append(data)
self._tempDataLen += len(data)
if self.producer is not None and self.streamingProducer:
if (len(self.dataBuffer) + self._tempDataLen
> self.writeBufferSize):
self.producerPaused = True
self.producer.pauseProducing()
self.startWriting()
def writeSequence(self, iovec):
for i in iovec:
if isinstance(i, unicode): # no, really, I mean it
raise TypeError("Data must not be unicode")
if not self.connected or not iovec or self._writeDisconnected:
return
self._tempDataBuffer.extend(iovec)
for i in iovec:
self._tempDataLen += len(i)
if self.producer is not None and self.streamingProducer:
if len(self.dataBuffer) + self._tempDataLen > self.writeBufferSize:
self.producerPaused = True
self.producer.pauseProducing()
self.startWriting()
# general stuff
connected = False
disconnected = False
disconnecting = False
logstr = "Uninitialized"
SEND_LIMIT = 128*1024
def __init__(self, reactor = None):
if not reactor:
from twisted.internet import reactor
self.reactor = reactor
self._tempDataBuffer = [] # will be added to dataBuffer in doWrite
self._tempDataLen = 0
self._readBuffers = [bytearray(self.readBufferSize)]
def connectionLost(self, reason):
"""
The connection was lost.
This is called when the connection on a selectable object has been
lost. It will be called whether the connection was closed explicitly,
an exception occurred in an event handler, or the other end of the
connection closed it first.
Clean up state here, but make sure to call back up to FileDescriptor.
"""
self.disconnected = True
self.connected = False
if self.producer is not None:
self.producer.stopProducing()
self.producer = None
self.stopReading()
self.stopWriting()
self.reactor.removeActiveHandle(self)
def getFileHandle(self):
return -1
def loseConnection(self, _connDone=failure.Failure(main.CONNECTION_DONE)):
"""
Close the connection at the next available opportunity.
Call this to cause this FileDescriptor to lose its connection. It will
first write any data that it has buffered.
If there is data buffered yet to be written, this method will cause the
transport to lose its connection as soon as it's done flushing its
write buffer. If you have a producer registered, the connection won't
be closed until the producer is finished. Therefore, make sure you
unregister your producer when it's finished, or the connection will
never close.
"""
if self.connected and not self.disconnecting:
if self._writeDisconnected:
# doWrite won't trigger the connection close anymore
self.stopReading()
self.stopWriting
self.connectionLost(_connDone)
else:
self.stopReading()
self.startWriting()
self.disconnecting = 1
# Producer/consumer implementation
def stopConsuming(self):
"""
Stop consuming data.
This is called when a producer has lost its connection, to tell the
consumer to go lose its connection (and break potential circular
references).
"""
self.unregisterProducer()
self.loseConnection()
# producer interface implementation
def resumeProducing(self):
if self.connected and not self.disconnecting:
self.startReading()
def pauseProducing(self):
self.stopReading()
def stopProducing(self):
self.loseConnection()
__all__ = ['FileHandle']
@@ -0,0 +1,23 @@
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
Distutils file for building low-level IOCP bindings from their Pyrex source
"""
from distutils.core import setup
from distutils.extension import Extension
from Cython.Distutils import build_ext
setup(name='iocpsupport',
ext_modules=[Extension('iocpsupport',
['iocpsupport/iocpsupport.pyx',
'iocpsupport/winsock_pointers.c'],
libraries = ['ws2_32'],
)
],
cmdclass = {'build_ext': build_ext},
)
@@ -0,0 +1,37 @@
# -*- test-case-name: twisted.internet.test.test_main -*-
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
Backwards compatibility, and utility functions.
In general, this module should not be used, other than by reactor authors
who need to use the 'installReactor' method.
"""
from __future__ import division, absolute_import
from twisted.internet import error
CONNECTION_DONE = error.ConnectionDone('Connection done')
CONNECTION_LOST = error.ConnectionLost('Connection lost')
def installReactor(reactor):
"""
Install reactor C{reactor}.
@param reactor: An object that provides one or more IReactor* interfaces.
"""
# this stuff should be common to all reactors.
import twisted.internet
import sys
if 'twisted.internet.reactor' in sys.modules:
raise error.ReactorAlreadyInstalledError("reactor already installed")
twisted.internet.reactor = reactor
sys.modules['twisted.internet.reactor'] = reactor
__all__ = ["CONNECTION_LOST", "CONNECTION_DONE", "installReactor"]
@@ -0,0 +1,37 @@
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
This module integrates PyUI with twisted.internet's mainloop.
Maintainer: Jp Calderone
See doc/examples/pyuidemo.py for example usage.
"""
# System imports
import pyui
def _guiUpdate(reactor, delay):
pyui.draw()
if pyui.update() == 0:
pyui.quit()
reactor.stop()
else:
reactor.callLater(delay, _guiUpdate, reactor, delay)
def install(ms=10, reactor=None, args=(), kw={}):
"""
Schedule PyUI's display to be updated approximately every C{ms}
milliseconds, and initialize PyUI with the specified arguments.
"""
d = pyui.init(*args, **kw)
if reactor is None:
from twisted.internet import reactor
_guiUpdate(reactor, ms / 1000.0)
return d
__all__ = ["install"]
@@ -0,0 +1,39 @@
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
The reactor is the Twisted event loop within Twisted, the loop which drives
applications using Twisted. The reactor provides APIs for networking,
threading, dispatching events, and more.
The default reactor depends on the platform and will be installed if this
module is imported without another reactor being explicitly installed
beforehand. Regardless of which reactor is installed, importing this module is
the correct way to get a reference to it.
New application code should prefer to pass and accept the reactor as a
parameter where it is needed, rather than relying on being able to import this
module to get a reference. This simplifies unit testing and may make it easier
to one day support multiple reactors (as a performance enhancement), though
this is not currently possible.
@see: L{IReactorCore<twisted.internet.interfaces.IReactorCore>}
@see: L{IReactorTime<twisted.internet.interfaces.IReactorTime>}
@see: L{IReactorProcess<twisted.internet.interfaces.IReactorProcess>}
@see: L{IReactorTCP<twisted.internet.interfaces.IReactorTCP>}
@see: L{IReactorSSL<twisted.internet.interfaces.IReactorSSL>}
@see: L{IReactorUDP<twisted.internet.interfaces.IReactorUDP>}
@see: L{IReactorMulticast<twisted.internet.interfaces.IReactorMulticast>}
@see: L{IReactorUNIX<twisted.internet.interfaces.IReactorUNIX>}
@see: L{IReactorUNIXDatagram<twisted.internet.interfaces.IReactorUNIXDatagram>}
@see: L{IReactorFDSet<twisted.internet.interfaces.IReactorFDSet>}
@see: L{IReactorThreads<twisted.internet.interfaces.IReactorThreads>}
@see: L{IReactorPluggableResolver<twisted.internet.interfaces.IReactorPluggableResolver>}
"""
from __future__ import division, absolute_import
import sys
del sys.modules['twisted.internet.reactor']
from twisted.internet import default
default.install()
@@ -0,0 +1,948 @@
# -*- test-case-name: twisted.test.test_task,twisted.test.test_cooperator -*-
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
Scheduling utility methods and classes.
"""
from __future__ import division, absolute_import
__metaclass__ = type
import sys
import time
import warnings
from zope.interface import implementer
from twisted.python import log
from twisted.python import reflect
from twisted.python.deprecate import _getDeprecationWarningString
from twisted.python.failure import Failure
from incremental import Version
from twisted.internet import base, defer
from twisted.internet.interfaces import IReactorTime
from twisted.internet.error import ReactorNotRunning
class LoopingCall:
"""Call a function repeatedly.
If C{f} returns a deferred, rescheduling will not take place until the
deferred has fired. The result value is ignored.
@ivar f: The function to call.
@ivar a: A tuple of arguments to pass the function.
@ivar kw: A dictionary of keyword arguments to pass to the function.
@ivar clock: A provider of
L{twisted.internet.interfaces.IReactorTime}. The default is
L{twisted.internet.reactor}. Feel free to set this to
something else, but it probably ought to be set *before*
calling L{start}.
@type running: C{bool}
@ivar running: A flag which is C{True} while C{f} is scheduled to be called
(or is currently being called). It is set to C{True} when L{start} is
called and set to C{False} when L{stop} is called or if C{f} raises an
exception. In either case, it will be C{False} by the time the
C{Deferred} returned by L{start} fires its callback or errback.
@type _realLastTime: C{float}
@ivar _realLastTime: When counting skips, the time at which the skip
counter was last invoked.
@type _runAtStart: C{bool}
@ivar _runAtStart: A flag indicating whether the 'now' argument was passed
to L{LoopingCall.start}.
"""
call = None
running = False
_deferred = None
interval = None
_runAtStart = False
starttime = None
def __init__(self, f, *a, **kw):
self.f = f
self.a = a
self.kw = kw
from twisted.internet import reactor
self.clock = reactor
@property
def deferred(self):
"""
DEPRECATED. L{Deferred} fired when loop stops or fails.
Use the L{Deferred} returned by L{LoopingCall.start}.
"""
warningString = _getDeprecationWarningString(
"twisted.internet.task.LoopingCall.deferred",
Version("Twisted", 16, 0, 0),
replacement='the deferred returned by start()')
warnings.warn(warningString, DeprecationWarning, stacklevel=2)
return self._deferred
def withCount(cls, countCallable):
"""
An alternate constructor for L{LoopingCall} that makes available the
number of calls which should have occurred since it was last invoked.
Note that this number is an C{int} value; It represents the discrete
number of calls that should have been made. For example, if you are
using a looping call to display an animation with discrete frames, this
number would be the number of frames to advance.
The count is normally 1, but can be higher. For example, if the reactor
is blocked and takes too long to invoke the L{LoopingCall}, a Deferred
returned from a previous call is not fired before an interval has
elapsed, or if the callable itself blocks for longer than an interval,
preventing I{itself} from being called.
When running with an interval if 0, count will be always 1.
@param countCallable: A callable that will be invoked each time the
resulting LoopingCall is run, with an integer specifying the number
of calls that should have been invoked.
@type countCallable: 1-argument callable which takes an C{int}
@return: An instance of L{LoopingCall} with call counting enabled,
which provides the count as the first positional argument.
@rtype: L{LoopingCall}
@since: 9.0
"""
def counter():
now = self.clock.seconds()
if self.interval == 0:
self._realLastTime = now
return countCallable(1)
lastTime = self._realLastTime
if lastTime is None:
lastTime = self.starttime
if self._runAtStart:
lastTime -= self.interval
lastInterval = self._intervalOf(lastTime)
thisInterval = self._intervalOf(now)
count = thisInterval - lastInterval
if count > 0:
self._realLastTime = now
return countCallable(count)
self = cls(counter)
self._realLastTime = None
return self
withCount = classmethod(withCount)
def _intervalOf(self, t):
"""
Determine the number of intervals passed as of the given point in
time.
@param t: The specified time (from the start of the L{LoopingCall}) to
be measured in intervals
@return: The C{int} number of intervals which have passed as of the
given point in time.
"""
elapsedTime = t - self.starttime
intervalNum = int(elapsedTime / self.interval)
return intervalNum
def start(self, interval, now=True):
"""
Start running function every interval seconds.
@param interval: The number of seconds between calls. May be
less than one. Precision will depend on the underlying
platform, the available hardware, and the load on the system.
@param now: If True, run this call right now. Otherwise, wait
until the interval has elapsed before beginning.
@return: A Deferred whose callback will be invoked with
C{self} when C{self.stop} is called, or whose errback will be
invoked when the function raises an exception or returned a
deferred that has its errback invoked.
"""
assert not self.running, ("Tried to start an already running "
"LoopingCall.")
if interval < 0:
raise ValueError("interval must be >= 0")
self.running = True
# Loop might fail to start and then self._deferred will be cleared.
# This why the local C{deferred} variable is used.
deferred = self._deferred = defer.Deferred()
self.starttime = self.clock.seconds()
self.interval = interval
self._runAtStart = now
if now:
self()
else:
self._scheduleFrom(self.starttime)
return deferred
def stop(self):
"""Stop running function.
"""
assert self.running, ("Tried to stop a LoopingCall that was "
"not running.")
self.running = False
if self.call is not None:
self.call.cancel()
self.call = None
d, self._deferred = self._deferred, None
d.callback(self)
def reset(self):
"""
Skip the next iteration and reset the timer.
@since: 11.1
"""
assert self.running, ("Tried to reset a LoopingCall that was "
"not running.")
if self.call is not None:
self.call.cancel()
self.call = None
self.starttime = self.clock.seconds()
self._scheduleFrom(self.starttime)
def __call__(self):
def cb(result):
if self.running:
self._scheduleFrom(self.clock.seconds())
else:
d, self._deferred = self._deferred, None
d.callback(self)
def eb(failure):
self.running = False
d, self._deferred = self._deferred, None
d.errback(failure)
self.call = None
d = defer.maybeDeferred(self.f, *self.a, **self.kw)
d.addCallback(cb)
d.addErrback(eb)
def _scheduleFrom(self, when):
"""
Schedule the next iteration of this looping call.
@param when: The present time from whence the call is scheduled.
"""
def howLong():
# How long should it take until the next invocation of our
# callable? Split out into a function because there are multiple
# places we want to 'return' out of this.
if self.interval == 0:
# If the interval is 0, just go as fast as possible, always
# return zero, call ourselves ASAP.
return 0
# Compute the time until the next interval; how long has this call
# been running for?
runningFor = when - self.starttime
# And based on that start time, when does the current interval end?
untilNextInterval = self.interval - (runningFor % self.interval)
# Now that we know how long it would be, we have to tell if the
# number is effectively zero. However, we can't just test against
# zero. If a number with a small exponent is added to a number
# with a large exponent, it may be so small that the digits just
# fall off the end, which means that adding the increment makes no
# difference; it's time to tick over into the next interval.
if when == when + untilNextInterval:
# If it's effectively zero, then we need to add another
# interval.
return self.interval
# Finally, if everything else is normal, we just return the
# computed delay.
return untilNextInterval
self.call = self.clock.callLater(howLong(), self)
def __repr__(self):
if hasattr(self.f, '__qualname__'):
func = self.f.__qualname__
elif hasattr(self.f, '__name__'):
func = self.f.__name__
if hasattr(self.f, 'im_class'):
func = self.f.im_class.__name__ + '.' + func
else:
func = reflect.safe_repr(self.f)
return 'LoopingCall<%r>(%s, *%s, **%s)' % (
self.interval, func, reflect.safe_repr(self.a),
reflect.safe_repr(self.kw))
class SchedulerError(Exception):
"""
The operation could not be completed because the scheduler or one of its
tasks was in an invalid state. This exception should not be raised
directly, but is a superclass of various scheduler-state-related
exceptions.
"""
class SchedulerStopped(SchedulerError):
"""
The operation could not complete because the scheduler was stopped in
progress or was already stopped.
"""
class TaskFinished(SchedulerError):
"""
The operation could not complete because the task was already completed,
stopped, encountered an error or otherwise permanently stopped running.
"""
class TaskDone(TaskFinished):
"""
The operation could not complete because the task was already completed.
"""
class TaskStopped(TaskFinished):
"""
The operation could not complete because the task was stopped.
"""
class TaskFailed(TaskFinished):
"""
The operation could not complete because the task died with an unhandled
error.
"""
class NotPaused(SchedulerError):
"""
This exception is raised when a task is resumed which was not previously
paused.
"""
class _Timer(object):
MAX_SLICE = 0.01
def __init__(self):
self.end = time.time() + self.MAX_SLICE
def __call__(self):
return time.time() >= self.end
_EPSILON = 0.00000001
def _defaultScheduler(x):
from twisted.internet import reactor
return reactor.callLater(_EPSILON, x)
class CooperativeTask(object):
"""
A L{CooperativeTask} is a task object inside a L{Cooperator}, which can be
paused, resumed, and stopped. It can also have its completion (or
termination) monitored.
@see: L{Cooperator.cooperate}
@ivar _iterator: the iterator to iterate when this L{CooperativeTask} is
asked to do work.
@ivar _cooperator: the L{Cooperator} that this L{CooperativeTask}
participates in, which is used to re-insert it upon resume.
@ivar _deferreds: the list of L{defer.Deferred}s to fire when this task
completes, fails, or finishes.
@type _deferreds: C{list}
@type _cooperator: L{Cooperator}
@ivar _pauseCount: the number of times that this L{CooperativeTask} has
been paused; if 0, it is running.
@type _pauseCount: C{int}
@ivar _completionState: The completion-state of this L{CooperativeTask}.
L{None} if the task is not yet completed, an instance of L{TaskStopped}
if C{stop} was called to stop this task early, of L{TaskFailed} if the
application code in the iterator raised an exception which caused it to
terminate, and of L{TaskDone} if it terminated normally via raising
C{StopIteration}.
@type _completionState: L{TaskFinished}
"""
def __init__(self, iterator, cooperator):
"""
A private constructor: to create a new L{CooperativeTask}, see
L{Cooperator.cooperate}.
"""
self._iterator = iterator
self._cooperator = cooperator
self._deferreds = []
self._pauseCount = 0
self._completionState = None
self._completionResult = None
cooperator._addTask(self)
def whenDone(self):
"""
Get a L{defer.Deferred} notification of when this task is complete.
@return: a L{defer.Deferred} that fires with the C{iterator} that this
L{CooperativeTask} was created with when the iterator has been
exhausted (i.e. its C{next} method has raised C{StopIteration}), or
fails with the exception raised by C{next} if it raises some other
exception.
@rtype: L{defer.Deferred}
"""
d = defer.Deferred()
if self._completionState is None:
self._deferreds.append(d)
else:
d.callback(self._completionResult)
return d
def pause(self):
"""
Pause this L{CooperativeTask}. Stop doing work until
L{CooperativeTask.resume} is called. If C{pause} is called more than
once, C{resume} must be called an equal number of times to resume this
task.
@raise TaskFinished: if this task has already finished or completed.
"""
self._checkFinish()
self._pauseCount += 1
if self._pauseCount == 1:
self._cooperator._removeTask(self)
def resume(self):
"""
Resume processing of a paused L{CooperativeTask}.
@raise NotPaused: if this L{CooperativeTask} is not paused.
"""
if self._pauseCount == 0:
raise NotPaused()
self._pauseCount -= 1
if self._pauseCount == 0 and self._completionState is None:
self._cooperator._addTask(self)
def _completeWith(self, completionState, deferredResult):
"""
@param completionState: a L{TaskFinished} exception or a subclass
thereof, indicating what exception should be raised when subsequent
operations are performed.
@param deferredResult: the result to fire all the deferreds with.
"""
self._completionState = completionState
self._completionResult = deferredResult
if not self._pauseCount:
self._cooperator._removeTask(self)
# The Deferreds need to be invoked after all this is completed, because
# a Deferred may want to manipulate other tasks in a Cooperator. For
# example, if you call "stop()" on a cooperator in a callback on a
# Deferred returned from whenDone(), this CooperativeTask must be gone
# from the Cooperator by that point so that _completeWith is not
# invoked reentrantly; that would cause these Deferreds to blow up with
# an AlreadyCalledError, or the _removeTask to fail with a ValueError.
for d in self._deferreds:
d.callback(deferredResult)
def stop(self):
"""
Stop further processing of this task.
@raise TaskFinished: if this L{CooperativeTask} has previously
completed, via C{stop}, completion, or failure.
"""
self._checkFinish()
self._completeWith(TaskStopped(), Failure(TaskStopped()))
def _checkFinish(self):
"""
If this task has been stopped, raise the appropriate subclass of
L{TaskFinished}.
"""
if self._completionState is not None:
raise self._completionState
def _oneWorkUnit(self):
"""
Perform one unit of work for this task, retrieving one item from its
iterator, stopping if there are no further items in the iterator, and
pausing if the result was a L{defer.Deferred}.
"""
try:
result = next(self._iterator)
except StopIteration:
self._completeWith(TaskDone(), self._iterator)
except:
self._completeWith(TaskFailed(), Failure())
else:
if isinstance(result, defer.Deferred):
self.pause()
def failLater(f):
self._completeWith(TaskFailed(), f)
result.addCallbacks(lambda result: self.resume(),
failLater)
class Cooperator(object):
"""
Cooperative task scheduler.
A cooperative task is an iterator where each iteration represents an
atomic unit of work. When the iterator yields, it allows the
L{Cooperator} to decide which of its tasks to execute next. If the
iterator yields a L{defer.Deferred} then work will pause until the
L{defer.Deferred} fires and completes its callback chain.
When a L{Cooperator} has more than one task, it distributes work between
all tasks.
There are two ways to add tasks to a L{Cooperator}, L{cooperate} and
L{coiterate}. L{cooperate} is the more useful of the two, as it returns a
L{CooperativeTask}, which can be L{paused<CooperativeTask.pause>},
L{resumed<CooperativeTask.resume>} and L{waited
on<CooperativeTask.whenDone>}. L{coiterate} has the same effect, but
returns only a L{defer.Deferred} that fires when the task is done.
L{Cooperator} can be used for many things, including but not limited to:
- running one or more computationally intensive tasks without blocking
- limiting parallelism by running a subset of the total tasks
simultaneously
- doing one thing, waiting for a L{Deferred<defer.Deferred>} to fire,
doing the next thing, repeat (i.e. serializing a sequence of
asynchronous tasks)
Multiple L{Cooperator}s do not cooperate with each other, so for most
cases you should use the L{global cooperator<task.cooperate>}.
"""
def __init__(self,
terminationPredicateFactory=_Timer,
scheduler=_defaultScheduler,
started=True):
"""
Create a scheduler-like object to which iterators may be added.
@param terminationPredicateFactory: A no-argument callable which will
be invoked at the beginning of each step and should return a
no-argument callable which will return True when the step should be
terminated. The default factory is time-based and allows iterators to
run for 1/100th of a second at a time.
@param scheduler: A one-argument callable which takes a no-argument
callable and should invoke it at some future point. This will be used
to schedule each step of this Cooperator.
@param started: A boolean which indicates whether iterators should be
stepped as soon as they are added, or if they will be queued up until
L{Cooperator.start} is called.
"""
self._tasks = []
self._metarator = iter(())
self._terminationPredicateFactory = terminationPredicateFactory
self._scheduler = scheduler
self._delayedCall = None
self._stopped = False
self._started = started
def coiterate(self, iterator, doneDeferred=None):
"""
Add an iterator to the list of iterators this L{Cooperator} is
currently running.
Equivalent to L{cooperate}, but returns a L{defer.Deferred} that will
be fired when the task is done.
@param doneDeferred: If specified, this will be the Deferred used as
the completion deferred. It is suggested that you use the default,
which creates a new Deferred for you.
@return: a Deferred that will fire when the iterator finishes.
"""
if doneDeferred is None:
doneDeferred = defer.Deferred()
CooperativeTask(iterator, self).whenDone().chainDeferred(doneDeferred)
return doneDeferred
def cooperate(self, iterator):
"""
Start running the given iterator as a long-running cooperative task, by
calling next() on it as a periodic timed event.
@param iterator: the iterator to invoke.
@return: a L{CooperativeTask} object representing this task.
"""
return CooperativeTask(iterator, self)
def _addTask(self, task):
"""
Add a L{CooperativeTask} object to this L{Cooperator}.
"""
if self._stopped:
self._tasks.append(task) # XXX silly, I know, but _completeWith
# does the inverse
task._completeWith(SchedulerStopped(), Failure(SchedulerStopped()))
else:
self._tasks.append(task)
self._reschedule()
def _removeTask(self, task):
"""
Remove a L{CooperativeTask} from this L{Cooperator}.
"""
self._tasks.remove(task)
# If no work left to do, cancel the delayed call:
if not self._tasks and self._delayedCall:
self._delayedCall.cancel()
self._delayedCall = None
def _tasksWhileNotStopped(self):
"""
Yield all L{CooperativeTask} objects in a loop as long as this
L{Cooperator}'s termination condition has not been met.
"""
terminator = self._terminationPredicateFactory()
while self._tasks:
for t in self._metarator:
yield t
if terminator():
return
self._metarator = iter(self._tasks)
def _tick(self):
"""
Run one scheduler tick.
"""
self._delayedCall = None
for taskObj in self._tasksWhileNotStopped():
taskObj._oneWorkUnit()
self._reschedule()
_mustScheduleOnStart = False
def _reschedule(self):
if not self._started:
self._mustScheduleOnStart = True
return
if self._delayedCall is None and self._tasks:
self._delayedCall = self._scheduler(self._tick)
def start(self):
"""
Begin scheduling steps.
"""
self._stopped = False
self._started = True
if self._mustScheduleOnStart:
del self._mustScheduleOnStart
self._reschedule()
def stop(self):
"""
Stop scheduling steps. Errback the completion Deferreds of all
iterators which have been added and forget about them.
"""
self._stopped = True
for taskObj in self._tasks:
taskObj._completeWith(SchedulerStopped(),
Failure(SchedulerStopped()))
self._tasks = []
if self._delayedCall is not None:
self._delayedCall.cancel()
self._delayedCall = None
@property
def running(self):
"""
Is this L{Cooperator} is currently running?
@return: C{True} if the L{Cooperator} is running, C{False} otherwise.
@rtype: C{bool}
"""
return (self._started and not self._stopped)
_theCooperator = Cooperator()
def coiterate(iterator):
"""
Cooperatively iterate over the given iterator, dividing runtime between it
and all other iterators which have been passed to this function and not yet
exhausted.
@param iterator: the iterator to invoke.
@return: a Deferred that will fire when the iterator finishes.
"""
return _theCooperator.coiterate(iterator)
def cooperate(iterator):
"""
Start running the given iterator as a long-running cooperative task, by
calling next() on it as a periodic timed event.
This is very useful if you have computationally expensive tasks that you
want to run without blocking the reactor. Just break each task up so that
it yields frequently, pass it in here and the global L{Cooperator} will
make sure work is distributed between them without blocking longer than a
single iteration of a single task.
@param iterator: the iterator to invoke.
@return: a L{CooperativeTask} object representing this task.
"""
return _theCooperator.cooperate(iterator)
@implementer(IReactorTime)
class Clock:
"""
Provide a deterministic, easily-controlled implementation of
L{IReactorTime.callLater}. This is commonly useful for writing
deterministic unit tests for code which schedules events using this API.
"""
rightNow = 0.0
def __init__(self):
self.calls = []
def seconds(self):
"""
Pretend to be time.time(). This is used internally when an operation
such as L{IDelayedCall.reset} needs to determine a time value
relative to the current time.
@rtype: C{float}
@return: The time which should be considered the current time.
"""
return self.rightNow
def _sortCalls(self):
"""
Sort the pending calls according to the time they are scheduled.
"""
self.calls.sort(key=lambda a: a.getTime())
def callLater(self, when, what, *a, **kw):
"""
See L{twisted.internet.interfaces.IReactorTime.callLater}.
"""
dc = base.DelayedCall(self.seconds() + when,
what, a, kw,
self.calls.remove,
lambda c: None,
self.seconds)
self.calls.append(dc)
self._sortCalls()
return dc
def getDelayedCalls(self):
"""
See L{twisted.internet.interfaces.IReactorTime.getDelayedCalls}
"""
return self.calls
def advance(self, amount):
"""
Move time on this clock forward by the given amount and run whatever
pending calls should be run.
@type amount: C{float}
@param amount: The number of seconds which to advance this clock's
time.
"""
self.rightNow += amount
self._sortCalls()
while self.calls and self.calls[0].getTime() <= self.seconds():
call = self.calls.pop(0)
call.called = 1
call.func(*call.args, **call.kw)
self._sortCalls()
def pump(self, timings):
"""
Advance incrementally by the given set of times.
@type timings: iterable of C{float}
"""
for amount in timings:
self.advance(amount)
def deferLater(clock, delay, callable=None, *args, **kw):
"""
Call the given function after a certain period of time has passed.
@type clock: L{IReactorTime} provider
@param clock: The object which will be used to schedule the delayed
call.
@type delay: C{float} or C{int}
@param delay: The number of seconds to wait before calling the function.
@param callable: The object to call after the delay.
@param *args: The positional arguments to pass to C{callable}.
@param **kw: The keyword arguments to pass to C{callable}.
@rtype: L{defer.Deferred}
@return: A deferred that fires with the result of the callable when the
specified time has elapsed.
"""
def deferLaterCancel(deferred):
delayedCall.cancel()
d = defer.Deferred(deferLaterCancel)
if callable is not None:
d.addCallback(lambda ignored: callable(*args, **kw))
delayedCall = clock.callLater(delay, d.callback, None)
return d
def react(main, argv=(), _reactor=None):
"""
Call C{main} and run the reactor until the L{Deferred} it returns fires.
This is intended as the way to start up an application with a well-defined
completion condition. Use it to write clients or one-off asynchronous
operations. Prefer this to calling C{reactor.run} directly, as this
function will also:
- Take care to call C{reactor.stop} once and only once, and at the right
time.
- Log any failures from the C{Deferred} returned by C{main}.
- Exit the application when done, with exit code 0 in case of success and
1 in case of failure. If C{main} fails with a C{SystemExit} error, the
code returned is used.
The following demonstrates the signature of a C{main} function which can be
used with L{react}::
def main(reactor, username, password):
return defer.succeed('ok')
task.react(main, ('alice', 'secret'))
@param main: A callable which returns a L{Deferred}. It should
take the reactor as its first parameter, followed by the elements of
C{argv}.
@param argv: A list of arguments to pass to C{main}. If omitted the
callable will be invoked with no additional arguments.
@param _reactor: An implementation detail to allow easier unit testing. Do
not supply this parameter.
@since: 12.3
"""
if _reactor is None:
from twisted.internet import reactor as _reactor
finished = main(_reactor, *argv)
codes = [0]
stopping = []
_reactor.addSystemEventTrigger('before', 'shutdown', stopping.append, True)
def stop(result, stopReactor):
if stopReactor:
try:
_reactor.stop()
except ReactorNotRunning:
pass
if isinstance(result, Failure):
if result.check(SystemExit) is not None:
code = result.value.code
else:
log.err(result, "main function encountered error")
code = 1
codes[0] = code
def cbFinish(result):
if stopping:
stop(result, False)
else:
_reactor.callWhenRunning(stop, result, True)
finished.addBoth(cbFinish)
_reactor.run()
sys.exit(codes[0])
__all__ = [
'LoopingCall',
'Clock',
'SchedulerStopped', 'Cooperator', 'coiterate',
'deferLater', 'react']
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,221 @@
# -*- test-case-name: twisted.internet.test.test_coroutines -*-
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
Tests for C{await} support in Deferreds.
These tests can only work and be imported on Python 3.5+!
"""
import types
from twisted.python.failure import Failure
from twisted.internet.defer import (
Deferred, maybeDeferred, ensureDeferred, fail
)
from twisted.trial.unittest import TestCase
from twisted.internet.task import Clock
class SampleException(Exception):
"""
A specific sample exception for testing.
"""
class AwaitTests(TestCase):
"""
Tests for using Deferreds in conjunction with PEP-492.
"""
def test_awaitReturnsIterable(self):
"""
C{Deferred.__await__} returns an iterable.
"""
d = Deferred()
awaitedDeferred = d.__await__()
self.assertEqual(awaitedDeferred, iter(awaitedDeferred))
def test_ensureDeferred(self):
"""
L{ensureDeferred} will turn a coroutine into a L{Deferred}.
"""
async def run():
d = Deferred()
d.callback("bar")
await d
res = await run2()
return res
async def run2():
d = Deferred()
d.callback("foo")
res = await d
return res
# It's a coroutine...
r = run()
self.assertIsInstance(r, types.CoroutineType)
# Now it's a Deferred.
d = ensureDeferred(r)
self.assertIsInstance(d, Deferred)
# The Deferred has the result we want.
res = self.successResultOf(d)
self.assertEqual(res, "foo")
def test_basic(self):
"""
L{ensureDeferred} allows a function to C{await} on a L{Deferred}.
"""
async def run():
d = Deferred()
d.callback("foo")
res = await d
return res
d = ensureDeferred(run())
res = self.successResultOf(d)
self.assertEqual(res, "foo")
def test_exception(self):
"""
An exception in a coroutine wrapped with L{ensureDeferred} will cause
the returned L{Deferred} to fire with a failure.
"""
async def run():
d = Deferred()
d.callback("foo")
await d
raise ValueError("Oh no!")
d = ensureDeferred(run())
res = self.failureResultOf(d)
self.assertEqual(type(res.value), ValueError)
self.assertEqual(res.value.args, ("Oh no!",))
def test_synchronousDeferredFailureTraceback(self):
"""
When a Deferred is awaited upon that has already failed with a Failure
that has a traceback, both the place that the synchronous traceback
comes from and the awaiting line are shown in the traceback.
"""
def raises():
raise SampleException()
it = maybeDeferred(raises)
async def doomed():
return await it
failure = self.failureResultOf(ensureDeferred(doomed()))
self.assertIn(", in doomed\n", failure.getTraceback())
self.assertIn(", in raises\n", failure.getTraceback())
def test_asyncDeferredFailureTraceback(self):
"""
When a Deferred is awaited upon that later fails with a Failure that
has a traceback, both the place that the synchronous traceback comes
from and the awaiting line are shown in the traceback.
"""
def returnsFailure():
try:
raise SampleException()
except SampleException:
return Failure()
it = Deferred()
async def doomed():
return await it
started = ensureDeferred(doomed())
self.assertNoResult(started)
it.errback(returnsFailure())
failure = self.failureResultOf(started)
self.assertIn(", in doomed\n", failure.getTraceback())
self.assertIn(", in returnsFailure\n", failure.getTraceback())
def test_twoDeep(self):
"""
A coroutine wrapped with L{ensureDeferred} that awaits a L{Deferred}
suspends its execution until the inner L{Deferred} fires.
"""
reactor = Clock()
sections = []
async def runone():
sections.append(2)
d = Deferred()
reactor.callLater(1, d.callback, 2)
await d
sections.append(3)
return "Yay!"
async def run():
sections.append(1)
result = await runone()
sections.append(4)
d = Deferred()
reactor.callLater(1, d.callback, 1)
await d
sections.append(5)
return result
d = ensureDeferred(run())
reactor.advance(0.9)
self.assertEqual(sections, [1, 2])
reactor.advance(0.1)
self.assertEqual(sections, [1, 2, 3, 4])
reactor.advance(0.9)
self.assertEqual(sections, [1, 2, 3, 4])
reactor.advance(0.1)
self.assertEqual(sections, [1, 2, 3, 4, 5])
res = self.successResultOf(d)
self.assertEqual(res, "Yay!")
def test_reraise(self):
"""
Awaiting an already failed Deferred will raise the exception.
"""
async def test():
try:
await fail(ValueError("Boom"))
except ValueError as e:
self.assertEqual(e.args, ("Boom",))
return 1
return 0
res = self.successResultOf(ensureDeferred(test()))
self.assertEqual(res, 1)
def test_chained(self):
"""
Awaiting a paused & chained Deferred will give the result when it has
one.
"""
reactor = Clock()
async def test():
d = Deferred()
d2 = Deferred()
d.addCallback(lambda ignored: d2)
d.callback(None)
reactor.callLater(0, d2.callback, "bye")
return await d
d = ensureDeferred(test())
reactor.advance(0.1)
res = self.successResultOf(d)
self.assertEqual(res, "bye")
@@ -0,0 +1,177 @@
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
POSIX implementation of local network interface enumeration.
"""
from __future__ import division, absolute_import
import sys, socket
from socket import AF_INET, AF_INET6, inet_ntop
from ctypes import (
CDLL, POINTER, Structure, c_char_p, c_ushort, c_int,
c_uint32, c_uint8, c_void_p, c_ubyte, pointer, cast)
from ctypes.util import find_library
from twisted.python.compat import _PY3, nativeString
if _PY3:
# Once #6070 is implemented, this can be replaced with the implementation
# from that ticket:
def chr(i):
"""
Python 3 implementation of Python 2 chr(), i.e. convert an integer to
corresponding byte.
"""
return bytes([i])
libc = CDLL(find_library("c"))
if sys.platform.startswith('freebsd') or sys.platform == 'darwin':
_sockaddrCommon = [
("sin_len", c_uint8),
("sin_family", c_uint8),
]
else:
_sockaddrCommon = [
("sin_family", c_ushort),
]
class in_addr(Structure):
_fields_ = [
("in_addr", c_ubyte * 4),
]
class in6_addr(Structure):
_fields_ = [
("in_addr", c_ubyte * 16),
]
class sockaddr(Structure):
_fields_ = _sockaddrCommon + [
("sin_port", c_ushort),
]
class sockaddr_in(Structure):
_fields_ = _sockaddrCommon + [
("sin_port", c_ushort),
("sin_addr", in_addr),
]
class sockaddr_in6(Structure):
_fields_ = _sockaddrCommon + [
("sin_port", c_ushort),
("sin_flowinfo", c_uint32),
("sin_addr", in6_addr),
]
class ifaddrs(Structure):
pass
ifaddrs_p = POINTER(ifaddrs)
ifaddrs._fields_ = [
('ifa_next', ifaddrs_p),
('ifa_name', c_char_p),
('ifa_flags', c_uint32),
('ifa_addr', POINTER(sockaddr)),
('ifa_netmask', POINTER(sockaddr)),
('ifa_dstaddr', POINTER(sockaddr)),
('ifa_data', c_void_p)]
getifaddrs = libc.getifaddrs
getifaddrs.argtypes = [POINTER(ifaddrs_p)]
getifaddrs.restype = c_int
freeifaddrs = libc.freeifaddrs
freeifaddrs.argtypes = [ifaddrs_p]
def _maybeCleanupScopeIndex(family, packed):
"""
On FreeBSD, kill the embedded interface indices in link-local scoped
addresses.
@param family: The address family of the packed address - one of the
I{socket.AF_*} constants.
@param packed: The packed representation of the address (ie, the bytes of a
I{in_addr} field).
@type packed: L{bytes}
@return: The packed address with any FreeBSD-specific extra bits cleared.
@rtype: L{bytes}
@see: U{https://twistedmatrix.com/trac/ticket/6843}
@see: U{http://www.freebsd.org/doc/en/books/developers-handbook/ipv6.html#ipv6-scope-index}
@note: Indications are that the need for this will be gone in FreeBSD >=10.
"""
if sys.platform.startswith('freebsd') and packed[:2] == b"\xfe\x80":
return packed[:2] + b"\x00\x00" + packed[4:]
return packed
def _interfaces():
"""
Call C{getifaddrs(3)} and return a list of tuples of interface name, address
family, and human-readable address representing its results.
"""
ifaddrs = ifaddrs_p()
if getifaddrs(pointer(ifaddrs)) < 0:
raise OSError()
results = []
try:
while ifaddrs:
if ifaddrs[0].ifa_addr:
family = ifaddrs[0].ifa_addr[0].sin_family
if family == AF_INET:
addr = cast(ifaddrs[0].ifa_addr, POINTER(sockaddr_in))
elif family == AF_INET6:
addr = cast(ifaddrs[0].ifa_addr, POINTER(sockaddr_in6))
else:
addr = None
if addr:
packed = b''.join(map(chr, addr[0].sin_addr.in_addr[:]))
packed = _maybeCleanupScopeIndex(family, packed)
results.append((
ifaddrs[0].ifa_name,
family,
inet_ntop(family, packed)))
ifaddrs = ifaddrs[0].ifa_next
finally:
freeifaddrs(ifaddrs)
return results
def posixGetLinkLocalIPv6Addresses():
"""
Return a list of strings in colon-hex format representing all the link local
IPv6 addresses available on the system, as reported by I{getifaddrs(3)}.
"""
retList = []
for (interface, family, address) in _interfaces():
interface = nativeString(interface)
address = nativeString(address)
if family == socket.AF_INET6 and address.startswith('fe80:'):
retList.append('%s%%%s' % (address, interface))
return retList
@@ -0,0 +1,26 @@
This is a self-signed certificate authority certificate to be used in tests.
It was created with the following command:
certcreate -f thing1.pem -h fake-ca-1.example.com -e noreply@example.com \
-S 1234 -o 'Twisted Matrix Labs'
'certcreate' may be obtained from <http://divmod.org/trac/wiki/DivmodEpsilon>
-----BEGIN CERTIFICATE-----
MIICwjCCAisCAgTSMA0GCSqGSIb3DQEBBAUAMIGoMREwDwYDVQQLEwhTZWN1cml0
eTEcMBoGA1UEChMTVHdpc3RlZCBNYXRyaXggTGFiczEeMBwGA1UEAxMVZmFrZS1j
YS0xLmV4YW1wbGUuY29tMREwDwYDVQQIEwhOZXcgWW9yazELMAkGA1UEBhMCVVMx
IjAgBgkqhkiG9w0BCQEWE25vcmVwbHlAZXhhbXBsZS5jb20xETAPBgNVBAcTCE5l
dyBZb3JrMB4XDTEwMDkyMTAxMjUxNFoXDTExMDkyMTAxMjUxNFowgagxETAPBgNV
BAsTCFNlY3VyaXR5MRwwGgYDVQQKExNUd2lzdGVkIE1hdHJpeCBMYWJzMR4wHAYD
VQQDExVmYWtlLWNhLTEuZXhhbXBsZS5jb20xETAPBgNVBAgTCE5ldyBZb3JrMQsw
CQYDVQQGEwJVUzEiMCAGCSqGSIb3DQEJARYTbm9yZXBseUBleGFtcGxlLmNvbTER
MA8GA1UEBxMITmV3IFlvcmswgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBALRb
VqC0CsaFgq1vbwPfs8zoP3ZYC/0sPMv0RJN+f3Dc7Q6YgNHS7o7TM3uAy/McADeW
rwVuNJGe9k+4ZBHysmBH1sG64fHT5TlK9saPcUQqkubSWj4cKSDtVbQERWqC5Dy+
qTQeZGYoPEMlnRXgMpST04DG//Dgzi4PYqUOjwxTAgMBAAEwDQYJKoZIhvcNAQEE
BQADgYEAqNEdMXWEs8Co76wxL3/cSV3MjiAroVxJdI/3EzlnfPi1JeibbdWw31fC
bn6428KTjjfhS31zo1yHG3YNXFEJXRscwLAH7ogz5kJwZMy/oS/96EFM10bkNwkK
v+nWKN8i3t/E5TEIl3BPN8tchtWmH0rycVuzs5LwaewwR1AnUE4=
-----END CERTIFICATE-----
@@ -0,0 +1,27 @@
from __future__ import absolute_import, division
import sys
import os
try:
# On Windows, stdout is not opened in binary mode by default,
# so newline characters are munged on writing, interfering with
# the tests.
import msvcrt
msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
except ImportError:
pass
# Loop over each of the arguments given and print it to stdout
for arg in sys.argv[1:]:
res = arg + chr(0)
if sys.version_info < (3, 0):
stdout = sys.stdout
else:
stdout = sys.stdout.buffer
res = res.encode(sys.getfilesystemencoding(), "surrogateescape")
stdout.write(res)
stdout.flush()
@@ -0,0 +1,350 @@
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
Utilities for unit testing reactor implementations.
The main feature of this module is L{ReactorBuilder}, a base class for use when
writing interface/blackbox tests for reactor implementations. Test case classes
for reactor features should subclass L{ReactorBuilder} instead of
L{SynchronousTestCase}. All of the features of L{SynchronousTestCase} will be
available. Additionally, the tests will automatically be applied to all
available reactor implementations.
"""
from __future__ import division, absolute_import
__metaclass__ = type
__all__ = ['TestTimeoutError', 'ReactorBuilder', 'needsRunningReactor']
import os, signal, time
from twisted.trial.unittest import SynchronousTestCase, SkipTest
from twisted.trial.util import DEFAULT_TIMEOUT_DURATION, acquireAttribute
from twisted.python.runtime import platform
from twisted.python.reflect import namedAny
from twisted.python.deprecate import _fullyQualifiedName as fullyQualifiedName
from twisted.python import log
from twisted.python.failure import Failure
from twisted.python.compat import _PY3
# Access private APIs.
if platform.isWindows():
process = None
else:
from twisted.internet import process
class TestTimeoutError(Exception):
"""
The reactor was still running after the timeout period elapsed in
L{ReactorBuilder.runReactor}.
"""
def needsRunningReactor(reactor, thunk):
"""
Various functions within these tests need an already-running reactor at
some point. They need to stop the reactor when the test has completed, and
that means calling reactor.stop(). However, reactor.stop() raises an
exception if the reactor isn't already running, so if the L{Deferred} that
a particular API under test returns fires synchronously (as especially an
endpoint's C{connect()} method may do, if the connect is to a local
interface address) then the test won't be able to stop the reactor being
tested and finish. So this calls C{thunk} only once C{reactor} is running.
(This is just an alias for
L{twisted.internet.interfaces.IReactorCore.callWhenRunning} on the given
reactor parameter, in order to centrally reference the above paragraph and
repeating it everywhere as a comment.)
@param reactor: the L{twisted.internet.interfaces.IReactorCore} under test
@param thunk: a 0-argument callable, which eventually finishes the test in
question, probably in a L{Deferred} callback.
"""
reactor.callWhenRunning(thunk)
def stopOnError(case, reactor, publisher=None):
"""
Stop the reactor as soon as any error is logged on the given publisher.
This is beneficial for tests which will wait for a L{Deferred} to fire
before completing (by passing or failing). Certain implementation bugs may
prevent the L{Deferred} from firing with any result at all (consider a
protocol's {dataReceived} method that raises an exception: this exception
is logged but it won't ever cause a L{Deferred} to fire). In that case the
test would have to complete by timing out which is a much less desirable
outcome than completing as soon as the unexpected error is encountered.
@param case: A L{SynchronousTestCase} to use to clean up the necessary log
observer when the test is over.
@param reactor: The reactor to stop.
@param publisher: A L{LogPublisher} to watch for errors. If L{None}, the
global log publisher will be watched.
"""
if publisher is None:
from twisted.python import log as publisher
running = [None]
def stopIfError(event):
if running and event.get('isError'):
running.pop()
reactor.stop()
publisher.addObserver(stopIfError)
case.addCleanup(publisher.removeObserver, stopIfError)
class ReactorBuilder:
"""
L{SynchronousTestCase} mixin which provides a reactor-creation API. This
mixin defines C{setUp} and C{tearDown}, so mix it in before
L{SynchronousTestCase} or call its methods from the overridden ones in the
subclass.
@cvar skippedReactors: A dict mapping FQPN strings of reactors for
which the tests defined by this class will be skipped to strings
giving the skip message.
@cvar requiredInterfaces: A C{list} of interfaces which the reactor must
provide or these tests will be skipped. The default, L{None}, means
that no interfaces are required.
@ivar reactorFactory: A no-argument callable which returns the reactor to
use for testing.
@ivar originalHandler: The SIGCHLD handler which was installed when setUp
ran and which will be re-installed when tearDown runs.
@ivar _reactors: A list of FQPN strings giving the reactors for which
L{SynchronousTestCase}s will be created.
"""
_reactors = [
# Select works everywhere
"twisted.internet.selectreactor.SelectReactor",
]
if platform.isWindows():
# PortableGtkReactor is only really interesting on Windows,
# but not really Windows specific; if you want you can
# temporarily move this up to the all-platforms list to test
# it on other platforms. It's not there in general because
# it's not _really_ worth it to support on other platforms,
# since no one really wants to use it on other platforms.
_reactors.extend([
"twisted.internet.gtk2reactor.PortableGtkReactor",
"twisted.internet.gireactor.PortableGIReactor",
"twisted.internet.gtk3reactor.PortableGtk3Reactor",
"twisted.internet.win32eventreactor.Win32Reactor",
"twisted.internet.iocpreactor.reactor.IOCPReactor"])
else:
_reactors.extend([
"twisted.internet.glib2reactor.Glib2Reactor",
"twisted.internet.gtk2reactor.Gtk2Reactor",
"twisted.internet.gireactor.GIReactor",
"twisted.internet.gtk3reactor.Gtk3Reactor"])
if _PY3:
_reactors.append(
"twisted.internet.asyncioreactor.AsyncioSelectorReactor")
if platform.isMacOSX():
_reactors.append("twisted.internet.cfreactor.CFReactor")
else:
_reactors.extend([
"twisted.internet.pollreactor.PollReactor",
"twisted.internet.epollreactor.EPollReactor"])
if not platform.isLinux():
# Presumably Linux is not going to start supporting kqueue, so
# skip even trying this configuration.
_reactors.extend([
# Support KQueue on non-OS-X POSIX platforms for now.
"twisted.internet.kqreactor.KQueueReactor",
])
reactorFactory = None
originalHandler = None
requiredInterfaces = None
skippedReactors = {}
def setUp(self):
"""
Clear the SIGCHLD handler, if there is one, to ensure an environment
like the one which exists prior to a call to L{reactor.run}.
"""
if not platform.isWindows():
self.originalHandler = signal.signal(signal.SIGCHLD, signal.SIG_DFL)
def tearDown(self):
"""
Restore the original SIGCHLD handler and reap processes as long as
there seem to be any remaining.
"""
if self.originalHandler is not None:
signal.signal(signal.SIGCHLD, self.originalHandler)
if process is not None:
begin = time.time()
while process.reapProcessHandlers:
log.msg(
"ReactorBuilder.tearDown reaping some processes %r" % (
process.reapProcessHandlers,))
process.reapAllProcesses()
# The process should exit on its own. However, if it
# doesn't, we're stuck in this loop forever. To avoid
# hanging the test suite, eventually give the process some
# help exiting and move on.
time.sleep(0.001)
if time.time() - begin > 60:
for pid in process.reapProcessHandlers:
os.kill(pid, signal.SIGKILL)
raise Exception(
"Timeout waiting for child processes to exit: %r" % (
process.reapProcessHandlers,))
def unbuildReactor(self, reactor):
"""
Clean up any resources which may have been allocated for the given
reactor by its creation or by a test which used it.
"""
# Chris says:
#
# XXX These explicit calls to clean up the waker (and any other
# internal readers) should become obsolete when bug #3063 is
# fixed. -radix, 2008-02-29. Fortunately it should probably cause an
# error when bug #3063 is fixed, so it should be removed in the same
# branch that fixes it.
#
# -exarkun
reactor._uninstallHandler()
if getattr(reactor, '_internalReaders', None) is not None:
for reader in reactor._internalReaders:
reactor.removeReader(reader)
reader.connectionLost(None)
reactor._internalReaders.clear()
# Here's an extra thing unrelated to wakers but necessary for
# cleaning up after the reactors we make. -exarkun
reactor.disconnectAll()
# It would also be bad if any timed calls left over were allowed to
# run.
calls = reactor.getDelayedCalls()
for c in calls:
c.cancel()
def buildReactor(self):
"""
Create and return a reactor using C{self.reactorFactory}.
"""
try:
from twisted.internet.cfreactor import CFReactor
from twisted.internet import reactor as globalReactor
except ImportError:
pass
else:
if (isinstance(globalReactor, CFReactor)
and self.reactorFactory is CFReactor):
raise SkipTest(
"CFReactor uses APIs which manipulate global state, "
"so it's not safe to run its own reactor-builder tests "
"under itself")
try:
reactor = self.reactorFactory()
except:
# Unfortunately, not all errors which result in a reactor
# being unusable are detectable without actually
# instantiating the reactor. So we catch some more here
# and skip the test if necessary. We also log it to aid
# with debugging, but flush the logged error so the test
# doesn't fail.
log.err(None, "Failed to install reactor")
self.flushLoggedErrors()
raise SkipTest(Failure().getErrorMessage())
else:
if self.requiredInterfaces is not None:
missing = [
required for required in self.requiredInterfaces
if not required.providedBy(reactor)]
if missing:
self.unbuildReactor(reactor)
raise SkipTest("%s does not provide %s" % (
fullyQualifiedName(reactor.__class__),
",".join([fullyQualifiedName(x) for x in missing])))
self.addCleanup(self.unbuildReactor, reactor)
return reactor
def getTimeout(self):
"""
Determine how long to run the test before considering it failed.
@return: A C{int} or C{float} giving a number of seconds.
"""
return acquireAttribute(self._parents, 'timeout', DEFAULT_TIMEOUT_DURATION)
def runReactor(self, reactor, timeout=None):
"""
Run the reactor for at most the given amount of time.
@param reactor: The reactor to run.
@type timeout: C{int} or C{float}
@param timeout: The maximum amount of time, specified in seconds, to
allow the reactor to run. If the reactor is still running after
this much time has elapsed, it will be stopped and an exception
raised. If L{None}, the default test method timeout imposed by
Trial will be used. This depends on the L{IReactorTime}
implementation of C{reactor} for correct operation.
@raise TestTimeoutError: If the reactor is still running after
C{timeout} seconds.
"""
if timeout is None:
timeout = self.getTimeout()
timedOut = []
def stop():
timedOut.append(None)
reactor.stop()
timedOutCall = reactor.callLater(timeout, stop)
reactor.run()
if timedOut:
raise TestTimeoutError(
"reactor still running after %s seconds" % (timeout,))
else:
timedOutCall.cancel()
def makeTestCaseClasses(cls):
"""
Create a L{SynchronousTestCase} subclass which mixes in C{cls} for each
known reactor and return a dict mapping their names to them.
"""
classes = {}
for reactor in cls._reactors:
shortReactorName = reactor.split(".")[-1]
name = (cls.__name__ + "." + shortReactorName + "Tests").replace(".", "_")
class testcase(cls, SynchronousTestCase):
__module__ = cls.__module__
if reactor in cls.skippedReactors:
skip = cls.skippedReactors[reactor]
try:
reactorFactory = namedAny(reactor)
except:
skip = Failure().getErrorMessage()
testcase.__name__ = name
if hasattr(cls, "__qualname__"):
testcase.__qualname__ = ".".join(cls.__qualname__.split()[0:-1] + [name])
classes[testcase.__name__] = testcase
return classes
makeTestCaseClasses = classmethod(makeTestCaseClasses)
@@ -0,0 +1,450 @@
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
Tests for L{twisted.internet.base}.
"""
import socket
try:
from Queue import Queue
except ImportError:
from queue import Queue
from zope.interface import implementer
from twisted.python.threadpool import ThreadPool
from twisted.internet.interfaces import (IReactorTime, IReactorThreads,
IResolverSimple)
from twisted.internet.error import DNSLookupError
from twisted.internet._resolver import FirstOneWins
from twisted.internet.defer import Deferred
from twisted.internet.base import ThreadedResolver, DelayedCall, ReactorBase
from twisted.internet.task import Clock
from twisted.trial.unittest import TestCase, SkipTest
@implementer(IReactorTime, IReactorThreads)
class FakeReactor(object):
"""
A fake reactor implementation which just supports enough reactor APIs for
L{ThreadedResolver}.
"""
def __init__(self):
self._clock = Clock()
self.callLater = self._clock.callLater
self._threadpool = ThreadPool()
self._threadpool.start()
self.getThreadPool = lambda: self._threadpool
self._threadCalls = Queue()
def callFromThread(self, f, *args, **kwargs):
self._threadCalls.put((f, args, kwargs))
def _runThreadCalls(self):
f, args, kwargs = self._threadCalls.get()
f(*args, **kwargs)
def _stop(self):
self._threadpool.stop()
class ThreadedResolverTests(TestCase):
"""
Tests for L{ThreadedResolver}.
"""
def test_success(self):
"""
L{ThreadedResolver.getHostByName} returns a L{Deferred} which fires
with the value returned by the call to L{socket.gethostbyname} in the
threadpool of the reactor passed to L{ThreadedResolver.__init__}.
"""
ip = "10.0.0.17"
name = "foo.bar.example.com"
timeout = 30
reactor = FakeReactor()
self.addCleanup(reactor._stop)
lookedUp = []
resolvedTo = []
def fakeGetHostByName(name):
lookedUp.append(name)
return ip
self.patch(socket, 'gethostbyname', fakeGetHostByName)
resolver = ThreadedResolver(reactor)
d = resolver.getHostByName(name, (timeout,))
d.addCallback(resolvedTo.append)
reactor._runThreadCalls()
self.assertEqual(lookedUp, [name])
self.assertEqual(resolvedTo, [ip])
# Make sure that any timeout-related stuff gets cleaned up.
reactor._clock.advance(timeout + 1)
self.assertEqual(reactor._clock.calls, [])
def test_failure(self):
"""
L{ThreadedResolver.getHostByName} returns a L{Deferred} which fires a
L{Failure} if the call to L{socket.gethostbyname} raises an exception.
"""
timeout = 30
reactor = FakeReactor()
self.addCleanup(reactor._stop)
def fakeGetHostByName(name):
raise IOError("ENOBUFS (this is a funny joke)")
self.patch(socket, 'gethostbyname', fakeGetHostByName)
failedWith = []
resolver = ThreadedResolver(reactor)
d = resolver.getHostByName("some.name", (timeout,))
self.assertFailure(d, DNSLookupError)
d.addCallback(failedWith.append)
reactor._runThreadCalls()
self.assertEqual(len(failedWith), 1)
# Make sure that any timeout-related stuff gets cleaned up.
reactor._clock.advance(timeout + 1)
self.assertEqual(reactor._clock.calls, [])
def test_timeout(self):
"""
If L{socket.gethostbyname} does not complete before the specified
timeout elapsed, the L{Deferred} returned by
L{ThreadedResolver.getHostByName} fails with L{DNSLookupError}.
"""
timeout = 10
reactor = FakeReactor()
self.addCleanup(reactor._stop)
result = Queue()
def fakeGetHostByName(name):
raise result.get()
self.patch(socket, 'gethostbyname', fakeGetHostByName)
failedWith = []
resolver = ThreadedResolver(reactor)
d = resolver.getHostByName("some.name", (timeout,))
self.assertFailure(d, DNSLookupError)
d.addCallback(failedWith.append)
reactor._clock.advance(timeout - 1)
self.assertEqual(failedWith, [])
reactor._clock.advance(1)
self.assertEqual(len(failedWith), 1)
# Eventually the socket.gethostbyname does finish - in this case, with
# an exception. Nobody cares, though.
result.put(IOError("The I/O was errorful"))
def test_resolverGivenStr(self):
"""
L{ThreadedResolver.getHostByName} is passed L{str}, encoded using IDNA
if required.
"""
calls = []
@implementer(IResolverSimple)
class FakeResolver(object):
def getHostByName(self, name, timeouts=()):
calls.append(name)
return Deferred()
class JustEnoughReactor(ReactorBase):
def installWaker(self):
pass
fake = FakeResolver()
reactor = JustEnoughReactor()
reactor.installResolver(fake)
rec = FirstOneWins(Deferred())
reactor.nameResolver.resolveHostName(
rec, u"example.example")
reactor.nameResolver.resolveHostName(
rec, "example.example")
reactor.nameResolver.resolveHostName(
rec, u"v\xe4\xe4ntynyt.example")
reactor.nameResolver.resolveHostName(
rec, u"\u0440\u0444.example")
reactor.nameResolver.resolveHostName(
rec, "xn----7sbb4ac0ad0be6cf.xn--p1ai")
self.assertEqual(len(calls), 5)
self.assertEqual(list(map(type, calls)), [str]*5)
self.assertEqual("example.example", calls[0])
self.assertEqual("example.example", calls[1])
self.assertEqual("xn--vntynyt-5waa.example", calls[2])
self.assertEqual("xn--p1ai.example", calls[3])
self.assertEqual("xn----7sbb4ac0ad0be6cf.xn--p1ai", calls[4])
def nothing():
"""
Function used by L{DelayedCallTests.test_str}.
"""
class DelayedCallMixin(object):
"""
L{DelayedCall}
"""
def _getDelayedCallAt(self, time):
"""
Get a L{DelayedCall} instance at a given C{time}.
@param time: The absolute time at which the returned L{DelayedCall}
will be scheduled.
"""
def noop(call):
pass
return DelayedCall(time, lambda: None, (), {}, noop, noop, None)
def setUp(self):
"""
Create two L{DelayedCall} instanced scheduled to run at different
times.
"""
self.zero = self._getDelayedCallAt(0)
self.one = self._getDelayedCallAt(1)
def test_str(self):
"""
The string representation of a L{DelayedCall} instance, as returned by
L{str}, includes the unsigned id of the instance, as well as its state,
the function to be called, and the function arguments.
"""
dc = DelayedCall(12, nothing, (3, ), {"A": 5}, None, None, lambda: 1.5)
self.assertEqual(
str(dc),
"<DelayedCall 0x%x [10.5s] called=0 cancelled=0 nothing(3, A=5)>"
% (id(dc),),
)
def test_repr(self):
"""
The string representation of a L{DelayedCall} instance, as returned by
{repr}, is identical to that returned by L{str}.
"""
dc = DelayedCall(13, nothing, (6, ), {"A": 9}, None, None, lambda: 1.6)
self.assertEqual(str(dc), repr(dc))
def test_lt(self):
"""
For two instances of L{DelayedCall} C{a} and C{b}, C{a < b} is true
if and only if C{a} is scheduled to run before C{b}.
"""
zero, one = self.zero, self.one
self.assertTrue(zero < one)
self.assertFalse(one < zero)
self.assertFalse(zero < zero)
self.assertFalse(one < one)
def test_le(self):
"""
For two instances of L{DelayedCall} C{a} and C{b}, C{a <= b} is true
if and only if C{a} is scheduled to run before C{b} or at the same
time as C{b}.
"""
zero, one = self.zero, self.one
self.assertTrue(zero <= one)
self.assertFalse(one <= zero)
self.assertTrue(zero <= zero)
self.assertTrue(one <= one)
def test_gt(self):
"""
For two instances of L{DelayedCall} C{a} and C{b}, C{a > b} is true
if and only if C{a} is scheduled to run after C{b}.
"""
zero, one = self.zero, self.one
self.assertTrue(one > zero)
self.assertFalse(zero > one)
self.assertFalse(zero > zero)
self.assertFalse(one > one)
def test_ge(self):
"""
For two instances of L{DelayedCall} C{a} and C{b}, C{a > b} is true
if and only if C{a} is scheduled to run after C{b} or at the same
time as C{b}.
"""
zero, one = self.zero, self.one
self.assertTrue(one >= zero)
self.assertFalse(zero >= one)
self.assertTrue(zero >= zero)
self.assertTrue(one >= one)
def test_eq(self):
"""
A L{DelayedCall} instance is only equal to itself.
"""
# Explicitly use == here, instead of assertEqual, to be more
# confident __eq__ is being tested.
self.assertFalse(self.zero == self.one)
self.assertTrue(self.zero == self.zero)
self.assertTrue(self.one == self.one)
def test_ne(self):
"""
A L{DelayedCall} instance is not equal to any other object.
"""
# Explicitly use != here, instead of assertEqual, to be more
# confident __ne__ is being tested.
self.assertTrue(self.zero != self.one)
self.assertFalse(self.zero != self.zero)
self.assertFalse(self.one != self.one)
class DelayedCallNoDebugTests(DelayedCallMixin, TestCase):
"""
L{DelayedCall}
"""
def setUp(self):
"""
Turn debug off.
"""
self.patch(DelayedCall, 'debug', False)
DelayedCallMixin.setUp(self)
def test_str(self):
"""
The string representation of a L{DelayedCall} instance, as returned by
L{str}, includes the unsigned id of the instance, as well as its state,
the function to be called, and the function arguments.
"""
dc = DelayedCall(12, nothing, (3, ), {"A": 5}, None, None, lambda: 1.5)
expected = (
"<DelayedCall 0x{:x} [10.5s] called=0 cancelled=0 "
"nothing(3, A=5)>".format(id(dc)))
self.assertEqual(str(dc), expected)
class DelayedCallDebugTests(DelayedCallMixin, TestCase):
"""
L{DelayedCall}
"""
def setUp(self):
"""
Turn debug on.
"""
self.patch(DelayedCall, 'debug', True)
DelayedCallMixin.setUp(self)
def test_str(self):
"""
The string representation of a L{DelayedCall} instance, as returned by
L{str}, includes the unsigned id of the instance, as well as its state,
the function to be called, and the function arguments.
"""
dc = DelayedCall(12, nothing, (3, ), {"A": 5}, None, None, lambda: 1.5)
expectedRegexp = (
"<DelayedCall 0x{:x} \\[10.5s\\] called=0 cancelled=0 "
"nothing\\(3, A=5\\)\n\n"
"traceback at creation:".format(id(dc)))
self.assertRegex(
str(dc), expectedRegexp)
class TestSpySignalCapturingReactor(ReactorBase):
"""
Subclass of ReactorBase to capture signals delivered to the
reactor for inspection.
"""
def installWaker(self):
"""
Required method, unused.
"""
class ReactorBaseSignalTests(TestCase):
"""
Tests to exercise ReactorBase's signal exit reporting path.
"""
def test_exitSignalDefaultsToNone(self):
"""
The default value of the _exitSignal attribute is None.
"""
reactor = TestSpySignalCapturingReactor()
self.assertIs(None, reactor._exitSignal)
def test_captureSIGINT(self):
"""
ReactorBase's SIGINT handler saves the value of SIGINT to the
_exitSignal attribute.
"""
reactor = TestSpySignalCapturingReactor()
reactor.sigInt(signal.SIGINT, None)
self.assertEquals(signal.SIGINT, reactor._exitSignal)
def test_captureSIGTERM(self):
"""
ReactorBase's SIGTERM handler saves the value of SIGTERM to the
_exitSignal attribute.
"""
reactor = TestSpySignalCapturingReactor()
reactor.sigTerm(signal.SIGTERM, None)
self.assertEquals(signal.SIGTERM, reactor._exitSignal)
def test_captureSIGBREAK(self):
"""
ReactorBase's SIGBREAK handler saves the value of SIGBREAK to the
_exitSignal attribute.
"""
if not hasattr(signal, "SIGBREAK"):
raise SkipTest("signal module does not have SIGBREAK")
reactor = TestSpySignalCapturingReactor()
reactor.sigBreak(signal.SIGBREAK, None)
self.assertEquals(signal.SIGBREAK, reactor._exitSignal)
try:
import signal
except ImportError:
ReactorBaseSignalTests.skip = "signal module not available"
@@ -0,0 +1,73 @@
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
Tests for L{twisted.internet._baseprocess} which implements process-related
functionality that is useful in all platforms supporting L{IReactorProcess}.
"""
__metaclass__ = type
from twisted.python.deprecate import getWarningMethod, setWarningMethod
from twisted.trial.unittest import TestCase
from twisted.internet._baseprocess import BaseProcess
class BaseProcessTests(TestCase):
"""
Tests for L{BaseProcess}, a parent class for other classes which represent
processes which implements functionality common to many different process
implementations.
"""
def test_callProcessExited(self):
"""
L{BaseProcess._callProcessExited} calls the C{processExited} method of
its C{proto} attribute and passes it a L{Failure} wrapping the given
exception.
"""
class FakeProto:
reason = None
def processExited(self, reason):
self.reason = reason
reason = RuntimeError("fake reason")
process = BaseProcess(FakeProto())
process._callProcessExited(reason)
process.proto.reason.trap(RuntimeError)
self.assertIs(reason, process.proto.reason.value)
def test_callProcessExitedMissing(self):
"""
L{BaseProcess._callProcessExited} emits a L{DeprecationWarning} if the
object referred to by its C{proto} attribute has no C{processExited}
method.
"""
class FakeProto:
pass
reason = object()
process = BaseProcess(FakeProto())
self.addCleanup(setWarningMethod, getWarningMethod())
warnings = []
def collect(message, category, stacklevel):
warnings.append((message, category, stacklevel))
setWarningMethod(collect)
process._callProcessExited(reason)
[(message, category, stacklevel)] = warnings
self.assertEqual(
message,
"Since Twisted 8.2, IProcessProtocol.processExited is required. "
"%s.%s must implement it." % (
FakeProto.__module__, FakeProto.__name__))
self.assertIs(category, DeprecationWarning)
# The stacklevel doesn't really make sense for this kind of
# deprecation. Requiring it to be 0 will at least avoid pointing to
# any part of Twisted or a random part of the application's code, which
# I think would be more misleading than having it point inside the
# warning system itself. -exarkun
self.assertEqual(stacklevel, 0)
@@ -0,0 +1,58 @@
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
A wrapper for L{twisted.internet.test._awaittests}, as that test module
includes keywords not valid in Pythons before 3.5.
"""
from __future__ import absolute_import, division
from twisted.python.compat import _PY35PLUS, _PY3, execfile
from twisted.python.filepath import FilePath
from twisted.trial import unittest
if _PY35PLUS:
_path = FilePath(__file__).parent().child("_awaittests.py.3only")
_g = {"__name__": __name__ + ".3-only.awaittests"}
execfile(_path.path, _g)
AwaitTests = _g["AwaitTests"]
else:
class AwaitTests(unittest.SynchronousTestCase):
"""
A dummy class to show that this test file was discovered but the tests
are unable to be run in this version of Python.
"""
skip = "async/await is not available before Python 3.5"
def test_notAvailable(self):
"""
A skipped test to show that this was not run because the Python is
too old.
"""
if _PY3:
_path = FilePath(__file__).parent().child("_yieldfromtests.py.3only")
_g = {"__name__": __name__ + ".3-only.yieldfromtests"}
execfile(_path.path, _g)
YieldFromTests = _g["YieldFromTests"]
else:
class YieldFromTests(unittest.SynchronousTestCase):
"""
A dummy class to show that this test file was discovered but the tests
are unable to be run in this version of Python.
"""
skip = "yield from is not available before Python 3"
def test_notAvailable(self):
"""
A skipped test to show that this was not run because the Python is
too old.
"""
__all__ = ["AwaitTests", "YieldFromTests"]
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,248 @@
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
Tests for L{twisted.internet.epollreactor}.
"""
from __future__ import division, absolute_import
from twisted.trial.unittest import TestCase
try:
from twisted.internet.epollreactor import _ContinuousPolling
except ImportError:
_ContinuousPolling = None
from twisted.internet.task import Clock
from twisted.internet.error import ConnectionDone
class Descriptor(object):
"""
Records reads and writes, as if it were a C{FileDescriptor}.
"""
def __init__(self):
self.events = []
def fileno(self):
return 1
def doRead(self):
self.events.append("read")
def doWrite(self):
self.events.append("write")
def connectionLost(self, reason):
reason.trap(ConnectionDone)
self.events.append("lost")
class ContinuousPollingTests(TestCase):
"""
L{_ContinuousPolling} can be used to read and write from C{FileDescriptor}
objects.
"""
def test_addReader(self):
"""
Adding a reader when there was previously no reader starts up a
C{LoopingCall}.
"""
poller = _ContinuousPolling(Clock())
self.assertIsNone(poller._loop)
reader = object()
self.assertFalse(poller.isReading(reader))
poller.addReader(reader)
self.assertIsNotNone(poller._loop)
self.assertTrue(poller._loop.running)
self.assertIs(poller._loop.clock, poller._reactor)
self.assertTrue(poller.isReading(reader))
def test_addWriter(self):
"""
Adding a writer when there was previously no writer starts up a
C{LoopingCall}.
"""
poller = _ContinuousPolling(Clock())
self.assertIsNone(poller._loop)
writer = object()
self.assertFalse(poller.isWriting(writer))
poller.addWriter(writer)
self.assertIsNotNone(poller._loop)
self.assertTrue(poller._loop.running)
self.assertIs(poller._loop.clock, poller._reactor)
self.assertTrue(poller.isWriting(writer))
def test_removeReader(self):
"""
Removing a reader stops the C{LoopingCall}.
"""
poller = _ContinuousPolling(Clock())
reader = object()
poller.addReader(reader)
poller.removeReader(reader)
self.assertIsNone(poller._loop)
self.assertEqual(poller._reactor.getDelayedCalls(), [])
self.assertFalse(poller.isReading(reader))
def test_removeWriter(self):
"""
Removing a writer stops the C{LoopingCall}.
"""
poller = _ContinuousPolling(Clock())
writer = object()
poller.addWriter(writer)
poller.removeWriter(writer)
self.assertIsNone(poller._loop)
self.assertEqual(poller._reactor.getDelayedCalls(), [])
self.assertFalse(poller.isWriting(writer))
def test_removeUnknown(self):
"""
Removing unknown readers and writers silently does nothing.
"""
poller = _ContinuousPolling(Clock())
poller.removeWriter(object())
poller.removeReader(object())
def test_multipleReadersAndWriters(self):
"""
Adding multiple readers and writers results in a single
C{LoopingCall}.
"""
poller = _ContinuousPolling(Clock())
writer = object()
poller.addWriter(writer)
self.assertIsNotNone(poller._loop)
poller.addWriter(object())
self.assertIsNotNone(poller._loop)
poller.addReader(object())
self.assertIsNotNone(poller._loop)
poller.addReader(object())
poller.removeWriter(writer)
self.assertIsNotNone(poller._loop)
self.assertTrue(poller._loop.running)
self.assertEqual(len(poller._reactor.getDelayedCalls()), 1)
def test_readerPolling(self):
"""
Adding a reader causes its C{doRead} to be called every 1
milliseconds.
"""
reactor = Clock()
poller = _ContinuousPolling(reactor)
desc = Descriptor()
poller.addReader(desc)
self.assertEqual(desc.events, [])
reactor.advance(0.00001)
self.assertEqual(desc.events, ["read"])
reactor.advance(0.00001)
self.assertEqual(desc.events, ["read", "read"])
reactor.advance(0.00001)
self.assertEqual(desc.events, ["read", "read", "read"])
def test_writerPolling(self):
"""
Adding a writer causes its C{doWrite} to be called every 1
milliseconds.
"""
reactor = Clock()
poller = _ContinuousPolling(reactor)
desc = Descriptor()
poller.addWriter(desc)
self.assertEqual(desc.events, [])
reactor.advance(0.001)
self.assertEqual(desc.events, ["write"])
reactor.advance(0.001)
self.assertEqual(desc.events, ["write", "write"])
reactor.advance(0.001)
self.assertEqual(desc.events, ["write", "write", "write"])
def test_connectionLostOnRead(self):
"""
If a C{doRead} returns a value indicating disconnection,
C{connectionLost} is called on it.
"""
reactor = Clock()
poller = _ContinuousPolling(reactor)
desc = Descriptor()
desc.doRead = lambda: ConnectionDone()
poller.addReader(desc)
self.assertEqual(desc.events, [])
reactor.advance(0.001)
self.assertEqual(desc.events, ["lost"])
def test_connectionLostOnWrite(self):
"""
If a C{doWrite} returns a value indicating disconnection,
C{connectionLost} is called on it.
"""
reactor = Clock()
poller = _ContinuousPolling(reactor)
desc = Descriptor()
desc.doWrite = lambda: ConnectionDone()
poller.addWriter(desc)
self.assertEqual(desc.events, [])
reactor.advance(0.001)
self.assertEqual(desc.events, ["lost"])
def test_removeAll(self):
"""
L{_ContinuousPolling.removeAll} removes all descriptors and returns
the readers and writers.
"""
poller = _ContinuousPolling(Clock())
reader = object()
writer = object()
both = object()
poller.addReader(reader)
poller.addReader(both)
poller.addWriter(writer)
poller.addWriter(both)
removed = poller.removeAll()
self.assertEqual(poller.getReaders(), [])
self.assertEqual(poller.getWriters(), [])
self.assertEqual(len(removed), 3)
self.assertEqual(set(removed), set([reader, writer, both]))
def test_getReaders(self):
"""
L{_ContinuousPolling.getReaders} returns a list of the read
descriptors.
"""
poller = _ContinuousPolling(Clock())
reader = object()
poller.addReader(reader)
self.assertIn(reader, poller.getReaders())
def test_getWriters(self):
"""
L{_ContinuousPolling.getWriters} returns a list of the write
descriptors.
"""
poller = _ContinuousPolling(Clock())
writer = object()
poller.addWriter(writer)
self.assertIn(writer, poller.getWriters())
if _ContinuousPolling is None:
skip = "epoll not supported in this environment."
@@ -0,0 +1,99 @@
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
Whitebox tests for L{twisted.internet.abstract.FileDescriptor}.
"""
from __future__ import division, absolute_import
from zope.interface.verify import verifyClass
from twisted.internet.abstract import FileDescriptor
from twisted.internet.interfaces import IPushProducer
from twisted.trial.unittest import SynchronousTestCase
class MemoryFile(FileDescriptor):
"""
A L{FileDescriptor} customization which writes to a Python list in memory
with certain limitations.
@ivar _written: A C{list} of C{bytes} which have been accepted as written.
@ivar _freeSpace: A C{int} giving the number of bytes which will be accepted
by future writes.
"""
connected = True
def __init__(self):
FileDescriptor.__init__(self, reactor=object())
self._written = []
self._freeSpace = 0
def startWriting(self):
pass
def stopWriting(self):
pass
def writeSomeData(self, data):
"""
Copy at most C{self._freeSpace} bytes from C{data} into C{self._written}.
@return: A C{int} indicating how many bytes were copied from C{data}.
"""
acceptLength = min(self._freeSpace, len(data))
if acceptLength:
self._freeSpace -= acceptLength
self._written.append(data[:acceptLength])
return acceptLength
class FileDescriptorTests(SynchronousTestCase):
"""
Tests for L{FileDescriptor}.
"""
def test_writeWithUnicodeRaisesException(self):
"""
L{FileDescriptor.write} doesn't accept unicode data.
"""
fileDescriptor = FileDescriptor(reactor=object())
self.assertRaises(TypeError, fileDescriptor.write, u'foo')
def test_writeSequenceWithUnicodeRaisesException(self):
"""
L{FileDescriptor.writeSequence} doesn't accept unicode data.
"""
fileDescriptor = FileDescriptor(reactor=object())
self.assertRaises(
TypeError, fileDescriptor.writeSequence, [b'foo', u'bar', b'baz'])
def test_implementInterfaceIPushProducer(self):
"""
L{FileDescriptor} should implement L{IPushProducer}.
"""
self.assertTrue(verifyClass(IPushProducer, FileDescriptor))
class WriteDescriptorTests(SynchronousTestCase):
"""
Tests for L{FileDescriptor}'s implementation of L{IWriteDescriptor}.
"""
def test_kernelBufferFull(self):
"""
When L{FileDescriptor.writeSomeData} returns C{0} to indicate no more
data can be written immediately, L{FileDescriptor.doWrite} returns
L{None}.
"""
descriptor = MemoryFile()
descriptor.write(b"hello, world")
self.assertIsNone(descriptor.doWrite())
@@ -0,0 +1,257 @@
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
GI/GTK3 reactor tests.
"""
from __future__ import division, absolute_import, print_function
import sys, os
try:
from twisted.internet import gireactor
from gi.repository import Gio
except ImportError:
gireactor = None
gtk3reactor = None
else:
# gtk3reactor may be unavailable even if gireactor is available; in
# particular in pygobject 3.4/gtk 3.6, when no X11 DISPLAY is found.
try:
from twisted.internet import gtk3reactor
except ImportError:
gtk3reactor = None
else:
from gi.repository import Gtk
from twisted.python.filepath import FilePath
from twisted.python.runtime import platform
from twisted.internet.defer import Deferred
from twisted.internet.error import ReactorAlreadyRunning
from twisted.internet.protocol import ProcessProtocol
from twisted.trial.unittest import TestCase, SkipTest
from twisted.internet.test.reactormixins import ReactorBuilder
from twisted.test.test_twisted import SetAsideModule
from twisted.internet.interfaces import IReactorProcess
from twisted.python.compat import _PY3
# Skip all tests if gi is unavailable:
if gireactor is None:
skip = "gtk3/gi not importable"
class GApplicationRegistrationTests(ReactorBuilder, TestCase):
"""
GtkApplication and GApplication are supported by
L{twisted.internet.gtk3reactor} and L{twisted.internet.gireactor}.
We inherit from L{ReactorBuilder} in order to use some of its
reactor-running infrastructure, but don't need its test-creation
functionality.
"""
def runReactor(self, app, reactor):
"""
Register the app, run the reactor, make sure app was activated, and
that reactor was running, and that reactor can be stopped.
"""
if not hasattr(app, "quit"):
raise SkipTest("Version of PyGObject is too old.")
result = []
def stop():
result.append("stopped")
reactor.stop()
def activate(widget):
result.append("activated")
reactor.callLater(0, stop)
app.connect('activate', activate)
# We want reactor.stop() to *always* stop the event loop, even if
# someone has called hold() on the application and never done the
# corresponding release() -- for more details see
# http://developer.gnome.org/gio/unstable/GApplication.html.
app.hold()
reactor.registerGApplication(app)
ReactorBuilder.runReactor(self, reactor)
self.assertEqual(result, ["activated", "stopped"])
def test_gApplicationActivate(self):
"""
L{Gio.Application} instances can be registered with a gireactor.
"""
reactor = gireactor.GIReactor(useGtk=False)
self.addCleanup(self.unbuildReactor, reactor)
app = Gio.Application(
application_id='com.twistedmatrix.trial.gireactor',
flags=Gio.ApplicationFlags.FLAGS_NONE)
self.runReactor(app, reactor)
def test_gtkApplicationActivate(self):
"""
L{Gtk.Application} instances can be registered with a gtk3reactor.
"""
reactor = gtk3reactor.Gtk3Reactor()
self.addCleanup(self.unbuildReactor, reactor)
app = Gtk.Application(
application_id='com.twistedmatrix.trial.gtk3reactor',
flags=Gio.ApplicationFlags.FLAGS_NONE)
self.runReactor(app, reactor)
if gtk3reactor is None:
test_gtkApplicationActivate.skip = (
"Gtk unavailable (may require running with X11 DISPLAY env set)")
def test_portable(self):
"""
L{gireactor.PortableGIReactor} doesn't support application
registration at this time.
"""
reactor = gireactor.PortableGIReactor()
self.addCleanup(self.unbuildReactor, reactor)
app = Gio.Application(
application_id='com.twistedmatrix.trial.gireactor',
flags=Gio.ApplicationFlags.FLAGS_NONE)
self.assertRaises(NotImplementedError,
reactor.registerGApplication, app)
def test_noQuit(self):
"""
Older versions of PyGObject lack C{Application.quit}, and so won't
allow registration.
"""
reactor = gireactor.GIReactor(useGtk=False)
self.addCleanup(self.unbuildReactor, reactor)
# An app with no "quit" method:
app = object()
exc = self.assertRaises(RuntimeError, reactor.registerGApplication, app)
self.assertTrue(exc.args[0].startswith(
"Application registration is not"))
def test_cantRegisterAfterRun(self):
"""
It is not possible to register a C{Application} after the reactor has
already started.
"""
reactor = gireactor.GIReactor(useGtk=False)
self.addCleanup(self.unbuildReactor, reactor)
app = Gio.Application(
application_id='com.twistedmatrix.trial.gireactor',
flags=Gio.ApplicationFlags.FLAGS_NONE)
def tryRegister():
exc = self.assertRaises(ReactorAlreadyRunning,
reactor.registerGApplication, app)
self.assertEqual(exc.args[0],
"Can't register application after reactor was started.")
reactor.stop()
reactor.callLater(0, tryRegister)
ReactorBuilder.runReactor(self, reactor)
def test_cantRegisterTwice(self):
"""
It is not possible to register more than one C{Application}.
"""
reactor = gireactor.GIReactor(useGtk=False)
self.addCleanup(self.unbuildReactor, reactor)
app = Gio.Application(
application_id='com.twistedmatrix.trial.gireactor',
flags=Gio.ApplicationFlags.FLAGS_NONE)
reactor.registerGApplication(app)
app2 = Gio.Application(
application_id='com.twistedmatrix.trial.gireactor2',
flags=Gio.ApplicationFlags.FLAGS_NONE)
exc = self.assertRaises(RuntimeError,
reactor.registerGApplication, app2)
self.assertEqual(exc.args[0],
"Can't register more than one application instance.")
class PygtkCompatibilityTests(TestCase):
"""
pygtk imports are either prevented, or a compatibility layer is used if
possible.
"""
def test_noCompatibilityLayer(self):
"""
If no compatibility layer is present, imports of gobject and friends
are disallowed.
We do this by running a process where we make sure gi.pygtkcompat
isn't present.
"""
if _PY3:
raise SkipTest("Python3 always has the compatibility layer.")
from twisted.internet import reactor
if not IReactorProcess.providedBy(reactor):
raise SkipTest("No process support available in this reactor.")
result = Deferred()
class Stdout(ProcessProtocol):
data = b""
def errReceived(self, err):
print(err)
def outReceived(self, data):
self.data += data
def processExited(self, reason):
result.callback(self.data)
path = FilePath(__file__).sibling(b"process_gireactornocompat.py").path
pyExe = FilePath(sys.executable)._asBytesPath()
# Pass in a PYTHONPATH that is the test runner's os.path, to make sure
# we're running from a checkout
reactor.spawnProcess(Stdout(), pyExe, [pyExe, path],
env={"PYTHONPATH": ":".join(sys.path)})
result.addCallback(self.assertEqual, b"success")
return result
def test_compatibilityLayer(self):
"""
If compatibility layer is present, importing gobject uses the gi
compatibility layer.
"""
if "gi.pygtkcompat" not in sys.modules:
raise SkipTest("This version of gi doesn't include pygtkcompat.")
import gobject
self.assertTrue(gobject.__name__.startswith("gi."))
class Gtk3ReactorTests(TestCase):
"""
Tests for L{gtk3reactor}.
"""
def test_requiresDISPLAY(self):
"""
On X11, L{gtk3reactor} is unimportable if the C{DISPLAY} environment
variable is not set.
"""
display = os.environ.get("DISPLAY", None)
if display is not None:
self.addCleanup(os.environ.__setitem__, "DISPLAY", display)
del os.environ["DISPLAY"]
with SetAsideModule("twisted.internet.gtk3reactor"):
exc = self.assertRaises(ImportError,
__import__, "twisted.internet.gtk3reactor")
self.assertEqual(
exc.args[0],
"Gtk3 requires X11, and no DISPLAY environment variable is set")
if platform.getType() != "posix" or platform.isMacOSX():
test_requiresDISPLAY.skip = "This test is only relevant when using X11"
@@ -0,0 +1,68 @@
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
Tests for twisted.internet.glibbase.
"""
from __future__ import division, absolute_import
import sys
from twisted.trial.unittest import TestCase
from twisted.internet._glibbase import ensureNotImported
class EnsureNotImportedTests(TestCase):
"""
L{ensureNotImported} protects against unwanted past and future imports.
"""
def test_ensureWhenNotImported(self):
"""
If the specified modules have never been imported, and import
prevention is requested, L{ensureNotImported} makes sure they will not
be imported in the future.
"""
modules = {}
self.patch(sys, "modules", modules)
ensureNotImported(["m1", "m2"], "A message.",
preventImports=["m1", "m2", "m3"])
self.assertEqual(modules, {"m1": None, "m2": None, "m3": None})
def test_ensureWhenNotImportedDontPrevent(self):
"""
If the specified modules have never been imported, and import
prevention is not requested, L{ensureNotImported} has no effect.
"""
modules = {}
self.patch(sys, "modules", modules)
ensureNotImported(["m1", "m2"], "A message.")
self.assertEqual(modules, {})
def test_ensureWhenFailedToImport(self):
"""
If the specified modules have been set to L{None} in C{sys.modules},
L{ensureNotImported} does not complain.
"""
modules = {"m2": None}
self.patch(sys, "modules", modules)
ensureNotImported(["m1", "m2"], "A message.", preventImports=["m1", "m2"])
self.assertEqual(modules, {"m1": None, "m2": None})
def test_ensureFailsWhenImported(self):
"""
If one of the specified modules has been previously imported,
L{ensureNotImported} raises an exception.
"""
module = object()
modules = {"m2": module}
self.patch(sys, "modules", modules)
e = self.assertRaises(ImportError, ensureNotImported,
["m1", "m2"], "A message.",
preventImports=["m1", "m2"])
self.assertEqual(modules, {"m2": module})
self.assertEqual(e.args, ("A message.",))
@@ -0,0 +1,384 @@
# -*- test-case-name: twisted.internet.test.test_inlinecb -*-
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
Tests for L{twisted.internet.defer.inlineCallbacks}.
Some tests for inlineCallbacks are defined in L{twisted.test.test_defgen} as
well.
"""
from __future__ import division, absolute_import
import sys
from twisted.trial.unittest import TestCase, SynchronousTestCase
from twisted.internet.defer import (
Deferred, returnValue, inlineCallbacks, CancelledError)
class StopIterationReturnTests(TestCase):
"""
On Python 3.4 and newer generator functions may use the C{return} statement
with a value, which is attached to the L{StopIteration} exception that is
raised.
L{inlineCallbacks} will use this value when it fires the C{callback}.
"""
def test_returnWithValue(self):
"""
If the C{return} statement has a value it is propagated back to the
L{Deferred} that the C{inlineCallbacks} function returned.
"""
environ = {"inlineCallbacks": inlineCallbacks}
exec("""
@inlineCallbacks
def f(d):
yield d
return 14
""", environ)
d1 = Deferred()
d2 = environ["f"](d1)
d1.callback(None)
self.assertEqual(self.successResultOf(d2), 14)
if sys.version_info < (3, 4):
StopIterationReturnTests.skip = "Test requires Python 3.4 or greater"
class NonLocalExitTests(TestCase):
"""
It's possible for L{returnValue} to be (accidentally) invoked at a stack
level below the L{inlineCallbacks}-decorated function which it is exiting.
If this happens, L{returnValue} should report useful errors.
If L{returnValue} is invoked from a function not decorated by
L{inlineCallbacks}, it will emit a warning if it causes an
L{inlineCallbacks} function further up the stack to exit.
"""
def mistakenMethod(self):
"""
This method mistakenly invokes L{returnValue}, despite the fact that it
is not decorated with L{inlineCallbacks}.
"""
returnValue(1)
def assertMistakenMethodWarning(self, resultList):
"""
Flush the current warnings and assert that we have been told that
C{mistakenMethod} was invoked, and that the result from the Deferred
that was fired (appended to the given list) is C{mistakenMethod}'s
result. The warning should indicate that an inlineCallbacks function
called 'inline' was made to exit.
"""
self.assertEqual(resultList, [1])
warnings = self.flushWarnings(offendingFunctions=[self.mistakenMethod])
self.assertEqual(len(warnings), 1)
self.assertEqual(warnings[0]['category'], DeprecationWarning)
self.assertEqual(
warnings[0]['message'],
"returnValue() in 'mistakenMethod' causing 'inline' to exit: "
"returnValue should only be invoked by functions decorated with "
"inlineCallbacks")
def test_returnValueNonLocalWarning(self):
"""
L{returnValue} will emit a non-local exit warning in the simplest case,
where the offending function is invoked immediately.
"""
@inlineCallbacks
def inline():
self.mistakenMethod()
returnValue(2)
yield 0
d = inline()
results = []
d.addCallback(results.append)
self.assertMistakenMethodWarning(results)
def test_returnValueNonLocalDeferred(self):
"""
L{returnValue} will emit a non-local warning in the case where the
L{inlineCallbacks}-decorated function has already yielded a Deferred
and therefore moved its generator function along.
"""
cause = Deferred()
@inlineCallbacks
def inline():
yield cause
self.mistakenMethod()
returnValue(2)
effect = inline()
results = []
effect.addCallback(results.append)
self.assertEqual(results, [])
cause.callback(1)
self.assertMistakenMethodWarning(results)
class ForwardTraceBackTests(SynchronousTestCase):
def test_forwardTracebacks(self):
"""
Chained inlineCallbacks are forwarding the traceback information
from generator to generator.
A first simple test with a couple of inline callbacks.
"""
@inlineCallbacks
def erroring():
yield "forcing generator"
raise Exception('Error Marker')
@inlineCallbacks
def calling():
yield erroring()
d = calling()
f = self.failureResultOf(d)
tb = f.getTraceback()
self.assertIn("in erroring", tb)
self.assertIn("in calling", tb)
self.assertIn("Error Marker", tb)
def test_forwardLotsOfTracebacks(self):
"""
Several Chained inlineCallbacks gives information about all generators.
A wider test with a 4 chained inline callbacks.
Application stack-trace should be reported, and implementation details
like "throwExceptionIntoGenerator" symbols are omitted from the stack.
Note that the previous test is testing the simple case, and this one is
testing the deep recursion case.
That case needs specific code in failure.py to accomodate to stack
breakage introduced by throwExceptionIntoGenerator.
Hence we keep the two tests in order to sort out which code we
might have regression in.
"""
@inlineCallbacks
def erroring():
yield "forcing generator"
raise Exception('Error Marker')
@inlineCallbacks
def calling3():
yield erroring()
@inlineCallbacks
def calling2():
yield calling3()
@inlineCallbacks
def calling():
yield calling2()
d = calling()
f = self.failureResultOf(d)
tb = f.getTraceback()
self.assertIn("in erroring", tb)
self.assertIn("in calling", tb)
self.assertIn("in calling2", tb)
self.assertIn("in calling3", tb)
self.assertNotIn("throwExceptionIntoGenerator", tb)
self.assertIn("Error Marker", tb)
self.assertIn("in erroring", f.getTraceback())
class UntranslatedError(Exception):
"""
Untranslated exception type when testing an exception translation.
"""
class TranslatedError(Exception):
"""
Translated exception type when testing an exception translation.
"""
class DontFail(Exception):
"""
Sample exception type.
"""
def __init__(self, actual):
Exception.__init__(self)
self.actualValue = actual
class CancellationTests(SynchronousTestCase):
"""
Tests for cancellation of L{Deferred}s returned by L{inlineCallbacks}.
For each of these tests, let:
- C{G} be a generator decorated with C{inlineCallbacks}
- C{D} be a L{Deferred} returned by C{G}
- C{C} be a L{Deferred} awaited by C{G} with C{yield}
"""
def setUp(self):
"""
Set up the list of outstanding L{Deferred}s.
"""
self.deferredsOutstanding = []
def tearDown(self):
"""
If any L{Deferred}s are still outstanding, fire them.
"""
while self.deferredsOutstanding:
self.deferredGotten()
@inlineCallbacks
def sampleInlineCB(self, getChildDeferred=None):
"""
Generator for testing cascade cancelling cases.
@param getChildDeferred: Some callable returning L{Deferred} that we
awaiting (with C{yield})
"""
if getChildDeferred is None:
getChildDeferred = self.getDeferred
try:
x = yield getChildDeferred()
except UntranslatedError:
raise TranslatedError()
except DontFail as df:
x = df.actualValue - 2
returnValue(x + 1)
def getDeferred(self):
"""
A sample function that returns a L{Deferred} that can be fired on
demand, by L{CancellationTests.deferredGotten}.
@return: L{Deferred} that can be fired on demand.
"""
self.deferredsOutstanding.append(Deferred())
return self.deferredsOutstanding[-1]
def deferredGotten(self, result=None):
"""
Fire the L{Deferred} returned from the least-recent call to
L{CancellationTests.getDeferred}.
@param result: result object to be used when firing the L{Deferred}.
"""
self.deferredsOutstanding.pop(0).callback(result)
def test_cascadeCancellingOnCancel(self):
"""
When C{D} cancelled, C{C} will be immediately cancelled too.
"""
childResultHolder = ['FAILURE']
def getChildDeferred():
d = Deferred()
def _eb(result):
childResultHolder[0] = result.check(CancelledError)
return result
d.addErrback(_eb)
return d
d = self.sampleInlineCB(getChildDeferred=getChildDeferred)
d.addErrback(lambda result: None)
d.cancel()
self.assertEqual(
childResultHolder[0],
CancelledError,
"no cascade cancelling occurs",
)
def test_errbackCancelledErrorOnCancel(self):
"""
When C{D} cancelled, CancelledError from C{C} will be errbacked
through C{D}.
"""
d = self.sampleInlineCB()
d.cancel()
self.assertRaises(
CancelledError,
self.failureResultOf(d).raiseException,
)
def test_errorToErrorTranslation(self):
"""
When C{D} is cancelled, and C raises a particular type of error, C{G}
may catch that error at the point of yielding and translate it into
a different error which may be received by application code.
"""
def cancel(it):
it.errback(UntranslatedError())
a = Deferred(cancel)
d = self.sampleInlineCB(lambda: a)
d.cancel()
self.assertRaises(
TranslatedError,
self.failureResultOf(d).raiseException,
)
def test_errorToSuccessTranslation(self):
"""
When C{D} is cancelled, and C{C} raises a particular type of error,
C{G} may catch that error at the point of yielding and translate it
into a result value which may be received by application code.
"""
def cancel(it):
it.errback(DontFail(4321))
a = Deferred(cancel)
d = self.sampleInlineCB(lambda: a)
results = []
d.addCallback(results.append)
d.cancel()
self.assertEquals(results, [4320])
def test_asynchronousCancellation(self):
"""
When C{D} is cancelled, it won't reach the callbacks added to it by
application code until C{C} reaches the point in its callback chain
where C{G} awaits it. Otherwise, application code won't be able to
track resource usage that C{D} may be using.
"""
moreDeferred = Deferred()
def deferMeMore(result):
result.trap(CancelledError)
return moreDeferred
def deferMe():
d = Deferred()
d.addErrback(deferMeMore)
return d
d = self.sampleInlineCB(getChildDeferred=deferMe)
d.cancel()
self.assertNoResult(d)
moreDeferred.callback(6543)
self.assertEqual(self.successResultOf(d), 6544)
@@ -0,0 +1,71 @@
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
Tests for L{twisted.internet.kqueuereactor}.
"""
from __future__ import division, absolute_import
import errno
from zope.interface import implementer
from twisted.trial.unittest import TestCase
try:
from twisted.internet.kqreactor import KQueueReactor, _IKQueue
kqueueSkip = None
except ImportError:
kqueueSkip = "KQueue not available."
def _fakeKEvent(*args, **kwargs):
"""
Do nothing.
"""
def makeFakeKQueue(testKQueue, testKEvent):
"""
Create a fake that implements L{_IKQueue}.
@param testKQueue: Something that acts like L{select.kqueue}.
@param testKEvent: Something that acts like L{select.kevent}.
@return: An implementation of L{_IKQueue} that includes C{testKQueue} and
C{testKEvent}.
"""
@implementer(_IKQueue)
class FakeKQueue(object):
kqueue = testKQueue
kevent = testKEvent
return FakeKQueue()
class KQueueTests(TestCase):
"""
These are tests for L{KQueueReactor}'s implementation, not its real world
behaviour. For that, look at
L{twisted.internet.test.reactormixins.ReactorBuilder}.
"""
skip = kqueueSkip
def test_EINTR(self):
"""
L{KQueueReactor} handles L{errno.EINTR} in C{doKEvent} by returning.
"""
class FakeKQueue(object):
"""
A fake KQueue that raises L{errno.EINTR} when C{control} is called,
like a real KQueue would if it was interrupted.
"""
def control(self, *args, **kwargs):
raise OSError(errno.EINTR, "Interrupted")
reactor = KQueueReactor(makeFakeKQueue(FakeKQueue, _fakeKEvent))
# This should return cleanly -- should not raise the OSError we're
# spawning, nor get upset and raise about the incomplete KQueue fake.
reactor.doKEvent(0)
@@ -0,0 +1,908 @@
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
Tests for implementations of L{IReactorProcess}.
@var properEnv: A copy of L{os.environ} which has L{bytes} keys/values on POSIX
platforms and native L{str} keys/values on Windows.
"""
from __future__ import division, absolute_import, print_function
import io
import os
import signal
import sys
import threading
import twisted
import subprocess
from twisted.trial.unittest import TestCase
from twisted.internet.test.reactormixins import ReactorBuilder
from twisted.python.log import msg, err
from twisted.python.runtime import platform
from twisted.python.filepath import FilePath, _asFilesystemBytes
from twisted.python.compat import (networkString, range, items,
bytesEnviron, unicode)
from twisted.internet import utils
from twisted.internet.interfaces import IReactorProcess, IProcessTransport
from twisted.internet.defer import Deferred, succeed
from twisted.internet.protocol import ProcessProtocol
from twisted.internet.error import ProcessDone, ProcessTerminated
# Get the current Python executable as a bytestring.
pyExe = FilePath(sys.executable)._asBytesPath()
twistedRoot = FilePath(twisted.__file__).parent().parent()
_uidgidSkip = None
if platform.isWindows():
resource = None
process = None
_uidgidSkip = "Cannot change UID/GID on Windows"
properEnv = dict(os.environ)
properEnv["PYTHONPATH"] = os.pathsep.join(sys.path)
else:
import resource
from twisted.internet import process
if os.getuid() != 0:
_uidgidSkip = "Cannot change UID/GID except as root"
properEnv = bytesEnviron()
properEnv[b"PYTHONPATH"] = os.pathsep.join(sys.path).encode(
sys.getfilesystemencoding())
def onlyOnPOSIX(testMethod):
"""
Only run this test on POSIX platforms.
@param testMethod: A test function, being decorated.
@return: the C{testMethod} argument.
"""
if resource is None:
testMethod.skip = "Test only applies to POSIX platforms."
return testMethod
class _ShutdownCallbackProcessProtocol(ProcessProtocol):
"""
An L{IProcessProtocol} which fires a Deferred when the process it is
associated with ends.
@ivar received: A C{dict} mapping file descriptors to lists of bytes
received from the child process on those file descriptors.
"""
def __init__(self, whenFinished):
self.whenFinished = whenFinished
self.received = {}
def childDataReceived(self, fd, bytes):
self.received.setdefault(fd, []).append(bytes)
def processEnded(self, reason):
self.whenFinished.callback(None)
class ProcessTestsBuilderBase(ReactorBuilder):
"""
Base class for L{IReactorProcess} tests which defines some tests which
can be applied to PTY or non-PTY uses of C{spawnProcess}.
Subclasses are expected to set the C{usePTY} attribute to C{True} or
C{False}.
"""
requiredInterfaces = [IReactorProcess]
def test_processTransportInterface(self):
"""
L{IReactorProcess.spawnProcess} connects the protocol passed to it
to a transport which provides L{IProcessTransport}.
"""
ended = Deferred()
protocol = _ShutdownCallbackProcessProtocol(ended)
reactor = self.buildReactor()
transport = reactor.spawnProcess(
protocol, pyExe, [pyExe, b"-c", b""],
usePTY=self.usePTY)
# The transport is available synchronously, so we can check it right
# away (unlike many transport-based tests). This is convenient even
# though it's probably not how the spawnProcess interface should really
# work.
# We're not using verifyObject here because part of
# IProcessTransport is a lie - there are no getHost or getPeer
# methods. See #1124.
self.assertTrue(IProcessTransport.providedBy(transport))
# Let the process run and exit so we don't leave a zombie around.
ended.addCallback(lambda ignored: reactor.stop())
self.runReactor(reactor)
def _writeTest(self, write):
"""
Helper for testing L{IProcessTransport} write functionality. This
method spawns a child process and gives C{write} a chance to write some
bytes to it. It then verifies that the bytes were actually written to
it (by relying on the child process to echo them back).
@param write: A two-argument callable. This is invoked with a process
transport and some bytes to write to it.
"""
reactor = self.buildReactor()
ended = Deferred()
protocol = _ShutdownCallbackProcessProtocol(ended)
bytesToSend = b"hello, world" + networkString(os.linesep)
program = (
b"import sys\n"
b"sys.stdout.write(sys.stdin.readline())\n"
)
def startup():
transport = reactor.spawnProcess(
protocol, pyExe, [pyExe, b"-c", program])
try:
write(transport, bytesToSend)
except:
err(None, "Unhandled exception while writing")
transport.signalProcess('KILL')
reactor.callWhenRunning(startup)
ended.addCallback(lambda ignored: reactor.stop())
self.runReactor(reactor)
self.assertEqual(bytesToSend, b"".join(protocol.received[1]))
def test_write(self):
"""
L{IProcessTransport.write} writes the specified C{bytes} to the standard
input of the child process.
"""
def write(transport, bytesToSend):
transport.write(bytesToSend)
self._writeTest(write)
def test_writeSequence(self):
"""
L{IProcessTransport.writeSequence} writes the specified C{list} of
C{bytes} to the standard input of the child process.
"""
def write(transport, bytesToSend):
transport.writeSequence([bytesToSend])
self._writeTest(write)
def test_writeToChild(self):
"""
L{IProcessTransport.writeToChild} writes the specified C{bytes} to the
specified file descriptor of the child process.
"""
def write(transport, bytesToSend):
transport.writeToChild(0, bytesToSend)
self._writeTest(write)
def test_writeToChildBadFileDescriptor(self):
"""
L{IProcessTransport.writeToChild} raises L{KeyError} if passed a file
descriptor which is was not set up by L{IReactorProcess.spawnProcess}.
"""
def write(transport, bytesToSend):
try:
self.assertRaises(KeyError, transport.writeToChild, 13, bytesToSend)
finally:
# Just get the process to exit so the test can complete
transport.write(bytesToSend)
self._writeTest(write)
def test_spawnProcessEarlyIsReaped(self):
"""
If, before the reactor is started with L{IReactorCore.run}, a
process is started with L{IReactorProcess.spawnProcess} and
terminates, the process is reaped once the reactor is started.
"""
reactor = self.buildReactor()
# Create the process with no shared file descriptors, so that there
# are no other events for the reactor to notice and "cheat" with.
# We want to be sure it's really dealing with the process exiting,
# not some associated event.
if self.usePTY:
childFDs = None
else:
childFDs = {}
# Arrange to notice the SIGCHLD.
signaled = threading.Event()
def handler(*args):
signaled.set()
signal.signal(signal.SIGCHLD, handler)
# Start a process - before starting the reactor!
ended = Deferred()
reactor.spawnProcess(
_ShutdownCallbackProcessProtocol(ended), pyExe,
[pyExe, b"-c", b""], usePTY=self.usePTY, childFDs=childFDs)
# Wait for the SIGCHLD (which might have been delivered before we got
# here, but that's okay because the signal handler was installed above,
# before we could have gotten it).
signaled.wait(120)
if not signaled.isSet():
self.fail("Timed out waiting for child process to exit.")
# Capture the processEnded callback.
result = []
ended.addCallback(result.append)
if result:
# The synchronous path through spawnProcess / Process.__init__ /
# registerReapProcessHandler was encountered. There's no reason to
# start the reactor, because everything is done already.
return
# Otherwise, though, start the reactor so it can tell us the process
# exited.
ended.addCallback(lambda ignored: reactor.stop())
self.runReactor(reactor)
# Make sure the reactor stopped because the Deferred fired.
self.assertTrue(result)
if getattr(signal, 'SIGCHLD', None) is None:
test_spawnProcessEarlyIsReaped.skip = (
"Platform lacks SIGCHLD, early-spawnProcess test can't work.")
def test_processExitedWithSignal(self):
"""
The C{reason} argument passed to L{IProcessProtocol.processExited} is a
L{ProcessTerminated} instance if the child process exits with a signal.
"""
sigName = 'TERM'
sigNum = getattr(signal, 'SIG' + sigName)
exited = Deferred()
source = (
b"import sys\n"
# Talk so the parent process knows the process is running. This is
# necessary because ProcessProtocol.makeConnection may be called
# before this process is exec'd. It would be unfortunate if we
# SIGTERM'd the Twisted process while it was on its way to doing
# the exec.
b"sys.stdout.write('x')\n"
b"sys.stdout.flush()\n"
b"sys.stdin.read()\n")
class Exiter(ProcessProtocol):
def childDataReceived(self, fd, data):
msg('childDataReceived(%d, %r)' % (fd, data))
self.transport.signalProcess(sigName)
def childConnectionLost(self, fd):
msg('childConnectionLost(%d)' % (fd,))
def processExited(self, reason):
msg('processExited(%r)' % (reason,))
# Protect the Deferred from the failure so that it follows
# the callback chain. This doesn't use the errback chain
# because it wants to make sure reason is a Failure. An
# Exception would also make an errback-based test pass, and
# that would be wrong.
exited.callback([reason])
def processEnded(self, reason):
msg('processEnded(%r)' % (reason,))
reactor = self.buildReactor()
reactor.callWhenRunning(
reactor.spawnProcess, Exiter(), pyExe,
[pyExe, b"-c", source], usePTY=self.usePTY)
def cbExited(args):
failure, = args
# Trapping implicitly verifies that it's a Failure (rather than
# an exception) and explicitly makes sure it's the right type.
failure.trap(ProcessTerminated)
err = failure.value
if platform.isWindows():
# Windows can't really /have/ signals, so it certainly can't
# report them as the reason for termination. Maybe there's
# something better we could be doing here, anyway? Hard to
# say. Anyway, this inconsistency between different platforms
# is extremely unfortunate and I would remove it if I
# could. -exarkun
self.assertIsNone(err.signal)
self.assertEqual(err.exitCode, 1)
else:
self.assertEqual(err.signal, sigNum)
self.assertIsNone(err.exitCode)
exited.addCallback(cbExited)
exited.addErrback(err)
exited.addCallback(lambda ign: reactor.stop())
self.runReactor(reactor)
def test_systemCallUninterruptedByChildExit(self):
"""
If a child process exits while a system call is in progress, the system
call should not be interfered with. In particular, it should not fail
with EINTR.
Older versions of Twisted installed a SIGCHLD handler on POSIX without
using the feature exposed by the SA_RESTART flag to sigaction(2). The
most noticeable problem this caused was for blocking reads and writes to
sometimes fail with EINTR.
"""
reactor = self.buildReactor()
result = []
def f():
try:
exe = pyExe.decode(sys.getfilesystemencoding())
subprocess.Popen([exe, "-c", "import time; time.sleep(0.1)"])
f2 = subprocess.Popen([exe, "-c",
("import time; time.sleep(0.5);"
"print(\'Foo\')")],
stdout=subprocess.PIPE)
# The read call below will blow up with an EINTR from the
# SIGCHLD from the first process exiting if we install a
# SIGCHLD handler without SA_RESTART. (which we used to do)
with f2.stdout:
result.append(f2.stdout.read())
finally:
reactor.stop()
reactor.callWhenRunning(f)
self.runReactor(reactor)
self.assertEqual(result, [b"Foo" + os.linesep.encode('ascii')])
@onlyOnPOSIX
def test_openFileDescriptors(self):
"""
Processes spawned with spawnProcess() close all extraneous file
descriptors in the parent. They do have a stdin, stdout, and stderr
open.
"""
# To test this, we are going to open a file descriptor in the parent
# that is unlikely to be opened in the child, then verify that it's not
# open in the child.
source = networkString("""
import sys
sys.path.insert(0, '{0}')
from twisted.internet import process
sys.stdout.write(repr(process._listOpenFDs()))
sys.stdout.flush()""".format(twistedRoot.path))
r, w = os.pipe()
self.addCleanup(os.close, r)
self.addCleanup(os.close, w)
# The call to "os.listdir()" (in _listOpenFDs's implementation) opens a
# file descriptor (with "opendir"), which shows up in _listOpenFDs's
# result. And speaking of "random" file descriptors, the code required
# for _listOpenFDs itself imports logger, which imports random, which
# (depending on your Python version) might leave /dev/urandom open.
# More generally though, even if we were to use an extremely minimal C
# program, the operating system would be within its rights to open file
# descriptors we might not know about in the C library's
# initialization; things like debuggers, profilers, or nsswitch plugins
# might open some and this test should pass in those environments.
# Although some of these file descriptors aren't predictable, we should
# at least be able to select a very large file descriptor which is very
# unlikely to be opened automatically in the subprocess. (Apply a
# fudge factor to avoid hard-coding something too near a limit
# condition like the maximum possible file descriptor, which a library
# might at least hypothetically select.)
fudgeFactor = 17
unlikelyFD = (resource.getrlimit(resource.RLIMIT_NOFILE)[0]
- fudgeFactor)
os.dup2(w, unlikelyFD)
self.addCleanup(os.close, unlikelyFD)
output = io.BytesIO()
class GatheringProtocol(ProcessProtocol):
outReceived = output.write
def processEnded(self, reason):
reactor.stop()
reactor = self.buildReactor()
reactor.callWhenRunning(
reactor.spawnProcess, GatheringProtocol(), pyExe,
[pyExe, b"-Wignore", b"-c", source], usePTY=self.usePTY)
self.runReactor(reactor)
reportedChildFDs = set(eval(output.getvalue()))
stdFDs = [0, 1, 2]
# Unfortunately this assertion is still not *entirely* deterministic,
# since hypothetically, any library could open any file descriptor at
# any time. See comment above.
self.assertEqual(
reportedChildFDs.intersection(set(stdFDs + [unlikelyFD])),
set(stdFDs)
)
@onlyOnPOSIX
def test_errorDuringExec(self):
"""
When L{os.execvpe} raises an exception, it will format that exception
on stderr as UTF-8, regardless of system encoding information.
"""
def execvpe(*args, **kw):
# Ensure that real traceback formatting has some non-ASCII in it,
# by forcing the filename of the last frame to contain non-ASCII.
filename = u"<\N{SNOWMAN}>"
if not isinstance(filename, str):
filename = filename.encode("utf-8")
codeobj = compile("1/0", filename, "single")
eval(codeobj)
self.patch(os, "execvpe", execvpe)
self.patch(sys, "getfilesystemencoding", lambda: "ascii")
reactor = self.buildReactor()
output = io.BytesIO()
@reactor.callWhenRunning
def whenRunning():
class TracebackCatcher(ProcessProtocol, object):
errReceived = output.write
def processEnded(self, reason):
reactor.stop()
reactor.spawnProcess(TracebackCatcher(), pyExe,
[pyExe, b"-c", b""])
self.runReactor(reactor, timeout=30)
self.assertIn(u"\N{SNOWMAN}".encode("utf-8"), output.getvalue())
def test_timelyProcessExited(self):
"""
If a spawned process exits, C{processExited} will be called in a
timely manner.
"""
reactor = self.buildReactor()
class ExitingProtocol(ProcessProtocol):
exited = False
def processExited(protoSelf, reason):
protoSelf.exited = True
reactor.stop()
self.assertEqual(reason.value.exitCode, 0)
protocol = ExitingProtocol()
reactor.callWhenRunning(
reactor.spawnProcess, protocol, pyExe,
[pyExe, b"-c", b"raise SystemExit(0)"],
usePTY=self.usePTY)
# This will timeout if processExited isn't called:
self.runReactor(reactor, timeout=30)
self.assertTrue(protocol.exited)
def _changeIDTest(self, which):
"""
Launch a child process, using either the C{uid} or C{gid} argument to
L{IReactorProcess.spawnProcess} to change either its UID or GID to a
different value. If the child process reports this hasn't happened,
raise an exception to fail the test.
@param which: Either C{b"uid"} or C{b"gid"}.
"""
program = [
"import os",
"raise SystemExit(os.get%s() != 1)" % (which,)]
container = []
class CaptureExitStatus(ProcessProtocol):
def processEnded(self, reason):
container.append(reason)
reactor.stop()
reactor = self.buildReactor()
protocol = CaptureExitStatus()
reactor.callWhenRunning(
reactor.spawnProcess, protocol, pyExe,
[pyExe, "-c", "\n".join(program)],
**{which: 1})
self.runReactor(reactor)
self.assertEqual(0, container[0].value.exitCode)
def test_changeUID(self):
"""
If a value is passed for L{IReactorProcess.spawnProcess}'s C{uid}, the
child process is run with that UID.
"""
self._changeIDTest("uid")
if _uidgidSkip is not None:
test_changeUID.skip = _uidgidSkip
def test_changeGID(self):
"""
If a value is passed for L{IReactorProcess.spawnProcess}'s C{gid}, the
child process is run with that GID.
"""
self._changeIDTest("gid")
if _uidgidSkip is not None:
test_changeGID.skip = _uidgidSkip
def test_processExitedRaises(self):
"""
If L{IProcessProtocol.processExited} raises an exception, it is logged.
"""
# Ideally we wouldn't need to poke the process module; see
# https://twistedmatrix.com/trac/ticket/6889
reactor = self.buildReactor()
class TestException(Exception):
pass
class Protocol(ProcessProtocol):
def processExited(self, reason):
reactor.stop()
raise TestException("processedExited raised")
protocol = Protocol()
transport = reactor.spawnProcess(
protocol, pyExe, [pyExe, b"-c", b""],
usePTY=self.usePTY)
self.runReactor(reactor)
# Manually clean-up broken process handler.
# Only required if the test fails on systems that support
# the process module.
if process is not None:
for pid, handler in items(process.reapProcessHandlers):
if handler is not transport:
continue
process.unregisterReapProcessHandler(pid, handler)
self.fail("After processExited raised, transport was left in"
" reapProcessHandlers")
self.assertEqual(1, len(self.flushLoggedErrors(TestException)))
class ProcessTestsBuilder(ProcessTestsBuilderBase):
"""
Builder defining tests relating to L{IReactorProcess} for child processes
which do not have a PTY.
"""
usePTY = False
keepStdioOpenProgram = b'twisted.internet.test.process_helper'
if platform.isWindows():
keepStdioOpenArg = b"windows"
else:
# Just a value that doesn't equal "windows"
keepStdioOpenArg = b""
# Define this test here because PTY-using processes only have stdin and
# stdout and the test would need to be different for that to work.
def test_childConnectionLost(self):
"""
L{IProcessProtocol.childConnectionLost} is called each time a file
descriptor associated with a child process is closed.
"""
connected = Deferred()
lost = {0: Deferred(), 1: Deferred(), 2: Deferred()}
class Closer(ProcessProtocol):
def makeConnection(self, transport):
connected.callback(transport)
def childConnectionLost(self, childFD):
lost[childFD].callback(None)
target = b"twisted.internet.test.process_loseconnection"
reactor = self.buildReactor()
reactor.callWhenRunning(
reactor.spawnProcess, Closer(), pyExe,
[pyExe, b"-m", target], env=properEnv, usePTY=self.usePTY)
def cbConnected(transport):
transport.write(b'2\n')
return lost[2].addCallback(lambda ign: transport)
connected.addCallback(cbConnected)
def lostSecond(transport):
transport.write(b'1\n')
return lost[1].addCallback(lambda ign: transport)
connected.addCallback(lostSecond)
def lostFirst(transport):
transport.write(b'\n')
connected.addCallback(lostFirst)
connected.addErrback(err)
def cbEnded(ignored):
reactor.stop()
connected.addCallback(cbEnded)
self.runReactor(reactor)
# This test is here because PTYProcess never delivers childConnectionLost.
def test_processEnded(self):
"""
L{IProcessProtocol.processEnded} is called after the child process
exits and L{IProcessProtocol.childConnectionLost} is called for each of
its file descriptors.
"""
ended = Deferred()
lost = []
class Ender(ProcessProtocol):
def childDataReceived(self, fd, data):
msg('childDataReceived(%d, %r)' % (fd, data))
self.transport.loseConnection()
def childConnectionLost(self, childFD):
msg('childConnectionLost(%d)' % (childFD,))
lost.append(childFD)
def processExited(self, reason):
msg('processExited(%r)' % (reason,))
def processEnded(self, reason):
msg('processEnded(%r)' % (reason,))
ended.callback([reason])
reactor = self.buildReactor()
reactor.callWhenRunning(
reactor.spawnProcess, Ender(), pyExe,
[pyExe, b"-m", self.keepStdioOpenProgram, b"child",
self.keepStdioOpenArg],
env=properEnv, usePTY=self.usePTY)
def cbEnded(args):
failure, = args
failure.trap(ProcessDone)
self.assertEqual(set(lost), set([0, 1, 2]))
ended.addCallback(cbEnded)
ended.addErrback(err)
ended.addCallback(lambda ign: reactor.stop())
self.runReactor(reactor)
# This test is here because PTYProcess.loseConnection does not actually
# close the file descriptors to the child process. This test needs to be
# written fairly differently for PTYProcess.
def test_processExited(self):
"""
L{IProcessProtocol.processExited} is called when the child process
exits, even if file descriptors associated with the child are still
open.
"""
exited = Deferred()
allLost = Deferred()
lost = []
class Waiter(ProcessProtocol):
def childDataReceived(self, fd, data):
msg('childDataReceived(%d, %r)' % (fd, data))
def childConnectionLost(self, childFD):
msg('childConnectionLost(%d)' % (childFD,))
lost.append(childFD)
if len(lost) == 3:
allLost.callback(None)
def processExited(self, reason):
msg('processExited(%r)' % (reason,))
# See test_processExitedWithSignal
exited.callback([reason])
self.transport.loseConnection()
reactor = self.buildReactor()
reactor.callWhenRunning(
reactor.spawnProcess, Waiter(), pyExe,
[pyExe, b"-u", b"-m", self.keepStdioOpenProgram, b"child",
self.keepStdioOpenArg],
env=properEnv, usePTY=self.usePTY)
def cbExited(args):
failure, = args
failure.trap(ProcessDone)
msg('cbExited; lost = %s' % (lost,))
self.assertEqual(lost, [])
return allLost
exited.addCallback(cbExited)
def cbAllLost(ignored):
self.assertEqual(set(lost), set([0, 1, 2]))
exited.addCallback(cbAllLost)
exited.addErrback(err)
exited.addCallback(lambda ign: reactor.stop())
self.runReactor(reactor)
def makeSourceFile(self, sourceLines):
"""
Write the given list of lines to a text file and return the absolute
path to it.
"""
script = _asFilesystemBytes(self.mktemp())
with open(script, 'wt') as scriptFile:
scriptFile.write(os.linesep.join(sourceLines) + os.linesep)
return os.path.abspath(script)
def test_shebang(self):
"""
Spawning a process with an executable which is a script starting
with an interpreter definition line (#!) uses that interpreter to
evaluate the script.
"""
shebangOutput = b'this is the shebang output'
scriptFile = self.makeSourceFile([
"#!%s" % (pyExe.decode('ascii'),),
"import sys",
"sys.stdout.write('%s')" % (shebangOutput.decode('ascii'),),
"sys.stdout.flush()"])
os.chmod(scriptFile, 0o700)
reactor = self.buildReactor()
def cbProcessExited(args):
out, err, code = args
msg("cbProcessExited((%r, %r, %d))" % (out, err, code))
self.assertEqual(out, shebangOutput)
self.assertEqual(err, b"")
self.assertEqual(code, 0)
def shutdown(passthrough):
reactor.stop()
return passthrough
def start():
d = utils.getProcessOutputAndValue(scriptFile, reactor=reactor)
d.addBoth(shutdown)
d.addCallback(cbProcessExited)
d.addErrback(err)
reactor.callWhenRunning(start)
self.runReactor(reactor)
def test_processCommandLineArguments(self):
"""
Arguments given to spawnProcess are passed to the child process as
originally intended.
"""
us = b"twisted.internet.test.process_cli"
args = [b'hello', b'"', b' \t|<>^&', br'"\\"hello\\"', br'"foo\ bar baz\""']
# Ensure that all non-NUL characters can be passed too.
allChars = "".join(map(chr, range(1, 255)))
if isinstance(allChars, unicode):
allChars.encode("utf-8")
reactor = self.buildReactor()
def processFinished(finishedArgs):
output, err, code = finishedArgs
output = output.split(b'\0')
# Drop the trailing \0.
output.pop()
self.assertEqual(args, output)
def shutdown(result):
reactor.stop()
return result
def spawnChild():
d = succeed(None)
d.addCallback(lambda dummy: utils.getProcessOutputAndValue(
pyExe, [b"-m", us] + args, env=properEnv,
reactor=reactor))
d.addCallback(processFinished)
d.addBoth(shutdown)
reactor.callWhenRunning(spawnChild)
self.runReactor(reactor)
globals().update(ProcessTestsBuilder.makeTestCaseClasses())
class PTYProcessTestsBuilder(ProcessTestsBuilderBase):
"""
Builder defining tests relating to L{IReactorProcess} for child processes
which have a PTY.
"""
usePTY = True
if platform.isWindows():
skip = "PTYs are not supported on Windows."
elif platform.isMacOSX():
skip = "PTYs are flaky from a Darwin bug. See #8840."
skippedReactors = {
"twisted.internet.pollreactor.PollReactor":
"macOS's poll() does not support PTYs"}
globals().update(PTYProcessTestsBuilder.makeTestCaseClasses())
class PotentialZombieWarningTests(TestCase):
"""
Tests for L{twisted.internet.error.PotentialZombieWarning}.
"""
def test_deprecated(self):
"""
Accessing L{PotentialZombieWarning} via the
I{PotentialZombieWarning} attribute of L{twisted.internet.error}
results in a deprecation warning being emitted.
"""
from twisted.internet import error
error.PotentialZombieWarning
warnings = self.flushWarnings([self.test_deprecated])
self.assertEqual(warnings[0]['category'], DeprecationWarning)
self.assertEqual(
warnings[0]['message'],
"twisted.internet.error.PotentialZombieWarning was deprecated in "
"Twisted 10.0.0: There is no longer any potential for zombie "
"process.")
self.assertEqual(len(warnings), 1)
class ProcessIsUnimportableOnUnsupportedPlatormsTests(TestCase):
"""
Tests to ensure that L{twisted.internet.process} is unimportable on
platforms where it does not work (namely Windows).
"""
def test_unimportableOnWindows(self):
"""
L{twisted.internet.process} is unimportable on Windows.
"""
with self.assertRaises(ImportError):
import twisted.internet.process
twisted.internet.process # shh pyflakes
if not platform.isWindows():
test_unimportableOnWindows.skip = "Only relevant on Windows."
@@ -0,0 +1,125 @@
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
Tests for L{twisted.internet._sigchld}, an alternate, superior SIGCHLD
monitoring API.
"""
from __future__ import division, absolute_import
import os, signal, errno
from twisted.python.runtime import platformType
from twisted.python.log import msg
from twisted.trial.unittest import SynchronousTestCase
if platformType == "posix":
from twisted.internet.fdesc import setNonBlocking
from twisted.internet._signals import installHandler, isDefaultHandler
else:
skip = "These tests can only run on POSIX platforms."
class SetWakeupSIGCHLDTests(SynchronousTestCase):
"""
Tests for the L{signal.set_wakeup_fd} implementation of the
L{installHandler} and L{isDefaultHandler} APIs.
"""
def pipe(self):
"""
Create a non-blocking pipe which will be closed after the currently
running test.
"""
read, write = os.pipe()
self.addCleanup(os.close, read)
self.addCleanup(os.close, write)
setNonBlocking(read)
setNonBlocking(write)
return read, write
def setUp(self):
"""
Save the current SIGCHLD handler as reported by L{signal.signal} and
the current file descriptor registered with L{installHandler}.
"""
handler = signal.getsignal(signal.SIGCHLD)
if handler != signal.SIG_DFL:
self.signalModuleHandler = handler
signal.signal(signal.SIGCHLD, signal.SIG_DFL)
else:
self.signalModuleHandler = None
self.oldFD = installHandler(-1)
if self.signalModuleHandler is not None and self.oldFD != -1:
msg("Previous test didn't clean up after its SIGCHLD setup: %r %r"
% (self.signalModuleHandler, self.oldFD))
def tearDown(self):
"""
Restore whatever signal handler was present when setUp ran.
"""
# If tests set up any kind of handlers, clear them out.
installHandler(-1)
signal.signal(signal.SIGCHLD, signal.SIG_DFL)
# Now restore whatever the setup was before the test ran.
if self.signalModuleHandler is not None:
signal.signal(signal.SIGCHLD, self.signalModuleHandler)
elif self.oldFD != -1:
installHandler(self.oldFD)
def test_isDefaultHandler(self):
"""
L{isDefaultHandler} returns true if the SIGCHLD handler is SIG_DFL,
false otherwise.
"""
self.assertTrue(isDefaultHandler())
signal.signal(signal.SIGCHLD, signal.SIG_IGN)
self.assertFalse(isDefaultHandler())
signal.signal(signal.SIGCHLD, signal.SIG_DFL)
self.assertTrue(isDefaultHandler())
signal.signal(signal.SIGCHLD, lambda *args: None)
self.assertFalse(isDefaultHandler())
def test_returnOldFD(self):
"""
L{installHandler} returns the previously registered file descriptor.
"""
read, write = self.pipe()
oldFD = installHandler(write)
self.assertEqual(installHandler(oldFD), write)
def test_uninstallHandler(self):
"""
C{installHandler(-1)} removes the SIGCHLD handler completely.
"""
read, write = self.pipe()
self.assertTrue(isDefaultHandler())
installHandler(write)
self.assertFalse(isDefaultHandler())
installHandler(-1)
self.assertTrue(isDefaultHandler())
def test_installHandler(self):
"""
The file descriptor passed to L{installHandler} has a byte written to
it when SIGCHLD is delivered to the process.
"""
read, write = self.pipe()
installHandler(write)
exc = self.assertRaises(OSError, os.read, read, 1)
self.assertEqual(exc.errno, errno.EAGAIN)
os.kill(os.getpid(), signal.SIGCHLD)
self.assertEqual(len(os.read(read, 5)), 1)
@@ -0,0 +1,199 @@
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
Tests for L{twisted.internet.stdio}.
"""
from __future__ import absolute_import, division
from twisted.python.runtime import platform
from twisted.internet.test.reactormixins import ReactorBuilder
from twisted.internet.protocol import Protocol
if not platform.isWindows():
from twisted.internet.stdio import StandardIO
class StdioFilesTests(ReactorBuilder):
"""
L{StandardIO} supports reading and writing to filesystem files.
"""
def setUp(self):
path = self.mktemp()
open(path, "wb").close()
self.extraFile = open(path, "rb+")
self.addCleanup(self.extraFile.close)
def test_addReader(self):
"""
Adding a filesystem file reader to a reactor will make sure it is
polled.
"""
reactor = self.buildReactor()
class DataProtocol(Protocol):
data = b""
def dataReceived(self, data):
self.data += data
# It'd be better to stop reactor on connectionLost, but that
# fails on FreeBSD, probably due to
# http://bugs.python.org/issue9591:
if self.data == b"hello!":
reactor.stop()
path = self.mktemp()
with open(path, "wb") as f:
f.write(b"hello!")
with open(path, "rb") as f:
# Read bytes from a file, deliver them to a protocol instance:
protocol = DataProtocol()
StandardIO(protocol, stdin=f.fileno(),
stdout=self.extraFile.fileno(),
reactor=reactor)
self.runReactor(reactor)
self.assertEqual(protocol.data, b"hello!")
def test_addWriter(self):
"""
Adding a filesystem file writer to a reactor will make sure it is
polled.
"""
reactor = self.buildReactor()
class DisconnectProtocol(Protocol):
def connectionLost(self, reason):
reactor.stop()
path = self.mktemp()
with open(path, "wb") as f:
# Write bytes to a transport, hopefully have them written to a
# file:
protocol = DisconnectProtocol()
StandardIO(protocol, stdout=f.fileno(),
stdin=self.extraFile.fileno(), reactor=reactor)
protocol.transport.write(b"hello")
protocol.transport.write(b", world")
protocol.transport.loseConnection()
self.runReactor(reactor)
with open(path, "rb") as f:
self.assertEqual(f.read(), b"hello, world")
def test_removeReader(self):
"""
Removing a filesystem file reader from a reactor will make sure it is
no longer polled.
"""
reactor = self.buildReactor()
self.addCleanup(self.unbuildReactor, reactor)
path = self.mktemp()
open(path, "wb").close()
with open(path, "rb") as f:
# Have the reader added:
stdio = StandardIO(Protocol(), stdin=f.fileno(),
stdout=self.extraFile.fileno(),
reactor=reactor)
self.assertIn(stdio._reader, reactor.getReaders())
stdio._reader.stopReading()
self.assertNotIn(stdio._reader, reactor.getReaders())
def test_removeWriter(self):
"""
Removing a filesystem file writer from a reactor will make sure it is
no longer polled.
"""
reactor = self.buildReactor()
self.addCleanup(self.unbuildReactor, reactor)
# Cleanup might fail if file is GCed too soon:
self.f = f = open(self.mktemp(), "wb")
# Have the reader added:
protocol = Protocol()
stdio = StandardIO(protocol, stdout=f.fileno(),
stdin=self.extraFile.fileno(),
reactor=reactor)
protocol.transport.write(b"hello")
self.assertIn(stdio._writer, reactor.getWriters())
stdio._writer.stopWriting()
self.assertNotIn(stdio._writer, reactor.getWriters())
def test_removeAll(self):
"""
Calling C{removeAll} on a reactor includes descriptors that are
filesystem files.
"""
reactor = self.buildReactor()
self.addCleanup(self.unbuildReactor, reactor)
path = self.mktemp()
open(path, "wb").close()
# Cleanup might fail if file is GCed too soon:
self.f = f = open(path, "rb")
# Have the reader added:
stdio = StandardIO(Protocol(), stdin=f.fileno(),
stdout=self.extraFile.fileno(), reactor=reactor)
# And then removed:
removed = reactor.removeAll()
self.assertIn(stdio._reader, removed)
self.assertNotIn(stdio._reader, reactor.getReaders())
def test_getReaders(self):
"""
C{reactor.getReaders} includes descriptors that are filesystem files.
"""
reactor = self.buildReactor()
self.addCleanup(self.unbuildReactor, reactor)
path = self.mktemp()
open(path, "wb").close()
# Cleanup might fail if file is GCed too soon:
with open(path, "rb") as f:
# Have the reader added:
stdio = StandardIO(Protocol(), stdin=f.fileno(),
stdout=self.extraFile.fileno(), reactor=reactor)
self.assertIn(stdio._reader, reactor.getReaders())
def test_getWriters(self):
"""
C{reactor.getWriters} includes descriptors that are filesystem files.
"""
reactor = self.buildReactor()
self.addCleanup(self.unbuildReactor, reactor)
# Cleanup might fail if file is GCed too soon:
self.f = f = open(self.mktemp(), "wb")
# Have the reader added:
stdio = StandardIO(Protocol(), stdout=f.fileno(),
stdin=self.extraFile.fileno(), reactor=reactor)
self.assertNotIn(stdio._writer, reactor.getWriters())
stdio._writer.startWriting()
self.assertIn(stdio._writer, reactor.getWriters())
if platform.isWindows():
skip = ("StandardIO does not accept stdout as an argument to Windows. "
"Testing redirection to a file is therefore harder.")
globals().update(StdioFilesTests.makeTestCaseClasses())
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,515 @@
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
Tests for L{twisted.internet.testing}.
"""
from zope.interface.verify import verifyObject
from twisted.internet.interfaces import (
ITransport,
IPushProducer,
IConsumer,
IReactorTCP,
IReactorSSL,
IReactorUNIX,
IAddress,
IListeningPort,
IConnector
)
from twisted.internet.address import IPv4Address
from twisted.trial.unittest import TestCase
from twisted.internet.testing import (
StringTransport,
MemoryReactor,
RaisingMemoryReactor,
NonStreamingProducer
)
from twisted.internet.protocol import ClientFactory, Factory
from twisted.python.reflect import namedAny
class StringTransportTests(TestCase):
"""
Tests for L{twisted.internet.testing.StringTransport}.
"""
def setUp(self):
self.transport = StringTransport()
def test_interfaces(self):
"""
L{StringTransport} instances provide L{ITransport}, L{IPushProducer},
and L{IConsumer}.
"""
self.assertTrue(verifyObject(ITransport, self.transport))
self.assertTrue(verifyObject(IPushProducer, self.transport))
self.assertTrue(verifyObject(IConsumer, self.transport))
def test_registerProducer(self):
"""
L{StringTransport.registerProducer} records the arguments supplied to
it as instance attributes.
"""
producer = object()
streaming = object()
self.transport.registerProducer(producer, streaming)
self.assertIs(self.transport.producer, producer)
self.assertIs(self.transport.streaming, streaming)
def test_disallowedRegisterProducer(self):
"""
L{StringTransport.registerProducer} raises L{RuntimeError} if a
producer is already registered.
"""
producer = object()
self.transport.registerProducer(producer, True)
self.assertRaises(
RuntimeError, self.transport.registerProducer, object(), False)
self.assertIs(self.transport.producer, producer)
self.assertTrue(self.transport.streaming)
def test_unregisterProducer(self):
"""
L{StringTransport.unregisterProducer} causes the transport to forget
about the registered producer and makes it possible to register a new
one.
"""
oldProducer = object()
newProducer = object()
self.transport.registerProducer(oldProducer, False)
self.transport.unregisterProducer()
self.assertIsNone(self.transport.producer)
self.transport.registerProducer(newProducer, True)
self.assertIs(self.transport.producer, newProducer)
self.assertTrue(self.transport.streaming)
def test_invalidUnregisterProducer(self):
"""
L{StringTransport.unregisterProducer} raises L{RuntimeError} if called
when no producer is registered.
"""
self.assertRaises(RuntimeError, self.transport.unregisterProducer)
def test_initialProducerState(self):
"""
L{StringTransport.producerState} is initially C{'producing'}.
"""
self.assertEqual(self.transport.producerState, 'producing')
def test_pauseProducing(self):
"""
L{StringTransport.pauseProducing} changes the C{producerState} of the
transport to C{'paused'}.
"""
self.transport.pauseProducing()
self.assertEqual(self.transport.producerState, 'paused')
def test_resumeProducing(self):
"""
L{StringTransport.resumeProducing} changes the C{producerState} of the
transport to C{'producing'}.
"""
self.transport.pauseProducing()
self.transport.resumeProducing()
self.assertEqual(self.transport.producerState, 'producing')
def test_stopProducing(self):
"""
L{StringTransport.stopProducing} changes the C{'producerState'} of the
transport to C{'stopped'}.
"""
self.transport.stopProducing()
self.assertEqual(self.transport.producerState, 'stopped')
def test_stoppedTransportCannotPause(self):
"""
L{StringTransport.pauseProducing} raises L{RuntimeError} if the
transport has been stopped.
"""
self.transport.stopProducing()
self.assertRaises(RuntimeError, self.transport.pauseProducing)
def test_stoppedTransportCannotResume(self):
"""
L{StringTransport.resumeProducing} raises L{RuntimeError} if the
transport has been stopped.
"""
self.transport.stopProducing()
self.assertRaises(RuntimeError, self.transport.resumeProducing)
def test_disconnectingTransportCannotPause(self):
"""
L{StringTransport.pauseProducing} raises L{RuntimeError} if the
transport is being disconnected.
"""
self.transport.loseConnection()
self.assertRaises(RuntimeError, self.transport.pauseProducing)
def test_disconnectingTransportCannotResume(self):
"""
L{StringTransport.resumeProducing} raises L{RuntimeError} if the
transport is being disconnected.
"""
self.transport.loseConnection()
self.assertRaises(RuntimeError, self.transport.resumeProducing)
def test_loseConnectionSetsDisconnecting(self):
"""
L{StringTransport.loseConnection} toggles the C{disconnecting} instance
variable to C{True}.
"""
self.assertFalse(self.transport.disconnecting)
self.transport.loseConnection()
self.assertTrue(self.transport.disconnecting)
def test_specifiedHostAddress(self):
"""
If a host address is passed to L{StringTransport.__init__}, that
value is returned from L{StringTransport.getHost}.
"""
address = object()
self.assertIs(StringTransport(address).getHost(), address)
def test_specifiedPeerAddress(self):
"""
If a peer address is passed to L{StringTransport.__init__}, that
value is returned from L{StringTransport.getPeer}.
"""
address = object()
self.assertIs(
StringTransport(peerAddress=address).getPeer(), address)
def test_defaultHostAddress(self):
"""
If no host address is passed to L{StringTransport.__init__}, an
L{IPv4Address} is returned from L{StringTransport.getHost}.
"""
address = StringTransport().getHost()
self.assertIsInstance(address, IPv4Address)
def test_defaultPeerAddress(self):
"""
If no peer address is passed to L{StringTransport.__init__}, an
L{IPv4Address} is returned from L{StringTransport.getPeer}.
"""
address = StringTransport().getPeer()
self.assertIsInstance(address, IPv4Address)
class ReactorTests(TestCase):
"""
Tests for L{MemoryReactor} and L{RaisingMemoryReactor}.
"""
def test_memoryReactorProvides(self):
"""
L{MemoryReactor} provides all of the attributes described by the
interfaces it advertises.
"""
memoryReactor = MemoryReactor()
verifyObject(IReactorTCP, memoryReactor)
verifyObject(IReactorSSL, memoryReactor)
verifyObject(IReactorUNIX, memoryReactor)
def test_raisingReactorProvides(self):
"""
L{RaisingMemoryReactor} provides all of the attributes described by the
interfaces it advertises.
"""
raisingReactor = RaisingMemoryReactor()
verifyObject(IReactorTCP, raisingReactor)
verifyObject(IReactorSSL, raisingReactor)
verifyObject(IReactorUNIX, raisingReactor)
def test_connectDestination(self):
"""
L{MemoryReactor.connectTCP}, L{MemoryReactor.connectSSL}, and
L{MemoryReactor.connectUNIX} will return an L{IConnector} whose
C{getDestination} method returns an L{IAddress} with attributes which
reflect the values passed.
"""
memoryReactor = MemoryReactor()
for connector in [memoryReactor.connectTCP(
"test.example.com", 8321, ClientFactory()),
memoryReactor.connectSSL(
"test.example.com", 8321, ClientFactory(),
None)]:
verifyObject(IConnector, connector)
address = connector.getDestination()
verifyObject(IAddress, address)
self.assertEqual(address.host, "test.example.com")
self.assertEqual(address.port, 8321)
connector = memoryReactor.connectUNIX(b"/fake/path", ClientFactory())
verifyObject(IConnector, connector)
address = connector.getDestination()
verifyObject(IAddress, address)
self.assertEqual(address.name, b"/fake/path")
def test_listenDefaultHost(self):
"""
L{MemoryReactor.listenTCP}, L{MemoryReactor.listenSSL} and
L{MemoryReactor.listenUNIX} will return an L{IListeningPort} whose
C{getHost} method returns an L{IAddress}; C{listenTCP} and C{listenSSL}
will have a default host of C{'0.0.0.0'}, and a port that reflects the
value passed, and C{listenUNIX} will have a name that reflects the path
passed.
"""
memoryReactor = MemoryReactor()
for port in [memoryReactor.listenTCP(8242, Factory()),
memoryReactor.listenSSL(8242, Factory(), None)]:
verifyObject(IListeningPort, port)
address = port.getHost()
verifyObject(IAddress, address)
self.assertEqual(address.host, '0.0.0.0')
self.assertEqual(address.port, 8242)
port = memoryReactor.listenUNIX(b"/path/to/socket", Factory())
verifyObject(IListeningPort, port)
address = port.getHost()
verifyObject(IAddress, address)
self.assertEqual(address.name, b"/path/to/socket")
def test_readers(self):
"""
Adding, removing, and listing readers works.
"""
reader = object()
reactor = MemoryReactor()
reactor.addReader(reader)
reactor.addReader(reader)
self.assertEqual(reactor.getReaders(), [reader])
reactor.removeReader(reader)
self.assertEqual(reactor.getReaders(), [])
def test_writers(self):
"""
Adding, removing, and listing writers works.
"""
writer = object()
reactor = MemoryReactor()
reactor.addWriter(writer)
reactor.addWriter(writer)
self.assertEqual(reactor.getWriters(), [writer])
reactor.removeWriter(writer)
self.assertEqual(reactor.getWriters(), [])
class TestConsumer(object):
"""
A very basic test consumer for use with the NonStreamingProducerTests.
"""
def __init__(self):
self.writes = []
self.producer = None
self.producerStreaming = None
def registerProducer(self, producer, streaming):
"""
Registers a single producer with this consumer. Just keeps track of it.
@param producer: The producer to register.
@param streaming: Whether the producer is a streaming one or not.
"""
self.producer = producer
self.producerStreaming = streaming
def unregisterProducer(self):
"""
Forget the producer we had previously registered.
"""
self.producer = None
self.producerStreaming = None
def write(self, data):
"""
Some data was written to the consumer: stores it for later use.
@param data: The data to write.
"""
self.writes.append(data)
class NonStreamingProducerTests(TestCase):
"""
Tests for the L{NonStreamingProducer} to validate behaviour.
"""
def test_producesOnly10Times(self):
"""
When the L{NonStreamingProducer} has resumeProducing called 10 times,
it writes the counter each time and then fails.
"""
consumer = TestConsumer()
producer = NonStreamingProducer(consumer)
consumer.registerProducer(producer, False)
self.assertIs(consumer.producer, producer)
self.assertIs(producer.consumer, consumer)
self.assertFalse(consumer.producerStreaming)
for _ in range(10):
producer.resumeProducing()
# We should have unregistered the producer and printed the 10 results.
expectedWrites = [
b'0', b'1', b'2', b'3', b'4', b'5', b'6', b'7', b'8', b'9'
]
self.assertIsNone(consumer.producer)
self.assertIsNone(consumer.producerStreaming)
self.assertIsNone(producer.consumer)
self.assertEqual(consumer.writes, expectedWrites)
# Another attempt to produce fails.
self.assertRaises(RuntimeError, producer.resumeProducing)
def test_cannotPauseProduction(self):
"""
When the L{NonStreamingProducer} is paused, it raises a
L{RuntimeError}.
"""
consumer = TestConsumer()
producer = NonStreamingProducer(consumer)
consumer.registerProducer(producer, False)
# Produce once, just to be safe.
producer.resumeProducing()
self.assertRaises(RuntimeError, producer.pauseProducing)
class DeprecationTests(TestCase):
"""
Deprecations in L{twisted.test.proto_helpers}.
"""
def helper(self, test, obj):
new_path = 'twisted.internet.testing.{}'.format(obj.__name__)
warnings = self.flushWarnings(
[test])
self.assertEqual(DeprecationWarning, warnings[0]['category'])
self.assertEqual(1, len(warnings))
self.assertIn(new_path, warnings[0]['message'])
self.assertIs(obj, namedAny(new_path))
def test_accumulatingProtocol(self):
from twisted.test.proto_helpers import AccumulatingProtocol
self.helper(self.test_accumulatingProtocol,
AccumulatingProtocol)
def test_lineSendingProtocol(self):
from twisted.test.proto_helpers import LineSendingProtocol
self.helper(self.test_lineSendingProtocol,
LineSendingProtocol)
def test_fakeDatagramTransport(self):
from twisted.test.proto_helpers import FakeDatagramTransport
self.helper(self.test_fakeDatagramTransport,
FakeDatagramTransport)
def test_stringTransport(self):
from twisted.test.proto_helpers import StringTransport
self.helper(self.test_stringTransport,
StringTransport)
def test_stringTransportWithDisconnection(self):
from twisted.test.proto_helpers import (
StringTransportWithDisconnection)
self.helper(self.test_stringTransportWithDisconnection,
StringTransportWithDisconnection)
def test_stringIOWithoutClosing(self):
from twisted.test.proto_helpers import StringIOWithoutClosing
self.helper(self.test_stringIOWithoutClosing,
StringIOWithoutClosing)
def test__fakeConnector(self):
from twisted.test.proto_helpers import _FakeConnector
self.helper(self.test__fakeConnector,
_FakeConnector)
def test__fakePort(self):
from twisted.test.proto_helpers import _FakePort
self.helper(self.test__fakePort,
_FakePort)
def test_memoryReactor(self):
from twisted.test.proto_helpers import MemoryReactor
self.helper(self.test_memoryReactor,
MemoryReactor)
def test_memoryReactorClock(self):
from twisted.test.proto_helpers import MemoryReactorClock
self.helper(self.test_memoryReactorClock,
MemoryReactorClock)
def test_raisingMemoryReactor(self):
from twisted.test.proto_helpers import RaisingMemoryReactor
self.helper(self.test_raisingMemoryReactor,
RaisingMemoryReactor)
def test_nonStreamingProducer(self):
from twisted.test.proto_helpers import NonStreamingProducer
self.helper(self.test_nonStreamingProducer,
NonStreamingProducer)
def test_waitUntilAllDisconnected(self):
from twisted.test.proto_helpers import (
waitUntilAllDisconnected)
self.helper(self.test_waitUntilAllDisconnected,
waitUntilAllDisconnected)
def test_eventLoggingObserver(self):
from twisted.test.proto_helpers import EventLoggingObserver
self.helper(self.test_eventLoggingObserver,
EventLoggingObserver)
@@ -0,0 +1,175 @@
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
Tests for L{twisted.internet.serialport}.
"""
import os
import shutil
import tempfile
from twisted.trial import unittest
from twisted.internet.protocol import Protocol
from twisted.python.failure import Failure
from twisted.python.runtime import platform
from twisted.internet.test.test_serialport import DoNothing
testingForced = 'TWISTED_FORCE_SERIAL_TESTS' in os.environ
try:
from twisted.internet import serialport
import serial
except ImportError:
if testingForced:
raise
serialport = None
serial = None
if serialport is not None:
class RegularFileSerial(serial.Serial):
def __init__(self, *args, **kwargs):
super(RegularFileSerial, self).__init__(*args, **kwargs)
self.captured_args = args
self.captured_kwargs = kwargs
def _reconfigurePort(self):
pass
def _reconfigure_port(self):
pass
class RegularFileSerialPort(serialport.SerialPort):
_serialFactory = RegularFileSerial
def __init__(self, *args, **kwargs):
cbInQue = kwargs.get('cbInQue')
if 'cbInQue' in kwargs:
del kwargs['cbInQue']
self.comstat = serial.win32.COMSTAT
self.comstat.cbInQue = cbInQue
super(RegularFileSerialPort, self).__init__(*args, **kwargs)
def _clearCommError(self):
return True, self.comstat
class CollectReceivedProtocol(Protocol):
def __init__(self):
self.received_data = []
def dataReceived(self, data):
self.received_data.append(data)
class Win32SerialPortTests(unittest.TestCase):
"""
Minimal testing for Twisted's Win32 serial port support.
"""
if not testingForced:
if not platform.isWindows():
skip = "This test must run on Windows."
elif not serialport:
skip = "Windows serial port support is not available."
def setUp(self):
# Re-usable protocol and reactor
self.protocol = Protocol()
self.reactor = DoNothing()
self.directory = tempfile.mkdtemp()
self.path = os.path.join(self.directory, 'fake_serial')
data = b'1234'
with open(self.path, 'wb') as f:
f.write(data)
def tearDown(self):
shutil.rmtree(self.directory)
def test_serialPortDefaultArgs(self):
"""
Test correct positional and keyword arguments have been
passed to the C{serial.Serial} object.
"""
port = RegularFileSerialPort(self.protocol, self.path, self.reactor)
# Validate args
self.assertEqual((self.path,), port._serial.captured_args)
# Validate kwargs
kwargs = port._serial.captured_kwargs
self.assertEqual(9600, kwargs["baudrate"])
self.assertEqual(serial.EIGHTBITS, kwargs["bytesize"])
self.assertEqual(serial.PARITY_NONE, kwargs["parity"])
self.assertEqual(serial.STOPBITS_ONE, kwargs["stopbits"])
self.assertEqual(0, kwargs["xonxoff"])
self.assertEqual(0, kwargs["rtscts"])
self.assertEqual(None, kwargs["timeout"])
port.connectionLost(Failure(Exception("Cleanup")))
def test_serialPortInitiallyConnected(self):
"""
Test the port is connected at initialization time, and
C{Protocol.makeConnection} has been called on the desired protocol.
"""
self.assertEqual(0, self.protocol.connected)
port = RegularFileSerialPort(self.protocol, self.path, self.reactor)
self.assertEqual(1, port.connected)
self.assertEqual(1, self.protocol.connected)
self.assertEqual(port, self.protocol.transport)
port.connectionLost(Failure(Exception("Cleanup")))
def common_exerciseHandleAccess(self, cbInQue):
port = RegularFileSerialPort(
protocol=self.protocol,
deviceNameOrPortNumber=self.path,
reactor=self.reactor,
cbInQue=cbInQue,
)
port.serialReadEvent()
port.write(b'')
port.write(b'abcd')
port.write(b'ABCD')
port.serialWriteEvent()
port.serialWriteEvent()
port.connectionLost(Failure(Exception("Cleanup")))
# No assertion since the point is simply to make sure that in all cases
# the port handle resolves instead of raising an exception.
def test_exerciseHandleAccess_1(self):
self.common_exerciseHandleAccess(cbInQue=False)
def test_exerciseHandleAccess_2(self):
self.common_exerciseHandleAccess(cbInQue=True)
def common_serialPortReturnsBytes(self, cbInQue):
protocol = CollectReceivedProtocol()
port = RegularFileSerialPort(
protocol=protocol,
deviceNameOrPortNumber=self.path,
reactor=self.reactor,
cbInQue=cbInQue,
)
port.serialReadEvent()
self.assertTrue(all(
isinstance(d, bytes) for d in protocol.received_data
))
port.connectionLost(Failure(Exception("Cleanup")))
def test_serialPortReturnsBytes_1(self):
self.common_serialPortReturnsBytes(cbInQue=False)
def test_serialPortReturnsBytes_2(self):
self.common_serialPortReturnsBytes(cbInQue=True)
@@ -0,0 +1,78 @@
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
This module integrates Tkinter with twisted.internet's mainloop.
Maintainer: Itamar Shtull-Trauring
To use, do::
| tksupport.install(rootWidget)
and then run your reactor as usual - do *not* call Tk's mainloop(),
use Twisted's regular mechanism for running the event loop.
Likewise, to stop your program you will need to stop Twisted's
event loop. For example, if you want closing your root widget to
stop Twisted::
| root.protocol('WM_DELETE_WINDOW', reactor.stop)
When using Aqua Tcl/Tk on macOS the standard Quit menu item in
your application might become unresponsive without the additional
fix::
| root.createcommand("::tk::mac::Quit", reactor.stop)
@see: U{Tcl/TkAqua FAQ for more info<http://wiki.tcl.tk/12987>}
"""
from twisted.internet import task
from twisted.python.compat import _PY3
if _PY3:
import tkinter.simpledialog as tkSimpleDialog
import tkinter.messagebox as tkMessageBox
else:
import tkSimpleDialog, tkMessageBox
_task = None
def install(widget, ms=10, reactor=None):
"""Install a Tkinter.Tk() object into the reactor."""
installTkFunctions()
global _task
_task = task.LoopingCall(widget.update)
_task.start(ms / 1000.0, False)
def uninstall():
"""Remove the root Tk widget from the reactor.
Call this before destroy()ing the root widget.
"""
global _task
_task.stop()
_task = None
def installTkFunctions():
import twisted.python.util
twisted.python.util.getPassword = getPassword
def getPassword(prompt = '', confirm = 0):
while 1:
try1 = tkSimpleDialog.askstring('Password Dialog', prompt, show='*')
if not confirm:
return try1
try2 = tkSimpleDialog.askstring('Password Dialog', 'Confirm Password', show='*')
if try1 == try2:
return try1
else:
tkMessageBox.showerror('Password Mismatch', 'Passwords did not match, starting over')
__all__ = ["install", "uninstall"]
@@ -0,0 +1,188 @@
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
This module provides wxPython event loop support for Twisted.
In order to use this support, simply do the following::
| from twisted.internet import wxreactor
| wxreactor.install()
Then, when your root wxApp has been created::
| from twisted.internet import reactor
| reactor.registerWxApp(yourApp)
| reactor.run()
Then use twisted.internet APIs as usual. Stop the event loop using
reactor.stop(), not yourApp.ExitMainLoop().
IMPORTANT: tests will fail when run under this reactor. This is
expected and probably does not reflect on the reactor's ability to run
real applications.
"""
try:
from queue import Empty, Queue
except ImportError:
from Queue import Empty, Queue
try:
from wx import PySimpleApp as wxPySimpleApp, CallAfter as wxCallAfter, \
Timer as wxTimer
except ImportError:
# older version of wxPython:
from wxPython.wx import wxPySimpleApp, wxCallAfter, wxTimer
from twisted.python import log, runtime
from twisted.internet import _threadedselect
class ProcessEventsTimer(wxTimer):
"""
Timer that tells wx to process pending events.
This is necessary on macOS, probably due to a bug in wx, if we want
wxCallAfters to be handled when modal dialogs, menus, etc. are open.
"""
def __init__(self, wxapp):
wxTimer.__init__(self)
self.wxapp = wxapp
def Notify(self):
"""
Called repeatedly by wx event loop.
"""
self.wxapp.ProcessPendingEvents()
class WxReactor(_threadedselect.ThreadedSelectReactor):
"""
wxPython reactor.
wxPython drives the event loop, select() runs in a thread.
"""
_stopping = False
def registerWxApp(self, wxapp):
"""
Register wxApp instance with the reactor.
"""
self.wxapp = wxapp
def _installSignalHandlersAgain(self):
"""
wx sometimes removes our own signal handlers, so re-add them.
"""
try:
# make _handleSignals happy:
import signal
signal.signal(signal.SIGINT, signal.default_int_handler)
except ImportError:
return
self._handleSignals()
def stop(self):
"""
Stop the reactor.
"""
if self._stopping:
return
self._stopping = True
_threadedselect.ThreadedSelectReactor.stop(self)
def _runInMainThread(self, f):
"""
Schedule function to run in main wx/Twisted thread.
Called by the select() thread.
"""
if hasattr(self, "wxapp"):
wxCallAfter(f)
else:
# wx shutdown but twisted hasn't
self._postQueue.put(f)
def _stopWx(self):
"""
Stop the wx event loop if it hasn't already been stopped.
Called during Twisted event loop shutdown.
"""
if hasattr(self, "wxapp"):
self.wxapp.ExitMainLoop()
def run(self, installSignalHandlers=True):
"""
Start the reactor.
"""
self._postQueue = Queue()
if not hasattr(self, "wxapp"):
log.msg("registerWxApp() was not called on reactor, "
"registering my own wxApp instance.")
self.registerWxApp(wxPySimpleApp())
# start select() thread:
self.interleave(self._runInMainThread,
installSignalHandlers=installSignalHandlers)
if installSignalHandlers:
self.callLater(0, self._installSignalHandlersAgain)
# add cleanup events:
self.addSystemEventTrigger("after", "shutdown", self._stopWx)
self.addSystemEventTrigger("after", "shutdown",
lambda: self._postQueue.put(None))
# On macOS, work around wx bug by starting timer to ensure
# wxCallAfter calls are always processed. We don't wake up as
# often as we could since that uses too much CPU.
if runtime.platform.isMacOSX():
t = ProcessEventsTimer(self.wxapp)
t.Start(2) # wake up every 2ms
self.wxapp.MainLoop()
wxapp = self.wxapp
del self.wxapp
if not self._stopping:
# wx event loop exited without reactor.stop() being
# called. At this point events from select() thread will
# be added to _postQueue, but some may still be waiting
# unprocessed in wx, thus the ProcessPendingEvents()
# below.
self.stop()
wxapp.ProcessPendingEvents() # deal with any queued wxCallAfters
while 1:
try:
f = self._postQueue.get(timeout=0.01)
except Empty:
continue
else:
if f is None:
break
try:
f()
except:
log.err()
def install():
"""
Configure the twisted mainloop to be run inside the wxPython mainloop.
"""
reactor = WxReactor()
from twisted.internet.main import installReactor
installReactor(reactor)
return reactor
__all__ = ['install']
@@ -0,0 +1,59 @@
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
#
"""Old method of wxPython support for Twisted.
twisted.internet.wxreactor is probably a better choice.
To use::
| # given a wxApp instance called myWxAppInstance:
| from twisted.internet import wxsupport
| wxsupport.install(myWxAppInstance)
Use Twisted's APIs for running and stopping the event loop, don't use
wxPython's methods.
On Windows the Twisted event loop might block when dialogs are open
or menus are selected.
Maintainer: Itamar Shtull-Trauring
"""
import warnings
warnings.warn("wxsupport is not fully functional on Windows, wxreactor is better.")
from twisted.python._oldstyle import _oldStyle
from twisted.internet import reactor
@_oldStyle
class wxRunner:
"""Make sure GUI events are handled."""
def __init__(self, app):
self.app = app
def run(self):
"""
Execute pending WX events followed by WX idle events and
reschedule.
"""
# run wx events
while self.app.Pending():
self.app.Dispatch()
# run wx idle events
self.app.ProcessIdle()
reactor.callLater(0.02, self.run)
def install(app):
"""Install the wxPython support, given a wxApp instance"""
runner = wxRunner(app)
reactor.callLater(0.02, runner.run)
__all__ = ["install"]