Funktionierender Prototyp des Serious Games zur Vermittlung von Wissen zu Software-Engineering-Arbeitsmodellen.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

basic.py 31KB

1 year ago
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912
  1. # -*- test-case-name: twisted.protocols.test.test_basic -*-
  2. # Copyright (c) Twisted Matrix Laboratories.
  3. # See LICENSE for details.
  4. """
  5. Basic protocols, such as line-oriented, netstring, and int prefixed strings.
  6. """
  7. import math
  8. # System imports
  9. import re
  10. from io import BytesIO
  11. from struct import calcsize, pack, unpack
  12. from zope.interface import implementer
  13. # Twisted imports
  14. from twisted.internet import defer, interfaces, protocol
  15. from twisted.python import log
  16. # Unfortunately we cannot use regular string formatting on Python 3; see
  17. # http://bugs.python.org/issue3982 for details.
  18. def _formatNetstring(data):
  19. return b"".join([str(len(data)).encode("ascii"), b":", data, b","])
  20. _formatNetstring.__doc__ = """
  21. Convert some C{bytes} into netstring format.
  22. @param data: C{bytes} that will be reformatted.
  23. """
  24. DEBUG = 0
  25. class NetstringParseError(ValueError):
  26. """
  27. The incoming data is not in valid Netstring format.
  28. """
  29. class IncompleteNetstring(Exception):
  30. """
  31. Not enough data to complete a netstring.
  32. """
  33. class NetstringReceiver(protocol.Protocol):
  34. """
  35. A protocol that sends and receives netstrings.
  36. See U{http://cr.yp.to/proto/netstrings.txt} for the specification of
  37. netstrings. Every netstring starts with digits that specify the length
  38. of the data. This length specification is separated from the data by
  39. a colon. The data is terminated with a comma.
  40. Override L{stringReceived} to handle received netstrings. This
  41. method is called with the netstring payload as a single argument
  42. whenever a complete netstring is received.
  43. Security features:
  44. 1. Messages are limited in size, useful if you don't want
  45. someone sending you a 500MB netstring (change C{self.MAX_LENGTH}
  46. to the maximum length you wish to accept).
  47. 2. The connection is lost if an illegal message is received.
  48. @ivar MAX_LENGTH: Defines the maximum length of netstrings that can be
  49. received.
  50. @type MAX_LENGTH: C{int}
  51. @ivar _LENGTH: A pattern describing all strings that contain a netstring
  52. length specification. Examples for length specifications are C{b'0:'},
  53. C{b'12:'}, and C{b'179:'}. C{b'007:'} is not a valid length
  54. specification, since leading zeros are not allowed.
  55. @type _LENGTH: C{re.Match}
  56. @ivar _LENGTH_PREFIX: A pattern describing all strings that contain
  57. the first part of a netstring length specification (without the
  58. trailing comma). Examples are '0', '12', and '179'. '007' does not
  59. start a netstring length specification, since leading zeros are
  60. not allowed.
  61. @type _LENGTH_PREFIX: C{re.Match}
  62. @ivar _PARSING_LENGTH: Indicates that the C{NetstringReceiver} is in
  63. the state of parsing the length portion of a netstring.
  64. @type _PARSING_LENGTH: C{int}
  65. @ivar _PARSING_PAYLOAD: Indicates that the C{NetstringReceiver} is in
  66. the state of parsing the payload portion (data and trailing comma)
  67. of a netstring.
  68. @type _PARSING_PAYLOAD: C{int}
  69. @ivar brokenPeer: Indicates if the connection is still functional
  70. @type brokenPeer: C{int}
  71. @ivar _state: Indicates if the protocol is consuming the length portion
  72. (C{PARSING_LENGTH}) or the payload (C{PARSING_PAYLOAD}) of a netstring
  73. @type _state: C{int}
  74. @ivar _remainingData: Holds the chunk of data that has not yet been consumed
  75. @type _remainingData: C{string}
  76. @ivar _payload: Holds the payload portion of a netstring including the
  77. trailing comma
  78. @type _payload: C{BytesIO}
  79. @ivar _expectedPayloadSize: Holds the payload size plus one for the trailing
  80. comma.
  81. @type _expectedPayloadSize: C{int}
  82. """
  83. MAX_LENGTH = 99999
  84. _LENGTH = re.compile(br"(0|[1-9]\d*)(:)")
  85. _LENGTH_PREFIX = re.compile(br"(0|[1-9]\d*)$")
  86. # Some error information for NetstringParseError instances.
  87. _MISSING_LENGTH = (
  88. "The received netstring does not start with a " "length specification."
  89. )
  90. _OVERFLOW = (
  91. "The length specification of the received netstring "
  92. "cannot be represented in Python - it causes an "
  93. "OverflowError!"
  94. )
  95. _TOO_LONG = (
  96. "The received netstring is longer than the maximum %s "
  97. "specified by self.MAX_LENGTH"
  98. )
  99. _MISSING_COMMA = "The received netstring is not terminated by a comma."
  100. # The following constants are used for determining if the NetstringReceiver
  101. # is parsing the length portion of a netstring, or the payload.
  102. _PARSING_LENGTH, _PARSING_PAYLOAD = range(2)
  103. def makeConnection(self, transport):
  104. """
  105. Initializes the protocol.
  106. """
  107. protocol.Protocol.makeConnection(self, transport)
  108. self._remainingData = b""
  109. self._currentPayloadSize = 0
  110. self._payload = BytesIO()
  111. self._state = self._PARSING_LENGTH
  112. self._expectedPayloadSize = 0
  113. self.brokenPeer = 0
  114. def sendString(self, string):
  115. """
  116. Sends a netstring.
  117. Wraps up C{string} by adding length information and a
  118. trailing comma; writes the result to the transport.
  119. @param string: The string to send. The necessary framing (length
  120. prefix, etc) will be added.
  121. @type string: C{bytes}
  122. """
  123. self.transport.write(_formatNetstring(string))
  124. def dataReceived(self, data):
  125. """
  126. Receives some characters of a netstring.
  127. Whenever a complete netstring is received, this method extracts
  128. its payload and calls L{stringReceived} to process it.
  129. @param data: A chunk of data representing a (possibly partial)
  130. netstring
  131. @type data: C{bytes}
  132. """
  133. self._remainingData += data
  134. while self._remainingData:
  135. try:
  136. self._consumeData()
  137. except IncompleteNetstring:
  138. break
  139. except NetstringParseError:
  140. self._handleParseError()
  141. break
  142. def stringReceived(self, string):
  143. """
  144. Override this for notification when each complete string is received.
  145. @param string: The complete string which was received with all
  146. framing (length prefix, etc) removed.
  147. @type string: C{bytes}
  148. @raise NotImplementedError: because the method has to be implemented
  149. by the child class.
  150. """
  151. raise NotImplementedError()
  152. def _maxLengthSize(self):
  153. """
  154. Calculate and return the string size of C{self.MAX_LENGTH}.
  155. @return: The size of the string representation for C{self.MAX_LENGTH}
  156. @rtype: C{float}
  157. """
  158. return math.ceil(math.log10(self.MAX_LENGTH)) + 1
  159. def _consumeData(self):
  160. """
  161. Consumes the content of C{self._remainingData}.
  162. @raise IncompleteNetstring: if C{self._remainingData} does not
  163. contain enough data to complete the current netstring.
  164. @raise NetstringParseError: if the received data do not
  165. form a valid netstring.
  166. """
  167. if self._state == self._PARSING_LENGTH:
  168. self._consumeLength()
  169. self._prepareForPayloadConsumption()
  170. if self._state == self._PARSING_PAYLOAD:
  171. self._consumePayload()
  172. def _consumeLength(self):
  173. """
  174. Consumes the length portion of C{self._remainingData}.
  175. @raise IncompleteNetstring: if C{self._remainingData} contains
  176. a partial length specification (digits without trailing
  177. comma).
  178. @raise NetstringParseError: if the received data do not form a valid
  179. netstring.
  180. """
  181. lengthMatch = self._LENGTH.match(self._remainingData)
  182. if not lengthMatch:
  183. self._checkPartialLengthSpecification()
  184. raise IncompleteNetstring()
  185. self._processLength(lengthMatch)
  186. def _checkPartialLengthSpecification(self):
  187. """
  188. Makes sure that the received data represents a valid number.
  189. Checks if C{self._remainingData} represents a number smaller or
  190. equal to C{self.MAX_LENGTH}.
  191. @raise NetstringParseError: if C{self._remainingData} is no
  192. number or is too big (checked by L{_extractLength}).
  193. """
  194. partialLengthMatch = self._LENGTH_PREFIX.match(self._remainingData)
  195. if not partialLengthMatch:
  196. raise NetstringParseError(self._MISSING_LENGTH)
  197. lengthSpecification = partialLengthMatch.group(1)
  198. self._extractLength(lengthSpecification)
  199. def _processLength(self, lengthMatch):
  200. """
  201. Processes the length definition of a netstring.
  202. Extracts and stores in C{self._expectedPayloadSize} the number
  203. representing the netstring size. Removes the prefix
  204. representing the length specification from
  205. C{self._remainingData}.
  206. @raise NetstringParseError: if the received netstring does not
  207. start with a number or the number is bigger than
  208. C{self.MAX_LENGTH}.
  209. @param lengthMatch: A regular expression match object matching
  210. a netstring length specification
  211. @type lengthMatch: C{re.Match}
  212. """
  213. endOfNumber = lengthMatch.end(1)
  214. startOfData = lengthMatch.end(2)
  215. lengthString = self._remainingData[:endOfNumber]
  216. # Expect payload plus trailing comma:
  217. self._expectedPayloadSize = self._extractLength(lengthString) + 1
  218. self._remainingData = self._remainingData[startOfData:]
  219. def _extractLength(self, lengthAsString):
  220. """
  221. Attempts to extract the length information of a netstring.
  222. @raise NetstringParseError: if the number is bigger than
  223. C{self.MAX_LENGTH}.
  224. @param lengthAsString: A chunk of data starting with a length
  225. specification
  226. @type lengthAsString: C{bytes}
  227. @return: The length of the netstring
  228. @rtype: C{int}
  229. """
  230. self._checkStringSize(lengthAsString)
  231. length = int(lengthAsString)
  232. if length > self.MAX_LENGTH:
  233. raise NetstringParseError(self._TOO_LONG % (self.MAX_LENGTH,))
  234. return length
  235. def _checkStringSize(self, lengthAsString):
  236. """
  237. Checks the sanity of lengthAsString.
  238. Checks if the size of the length specification exceeds the
  239. size of the string representing self.MAX_LENGTH. If this is
  240. not the case, the number represented by lengthAsString is
  241. certainly bigger than self.MAX_LENGTH, and a
  242. NetstringParseError can be raised.
  243. This method should make sure that netstrings with extremely
  244. long length specifications are refused before even attempting
  245. to convert them to an integer (which might trigger a
  246. MemoryError).
  247. """
  248. if len(lengthAsString) > self._maxLengthSize():
  249. raise NetstringParseError(self._TOO_LONG % (self.MAX_LENGTH,))
  250. def _prepareForPayloadConsumption(self):
  251. """
  252. Sets up variables necessary for consuming the payload of a netstring.
  253. """
  254. self._state = self._PARSING_PAYLOAD
  255. self._currentPayloadSize = 0
  256. self._payload.seek(0)
  257. self._payload.truncate()
  258. def _consumePayload(self):
  259. """
  260. Consumes the payload portion of C{self._remainingData}.
  261. If the payload is complete, checks for the trailing comma and
  262. processes the payload. If not, raises an L{IncompleteNetstring}
  263. exception.
  264. @raise IncompleteNetstring: if the payload received so far
  265. contains fewer characters than expected.
  266. @raise NetstringParseError: if the payload does not end with a
  267. comma.
  268. """
  269. self._extractPayload()
  270. if self._currentPayloadSize < self._expectedPayloadSize:
  271. raise IncompleteNetstring()
  272. self._checkForTrailingComma()
  273. self._state = self._PARSING_LENGTH
  274. self._processPayload()
  275. def _extractPayload(self):
  276. """
  277. Extracts payload information from C{self._remainingData}.
  278. Splits C{self._remainingData} at the end of the netstring. The
  279. first part becomes C{self._payload}, the second part is stored
  280. in C{self._remainingData}.
  281. If the netstring is not yet complete, the whole content of
  282. C{self._remainingData} is moved to C{self._payload}.
  283. """
  284. if self._payloadComplete():
  285. remainingPayloadSize = self._expectedPayloadSize - self._currentPayloadSize
  286. self._payload.write(self._remainingData[:remainingPayloadSize])
  287. self._remainingData = self._remainingData[remainingPayloadSize:]
  288. self._currentPayloadSize = self._expectedPayloadSize
  289. else:
  290. self._payload.write(self._remainingData)
  291. self._currentPayloadSize += len(self._remainingData)
  292. self._remainingData = b""
  293. def _payloadComplete(self):
  294. """
  295. Checks if enough data have been received to complete the netstring.
  296. @return: C{True} iff the received data contain at least as many
  297. characters as specified in the length section of the
  298. netstring
  299. @rtype: C{bool}
  300. """
  301. return (
  302. len(self._remainingData) + self._currentPayloadSize
  303. >= self._expectedPayloadSize
  304. )
  305. def _processPayload(self):
  306. """
  307. Processes the actual payload with L{stringReceived}.
  308. Strips C{self._payload} of the trailing comma and calls
  309. L{stringReceived} with the result.
  310. """
  311. self.stringReceived(self._payload.getvalue()[:-1])
  312. def _checkForTrailingComma(self):
  313. """
  314. Checks if the netstring has a trailing comma at the expected position.
  315. @raise NetstringParseError: if the last payload character is
  316. anything but a comma.
  317. """
  318. if self._payload.getvalue()[-1:] != b",":
  319. raise NetstringParseError(self._MISSING_COMMA)
  320. def _handleParseError(self):
  321. """
  322. Terminates the connection and sets the flag C{self.brokenPeer}.
  323. """
  324. self.transport.loseConnection()
  325. self.brokenPeer = 1
  326. class LineOnlyReceiver(protocol.Protocol):
  327. """
  328. A protocol that receives only lines.
  329. This is purely a speed optimisation over LineReceiver, for the
  330. cases that raw mode is known to be unnecessary.
  331. @cvar delimiter: The line-ending delimiter to use. By default this is
  332. C{b'\\r\\n'}.
  333. @cvar MAX_LENGTH: The maximum length of a line to allow (If a
  334. sent line is longer than this, the connection is dropped).
  335. Default is 16384.
  336. """
  337. _buffer = b""
  338. delimiter = b"\r\n"
  339. MAX_LENGTH = 16384
  340. def dataReceived(self, data):
  341. """
  342. Translates bytes into lines, and calls lineReceived.
  343. """
  344. lines = (self._buffer + data).split(self.delimiter)
  345. self._buffer = lines.pop(-1)
  346. for line in lines:
  347. if self.transport.disconnecting:
  348. # this is necessary because the transport may be told to lose
  349. # the connection by a line within a larger packet, and it is
  350. # important to disregard all the lines in that packet following
  351. # the one that told it to close.
  352. return
  353. if len(line) > self.MAX_LENGTH:
  354. return self.lineLengthExceeded(line)
  355. else:
  356. self.lineReceived(line)
  357. if len(self._buffer) > self.MAX_LENGTH:
  358. return self.lineLengthExceeded(self._buffer)
  359. def lineReceived(self, line):
  360. """
  361. Override this for when each line is received.
  362. @param line: The line which was received with the delimiter removed.
  363. @type line: C{bytes}
  364. """
  365. raise NotImplementedError
  366. def sendLine(self, line):
  367. """
  368. Sends a line to the other end of the connection.
  369. @param line: The line to send, not including the delimiter.
  370. @type line: C{bytes}
  371. """
  372. return self.transport.writeSequence((line, self.delimiter))
  373. def lineLengthExceeded(self, line):
  374. """
  375. Called when the maximum line length has been reached.
  376. Override if it needs to be dealt with in some special way.
  377. """
  378. return self.transport.loseConnection()
  379. class _PauseableMixin:
  380. paused = False
  381. def pauseProducing(self):
  382. self.paused = True
  383. self.transport.pauseProducing()
  384. def resumeProducing(self):
  385. self.paused = False
  386. self.transport.resumeProducing()
  387. self.dataReceived(b"")
  388. def stopProducing(self):
  389. self.paused = True
  390. self.transport.stopProducing()
  391. class LineReceiver(protocol.Protocol, _PauseableMixin):
  392. """
  393. A protocol that receives lines and/or raw data, depending on mode.
  394. In line mode, each line that's received becomes a callback to
  395. L{lineReceived}. In raw data mode, each chunk of raw data becomes a
  396. callback to L{LineReceiver.rawDataReceived}.
  397. The L{setLineMode} and L{setRawMode} methods switch between the two modes.
  398. This is useful for line-oriented protocols such as IRC, HTTP, POP, etc.
  399. @cvar delimiter: The line-ending delimiter to use. By default this is
  400. C{b'\\r\\n'}.
  401. @cvar MAX_LENGTH: The maximum length of a line to allow (If a
  402. sent line is longer than this, the connection is dropped).
  403. Default is 16384.
  404. """
  405. line_mode = 1
  406. _buffer = b""
  407. _busyReceiving = False
  408. delimiter = b"\r\n"
  409. MAX_LENGTH = 16384
  410. def clearLineBuffer(self):
  411. """
  412. Clear buffered data.
  413. @return: All of the cleared buffered data.
  414. @rtype: C{bytes}
  415. """
  416. b, self._buffer = self._buffer, b""
  417. return b
  418. def dataReceived(self, data):
  419. """
  420. Protocol.dataReceived.
  421. Translates bytes into lines, and calls lineReceived (or
  422. rawDataReceived, depending on mode.)
  423. """
  424. if self._busyReceiving:
  425. self._buffer += data
  426. return
  427. try:
  428. self._busyReceiving = True
  429. self._buffer += data
  430. while self._buffer and not self.paused:
  431. if self.line_mode:
  432. try:
  433. line, self._buffer = self._buffer.split(self.delimiter, 1)
  434. except ValueError:
  435. if len(self._buffer) >= (self.MAX_LENGTH + len(self.delimiter)):
  436. line, self._buffer = self._buffer, b""
  437. return self.lineLengthExceeded(line)
  438. return
  439. else:
  440. lineLength = len(line)
  441. if lineLength > self.MAX_LENGTH:
  442. exceeded = line + self.delimiter + self._buffer
  443. self._buffer = b""
  444. return self.lineLengthExceeded(exceeded)
  445. why = self.lineReceived(line)
  446. if why or self.transport and self.transport.disconnecting:
  447. return why
  448. else:
  449. data = self._buffer
  450. self._buffer = b""
  451. why = self.rawDataReceived(data)
  452. if why:
  453. return why
  454. finally:
  455. self._busyReceiving = False
  456. def setLineMode(self, extra=b""):
  457. """
  458. Sets the line-mode of this receiver.
  459. If you are calling this from a rawDataReceived callback,
  460. you can pass in extra unhandled data, and that data will
  461. be parsed for lines. Further data received will be sent
  462. to lineReceived rather than rawDataReceived.
  463. Do not pass extra data if calling this function from
  464. within a lineReceived callback.
  465. """
  466. self.line_mode = 1
  467. if extra:
  468. return self.dataReceived(extra)
  469. def setRawMode(self):
  470. """
  471. Sets the raw mode of this receiver.
  472. Further data received will be sent to rawDataReceived rather
  473. than lineReceived.
  474. """
  475. self.line_mode = 0
  476. def rawDataReceived(self, data):
  477. """
  478. Override this for when raw data is received.
  479. """
  480. raise NotImplementedError
  481. def lineReceived(self, line):
  482. """
  483. Override this for when each line is received.
  484. @param line: The line which was received with the delimiter removed.
  485. @type line: C{bytes}
  486. """
  487. raise NotImplementedError
  488. def sendLine(self, line):
  489. """
  490. Sends a line to the other end of the connection.
  491. @param line: The line to send, not including the delimiter.
  492. @type line: C{bytes}
  493. """
  494. return self.transport.write(line + self.delimiter)
  495. def lineLengthExceeded(self, line):
  496. """
  497. Called when the maximum line length has been reached.
  498. Override if it needs to be dealt with in some special way.
  499. The argument 'line' contains the remainder of the buffer, starting
  500. with (at least some part) of the line which is too long. This may
  501. be more than one line, or may be only the initial portion of the
  502. line.
  503. """
  504. return self.transport.loseConnection()
  505. class StringTooLongError(AssertionError):
  506. """
  507. Raised when trying to send a string too long for a length prefixed
  508. protocol.
  509. """
  510. class _RecvdCompatHack:
  511. """
  512. Emulates the to-be-deprecated C{IntNStringReceiver.recvd} attribute.
  513. The C{recvd} attribute was where the working buffer for buffering and
  514. parsing netstrings was kept. It was updated each time new data arrived and
  515. each time some of that data was parsed and delivered to application code.
  516. The piecemeal updates to its string value were expensive and have been
  517. removed from C{IntNStringReceiver} in the normal case. However, for
  518. applications directly reading this attribute, this descriptor restores that
  519. behavior. It only copies the working buffer when necessary (ie, when
  520. accessed). This avoids the cost for applications not using the data.
  521. This is a custom descriptor rather than a property, because we still need
  522. the default __set__ behavior in both new-style and old-style subclasses.
  523. """
  524. def __get__(self, oself, type=None):
  525. return oself._unprocessed[oself._compatibilityOffset :]
  526. class IntNStringReceiver(protocol.Protocol, _PauseableMixin):
  527. """
  528. Generic class for length prefixed protocols.
  529. @ivar _unprocessed: bytes received, but not yet broken up into messages /
  530. sent to stringReceived. _compatibilityOffset must be updated when this
  531. value is updated so that the C{recvd} attribute can be generated
  532. correctly.
  533. @type _unprocessed: C{bytes}
  534. @ivar structFormat: format used for struct packing/unpacking. Define it in
  535. subclass.
  536. @type structFormat: C{str}
  537. @ivar prefixLength: length of the prefix, in bytes. Define it in subclass,
  538. using C{struct.calcsize(structFormat)}
  539. @type prefixLength: C{int}
  540. @ivar _compatibilityOffset: the offset within C{_unprocessed} to the next
  541. message to be parsed. (used to generate the recvd attribute)
  542. @type _compatibilityOffset: C{int}
  543. """
  544. MAX_LENGTH = 99999
  545. _unprocessed = b""
  546. _compatibilityOffset = 0
  547. # Backwards compatibility support for applications which directly touch the
  548. # "internal" parse buffer.
  549. recvd = _RecvdCompatHack()
  550. def stringReceived(self, string):
  551. """
  552. Override this for notification when each complete string is received.
  553. @param string: The complete string which was received with all
  554. framing (length prefix, etc) removed.
  555. @type string: C{bytes}
  556. """
  557. raise NotImplementedError
  558. def lengthLimitExceeded(self, length):
  559. """
  560. Callback invoked when a length prefix greater than C{MAX_LENGTH} is
  561. received. The default implementation disconnects the transport.
  562. Override this.
  563. @param length: The length prefix which was received.
  564. @type length: C{int}
  565. """
  566. self.transport.loseConnection()
  567. def dataReceived(self, data):
  568. """
  569. Convert int prefixed strings into calls to stringReceived.
  570. """
  571. # Try to minimize string copying (via slices) by keeping one buffer
  572. # containing all the data we have so far and a separate offset into that
  573. # buffer.
  574. alldata = self._unprocessed + data
  575. currentOffset = 0
  576. prefixLength = self.prefixLength
  577. fmt = self.structFormat
  578. self._unprocessed = alldata
  579. while len(alldata) >= (currentOffset + prefixLength) and not self.paused:
  580. messageStart = currentOffset + prefixLength
  581. (length,) = unpack(fmt, alldata[currentOffset:messageStart])
  582. if length > self.MAX_LENGTH:
  583. self._unprocessed = alldata
  584. self._compatibilityOffset = currentOffset
  585. self.lengthLimitExceeded(length)
  586. return
  587. messageEnd = messageStart + length
  588. if len(alldata) < messageEnd:
  589. break
  590. # Here we have to slice the working buffer so we can send just the
  591. # netstring into the stringReceived callback.
  592. packet = alldata[messageStart:messageEnd]
  593. currentOffset = messageEnd
  594. self._compatibilityOffset = currentOffset
  595. self.stringReceived(packet)
  596. # Check to see if the backwards compat "recvd" attribute got written
  597. # to by application code. If so, drop the current data buffer and
  598. # switch to the new buffer given by that attribute's value.
  599. if "recvd" in self.__dict__:
  600. alldata = self.__dict__.pop("recvd")
  601. self._unprocessed = alldata
  602. self._compatibilityOffset = currentOffset = 0
  603. if alldata:
  604. continue
  605. return
  606. # Slice off all the data that has been processed, avoiding holding onto
  607. # memory to store it, and update the compatibility attributes to reflect
  608. # that change.
  609. self._unprocessed = alldata[currentOffset:]
  610. self._compatibilityOffset = 0
  611. def sendString(self, string):
  612. """
  613. Send a prefixed string to the other end of the connection.
  614. @param string: The string to send. The necessary framing (length
  615. prefix, etc) will be added.
  616. @type string: C{bytes}
  617. """
  618. if len(string) >= 2 ** (8 * self.prefixLength):
  619. raise StringTooLongError(
  620. "Try to send %s bytes whereas maximum is %s"
  621. % (len(string), 2 ** (8 * self.prefixLength))
  622. )
  623. self.transport.write(pack(self.structFormat, len(string)) + string)
  624. class Int32StringReceiver(IntNStringReceiver):
  625. """
  626. A receiver for int32-prefixed strings.
  627. An int32 string is a string prefixed by 4 bytes, the 32-bit length of
  628. the string encoded in network byte order.
  629. This class publishes the same interface as NetstringReceiver.
  630. """
  631. structFormat = "!I"
  632. prefixLength = calcsize(structFormat)
  633. class Int16StringReceiver(IntNStringReceiver):
  634. """
  635. A receiver for int16-prefixed strings.
  636. An int16 string is a string prefixed by 2 bytes, the 16-bit length of
  637. the string encoded in network byte order.
  638. This class publishes the same interface as NetstringReceiver.
  639. """
  640. structFormat = "!H"
  641. prefixLength = calcsize(structFormat)
  642. class Int8StringReceiver(IntNStringReceiver):
  643. """
  644. A receiver for int8-prefixed strings.
  645. An int8 string is a string prefixed by 1 byte, the 8-bit length of
  646. the string.
  647. This class publishes the same interface as NetstringReceiver.
  648. """
  649. structFormat = "!B"
  650. prefixLength = calcsize(structFormat)
  651. class StatefulStringProtocol:
  652. """
  653. A stateful string protocol.
  654. This is a mixin for string protocols (L{Int32StringReceiver},
  655. L{NetstringReceiver}) which translates L{stringReceived} into a callback
  656. (prefixed with C{'proto_'}) depending on state.
  657. The state C{'done'} is special; if a C{proto_*} method returns it, the
  658. connection will be closed immediately.
  659. @ivar state: Current state of the protocol. Defaults to C{'init'}.
  660. @type state: C{str}
  661. """
  662. state = "init"
  663. def stringReceived(self, string):
  664. """
  665. Choose a protocol phase function and call it.
  666. Call back to the appropriate protocol phase; this begins with
  667. the function C{proto_init} and moves on to C{proto_*} depending on
  668. what each C{proto_*} function returns. (For example, if
  669. C{self.proto_init} returns 'foo', then C{self.proto_foo} will be the
  670. next function called when a protocol message is received.
  671. """
  672. try:
  673. pto = "proto_" + self.state
  674. statehandler = getattr(self, pto)
  675. except AttributeError:
  676. log.msg("callback", self.state, "not found")
  677. else:
  678. self.state = statehandler(string)
  679. if self.state == "done":
  680. self.transport.loseConnection()
  681. @implementer(interfaces.IProducer)
  682. class FileSender:
  683. """
  684. A producer that sends the contents of a file to a consumer.
  685. This is a helper for protocols that, at some point, will take a
  686. file-like object, read its contents, and write them out to the network,
  687. optionally performing some transformation on the bytes in between.
  688. """
  689. CHUNK_SIZE = 2 ** 14
  690. lastSent = ""
  691. deferred = None
  692. def beginFileTransfer(self, file, consumer, transform=None):
  693. """
  694. Begin transferring a file
  695. @type file: Any file-like object
  696. @param file: The file object to read data from
  697. @type consumer: Any implementor of IConsumer
  698. @param consumer: The object to write data to
  699. @param transform: A callable taking one string argument and returning
  700. the same. All bytes read from the file are passed through this before
  701. being written to the consumer.
  702. @rtype: C{Deferred}
  703. @return: A deferred whose callback will be invoked when the file has
  704. been completely written to the consumer. The last byte written to the
  705. consumer is passed to the callback.
  706. """
  707. self.file = file
  708. self.consumer = consumer
  709. self.transform = transform
  710. self.deferred = deferred = defer.Deferred()
  711. self.consumer.registerProducer(self, False)
  712. return deferred
  713. def resumeProducing(self):
  714. chunk = ""
  715. if self.file:
  716. chunk = self.file.read(self.CHUNK_SIZE)
  717. if not chunk:
  718. self.file = None
  719. self.consumer.unregisterProducer()
  720. if self.deferred:
  721. self.deferred.callback(self.lastSent)
  722. self.deferred = None
  723. return
  724. if self.transform:
  725. chunk = self.transform(chunk)
  726. self.consumer.write(chunk)
  727. self.lastSent = chunk[-1:]
  728. def pauseProducing(self):
  729. pass
  730. def stopProducing(self):
  731. if self.deferred:
  732. self.deferred.errback(Exception("Consumer asked us to stop producing"))
  733. self.deferred = None